googlecloud 0.0.4 → 0.0.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (768) hide show
  1. data.tar.gz.sig +0 -0
  2. data/CHANGELOG +4 -0
  3. data/Makefile +4 -0
  4. data/Manifest +760 -1
  5. data/bin/gcloud +7 -0
  6. data/bin/{gcutil → gcutil.symlink} +0 -0
  7. data/bin/gsutil-symlink +377 -0
  8. data/googlecloud.gemspec +5 -5
  9. data/packages/gsutil/CHECKSUM +1 -0
  10. data/packages/gsutil/COPYING +202 -0
  11. data/packages/gsutil/LICENSE.third_party +295 -0
  12. data/packages/gsutil/MANIFEST.in +5 -0
  13. data/packages/gsutil/README +38 -0
  14. data/packages/gsutil/README.pkg +49 -0
  15. data/packages/gsutil/ReleaseNotes.txt +780 -0
  16. data/packages/gsutil/VERSION +1 -0
  17. data/packages/gsutil/boto/Changelog.rst +35 -0
  18. data/packages/gsutil/boto/LICENSE +18 -0
  19. data/packages/gsutil/boto/MANIFEST.in +12 -0
  20. data/packages/gsutil/boto/README.rst +163 -0
  21. data/packages/gsutil/boto/bin/asadmin +290 -0
  22. data/packages/gsutil/boto/bin/bundle_image +27 -0
  23. data/packages/gsutil/boto/bin/cfadmin +108 -0
  24. data/packages/gsutil/boto/bin/cq +89 -0
  25. data/packages/gsutil/boto/bin/cwutil +140 -0
  26. data/packages/gsutil/boto/bin/elbadmin +284 -0
  27. data/packages/gsutil/boto/bin/fetch_file +43 -0
  28. data/packages/gsutil/boto/bin/glacier +154 -0
  29. data/packages/gsutil/boto/bin/instance_events +145 -0
  30. data/packages/gsutil/boto/bin/kill_instance +35 -0
  31. data/packages/gsutil/boto/bin/launch_instance +252 -0
  32. data/packages/gsutil/boto/bin/list_instances +90 -0
  33. data/packages/gsutil/boto/bin/lss3 +77 -0
  34. data/packages/gsutil/boto/bin/mturk +465 -0
  35. data/packages/gsutil/boto/bin/pyami_sendmail +52 -0
  36. data/packages/gsutil/boto/bin/route53 +205 -0
  37. data/packages/gsutil/boto/bin/s3put +374 -0
  38. data/packages/gsutil/boto/bin/sdbadmin +194 -0
  39. data/packages/gsutil/boto/bin/taskadmin +116 -0
  40. data/packages/gsutil/boto/boto/__init__.py +793 -0
  41. data/packages/gsutil/boto/boto/__init__.pyc +0 -0
  42. data/packages/gsutil/boto/boto/auth.py +682 -0
  43. data/packages/gsutil/boto/boto/auth.pyc +0 -0
  44. data/packages/gsutil/boto/boto/auth_handler.py +58 -0
  45. data/packages/gsutil/boto/boto/auth_handler.pyc +0 -0
  46. data/packages/gsutil/boto/boto/beanstalk/__init__.py +65 -0
  47. data/packages/gsutil/boto/boto/beanstalk/exception.py +64 -0
  48. data/packages/gsutil/boto/boto/beanstalk/layer1.py +1153 -0
  49. data/packages/gsutil/boto/boto/beanstalk/response.py +703 -0
  50. data/packages/gsutil/boto/boto/beanstalk/wrapper.py +29 -0
  51. data/packages/gsutil/boto/boto/cacerts/__init__.py +22 -0
  52. data/packages/gsutil/boto/boto/cacerts/__init__.pyc +0 -0
  53. data/packages/gsutil/boto/boto/cacerts/cacerts.txt +633 -0
  54. data/packages/gsutil/boto/boto/cloudformation/__init__.py +68 -0
  55. data/packages/gsutil/boto/boto/cloudformation/connection.py +364 -0
  56. data/packages/gsutil/boto/boto/cloudformation/stack.py +360 -0
  57. data/packages/gsutil/boto/boto/cloudformation/template.py +43 -0
  58. data/packages/gsutil/boto/boto/cloudfront/__init__.py +324 -0
  59. data/packages/gsutil/boto/boto/cloudfront/distribution.py +745 -0
  60. data/packages/gsutil/boto/boto/cloudfront/exception.py +26 -0
  61. data/packages/gsutil/boto/boto/cloudfront/identity.py +122 -0
  62. data/packages/gsutil/boto/boto/cloudfront/invalidation.py +216 -0
  63. data/packages/gsutil/boto/boto/cloudfront/logging.py +38 -0
  64. data/packages/gsutil/boto/boto/cloudfront/object.py +48 -0
  65. data/packages/gsutil/boto/boto/cloudfront/origin.py +150 -0
  66. data/packages/gsutil/boto/boto/cloudfront/signers.py +60 -0
  67. data/packages/gsutil/boto/boto/cloudsearch/__init__.py +45 -0
  68. data/packages/gsutil/boto/boto/cloudsearch/document.py +265 -0
  69. data/packages/gsutil/boto/boto/cloudsearch/domain.py +394 -0
  70. data/packages/gsutil/boto/boto/cloudsearch/layer1.py +738 -0
  71. data/packages/gsutil/boto/boto/cloudsearch/layer2.py +67 -0
  72. data/packages/gsutil/boto/boto/cloudsearch/optionstatus.py +248 -0
  73. data/packages/gsutil/boto/boto/cloudsearch/search.py +365 -0
  74. data/packages/gsutil/boto/boto/cloudsearch/sourceattribute.py +75 -0
  75. data/packages/gsutil/boto/boto/compat.py +28 -0
  76. data/packages/gsutil/boto/boto/compat.pyc +0 -0
  77. data/packages/gsutil/boto/boto/connection.py +1081 -0
  78. data/packages/gsutil/boto/boto/connection.pyc +0 -0
  79. data/packages/gsutil/boto/boto/contrib/__init__.py +22 -0
  80. data/packages/gsutil/boto/boto/contrib/ymlmessage.py +52 -0
  81. data/packages/gsutil/boto/boto/core/README +58 -0
  82. data/packages/gsutil/boto/boto/core/__init__.py +23 -0
  83. data/packages/gsutil/boto/boto/core/auth.py +78 -0
  84. data/packages/gsutil/boto/boto/core/credentials.py +154 -0
  85. data/packages/gsutil/boto/boto/core/dictresponse.py +178 -0
  86. data/packages/gsutil/boto/boto/core/service.py +67 -0
  87. data/packages/gsutil/boto/boto/datapipeline/__init__.py +0 -0
  88. data/packages/gsutil/boto/boto/datapipeline/exceptions.py +42 -0
  89. data/packages/gsutil/boto/boto/datapipeline/layer1.py +546 -0
  90. data/packages/gsutil/boto/boto/dynamodb/__init__.py +66 -0
  91. data/packages/gsutil/boto/boto/dynamodb/batch.py +262 -0
  92. data/packages/gsutil/boto/boto/dynamodb/condition.py +170 -0
  93. data/packages/gsutil/boto/boto/dynamodb/exceptions.py +64 -0
  94. data/packages/gsutil/boto/boto/dynamodb/item.py +196 -0
  95. data/packages/gsutil/boto/boto/dynamodb/layer1.py +575 -0
  96. data/packages/gsutil/boto/boto/dynamodb/layer2.py +798 -0
  97. data/packages/gsutil/boto/boto/dynamodb/schema.py +112 -0
  98. data/packages/gsutil/boto/boto/dynamodb/table.py +540 -0
  99. data/packages/gsutil/boto/boto/dynamodb/types.py +326 -0
  100. data/packages/gsutil/boto/boto/ec2/__init__.py +96 -0
  101. data/packages/gsutil/boto/boto/ec2/address.py +103 -0
  102. data/packages/gsutil/boto/boto/ec2/autoscale/__init__.py +781 -0
  103. data/packages/gsutil/boto/boto/ec2/autoscale/activity.py +74 -0
  104. data/packages/gsutil/boto/boto/ec2/autoscale/group.py +337 -0
  105. data/packages/gsutil/boto/boto/ec2/autoscale/instance.py +60 -0
  106. data/packages/gsutil/boto/boto/ec2/autoscale/launchconfig.py +209 -0
  107. data/packages/gsutil/boto/boto/ec2/autoscale/policy.py +166 -0
  108. data/packages/gsutil/boto/boto/ec2/autoscale/request.py +38 -0
  109. data/packages/gsutil/boto/boto/ec2/autoscale/scheduled.py +60 -0
  110. data/packages/gsutil/boto/boto/ec2/autoscale/tag.py +84 -0
  111. data/packages/gsutil/boto/boto/ec2/blockdevicemapping.py +141 -0
  112. data/packages/gsutil/boto/boto/ec2/bundleinstance.py +78 -0
  113. data/packages/gsutil/boto/boto/ec2/buyreservation.py +84 -0
  114. data/packages/gsutil/boto/boto/ec2/cloudwatch/__init__.py +603 -0
  115. data/packages/gsutil/boto/boto/ec2/cloudwatch/alarm.py +316 -0
  116. data/packages/gsutil/boto/boto/ec2/cloudwatch/datapoint.py +40 -0
  117. data/packages/gsutil/boto/boto/ec2/cloudwatch/dimension.py +38 -0
  118. data/packages/gsutil/boto/boto/ec2/cloudwatch/listelement.py +31 -0
  119. data/packages/gsutil/boto/boto/ec2/cloudwatch/metric.py +175 -0
  120. data/packages/gsutil/boto/boto/ec2/connection.py +3409 -0
  121. data/packages/gsutil/boto/boto/ec2/ec2object.py +107 -0
  122. data/packages/gsutil/boto/boto/ec2/elb/__init__.py +553 -0
  123. data/packages/gsutil/boto/boto/ec2/elb/healthcheck.py +89 -0
  124. data/packages/gsutil/boto/boto/ec2/elb/instancestate.py +62 -0
  125. data/packages/gsutil/boto/boto/ec2/elb/listelement.py +36 -0
  126. data/packages/gsutil/boto/boto/ec2/elb/listener.py +75 -0
  127. data/packages/gsutil/boto/boto/ec2/elb/loadbalancer.py +324 -0
  128. data/packages/gsutil/boto/boto/ec2/elb/policies.py +89 -0
  129. data/packages/gsutil/boto/boto/ec2/elb/securitygroup.py +38 -0
  130. data/packages/gsutil/boto/boto/ec2/group.py +39 -0
  131. data/packages/gsutil/boto/boto/ec2/image.py +350 -0
  132. data/packages/gsutil/boto/boto/ec2/instance.py +661 -0
  133. data/packages/gsutil/boto/boto/ec2/instanceinfo.py +51 -0
  134. data/packages/gsutil/boto/boto/ec2/instancestatus.py +207 -0
  135. data/packages/gsutil/boto/boto/ec2/keypair.py +113 -0
  136. data/packages/gsutil/boto/boto/ec2/launchspecification.py +105 -0
  137. data/packages/gsutil/boto/boto/ec2/networkinterface.py +247 -0
  138. data/packages/gsutil/boto/boto/ec2/placementgroup.py +51 -0
  139. data/packages/gsutil/boto/boto/ec2/regioninfo.py +34 -0
  140. data/packages/gsutil/boto/boto/ec2/reservedinstance.py +227 -0
  141. data/packages/gsutil/boto/boto/ec2/securitygroup.py +357 -0
  142. data/packages/gsutil/boto/boto/ec2/snapshot.py +170 -0
  143. data/packages/gsutil/boto/boto/ec2/spotdatafeedsubscription.py +63 -0
  144. data/packages/gsutil/boto/boto/ec2/spotinstancerequest.py +188 -0
  145. data/packages/gsutil/boto/boto/ec2/spotpricehistory.py +55 -0
  146. data/packages/gsutil/boto/boto/ec2/tag.py +87 -0
  147. data/packages/gsutil/boto/boto/ec2/vmtype.py +58 -0
  148. data/packages/gsutil/boto/boto/ec2/volume.py +293 -0
  149. data/packages/gsutil/boto/boto/ec2/volumestatus.py +200 -0
  150. data/packages/gsutil/boto/boto/ec2/zone.py +80 -0
  151. data/packages/gsutil/boto/boto/ecs/__init__.py +90 -0
  152. data/packages/gsutil/boto/boto/ecs/item.py +153 -0
  153. data/packages/gsutil/boto/boto/elasticache/__init__.py +62 -0
  154. data/packages/gsutil/boto/boto/elasticache/layer1.py +1252 -0
  155. data/packages/gsutil/boto/boto/elastictranscoder/__init__.py +62 -0
  156. data/packages/gsutil/boto/boto/elastictranscoder/exceptions.py +46 -0
  157. data/packages/gsutil/boto/boto/elastictranscoder/layer1.py +509 -0
  158. data/packages/gsutil/boto/boto/emr/__init__.py +73 -0
  159. data/packages/gsutil/boto/boto/emr/bootstrap_action.py +44 -0
  160. data/packages/gsutil/boto/boto/emr/connection.py +531 -0
  161. data/packages/gsutil/boto/boto/emr/emrobject.py +176 -0
  162. data/packages/gsutil/boto/boto/emr/instance_group.py +43 -0
  163. data/packages/gsutil/boto/boto/emr/step.py +281 -0
  164. data/packages/gsutil/boto/boto/exception.py +476 -0
  165. data/packages/gsutil/boto/boto/exception.pyc +0 -0
  166. data/packages/gsutil/boto/boto/file/README +49 -0
  167. data/packages/gsutil/boto/boto/file/__init__.py +28 -0
  168. data/packages/gsutil/boto/boto/file/bucket.py +112 -0
  169. data/packages/gsutil/boto/boto/file/connection.py +33 -0
  170. data/packages/gsutil/boto/boto/file/key.py +199 -0
  171. data/packages/gsutil/boto/boto/file/simpleresultset.py +30 -0
  172. data/packages/gsutil/boto/boto/fps/__init__.py +21 -0
  173. data/packages/gsutil/boto/boto/fps/connection.py +369 -0
  174. data/packages/gsutil/boto/boto/fps/exception.py +344 -0
  175. data/packages/gsutil/boto/boto/fps/response.py +175 -0
  176. data/packages/gsutil/boto/boto/glacier/__init__.py +57 -0
  177. data/packages/gsutil/boto/boto/glacier/concurrent.py +409 -0
  178. data/packages/gsutil/boto/boto/glacier/exceptions.py +58 -0
  179. data/packages/gsutil/boto/boto/glacier/job.py +152 -0
  180. data/packages/gsutil/boto/boto/glacier/layer1.py +637 -0
  181. data/packages/gsutil/boto/boto/glacier/layer2.py +93 -0
  182. data/packages/gsutil/boto/boto/glacier/response.py +48 -0
  183. data/packages/gsutil/boto/boto/glacier/utils.py +163 -0
  184. data/packages/gsutil/boto/boto/glacier/vault.py +387 -0
  185. data/packages/gsutil/boto/boto/glacier/writer.py +242 -0
  186. data/packages/gsutil/boto/boto/gs/__init__.py +22 -0
  187. data/packages/gsutil/boto/boto/gs/__init__.pyc +0 -0
  188. data/packages/gsutil/boto/boto/gs/acl.py +304 -0
  189. data/packages/gsutil/boto/boto/gs/acl.pyc +0 -0
  190. data/packages/gsutil/boto/boto/gs/bucket.py +870 -0
  191. data/packages/gsutil/boto/boto/gs/bucket.pyc +0 -0
  192. data/packages/gsutil/boto/boto/gs/bucketlistresultset.py +64 -0
  193. data/packages/gsutil/boto/boto/gs/bucketlistresultset.pyc +0 -0
  194. data/packages/gsutil/boto/boto/gs/connection.py +103 -0
  195. data/packages/gsutil/boto/boto/gs/connection.pyc +0 -0
  196. data/packages/gsutil/boto/boto/gs/cors.py +169 -0
  197. data/packages/gsutil/boto/boto/gs/cors.pyc +0 -0
  198. data/packages/gsutil/boto/boto/gs/key.py +704 -0
  199. data/packages/gsutil/boto/boto/gs/key.pyc +0 -0
  200. data/packages/gsutil/boto/boto/gs/resumable_upload_handler.py +659 -0
  201. data/packages/gsutil/boto/boto/gs/resumable_upload_handler.pyc +0 -0
  202. data/packages/gsutil/boto/boto/gs/user.py +54 -0
  203. data/packages/gsutil/boto/boto/gs/user.pyc +0 -0
  204. data/packages/gsutil/boto/boto/handler.py +44 -0
  205. data/packages/gsutil/boto/boto/handler.pyc +0 -0
  206. data/packages/gsutil/boto/boto/https_connection.py +124 -0
  207. data/packages/gsutil/boto/boto/https_connection.pyc +0 -0
  208. data/packages/gsutil/boto/boto/iam/__init__.py +74 -0
  209. data/packages/gsutil/boto/boto/iam/connection.py +1317 -0
  210. data/packages/gsutil/boto/boto/iam/summarymap.py +42 -0
  211. data/packages/gsutil/boto/boto/jsonresponse.py +163 -0
  212. data/packages/gsutil/boto/boto/jsonresponse.pyc +0 -0
  213. data/packages/gsutil/boto/boto/manage/__init__.py +23 -0
  214. data/packages/gsutil/boto/boto/manage/cmdshell.py +241 -0
  215. data/packages/gsutil/boto/boto/manage/propget.py +64 -0
  216. data/packages/gsutil/boto/boto/manage/server.py +556 -0
  217. data/packages/gsutil/boto/boto/manage/task.py +175 -0
  218. data/packages/gsutil/boto/boto/manage/test_manage.py +34 -0
  219. data/packages/gsutil/boto/boto/manage/volume.py +420 -0
  220. data/packages/gsutil/boto/boto/mashups/__init__.py +23 -0
  221. data/packages/gsutil/boto/boto/mashups/interactive.py +97 -0
  222. data/packages/gsutil/boto/boto/mashups/iobject.py +115 -0
  223. data/packages/gsutil/boto/boto/mashups/order.py +211 -0
  224. data/packages/gsutil/boto/boto/mashups/server.py +395 -0
  225. data/packages/gsutil/boto/boto/mturk/__init__.py +23 -0
  226. data/packages/gsutil/boto/boto/mturk/connection.py +1027 -0
  227. data/packages/gsutil/boto/boto/mturk/layoutparam.py +55 -0
  228. data/packages/gsutil/boto/boto/mturk/notification.py +103 -0
  229. data/packages/gsutil/boto/boto/mturk/price.py +48 -0
  230. data/packages/gsutil/boto/boto/mturk/qualification.py +137 -0
  231. data/packages/gsutil/boto/boto/mturk/question.py +455 -0
  232. data/packages/gsutil/boto/boto/mws/__init__.py +21 -0
  233. data/packages/gsutil/boto/boto/mws/connection.py +813 -0
  234. data/packages/gsutil/boto/boto/mws/exception.py +75 -0
  235. data/packages/gsutil/boto/boto/mws/response.py +655 -0
  236. data/packages/gsutil/boto/boto/plugin.py +90 -0
  237. data/packages/gsutil/boto/boto/plugin.pyc +0 -0
  238. data/packages/gsutil/boto/boto/provider.py +337 -0
  239. data/packages/gsutil/boto/boto/provider.pyc +0 -0
  240. data/packages/gsutil/boto/boto/pyami/__init__.py +22 -0
  241. data/packages/gsutil/boto/boto/pyami/__init__.pyc +0 -0
  242. data/packages/gsutil/boto/boto/pyami/bootstrap.py +134 -0
  243. data/packages/gsutil/boto/boto/pyami/config.py +229 -0
  244. data/packages/gsutil/boto/boto/pyami/config.pyc +0 -0
  245. data/packages/gsutil/boto/boto/pyami/copybot.cfg +60 -0
  246. data/packages/gsutil/boto/boto/pyami/copybot.py +97 -0
  247. data/packages/gsutil/boto/boto/pyami/helloworld.py +28 -0
  248. data/packages/gsutil/boto/boto/pyami/installers/__init__.py +64 -0
  249. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/__init__.py +22 -0
  250. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/apache.py +43 -0
  251. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/ebs.py +238 -0
  252. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/installer.py +96 -0
  253. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/mysql.py +109 -0
  254. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/trac.py +139 -0
  255. data/packages/gsutil/boto/boto/pyami/launch_ami.py +178 -0
  256. data/packages/gsutil/boto/boto/pyami/scriptbase.py +44 -0
  257. data/packages/gsutil/boto/boto/pyami/startup.py +60 -0
  258. data/packages/gsutil/boto/boto/rds/__init__.py +1194 -0
  259. data/packages/gsutil/boto/boto/rds/dbinstance.py +357 -0
  260. data/packages/gsutil/boto/boto/rds/dbsecuritygroup.py +177 -0
  261. data/packages/gsutil/boto/boto/rds/dbsnapshot.py +108 -0
  262. data/packages/gsutil/boto/boto/rds/event.py +49 -0
  263. data/packages/gsutil/boto/boto/rds/parametergroup.py +201 -0
  264. data/packages/gsutil/boto/boto/rds/regioninfo.py +32 -0
  265. data/packages/gsutil/boto/boto/regioninfo.py +63 -0
  266. data/packages/gsutil/boto/boto/regioninfo.pyc +0 -0
  267. data/packages/gsutil/boto/boto/resultset.py +169 -0
  268. data/packages/gsutil/boto/boto/resultset.pyc +0 -0
  269. data/packages/gsutil/boto/boto/roboto/__init__.py +1 -0
  270. data/packages/gsutil/boto/boto/roboto/awsqueryrequest.py +504 -0
  271. data/packages/gsutil/boto/boto/roboto/awsqueryservice.py +121 -0
  272. data/packages/gsutil/boto/boto/roboto/param.py +147 -0
  273. data/packages/gsutil/boto/boto/route53/__init__.py +75 -0
  274. data/packages/gsutil/boto/boto/route53/connection.py +403 -0
  275. data/packages/gsutil/boto/boto/route53/exception.py +27 -0
  276. data/packages/gsutil/boto/boto/route53/hostedzone.py +56 -0
  277. data/packages/gsutil/boto/boto/route53/record.py +306 -0
  278. data/packages/gsutil/boto/boto/route53/status.py +42 -0
  279. data/packages/gsutil/boto/boto/route53/zone.py +412 -0
  280. data/packages/gsutil/boto/boto/s3/__init__.py +84 -0
  281. data/packages/gsutil/boto/boto/s3/__init__.pyc +0 -0
  282. data/packages/gsutil/boto/boto/s3/acl.py +164 -0
  283. data/packages/gsutil/boto/boto/s3/acl.pyc +0 -0
  284. data/packages/gsutil/boto/boto/s3/bucket.py +1634 -0
  285. data/packages/gsutil/boto/boto/s3/bucket.pyc +0 -0
  286. data/packages/gsutil/boto/boto/s3/bucketlistresultset.py +139 -0
  287. data/packages/gsutil/boto/boto/s3/bucketlistresultset.pyc +0 -0
  288. data/packages/gsutil/boto/boto/s3/bucketlogging.py +83 -0
  289. data/packages/gsutil/boto/boto/s3/bucketlogging.pyc +0 -0
  290. data/packages/gsutil/boto/boto/s3/connection.py +540 -0
  291. data/packages/gsutil/boto/boto/s3/connection.pyc +0 -0
  292. data/packages/gsutil/boto/boto/s3/cors.py +210 -0
  293. data/packages/gsutil/boto/boto/s3/cors.pyc +0 -0
  294. data/packages/gsutil/boto/boto/s3/deletemarker.py +55 -0
  295. data/packages/gsutil/boto/boto/s3/deletemarker.pyc +0 -0
  296. data/packages/gsutil/boto/boto/s3/key.py +1712 -0
  297. data/packages/gsutil/boto/boto/s3/key.pyc +0 -0
  298. data/packages/gsutil/boto/boto/s3/keyfile.py +134 -0
  299. data/packages/gsutil/boto/boto/s3/keyfile.pyc +0 -0
  300. data/packages/gsutil/boto/boto/s3/lifecycle.py +231 -0
  301. data/packages/gsutil/boto/boto/s3/lifecycle.pyc +0 -0
  302. data/packages/gsutil/boto/boto/s3/multidelete.py +138 -0
  303. data/packages/gsutil/boto/boto/s3/multidelete.pyc +0 -0
  304. data/packages/gsutil/boto/boto/s3/multipart.py +315 -0
  305. data/packages/gsutil/boto/boto/s3/multipart.pyc +0 -0
  306. data/packages/gsutil/boto/boto/s3/prefix.py +42 -0
  307. data/packages/gsutil/boto/boto/s3/prefix.pyc +0 -0
  308. data/packages/gsutil/boto/boto/s3/resumable_download_handler.py +339 -0
  309. data/packages/gsutil/boto/boto/s3/resumable_download_handler.pyc +0 -0
  310. data/packages/gsutil/boto/boto/s3/tagging.py +71 -0
  311. data/packages/gsutil/boto/boto/s3/tagging.pyc +0 -0
  312. data/packages/gsutil/boto/boto/s3/user.py +49 -0
  313. data/packages/gsutil/boto/boto/s3/user.pyc +0 -0
  314. data/packages/gsutil/boto/boto/s3/website.py +237 -0
  315. data/packages/gsutil/boto/boto/s3/website.pyc +0 -0
  316. data/packages/gsutil/boto/boto/sdb/__init__.py +67 -0
  317. data/packages/gsutil/boto/boto/sdb/connection.py +617 -0
  318. data/packages/gsutil/boto/boto/sdb/db/__init__.py +20 -0
  319. data/packages/gsutil/boto/boto/sdb/db/blob.py +75 -0
  320. data/packages/gsutil/boto/boto/sdb/db/key.py +59 -0
  321. data/packages/gsutil/boto/boto/sdb/db/manager/__init__.py +85 -0
  322. data/packages/gsutil/boto/boto/sdb/db/manager/sdbmanager.py +732 -0
  323. data/packages/gsutil/boto/boto/sdb/db/manager/xmlmanager.py +517 -0
  324. data/packages/gsutil/boto/boto/sdb/db/model.py +294 -0
  325. data/packages/gsutil/boto/boto/sdb/db/property.py +703 -0
  326. data/packages/gsutil/boto/boto/sdb/db/query.py +85 -0
  327. data/packages/gsutil/boto/boto/sdb/db/sequence.py +226 -0
  328. data/packages/gsutil/boto/boto/sdb/db/test_db.py +231 -0
  329. data/packages/gsutil/boto/boto/sdb/domain.py +377 -0
  330. data/packages/gsutil/boto/boto/sdb/item.py +181 -0
  331. data/packages/gsutil/boto/boto/sdb/queryresultset.py +92 -0
  332. data/packages/gsutil/boto/boto/sdb/regioninfo.py +32 -0
  333. data/packages/gsutil/boto/boto/services/__init__.py +23 -0
  334. data/packages/gsutil/boto/boto/services/bs.py +179 -0
  335. data/packages/gsutil/boto/boto/services/message.py +58 -0
  336. data/packages/gsutil/boto/boto/services/result.py +136 -0
  337. data/packages/gsutil/boto/boto/services/service.py +161 -0
  338. data/packages/gsutil/boto/boto/services/servicedef.py +91 -0
  339. data/packages/gsutil/boto/boto/services/sonofmmm.cfg +43 -0
  340. data/packages/gsutil/boto/boto/services/sonofmmm.py +81 -0
  341. data/packages/gsutil/boto/boto/services/submit.py +88 -0
  342. data/packages/gsutil/boto/boto/ses/__init__.py +54 -0
  343. data/packages/gsutil/boto/boto/ses/connection.py +521 -0
  344. data/packages/gsutil/boto/boto/ses/exceptions.py +77 -0
  345. data/packages/gsutil/boto/boto/sns/__init__.py +78 -0
  346. data/packages/gsutil/boto/boto/sns/connection.py +431 -0
  347. data/packages/gsutil/boto/boto/sqs/__init__.py +56 -0
  348. data/packages/gsutil/boto/boto/sqs/attributes.py +46 -0
  349. data/packages/gsutil/boto/boto/sqs/batchresults.py +95 -0
  350. data/packages/gsutil/boto/boto/sqs/connection.py +417 -0
  351. data/packages/gsutil/boto/boto/sqs/jsonmessage.py +43 -0
  352. data/packages/gsutil/boto/boto/sqs/message.py +253 -0
  353. data/packages/gsutil/boto/boto/sqs/queue.py +478 -0
  354. data/packages/gsutil/boto/boto/sqs/regioninfo.py +32 -0
  355. data/packages/gsutil/boto/boto/storage_uri.py +835 -0
  356. data/packages/gsutil/boto/boto/storage_uri.pyc +0 -0
  357. data/packages/gsutil/boto/boto/sts/__init__.py +55 -0
  358. data/packages/gsutil/boto/boto/sts/connection.py +207 -0
  359. data/packages/gsutil/boto/boto/sts/credentials.py +215 -0
  360. data/packages/gsutil/boto/boto/swf/__init__.py +60 -0
  361. data/packages/gsutil/boto/boto/swf/exceptions.py +44 -0
  362. data/packages/gsutil/boto/boto/swf/layer1.py +1512 -0
  363. data/packages/gsutil/boto/boto/swf/layer1_decisions.py +287 -0
  364. data/packages/gsutil/boto/boto/swf/layer2.py +342 -0
  365. data/packages/gsutil/boto/boto/utils.py +927 -0
  366. data/packages/gsutil/boto/boto/utils.pyc +0 -0
  367. data/packages/gsutil/boto/boto/vpc/__init__.py +910 -0
  368. data/packages/gsutil/boto/boto/vpc/customergateway.py +54 -0
  369. data/packages/gsutil/boto/boto/vpc/dhcpoptions.py +72 -0
  370. data/packages/gsutil/boto/boto/vpc/internetgateway.py +72 -0
  371. data/packages/gsutil/boto/boto/vpc/routetable.py +109 -0
  372. data/packages/gsutil/boto/boto/vpc/subnet.py +57 -0
  373. data/packages/gsutil/boto/boto/vpc/vpc.py +54 -0
  374. data/packages/gsutil/boto/boto/vpc/vpnconnection.py +60 -0
  375. data/packages/gsutil/boto/boto/vpc/vpngateway.py +83 -0
  376. data/packages/gsutil/boto/docs/BotoCheatSheet.pdf +0 -0
  377. data/packages/gsutil/boto/docs/Makefile +89 -0
  378. data/packages/gsutil/boto/docs/make.bat +113 -0
  379. data/packages/gsutil/boto/docs/source/_templates/layout.html +3 -0
  380. data/packages/gsutil/boto/docs/source/autoscale_tut.rst +220 -0
  381. data/packages/gsutil/boto/docs/source/boto_config_tut.rst +125 -0
  382. data/packages/gsutil/boto/docs/source/boto_theme/static/boto.css_t +239 -0
  383. data/packages/gsutil/boto/docs/source/boto_theme/static/pygments.css +61 -0
  384. data/packages/gsutil/boto/docs/source/boto_theme/theme.conf +3 -0
  385. data/packages/gsutil/boto/docs/source/cloudfront_tut.rst +196 -0
  386. data/packages/gsutil/boto/docs/source/cloudsearch_tut.rst +411 -0
  387. data/packages/gsutil/boto/docs/source/cloudwatch_tut.rst +116 -0
  388. data/packages/gsutil/boto/docs/source/conf.py +32 -0
  389. data/packages/gsutil/boto/docs/source/contributing.rst +204 -0
  390. data/packages/gsutil/boto/docs/source/documentation.rst +59 -0
  391. data/packages/gsutil/boto/docs/source/dynamodb_tut.rst +339 -0
  392. data/packages/gsutil/boto/docs/source/ec2_tut.rst +86 -0
  393. data/packages/gsutil/boto/docs/source/elb_tut.rst +257 -0
  394. data/packages/gsutil/boto/docs/source/emr_tut.rst +108 -0
  395. data/packages/gsutil/boto/docs/source/index.rst +146 -0
  396. data/packages/gsutil/boto/docs/source/rds_tut.rst +108 -0
  397. data/packages/gsutil/boto/docs/source/ref/autoscale.rst +62 -0
  398. data/packages/gsutil/boto/docs/source/ref/beanstalk.rst +26 -0
  399. data/packages/gsutil/boto/docs/source/ref/boto.rst +47 -0
  400. data/packages/gsutil/boto/docs/source/ref/cloudformation.rst +34 -0
  401. data/packages/gsutil/boto/docs/source/ref/cloudfront.rst +68 -0
  402. data/packages/gsutil/boto/docs/source/ref/cloudsearch.rst +59 -0
  403. data/packages/gsutil/boto/docs/source/ref/cloudwatch.rst +27 -0
  404. data/packages/gsutil/boto/docs/source/ref/contrib.rst +32 -0
  405. data/packages/gsutil/boto/docs/source/ref/datapipeline.rst +26 -0
  406. data/packages/gsutil/boto/docs/source/ref/dynamodb.rst +61 -0
  407. data/packages/gsutil/boto/docs/source/ref/ec2.rst +140 -0
  408. data/packages/gsutil/boto/docs/source/ref/ecs.rst +19 -0
  409. data/packages/gsutil/boto/docs/source/ref/elasticache.rst +19 -0
  410. data/packages/gsutil/boto/docs/source/ref/elastictranscoder.rst +26 -0
  411. data/packages/gsutil/boto/docs/source/ref/elb.rst +47 -0
  412. data/packages/gsutil/boto/docs/source/ref/emr.rst +34 -0
  413. data/packages/gsutil/boto/docs/source/ref/file.rst +34 -0
  414. data/packages/gsutil/boto/docs/source/ref/fps.rst +19 -0
  415. data/packages/gsutil/boto/docs/source/ref/glacier.rst +63 -0
  416. data/packages/gsutil/boto/docs/source/ref/gs.rst +72 -0
  417. data/packages/gsutil/boto/docs/source/ref/iam.rst +27 -0
  418. data/packages/gsutil/boto/docs/source/ref/index.rst +40 -0
  419. data/packages/gsutil/boto/docs/source/ref/manage.rst +47 -0
  420. data/packages/gsutil/boto/docs/source/ref/mturk.rst +54 -0
  421. data/packages/gsutil/boto/docs/source/ref/mws.rst +33 -0
  422. data/packages/gsutil/boto/docs/source/ref/pyami.rst +103 -0
  423. data/packages/gsutil/boto/docs/source/ref/rds.rst +47 -0
  424. data/packages/gsutil/boto/docs/source/ref/route53.rst +34 -0
  425. data/packages/gsutil/boto/docs/source/ref/s3.rst +111 -0
  426. data/packages/gsutil/boto/docs/source/ref/sdb.rst +45 -0
  427. data/packages/gsutil/boto/docs/source/ref/sdb_db.rst +83 -0
  428. data/packages/gsutil/boto/docs/source/ref/services.rst +61 -0
  429. data/packages/gsutil/boto/docs/source/ref/ses.rst +21 -0
  430. data/packages/gsutil/boto/docs/source/ref/sns.rst +17 -0
  431. data/packages/gsutil/boto/docs/source/ref/sqs.rst +61 -0
  432. data/packages/gsutil/boto/docs/source/ref/sts.rst +25 -0
  433. data/packages/gsutil/boto/docs/source/ref/swf.rst +22 -0
  434. data/packages/gsutil/boto/docs/source/ref/vpc.rst +54 -0
  435. data/packages/gsutil/boto/docs/source/s3_tut.rst +450 -0
  436. data/packages/gsutil/boto/docs/source/security_groups.rst +82 -0
  437. data/packages/gsutil/boto/docs/source/ses_tut.rst +171 -0
  438. data/packages/gsutil/boto/docs/source/simpledb_tut.rst +188 -0
  439. data/packages/gsutil/boto/docs/source/sqs_tut.rst +246 -0
  440. data/packages/gsutil/boto/docs/source/vpc_tut.rst +100 -0
  441. data/packages/gsutil/boto/pylintrc +305 -0
  442. data/packages/gsutil/boto/requirements.txt +10 -0
  443. data/packages/gsutil/boto/setup.py +89 -0
  444. data/packages/gsutil/boto/tests/__init__.py +20 -0
  445. data/packages/gsutil/boto/tests/db/test_lists.py +96 -0
  446. data/packages/gsutil/boto/tests/db/test_password.py +128 -0
  447. data/packages/gsutil/boto/tests/db/test_query.py +152 -0
  448. data/packages/gsutil/boto/tests/db/test_sequence.py +109 -0
  449. data/packages/gsutil/boto/tests/devpay/__init__.py +0 -0
  450. data/packages/gsutil/boto/tests/devpay/test_s3.py +181 -0
  451. data/packages/gsutil/boto/tests/fps/__init__.py +0 -0
  452. data/packages/gsutil/boto/tests/fps/test.py +100 -0
  453. data/packages/gsutil/boto/tests/fps/test_verify_signature.py +12 -0
  454. data/packages/gsutil/boto/tests/integration/__init__.py +0 -0
  455. data/packages/gsutil/boto/tests/integration/beanstalk/test_wrapper.py +209 -0
  456. data/packages/gsutil/boto/tests/integration/cloudformation/__init__.py +21 -0
  457. data/packages/gsutil/boto/tests/integration/cloudformation/test_cert_verification.py +40 -0
  458. data/packages/gsutil/boto/tests/integration/cloudformation/test_connection.py +110 -0
  459. data/packages/gsutil/boto/tests/integration/cloudsearch/__init__.py +21 -0
  460. data/packages/gsutil/boto/tests/integration/cloudsearch/test_cert_verification.py +40 -0
  461. data/packages/gsutil/boto/tests/integration/datapipeline/test_layer1.py +122 -0
  462. data/packages/gsutil/boto/tests/integration/dynamodb/__init__.py +20 -0
  463. data/packages/gsutil/boto/tests/integration/dynamodb/test_cert_verification.py +40 -0
  464. data/packages/gsutil/boto/tests/integration/dynamodb/test_layer1.py +266 -0
  465. data/packages/gsutil/boto/tests/integration/dynamodb/test_layer2.py +484 -0
  466. data/packages/gsutil/boto/tests/integration/dynamodb/test_table.py +84 -0
  467. data/packages/gsutil/boto/tests/integration/ec2/__init__.py +20 -0
  468. data/packages/gsutil/boto/tests/integration/ec2/autoscale/__init__.py +21 -0
  469. data/packages/gsutil/boto/tests/integration/ec2/autoscale/test_cert_verification.py +40 -0
  470. data/packages/gsutil/boto/tests/integration/ec2/autoscale/test_connection.py +167 -0
  471. data/packages/gsutil/boto/tests/integration/ec2/cloudwatch/__init__.py +20 -0
  472. data/packages/gsutil/boto/tests/integration/ec2/cloudwatch/test_cert_verification.py +40 -0
  473. data/packages/gsutil/boto/tests/integration/ec2/cloudwatch/test_connection.py +277 -0
  474. data/packages/gsutil/boto/tests/integration/ec2/elb/__init__.py +20 -0
  475. data/packages/gsutil/boto/tests/integration/ec2/elb/test_cert_verification.py +40 -0
  476. data/packages/gsutil/boto/tests/integration/ec2/elb/test_connection.py +130 -0
  477. data/packages/gsutil/boto/tests/integration/ec2/test_cert_verification.py +40 -0
  478. data/packages/gsutil/boto/tests/integration/ec2/test_connection.py +192 -0
  479. data/packages/gsutil/boto/tests/integration/ec2/vpc/__init__.py +0 -0
  480. data/packages/gsutil/boto/tests/integration/ec2/vpc/test_connection.py +95 -0
  481. data/packages/gsutil/boto/tests/integration/elasticache/__init__.py +0 -0
  482. data/packages/gsutil/boto/tests/integration/elasticache/test_layer1.py +67 -0
  483. data/packages/gsutil/boto/tests/integration/elastictranscoder/__init__.py +0 -0
  484. data/packages/gsutil/boto/tests/integration/elastictranscoder/test_cert_verification.py +35 -0
  485. data/packages/gsutil/boto/tests/integration/elastictranscoder/test_layer1.py +115 -0
  486. data/packages/gsutil/boto/tests/integration/emr/__init__.py +20 -0
  487. data/packages/gsutil/boto/tests/integration/emr/test_cert_verification.py +40 -0
  488. data/packages/gsutil/boto/tests/integration/glacier/__init__.py +22 -0
  489. data/packages/gsutil/boto/tests/integration/glacier/test_cert_verification.py +40 -0
  490. data/packages/gsutil/boto/tests/integration/glacier/test_layer1.py +44 -0
  491. data/packages/gsutil/boto/tests/integration/glacier/test_layer2.py +45 -0
  492. data/packages/gsutil/boto/tests/integration/gs/__init__.py +0 -0
  493. data/packages/gsutil/boto/tests/integration/gs/cb_test_harness.py +71 -0
  494. data/packages/gsutil/boto/tests/integration/gs/test_basic.py +379 -0
  495. data/packages/gsutil/boto/tests/integration/gs/test_generation_conditionals.py +399 -0
  496. data/packages/gsutil/boto/tests/integration/gs/test_resumable_downloads.py +358 -0
  497. data/packages/gsutil/boto/tests/integration/gs/test_resumable_uploads.py +525 -0
  498. data/packages/gsutil/boto/tests/integration/gs/test_storage_uri.py +125 -0
  499. data/packages/gsutil/boto/tests/integration/gs/test_versioning.py +268 -0
  500. data/packages/gsutil/boto/tests/integration/gs/testcase.py +116 -0
  501. data/packages/gsutil/boto/tests/integration/gs/util.py +63 -0
  502. data/packages/gsutil/boto/tests/integration/iam/__init__.py +20 -0
  503. data/packages/gsutil/boto/tests/integration/iam/test_cert_verification.py +40 -0
  504. data/packages/gsutil/boto/tests/integration/mws/__init__.py +0 -0
  505. data/packages/gsutil/boto/tests/integration/mws/test.py +100 -0
  506. data/packages/gsutil/boto/tests/integration/rds/__init__.py +21 -0
  507. data/packages/gsutil/boto/tests/integration/rds/test_cert_verification.py +40 -0
  508. data/packages/gsutil/boto/tests/integration/route53/__init__.py +20 -0
  509. data/packages/gsutil/boto/tests/integration/route53/test_cert_verification.py +40 -0
  510. data/packages/gsutil/boto/tests/integration/route53/test_zone.py +132 -0
  511. data/packages/gsutil/boto/tests/integration/s3/__init__.py +20 -0
  512. data/packages/gsutil/boto/tests/integration/s3/mock_storage_service.py +589 -0
  513. data/packages/gsutil/boto/tests/integration/s3/other_cacerts.txt +70 -0
  514. data/packages/gsutil/boto/tests/integration/s3/test_bucket.py +263 -0
  515. data/packages/gsutil/boto/tests/integration/s3/test_cert_verification.py +40 -0
  516. data/packages/gsutil/boto/tests/integration/s3/test_connection.py +245 -0
  517. data/packages/gsutil/boto/tests/integration/s3/test_cors.py +78 -0
  518. data/packages/gsutil/boto/tests/integration/s3/test_encryption.py +115 -0
  519. data/packages/gsutil/boto/tests/integration/s3/test_https_cert_validation.py +141 -0
  520. data/packages/gsutil/boto/tests/integration/s3/test_key.py +375 -0
  521. data/packages/gsutil/boto/tests/integration/s3/test_mfa.py +95 -0
  522. data/packages/gsutil/boto/tests/integration/s3/test_multidelete.py +181 -0
  523. data/packages/gsutil/boto/tests/integration/s3/test_multipart.py +139 -0
  524. data/packages/gsutil/boto/tests/integration/s3/test_pool.py +246 -0
  525. data/packages/gsutil/boto/tests/integration/s3/test_versioning.py +158 -0
  526. data/packages/gsutil/boto/tests/integration/sdb/__init__.py +20 -0
  527. data/packages/gsutil/boto/tests/integration/sdb/test_cert_verification.py +40 -0
  528. data/packages/gsutil/boto/tests/integration/sdb/test_connection.py +119 -0
  529. data/packages/gsutil/boto/tests/integration/ses/__init__.py +0 -0
  530. data/packages/gsutil/boto/tests/integration/ses/test_cert_verification.py +40 -0
  531. data/packages/gsutil/boto/tests/integration/ses/test_connection.py +38 -0
  532. data/packages/gsutil/boto/tests/integration/sns/__init__.py +20 -0
  533. data/packages/gsutil/boto/tests/integration/sns/test_cert_verification.py +40 -0
  534. data/packages/gsutil/boto/tests/integration/sqs/__init__.py +20 -0
  535. data/packages/gsutil/boto/tests/integration/sqs/test_cert_verification.py +40 -0
  536. data/packages/gsutil/boto/tests/integration/sqs/test_connection.py +217 -0
  537. data/packages/gsutil/boto/tests/integration/sts/__init__.py +20 -0
  538. data/packages/gsutil/boto/tests/integration/sts/test_cert_verification.py +40 -0
  539. data/packages/gsutil/boto/tests/integration/sts/test_session_token.py +65 -0
  540. data/packages/gsutil/boto/tests/integration/swf/__init__.py +0 -0
  541. data/packages/gsutil/boto/tests/integration/swf/test_cert_verification.py +40 -0
  542. data/packages/gsutil/boto/tests/integration/swf/test_layer1.py +246 -0
  543. data/packages/gsutil/boto/tests/integration/swf/test_layer1_workflow_execution.py +173 -0
  544. data/packages/gsutil/boto/tests/mturk/__init__.py +0 -0
  545. data/packages/gsutil/boto/tests/mturk/_init_environment.py +28 -0
  546. data/packages/gsutil/boto/tests/mturk/all_tests.py +24 -0
  547. data/packages/gsutil/boto/tests/mturk/cleanup_tests.py +47 -0
  548. data/packages/gsutil/boto/tests/mturk/common.py +45 -0
  549. data/packages/gsutil/boto/tests/mturk/create_free_text_question_regex.doctest +100 -0
  550. data/packages/gsutil/boto/tests/mturk/create_hit.doctest +92 -0
  551. data/packages/gsutil/boto/tests/mturk/create_hit_binary.doctest +94 -0
  552. data/packages/gsutil/boto/tests/mturk/create_hit_external.py +21 -0
  553. data/packages/gsutil/boto/tests/mturk/create_hit_from_hit_type.doctest +103 -0
  554. data/packages/gsutil/boto/tests/mturk/create_hit_test.py +21 -0
  555. data/packages/gsutil/boto/tests/mturk/create_hit_with_qualifications.py +16 -0
  556. data/packages/gsutil/boto/tests/mturk/hit_persistence.py +27 -0
  557. data/packages/gsutil/boto/tests/mturk/mocks.py +11 -0
  558. data/packages/gsutil/boto/tests/mturk/reviewable_hits.doctest +129 -0
  559. data/packages/gsutil/boto/tests/mturk/run-doctest.py +13 -0
  560. data/packages/gsutil/boto/tests/mturk/search_hits.doctest +16 -0
  561. data/packages/gsutil/boto/tests/mturk/selenium_support.py +61 -0
  562. data/packages/gsutil/boto/tests/mturk/support.py +7 -0
  563. data/packages/gsutil/boto/tests/mturk/test_disable_hit.py +11 -0
  564. data/packages/gsutil/boto/tests/test.py +59 -0
  565. data/packages/gsutil/boto/tests/unit/__init__.py +79 -0
  566. data/packages/gsutil/boto/tests/unit/auth/__init__.py +0 -0
  567. data/packages/gsutil/boto/tests/unit/auth/test_sigv4.py +73 -0
  568. data/packages/gsutil/boto/tests/unit/beanstalk/__init__.py +0 -0
  569. data/packages/gsutil/boto/tests/unit/beanstalk/test_layer1.py +128 -0
  570. data/packages/gsutil/boto/tests/unit/cloudformation/__init__.py +0 -0
  571. data/packages/gsutil/boto/tests/unit/cloudformation/test_connection.py +605 -0
  572. data/packages/gsutil/boto/tests/unit/cloudformation/test_stack.py +63 -0
  573. data/packages/gsutil/boto/tests/unit/cloudfront/__init__.py +0 -0
  574. data/packages/gsutil/boto/tests/unit/cloudfront/test_invalidation_list.py +113 -0
  575. data/packages/gsutil/boto/tests/unit/cloudfront/test_signed_urls.py +354 -0
  576. data/packages/gsutil/boto/tests/unit/cloudsearch/__init__.py +1 -0
  577. data/packages/gsutil/boto/tests/unit/cloudsearch/test_connection.py +241 -0
  578. data/packages/gsutil/boto/tests/unit/cloudsearch/test_document.py +324 -0
  579. data/packages/gsutil/boto/tests/unit/cloudsearch/test_search.py +325 -0
  580. data/packages/gsutil/boto/tests/unit/dynamodb/__init__.py +0 -0
  581. data/packages/gsutil/boto/tests/unit/dynamodb/test_batch.py +103 -0
  582. data/packages/gsutil/boto/tests/unit/dynamodb/test_layer2.py +119 -0
  583. data/packages/gsutil/boto/tests/unit/dynamodb/test_types.py +82 -0
  584. data/packages/gsutil/boto/tests/unit/ec2/__init__.py +0 -0
  585. data/packages/gsutil/boto/tests/unit/ec2/autoscale/__init__.py +0 -0
  586. data/packages/gsutil/boto/tests/unit/ec2/autoscale/test_group.py +162 -0
  587. data/packages/gsutil/boto/tests/unit/ec2/test_address.py +39 -0
  588. data/packages/gsutil/boto/tests/unit/ec2/test_blockdevicemapping.py +79 -0
  589. data/packages/gsutil/boto/tests/unit/ec2/test_connection.py +480 -0
  590. data/packages/gsutil/boto/tests/unit/ec2/test_instance.py +243 -0
  591. data/packages/gsutil/boto/tests/unit/ec2/test_networkinterface.py +140 -0
  592. data/packages/gsutil/boto/tests/unit/ec2/test_volume.py +248 -0
  593. data/packages/gsutil/boto/tests/unit/emr/test_emr_responses.py +373 -0
  594. data/packages/gsutil/boto/tests/unit/glacier/__init__.py +0 -0
  595. data/packages/gsutil/boto/tests/unit/glacier/test_concurrent.py +120 -0
  596. data/packages/gsutil/boto/tests/unit/glacier/test_job.py +60 -0
  597. data/packages/gsutil/boto/tests/unit/glacier/test_layer1.py +98 -0
  598. data/packages/gsutil/boto/tests/unit/glacier/test_layer2.py +266 -0
  599. data/packages/gsutil/boto/tests/unit/glacier/test_utils.py +116 -0
  600. data/packages/gsutil/boto/tests/unit/glacier/test_vault.py +100 -0
  601. data/packages/gsutil/boto/tests/unit/glacier/test_writer.py +185 -0
  602. data/packages/gsutil/boto/tests/unit/provider/__init__.py +0 -0
  603. data/packages/gsutil/boto/tests/unit/provider/test_provider.py +176 -0
  604. data/packages/gsutil/boto/tests/unit/rds/__init__.py +0 -0
  605. data/packages/gsutil/boto/tests/unit/rds/test_connection.py +131 -0
  606. data/packages/gsutil/boto/tests/unit/s3/__init__.py +0 -0
  607. data/packages/gsutil/boto/tests/unit/s3/test_cors_configuration.py +77 -0
  608. data/packages/gsutil/boto/tests/unit/s3/test_key.py +75 -0
  609. data/packages/gsutil/boto/tests/unit/s3/test_keyfile.py +101 -0
  610. data/packages/gsutil/boto/tests/unit/s3/test_lifecycle.py +97 -0
  611. data/packages/gsutil/boto/tests/unit/s3/test_tagging.py +47 -0
  612. data/packages/gsutil/boto/tests/unit/s3/test_uri.py +257 -0
  613. data/packages/gsutil/boto/tests/unit/s3/test_website.py +188 -0
  614. data/packages/gsutil/boto/tests/unit/sns/__init__.py +0 -0
  615. data/packages/gsutil/boto/tests/unit/sns/test_connection.py +99 -0
  616. data/packages/gsutil/boto/tests/unit/sqs/__init__.py +0 -0
  617. data/packages/gsutil/boto/tests/unit/sqs/test_connection.py +98 -0
  618. data/packages/gsutil/boto/tests/unit/sqs/test_queue.py +40 -0
  619. data/packages/gsutil/boto/tests/unit/sts/test_connection.py +74 -0
  620. data/packages/gsutil/boto/tests/unit/test_connection.py +60 -0
  621. data/packages/gsutil/boto/tests/unit/utils/test_utils.py +109 -0
  622. data/packages/gsutil/boto/tox.ini +8 -0
  623. data/packages/gsutil/gslib/README +5 -0
  624. data/packages/gsutil/gslib/__init__.py +22 -0
  625. data/packages/gsutil/gslib/__init__.pyc +0 -0
  626. data/packages/gsutil/gslib/addlhelp/__init__.py +15 -0
  627. data/packages/gsutil/gslib/addlhelp/acls.py +234 -0
  628. data/packages/gsutil/gslib/addlhelp/anon.py +57 -0
  629. data/packages/gsutil/gslib/addlhelp/command_opts.py +116 -0
  630. data/packages/gsutil/gslib/addlhelp/dev.py +139 -0
  631. data/packages/gsutil/gslib/addlhelp/metadata.py +186 -0
  632. data/packages/gsutil/gslib/addlhelp/naming.py +173 -0
  633. data/packages/gsutil/gslib/addlhelp/prod.py +160 -0
  634. data/packages/gsutil/gslib/addlhelp/projects.py +130 -0
  635. data/packages/gsutil/gslib/addlhelp/subdirs.py +110 -0
  636. data/packages/gsutil/gslib/addlhelp/support.py +86 -0
  637. data/packages/gsutil/gslib/addlhelp/versioning.py +242 -0
  638. data/packages/gsutil/gslib/addlhelp/wildcards.py +170 -0
  639. data/packages/gsutil/gslib/bucket_listing_ref.py +175 -0
  640. data/packages/gsutil/gslib/bucket_listing_ref.pyc +0 -0
  641. data/packages/gsutil/gslib/command.py +722 -0
  642. data/packages/gsutil/gslib/command.pyc +0 -0
  643. data/packages/gsutil/gslib/command_runner.py +101 -0
  644. data/packages/gsutil/gslib/command_runner.pyc +0 -0
  645. data/packages/gsutil/gslib/commands/__init__.py +15 -0
  646. data/packages/gsutil/gslib/commands/__init__.pyc +0 -0
  647. data/packages/gsutil/gslib/commands/cat.py +131 -0
  648. data/packages/gsutil/gslib/commands/cat.pyc +0 -0
  649. data/packages/gsutil/gslib/commands/chacl.py +532 -0
  650. data/packages/gsutil/gslib/commands/chacl.pyc +0 -0
  651. data/packages/gsutil/gslib/commands/config.py +694 -0
  652. data/packages/gsutil/gslib/commands/config.pyc +0 -0
  653. data/packages/gsutil/gslib/commands/cp.py +1818 -0
  654. data/packages/gsutil/gslib/commands/cp.pyc +0 -0
  655. data/packages/gsutil/gslib/commands/disablelogging.py +101 -0
  656. data/packages/gsutil/gslib/commands/disablelogging.pyc +0 -0
  657. data/packages/gsutil/gslib/commands/enablelogging.py +149 -0
  658. data/packages/gsutil/gslib/commands/enablelogging.pyc +0 -0
  659. data/packages/gsutil/gslib/commands/getacl.py +82 -0
  660. data/packages/gsutil/gslib/commands/getacl.pyc +0 -0
  661. data/packages/gsutil/gslib/commands/getcors.py +121 -0
  662. data/packages/gsutil/gslib/commands/getcors.pyc +0 -0
  663. data/packages/gsutil/gslib/commands/getdefacl.py +86 -0
  664. data/packages/gsutil/gslib/commands/getdefacl.pyc +0 -0
  665. data/packages/gsutil/gslib/commands/getlogging.py +137 -0
  666. data/packages/gsutil/gslib/commands/getlogging.pyc +0 -0
  667. data/packages/gsutil/gslib/commands/getversioning.py +116 -0
  668. data/packages/gsutil/gslib/commands/getversioning.pyc +0 -0
  669. data/packages/gsutil/gslib/commands/getwebcfg.py +122 -0
  670. data/packages/gsutil/gslib/commands/getwebcfg.pyc +0 -0
  671. data/packages/gsutil/gslib/commands/help.py +218 -0
  672. data/packages/gsutil/gslib/commands/help.pyc +0 -0
  673. data/packages/gsutil/gslib/commands/ls.py +578 -0
  674. data/packages/gsutil/gslib/commands/ls.pyc +0 -0
  675. data/packages/gsutil/gslib/commands/mb.py +172 -0
  676. data/packages/gsutil/gslib/commands/mb.pyc +0 -0
  677. data/packages/gsutil/gslib/commands/mv.py +159 -0
  678. data/packages/gsutil/gslib/commands/mv.pyc +0 -0
  679. data/packages/gsutil/gslib/commands/perfdiag.py +903 -0
  680. data/packages/gsutil/gslib/commands/perfdiag.pyc +0 -0
  681. data/packages/gsutil/gslib/commands/rb.py +113 -0
  682. data/packages/gsutil/gslib/commands/rb.pyc +0 -0
  683. data/packages/gsutil/gslib/commands/rm.py +239 -0
  684. data/packages/gsutil/gslib/commands/rm.pyc +0 -0
  685. data/packages/gsutil/gslib/commands/setacl.py +138 -0
  686. data/packages/gsutil/gslib/commands/setacl.pyc +0 -0
  687. data/packages/gsutil/gslib/commands/setcors.py +145 -0
  688. data/packages/gsutil/gslib/commands/setcors.pyc +0 -0
  689. data/packages/gsutil/gslib/commands/setdefacl.py +105 -0
  690. data/packages/gsutil/gslib/commands/setdefacl.pyc +0 -0
  691. data/packages/gsutil/gslib/commands/setmeta.py +428 -0
  692. data/packages/gsutil/gslib/commands/setmeta.pyc +0 -0
  693. data/packages/gsutil/gslib/commands/setversioning.py +114 -0
  694. data/packages/gsutil/gslib/commands/setversioning.pyc +0 -0
  695. data/packages/gsutil/gslib/commands/setwebcfg.py +190 -0
  696. data/packages/gsutil/gslib/commands/setwebcfg.pyc +0 -0
  697. data/packages/gsutil/gslib/commands/test.py +228 -0
  698. data/packages/gsutil/gslib/commands/test.pyc +0 -0
  699. data/packages/gsutil/gslib/commands/update.py +305 -0
  700. data/packages/gsutil/gslib/commands/update.pyc +0 -0
  701. data/packages/gsutil/gslib/commands/version.py +150 -0
  702. data/packages/gsutil/gslib/commands/version.pyc +0 -0
  703. data/packages/gsutil/gslib/exception.py +76 -0
  704. data/packages/gsutil/gslib/exception.pyc +0 -0
  705. data/packages/gsutil/gslib/help_provider.py +81 -0
  706. data/packages/gsutil/gslib/help_provider.pyc +0 -0
  707. data/packages/gsutil/gslib/name_expansion.py +550 -0
  708. data/packages/gsutil/gslib/name_expansion.pyc +0 -0
  709. data/packages/gsutil/gslib/no_op_auth_plugin.py +30 -0
  710. data/packages/gsutil/gslib/plurality_checkable_iterator.py +56 -0
  711. data/packages/gsutil/gslib/plurality_checkable_iterator.pyc +0 -0
  712. data/packages/gsutil/gslib/project_id.py +67 -0
  713. data/packages/gsutil/gslib/project_id.pyc +0 -0
  714. data/packages/gsutil/gslib/storage_uri_builder.py +56 -0
  715. data/packages/gsutil/gslib/storage_uri_builder.pyc +0 -0
  716. data/packages/gsutil/gslib/tests/__init__.py +15 -0
  717. data/packages/gsutil/gslib/tests/__init__.pyc +0 -0
  718. data/packages/gsutil/gslib/tests/test_chacl.py +236 -0
  719. data/packages/gsutil/gslib/tests/test_cp.py +267 -0
  720. data/packages/gsutil/gslib/tests/test_data/test.gif +0 -0
  721. data/packages/gsutil/gslib/tests/test_data/test.mp3 +0 -0
  722. data/packages/gsutil/gslib/tests/test_ls.py +66 -0
  723. data/packages/gsutil/gslib/tests/test_mv.py +69 -0
  724. data/packages/gsutil/gslib/tests/test_naming.py +989 -0
  725. data/packages/gsutil/gslib/tests/test_perfdiag.py +41 -0
  726. data/packages/gsutil/gslib/tests/test_plurality_checkable_iterator.py +67 -0
  727. data/packages/gsutil/gslib/tests/test_rm.py +143 -0
  728. data/packages/gsutil/gslib/tests/test_setacl.py +152 -0
  729. data/packages/gsutil/gslib/tests/test_setcors.py +168 -0
  730. data/packages/gsutil/gslib/tests/test_setmeta.py +91 -0
  731. data/packages/gsutil/gslib/tests/test_setversioning.py +44 -0
  732. data/packages/gsutil/gslib/tests/test_setwebcfg.py +63 -0
  733. data/packages/gsutil/gslib/tests/test_thread_pool.py +92 -0
  734. data/packages/gsutil/gslib/tests/test_wildcard_iterator.py +364 -0
  735. data/packages/gsutil/gslib/tests/testcase/__init__.py +18 -0
  736. data/packages/gsutil/gslib/tests/testcase/base.py +89 -0
  737. data/packages/gsutil/gslib/tests/testcase/integration_testcase.py +197 -0
  738. data/packages/gsutil/gslib/tests/testcase/unit_testcase.py +230 -0
  739. data/packages/gsutil/gslib/tests/util.py +125 -0
  740. data/packages/gsutil/gslib/tests/util.pyc +0 -0
  741. data/packages/gsutil/gslib/thread_pool.py +79 -0
  742. data/packages/gsutil/gslib/thread_pool.pyc +0 -0
  743. data/packages/gsutil/gslib/util.py +151 -0
  744. data/packages/gsutil/gslib/util.pyc +0 -0
  745. data/packages/gsutil/gslib/wildcard_iterator.py +492 -0
  746. data/packages/gsutil/gslib/wildcard_iterator.pyc +0 -0
  747. data/packages/gsutil/gsutil +377 -0
  748. data/packages/gsutil/gsutil.spec.in +75 -0
  749. data/packages/gsutil/oauth2_plugin/__init__.py +22 -0
  750. data/packages/gsutil/oauth2_plugin/__init__.pyc +0 -0
  751. data/packages/gsutil/oauth2_plugin/oauth2_client.py +642 -0
  752. data/packages/gsutil/oauth2_plugin/oauth2_client.pyc +0 -0
  753. data/packages/gsutil/oauth2_plugin/oauth2_client_test.py +374 -0
  754. data/packages/gsutil/oauth2_plugin/oauth2_helper.py +110 -0
  755. data/packages/gsutil/oauth2_plugin/oauth2_helper.pyc +0 -0
  756. data/packages/gsutil/oauth2_plugin/oauth2_plugin.py +24 -0
  757. data/packages/gsutil/oauth2_plugin/oauth2_plugin.pyc +0 -0
  758. data/packages/gsutil/pkg_gen.sh +54 -0
  759. data/packages/gsutil/pkg_util.py +60 -0
  760. data/packages/gsutil/setup.py +141 -0
  761. data/packages/gsutil/third_party/__init__.py +0 -0
  762. data/packages/gsutil/third_party/__init__.pyc +0 -0
  763. data/packages/gsutil/third_party/fancy_urllib/README +21 -0
  764. data/packages/gsutil/third_party/fancy_urllib/__init__.py +398 -0
  765. data/packages/gsutil/third_party/fancy_urllib/__init__.pyc +0 -0
  766. data/tasks/rubygem.rake +3 -1
  767. metadata +770 -7
  768. metadata.gz.sig +0 -0
@@ -0,0 +1,659 @@
1
+ # Copyright 2010 Google Inc.
2
+ #
3
+ # Permission is hereby granted, free of charge, to any person obtaining a
4
+ # copy of this software and associated documentation files (the
5
+ # "Software"), to deal in the Software without restriction, including
6
+ # without limitation the rights to use, copy, modify, merge, publish, dis-
7
+ # tribute, sublicense, and/or sell copies of the Software, and to permit
8
+ # persons to whom the Software is furnished to do so, subject to the fol-
9
+ # lowing conditions:
10
+ #
11
+ # The above copyright notice and this permission notice shall be included
12
+ # in all copies or substantial portions of the Software.
13
+ #
14
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
15
+ # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
16
+ # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
17
+ # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
18
+ # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
20
+ # IN THE SOFTWARE.
21
+
22
+ import cgi
23
+ import errno
24
+ import httplib
25
+ import os
26
+ import random
27
+ import re
28
+ import socket
29
+ import time
30
+ import urlparse
31
+ import boto
32
+ from boto import config, UserAgent
33
+ from boto.connection import AWSAuthConnection
34
+ from boto.exception import InvalidUriError
35
+ from boto.exception import ResumableTransferDisposition
36
+ from boto.exception import ResumableUploadException
37
+ from boto.s3.keyfile import KeyFile
38
+ try:
39
+ from hashlib import md5
40
+ except ImportError:
41
+ from md5 import md5
42
+
43
+ """
44
+ Handler for Google Cloud Storage resumable uploads. See
45
+ http://code.google.com/apis/storage/docs/developer-guide.html#resumable
46
+ for details.
47
+
48
+ Resumable uploads will retry failed uploads, resuming at the byte
49
+ count completed by the last upload attempt. If too many retries happen with
50
+ no progress (per configurable num_retries param), the upload will be
51
+ aborted in the current process.
52
+
53
+ The caller can optionally specify a tracker_file_name param in the
54
+ ResumableUploadHandler constructor. If you do this, that file will
55
+ save the state needed to allow retrying later, in a separate process
56
+ (e.g., in a later run of gsutil).
57
+ """
58
+
59
+
60
+ class ResumableUploadHandler(object):
61
+
62
+ BUFFER_SIZE = 8192
63
+ RETRYABLE_EXCEPTIONS = (httplib.HTTPException, IOError, socket.error,
64
+ socket.gaierror)
65
+
66
+ # (start, end) response indicating server has nothing (upload protocol uses
67
+ # inclusive numbering).
68
+ SERVER_HAS_NOTHING = (0, -1)
69
+
70
+ def __init__(self, tracker_file_name=None, num_retries=None):
71
+ """
72
+ Constructor. Instantiate once for each uploaded file.
73
+
74
+ :type tracker_file_name: string
75
+ :param tracker_file_name: optional file name to save tracker URI.
76
+ If supplied and the current process fails the upload, it can be
77
+ retried in a new process. If called with an existing file containing
78
+ a valid tracker URI, we'll resume the upload from this URI; else
79
+ we'll start a new resumable upload (and write the URI to this
80
+ tracker file).
81
+
82
+ :type num_retries: int
83
+ :param num_retries: the number of times we'll re-try a resumable upload
84
+ making no progress. (Count resets every time we get progress, so
85
+ upload can span many more than this number of retries.)
86
+ """
87
+ self.tracker_file_name = tracker_file_name
88
+ self.num_retries = num_retries
89
+ self.server_has_bytes = 0 # Byte count at last server check.
90
+ self.tracker_uri = None
91
+ if tracker_file_name:
92
+ self._load_tracker_uri_from_file()
93
+ # Save upload_start_point in instance state so caller can find how
94
+ # much was transferred by this ResumableUploadHandler (across retries).
95
+ self.upload_start_point = None
96
+
97
+ def _load_tracker_uri_from_file(self):
98
+ f = None
99
+ try:
100
+ f = open(self.tracker_file_name, 'r')
101
+ uri = f.readline().strip()
102
+ self._set_tracker_uri(uri)
103
+ except IOError, e:
104
+ # Ignore non-existent file (happens first time an upload
105
+ # is attempted on a file), but warn user for other errors.
106
+ if e.errno != errno.ENOENT:
107
+ # Will restart because self.tracker_uri == None.
108
+ print('Couldn\'t read URI tracker file (%s): %s. Restarting '
109
+ 'upload from scratch.' %
110
+ (self.tracker_file_name, e.strerror))
111
+ except InvalidUriError, e:
112
+ # Warn user, but proceed (will restart because
113
+ # self.tracker_uri == None).
114
+ print('Invalid tracker URI (%s) found in URI tracker file '
115
+ '(%s). Restarting upload from scratch.' %
116
+ (uri, self.tracker_file_name))
117
+ finally:
118
+ if f:
119
+ f.close()
120
+
121
+ def _save_tracker_uri_to_file(self):
122
+ """
123
+ Saves URI to tracker file if one was passed to constructor.
124
+ """
125
+ if not self.tracker_file_name:
126
+ return
127
+ f = None
128
+ try:
129
+ f = open(self.tracker_file_name, 'w')
130
+ f.write(self.tracker_uri)
131
+ except IOError, e:
132
+ raise ResumableUploadException(
133
+ 'Couldn\'t write URI tracker file (%s): %s.\nThis can happen'
134
+ 'if you\'re using an incorrectly configured upload tool\n'
135
+ '(e.g., gsutil configured to save tracker files to an '
136
+ 'unwritable directory)' %
137
+ (self.tracker_file_name, e.strerror),
138
+ ResumableTransferDisposition.ABORT)
139
+ finally:
140
+ if f:
141
+ f.close()
142
+
143
+ def _set_tracker_uri(self, uri):
144
+ """
145
+ Called when we start a new resumable upload or get a new tracker
146
+ URI for the upload. Saves URI and resets upload state.
147
+
148
+ Raises InvalidUriError if URI is syntactically invalid.
149
+ """
150
+ parse_result = urlparse.urlparse(uri)
151
+ if (parse_result.scheme.lower() not in ['http', 'https'] or
152
+ not parse_result.netloc):
153
+ raise InvalidUriError('Invalid tracker URI (%s)' % uri)
154
+ self.tracker_uri = uri
155
+ self.tracker_uri_host = parse_result.netloc
156
+ self.tracker_uri_path = '%s?%s' % (
157
+ parse_result.path, parse_result.query)
158
+ self.server_has_bytes = 0
159
+
160
+ def get_tracker_uri(self):
161
+ """
162
+ Returns upload tracker URI, or None if the upload has not yet started.
163
+ """
164
+ return self.tracker_uri
165
+
166
+ def _remove_tracker_file(self):
167
+ if (self.tracker_file_name and
168
+ os.path.exists(self.tracker_file_name)):
169
+ os.unlink(self.tracker_file_name)
170
+
171
+ def _build_content_range_header(self, range_spec='*', length_spec='*'):
172
+ return 'bytes %s/%s' % (range_spec, length_spec)
173
+
174
+ def _query_server_state(self, conn, file_length):
175
+ """
176
+ Queries server to find out state of given upload.
177
+
178
+ Note that this method really just makes special case use of the
179
+ fact that the upload server always returns the current start/end
180
+ state whenever a PUT doesn't complete.
181
+
182
+ Returns HTTP response from sending request.
183
+
184
+ Raises ResumableUploadException if problem querying server.
185
+ """
186
+ # Send an empty PUT so that server replies with this resumable
187
+ # transfer's state.
188
+ put_headers = {}
189
+ put_headers['Content-Range'] = (
190
+ self._build_content_range_header('*', file_length))
191
+ put_headers['Content-Length'] = '0'
192
+ return AWSAuthConnection.make_request(conn, 'PUT',
193
+ path=self.tracker_uri_path,
194
+ auth_path=self.tracker_uri_path,
195
+ headers=put_headers,
196
+ host=self.tracker_uri_host)
197
+
198
+ def _query_server_pos(self, conn, file_length):
199
+ """
200
+ Queries server to find out what bytes it currently has.
201
+
202
+ Returns (server_start, server_end), where the values are inclusive.
203
+ For example, (0, 2) would mean that the server has bytes 0, 1, *and* 2.
204
+
205
+ Raises ResumableUploadException if problem querying server.
206
+ """
207
+ resp = self._query_server_state(conn, file_length)
208
+ if resp.status == 200:
209
+ # To handle the boundary condition where the server has the complete
210
+ # file, we return (server_start, file_length-1). That way the
211
+ # calling code can always simply read up through server_end. (If we
212
+ # didn't handle this boundary condition here, the caller would have
213
+ # to check whether server_end == file_length and read one fewer byte
214
+ # in that case.)
215
+ return (0, file_length - 1) # Completed upload.
216
+ if resp.status != 308:
217
+ # This means the server didn't have any state for the given
218
+ # upload ID, which can happen (for example) if the caller saved
219
+ # the tracker URI to a file and then tried to restart the transfer
220
+ # after that upload ID has gone stale. In that case we need to
221
+ # start a new transfer (and the caller will then save the new
222
+ # tracker URI to the tracker file).
223
+ raise ResumableUploadException(
224
+ 'Got non-308 response (%s) from server state query' %
225
+ resp.status, ResumableTransferDisposition.START_OVER)
226
+ got_valid_response = False
227
+ range_spec = resp.getheader('range')
228
+ if range_spec:
229
+ # Parse 'bytes=<from>-<to>' range_spec.
230
+ m = re.search('bytes=(\d+)-(\d+)', range_spec)
231
+ if m:
232
+ server_start = long(m.group(1))
233
+ server_end = long(m.group(2))
234
+ got_valid_response = True
235
+ else:
236
+ # No Range header, which means the server does not yet have
237
+ # any bytes. Note that the Range header uses inclusive 'from'
238
+ # and 'to' values. Since Range 0-0 would mean that the server
239
+ # has byte 0, omitting the Range header is used to indicate that
240
+ # the server doesn't have any bytes.
241
+ return self.SERVER_HAS_NOTHING
242
+ if not got_valid_response:
243
+ raise ResumableUploadException(
244
+ 'Couldn\'t parse upload server state query response (%s)' %
245
+ str(resp.getheaders()), ResumableTransferDisposition.START_OVER)
246
+ if conn.debug >= 1:
247
+ print 'Server has: Range: %d - %d.' % (server_start, server_end)
248
+ return (server_start, server_end)
249
+
250
+ def _start_new_resumable_upload(self, key, headers=None):
251
+ """
252
+ Starts a new resumable upload.
253
+
254
+ Raises ResumableUploadException if any errors occur.
255
+ """
256
+ conn = key.bucket.connection
257
+ if conn.debug >= 1:
258
+ print 'Starting new resumable upload.'
259
+ self.server_has_bytes = 0
260
+
261
+ # Start a new resumable upload by sending a POST request with an
262
+ # empty body and the "X-Goog-Resumable: start" header. Include any
263
+ # caller-provided headers (e.g., Content-Type) EXCEPT Content-Length
264
+ # (and raise an exception if they tried to pass one, since it's
265
+ # a semantic error to specify it at this point, and if we were to
266
+ # include one now it would cause the server to expect that many
267
+ # bytes; the POST doesn't include the actual file bytes We set
268
+ # the Content-Length in the subsequent PUT, based on the uploaded
269
+ # file size.
270
+ post_headers = {}
271
+ for k in headers:
272
+ if k.lower() == 'content-length':
273
+ raise ResumableUploadException(
274
+ 'Attempt to specify Content-Length header (disallowed)',
275
+ ResumableTransferDisposition.ABORT)
276
+ post_headers[k] = headers[k]
277
+ post_headers[conn.provider.resumable_upload_header] = 'start'
278
+
279
+ resp = conn.make_request(
280
+ 'POST', key.bucket.name, key.name, post_headers)
281
+ # Get tracker URI from response 'Location' header.
282
+ body = resp.read()
283
+
284
+ # Check for various status conditions.
285
+ if resp.status in [500, 503]:
286
+ # Retry status 500 and 503 errors after a delay.
287
+ raise ResumableUploadException(
288
+ 'Got status %d from attempt to start resumable upload. '
289
+ 'Will wait/retry' % resp.status,
290
+ ResumableTransferDisposition.WAIT_BEFORE_RETRY)
291
+ elif resp.status != 200 and resp.status != 201:
292
+ raise ResumableUploadException(
293
+ 'Got status %d from attempt to start resumable upload. '
294
+ 'Aborting' % resp.status,
295
+ ResumableTransferDisposition.ABORT)
296
+
297
+ # Else we got 200 or 201 response code, indicating the resumable
298
+ # upload was created.
299
+ tracker_uri = resp.getheader('Location')
300
+ if not tracker_uri:
301
+ raise ResumableUploadException(
302
+ 'No resumable tracker URI found in resumable initiation '
303
+ 'POST response (%s)' % body,
304
+ ResumableTransferDisposition.WAIT_BEFORE_RETRY)
305
+ self._set_tracker_uri(tracker_uri)
306
+ self._save_tracker_uri_to_file()
307
+
308
+ def _upload_file_bytes(self, conn, http_conn, fp, file_length,
309
+ total_bytes_uploaded, cb, num_cb, md5sum, headers):
310
+ """
311
+ Makes one attempt to upload file bytes, using an existing resumable
312
+ upload connection.
313
+
314
+ Returns (etag, generation, meta_generation) from server upon success.
315
+
316
+ Raises ResumableUploadException if any problems occur.
317
+ """
318
+ buf = fp.read(self.BUFFER_SIZE)
319
+ if cb:
320
+ # The cb_count represents the number of full buffers to send between
321
+ # cb executions.
322
+ if num_cb > 2:
323
+ cb_count = file_length / self.BUFFER_SIZE / (num_cb-2)
324
+ elif num_cb < 0:
325
+ cb_count = -1
326
+ else:
327
+ cb_count = 0
328
+ i = 0
329
+ cb(total_bytes_uploaded, file_length)
330
+
331
+ # Build resumable upload headers for the transfer. Don't send a
332
+ # Content-Range header if the file is 0 bytes long, because the
333
+ # resumable upload protocol uses an *inclusive* end-range (so, sending
334
+ # 'bytes 0-0/1' would actually mean you're sending a 1-byte file).
335
+ if not headers:
336
+ put_headers = {}
337
+ else:
338
+ put_headers = headers.copy()
339
+ if file_length:
340
+ if total_bytes_uploaded == file_length:
341
+ range_header = self._build_content_range_header(
342
+ '*', file_length)
343
+ else:
344
+ range_header = self._build_content_range_header(
345
+ '%d-%d' % (total_bytes_uploaded, file_length - 1),
346
+ file_length)
347
+ put_headers['Content-Range'] = range_header
348
+ # Set Content-Length to the total bytes we'll send with this PUT.
349
+ put_headers['Content-Length'] = str(file_length - total_bytes_uploaded)
350
+ http_request = AWSAuthConnection.build_base_http_request(
351
+ conn, 'PUT', path=self.tracker_uri_path, auth_path=None,
352
+ headers=put_headers, host=self.tracker_uri_host)
353
+ http_conn.putrequest('PUT', http_request.path)
354
+ for k in put_headers:
355
+ http_conn.putheader(k, put_headers[k])
356
+ http_conn.endheaders()
357
+
358
+ # Turn off debug on http connection so upload content isn't included
359
+ # in debug stream.
360
+ http_conn.set_debuglevel(0)
361
+ while buf:
362
+ http_conn.send(buf)
363
+ md5sum.update(buf)
364
+ total_bytes_uploaded += len(buf)
365
+ if cb:
366
+ i += 1
367
+ if i == cb_count or cb_count == -1:
368
+ cb(total_bytes_uploaded, file_length)
369
+ i = 0
370
+ buf = fp.read(self.BUFFER_SIZE)
371
+ http_conn.set_debuglevel(conn.debug)
372
+ if cb:
373
+ cb(total_bytes_uploaded, file_length)
374
+ if total_bytes_uploaded != file_length:
375
+ # Abort (and delete the tracker file) so if the user retries
376
+ # they'll start a new resumable upload rather than potentially
377
+ # attempting to pick back up later where we left off.
378
+ raise ResumableUploadException(
379
+ 'File changed during upload: EOF at %d bytes of %d byte file.' %
380
+ (total_bytes_uploaded, file_length),
381
+ ResumableTransferDisposition.ABORT)
382
+ resp = http_conn.getresponse()
383
+ body = resp.read()
384
+ # Restore http connection debug level.
385
+ http_conn.set_debuglevel(conn.debug)
386
+
387
+ if resp.status == 200:
388
+ # Success.
389
+ return (resp.getheader('etag'),
390
+ resp.getheader('x-goog-generation'),
391
+ resp.getheader('x-goog-metageneration'))
392
+ # Retry timeout (408) and status 500 and 503 errors after a delay.
393
+ elif resp.status in [408, 500, 503]:
394
+ disposition = ResumableTransferDisposition.WAIT_BEFORE_RETRY
395
+ else:
396
+ # Catch all for any other error codes.
397
+ disposition = ResumableTransferDisposition.ABORT
398
+ raise ResumableUploadException('Got response code %d while attempting '
399
+ 'upload (%s)' %
400
+ (resp.status, resp.reason), disposition)
401
+
402
+ def _attempt_resumable_upload(self, key, fp, file_length, headers, cb,
403
+ num_cb, md5sum):
404
+ """
405
+ Attempts a resumable upload.
406
+
407
+ Returns (etag, generation, meta_generation) from server upon success.
408
+
409
+ Raises ResumableUploadException if any problems occur.
410
+ """
411
+ (server_start, server_end) = self.SERVER_HAS_NOTHING
412
+ conn = key.bucket.connection
413
+ if self.tracker_uri:
414
+ # Try to resume existing resumable upload.
415
+ try:
416
+ (server_start, server_end) = (
417
+ self._query_server_pos(conn, file_length))
418
+ self.server_has_bytes = server_start
419
+
420
+ if server_end:
421
+ # If the server already has some of the content, we need to
422
+ # update the md5 with the bytes that have already been
423
+ # uploaded to ensure we get a complete hash in the end.
424
+ print 'Catching up md5 for resumed upload'
425
+ fp.seek(0)
426
+ # Read local file's bytes through position server has. For
427
+ # example, if server has (0, 3) we want to read 3-0+1=4 bytes.
428
+ bytes_to_go = server_end + 1
429
+ while bytes_to_go:
430
+ chunk = fp.read(min(key.BufferSize, bytes_to_go))
431
+ if not chunk:
432
+ raise ResumableUploadException(
433
+ 'Hit end of file during resumable upload md5 '
434
+ 'catchup. This should not happen under\n'
435
+ 'normal circumstances, as it indicates the '
436
+ 'server has more bytes of this transfer\nthan'
437
+ ' the current file size. Restarting upload.',
438
+ ResumableTransferDisposition.START_OVER)
439
+ md5sum.update(chunk)
440
+ bytes_to_go -= len(chunk)
441
+
442
+ if conn.debug >= 1:
443
+ print 'Resuming transfer.'
444
+ except ResumableUploadException, e:
445
+ if conn.debug >= 1:
446
+ print 'Unable to resume transfer (%s).' % e.message
447
+ self._start_new_resumable_upload(key, headers)
448
+ else:
449
+ self._start_new_resumable_upload(key, headers)
450
+
451
+ # upload_start_point allows the code that instantiated the
452
+ # ResumableUploadHandler to find out the point from which it started
453
+ # uploading (e.g., so it can correctly compute throughput).
454
+ if self.upload_start_point is None:
455
+ self.upload_start_point = server_end
456
+
457
+ total_bytes_uploaded = server_end + 1
458
+ # Corner case: Don't attempt to seek if we've already uploaded the
459
+ # entire file, because if the file is a stream (e.g., the KeyFile
460
+ # wrapper around input key when copying between providers), attempting
461
+ # to seek to the end of file would result in an InvalidRange error.
462
+ if file_length < total_bytes_uploaded:
463
+ fp.seek(total_bytes_uploaded)
464
+ conn = key.bucket.connection
465
+
466
+ # Get a new HTTP connection (vs conn.get_http_connection(), which reuses
467
+ # pool connections) because httplib requires a new HTTP connection per
468
+ # transaction. (Without this, calling http_conn.getresponse() would get
469
+ # "ResponseNotReady".)
470
+ http_conn = conn.new_http_connection(self.tracker_uri_host,
471
+ conn.is_secure)
472
+ http_conn.set_debuglevel(conn.debug)
473
+
474
+ # Make sure to close http_conn at end so if a local file read
475
+ # failure occurs partway through server will terminate current upload
476
+ # and can report that progress on next attempt.
477
+ try:
478
+ return self._upload_file_bytes(conn, http_conn, fp, file_length,
479
+ total_bytes_uploaded, cb, num_cb, md5sum,
480
+ headers)
481
+ except (ResumableUploadException, socket.error):
482
+ resp = self._query_server_state(conn, file_length)
483
+ if resp.status == 400:
484
+ raise ResumableUploadException('Got 400 response from server '
485
+ 'state query after failed resumable upload attempt. This '
486
+ 'can happen for various reasons, including specifying an '
487
+ 'invalid request (e.g., an invalid canned ACL) or if the '
488
+ 'file size changed between upload attempts',
489
+ ResumableTransferDisposition.ABORT)
490
+ else:
491
+ raise
492
+ finally:
493
+ http_conn.close()
494
+
495
+ def _check_final_md5(self, key, etag):
496
+ """
497
+ Checks that etag from server agrees with md5 computed before upload.
498
+ This is important, since the upload could have spanned a number of
499
+ hours and multiple processes (e.g., gsutil runs), and the user could
500
+ change some of the file and not realize they have inconsistent data.
501
+ """
502
+ if key.bucket.connection.debug >= 1:
503
+ print 'Checking md5 against etag.'
504
+ if key.md5 != etag.strip('"\''):
505
+ # Call key.open_read() before attempting to delete the
506
+ # (incorrect-content) key, so we perform that request on a
507
+ # different HTTP connection. This is neededb because httplib
508
+ # will return a "Response not ready" error if you try to perform
509
+ # a second transaction on the connection.
510
+ key.open_read()
511
+ key.close()
512
+ key.delete()
513
+ raise ResumableUploadException(
514
+ 'File changed during upload: md5 signature doesn\'t match etag '
515
+ '(incorrect uploaded object deleted)',
516
+ ResumableTransferDisposition.ABORT)
517
+
518
+ def handle_resumable_upload_exception(self, e, debug):
519
+ if (e.disposition == ResumableTransferDisposition.ABORT_CUR_PROCESS):
520
+ if debug >= 1:
521
+ print('Caught non-retryable ResumableUploadException (%s); '
522
+ 'aborting but retaining tracker file' % e.message)
523
+ raise
524
+ elif (e.disposition == ResumableTransferDisposition.ABORT):
525
+ if debug >= 1:
526
+ print('Caught non-retryable ResumableUploadException (%s); '
527
+ 'aborting and removing tracker file' % e.message)
528
+ self._remove_tracker_file()
529
+ raise
530
+ else:
531
+ if debug >= 1:
532
+ print('Caught ResumableUploadException (%s) - will retry' %
533
+ e.message)
534
+
535
+ def track_progress_less_iterations(self, server_had_bytes_before_attempt,
536
+ roll_back_md5=True, debug=0):
537
+ # At this point we had a re-tryable failure; see if made progress.
538
+ if self.server_has_bytes > server_had_bytes_before_attempt:
539
+ self.progress_less_iterations = 0 # If progress, reset counter.
540
+ else:
541
+ self.progress_less_iterations += 1
542
+ if roll_back_md5:
543
+ # Rollback any potential md5sum updates, as we did not
544
+ # make any progress in this iteration.
545
+ self.md5sum = self.md5sum_before_attempt
546
+
547
+ if self.progress_less_iterations > self.num_retries:
548
+ # Don't retry any longer in the current process.
549
+ raise ResumableUploadException(
550
+ 'Too many resumable upload attempts failed without '
551
+ 'progress. You might try this upload again later',
552
+ ResumableTransferDisposition.ABORT_CUR_PROCESS)
553
+
554
+ # Use binary exponential backoff to desynchronize client requests.
555
+ sleep_time_secs = random.random() * (2**self.progress_less_iterations)
556
+ if debug >= 1:
557
+ print ('Got retryable failure (%d progress-less in a row).\n'
558
+ 'Sleeping %3.1f seconds before re-trying' %
559
+ (self.progress_less_iterations, sleep_time_secs))
560
+ time.sleep(sleep_time_secs)
561
+
562
+ def send_file(self, key, fp, headers, cb=None, num_cb=10):
563
+ """
564
+ Upload a file to a key into a bucket on GS, using GS resumable upload
565
+ protocol.
566
+
567
+ :type key: :class:`boto.s3.key.Key` or subclass
568
+ :param key: The Key object to which data is to be uploaded
569
+
570
+ :type fp: file-like object
571
+ :param fp: The file pointer to upload
572
+
573
+ :type headers: dict
574
+ :param headers: The headers to pass along with the PUT request
575
+
576
+ :type cb: function
577
+ :param cb: a callback function that will be called to report progress on
578
+ the upload. The callback should accept two integer parameters, the
579
+ first representing the number of bytes that have been successfully
580
+ transmitted to GS, and the second representing the total number of
581
+ bytes that need to be transmitted.
582
+
583
+ :type num_cb: int
584
+ :param num_cb: (optional) If a callback is specified with the cb
585
+ parameter, this parameter determines the granularity of the callback
586
+ by defining the maximum number of times the callback will be called
587
+ during the file transfer. Providing a negative integer will cause
588
+ your callback to be called with each buffer read.
589
+
590
+ Raises ResumableUploadException if a problem occurs during the transfer.
591
+ """
592
+
593
+ if not headers:
594
+ headers = {}
595
+ # If Content-Type header is present and set to None, remove it.
596
+ # This is gsutil's way of asking boto to refrain from auto-generating
597
+ # that header.
598
+ CT = 'Content-Type'
599
+ if CT in headers and headers[CT] is None:
600
+ del headers[CT]
601
+
602
+ headers['User-Agent'] = UserAgent
603
+
604
+ # Determine file size different ways for case where fp is actually a
605
+ # wrapper around a Key vs an actual file.
606
+ if isinstance(fp, KeyFile):
607
+ file_length = fp.getkey().size
608
+ else:
609
+ fp.seek(0, os.SEEK_END)
610
+ file_length = fp.tell()
611
+ fp.seek(0)
612
+ debug = key.bucket.connection.debug
613
+
614
+ # Compute the MD5 checksum on the fly.
615
+ self.md5sum = md5()
616
+
617
+ # Use num-retries from constructor if one was provided; else check
618
+ # for a value specified in the boto config file; else default to 5.
619
+ if self.num_retries is None:
620
+ self.num_retries = config.getint('Boto', 'num_retries', 6)
621
+ self.progress_less_iterations = 0
622
+
623
+ while True: # Retry as long as we're making progress.
624
+ server_had_bytes_before_attempt = self.server_has_bytes
625
+ self.md5sum_before_attempt = self.md5sum.copy()
626
+ try:
627
+ # Save generation and meta_generation in class state so caller
628
+ # can find these values, for use in preconditions of future
629
+ # operations on the uploaded object.
630
+ (etag, self.generation, self.meta_generation) = (
631
+ self._attempt_resumable_upload(key, fp, file_length,
632
+ headers, cb, num_cb,
633
+ self.md5sum))
634
+
635
+ # Get the final md5 for the uploaded content.
636
+ hd = self.md5sum.hexdigest()
637
+ key.md5, key.base64md5 = key.get_md5_from_hexdigest(hd)
638
+
639
+ # Upload succceded, so remove the tracker file (if have one).
640
+ self._remove_tracker_file()
641
+ self._check_final_md5(key, etag)
642
+ if debug >= 1:
643
+ print 'Resumable upload complete.'
644
+ return
645
+ except self.RETRYABLE_EXCEPTIONS, e:
646
+ if debug >= 1:
647
+ print('Caught exception (%s)' % e.__repr__())
648
+ if isinstance(e, IOError) and e.errno == errno.EPIPE:
649
+ # Broken pipe error causes httplib to immediately
650
+ # close the socket (http://bugs.python.org/issue5542),
651
+ # so we need to close the connection before we resume
652
+ # the upload (which will cause a new connection to be
653
+ # opened the next time an HTTP request is sent).
654
+ key.bucket.connection.connection.close()
655
+ except ResumableUploadException, e:
656
+ self.handle_resumable_upload_exception(e, debug)
657
+
658
+ self.track_progress_less_iterations(server_had_bytes_before_attempt,
659
+ True, debug)