gcloud 0.0.4 → 0.0.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (768) hide show
  1. data.tar.gz.sig +0 -0
  2. data/CHANGELOG +4 -0
  3. data/Makefile +4 -0
  4. data/Manifest +760 -1
  5. data/bin/gcloud +7 -0
  6. data/bin/{gcutil → gcutil.symlink} +0 -0
  7. data/bin/gsutil-symlink +377 -0
  8. data/gcloud.gemspec +5 -5
  9. data/packages/gsutil/CHECKSUM +1 -0
  10. data/packages/gsutil/COPYING +202 -0
  11. data/packages/gsutil/LICENSE.third_party +295 -0
  12. data/packages/gsutil/MANIFEST.in +5 -0
  13. data/packages/gsutil/README +38 -0
  14. data/packages/gsutil/README.pkg +49 -0
  15. data/packages/gsutil/ReleaseNotes.txt +780 -0
  16. data/packages/gsutil/VERSION +1 -0
  17. data/packages/gsutil/boto/Changelog.rst +35 -0
  18. data/packages/gsutil/boto/LICENSE +18 -0
  19. data/packages/gsutil/boto/MANIFEST.in +12 -0
  20. data/packages/gsutil/boto/README.rst +163 -0
  21. data/packages/gsutil/boto/bin/asadmin +290 -0
  22. data/packages/gsutil/boto/bin/bundle_image +27 -0
  23. data/packages/gsutil/boto/bin/cfadmin +108 -0
  24. data/packages/gsutil/boto/bin/cq +89 -0
  25. data/packages/gsutil/boto/bin/cwutil +140 -0
  26. data/packages/gsutil/boto/bin/elbadmin +284 -0
  27. data/packages/gsutil/boto/bin/fetch_file +43 -0
  28. data/packages/gsutil/boto/bin/glacier +154 -0
  29. data/packages/gsutil/boto/bin/instance_events +145 -0
  30. data/packages/gsutil/boto/bin/kill_instance +35 -0
  31. data/packages/gsutil/boto/bin/launch_instance +252 -0
  32. data/packages/gsutil/boto/bin/list_instances +90 -0
  33. data/packages/gsutil/boto/bin/lss3 +77 -0
  34. data/packages/gsutil/boto/bin/mturk +465 -0
  35. data/packages/gsutil/boto/bin/pyami_sendmail +52 -0
  36. data/packages/gsutil/boto/bin/route53 +205 -0
  37. data/packages/gsutil/boto/bin/s3put +374 -0
  38. data/packages/gsutil/boto/bin/sdbadmin +194 -0
  39. data/packages/gsutil/boto/bin/taskadmin +116 -0
  40. data/packages/gsutil/boto/boto/__init__.py +793 -0
  41. data/packages/gsutil/boto/boto/__init__.pyc +0 -0
  42. data/packages/gsutil/boto/boto/auth.py +682 -0
  43. data/packages/gsutil/boto/boto/auth.pyc +0 -0
  44. data/packages/gsutil/boto/boto/auth_handler.py +58 -0
  45. data/packages/gsutil/boto/boto/auth_handler.pyc +0 -0
  46. data/packages/gsutil/boto/boto/beanstalk/__init__.py +65 -0
  47. data/packages/gsutil/boto/boto/beanstalk/exception.py +64 -0
  48. data/packages/gsutil/boto/boto/beanstalk/layer1.py +1153 -0
  49. data/packages/gsutil/boto/boto/beanstalk/response.py +703 -0
  50. data/packages/gsutil/boto/boto/beanstalk/wrapper.py +29 -0
  51. data/packages/gsutil/boto/boto/cacerts/__init__.py +22 -0
  52. data/packages/gsutil/boto/boto/cacerts/__init__.pyc +0 -0
  53. data/packages/gsutil/boto/boto/cacerts/cacerts.txt +633 -0
  54. data/packages/gsutil/boto/boto/cloudformation/__init__.py +68 -0
  55. data/packages/gsutil/boto/boto/cloudformation/connection.py +364 -0
  56. data/packages/gsutil/boto/boto/cloudformation/stack.py +360 -0
  57. data/packages/gsutil/boto/boto/cloudformation/template.py +43 -0
  58. data/packages/gsutil/boto/boto/cloudfront/__init__.py +324 -0
  59. data/packages/gsutil/boto/boto/cloudfront/distribution.py +745 -0
  60. data/packages/gsutil/boto/boto/cloudfront/exception.py +26 -0
  61. data/packages/gsutil/boto/boto/cloudfront/identity.py +122 -0
  62. data/packages/gsutil/boto/boto/cloudfront/invalidation.py +216 -0
  63. data/packages/gsutil/boto/boto/cloudfront/logging.py +38 -0
  64. data/packages/gsutil/boto/boto/cloudfront/object.py +48 -0
  65. data/packages/gsutil/boto/boto/cloudfront/origin.py +150 -0
  66. data/packages/gsutil/boto/boto/cloudfront/signers.py +60 -0
  67. data/packages/gsutil/boto/boto/cloudsearch/__init__.py +45 -0
  68. data/packages/gsutil/boto/boto/cloudsearch/document.py +265 -0
  69. data/packages/gsutil/boto/boto/cloudsearch/domain.py +394 -0
  70. data/packages/gsutil/boto/boto/cloudsearch/layer1.py +738 -0
  71. data/packages/gsutil/boto/boto/cloudsearch/layer2.py +67 -0
  72. data/packages/gsutil/boto/boto/cloudsearch/optionstatus.py +248 -0
  73. data/packages/gsutil/boto/boto/cloudsearch/search.py +365 -0
  74. data/packages/gsutil/boto/boto/cloudsearch/sourceattribute.py +75 -0
  75. data/packages/gsutil/boto/boto/compat.py +28 -0
  76. data/packages/gsutil/boto/boto/compat.pyc +0 -0
  77. data/packages/gsutil/boto/boto/connection.py +1081 -0
  78. data/packages/gsutil/boto/boto/connection.pyc +0 -0
  79. data/packages/gsutil/boto/boto/contrib/__init__.py +22 -0
  80. data/packages/gsutil/boto/boto/contrib/ymlmessage.py +52 -0
  81. data/packages/gsutil/boto/boto/core/README +58 -0
  82. data/packages/gsutil/boto/boto/core/__init__.py +23 -0
  83. data/packages/gsutil/boto/boto/core/auth.py +78 -0
  84. data/packages/gsutil/boto/boto/core/credentials.py +154 -0
  85. data/packages/gsutil/boto/boto/core/dictresponse.py +178 -0
  86. data/packages/gsutil/boto/boto/core/service.py +67 -0
  87. data/packages/gsutil/boto/boto/datapipeline/__init__.py +0 -0
  88. data/packages/gsutil/boto/boto/datapipeline/exceptions.py +42 -0
  89. data/packages/gsutil/boto/boto/datapipeline/layer1.py +546 -0
  90. data/packages/gsutil/boto/boto/dynamodb/__init__.py +66 -0
  91. data/packages/gsutil/boto/boto/dynamodb/batch.py +262 -0
  92. data/packages/gsutil/boto/boto/dynamodb/condition.py +170 -0
  93. data/packages/gsutil/boto/boto/dynamodb/exceptions.py +64 -0
  94. data/packages/gsutil/boto/boto/dynamodb/item.py +196 -0
  95. data/packages/gsutil/boto/boto/dynamodb/layer1.py +575 -0
  96. data/packages/gsutil/boto/boto/dynamodb/layer2.py +798 -0
  97. data/packages/gsutil/boto/boto/dynamodb/schema.py +112 -0
  98. data/packages/gsutil/boto/boto/dynamodb/table.py +540 -0
  99. data/packages/gsutil/boto/boto/dynamodb/types.py +326 -0
  100. data/packages/gsutil/boto/boto/ec2/__init__.py +96 -0
  101. data/packages/gsutil/boto/boto/ec2/address.py +103 -0
  102. data/packages/gsutil/boto/boto/ec2/autoscale/__init__.py +781 -0
  103. data/packages/gsutil/boto/boto/ec2/autoscale/activity.py +74 -0
  104. data/packages/gsutil/boto/boto/ec2/autoscale/group.py +337 -0
  105. data/packages/gsutil/boto/boto/ec2/autoscale/instance.py +60 -0
  106. data/packages/gsutil/boto/boto/ec2/autoscale/launchconfig.py +209 -0
  107. data/packages/gsutil/boto/boto/ec2/autoscale/policy.py +166 -0
  108. data/packages/gsutil/boto/boto/ec2/autoscale/request.py +38 -0
  109. data/packages/gsutil/boto/boto/ec2/autoscale/scheduled.py +60 -0
  110. data/packages/gsutil/boto/boto/ec2/autoscale/tag.py +84 -0
  111. data/packages/gsutil/boto/boto/ec2/blockdevicemapping.py +141 -0
  112. data/packages/gsutil/boto/boto/ec2/bundleinstance.py +78 -0
  113. data/packages/gsutil/boto/boto/ec2/buyreservation.py +84 -0
  114. data/packages/gsutil/boto/boto/ec2/cloudwatch/__init__.py +603 -0
  115. data/packages/gsutil/boto/boto/ec2/cloudwatch/alarm.py +316 -0
  116. data/packages/gsutil/boto/boto/ec2/cloudwatch/datapoint.py +40 -0
  117. data/packages/gsutil/boto/boto/ec2/cloudwatch/dimension.py +38 -0
  118. data/packages/gsutil/boto/boto/ec2/cloudwatch/listelement.py +31 -0
  119. data/packages/gsutil/boto/boto/ec2/cloudwatch/metric.py +175 -0
  120. data/packages/gsutil/boto/boto/ec2/connection.py +3409 -0
  121. data/packages/gsutil/boto/boto/ec2/ec2object.py +107 -0
  122. data/packages/gsutil/boto/boto/ec2/elb/__init__.py +553 -0
  123. data/packages/gsutil/boto/boto/ec2/elb/healthcheck.py +89 -0
  124. data/packages/gsutil/boto/boto/ec2/elb/instancestate.py +62 -0
  125. data/packages/gsutil/boto/boto/ec2/elb/listelement.py +36 -0
  126. data/packages/gsutil/boto/boto/ec2/elb/listener.py +75 -0
  127. data/packages/gsutil/boto/boto/ec2/elb/loadbalancer.py +324 -0
  128. data/packages/gsutil/boto/boto/ec2/elb/policies.py +89 -0
  129. data/packages/gsutil/boto/boto/ec2/elb/securitygroup.py +38 -0
  130. data/packages/gsutil/boto/boto/ec2/group.py +39 -0
  131. data/packages/gsutil/boto/boto/ec2/image.py +350 -0
  132. data/packages/gsutil/boto/boto/ec2/instance.py +661 -0
  133. data/packages/gsutil/boto/boto/ec2/instanceinfo.py +51 -0
  134. data/packages/gsutil/boto/boto/ec2/instancestatus.py +207 -0
  135. data/packages/gsutil/boto/boto/ec2/keypair.py +113 -0
  136. data/packages/gsutil/boto/boto/ec2/launchspecification.py +105 -0
  137. data/packages/gsutil/boto/boto/ec2/networkinterface.py +247 -0
  138. data/packages/gsutil/boto/boto/ec2/placementgroup.py +51 -0
  139. data/packages/gsutil/boto/boto/ec2/regioninfo.py +34 -0
  140. data/packages/gsutil/boto/boto/ec2/reservedinstance.py +227 -0
  141. data/packages/gsutil/boto/boto/ec2/securitygroup.py +357 -0
  142. data/packages/gsutil/boto/boto/ec2/snapshot.py +170 -0
  143. data/packages/gsutil/boto/boto/ec2/spotdatafeedsubscription.py +63 -0
  144. data/packages/gsutil/boto/boto/ec2/spotinstancerequest.py +188 -0
  145. data/packages/gsutil/boto/boto/ec2/spotpricehistory.py +55 -0
  146. data/packages/gsutil/boto/boto/ec2/tag.py +87 -0
  147. data/packages/gsutil/boto/boto/ec2/vmtype.py +58 -0
  148. data/packages/gsutil/boto/boto/ec2/volume.py +293 -0
  149. data/packages/gsutil/boto/boto/ec2/volumestatus.py +200 -0
  150. data/packages/gsutil/boto/boto/ec2/zone.py +80 -0
  151. data/packages/gsutil/boto/boto/ecs/__init__.py +90 -0
  152. data/packages/gsutil/boto/boto/ecs/item.py +153 -0
  153. data/packages/gsutil/boto/boto/elasticache/__init__.py +62 -0
  154. data/packages/gsutil/boto/boto/elasticache/layer1.py +1252 -0
  155. data/packages/gsutil/boto/boto/elastictranscoder/__init__.py +62 -0
  156. data/packages/gsutil/boto/boto/elastictranscoder/exceptions.py +46 -0
  157. data/packages/gsutil/boto/boto/elastictranscoder/layer1.py +509 -0
  158. data/packages/gsutil/boto/boto/emr/__init__.py +73 -0
  159. data/packages/gsutil/boto/boto/emr/bootstrap_action.py +44 -0
  160. data/packages/gsutil/boto/boto/emr/connection.py +531 -0
  161. data/packages/gsutil/boto/boto/emr/emrobject.py +176 -0
  162. data/packages/gsutil/boto/boto/emr/instance_group.py +43 -0
  163. data/packages/gsutil/boto/boto/emr/step.py +281 -0
  164. data/packages/gsutil/boto/boto/exception.py +476 -0
  165. data/packages/gsutil/boto/boto/exception.pyc +0 -0
  166. data/packages/gsutil/boto/boto/file/README +49 -0
  167. data/packages/gsutil/boto/boto/file/__init__.py +28 -0
  168. data/packages/gsutil/boto/boto/file/bucket.py +112 -0
  169. data/packages/gsutil/boto/boto/file/connection.py +33 -0
  170. data/packages/gsutil/boto/boto/file/key.py +199 -0
  171. data/packages/gsutil/boto/boto/file/simpleresultset.py +30 -0
  172. data/packages/gsutil/boto/boto/fps/__init__.py +21 -0
  173. data/packages/gsutil/boto/boto/fps/connection.py +369 -0
  174. data/packages/gsutil/boto/boto/fps/exception.py +344 -0
  175. data/packages/gsutil/boto/boto/fps/response.py +175 -0
  176. data/packages/gsutil/boto/boto/glacier/__init__.py +57 -0
  177. data/packages/gsutil/boto/boto/glacier/concurrent.py +409 -0
  178. data/packages/gsutil/boto/boto/glacier/exceptions.py +58 -0
  179. data/packages/gsutil/boto/boto/glacier/job.py +152 -0
  180. data/packages/gsutil/boto/boto/glacier/layer1.py +637 -0
  181. data/packages/gsutil/boto/boto/glacier/layer2.py +93 -0
  182. data/packages/gsutil/boto/boto/glacier/response.py +48 -0
  183. data/packages/gsutil/boto/boto/glacier/utils.py +163 -0
  184. data/packages/gsutil/boto/boto/glacier/vault.py +387 -0
  185. data/packages/gsutil/boto/boto/glacier/writer.py +242 -0
  186. data/packages/gsutil/boto/boto/gs/__init__.py +22 -0
  187. data/packages/gsutil/boto/boto/gs/__init__.pyc +0 -0
  188. data/packages/gsutil/boto/boto/gs/acl.py +304 -0
  189. data/packages/gsutil/boto/boto/gs/acl.pyc +0 -0
  190. data/packages/gsutil/boto/boto/gs/bucket.py +870 -0
  191. data/packages/gsutil/boto/boto/gs/bucket.pyc +0 -0
  192. data/packages/gsutil/boto/boto/gs/bucketlistresultset.py +64 -0
  193. data/packages/gsutil/boto/boto/gs/bucketlistresultset.pyc +0 -0
  194. data/packages/gsutil/boto/boto/gs/connection.py +103 -0
  195. data/packages/gsutil/boto/boto/gs/connection.pyc +0 -0
  196. data/packages/gsutil/boto/boto/gs/cors.py +169 -0
  197. data/packages/gsutil/boto/boto/gs/cors.pyc +0 -0
  198. data/packages/gsutil/boto/boto/gs/key.py +704 -0
  199. data/packages/gsutil/boto/boto/gs/key.pyc +0 -0
  200. data/packages/gsutil/boto/boto/gs/resumable_upload_handler.py +659 -0
  201. data/packages/gsutil/boto/boto/gs/resumable_upload_handler.pyc +0 -0
  202. data/packages/gsutil/boto/boto/gs/user.py +54 -0
  203. data/packages/gsutil/boto/boto/gs/user.pyc +0 -0
  204. data/packages/gsutil/boto/boto/handler.py +44 -0
  205. data/packages/gsutil/boto/boto/handler.pyc +0 -0
  206. data/packages/gsutil/boto/boto/https_connection.py +124 -0
  207. data/packages/gsutil/boto/boto/https_connection.pyc +0 -0
  208. data/packages/gsutil/boto/boto/iam/__init__.py +74 -0
  209. data/packages/gsutil/boto/boto/iam/connection.py +1317 -0
  210. data/packages/gsutil/boto/boto/iam/summarymap.py +42 -0
  211. data/packages/gsutil/boto/boto/jsonresponse.py +163 -0
  212. data/packages/gsutil/boto/boto/jsonresponse.pyc +0 -0
  213. data/packages/gsutil/boto/boto/manage/__init__.py +23 -0
  214. data/packages/gsutil/boto/boto/manage/cmdshell.py +241 -0
  215. data/packages/gsutil/boto/boto/manage/propget.py +64 -0
  216. data/packages/gsutil/boto/boto/manage/server.py +556 -0
  217. data/packages/gsutil/boto/boto/manage/task.py +175 -0
  218. data/packages/gsutil/boto/boto/manage/test_manage.py +34 -0
  219. data/packages/gsutil/boto/boto/manage/volume.py +420 -0
  220. data/packages/gsutil/boto/boto/mashups/__init__.py +23 -0
  221. data/packages/gsutil/boto/boto/mashups/interactive.py +97 -0
  222. data/packages/gsutil/boto/boto/mashups/iobject.py +115 -0
  223. data/packages/gsutil/boto/boto/mashups/order.py +211 -0
  224. data/packages/gsutil/boto/boto/mashups/server.py +395 -0
  225. data/packages/gsutil/boto/boto/mturk/__init__.py +23 -0
  226. data/packages/gsutil/boto/boto/mturk/connection.py +1027 -0
  227. data/packages/gsutil/boto/boto/mturk/layoutparam.py +55 -0
  228. data/packages/gsutil/boto/boto/mturk/notification.py +103 -0
  229. data/packages/gsutil/boto/boto/mturk/price.py +48 -0
  230. data/packages/gsutil/boto/boto/mturk/qualification.py +137 -0
  231. data/packages/gsutil/boto/boto/mturk/question.py +455 -0
  232. data/packages/gsutil/boto/boto/mws/__init__.py +21 -0
  233. data/packages/gsutil/boto/boto/mws/connection.py +813 -0
  234. data/packages/gsutil/boto/boto/mws/exception.py +75 -0
  235. data/packages/gsutil/boto/boto/mws/response.py +655 -0
  236. data/packages/gsutil/boto/boto/plugin.py +90 -0
  237. data/packages/gsutil/boto/boto/plugin.pyc +0 -0
  238. data/packages/gsutil/boto/boto/provider.py +337 -0
  239. data/packages/gsutil/boto/boto/provider.pyc +0 -0
  240. data/packages/gsutil/boto/boto/pyami/__init__.py +22 -0
  241. data/packages/gsutil/boto/boto/pyami/__init__.pyc +0 -0
  242. data/packages/gsutil/boto/boto/pyami/bootstrap.py +134 -0
  243. data/packages/gsutil/boto/boto/pyami/config.py +229 -0
  244. data/packages/gsutil/boto/boto/pyami/config.pyc +0 -0
  245. data/packages/gsutil/boto/boto/pyami/copybot.cfg +60 -0
  246. data/packages/gsutil/boto/boto/pyami/copybot.py +97 -0
  247. data/packages/gsutil/boto/boto/pyami/helloworld.py +28 -0
  248. data/packages/gsutil/boto/boto/pyami/installers/__init__.py +64 -0
  249. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/__init__.py +22 -0
  250. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/apache.py +43 -0
  251. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/ebs.py +238 -0
  252. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/installer.py +96 -0
  253. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/mysql.py +109 -0
  254. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/trac.py +139 -0
  255. data/packages/gsutil/boto/boto/pyami/launch_ami.py +178 -0
  256. data/packages/gsutil/boto/boto/pyami/scriptbase.py +44 -0
  257. data/packages/gsutil/boto/boto/pyami/startup.py +60 -0
  258. data/packages/gsutil/boto/boto/rds/__init__.py +1194 -0
  259. data/packages/gsutil/boto/boto/rds/dbinstance.py +357 -0
  260. data/packages/gsutil/boto/boto/rds/dbsecuritygroup.py +177 -0
  261. data/packages/gsutil/boto/boto/rds/dbsnapshot.py +108 -0
  262. data/packages/gsutil/boto/boto/rds/event.py +49 -0
  263. data/packages/gsutil/boto/boto/rds/parametergroup.py +201 -0
  264. data/packages/gsutil/boto/boto/rds/regioninfo.py +32 -0
  265. data/packages/gsutil/boto/boto/regioninfo.py +63 -0
  266. data/packages/gsutil/boto/boto/regioninfo.pyc +0 -0
  267. data/packages/gsutil/boto/boto/resultset.py +169 -0
  268. data/packages/gsutil/boto/boto/resultset.pyc +0 -0
  269. data/packages/gsutil/boto/boto/roboto/__init__.py +1 -0
  270. data/packages/gsutil/boto/boto/roboto/awsqueryrequest.py +504 -0
  271. data/packages/gsutil/boto/boto/roboto/awsqueryservice.py +121 -0
  272. data/packages/gsutil/boto/boto/roboto/param.py +147 -0
  273. data/packages/gsutil/boto/boto/route53/__init__.py +75 -0
  274. data/packages/gsutil/boto/boto/route53/connection.py +403 -0
  275. data/packages/gsutil/boto/boto/route53/exception.py +27 -0
  276. data/packages/gsutil/boto/boto/route53/hostedzone.py +56 -0
  277. data/packages/gsutil/boto/boto/route53/record.py +306 -0
  278. data/packages/gsutil/boto/boto/route53/status.py +42 -0
  279. data/packages/gsutil/boto/boto/route53/zone.py +412 -0
  280. data/packages/gsutil/boto/boto/s3/__init__.py +84 -0
  281. data/packages/gsutil/boto/boto/s3/__init__.pyc +0 -0
  282. data/packages/gsutil/boto/boto/s3/acl.py +164 -0
  283. data/packages/gsutil/boto/boto/s3/acl.pyc +0 -0
  284. data/packages/gsutil/boto/boto/s3/bucket.py +1634 -0
  285. data/packages/gsutil/boto/boto/s3/bucket.pyc +0 -0
  286. data/packages/gsutil/boto/boto/s3/bucketlistresultset.py +139 -0
  287. data/packages/gsutil/boto/boto/s3/bucketlistresultset.pyc +0 -0
  288. data/packages/gsutil/boto/boto/s3/bucketlogging.py +83 -0
  289. data/packages/gsutil/boto/boto/s3/bucketlogging.pyc +0 -0
  290. data/packages/gsutil/boto/boto/s3/connection.py +540 -0
  291. data/packages/gsutil/boto/boto/s3/connection.pyc +0 -0
  292. data/packages/gsutil/boto/boto/s3/cors.py +210 -0
  293. data/packages/gsutil/boto/boto/s3/cors.pyc +0 -0
  294. data/packages/gsutil/boto/boto/s3/deletemarker.py +55 -0
  295. data/packages/gsutil/boto/boto/s3/deletemarker.pyc +0 -0
  296. data/packages/gsutil/boto/boto/s3/key.py +1712 -0
  297. data/packages/gsutil/boto/boto/s3/key.pyc +0 -0
  298. data/packages/gsutil/boto/boto/s3/keyfile.py +134 -0
  299. data/packages/gsutil/boto/boto/s3/keyfile.pyc +0 -0
  300. data/packages/gsutil/boto/boto/s3/lifecycle.py +231 -0
  301. data/packages/gsutil/boto/boto/s3/lifecycle.pyc +0 -0
  302. data/packages/gsutil/boto/boto/s3/multidelete.py +138 -0
  303. data/packages/gsutil/boto/boto/s3/multidelete.pyc +0 -0
  304. data/packages/gsutil/boto/boto/s3/multipart.py +315 -0
  305. data/packages/gsutil/boto/boto/s3/multipart.pyc +0 -0
  306. data/packages/gsutil/boto/boto/s3/prefix.py +42 -0
  307. data/packages/gsutil/boto/boto/s3/prefix.pyc +0 -0
  308. data/packages/gsutil/boto/boto/s3/resumable_download_handler.py +339 -0
  309. data/packages/gsutil/boto/boto/s3/resumable_download_handler.pyc +0 -0
  310. data/packages/gsutil/boto/boto/s3/tagging.py +71 -0
  311. data/packages/gsutil/boto/boto/s3/tagging.pyc +0 -0
  312. data/packages/gsutil/boto/boto/s3/user.py +49 -0
  313. data/packages/gsutil/boto/boto/s3/user.pyc +0 -0
  314. data/packages/gsutil/boto/boto/s3/website.py +237 -0
  315. data/packages/gsutil/boto/boto/s3/website.pyc +0 -0
  316. data/packages/gsutil/boto/boto/sdb/__init__.py +67 -0
  317. data/packages/gsutil/boto/boto/sdb/connection.py +617 -0
  318. data/packages/gsutil/boto/boto/sdb/db/__init__.py +20 -0
  319. data/packages/gsutil/boto/boto/sdb/db/blob.py +75 -0
  320. data/packages/gsutil/boto/boto/sdb/db/key.py +59 -0
  321. data/packages/gsutil/boto/boto/sdb/db/manager/__init__.py +85 -0
  322. data/packages/gsutil/boto/boto/sdb/db/manager/sdbmanager.py +732 -0
  323. data/packages/gsutil/boto/boto/sdb/db/manager/xmlmanager.py +517 -0
  324. data/packages/gsutil/boto/boto/sdb/db/model.py +294 -0
  325. data/packages/gsutil/boto/boto/sdb/db/property.py +703 -0
  326. data/packages/gsutil/boto/boto/sdb/db/query.py +85 -0
  327. data/packages/gsutil/boto/boto/sdb/db/sequence.py +226 -0
  328. data/packages/gsutil/boto/boto/sdb/db/test_db.py +231 -0
  329. data/packages/gsutil/boto/boto/sdb/domain.py +377 -0
  330. data/packages/gsutil/boto/boto/sdb/item.py +181 -0
  331. data/packages/gsutil/boto/boto/sdb/queryresultset.py +92 -0
  332. data/packages/gsutil/boto/boto/sdb/regioninfo.py +32 -0
  333. data/packages/gsutil/boto/boto/services/__init__.py +23 -0
  334. data/packages/gsutil/boto/boto/services/bs.py +179 -0
  335. data/packages/gsutil/boto/boto/services/message.py +58 -0
  336. data/packages/gsutil/boto/boto/services/result.py +136 -0
  337. data/packages/gsutil/boto/boto/services/service.py +161 -0
  338. data/packages/gsutil/boto/boto/services/servicedef.py +91 -0
  339. data/packages/gsutil/boto/boto/services/sonofmmm.cfg +43 -0
  340. data/packages/gsutil/boto/boto/services/sonofmmm.py +81 -0
  341. data/packages/gsutil/boto/boto/services/submit.py +88 -0
  342. data/packages/gsutil/boto/boto/ses/__init__.py +54 -0
  343. data/packages/gsutil/boto/boto/ses/connection.py +521 -0
  344. data/packages/gsutil/boto/boto/ses/exceptions.py +77 -0
  345. data/packages/gsutil/boto/boto/sns/__init__.py +78 -0
  346. data/packages/gsutil/boto/boto/sns/connection.py +431 -0
  347. data/packages/gsutil/boto/boto/sqs/__init__.py +56 -0
  348. data/packages/gsutil/boto/boto/sqs/attributes.py +46 -0
  349. data/packages/gsutil/boto/boto/sqs/batchresults.py +95 -0
  350. data/packages/gsutil/boto/boto/sqs/connection.py +417 -0
  351. data/packages/gsutil/boto/boto/sqs/jsonmessage.py +43 -0
  352. data/packages/gsutil/boto/boto/sqs/message.py +253 -0
  353. data/packages/gsutil/boto/boto/sqs/queue.py +478 -0
  354. data/packages/gsutil/boto/boto/sqs/regioninfo.py +32 -0
  355. data/packages/gsutil/boto/boto/storage_uri.py +835 -0
  356. data/packages/gsutil/boto/boto/storage_uri.pyc +0 -0
  357. data/packages/gsutil/boto/boto/sts/__init__.py +55 -0
  358. data/packages/gsutil/boto/boto/sts/connection.py +207 -0
  359. data/packages/gsutil/boto/boto/sts/credentials.py +215 -0
  360. data/packages/gsutil/boto/boto/swf/__init__.py +60 -0
  361. data/packages/gsutil/boto/boto/swf/exceptions.py +44 -0
  362. data/packages/gsutil/boto/boto/swf/layer1.py +1512 -0
  363. data/packages/gsutil/boto/boto/swf/layer1_decisions.py +287 -0
  364. data/packages/gsutil/boto/boto/swf/layer2.py +342 -0
  365. data/packages/gsutil/boto/boto/utils.py +927 -0
  366. data/packages/gsutil/boto/boto/utils.pyc +0 -0
  367. data/packages/gsutil/boto/boto/vpc/__init__.py +910 -0
  368. data/packages/gsutil/boto/boto/vpc/customergateway.py +54 -0
  369. data/packages/gsutil/boto/boto/vpc/dhcpoptions.py +72 -0
  370. data/packages/gsutil/boto/boto/vpc/internetgateway.py +72 -0
  371. data/packages/gsutil/boto/boto/vpc/routetable.py +109 -0
  372. data/packages/gsutil/boto/boto/vpc/subnet.py +57 -0
  373. data/packages/gsutil/boto/boto/vpc/vpc.py +54 -0
  374. data/packages/gsutil/boto/boto/vpc/vpnconnection.py +60 -0
  375. data/packages/gsutil/boto/boto/vpc/vpngateway.py +83 -0
  376. data/packages/gsutil/boto/docs/BotoCheatSheet.pdf +0 -0
  377. data/packages/gsutil/boto/docs/Makefile +89 -0
  378. data/packages/gsutil/boto/docs/make.bat +113 -0
  379. data/packages/gsutil/boto/docs/source/_templates/layout.html +3 -0
  380. data/packages/gsutil/boto/docs/source/autoscale_tut.rst +220 -0
  381. data/packages/gsutil/boto/docs/source/boto_config_tut.rst +125 -0
  382. data/packages/gsutil/boto/docs/source/boto_theme/static/boto.css_t +239 -0
  383. data/packages/gsutil/boto/docs/source/boto_theme/static/pygments.css +61 -0
  384. data/packages/gsutil/boto/docs/source/boto_theme/theme.conf +3 -0
  385. data/packages/gsutil/boto/docs/source/cloudfront_tut.rst +196 -0
  386. data/packages/gsutil/boto/docs/source/cloudsearch_tut.rst +411 -0
  387. data/packages/gsutil/boto/docs/source/cloudwatch_tut.rst +116 -0
  388. data/packages/gsutil/boto/docs/source/conf.py +32 -0
  389. data/packages/gsutil/boto/docs/source/contributing.rst +204 -0
  390. data/packages/gsutil/boto/docs/source/documentation.rst +59 -0
  391. data/packages/gsutil/boto/docs/source/dynamodb_tut.rst +339 -0
  392. data/packages/gsutil/boto/docs/source/ec2_tut.rst +86 -0
  393. data/packages/gsutil/boto/docs/source/elb_tut.rst +257 -0
  394. data/packages/gsutil/boto/docs/source/emr_tut.rst +108 -0
  395. data/packages/gsutil/boto/docs/source/index.rst +146 -0
  396. data/packages/gsutil/boto/docs/source/rds_tut.rst +108 -0
  397. data/packages/gsutil/boto/docs/source/ref/autoscale.rst +62 -0
  398. data/packages/gsutil/boto/docs/source/ref/beanstalk.rst +26 -0
  399. data/packages/gsutil/boto/docs/source/ref/boto.rst +47 -0
  400. data/packages/gsutil/boto/docs/source/ref/cloudformation.rst +34 -0
  401. data/packages/gsutil/boto/docs/source/ref/cloudfront.rst +68 -0
  402. data/packages/gsutil/boto/docs/source/ref/cloudsearch.rst +59 -0
  403. data/packages/gsutil/boto/docs/source/ref/cloudwatch.rst +27 -0
  404. data/packages/gsutil/boto/docs/source/ref/contrib.rst +32 -0
  405. data/packages/gsutil/boto/docs/source/ref/datapipeline.rst +26 -0
  406. data/packages/gsutil/boto/docs/source/ref/dynamodb.rst +61 -0
  407. data/packages/gsutil/boto/docs/source/ref/ec2.rst +140 -0
  408. data/packages/gsutil/boto/docs/source/ref/ecs.rst +19 -0
  409. data/packages/gsutil/boto/docs/source/ref/elasticache.rst +19 -0
  410. data/packages/gsutil/boto/docs/source/ref/elastictranscoder.rst +26 -0
  411. data/packages/gsutil/boto/docs/source/ref/elb.rst +47 -0
  412. data/packages/gsutil/boto/docs/source/ref/emr.rst +34 -0
  413. data/packages/gsutil/boto/docs/source/ref/file.rst +34 -0
  414. data/packages/gsutil/boto/docs/source/ref/fps.rst +19 -0
  415. data/packages/gsutil/boto/docs/source/ref/glacier.rst +63 -0
  416. data/packages/gsutil/boto/docs/source/ref/gs.rst +72 -0
  417. data/packages/gsutil/boto/docs/source/ref/iam.rst +27 -0
  418. data/packages/gsutil/boto/docs/source/ref/index.rst +40 -0
  419. data/packages/gsutil/boto/docs/source/ref/manage.rst +47 -0
  420. data/packages/gsutil/boto/docs/source/ref/mturk.rst +54 -0
  421. data/packages/gsutil/boto/docs/source/ref/mws.rst +33 -0
  422. data/packages/gsutil/boto/docs/source/ref/pyami.rst +103 -0
  423. data/packages/gsutil/boto/docs/source/ref/rds.rst +47 -0
  424. data/packages/gsutil/boto/docs/source/ref/route53.rst +34 -0
  425. data/packages/gsutil/boto/docs/source/ref/s3.rst +111 -0
  426. data/packages/gsutil/boto/docs/source/ref/sdb.rst +45 -0
  427. data/packages/gsutil/boto/docs/source/ref/sdb_db.rst +83 -0
  428. data/packages/gsutil/boto/docs/source/ref/services.rst +61 -0
  429. data/packages/gsutil/boto/docs/source/ref/ses.rst +21 -0
  430. data/packages/gsutil/boto/docs/source/ref/sns.rst +17 -0
  431. data/packages/gsutil/boto/docs/source/ref/sqs.rst +61 -0
  432. data/packages/gsutil/boto/docs/source/ref/sts.rst +25 -0
  433. data/packages/gsutil/boto/docs/source/ref/swf.rst +22 -0
  434. data/packages/gsutil/boto/docs/source/ref/vpc.rst +54 -0
  435. data/packages/gsutil/boto/docs/source/s3_tut.rst +450 -0
  436. data/packages/gsutil/boto/docs/source/security_groups.rst +82 -0
  437. data/packages/gsutil/boto/docs/source/ses_tut.rst +171 -0
  438. data/packages/gsutil/boto/docs/source/simpledb_tut.rst +188 -0
  439. data/packages/gsutil/boto/docs/source/sqs_tut.rst +246 -0
  440. data/packages/gsutil/boto/docs/source/vpc_tut.rst +100 -0
  441. data/packages/gsutil/boto/pylintrc +305 -0
  442. data/packages/gsutil/boto/requirements.txt +10 -0
  443. data/packages/gsutil/boto/setup.py +89 -0
  444. data/packages/gsutil/boto/tests/__init__.py +20 -0
  445. data/packages/gsutil/boto/tests/db/test_lists.py +96 -0
  446. data/packages/gsutil/boto/tests/db/test_password.py +128 -0
  447. data/packages/gsutil/boto/tests/db/test_query.py +152 -0
  448. data/packages/gsutil/boto/tests/db/test_sequence.py +109 -0
  449. data/packages/gsutil/boto/tests/devpay/__init__.py +0 -0
  450. data/packages/gsutil/boto/tests/devpay/test_s3.py +181 -0
  451. data/packages/gsutil/boto/tests/fps/__init__.py +0 -0
  452. data/packages/gsutil/boto/tests/fps/test.py +100 -0
  453. data/packages/gsutil/boto/tests/fps/test_verify_signature.py +12 -0
  454. data/packages/gsutil/boto/tests/integration/__init__.py +0 -0
  455. data/packages/gsutil/boto/tests/integration/beanstalk/test_wrapper.py +209 -0
  456. data/packages/gsutil/boto/tests/integration/cloudformation/__init__.py +21 -0
  457. data/packages/gsutil/boto/tests/integration/cloudformation/test_cert_verification.py +40 -0
  458. data/packages/gsutil/boto/tests/integration/cloudformation/test_connection.py +110 -0
  459. data/packages/gsutil/boto/tests/integration/cloudsearch/__init__.py +21 -0
  460. data/packages/gsutil/boto/tests/integration/cloudsearch/test_cert_verification.py +40 -0
  461. data/packages/gsutil/boto/tests/integration/datapipeline/test_layer1.py +122 -0
  462. data/packages/gsutil/boto/tests/integration/dynamodb/__init__.py +20 -0
  463. data/packages/gsutil/boto/tests/integration/dynamodb/test_cert_verification.py +40 -0
  464. data/packages/gsutil/boto/tests/integration/dynamodb/test_layer1.py +266 -0
  465. data/packages/gsutil/boto/tests/integration/dynamodb/test_layer2.py +484 -0
  466. data/packages/gsutil/boto/tests/integration/dynamodb/test_table.py +84 -0
  467. data/packages/gsutil/boto/tests/integration/ec2/__init__.py +20 -0
  468. data/packages/gsutil/boto/tests/integration/ec2/autoscale/__init__.py +21 -0
  469. data/packages/gsutil/boto/tests/integration/ec2/autoscale/test_cert_verification.py +40 -0
  470. data/packages/gsutil/boto/tests/integration/ec2/autoscale/test_connection.py +167 -0
  471. data/packages/gsutil/boto/tests/integration/ec2/cloudwatch/__init__.py +20 -0
  472. data/packages/gsutil/boto/tests/integration/ec2/cloudwatch/test_cert_verification.py +40 -0
  473. data/packages/gsutil/boto/tests/integration/ec2/cloudwatch/test_connection.py +277 -0
  474. data/packages/gsutil/boto/tests/integration/ec2/elb/__init__.py +20 -0
  475. data/packages/gsutil/boto/tests/integration/ec2/elb/test_cert_verification.py +40 -0
  476. data/packages/gsutil/boto/tests/integration/ec2/elb/test_connection.py +130 -0
  477. data/packages/gsutil/boto/tests/integration/ec2/test_cert_verification.py +40 -0
  478. data/packages/gsutil/boto/tests/integration/ec2/test_connection.py +192 -0
  479. data/packages/gsutil/boto/tests/integration/ec2/vpc/__init__.py +0 -0
  480. data/packages/gsutil/boto/tests/integration/ec2/vpc/test_connection.py +95 -0
  481. data/packages/gsutil/boto/tests/integration/elasticache/__init__.py +0 -0
  482. data/packages/gsutil/boto/tests/integration/elasticache/test_layer1.py +67 -0
  483. data/packages/gsutil/boto/tests/integration/elastictranscoder/__init__.py +0 -0
  484. data/packages/gsutil/boto/tests/integration/elastictranscoder/test_cert_verification.py +35 -0
  485. data/packages/gsutil/boto/tests/integration/elastictranscoder/test_layer1.py +115 -0
  486. data/packages/gsutil/boto/tests/integration/emr/__init__.py +20 -0
  487. data/packages/gsutil/boto/tests/integration/emr/test_cert_verification.py +40 -0
  488. data/packages/gsutil/boto/tests/integration/glacier/__init__.py +22 -0
  489. data/packages/gsutil/boto/tests/integration/glacier/test_cert_verification.py +40 -0
  490. data/packages/gsutil/boto/tests/integration/glacier/test_layer1.py +44 -0
  491. data/packages/gsutil/boto/tests/integration/glacier/test_layer2.py +45 -0
  492. data/packages/gsutil/boto/tests/integration/gs/__init__.py +0 -0
  493. data/packages/gsutil/boto/tests/integration/gs/cb_test_harness.py +71 -0
  494. data/packages/gsutil/boto/tests/integration/gs/test_basic.py +379 -0
  495. data/packages/gsutil/boto/tests/integration/gs/test_generation_conditionals.py +399 -0
  496. data/packages/gsutil/boto/tests/integration/gs/test_resumable_downloads.py +358 -0
  497. data/packages/gsutil/boto/tests/integration/gs/test_resumable_uploads.py +525 -0
  498. data/packages/gsutil/boto/tests/integration/gs/test_storage_uri.py +125 -0
  499. data/packages/gsutil/boto/tests/integration/gs/test_versioning.py +268 -0
  500. data/packages/gsutil/boto/tests/integration/gs/testcase.py +116 -0
  501. data/packages/gsutil/boto/tests/integration/gs/util.py +63 -0
  502. data/packages/gsutil/boto/tests/integration/iam/__init__.py +20 -0
  503. data/packages/gsutil/boto/tests/integration/iam/test_cert_verification.py +40 -0
  504. data/packages/gsutil/boto/tests/integration/mws/__init__.py +0 -0
  505. data/packages/gsutil/boto/tests/integration/mws/test.py +100 -0
  506. data/packages/gsutil/boto/tests/integration/rds/__init__.py +21 -0
  507. data/packages/gsutil/boto/tests/integration/rds/test_cert_verification.py +40 -0
  508. data/packages/gsutil/boto/tests/integration/route53/__init__.py +20 -0
  509. data/packages/gsutil/boto/tests/integration/route53/test_cert_verification.py +40 -0
  510. data/packages/gsutil/boto/tests/integration/route53/test_zone.py +132 -0
  511. data/packages/gsutil/boto/tests/integration/s3/__init__.py +20 -0
  512. data/packages/gsutil/boto/tests/integration/s3/mock_storage_service.py +589 -0
  513. data/packages/gsutil/boto/tests/integration/s3/other_cacerts.txt +70 -0
  514. data/packages/gsutil/boto/tests/integration/s3/test_bucket.py +263 -0
  515. data/packages/gsutil/boto/tests/integration/s3/test_cert_verification.py +40 -0
  516. data/packages/gsutil/boto/tests/integration/s3/test_connection.py +245 -0
  517. data/packages/gsutil/boto/tests/integration/s3/test_cors.py +78 -0
  518. data/packages/gsutil/boto/tests/integration/s3/test_encryption.py +115 -0
  519. data/packages/gsutil/boto/tests/integration/s3/test_https_cert_validation.py +141 -0
  520. data/packages/gsutil/boto/tests/integration/s3/test_key.py +375 -0
  521. data/packages/gsutil/boto/tests/integration/s3/test_mfa.py +95 -0
  522. data/packages/gsutil/boto/tests/integration/s3/test_multidelete.py +181 -0
  523. data/packages/gsutil/boto/tests/integration/s3/test_multipart.py +139 -0
  524. data/packages/gsutil/boto/tests/integration/s3/test_pool.py +246 -0
  525. data/packages/gsutil/boto/tests/integration/s3/test_versioning.py +158 -0
  526. data/packages/gsutil/boto/tests/integration/sdb/__init__.py +20 -0
  527. data/packages/gsutil/boto/tests/integration/sdb/test_cert_verification.py +40 -0
  528. data/packages/gsutil/boto/tests/integration/sdb/test_connection.py +119 -0
  529. data/packages/gsutil/boto/tests/integration/ses/__init__.py +0 -0
  530. data/packages/gsutil/boto/tests/integration/ses/test_cert_verification.py +40 -0
  531. data/packages/gsutil/boto/tests/integration/ses/test_connection.py +38 -0
  532. data/packages/gsutil/boto/tests/integration/sns/__init__.py +20 -0
  533. data/packages/gsutil/boto/tests/integration/sns/test_cert_verification.py +40 -0
  534. data/packages/gsutil/boto/tests/integration/sqs/__init__.py +20 -0
  535. data/packages/gsutil/boto/tests/integration/sqs/test_cert_verification.py +40 -0
  536. data/packages/gsutil/boto/tests/integration/sqs/test_connection.py +217 -0
  537. data/packages/gsutil/boto/tests/integration/sts/__init__.py +20 -0
  538. data/packages/gsutil/boto/tests/integration/sts/test_cert_verification.py +40 -0
  539. data/packages/gsutil/boto/tests/integration/sts/test_session_token.py +65 -0
  540. data/packages/gsutil/boto/tests/integration/swf/__init__.py +0 -0
  541. data/packages/gsutil/boto/tests/integration/swf/test_cert_verification.py +40 -0
  542. data/packages/gsutil/boto/tests/integration/swf/test_layer1.py +246 -0
  543. data/packages/gsutil/boto/tests/integration/swf/test_layer1_workflow_execution.py +173 -0
  544. data/packages/gsutil/boto/tests/mturk/__init__.py +0 -0
  545. data/packages/gsutil/boto/tests/mturk/_init_environment.py +28 -0
  546. data/packages/gsutil/boto/tests/mturk/all_tests.py +24 -0
  547. data/packages/gsutil/boto/tests/mturk/cleanup_tests.py +47 -0
  548. data/packages/gsutil/boto/tests/mturk/common.py +45 -0
  549. data/packages/gsutil/boto/tests/mturk/create_free_text_question_regex.doctest +100 -0
  550. data/packages/gsutil/boto/tests/mturk/create_hit.doctest +92 -0
  551. data/packages/gsutil/boto/tests/mturk/create_hit_binary.doctest +94 -0
  552. data/packages/gsutil/boto/tests/mturk/create_hit_external.py +21 -0
  553. data/packages/gsutil/boto/tests/mturk/create_hit_from_hit_type.doctest +103 -0
  554. data/packages/gsutil/boto/tests/mturk/create_hit_test.py +21 -0
  555. data/packages/gsutil/boto/tests/mturk/create_hit_with_qualifications.py +16 -0
  556. data/packages/gsutil/boto/tests/mturk/hit_persistence.py +27 -0
  557. data/packages/gsutil/boto/tests/mturk/mocks.py +11 -0
  558. data/packages/gsutil/boto/tests/mturk/reviewable_hits.doctest +129 -0
  559. data/packages/gsutil/boto/tests/mturk/run-doctest.py +13 -0
  560. data/packages/gsutil/boto/tests/mturk/search_hits.doctest +16 -0
  561. data/packages/gsutil/boto/tests/mturk/selenium_support.py +61 -0
  562. data/packages/gsutil/boto/tests/mturk/support.py +7 -0
  563. data/packages/gsutil/boto/tests/mturk/test_disable_hit.py +11 -0
  564. data/packages/gsutil/boto/tests/test.py +59 -0
  565. data/packages/gsutil/boto/tests/unit/__init__.py +79 -0
  566. data/packages/gsutil/boto/tests/unit/auth/__init__.py +0 -0
  567. data/packages/gsutil/boto/tests/unit/auth/test_sigv4.py +73 -0
  568. data/packages/gsutil/boto/tests/unit/beanstalk/__init__.py +0 -0
  569. data/packages/gsutil/boto/tests/unit/beanstalk/test_layer1.py +128 -0
  570. data/packages/gsutil/boto/tests/unit/cloudformation/__init__.py +0 -0
  571. data/packages/gsutil/boto/tests/unit/cloudformation/test_connection.py +605 -0
  572. data/packages/gsutil/boto/tests/unit/cloudformation/test_stack.py +63 -0
  573. data/packages/gsutil/boto/tests/unit/cloudfront/__init__.py +0 -0
  574. data/packages/gsutil/boto/tests/unit/cloudfront/test_invalidation_list.py +113 -0
  575. data/packages/gsutil/boto/tests/unit/cloudfront/test_signed_urls.py +354 -0
  576. data/packages/gsutil/boto/tests/unit/cloudsearch/__init__.py +1 -0
  577. data/packages/gsutil/boto/tests/unit/cloudsearch/test_connection.py +241 -0
  578. data/packages/gsutil/boto/tests/unit/cloudsearch/test_document.py +324 -0
  579. data/packages/gsutil/boto/tests/unit/cloudsearch/test_search.py +325 -0
  580. data/packages/gsutil/boto/tests/unit/dynamodb/__init__.py +0 -0
  581. data/packages/gsutil/boto/tests/unit/dynamodb/test_batch.py +103 -0
  582. data/packages/gsutil/boto/tests/unit/dynamodb/test_layer2.py +119 -0
  583. data/packages/gsutil/boto/tests/unit/dynamodb/test_types.py +82 -0
  584. data/packages/gsutil/boto/tests/unit/ec2/__init__.py +0 -0
  585. data/packages/gsutil/boto/tests/unit/ec2/autoscale/__init__.py +0 -0
  586. data/packages/gsutil/boto/tests/unit/ec2/autoscale/test_group.py +162 -0
  587. data/packages/gsutil/boto/tests/unit/ec2/test_address.py +39 -0
  588. data/packages/gsutil/boto/tests/unit/ec2/test_blockdevicemapping.py +79 -0
  589. data/packages/gsutil/boto/tests/unit/ec2/test_connection.py +480 -0
  590. data/packages/gsutil/boto/tests/unit/ec2/test_instance.py +243 -0
  591. data/packages/gsutil/boto/tests/unit/ec2/test_networkinterface.py +140 -0
  592. data/packages/gsutil/boto/tests/unit/ec2/test_volume.py +248 -0
  593. data/packages/gsutil/boto/tests/unit/emr/test_emr_responses.py +373 -0
  594. data/packages/gsutil/boto/tests/unit/glacier/__init__.py +0 -0
  595. data/packages/gsutil/boto/tests/unit/glacier/test_concurrent.py +120 -0
  596. data/packages/gsutil/boto/tests/unit/glacier/test_job.py +60 -0
  597. data/packages/gsutil/boto/tests/unit/glacier/test_layer1.py +98 -0
  598. data/packages/gsutil/boto/tests/unit/glacier/test_layer2.py +266 -0
  599. data/packages/gsutil/boto/tests/unit/glacier/test_utils.py +116 -0
  600. data/packages/gsutil/boto/tests/unit/glacier/test_vault.py +100 -0
  601. data/packages/gsutil/boto/tests/unit/glacier/test_writer.py +185 -0
  602. data/packages/gsutil/boto/tests/unit/provider/__init__.py +0 -0
  603. data/packages/gsutil/boto/tests/unit/provider/test_provider.py +176 -0
  604. data/packages/gsutil/boto/tests/unit/rds/__init__.py +0 -0
  605. data/packages/gsutil/boto/tests/unit/rds/test_connection.py +131 -0
  606. data/packages/gsutil/boto/tests/unit/s3/__init__.py +0 -0
  607. data/packages/gsutil/boto/tests/unit/s3/test_cors_configuration.py +77 -0
  608. data/packages/gsutil/boto/tests/unit/s3/test_key.py +75 -0
  609. data/packages/gsutil/boto/tests/unit/s3/test_keyfile.py +101 -0
  610. data/packages/gsutil/boto/tests/unit/s3/test_lifecycle.py +97 -0
  611. data/packages/gsutil/boto/tests/unit/s3/test_tagging.py +47 -0
  612. data/packages/gsutil/boto/tests/unit/s3/test_uri.py +257 -0
  613. data/packages/gsutil/boto/tests/unit/s3/test_website.py +188 -0
  614. data/packages/gsutil/boto/tests/unit/sns/__init__.py +0 -0
  615. data/packages/gsutil/boto/tests/unit/sns/test_connection.py +99 -0
  616. data/packages/gsutil/boto/tests/unit/sqs/__init__.py +0 -0
  617. data/packages/gsutil/boto/tests/unit/sqs/test_connection.py +98 -0
  618. data/packages/gsutil/boto/tests/unit/sqs/test_queue.py +40 -0
  619. data/packages/gsutil/boto/tests/unit/sts/test_connection.py +74 -0
  620. data/packages/gsutil/boto/tests/unit/test_connection.py +60 -0
  621. data/packages/gsutil/boto/tests/unit/utils/test_utils.py +109 -0
  622. data/packages/gsutil/boto/tox.ini +8 -0
  623. data/packages/gsutil/gslib/README +5 -0
  624. data/packages/gsutil/gslib/__init__.py +22 -0
  625. data/packages/gsutil/gslib/__init__.pyc +0 -0
  626. data/packages/gsutil/gslib/addlhelp/__init__.py +15 -0
  627. data/packages/gsutil/gslib/addlhelp/acls.py +234 -0
  628. data/packages/gsutil/gslib/addlhelp/anon.py +57 -0
  629. data/packages/gsutil/gslib/addlhelp/command_opts.py +116 -0
  630. data/packages/gsutil/gslib/addlhelp/dev.py +139 -0
  631. data/packages/gsutil/gslib/addlhelp/metadata.py +186 -0
  632. data/packages/gsutil/gslib/addlhelp/naming.py +173 -0
  633. data/packages/gsutil/gslib/addlhelp/prod.py +160 -0
  634. data/packages/gsutil/gslib/addlhelp/projects.py +130 -0
  635. data/packages/gsutil/gslib/addlhelp/subdirs.py +110 -0
  636. data/packages/gsutil/gslib/addlhelp/support.py +86 -0
  637. data/packages/gsutil/gslib/addlhelp/versioning.py +242 -0
  638. data/packages/gsutil/gslib/addlhelp/wildcards.py +170 -0
  639. data/packages/gsutil/gslib/bucket_listing_ref.py +175 -0
  640. data/packages/gsutil/gslib/bucket_listing_ref.pyc +0 -0
  641. data/packages/gsutil/gslib/command.py +722 -0
  642. data/packages/gsutil/gslib/command.pyc +0 -0
  643. data/packages/gsutil/gslib/command_runner.py +101 -0
  644. data/packages/gsutil/gslib/command_runner.pyc +0 -0
  645. data/packages/gsutil/gslib/commands/__init__.py +15 -0
  646. data/packages/gsutil/gslib/commands/__init__.pyc +0 -0
  647. data/packages/gsutil/gslib/commands/cat.py +131 -0
  648. data/packages/gsutil/gslib/commands/cat.pyc +0 -0
  649. data/packages/gsutil/gslib/commands/chacl.py +532 -0
  650. data/packages/gsutil/gslib/commands/chacl.pyc +0 -0
  651. data/packages/gsutil/gslib/commands/config.py +694 -0
  652. data/packages/gsutil/gslib/commands/config.pyc +0 -0
  653. data/packages/gsutil/gslib/commands/cp.py +1818 -0
  654. data/packages/gsutil/gslib/commands/cp.pyc +0 -0
  655. data/packages/gsutil/gslib/commands/disablelogging.py +101 -0
  656. data/packages/gsutil/gslib/commands/disablelogging.pyc +0 -0
  657. data/packages/gsutil/gslib/commands/enablelogging.py +149 -0
  658. data/packages/gsutil/gslib/commands/enablelogging.pyc +0 -0
  659. data/packages/gsutil/gslib/commands/getacl.py +82 -0
  660. data/packages/gsutil/gslib/commands/getacl.pyc +0 -0
  661. data/packages/gsutil/gslib/commands/getcors.py +121 -0
  662. data/packages/gsutil/gslib/commands/getcors.pyc +0 -0
  663. data/packages/gsutil/gslib/commands/getdefacl.py +86 -0
  664. data/packages/gsutil/gslib/commands/getdefacl.pyc +0 -0
  665. data/packages/gsutil/gslib/commands/getlogging.py +137 -0
  666. data/packages/gsutil/gslib/commands/getlogging.pyc +0 -0
  667. data/packages/gsutil/gslib/commands/getversioning.py +116 -0
  668. data/packages/gsutil/gslib/commands/getversioning.pyc +0 -0
  669. data/packages/gsutil/gslib/commands/getwebcfg.py +122 -0
  670. data/packages/gsutil/gslib/commands/getwebcfg.pyc +0 -0
  671. data/packages/gsutil/gslib/commands/help.py +218 -0
  672. data/packages/gsutil/gslib/commands/help.pyc +0 -0
  673. data/packages/gsutil/gslib/commands/ls.py +578 -0
  674. data/packages/gsutil/gslib/commands/ls.pyc +0 -0
  675. data/packages/gsutil/gslib/commands/mb.py +172 -0
  676. data/packages/gsutil/gslib/commands/mb.pyc +0 -0
  677. data/packages/gsutil/gslib/commands/mv.py +159 -0
  678. data/packages/gsutil/gslib/commands/mv.pyc +0 -0
  679. data/packages/gsutil/gslib/commands/perfdiag.py +903 -0
  680. data/packages/gsutil/gslib/commands/perfdiag.pyc +0 -0
  681. data/packages/gsutil/gslib/commands/rb.py +113 -0
  682. data/packages/gsutil/gslib/commands/rb.pyc +0 -0
  683. data/packages/gsutil/gslib/commands/rm.py +239 -0
  684. data/packages/gsutil/gslib/commands/rm.pyc +0 -0
  685. data/packages/gsutil/gslib/commands/setacl.py +138 -0
  686. data/packages/gsutil/gslib/commands/setacl.pyc +0 -0
  687. data/packages/gsutil/gslib/commands/setcors.py +145 -0
  688. data/packages/gsutil/gslib/commands/setcors.pyc +0 -0
  689. data/packages/gsutil/gslib/commands/setdefacl.py +105 -0
  690. data/packages/gsutil/gslib/commands/setdefacl.pyc +0 -0
  691. data/packages/gsutil/gslib/commands/setmeta.py +428 -0
  692. data/packages/gsutil/gslib/commands/setmeta.pyc +0 -0
  693. data/packages/gsutil/gslib/commands/setversioning.py +114 -0
  694. data/packages/gsutil/gslib/commands/setversioning.pyc +0 -0
  695. data/packages/gsutil/gslib/commands/setwebcfg.py +190 -0
  696. data/packages/gsutil/gslib/commands/setwebcfg.pyc +0 -0
  697. data/packages/gsutil/gslib/commands/test.py +228 -0
  698. data/packages/gsutil/gslib/commands/test.pyc +0 -0
  699. data/packages/gsutil/gslib/commands/update.py +305 -0
  700. data/packages/gsutil/gslib/commands/update.pyc +0 -0
  701. data/packages/gsutil/gslib/commands/version.py +150 -0
  702. data/packages/gsutil/gslib/commands/version.pyc +0 -0
  703. data/packages/gsutil/gslib/exception.py +76 -0
  704. data/packages/gsutil/gslib/exception.pyc +0 -0
  705. data/packages/gsutil/gslib/help_provider.py +81 -0
  706. data/packages/gsutil/gslib/help_provider.pyc +0 -0
  707. data/packages/gsutil/gslib/name_expansion.py +550 -0
  708. data/packages/gsutil/gslib/name_expansion.pyc +0 -0
  709. data/packages/gsutil/gslib/no_op_auth_plugin.py +30 -0
  710. data/packages/gsutil/gslib/plurality_checkable_iterator.py +56 -0
  711. data/packages/gsutil/gslib/plurality_checkable_iterator.pyc +0 -0
  712. data/packages/gsutil/gslib/project_id.py +67 -0
  713. data/packages/gsutil/gslib/project_id.pyc +0 -0
  714. data/packages/gsutil/gslib/storage_uri_builder.py +56 -0
  715. data/packages/gsutil/gslib/storage_uri_builder.pyc +0 -0
  716. data/packages/gsutil/gslib/tests/__init__.py +15 -0
  717. data/packages/gsutil/gslib/tests/__init__.pyc +0 -0
  718. data/packages/gsutil/gslib/tests/test_chacl.py +236 -0
  719. data/packages/gsutil/gslib/tests/test_cp.py +267 -0
  720. data/packages/gsutil/gslib/tests/test_data/test.gif +0 -0
  721. data/packages/gsutil/gslib/tests/test_data/test.mp3 +0 -0
  722. data/packages/gsutil/gslib/tests/test_ls.py +66 -0
  723. data/packages/gsutil/gslib/tests/test_mv.py +69 -0
  724. data/packages/gsutil/gslib/tests/test_naming.py +989 -0
  725. data/packages/gsutil/gslib/tests/test_perfdiag.py +41 -0
  726. data/packages/gsutil/gslib/tests/test_plurality_checkable_iterator.py +67 -0
  727. data/packages/gsutil/gslib/tests/test_rm.py +143 -0
  728. data/packages/gsutil/gslib/tests/test_setacl.py +152 -0
  729. data/packages/gsutil/gslib/tests/test_setcors.py +168 -0
  730. data/packages/gsutil/gslib/tests/test_setmeta.py +91 -0
  731. data/packages/gsutil/gslib/tests/test_setversioning.py +44 -0
  732. data/packages/gsutil/gslib/tests/test_setwebcfg.py +63 -0
  733. data/packages/gsutil/gslib/tests/test_thread_pool.py +92 -0
  734. data/packages/gsutil/gslib/tests/test_wildcard_iterator.py +364 -0
  735. data/packages/gsutil/gslib/tests/testcase/__init__.py +18 -0
  736. data/packages/gsutil/gslib/tests/testcase/base.py +89 -0
  737. data/packages/gsutil/gslib/tests/testcase/integration_testcase.py +197 -0
  738. data/packages/gsutil/gslib/tests/testcase/unit_testcase.py +230 -0
  739. data/packages/gsutil/gslib/tests/util.py +125 -0
  740. data/packages/gsutil/gslib/tests/util.pyc +0 -0
  741. data/packages/gsutil/gslib/thread_pool.py +79 -0
  742. data/packages/gsutil/gslib/thread_pool.pyc +0 -0
  743. data/packages/gsutil/gslib/util.py +151 -0
  744. data/packages/gsutil/gslib/util.pyc +0 -0
  745. data/packages/gsutil/gslib/wildcard_iterator.py +492 -0
  746. data/packages/gsutil/gslib/wildcard_iterator.pyc +0 -0
  747. data/packages/gsutil/gsutil +377 -0
  748. data/packages/gsutil/gsutil.spec.in +75 -0
  749. data/packages/gsutil/oauth2_plugin/__init__.py +22 -0
  750. data/packages/gsutil/oauth2_plugin/__init__.pyc +0 -0
  751. data/packages/gsutil/oauth2_plugin/oauth2_client.py +642 -0
  752. data/packages/gsutil/oauth2_plugin/oauth2_client.pyc +0 -0
  753. data/packages/gsutil/oauth2_plugin/oauth2_client_test.py +374 -0
  754. data/packages/gsutil/oauth2_plugin/oauth2_helper.py +110 -0
  755. data/packages/gsutil/oauth2_plugin/oauth2_helper.pyc +0 -0
  756. data/packages/gsutil/oauth2_plugin/oauth2_plugin.py +24 -0
  757. data/packages/gsutil/oauth2_plugin/oauth2_plugin.pyc +0 -0
  758. data/packages/gsutil/pkg_gen.sh +54 -0
  759. data/packages/gsutil/pkg_util.py +60 -0
  760. data/packages/gsutil/setup.py +141 -0
  761. data/packages/gsutil/third_party/__init__.py +0 -0
  762. data/packages/gsutil/third_party/__init__.pyc +0 -0
  763. data/packages/gsutil/third_party/fancy_urllib/README +21 -0
  764. data/packages/gsutil/third_party/fancy_urllib/__init__.py +398 -0
  765. data/packages/gsutil/third_party/fancy_urllib/__init__.pyc +0 -0
  766. data/tasks/rubygem.rake +3 -1
  767. metadata +770 -7
  768. metadata.gz.sig +0 -0
@@ -0,0 +1,358 @@
1
+ #!/usr/bin/env python
2
+ #
3
+ # Copyright 2010 Google Inc.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a
6
+ # copy of this software and associated documentation files (the
7
+ # "Software"), to deal in the Software without restriction, including
8
+ # without limitation the rights to use, copy, modify, merge, publish, dis-
9
+ # tribute, sublicense, and/or sell copies of the Software, and to permit
10
+ # persons to whom the Software is furnished to do so, subject to the fol-
11
+ # lowing conditions:
12
+ #
13
+ # The above copyright notice and this permission notice shall be included
14
+ # in all copies or substantial portions of the Software.
15
+ #
16
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17
+ # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
18
+ # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
19
+ # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
20
+ # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22
+ # IN THE SOFTWARE.
23
+
24
+ """
25
+ Tests of resumable downloads.
26
+ """
27
+
28
+ import errno
29
+ import os
30
+ import re
31
+
32
+ import boto
33
+ from boto.s3.resumable_download_handler import get_cur_file_size
34
+ from boto.s3.resumable_download_handler import ResumableDownloadHandler
35
+ from boto.exception import ResumableTransferDisposition
36
+ from boto.exception import ResumableDownloadException
37
+ from cb_test_harness import CallbackTestHarness
38
+ from tests.integration.gs.testcase import GSTestCase
39
+
40
+
41
+ SMALL_KEY_SIZE = 2 * 1024 # 2 KB.
42
+ LARGE_KEY_SIZE = 500 * 1024 # 500 KB.
43
+
44
+
45
+ class ResumableDownloadTests(GSTestCase):
46
+ """Resumable download test suite."""
47
+
48
+ def make_small_key(self):
49
+ small_src_key_as_string = os.urandom(SMALL_KEY_SIZE)
50
+ small_src_key = self._MakeKey(data=small_src_key_as_string)
51
+ return small_src_key_as_string, small_src_key
52
+
53
+ def make_tracker_file(self, tmpdir=None):
54
+ if not tmpdir:
55
+ tmpdir = self._MakeTempDir()
56
+ tracker_file = os.path.join(tmpdir, 'tracker')
57
+ return tracker_file
58
+
59
+ def make_dst_fp(self, tmpdir=None):
60
+ if not tmpdir:
61
+ tmpdir = self._MakeTempDir()
62
+ dst_file = os.path.join(tmpdir, 'dstfile')
63
+ return open(dst_file, 'w')
64
+
65
+ def test_non_resumable_download(self):
66
+ """
67
+ Tests that non-resumable downloads work
68
+ """
69
+ dst_fp = self.make_dst_fp()
70
+ small_src_key_as_string, small_src_key = self.make_small_key()
71
+ small_src_key.get_contents_to_file(dst_fp)
72
+ self.assertEqual(SMALL_KEY_SIZE,
73
+ get_cur_file_size(dst_fp))
74
+ self.assertEqual(small_src_key_as_string,
75
+ small_src_key.get_contents_as_string())
76
+
77
+ def test_download_without_persistent_tracker(self):
78
+ """
79
+ Tests a single resumable download, with no tracker persistence
80
+ """
81
+ res_download_handler = ResumableDownloadHandler()
82
+ dst_fp = self.make_dst_fp()
83
+ small_src_key_as_string, small_src_key = self.make_small_key()
84
+ small_src_key.get_contents_to_file(
85
+ dst_fp, res_download_handler=res_download_handler)
86
+ self.assertEqual(SMALL_KEY_SIZE,
87
+ get_cur_file_size(dst_fp))
88
+ self.assertEqual(small_src_key_as_string,
89
+ small_src_key.get_contents_as_string())
90
+
91
+ def test_failed_download_with_persistent_tracker(self):
92
+ """
93
+ Tests that failed resumable download leaves a correct tracker file
94
+ """
95
+ harness = CallbackTestHarness()
96
+ tmpdir = self._MakeTempDir()
97
+ tracker_file_name = self.make_tracker_file(tmpdir)
98
+ dst_fp = self.make_dst_fp(tmpdir)
99
+ res_download_handler = ResumableDownloadHandler(
100
+ tracker_file_name=tracker_file_name, num_retries=0)
101
+ small_src_key_as_string, small_src_key = self.make_small_key()
102
+ try:
103
+ small_src_key.get_contents_to_file(
104
+ dst_fp, cb=harness.call,
105
+ res_download_handler=res_download_handler)
106
+ self.fail('Did not get expected ResumableDownloadException')
107
+ except ResumableDownloadException, e:
108
+ # We'll get a ResumableDownloadException at this point because
109
+ # of CallbackTestHarness (above). Check that the tracker file was
110
+ # created correctly.
111
+ self.assertEqual(e.disposition,
112
+ ResumableTransferDisposition.ABORT_CUR_PROCESS)
113
+ self.assertTrue(os.path.exists(tracker_file_name))
114
+ f = open(tracker_file_name)
115
+ etag_line = f.readline()
116
+ m = re.search(ResumableDownloadHandler.ETAG_REGEX, etag_line)
117
+ f.close()
118
+ self.assertTrue(m)
119
+
120
+ def test_retryable_exception_recovery(self):
121
+ """
122
+ Tests handling of a retryable exception
123
+ """
124
+ # Test one of the RETRYABLE_EXCEPTIONS.
125
+ exception = ResumableDownloadHandler.RETRYABLE_EXCEPTIONS[0]
126
+ harness = CallbackTestHarness(exception=exception)
127
+ res_download_handler = ResumableDownloadHandler(num_retries=1)
128
+ dst_fp = self.make_dst_fp()
129
+ small_src_key_as_string, small_src_key = self.make_small_key()
130
+ small_src_key.get_contents_to_file(
131
+ dst_fp, cb=harness.call,
132
+ res_download_handler=res_download_handler)
133
+ # Ensure downloaded object has correct content.
134
+ self.assertEqual(SMALL_KEY_SIZE,
135
+ get_cur_file_size(dst_fp))
136
+ self.assertEqual(small_src_key_as_string,
137
+ small_src_key.get_contents_as_string())
138
+
139
+ def test_broken_pipe_recovery(self):
140
+ """
141
+ Tests handling of a Broken Pipe (which interacts with an httplib bug)
142
+ """
143
+ exception = IOError(errno.EPIPE, "Broken pipe")
144
+ harness = CallbackTestHarness(exception=exception)
145
+ res_download_handler = ResumableDownloadHandler(num_retries=1)
146
+ dst_fp = self.make_dst_fp()
147
+ small_src_key_as_string, small_src_key = self.make_small_key()
148
+ small_src_key.get_contents_to_file(
149
+ dst_fp, cb=harness.call,
150
+ res_download_handler=res_download_handler)
151
+ # Ensure downloaded object has correct content.
152
+ self.assertEqual(SMALL_KEY_SIZE,
153
+ get_cur_file_size(dst_fp))
154
+ self.assertEqual(small_src_key_as_string,
155
+ small_src_key.get_contents_as_string())
156
+
157
+ def test_non_retryable_exception_handling(self):
158
+ """
159
+ Tests resumable download that fails with a non-retryable exception
160
+ """
161
+ harness = CallbackTestHarness(
162
+ exception=OSError(errno.EACCES, 'Permission denied'))
163
+ res_download_handler = ResumableDownloadHandler(num_retries=1)
164
+ dst_fp = self.make_dst_fp()
165
+ small_src_key_as_string, small_src_key = self.make_small_key()
166
+ try:
167
+ small_src_key.get_contents_to_file(
168
+ dst_fp, cb=harness.call,
169
+ res_download_handler=res_download_handler)
170
+ self.fail('Did not get expected OSError')
171
+ except OSError, e:
172
+ # Ensure the error was re-raised.
173
+ self.assertEqual(e.errno, 13)
174
+
175
+ def test_failed_and_restarted_download_with_persistent_tracker(self):
176
+ """
177
+ Tests resumable download that fails once and then completes,
178
+ with tracker file
179
+ """
180
+ harness = CallbackTestHarness()
181
+ tmpdir = self._MakeTempDir()
182
+ tracker_file_name = self.make_tracker_file(tmpdir)
183
+ dst_fp = self.make_dst_fp(tmpdir)
184
+ small_src_key_as_string, small_src_key = self.make_small_key()
185
+ res_download_handler = ResumableDownloadHandler(
186
+ tracker_file_name=tracker_file_name, num_retries=1)
187
+ small_src_key.get_contents_to_file(
188
+ dst_fp, cb=harness.call,
189
+ res_download_handler=res_download_handler)
190
+ # Ensure downloaded object has correct content.
191
+ self.assertEqual(SMALL_KEY_SIZE,
192
+ get_cur_file_size(dst_fp))
193
+ self.assertEqual(small_src_key_as_string,
194
+ small_src_key.get_contents_as_string())
195
+ # Ensure tracker file deleted.
196
+ self.assertFalse(os.path.exists(tracker_file_name))
197
+
198
+ def test_multiple_in_process_failures_then_succeed(self):
199
+ """
200
+ Tests resumable download that fails twice in one process, then completes
201
+ """
202
+ res_download_handler = ResumableDownloadHandler(num_retries=3)
203
+ dst_fp = self.make_dst_fp()
204
+ small_src_key_as_string, small_src_key = self.make_small_key()
205
+ small_src_key.get_contents_to_file(
206
+ dst_fp, res_download_handler=res_download_handler)
207
+ # Ensure downloaded object has correct content.
208
+ self.assertEqual(SMALL_KEY_SIZE,
209
+ get_cur_file_size(dst_fp))
210
+ self.assertEqual(small_src_key_as_string,
211
+ small_src_key.get_contents_as_string())
212
+
213
+ def test_multiple_in_process_failures_then_succeed_with_tracker_file(self):
214
+ """
215
+ Tests resumable download that fails completely in one process,
216
+ then when restarted completes, using a tracker file
217
+ """
218
+ # Set up test harness that causes more failures than a single
219
+ # ResumableDownloadHandler instance will handle, writing enough data
220
+ # before the first failure that some of it survives that process run.
221
+ harness = CallbackTestHarness(
222
+ fail_after_n_bytes=LARGE_KEY_SIZE/2, num_times_to_fail=2)
223
+ larger_src_key_as_string = os.urandom(LARGE_KEY_SIZE)
224
+ larger_src_key = self._MakeKey(data=larger_src_key_as_string)
225
+ tmpdir = self._MakeTempDir()
226
+ tracker_file_name = self.make_tracker_file(tmpdir)
227
+ dst_fp = self.make_dst_fp(tmpdir)
228
+ res_download_handler = ResumableDownloadHandler(
229
+ tracker_file_name=tracker_file_name, num_retries=0)
230
+ try:
231
+ larger_src_key.get_contents_to_file(
232
+ dst_fp, cb=harness.call,
233
+ res_download_handler=res_download_handler)
234
+ self.fail('Did not get expected ResumableDownloadException')
235
+ except ResumableDownloadException, e:
236
+ self.assertEqual(e.disposition,
237
+ ResumableTransferDisposition.ABORT_CUR_PROCESS)
238
+ # Ensure a tracker file survived.
239
+ self.assertTrue(os.path.exists(tracker_file_name))
240
+ # Try it one more time; this time should succeed.
241
+ larger_src_key.get_contents_to_file(
242
+ dst_fp, cb=harness.call,
243
+ res_download_handler=res_download_handler)
244
+ self.assertEqual(LARGE_KEY_SIZE,
245
+ get_cur_file_size(dst_fp))
246
+ self.assertEqual(larger_src_key_as_string,
247
+ larger_src_key.get_contents_as_string())
248
+ self.assertFalse(os.path.exists(tracker_file_name))
249
+ # Ensure some of the file was downloaded both before and after failure.
250
+ self.assertTrue(
251
+ len(harness.transferred_seq_before_first_failure) > 1 and
252
+ len(harness.transferred_seq_after_first_failure) > 1)
253
+
254
+ def test_download_with_inital_partial_download_before_failure(self):
255
+ """
256
+ Tests resumable download that successfully downloads some content
257
+ before it fails, then restarts and completes
258
+ """
259
+ # Set up harness to fail download after several hundred KB so download
260
+ # server will have saved something before we retry.
261
+ harness = CallbackTestHarness(
262
+ fail_after_n_bytes=LARGE_KEY_SIZE/2)
263
+ larger_src_key_as_string = os.urandom(LARGE_KEY_SIZE)
264
+ larger_src_key = self._MakeKey(data=larger_src_key_as_string)
265
+ res_download_handler = ResumableDownloadHandler(num_retries=1)
266
+ dst_fp = self.make_dst_fp()
267
+ larger_src_key.get_contents_to_file(
268
+ dst_fp, cb=harness.call,
269
+ res_download_handler=res_download_handler)
270
+ # Ensure downloaded object has correct content.
271
+ self.assertEqual(LARGE_KEY_SIZE,
272
+ get_cur_file_size(dst_fp))
273
+ self.assertEqual(larger_src_key_as_string,
274
+ larger_src_key.get_contents_as_string())
275
+ # Ensure some of the file was downloaded both before and after failure.
276
+ self.assertTrue(
277
+ len(harness.transferred_seq_before_first_failure) > 1 and
278
+ len(harness.transferred_seq_after_first_failure) > 1)
279
+
280
+ def test_zero_length_object_download(self):
281
+ """
282
+ Tests downloading a zero-length object (exercises boundary conditions).
283
+ """
284
+ res_download_handler = ResumableDownloadHandler()
285
+ dst_fp = self.make_dst_fp()
286
+ k = self._MakeKey()
287
+ k.get_contents_to_file(dst_fp,
288
+ res_download_handler=res_download_handler)
289
+ self.assertEqual(0, get_cur_file_size(dst_fp))
290
+
291
+ def test_download_with_invalid_tracker_etag(self):
292
+ """
293
+ Tests resumable download with a tracker file containing an invalid etag
294
+ """
295
+ tmp_dir = self._MakeTempDir()
296
+ dst_fp = self.make_dst_fp(tmp_dir)
297
+ small_src_key_as_string, small_src_key = self.make_small_key()
298
+ invalid_etag_tracker_file_name = os.path.join(tmp_dir,
299
+ 'invalid_etag_tracker')
300
+ f = open(invalid_etag_tracker_file_name, 'w')
301
+ f.write('3.14159\n')
302
+ f.close()
303
+ res_download_handler = ResumableDownloadHandler(
304
+ tracker_file_name=invalid_etag_tracker_file_name)
305
+ # An error should be printed about the invalid tracker, but then it
306
+ # should run the update successfully.
307
+ small_src_key.get_contents_to_file(
308
+ dst_fp, res_download_handler=res_download_handler)
309
+ self.assertEqual(SMALL_KEY_SIZE, get_cur_file_size(dst_fp))
310
+ self.assertEqual(small_src_key_as_string,
311
+ small_src_key.get_contents_as_string())
312
+
313
+ def test_download_with_inconsistent_etag_in_tracker(self):
314
+ """
315
+ Tests resumable download with an inconsistent etag in tracker file
316
+ """
317
+ tmp_dir = self._MakeTempDir()
318
+ dst_fp = self.make_dst_fp(tmp_dir)
319
+ small_src_key_as_string, small_src_key = self.make_small_key()
320
+ inconsistent_etag_tracker_file_name = os.path.join(tmp_dir,
321
+ 'inconsistent_etag_tracker')
322
+ f = open(inconsistent_etag_tracker_file_name, 'w')
323
+ good_etag = small_src_key.etag.strip('"\'')
324
+ new_val_as_list = []
325
+ for c in reversed(good_etag):
326
+ new_val_as_list.append(c)
327
+ f.write('%s\n' % ''.join(new_val_as_list))
328
+ f.close()
329
+ res_download_handler = ResumableDownloadHandler(
330
+ tracker_file_name=inconsistent_etag_tracker_file_name)
331
+ # An error should be printed about the expired tracker, but then it
332
+ # should run the update successfully.
333
+ small_src_key.get_contents_to_file(
334
+ dst_fp, res_download_handler=res_download_handler)
335
+ self.assertEqual(SMALL_KEY_SIZE,
336
+ get_cur_file_size(dst_fp))
337
+ self.assertEqual(small_src_key_as_string,
338
+ small_src_key.get_contents_as_string())
339
+
340
+ def test_download_with_unwritable_tracker_file(self):
341
+ """
342
+ Tests resumable download with an unwritable tracker file
343
+ """
344
+ # Make dir where tracker_file lives temporarily unwritable.
345
+ tmp_dir = self._MakeTempDir()
346
+ tracker_file_name = os.path.join(tmp_dir, 'tracker')
347
+ save_mod = os.stat(tmp_dir).st_mode
348
+ try:
349
+ os.chmod(tmp_dir, 0)
350
+ res_download_handler = ResumableDownloadHandler(
351
+ tracker_file_name=tracker_file_name)
352
+ except ResumableDownloadException, e:
353
+ self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT)
354
+ self.assertNotEqual(
355
+ e.message.find('Couldn\'t write URI tracker file'), -1)
356
+ finally:
357
+ # Restore original protection of dir where tracker_file lives.
358
+ os.chmod(tmp_dir, save_mod)
@@ -0,0 +1,525 @@
1
+ #!/usr/bin/env python
2
+ #
3
+ # Copyright 2010 Google Inc.
4
+ #
5
+ # Permission is hereby granted, free of charge, to any person obtaining a
6
+ # copy of this software and associated documentation files (the
7
+ # "Software"), to deal in the Software without restriction, including
8
+ # without limitation the rights to use, copy, modify, merge, publish, dis-
9
+ # tribute, sublicense, and/or sell copies of the Software, and to permit
10
+ # persons to whom the Software is furnished to do so, subject to the fol-
11
+ # lowing conditions:
12
+ #
13
+ # The above copyright notice and this permission notice shall be included
14
+ # in all copies or substantial portions of the Software.
15
+ #
16
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
17
+ # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
18
+ # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
19
+ # SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
20
+ # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21
+ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
22
+ # IN THE SOFTWARE.
23
+
24
+ """
25
+ Tests of Google Cloud Storage resumable uploads.
26
+ """
27
+
28
+ import StringIO
29
+ import errno
30
+ import random
31
+ import os
32
+ import time
33
+
34
+ import boto
35
+ from boto import storage_uri
36
+ from boto.gs.resumable_upload_handler import ResumableUploadHandler
37
+ from boto.exception import InvalidUriError
38
+ from boto.exception import ResumableTransferDisposition
39
+ from boto.exception import ResumableUploadException
40
+ from cb_test_harness import CallbackTestHarness
41
+ from tests.integration.gs.testcase import GSTestCase
42
+
43
+
44
+ SMALL_KEY_SIZE = 2 * 1024 # 2 KB.
45
+ LARGE_KEY_SIZE = 500 * 1024 # 500 KB.
46
+ LARGEST_KEY_SIZE = 1024 * 1024 # 1 MB.
47
+
48
+
49
+ class ResumableUploadTests(GSTestCase):
50
+ """Resumable upload test suite."""
51
+
52
+ def build_input_file(self, size):
53
+ buf = []
54
+ # I manually construct the random data here instead of calling
55
+ # os.urandom() because I want to constrain the range of data (in
56
+ # this case to 0'..'9') so the test
57
+ # code can easily overwrite part of the StringIO file with
58
+ # known-to-be-different values.
59
+ for i in range(size):
60
+ buf.append(str(random.randint(0, 9)))
61
+ file_as_string = ''.join(buf)
62
+ return (file_as_string, StringIO.StringIO(file_as_string))
63
+
64
+ def make_small_file(self):
65
+ return self.build_input_file(SMALL_KEY_SIZE)
66
+
67
+ def make_large_file(self):
68
+ return self.build_input_file(LARGE_KEY_SIZE)
69
+
70
+ def make_tracker_file(self, tmpdir=None):
71
+ if not tmpdir:
72
+ tmpdir = self._MakeTempDir()
73
+ tracker_file = os.path.join(tmpdir, 'tracker')
74
+ return tracker_file
75
+
76
+ def test_non_resumable_upload(self):
77
+ """
78
+ Tests that non-resumable uploads work
79
+ """
80
+ small_src_file_as_string, small_src_file = self.make_small_file()
81
+ # Seek to end incase its the first test.
82
+ small_src_file.seek(0, os.SEEK_END)
83
+ dst_key = self._MakeKey(set_contents=False)
84
+ try:
85
+ dst_key.set_contents_from_file(small_src_file)
86
+ self.fail("should fail as need to rewind the filepointer")
87
+ except AttributeError:
88
+ pass
89
+ # Now try calling with a proper rewind.
90
+ dst_key.set_contents_from_file(small_src_file, rewind=True)
91
+ self.assertEqual(SMALL_KEY_SIZE, dst_key.size)
92
+ self.assertEqual(small_src_file_as_string,
93
+ dst_key.get_contents_as_string())
94
+
95
+ def test_upload_without_persistent_tracker(self):
96
+ """
97
+ Tests a single resumable upload, with no tracker URI persistence
98
+ """
99
+ res_upload_handler = ResumableUploadHandler()
100
+ small_src_file_as_string, small_src_file = self.make_small_file()
101
+ small_src_file.seek(0)
102
+ dst_key = self._MakeKey(set_contents=False)
103
+ dst_key.set_contents_from_file(
104
+ small_src_file, res_upload_handler=res_upload_handler)
105
+ self.assertEqual(SMALL_KEY_SIZE, dst_key.size)
106
+ self.assertEqual(small_src_file_as_string,
107
+ dst_key.get_contents_as_string())
108
+
109
+ def test_failed_upload_with_persistent_tracker(self):
110
+ """
111
+ Tests that failed resumable upload leaves a correct tracker URI file
112
+ """
113
+ harness = CallbackTestHarness()
114
+ tracker_file_name = self.make_tracker_file()
115
+ res_upload_handler = ResumableUploadHandler(
116
+ tracker_file_name=tracker_file_name, num_retries=0)
117
+ small_src_file_as_string, small_src_file = self.make_small_file()
118
+ small_src_file.seek(0)
119
+ dst_key = self._MakeKey(set_contents=False)
120
+ try:
121
+ dst_key.set_contents_from_file(
122
+ small_src_file, cb=harness.call,
123
+ res_upload_handler=res_upload_handler)
124
+ self.fail('Did not get expected ResumableUploadException')
125
+ except ResumableUploadException, e:
126
+ # We'll get a ResumableUploadException at this point because
127
+ # of CallbackTestHarness (above). Check that the tracker file was
128
+ # created correctly.
129
+ self.assertEqual(e.disposition,
130
+ ResumableTransferDisposition.ABORT_CUR_PROCESS)
131
+ self.assertTrue(os.path.exists(tracker_file_name))
132
+ f = open(tracker_file_name)
133
+ uri_from_file = f.readline().strip()
134
+ f.close()
135
+ self.assertEqual(uri_from_file,
136
+ res_upload_handler.get_tracker_uri())
137
+
138
+ def test_retryable_exception_recovery(self):
139
+ """
140
+ Tests handling of a retryable exception
141
+ """
142
+ # Test one of the RETRYABLE_EXCEPTIONS.
143
+ exception = ResumableUploadHandler.RETRYABLE_EXCEPTIONS[0]
144
+ harness = CallbackTestHarness(exception=exception)
145
+ res_upload_handler = ResumableUploadHandler(num_retries=1)
146
+ small_src_file_as_string, small_src_file = self.make_small_file()
147
+ small_src_file.seek(0)
148
+ dst_key = self._MakeKey(set_contents=False)
149
+ dst_key.set_contents_from_file(
150
+ small_src_file, cb=harness.call,
151
+ res_upload_handler=res_upload_handler)
152
+ # Ensure uploaded object has correct content.
153
+ self.assertEqual(SMALL_KEY_SIZE, dst_key.size)
154
+ self.assertEqual(small_src_file_as_string,
155
+ dst_key.get_contents_as_string())
156
+
157
+ def test_broken_pipe_recovery(self):
158
+ """
159
+ Tests handling of a Broken Pipe (which interacts with an httplib bug)
160
+ """
161
+ exception = IOError(errno.EPIPE, "Broken pipe")
162
+ harness = CallbackTestHarness(exception=exception)
163
+ res_upload_handler = ResumableUploadHandler(num_retries=1)
164
+ small_src_file_as_string, small_src_file = self.make_small_file()
165
+ small_src_file.seek(0)
166
+ dst_key = self._MakeKey(set_contents=False)
167
+ dst_key.set_contents_from_file(
168
+ small_src_file, cb=harness.call,
169
+ res_upload_handler=res_upload_handler)
170
+ # Ensure uploaded object has correct content.
171
+ self.assertEqual(SMALL_KEY_SIZE, dst_key.size)
172
+ self.assertEqual(small_src_file_as_string,
173
+ dst_key.get_contents_as_string())
174
+
175
+ def test_non_retryable_exception_handling(self):
176
+ """
177
+ Tests a resumable upload that fails with a non-retryable exception
178
+ """
179
+ harness = CallbackTestHarness(
180
+ exception=OSError(errno.EACCES, 'Permission denied'))
181
+ res_upload_handler = ResumableUploadHandler(num_retries=1)
182
+ small_src_file_as_string, small_src_file = self.make_small_file()
183
+ small_src_file.seek(0)
184
+ dst_key = self._MakeKey(set_contents=False)
185
+ try:
186
+ dst_key.set_contents_from_file(
187
+ small_src_file, cb=harness.call,
188
+ res_upload_handler=res_upload_handler)
189
+ self.fail('Did not get expected OSError')
190
+ except OSError, e:
191
+ # Ensure the error was re-raised.
192
+ self.assertEqual(e.errno, 13)
193
+
194
+ def test_failed_and_restarted_upload_with_persistent_tracker(self):
195
+ """
196
+ Tests resumable upload that fails once and then completes, with tracker
197
+ file
198
+ """
199
+ harness = CallbackTestHarness()
200
+ tracker_file_name = self.make_tracker_file()
201
+ res_upload_handler = ResumableUploadHandler(
202
+ tracker_file_name=tracker_file_name, num_retries=1)
203
+ small_src_file_as_string, small_src_file = self.make_small_file()
204
+ small_src_file.seek(0)
205
+ dst_key = self._MakeKey(set_contents=False)
206
+ dst_key.set_contents_from_file(
207
+ small_src_file, cb=harness.call,
208
+ res_upload_handler=res_upload_handler)
209
+ # Ensure uploaded object has correct content.
210
+ self.assertEqual(SMALL_KEY_SIZE, dst_key.size)
211
+ self.assertEqual(small_src_file_as_string,
212
+ dst_key.get_contents_as_string())
213
+ # Ensure tracker file deleted.
214
+ self.assertFalse(os.path.exists(tracker_file_name))
215
+
216
+ def test_multiple_in_process_failures_then_succeed(self):
217
+ """
218
+ Tests resumable upload that fails twice in one process, then completes
219
+ """
220
+ res_upload_handler = ResumableUploadHandler(num_retries=3)
221
+ small_src_file_as_string, small_src_file = self.make_small_file()
222
+ small_src_file.seek(0)
223
+ dst_key = self._MakeKey(set_contents=False)
224
+ dst_key.set_contents_from_file(
225
+ small_src_file, res_upload_handler=res_upload_handler)
226
+ # Ensure uploaded object has correct content.
227
+ self.assertEqual(SMALL_KEY_SIZE, dst_key.size)
228
+ self.assertEqual(small_src_file_as_string,
229
+ dst_key.get_contents_as_string())
230
+
231
+ def test_multiple_in_process_failures_then_succeed_with_tracker_file(self):
232
+ """
233
+ Tests resumable upload that fails completely in one process,
234
+ then when restarted completes, using a tracker file
235
+ """
236
+ # Set up test harness that causes more failures than a single
237
+ # ResumableUploadHandler instance will handle, writing enough data
238
+ # before the first failure that some of it survives that process run.
239
+ harness = CallbackTestHarness(
240
+ fail_after_n_bytes=LARGE_KEY_SIZE/2, num_times_to_fail=2)
241
+ tracker_file_name = self.make_tracker_file()
242
+ res_upload_handler = ResumableUploadHandler(
243
+ tracker_file_name=tracker_file_name, num_retries=1)
244
+ larger_src_file_as_string, larger_src_file = self.make_large_file()
245
+ larger_src_file.seek(0)
246
+ dst_key = self._MakeKey(set_contents=False)
247
+ try:
248
+ dst_key.set_contents_from_file(
249
+ larger_src_file, cb=harness.call,
250
+ res_upload_handler=res_upload_handler)
251
+ self.fail('Did not get expected ResumableUploadException')
252
+ except ResumableUploadException, e:
253
+ self.assertEqual(e.disposition,
254
+ ResumableTransferDisposition.ABORT_CUR_PROCESS)
255
+ # Ensure a tracker file survived.
256
+ self.assertTrue(os.path.exists(tracker_file_name))
257
+ # Try it one more time; this time should succeed.
258
+ larger_src_file.seek(0)
259
+ dst_key.set_contents_from_file(
260
+ larger_src_file, cb=harness.call,
261
+ res_upload_handler=res_upload_handler)
262
+ self.assertEqual(LARGE_KEY_SIZE, dst_key.size)
263
+ self.assertEqual(larger_src_file_as_string,
264
+ dst_key.get_contents_as_string())
265
+ self.assertFalse(os.path.exists(tracker_file_name))
266
+ # Ensure some of the file was uploaded both before and after failure.
267
+ self.assertTrue(len(harness.transferred_seq_before_first_failure) > 1
268
+ and
269
+ len(harness.transferred_seq_after_first_failure) > 1)
270
+
271
+ def test_upload_with_inital_partial_upload_before_failure(self):
272
+ """
273
+ Tests resumable upload that successfully uploads some content
274
+ before it fails, then restarts and completes
275
+ """
276
+ # Set up harness to fail upload after several hundred KB so upload
277
+ # server will have saved something before we retry.
278
+ harness = CallbackTestHarness(
279
+ fail_after_n_bytes=LARGE_KEY_SIZE/2)
280
+ res_upload_handler = ResumableUploadHandler(num_retries=1)
281
+ larger_src_file_as_string, larger_src_file = self.make_large_file()
282
+ larger_src_file.seek(0)
283
+ dst_key = self._MakeKey(set_contents=False)
284
+ dst_key.set_contents_from_file(
285
+ larger_src_file, cb=harness.call,
286
+ res_upload_handler=res_upload_handler)
287
+ # Ensure uploaded object has correct content.
288
+ self.assertEqual(LARGE_KEY_SIZE, dst_key.size)
289
+ self.assertEqual(larger_src_file_as_string,
290
+ dst_key.get_contents_as_string())
291
+ # Ensure some of the file was uploaded both before and after failure.
292
+ self.assertTrue(len(harness.transferred_seq_before_first_failure) > 1
293
+ and
294
+ len(harness.transferred_seq_after_first_failure) > 1)
295
+
296
+ def test_empty_file_upload(self):
297
+ """
298
+ Tests uploading an empty file (exercises boundary conditions).
299
+ """
300
+ res_upload_handler = ResumableUploadHandler()
301
+ empty_src_file = StringIO.StringIO('')
302
+ empty_src_file.seek(0)
303
+ dst_key = self._MakeKey(set_contents=False)
304
+ dst_key.set_contents_from_file(
305
+ empty_src_file, res_upload_handler=res_upload_handler)
306
+ self.assertEqual(0, dst_key.size)
307
+
308
+ def test_upload_retains_metadata(self):
309
+ """
310
+ Tests that resumable upload correctly sets passed metadata
311
+ """
312
+ res_upload_handler = ResumableUploadHandler()
313
+ headers = {'Content-Type' : 'text/plain', 'Content-Encoding' : 'gzip',
314
+ 'x-goog-meta-abc' : 'my meta', 'x-goog-acl' : 'public-read'}
315
+ small_src_file_as_string, small_src_file = self.make_small_file()
316
+ small_src_file.seek(0)
317
+ dst_key = self._MakeKey(set_contents=False)
318
+ dst_key.set_contents_from_file(
319
+ small_src_file, headers=headers,
320
+ res_upload_handler=res_upload_handler)
321
+ self.assertEqual(SMALL_KEY_SIZE, dst_key.size)
322
+ self.assertEqual(small_src_file_as_string,
323
+ dst_key.get_contents_as_string())
324
+ dst_key.open_read()
325
+ self.assertEqual('text/plain', dst_key.content_type)
326
+ self.assertEqual('gzip', dst_key.content_encoding)
327
+ self.assertTrue('abc' in dst_key.metadata)
328
+ self.assertEqual('my meta', str(dst_key.metadata['abc']))
329
+ acl = dst_key.get_acl()
330
+ for entry in acl.entries.entry_list:
331
+ if str(entry.scope) == '<AllUsers>':
332
+ self.assertEqual('READ', str(acl.entries.entry_list[1].permission))
333
+ return
334
+ self.fail('No <AllUsers> scope found')
335
+
336
+ def test_upload_with_file_size_change_between_starts(self):
337
+ """
338
+ Tests resumable upload on a file that changes sizes between initial
339
+ upload start and restart
340
+ """
341
+ harness = CallbackTestHarness(
342
+ fail_after_n_bytes=LARGE_KEY_SIZE/2)
343
+ tracker_file_name = self.make_tracker_file()
344
+ # Set up first process' ResumableUploadHandler not to do any
345
+ # retries (initial upload request will establish expected size to
346
+ # upload server).
347
+ res_upload_handler = ResumableUploadHandler(
348
+ tracker_file_name=tracker_file_name, num_retries=0)
349
+ larger_src_file_as_string, larger_src_file = self.make_large_file()
350
+ larger_src_file.seek(0)
351
+ dst_key = self._MakeKey(set_contents=False)
352
+ try:
353
+ dst_key.set_contents_from_file(
354
+ larger_src_file, cb=harness.call,
355
+ res_upload_handler=res_upload_handler)
356
+ self.fail('Did not get expected ResumableUploadException')
357
+ except ResumableUploadException, e:
358
+ # First abort (from harness-forced failure) should be
359
+ # ABORT_CUR_PROCESS.
360
+ self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT_CUR_PROCESS)
361
+ # Ensure a tracker file survived.
362
+ self.assertTrue(os.path.exists(tracker_file_name))
363
+ # Try it again, this time with different size source file.
364
+ # Wait 1 second between retry attempts, to give upload server a
365
+ # chance to save state so it can respond to changed file size with
366
+ # 500 response in the next attempt.
367
+ time.sleep(1)
368
+ try:
369
+ largest_src_file = self.build_input_file(LARGEST_KEY_SIZE)[1]
370
+ largest_src_file.seek(0)
371
+ dst_key.set_contents_from_file(
372
+ largest_src_file, res_upload_handler=res_upload_handler)
373
+ self.fail('Did not get expected ResumableUploadException')
374
+ except ResumableUploadException, e:
375
+ # This abort should be a hard abort (file size changing during
376
+ # transfer).
377
+ self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT)
378
+ self.assertNotEqual(e.message.find('file size changed'), -1, e.message)
379
+
380
+ def test_upload_with_file_size_change_during_upload(self):
381
+ """
382
+ Tests resumable upload on a file that changes sizes while upload
383
+ in progress
384
+ """
385
+ # Create a file we can change during the upload.
386
+ test_file_size = 500 * 1024 # 500 KB.
387
+ test_file = self.build_input_file(test_file_size)[1]
388
+ harness = CallbackTestHarness(fp_to_change=test_file,
389
+ fp_change_pos=test_file_size)
390
+ res_upload_handler = ResumableUploadHandler(num_retries=1)
391
+ dst_key = self._MakeKey(set_contents=False)
392
+ try:
393
+ dst_key.set_contents_from_file(
394
+ test_file, cb=harness.call,
395
+ res_upload_handler=res_upload_handler)
396
+ self.fail('Did not get expected ResumableUploadException')
397
+ except ResumableUploadException, e:
398
+ self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT)
399
+ self.assertNotEqual(
400
+ e.message.find('File changed during upload'), -1)
401
+
402
+ def test_upload_with_file_content_change_during_upload(self):
403
+ """
404
+ Tests resumable upload on a file that changes one byte of content
405
+ (so, size stays the same) while upload in progress
406
+ """
407
+ test_file_size = 500 * 1024 # 500 KB.
408
+ test_file = self.build_input_file(test_file_size)[1]
409
+ harness = CallbackTestHarness(fail_after_n_bytes=test_file_size/2,
410
+ fp_to_change=test_file,
411
+ # Write to byte 1, as the CallbackTestHarness writes
412
+ # 3 bytes. This will result in the data on the server
413
+ # being different than the local file.
414
+ fp_change_pos=1)
415
+ res_upload_handler = ResumableUploadHandler(num_retries=1)
416
+ dst_key = self._MakeKey(set_contents=False)
417
+ bucket_uri = storage_uri('gs://' + dst_key.bucket.name)
418
+ dst_key_uri = bucket_uri.clone_replace_name(dst_key.name)
419
+ try:
420
+ dst_key.set_contents_from_file(
421
+ test_file, cb=harness.call,
422
+ res_upload_handler=res_upload_handler)
423
+ self.fail('Did not get expected ResumableUploadException')
424
+ except ResumableUploadException, e:
425
+ self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT)
426
+ # Ensure the file size didn't change.
427
+ test_file.seek(0, os.SEEK_END)
428
+ self.assertEqual(test_file_size, test_file.tell())
429
+ self.assertNotEqual(
430
+ e.message.find('md5 signature doesn\'t match etag'), -1)
431
+ # Ensure the bad data wasn't left around.
432
+ try:
433
+ dst_key_uri.get_key()
434
+ self.fail('Did not get expected InvalidUriError')
435
+ except InvalidUriError, e:
436
+ pass
437
+
438
+ def test_upload_with_content_length_header_set(self):
439
+ """
440
+ Tests resumable upload on a file when the user supplies a
441
+ Content-Length header. This is used by gsutil, for example,
442
+ to set the content length when gzipping a file.
443
+ """
444
+ res_upload_handler = ResumableUploadHandler()
445
+ small_src_file_as_string, small_src_file = self.make_small_file()
446
+ small_src_file.seek(0)
447
+ dst_key = self._MakeKey(set_contents=False)
448
+ try:
449
+ dst_key.set_contents_from_file(
450
+ small_src_file, res_upload_handler=res_upload_handler,
451
+ headers={'Content-Length' : SMALL_KEY_SIZE})
452
+ self.fail('Did not get expected ResumableUploadException')
453
+ except ResumableUploadException, e:
454
+ self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT)
455
+ self.assertNotEqual(
456
+ e.message.find('Attempt to specify Content-Length header'), -1)
457
+
458
+ def test_upload_with_syntactically_invalid_tracker_uri(self):
459
+ """
460
+ Tests resumable upload with a syntactically invalid tracker URI
461
+ """
462
+ tmp_dir = self._MakeTempDir()
463
+ syntactically_invalid_tracker_file_name = os.path.join(tmp_dir,
464
+ 'synt_invalid_uri_tracker')
465
+ with open(syntactically_invalid_tracker_file_name, 'w') as f:
466
+ f.write('ftp://example.com')
467
+ res_upload_handler = ResumableUploadHandler(
468
+ tracker_file_name=syntactically_invalid_tracker_file_name)
469
+ small_src_file_as_string, small_src_file = self.make_small_file()
470
+ # An error should be printed about the invalid URI, but then it
471
+ # should run the update successfully.
472
+ small_src_file.seek(0)
473
+ dst_key = self._MakeKey(set_contents=False)
474
+ dst_key.set_contents_from_file(
475
+ small_src_file, res_upload_handler=res_upload_handler)
476
+ self.assertEqual(SMALL_KEY_SIZE, dst_key.size)
477
+ self.assertEqual(small_src_file_as_string,
478
+ dst_key.get_contents_as_string())
479
+
480
+ def test_upload_with_invalid_upload_id_in_tracker_file(self):
481
+ """
482
+ Tests resumable upload with invalid upload ID
483
+ """
484
+ invalid_upload_id = ('http://pub.storage.googleapis.com/?upload_id='
485
+ 'AyzB2Uo74W4EYxyi5dp_-r68jz8rtbvshsv4TX7srJVkJ57CxTY5Dw2')
486
+ tmpdir = self._MakeTempDir()
487
+ invalid_upload_id_tracker_file_name = os.path.join(tmpdir,
488
+ 'invalid_upload_id_tracker')
489
+ with open(invalid_upload_id_tracker_file_name, 'w') as f:
490
+ f.write(invalid_upload_id)
491
+
492
+ res_upload_handler = ResumableUploadHandler(
493
+ tracker_file_name=invalid_upload_id_tracker_file_name)
494
+ small_src_file_as_string, small_src_file = self.make_small_file()
495
+ # An error should occur, but then the tracker URI should be
496
+ # regenerated and the the update should succeed.
497
+ small_src_file.seek(0)
498
+ dst_key = self._MakeKey(set_contents=False)
499
+ dst_key.set_contents_from_file(
500
+ small_src_file, res_upload_handler=res_upload_handler)
501
+ self.assertEqual(SMALL_KEY_SIZE, dst_key.size)
502
+ self.assertEqual(small_src_file_as_string,
503
+ dst_key.get_contents_as_string())
504
+ self.assertNotEqual(invalid_upload_id,
505
+ res_upload_handler.get_tracker_uri())
506
+
507
+ def test_upload_with_unwritable_tracker_file(self):
508
+ """
509
+ Tests resumable upload with an unwritable tracker file
510
+ """
511
+ # Make dir where tracker_file lives temporarily unwritable.
512
+ tmp_dir = self._MakeTempDir()
513
+ tracker_file_name = self.make_tracker_file(tmp_dir)
514
+ save_mod = os.stat(tmp_dir).st_mode
515
+ try:
516
+ os.chmod(tmp_dir, 0)
517
+ res_upload_handler = ResumableUploadHandler(
518
+ tracker_file_name=tracker_file_name)
519
+ except ResumableUploadException, e:
520
+ self.assertEqual(e.disposition, ResumableTransferDisposition.ABORT)
521
+ self.assertNotEqual(
522
+ e.message.find('Couldn\'t write URI tracker file'), -1)
523
+ finally:
524
+ # Restore original protection of dir where tracker_file lives.
525
+ os.chmod(tmp_dir, save_mod)