googlecloud 0.0.4 → 0.0.6

Sign up to get free protection for your applications and to get access to all the features.
Files changed (768) hide show
  1. data.tar.gz.sig +0 -0
  2. data/CHANGELOG +4 -0
  3. data/Makefile +4 -0
  4. data/Manifest +760 -1
  5. data/bin/gcloud +7 -0
  6. data/bin/{gcutil → gcutil.symlink} +0 -0
  7. data/bin/gsutil-symlink +377 -0
  8. data/googlecloud.gemspec +5 -5
  9. data/packages/gsutil/CHECKSUM +1 -0
  10. data/packages/gsutil/COPYING +202 -0
  11. data/packages/gsutil/LICENSE.third_party +295 -0
  12. data/packages/gsutil/MANIFEST.in +5 -0
  13. data/packages/gsutil/README +38 -0
  14. data/packages/gsutil/README.pkg +49 -0
  15. data/packages/gsutil/ReleaseNotes.txt +780 -0
  16. data/packages/gsutil/VERSION +1 -0
  17. data/packages/gsutil/boto/Changelog.rst +35 -0
  18. data/packages/gsutil/boto/LICENSE +18 -0
  19. data/packages/gsutil/boto/MANIFEST.in +12 -0
  20. data/packages/gsutil/boto/README.rst +163 -0
  21. data/packages/gsutil/boto/bin/asadmin +290 -0
  22. data/packages/gsutil/boto/bin/bundle_image +27 -0
  23. data/packages/gsutil/boto/bin/cfadmin +108 -0
  24. data/packages/gsutil/boto/bin/cq +89 -0
  25. data/packages/gsutil/boto/bin/cwutil +140 -0
  26. data/packages/gsutil/boto/bin/elbadmin +284 -0
  27. data/packages/gsutil/boto/bin/fetch_file +43 -0
  28. data/packages/gsutil/boto/bin/glacier +154 -0
  29. data/packages/gsutil/boto/bin/instance_events +145 -0
  30. data/packages/gsutil/boto/bin/kill_instance +35 -0
  31. data/packages/gsutil/boto/bin/launch_instance +252 -0
  32. data/packages/gsutil/boto/bin/list_instances +90 -0
  33. data/packages/gsutil/boto/bin/lss3 +77 -0
  34. data/packages/gsutil/boto/bin/mturk +465 -0
  35. data/packages/gsutil/boto/bin/pyami_sendmail +52 -0
  36. data/packages/gsutil/boto/bin/route53 +205 -0
  37. data/packages/gsutil/boto/bin/s3put +374 -0
  38. data/packages/gsutil/boto/bin/sdbadmin +194 -0
  39. data/packages/gsutil/boto/bin/taskadmin +116 -0
  40. data/packages/gsutil/boto/boto/__init__.py +793 -0
  41. data/packages/gsutil/boto/boto/__init__.pyc +0 -0
  42. data/packages/gsutil/boto/boto/auth.py +682 -0
  43. data/packages/gsutil/boto/boto/auth.pyc +0 -0
  44. data/packages/gsutil/boto/boto/auth_handler.py +58 -0
  45. data/packages/gsutil/boto/boto/auth_handler.pyc +0 -0
  46. data/packages/gsutil/boto/boto/beanstalk/__init__.py +65 -0
  47. data/packages/gsutil/boto/boto/beanstalk/exception.py +64 -0
  48. data/packages/gsutil/boto/boto/beanstalk/layer1.py +1153 -0
  49. data/packages/gsutil/boto/boto/beanstalk/response.py +703 -0
  50. data/packages/gsutil/boto/boto/beanstalk/wrapper.py +29 -0
  51. data/packages/gsutil/boto/boto/cacerts/__init__.py +22 -0
  52. data/packages/gsutil/boto/boto/cacerts/__init__.pyc +0 -0
  53. data/packages/gsutil/boto/boto/cacerts/cacerts.txt +633 -0
  54. data/packages/gsutil/boto/boto/cloudformation/__init__.py +68 -0
  55. data/packages/gsutil/boto/boto/cloudformation/connection.py +364 -0
  56. data/packages/gsutil/boto/boto/cloudformation/stack.py +360 -0
  57. data/packages/gsutil/boto/boto/cloudformation/template.py +43 -0
  58. data/packages/gsutil/boto/boto/cloudfront/__init__.py +324 -0
  59. data/packages/gsutil/boto/boto/cloudfront/distribution.py +745 -0
  60. data/packages/gsutil/boto/boto/cloudfront/exception.py +26 -0
  61. data/packages/gsutil/boto/boto/cloudfront/identity.py +122 -0
  62. data/packages/gsutil/boto/boto/cloudfront/invalidation.py +216 -0
  63. data/packages/gsutil/boto/boto/cloudfront/logging.py +38 -0
  64. data/packages/gsutil/boto/boto/cloudfront/object.py +48 -0
  65. data/packages/gsutil/boto/boto/cloudfront/origin.py +150 -0
  66. data/packages/gsutil/boto/boto/cloudfront/signers.py +60 -0
  67. data/packages/gsutil/boto/boto/cloudsearch/__init__.py +45 -0
  68. data/packages/gsutil/boto/boto/cloudsearch/document.py +265 -0
  69. data/packages/gsutil/boto/boto/cloudsearch/domain.py +394 -0
  70. data/packages/gsutil/boto/boto/cloudsearch/layer1.py +738 -0
  71. data/packages/gsutil/boto/boto/cloudsearch/layer2.py +67 -0
  72. data/packages/gsutil/boto/boto/cloudsearch/optionstatus.py +248 -0
  73. data/packages/gsutil/boto/boto/cloudsearch/search.py +365 -0
  74. data/packages/gsutil/boto/boto/cloudsearch/sourceattribute.py +75 -0
  75. data/packages/gsutil/boto/boto/compat.py +28 -0
  76. data/packages/gsutil/boto/boto/compat.pyc +0 -0
  77. data/packages/gsutil/boto/boto/connection.py +1081 -0
  78. data/packages/gsutil/boto/boto/connection.pyc +0 -0
  79. data/packages/gsutil/boto/boto/contrib/__init__.py +22 -0
  80. data/packages/gsutil/boto/boto/contrib/ymlmessage.py +52 -0
  81. data/packages/gsutil/boto/boto/core/README +58 -0
  82. data/packages/gsutil/boto/boto/core/__init__.py +23 -0
  83. data/packages/gsutil/boto/boto/core/auth.py +78 -0
  84. data/packages/gsutil/boto/boto/core/credentials.py +154 -0
  85. data/packages/gsutil/boto/boto/core/dictresponse.py +178 -0
  86. data/packages/gsutil/boto/boto/core/service.py +67 -0
  87. data/packages/gsutil/boto/boto/datapipeline/__init__.py +0 -0
  88. data/packages/gsutil/boto/boto/datapipeline/exceptions.py +42 -0
  89. data/packages/gsutil/boto/boto/datapipeline/layer1.py +546 -0
  90. data/packages/gsutil/boto/boto/dynamodb/__init__.py +66 -0
  91. data/packages/gsutil/boto/boto/dynamodb/batch.py +262 -0
  92. data/packages/gsutil/boto/boto/dynamodb/condition.py +170 -0
  93. data/packages/gsutil/boto/boto/dynamodb/exceptions.py +64 -0
  94. data/packages/gsutil/boto/boto/dynamodb/item.py +196 -0
  95. data/packages/gsutil/boto/boto/dynamodb/layer1.py +575 -0
  96. data/packages/gsutil/boto/boto/dynamodb/layer2.py +798 -0
  97. data/packages/gsutil/boto/boto/dynamodb/schema.py +112 -0
  98. data/packages/gsutil/boto/boto/dynamodb/table.py +540 -0
  99. data/packages/gsutil/boto/boto/dynamodb/types.py +326 -0
  100. data/packages/gsutil/boto/boto/ec2/__init__.py +96 -0
  101. data/packages/gsutil/boto/boto/ec2/address.py +103 -0
  102. data/packages/gsutil/boto/boto/ec2/autoscale/__init__.py +781 -0
  103. data/packages/gsutil/boto/boto/ec2/autoscale/activity.py +74 -0
  104. data/packages/gsutil/boto/boto/ec2/autoscale/group.py +337 -0
  105. data/packages/gsutil/boto/boto/ec2/autoscale/instance.py +60 -0
  106. data/packages/gsutil/boto/boto/ec2/autoscale/launchconfig.py +209 -0
  107. data/packages/gsutil/boto/boto/ec2/autoscale/policy.py +166 -0
  108. data/packages/gsutil/boto/boto/ec2/autoscale/request.py +38 -0
  109. data/packages/gsutil/boto/boto/ec2/autoscale/scheduled.py +60 -0
  110. data/packages/gsutil/boto/boto/ec2/autoscale/tag.py +84 -0
  111. data/packages/gsutil/boto/boto/ec2/blockdevicemapping.py +141 -0
  112. data/packages/gsutil/boto/boto/ec2/bundleinstance.py +78 -0
  113. data/packages/gsutil/boto/boto/ec2/buyreservation.py +84 -0
  114. data/packages/gsutil/boto/boto/ec2/cloudwatch/__init__.py +603 -0
  115. data/packages/gsutil/boto/boto/ec2/cloudwatch/alarm.py +316 -0
  116. data/packages/gsutil/boto/boto/ec2/cloudwatch/datapoint.py +40 -0
  117. data/packages/gsutil/boto/boto/ec2/cloudwatch/dimension.py +38 -0
  118. data/packages/gsutil/boto/boto/ec2/cloudwatch/listelement.py +31 -0
  119. data/packages/gsutil/boto/boto/ec2/cloudwatch/metric.py +175 -0
  120. data/packages/gsutil/boto/boto/ec2/connection.py +3409 -0
  121. data/packages/gsutil/boto/boto/ec2/ec2object.py +107 -0
  122. data/packages/gsutil/boto/boto/ec2/elb/__init__.py +553 -0
  123. data/packages/gsutil/boto/boto/ec2/elb/healthcheck.py +89 -0
  124. data/packages/gsutil/boto/boto/ec2/elb/instancestate.py +62 -0
  125. data/packages/gsutil/boto/boto/ec2/elb/listelement.py +36 -0
  126. data/packages/gsutil/boto/boto/ec2/elb/listener.py +75 -0
  127. data/packages/gsutil/boto/boto/ec2/elb/loadbalancer.py +324 -0
  128. data/packages/gsutil/boto/boto/ec2/elb/policies.py +89 -0
  129. data/packages/gsutil/boto/boto/ec2/elb/securitygroup.py +38 -0
  130. data/packages/gsutil/boto/boto/ec2/group.py +39 -0
  131. data/packages/gsutil/boto/boto/ec2/image.py +350 -0
  132. data/packages/gsutil/boto/boto/ec2/instance.py +661 -0
  133. data/packages/gsutil/boto/boto/ec2/instanceinfo.py +51 -0
  134. data/packages/gsutil/boto/boto/ec2/instancestatus.py +207 -0
  135. data/packages/gsutil/boto/boto/ec2/keypair.py +113 -0
  136. data/packages/gsutil/boto/boto/ec2/launchspecification.py +105 -0
  137. data/packages/gsutil/boto/boto/ec2/networkinterface.py +247 -0
  138. data/packages/gsutil/boto/boto/ec2/placementgroup.py +51 -0
  139. data/packages/gsutil/boto/boto/ec2/regioninfo.py +34 -0
  140. data/packages/gsutil/boto/boto/ec2/reservedinstance.py +227 -0
  141. data/packages/gsutil/boto/boto/ec2/securitygroup.py +357 -0
  142. data/packages/gsutil/boto/boto/ec2/snapshot.py +170 -0
  143. data/packages/gsutil/boto/boto/ec2/spotdatafeedsubscription.py +63 -0
  144. data/packages/gsutil/boto/boto/ec2/spotinstancerequest.py +188 -0
  145. data/packages/gsutil/boto/boto/ec2/spotpricehistory.py +55 -0
  146. data/packages/gsutil/boto/boto/ec2/tag.py +87 -0
  147. data/packages/gsutil/boto/boto/ec2/vmtype.py +58 -0
  148. data/packages/gsutil/boto/boto/ec2/volume.py +293 -0
  149. data/packages/gsutil/boto/boto/ec2/volumestatus.py +200 -0
  150. data/packages/gsutil/boto/boto/ec2/zone.py +80 -0
  151. data/packages/gsutil/boto/boto/ecs/__init__.py +90 -0
  152. data/packages/gsutil/boto/boto/ecs/item.py +153 -0
  153. data/packages/gsutil/boto/boto/elasticache/__init__.py +62 -0
  154. data/packages/gsutil/boto/boto/elasticache/layer1.py +1252 -0
  155. data/packages/gsutil/boto/boto/elastictranscoder/__init__.py +62 -0
  156. data/packages/gsutil/boto/boto/elastictranscoder/exceptions.py +46 -0
  157. data/packages/gsutil/boto/boto/elastictranscoder/layer1.py +509 -0
  158. data/packages/gsutil/boto/boto/emr/__init__.py +73 -0
  159. data/packages/gsutil/boto/boto/emr/bootstrap_action.py +44 -0
  160. data/packages/gsutil/boto/boto/emr/connection.py +531 -0
  161. data/packages/gsutil/boto/boto/emr/emrobject.py +176 -0
  162. data/packages/gsutil/boto/boto/emr/instance_group.py +43 -0
  163. data/packages/gsutil/boto/boto/emr/step.py +281 -0
  164. data/packages/gsutil/boto/boto/exception.py +476 -0
  165. data/packages/gsutil/boto/boto/exception.pyc +0 -0
  166. data/packages/gsutil/boto/boto/file/README +49 -0
  167. data/packages/gsutil/boto/boto/file/__init__.py +28 -0
  168. data/packages/gsutil/boto/boto/file/bucket.py +112 -0
  169. data/packages/gsutil/boto/boto/file/connection.py +33 -0
  170. data/packages/gsutil/boto/boto/file/key.py +199 -0
  171. data/packages/gsutil/boto/boto/file/simpleresultset.py +30 -0
  172. data/packages/gsutil/boto/boto/fps/__init__.py +21 -0
  173. data/packages/gsutil/boto/boto/fps/connection.py +369 -0
  174. data/packages/gsutil/boto/boto/fps/exception.py +344 -0
  175. data/packages/gsutil/boto/boto/fps/response.py +175 -0
  176. data/packages/gsutil/boto/boto/glacier/__init__.py +57 -0
  177. data/packages/gsutil/boto/boto/glacier/concurrent.py +409 -0
  178. data/packages/gsutil/boto/boto/glacier/exceptions.py +58 -0
  179. data/packages/gsutil/boto/boto/glacier/job.py +152 -0
  180. data/packages/gsutil/boto/boto/glacier/layer1.py +637 -0
  181. data/packages/gsutil/boto/boto/glacier/layer2.py +93 -0
  182. data/packages/gsutil/boto/boto/glacier/response.py +48 -0
  183. data/packages/gsutil/boto/boto/glacier/utils.py +163 -0
  184. data/packages/gsutil/boto/boto/glacier/vault.py +387 -0
  185. data/packages/gsutil/boto/boto/glacier/writer.py +242 -0
  186. data/packages/gsutil/boto/boto/gs/__init__.py +22 -0
  187. data/packages/gsutil/boto/boto/gs/__init__.pyc +0 -0
  188. data/packages/gsutil/boto/boto/gs/acl.py +304 -0
  189. data/packages/gsutil/boto/boto/gs/acl.pyc +0 -0
  190. data/packages/gsutil/boto/boto/gs/bucket.py +870 -0
  191. data/packages/gsutil/boto/boto/gs/bucket.pyc +0 -0
  192. data/packages/gsutil/boto/boto/gs/bucketlistresultset.py +64 -0
  193. data/packages/gsutil/boto/boto/gs/bucketlistresultset.pyc +0 -0
  194. data/packages/gsutil/boto/boto/gs/connection.py +103 -0
  195. data/packages/gsutil/boto/boto/gs/connection.pyc +0 -0
  196. data/packages/gsutil/boto/boto/gs/cors.py +169 -0
  197. data/packages/gsutil/boto/boto/gs/cors.pyc +0 -0
  198. data/packages/gsutil/boto/boto/gs/key.py +704 -0
  199. data/packages/gsutil/boto/boto/gs/key.pyc +0 -0
  200. data/packages/gsutil/boto/boto/gs/resumable_upload_handler.py +659 -0
  201. data/packages/gsutil/boto/boto/gs/resumable_upload_handler.pyc +0 -0
  202. data/packages/gsutil/boto/boto/gs/user.py +54 -0
  203. data/packages/gsutil/boto/boto/gs/user.pyc +0 -0
  204. data/packages/gsutil/boto/boto/handler.py +44 -0
  205. data/packages/gsutil/boto/boto/handler.pyc +0 -0
  206. data/packages/gsutil/boto/boto/https_connection.py +124 -0
  207. data/packages/gsutil/boto/boto/https_connection.pyc +0 -0
  208. data/packages/gsutil/boto/boto/iam/__init__.py +74 -0
  209. data/packages/gsutil/boto/boto/iam/connection.py +1317 -0
  210. data/packages/gsutil/boto/boto/iam/summarymap.py +42 -0
  211. data/packages/gsutil/boto/boto/jsonresponse.py +163 -0
  212. data/packages/gsutil/boto/boto/jsonresponse.pyc +0 -0
  213. data/packages/gsutil/boto/boto/manage/__init__.py +23 -0
  214. data/packages/gsutil/boto/boto/manage/cmdshell.py +241 -0
  215. data/packages/gsutil/boto/boto/manage/propget.py +64 -0
  216. data/packages/gsutil/boto/boto/manage/server.py +556 -0
  217. data/packages/gsutil/boto/boto/manage/task.py +175 -0
  218. data/packages/gsutil/boto/boto/manage/test_manage.py +34 -0
  219. data/packages/gsutil/boto/boto/manage/volume.py +420 -0
  220. data/packages/gsutil/boto/boto/mashups/__init__.py +23 -0
  221. data/packages/gsutil/boto/boto/mashups/interactive.py +97 -0
  222. data/packages/gsutil/boto/boto/mashups/iobject.py +115 -0
  223. data/packages/gsutil/boto/boto/mashups/order.py +211 -0
  224. data/packages/gsutil/boto/boto/mashups/server.py +395 -0
  225. data/packages/gsutil/boto/boto/mturk/__init__.py +23 -0
  226. data/packages/gsutil/boto/boto/mturk/connection.py +1027 -0
  227. data/packages/gsutil/boto/boto/mturk/layoutparam.py +55 -0
  228. data/packages/gsutil/boto/boto/mturk/notification.py +103 -0
  229. data/packages/gsutil/boto/boto/mturk/price.py +48 -0
  230. data/packages/gsutil/boto/boto/mturk/qualification.py +137 -0
  231. data/packages/gsutil/boto/boto/mturk/question.py +455 -0
  232. data/packages/gsutil/boto/boto/mws/__init__.py +21 -0
  233. data/packages/gsutil/boto/boto/mws/connection.py +813 -0
  234. data/packages/gsutil/boto/boto/mws/exception.py +75 -0
  235. data/packages/gsutil/boto/boto/mws/response.py +655 -0
  236. data/packages/gsutil/boto/boto/plugin.py +90 -0
  237. data/packages/gsutil/boto/boto/plugin.pyc +0 -0
  238. data/packages/gsutil/boto/boto/provider.py +337 -0
  239. data/packages/gsutil/boto/boto/provider.pyc +0 -0
  240. data/packages/gsutil/boto/boto/pyami/__init__.py +22 -0
  241. data/packages/gsutil/boto/boto/pyami/__init__.pyc +0 -0
  242. data/packages/gsutil/boto/boto/pyami/bootstrap.py +134 -0
  243. data/packages/gsutil/boto/boto/pyami/config.py +229 -0
  244. data/packages/gsutil/boto/boto/pyami/config.pyc +0 -0
  245. data/packages/gsutil/boto/boto/pyami/copybot.cfg +60 -0
  246. data/packages/gsutil/boto/boto/pyami/copybot.py +97 -0
  247. data/packages/gsutil/boto/boto/pyami/helloworld.py +28 -0
  248. data/packages/gsutil/boto/boto/pyami/installers/__init__.py +64 -0
  249. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/__init__.py +22 -0
  250. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/apache.py +43 -0
  251. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/ebs.py +238 -0
  252. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/installer.py +96 -0
  253. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/mysql.py +109 -0
  254. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/trac.py +139 -0
  255. data/packages/gsutil/boto/boto/pyami/launch_ami.py +178 -0
  256. data/packages/gsutil/boto/boto/pyami/scriptbase.py +44 -0
  257. data/packages/gsutil/boto/boto/pyami/startup.py +60 -0
  258. data/packages/gsutil/boto/boto/rds/__init__.py +1194 -0
  259. data/packages/gsutil/boto/boto/rds/dbinstance.py +357 -0
  260. data/packages/gsutil/boto/boto/rds/dbsecuritygroup.py +177 -0
  261. data/packages/gsutil/boto/boto/rds/dbsnapshot.py +108 -0
  262. data/packages/gsutil/boto/boto/rds/event.py +49 -0
  263. data/packages/gsutil/boto/boto/rds/parametergroup.py +201 -0
  264. data/packages/gsutil/boto/boto/rds/regioninfo.py +32 -0
  265. data/packages/gsutil/boto/boto/regioninfo.py +63 -0
  266. data/packages/gsutil/boto/boto/regioninfo.pyc +0 -0
  267. data/packages/gsutil/boto/boto/resultset.py +169 -0
  268. data/packages/gsutil/boto/boto/resultset.pyc +0 -0
  269. data/packages/gsutil/boto/boto/roboto/__init__.py +1 -0
  270. data/packages/gsutil/boto/boto/roboto/awsqueryrequest.py +504 -0
  271. data/packages/gsutil/boto/boto/roboto/awsqueryservice.py +121 -0
  272. data/packages/gsutil/boto/boto/roboto/param.py +147 -0
  273. data/packages/gsutil/boto/boto/route53/__init__.py +75 -0
  274. data/packages/gsutil/boto/boto/route53/connection.py +403 -0
  275. data/packages/gsutil/boto/boto/route53/exception.py +27 -0
  276. data/packages/gsutil/boto/boto/route53/hostedzone.py +56 -0
  277. data/packages/gsutil/boto/boto/route53/record.py +306 -0
  278. data/packages/gsutil/boto/boto/route53/status.py +42 -0
  279. data/packages/gsutil/boto/boto/route53/zone.py +412 -0
  280. data/packages/gsutil/boto/boto/s3/__init__.py +84 -0
  281. data/packages/gsutil/boto/boto/s3/__init__.pyc +0 -0
  282. data/packages/gsutil/boto/boto/s3/acl.py +164 -0
  283. data/packages/gsutil/boto/boto/s3/acl.pyc +0 -0
  284. data/packages/gsutil/boto/boto/s3/bucket.py +1634 -0
  285. data/packages/gsutil/boto/boto/s3/bucket.pyc +0 -0
  286. data/packages/gsutil/boto/boto/s3/bucketlistresultset.py +139 -0
  287. data/packages/gsutil/boto/boto/s3/bucketlistresultset.pyc +0 -0
  288. data/packages/gsutil/boto/boto/s3/bucketlogging.py +83 -0
  289. data/packages/gsutil/boto/boto/s3/bucketlogging.pyc +0 -0
  290. data/packages/gsutil/boto/boto/s3/connection.py +540 -0
  291. data/packages/gsutil/boto/boto/s3/connection.pyc +0 -0
  292. data/packages/gsutil/boto/boto/s3/cors.py +210 -0
  293. data/packages/gsutil/boto/boto/s3/cors.pyc +0 -0
  294. data/packages/gsutil/boto/boto/s3/deletemarker.py +55 -0
  295. data/packages/gsutil/boto/boto/s3/deletemarker.pyc +0 -0
  296. data/packages/gsutil/boto/boto/s3/key.py +1712 -0
  297. data/packages/gsutil/boto/boto/s3/key.pyc +0 -0
  298. data/packages/gsutil/boto/boto/s3/keyfile.py +134 -0
  299. data/packages/gsutil/boto/boto/s3/keyfile.pyc +0 -0
  300. data/packages/gsutil/boto/boto/s3/lifecycle.py +231 -0
  301. data/packages/gsutil/boto/boto/s3/lifecycle.pyc +0 -0
  302. data/packages/gsutil/boto/boto/s3/multidelete.py +138 -0
  303. data/packages/gsutil/boto/boto/s3/multidelete.pyc +0 -0
  304. data/packages/gsutil/boto/boto/s3/multipart.py +315 -0
  305. data/packages/gsutil/boto/boto/s3/multipart.pyc +0 -0
  306. data/packages/gsutil/boto/boto/s3/prefix.py +42 -0
  307. data/packages/gsutil/boto/boto/s3/prefix.pyc +0 -0
  308. data/packages/gsutil/boto/boto/s3/resumable_download_handler.py +339 -0
  309. data/packages/gsutil/boto/boto/s3/resumable_download_handler.pyc +0 -0
  310. data/packages/gsutil/boto/boto/s3/tagging.py +71 -0
  311. data/packages/gsutil/boto/boto/s3/tagging.pyc +0 -0
  312. data/packages/gsutil/boto/boto/s3/user.py +49 -0
  313. data/packages/gsutil/boto/boto/s3/user.pyc +0 -0
  314. data/packages/gsutil/boto/boto/s3/website.py +237 -0
  315. data/packages/gsutil/boto/boto/s3/website.pyc +0 -0
  316. data/packages/gsutil/boto/boto/sdb/__init__.py +67 -0
  317. data/packages/gsutil/boto/boto/sdb/connection.py +617 -0
  318. data/packages/gsutil/boto/boto/sdb/db/__init__.py +20 -0
  319. data/packages/gsutil/boto/boto/sdb/db/blob.py +75 -0
  320. data/packages/gsutil/boto/boto/sdb/db/key.py +59 -0
  321. data/packages/gsutil/boto/boto/sdb/db/manager/__init__.py +85 -0
  322. data/packages/gsutil/boto/boto/sdb/db/manager/sdbmanager.py +732 -0
  323. data/packages/gsutil/boto/boto/sdb/db/manager/xmlmanager.py +517 -0
  324. data/packages/gsutil/boto/boto/sdb/db/model.py +294 -0
  325. data/packages/gsutil/boto/boto/sdb/db/property.py +703 -0
  326. data/packages/gsutil/boto/boto/sdb/db/query.py +85 -0
  327. data/packages/gsutil/boto/boto/sdb/db/sequence.py +226 -0
  328. data/packages/gsutil/boto/boto/sdb/db/test_db.py +231 -0
  329. data/packages/gsutil/boto/boto/sdb/domain.py +377 -0
  330. data/packages/gsutil/boto/boto/sdb/item.py +181 -0
  331. data/packages/gsutil/boto/boto/sdb/queryresultset.py +92 -0
  332. data/packages/gsutil/boto/boto/sdb/regioninfo.py +32 -0
  333. data/packages/gsutil/boto/boto/services/__init__.py +23 -0
  334. data/packages/gsutil/boto/boto/services/bs.py +179 -0
  335. data/packages/gsutil/boto/boto/services/message.py +58 -0
  336. data/packages/gsutil/boto/boto/services/result.py +136 -0
  337. data/packages/gsutil/boto/boto/services/service.py +161 -0
  338. data/packages/gsutil/boto/boto/services/servicedef.py +91 -0
  339. data/packages/gsutil/boto/boto/services/sonofmmm.cfg +43 -0
  340. data/packages/gsutil/boto/boto/services/sonofmmm.py +81 -0
  341. data/packages/gsutil/boto/boto/services/submit.py +88 -0
  342. data/packages/gsutil/boto/boto/ses/__init__.py +54 -0
  343. data/packages/gsutil/boto/boto/ses/connection.py +521 -0
  344. data/packages/gsutil/boto/boto/ses/exceptions.py +77 -0
  345. data/packages/gsutil/boto/boto/sns/__init__.py +78 -0
  346. data/packages/gsutil/boto/boto/sns/connection.py +431 -0
  347. data/packages/gsutil/boto/boto/sqs/__init__.py +56 -0
  348. data/packages/gsutil/boto/boto/sqs/attributes.py +46 -0
  349. data/packages/gsutil/boto/boto/sqs/batchresults.py +95 -0
  350. data/packages/gsutil/boto/boto/sqs/connection.py +417 -0
  351. data/packages/gsutil/boto/boto/sqs/jsonmessage.py +43 -0
  352. data/packages/gsutil/boto/boto/sqs/message.py +253 -0
  353. data/packages/gsutil/boto/boto/sqs/queue.py +478 -0
  354. data/packages/gsutil/boto/boto/sqs/regioninfo.py +32 -0
  355. data/packages/gsutil/boto/boto/storage_uri.py +835 -0
  356. data/packages/gsutil/boto/boto/storage_uri.pyc +0 -0
  357. data/packages/gsutil/boto/boto/sts/__init__.py +55 -0
  358. data/packages/gsutil/boto/boto/sts/connection.py +207 -0
  359. data/packages/gsutil/boto/boto/sts/credentials.py +215 -0
  360. data/packages/gsutil/boto/boto/swf/__init__.py +60 -0
  361. data/packages/gsutil/boto/boto/swf/exceptions.py +44 -0
  362. data/packages/gsutil/boto/boto/swf/layer1.py +1512 -0
  363. data/packages/gsutil/boto/boto/swf/layer1_decisions.py +287 -0
  364. data/packages/gsutil/boto/boto/swf/layer2.py +342 -0
  365. data/packages/gsutil/boto/boto/utils.py +927 -0
  366. data/packages/gsutil/boto/boto/utils.pyc +0 -0
  367. data/packages/gsutil/boto/boto/vpc/__init__.py +910 -0
  368. data/packages/gsutil/boto/boto/vpc/customergateway.py +54 -0
  369. data/packages/gsutil/boto/boto/vpc/dhcpoptions.py +72 -0
  370. data/packages/gsutil/boto/boto/vpc/internetgateway.py +72 -0
  371. data/packages/gsutil/boto/boto/vpc/routetable.py +109 -0
  372. data/packages/gsutil/boto/boto/vpc/subnet.py +57 -0
  373. data/packages/gsutil/boto/boto/vpc/vpc.py +54 -0
  374. data/packages/gsutil/boto/boto/vpc/vpnconnection.py +60 -0
  375. data/packages/gsutil/boto/boto/vpc/vpngateway.py +83 -0
  376. data/packages/gsutil/boto/docs/BotoCheatSheet.pdf +0 -0
  377. data/packages/gsutil/boto/docs/Makefile +89 -0
  378. data/packages/gsutil/boto/docs/make.bat +113 -0
  379. data/packages/gsutil/boto/docs/source/_templates/layout.html +3 -0
  380. data/packages/gsutil/boto/docs/source/autoscale_tut.rst +220 -0
  381. data/packages/gsutil/boto/docs/source/boto_config_tut.rst +125 -0
  382. data/packages/gsutil/boto/docs/source/boto_theme/static/boto.css_t +239 -0
  383. data/packages/gsutil/boto/docs/source/boto_theme/static/pygments.css +61 -0
  384. data/packages/gsutil/boto/docs/source/boto_theme/theme.conf +3 -0
  385. data/packages/gsutil/boto/docs/source/cloudfront_tut.rst +196 -0
  386. data/packages/gsutil/boto/docs/source/cloudsearch_tut.rst +411 -0
  387. data/packages/gsutil/boto/docs/source/cloudwatch_tut.rst +116 -0
  388. data/packages/gsutil/boto/docs/source/conf.py +32 -0
  389. data/packages/gsutil/boto/docs/source/contributing.rst +204 -0
  390. data/packages/gsutil/boto/docs/source/documentation.rst +59 -0
  391. data/packages/gsutil/boto/docs/source/dynamodb_tut.rst +339 -0
  392. data/packages/gsutil/boto/docs/source/ec2_tut.rst +86 -0
  393. data/packages/gsutil/boto/docs/source/elb_tut.rst +257 -0
  394. data/packages/gsutil/boto/docs/source/emr_tut.rst +108 -0
  395. data/packages/gsutil/boto/docs/source/index.rst +146 -0
  396. data/packages/gsutil/boto/docs/source/rds_tut.rst +108 -0
  397. data/packages/gsutil/boto/docs/source/ref/autoscale.rst +62 -0
  398. data/packages/gsutil/boto/docs/source/ref/beanstalk.rst +26 -0
  399. data/packages/gsutil/boto/docs/source/ref/boto.rst +47 -0
  400. data/packages/gsutil/boto/docs/source/ref/cloudformation.rst +34 -0
  401. data/packages/gsutil/boto/docs/source/ref/cloudfront.rst +68 -0
  402. data/packages/gsutil/boto/docs/source/ref/cloudsearch.rst +59 -0
  403. data/packages/gsutil/boto/docs/source/ref/cloudwatch.rst +27 -0
  404. data/packages/gsutil/boto/docs/source/ref/contrib.rst +32 -0
  405. data/packages/gsutil/boto/docs/source/ref/datapipeline.rst +26 -0
  406. data/packages/gsutil/boto/docs/source/ref/dynamodb.rst +61 -0
  407. data/packages/gsutil/boto/docs/source/ref/ec2.rst +140 -0
  408. data/packages/gsutil/boto/docs/source/ref/ecs.rst +19 -0
  409. data/packages/gsutil/boto/docs/source/ref/elasticache.rst +19 -0
  410. data/packages/gsutil/boto/docs/source/ref/elastictranscoder.rst +26 -0
  411. data/packages/gsutil/boto/docs/source/ref/elb.rst +47 -0
  412. data/packages/gsutil/boto/docs/source/ref/emr.rst +34 -0
  413. data/packages/gsutil/boto/docs/source/ref/file.rst +34 -0
  414. data/packages/gsutil/boto/docs/source/ref/fps.rst +19 -0
  415. data/packages/gsutil/boto/docs/source/ref/glacier.rst +63 -0
  416. data/packages/gsutil/boto/docs/source/ref/gs.rst +72 -0
  417. data/packages/gsutil/boto/docs/source/ref/iam.rst +27 -0
  418. data/packages/gsutil/boto/docs/source/ref/index.rst +40 -0
  419. data/packages/gsutil/boto/docs/source/ref/manage.rst +47 -0
  420. data/packages/gsutil/boto/docs/source/ref/mturk.rst +54 -0
  421. data/packages/gsutil/boto/docs/source/ref/mws.rst +33 -0
  422. data/packages/gsutil/boto/docs/source/ref/pyami.rst +103 -0
  423. data/packages/gsutil/boto/docs/source/ref/rds.rst +47 -0
  424. data/packages/gsutil/boto/docs/source/ref/route53.rst +34 -0
  425. data/packages/gsutil/boto/docs/source/ref/s3.rst +111 -0
  426. data/packages/gsutil/boto/docs/source/ref/sdb.rst +45 -0
  427. data/packages/gsutil/boto/docs/source/ref/sdb_db.rst +83 -0
  428. data/packages/gsutil/boto/docs/source/ref/services.rst +61 -0
  429. data/packages/gsutil/boto/docs/source/ref/ses.rst +21 -0
  430. data/packages/gsutil/boto/docs/source/ref/sns.rst +17 -0
  431. data/packages/gsutil/boto/docs/source/ref/sqs.rst +61 -0
  432. data/packages/gsutil/boto/docs/source/ref/sts.rst +25 -0
  433. data/packages/gsutil/boto/docs/source/ref/swf.rst +22 -0
  434. data/packages/gsutil/boto/docs/source/ref/vpc.rst +54 -0
  435. data/packages/gsutil/boto/docs/source/s3_tut.rst +450 -0
  436. data/packages/gsutil/boto/docs/source/security_groups.rst +82 -0
  437. data/packages/gsutil/boto/docs/source/ses_tut.rst +171 -0
  438. data/packages/gsutil/boto/docs/source/simpledb_tut.rst +188 -0
  439. data/packages/gsutil/boto/docs/source/sqs_tut.rst +246 -0
  440. data/packages/gsutil/boto/docs/source/vpc_tut.rst +100 -0
  441. data/packages/gsutil/boto/pylintrc +305 -0
  442. data/packages/gsutil/boto/requirements.txt +10 -0
  443. data/packages/gsutil/boto/setup.py +89 -0
  444. data/packages/gsutil/boto/tests/__init__.py +20 -0
  445. data/packages/gsutil/boto/tests/db/test_lists.py +96 -0
  446. data/packages/gsutil/boto/tests/db/test_password.py +128 -0
  447. data/packages/gsutil/boto/tests/db/test_query.py +152 -0
  448. data/packages/gsutil/boto/tests/db/test_sequence.py +109 -0
  449. data/packages/gsutil/boto/tests/devpay/__init__.py +0 -0
  450. data/packages/gsutil/boto/tests/devpay/test_s3.py +181 -0
  451. data/packages/gsutil/boto/tests/fps/__init__.py +0 -0
  452. data/packages/gsutil/boto/tests/fps/test.py +100 -0
  453. data/packages/gsutil/boto/tests/fps/test_verify_signature.py +12 -0
  454. data/packages/gsutil/boto/tests/integration/__init__.py +0 -0
  455. data/packages/gsutil/boto/tests/integration/beanstalk/test_wrapper.py +209 -0
  456. data/packages/gsutil/boto/tests/integration/cloudformation/__init__.py +21 -0
  457. data/packages/gsutil/boto/tests/integration/cloudformation/test_cert_verification.py +40 -0
  458. data/packages/gsutil/boto/tests/integration/cloudformation/test_connection.py +110 -0
  459. data/packages/gsutil/boto/tests/integration/cloudsearch/__init__.py +21 -0
  460. data/packages/gsutil/boto/tests/integration/cloudsearch/test_cert_verification.py +40 -0
  461. data/packages/gsutil/boto/tests/integration/datapipeline/test_layer1.py +122 -0
  462. data/packages/gsutil/boto/tests/integration/dynamodb/__init__.py +20 -0
  463. data/packages/gsutil/boto/tests/integration/dynamodb/test_cert_verification.py +40 -0
  464. data/packages/gsutil/boto/tests/integration/dynamodb/test_layer1.py +266 -0
  465. data/packages/gsutil/boto/tests/integration/dynamodb/test_layer2.py +484 -0
  466. data/packages/gsutil/boto/tests/integration/dynamodb/test_table.py +84 -0
  467. data/packages/gsutil/boto/tests/integration/ec2/__init__.py +20 -0
  468. data/packages/gsutil/boto/tests/integration/ec2/autoscale/__init__.py +21 -0
  469. data/packages/gsutil/boto/tests/integration/ec2/autoscale/test_cert_verification.py +40 -0
  470. data/packages/gsutil/boto/tests/integration/ec2/autoscale/test_connection.py +167 -0
  471. data/packages/gsutil/boto/tests/integration/ec2/cloudwatch/__init__.py +20 -0
  472. data/packages/gsutil/boto/tests/integration/ec2/cloudwatch/test_cert_verification.py +40 -0
  473. data/packages/gsutil/boto/tests/integration/ec2/cloudwatch/test_connection.py +277 -0
  474. data/packages/gsutil/boto/tests/integration/ec2/elb/__init__.py +20 -0
  475. data/packages/gsutil/boto/tests/integration/ec2/elb/test_cert_verification.py +40 -0
  476. data/packages/gsutil/boto/tests/integration/ec2/elb/test_connection.py +130 -0
  477. data/packages/gsutil/boto/tests/integration/ec2/test_cert_verification.py +40 -0
  478. data/packages/gsutil/boto/tests/integration/ec2/test_connection.py +192 -0
  479. data/packages/gsutil/boto/tests/integration/ec2/vpc/__init__.py +0 -0
  480. data/packages/gsutil/boto/tests/integration/ec2/vpc/test_connection.py +95 -0
  481. data/packages/gsutil/boto/tests/integration/elasticache/__init__.py +0 -0
  482. data/packages/gsutil/boto/tests/integration/elasticache/test_layer1.py +67 -0
  483. data/packages/gsutil/boto/tests/integration/elastictranscoder/__init__.py +0 -0
  484. data/packages/gsutil/boto/tests/integration/elastictranscoder/test_cert_verification.py +35 -0
  485. data/packages/gsutil/boto/tests/integration/elastictranscoder/test_layer1.py +115 -0
  486. data/packages/gsutil/boto/tests/integration/emr/__init__.py +20 -0
  487. data/packages/gsutil/boto/tests/integration/emr/test_cert_verification.py +40 -0
  488. data/packages/gsutil/boto/tests/integration/glacier/__init__.py +22 -0
  489. data/packages/gsutil/boto/tests/integration/glacier/test_cert_verification.py +40 -0
  490. data/packages/gsutil/boto/tests/integration/glacier/test_layer1.py +44 -0
  491. data/packages/gsutil/boto/tests/integration/glacier/test_layer2.py +45 -0
  492. data/packages/gsutil/boto/tests/integration/gs/__init__.py +0 -0
  493. data/packages/gsutil/boto/tests/integration/gs/cb_test_harness.py +71 -0
  494. data/packages/gsutil/boto/tests/integration/gs/test_basic.py +379 -0
  495. data/packages/gsutil/boto/tests/integration/gs/test_generation_conditionals.py +399 -0
  496. data/packages/gsutil/boto/tests/integration/gs/test_resumable_downloads.py +358 -0
  497. data/packages/gsutil/boto/tests/integration/gs/test_resumable_uploads.py +525 -0
  498. data/packages/gsutil/boto/tests/integration/gs/test_storage_uri.py +125 -0
  499. data/packages/gsutil/boto/tests/integration/gs/test_versioning.py +268 -0
  500. data/packages/gsutil/boto/tests/integration/gs/testcase.py +116 -0
  501. data/packages/gsutil/boto/tests/integration/gs/util.py +63 -0
  502. data/packages/gsutil/boto/tests/integration/iam/__init__.py +20 -0
  503. data/packages/gsutil/boto/tests/integration/iam/test_cert_verification.py +40 -0
  504. data/packages/gsutil/boto/tests/integration/mws/__init__.py +0 -0
  505. data/packages/gsutil/boto/tests/integration/mws/test.py +100 -0
  506. data/packages/gsutil/boto/tests/integration/rds/__init__.py +21 -0
  507. data/packages/gsutil/boto/tests/integration/rds/test_cert_verification.py +40 -0
  508. data/packages/gsutil/boto/tests/integration/route53/__init__.py +20 -0
  509. data/packages/gsutil/boto/tests/integration/route53/test_cert_verification.py +40 -0
  510. data/packages/gsutil/boto/tests/integration/route53/test_zone.py +132 -0
  511. data/packages/gsutil/boto/tests/integration/s3/__init__.py +20 -0
  512. data/packages/gsutil/boto/tests/integration/s3/mock_storage_service.py +589 -0
  513. data/packages/gsutil/boto/tests/integration/s3/other_cacerts.txt +70 -0
  514. data/packages/gsutil/boto/tests/integration/s3/test_bucket.py +263 -0
  515. data/packages/gsutil/boto/tests/integration/s3/test_cert_verification.py +40 -0
  516. data/packages/gsutil/boto/tests/integration/s3/test_connection.py +245 -0
  517. data/packages/gsutil/boto/tests/integration/s3/test_cors.py +78 -0
  518. data/packages/gsutil/boto/tests/integration/s3/test_encryption.py +115 -0
  519. data/packages/gsutil/boto/tests/integration/s3/test_https_cert_validation.py +141 -0
  520. data/packages/gsutil/boto/tests/integration/s3/test_key.py +375 -0
  521. data/packages/gsutil/boto/tests/integration/s3/test_mfa.py +95 -0
  522. data/packages/gsutil/boto/tests/integration/s3/test_multidelete.py +181 -0
  523. data/packages/gsutil/boto/tests/integration/s3/test_multipart.py +139 -0
  524. data/packages/gsutil/boto/tests/integration/s3/test_pool.py +246 -0
  525. data/packages/gsutil/boto/tests/integration/s3/test_versioning.py +158 -0
  526. data/packages/gsutil/boto/tests/integration/sdb/__init__.py +20 -0
  527. data/packages/gsutil/boto/tests/integration/sdb/test_cert_verification.py +40 -0
  528. data/packages/gsutil/boto/tests/integration/sdb/test_connection.py +119 -0
  529. data/packages/gsutil/boto/tests/integration/ses/__init__.py +0 -0
  530. data/packages/gsutil/boto/tests/integration/ses/test_cert_verification.py +40 -0
  531. data/packages/gsutil/boto/tests/integration/ses/test_connection.py +38 -0
  532. data/packages/gsutil/boto/tests/integration/sns/__init__.py +20 -0
  533. data/packages/gsutil/boto/tests/integration/sns/test_cert_verification.py +40 -0
  534. data/packages/gsutil/boto/tests/integration/sqs/__init__.py +20 -0
  535. data/packages/gsutil/boto/tests/integration/sqs/test_cert_verification.py +40 -0
  536. data/packages/gsutil/boto/tests/integration/sqs/test_connection.py +217 -0
  537. data/packages/gsutil/boto/tests/integration/sts/__init__.py +20 -0
  538. data/packages/gsutil/boto/tests/integration/sts/test_cert_verification.py +40 -0
  539. data/packages/gsutil/boto/tests/integration/sts/test_session_token.py +65 -0
  540. data/packages/gsutil/boto/tests/integration/swf/__init__.py +0 -0
  541. data/packages/gsutil/boto/tests/integration/swf/test_cert_verification.py +40 -0
  542. data/packages/gsutil/boto/tests/integration/swf/test_layer1.py +246 -0
  543. data/packages/gsutil/boto/tests/integration/swf/test_layer1_workflow_execution.py +173 -0
  544. data/packages/gsutil/boto/tests/mturk/__init__.py +0 -0
  545. data/packages/gsutil/boto/tests/mturk/_init_environment.py +28 -0
  546. data/packages/gsutil/boto/tests/mturk/all_tests.py +24 -0
  547. data/packages/gsutil/boto/tests/mturk/cleanup_tests.py +47 -0
  548. data/packages/gsutil/boto/tests/mturk/common.py +45 -0
  549. data/packages/gsutil/boto/tests/mturk/create_free_text_question_regex.doctest +100 -0
  550. data/packages/gsutil/boto/tests/mturk/create_hit.doctest +92 -0
  551. data/packages/gsutil/boto/tests/mturk/create_hit_binary.doctest +94 -0
  552. data/packages/gsutil/boto/tests/mturk/create_hit_external.py +21 -0
  553. data/packages/gsutil/boto/tests/mturk/create_hit_from_hit_type.doctest +103 -0
  554. data/packages/gsutil/boto/tests/mturk/create_hit_test.py +21 -0
  555. data/packages/gsutil/boto/tests/mturk/create_hit_with_qualifications.py +16 -0
  556. data/packages/gsutil/boto/tests/mturk/hit_persistence.py +27 -0
  557. data/packages/gsutil/boto/tests/mturk/mocks.py +11 -0
  558. data/packages/gsutil/boto/tests/mturk/reviewable_hits.doctest +129 -0
  559. data/packages/gsutil/boto/tests/mturk/run-doctest.py +13 -0
  560. data/packages/gsutil/boto/tests/mturk/search_hits.doctest +16 -0
  561. data/packages/gsutil/boto/tests/mturk/selenium_support.py +61 -0
  562. data/packages/gsutil/boto/tests/mturk/support.py +7 -0
  563. data/packages/gsutil/boto/tests/mturk/test_disable_hit.py +11 -0
  564. data/packages/gsutil/boto/tests/test.py +59 -0
  565. data/packages/gsutil/boto/tests/unit/__init__.py +79 -0
  566. data/packages/gsutil/boto/tests/unit/auth/__init__.py +0 -0
  567. data/packages/gsutil/boto/tests/unit/auth/test_sigv4.py +73 -0
  568. data/packages/gsutil/boto/tests/unit/beanstalk/__init__.py +0 -0
  569. data/packages/gsutil/boto/tests/unit/beanstalk/test_layer1.py +128 -0
  570. data/packages/gsutil/boto/tests/unit/cloudformation/__init__.py +0 -0
  571. data/packages/gsutil/boto/tests/unit/cloudformation/test_connection.py +605 -0
  572. data/packages/gsutil/boto/tests/unit/cloudformation/test_stack.py +63 -0
  573. data/packages/gsutil/boto/tests/unit/cloudfront/__init__.py +0 -0
  574. data/packages/gsutil/boto/tests/unit/cloudfront/test_invalidation_list.py +113 -0
  575. data/packages/gsutil/boto/tests/unit/cloudfront/test_signed_urls.py +354 -0
  576. data/packages/gsutil/boto/tests/unit/cloudsearch/__init__.py +1 -0
  577. data/packages/gsutil/boto/tests/unit/cloudsearch/test_connection.py +241 -0
  578. data/packages/gsutil/boto/tests/unit/cloudsearch/test_document.py +324 -0
  579. data/packages/gsutil/boto/tests/unit/cloudsearch/test_search.py +325 -0
  580. data/packages/gsutil/boto/tests/unit/dynamodb/__init__.py +0 -0
  581. data/packages/gsutil/boto/tests/unit/dynamodb/test_batch.py +103 -0
  582. data/packages/gsutil/boto/tests/unit/dynamodb/test_layer2.py +119 -0
  583. data/packages/gsutil/boto/tests/unit/dynamodb/test_types.py +82 -0
  584. data/packages/gsutil/boto/tests/unit/ec2/__init__.py +0 -0
  585. data/packages/gsutil/boto/tests/unit/ec2/autoscale/__init__.py +0 -0
  586. data/packages/gsutil/boto/tests/unit/ec2/autoscale/test_group.py +162 -0
  587. data/packages/gsutil/boto/tests/unit/ec2/test_address.py +39 -0
  588. data/packages/gsutil/boto/tests/unit/ec2/test_blockdevicemapping.py +79 -0
  589. data/packages/gsutil/boto/tests/unit/ec2/test_connection.py +480 -0
  590. data/packages/gsutil/boto/tests/unit/ec2/test_instance.py +243 -0
  591. data/packages/gsutil/boto/tests/unit/ec2/test_networkinterface.py +140 -0
  592. data/packages/gsutil/boto/tests/unit/ec2/test_volume.py +248 -0
  593. data/packages/gsutil/boto/tests/unit/emr/test_emr_responses.py +373 -0
  594. data/packages/gsutil/boto/tests/unit/glacier/__init__.py +0 -0
  595. data/packages/gsutil/boto/tests/unit/glacier/test_concurrent.py +120 -0
  596. data/packages/gsutil/boto/tests/unit/glacier/test_job.py +60 -0
  597. data/packages/gsutil/boto/tests/unit/glacier/test_layer1.py +98 -0
  598. data/packages/gsutil/boto/tests/unit/glacier/test_layer2.py +266 -0
  599. data/packages/gsutil/boto/tests/unit/glacier/test_utils.py +116 -0
  600. data/packages/gsutil/boto/tests/unit/glacier/test_vault.py +100 -0
  601. data/packages/gsutil/boto/tests/unit/glacier/test_writer.py +185 -0
  602. data/packages/gsutil/boto/tests/unit/provider/__init__.py +0 -0
  603. data/packages/gsutil/boto/tests/unit/provider/test_provider.py +176 -0
  604. data/packages/gsutil/boto/tests/unit/rds/__init__.py +0 -0
  605. data/packages/gsutil/boto/tests/unit/rds/test_connection.py +131 -0
  606. data/packages/gsutil/boto/tests/unit/s3/__init__.py +0 -0
  607. data/packages/gsutil/boto/tests/unit/s3/test_cors_configuration.py +77 -0
  608. data/packages/gsutil/boto/tests/unit/s3/test_key.py +75 -0
  609. data/packages/gsutil/boto/tests/unit/s3/test_keyfile.py +101 -0
  610. data/packages/gsutil/boto/tests/unit/s3/test_lifecycle.py +97 -0
  611. data/packages/gsutil/boto/tests/unit/s3/test_tagging.py +47 -0
  612. data/packages/gsutil/boto/tests/unit/s3/test_uri.py +257 -0
  613. data/packages/gsutil/boto/tests/unit/s3/test_website.py +188 -0
  614. data/packages/gsutil/boto/tests/unit/sns/__init__.py +0 -0
  615. data/packages/gsutil/boto/tests/unit/sns/test_connection.py +99 -0
  616. data/packages/gsutil/boto/tests/unit/sqs/__init__.py +0 -0
  617. data/packages/gsutil/boto/tests/unit/sqs/test_connection.py +98 -0
  618. data/packages/gsutil/boto/tests/unit/sqs/test_queue.py +40 -0
  619. data/packages/gsutil/boto/tests/unit/sts/test_connection.py +74 -0
  620. data/packages/gsutil/boto/tests/unit/test_connection.py +60 -0
  621. data/packages/gsutil/boto/tests/unit/utils/test_utils.py +109 -0
  622. data/packages/gsutil/boto/tox.ini +8 -0
  623. data/packages/gsutil/gslib/README +5 -0
  624. data/packages/gsutil/gslib/__init__.py +22 -0
  625. data/packages/gsutil/gslib/__init__.pyc +0 -0
  626. data/packages/gsutil/gslib/addlhelp/__init__.py +15 -0
  627. data/packages/gsutil/gslib/addlhelp/acls.py +234 -0
  628. data/packages/gsutil/gslib/addlhelp/anon.py +57 -0
  629. data/packages/gsutil/gslib/addlhelp/command_opts.py +116 -0
  630. data/packages/gsutil/gslib/addlhelp/dev.py +139 -0
  631. data/packages/gsutil/gslib/addlhelp/metadata.py +186 -0
  632. data/packages/gsutil/gslib/addlhelp/naming.py +173 -0
  633. data/packages/gsutil/gslib/addlhelp/prod.py +160 -0
  634. data/packages/gsutil/gslib/addlhelp/projects.py +130 -0
  635. data/packages/gsutil/gslib/addlhelp/subdirs.py +110 -0
  636. data/packages/gsutil/gslib/addlhelp/support.py +86 -0
  637. data/packages/gsutil/gslib/addlhelp/versioning.py +242 -0
  638. data/packages/gsutil/gslib/addlhelp/wildcards.py +170 -0
  639. data/packages/gsutil/gslib/bucket_listing_ref.py +175 -0
  640. data/packages/gsutil/gslib/bucket_listing_ref.pyc +0 -0
  641. data/packages/gsutil/gslib/command.py +722 -0
  642. data/packages/gsutil/gslib/command.pyc +0 -0
  643. data/packages/gsutil/gslib/command_runner.py +101 -0
  644. data/packages/gsutil/gslib/command_runner.pyc +0 -0
  645. data/packages/gsutil/gslib/commands/__init__.py +15 -0
  646. data/packages/gsutil/gslib/commands/__init__.pyc +0 -0
  647. data/packages/gsutil/gslib/commands/cat.py +131 -0
  648. data/packages/gsutil/gslib/commands/cat.pyc +0 -0
  649. data/packages/gsutil/gslib/commands/chacl.py +532 -0
  650. data/packages/gsutil/gslib/commands/chacl.pyc +0 -0
  651. data/packages/gsutil/gslib/commands/config.py +694 -0
  652. data/packages/gsutil/gslib/commands/config.pyc +0 -0
  653. data/packages/gsutil/gslib/commands/cp.py +1818 -0
  654. data/packages/gsutil/gslib/commands/cp.pyc +0 -0
  655. data/packages/gsutil/gslib/commands/disablelogging.py +101 -0
  656. data/packages/gsutil/gslib/commands/disablelogging.pyc +0 -0
  657. data/packages/gsutil/gslib/commands/enablelogging.py +149 -0
  658. data/packages/gsutil/gslib/commands/enablelogging.pyc +0 -0
  659. data/packages/gsutil/gslib/commands/getacl.py +82 -0
  660. data/packages/gsutil/gslib/commands/getacl.pyc +0 -0
  661. data/packages/gsutil/gslib/commands/getcors.py +121 -0
  662. data/packages/gsutil/gslib/commands/getcors.pyc +0 -0
  663. data/packages/gsutil/gslib/commands/getdefacl.py +86 -0
  664. data/packages/gsutil/gslib/commands/getdefacl.pyc +0 -0
  665. data/packages/gsutil/gslib/commands/getlogging.py +137 -0
  666. data/packages/gsutil/gslib/commands/getlogging.pyc +0 -0
  667. data/packages/gsutil/gslib/commands/getversioning.py +116 -0
  668. data/packages/gsutil/gslib/commands/getversioning.pyc +0 -0
  669. data/packages/gsutil/gslib/commands/getwebcfg.py +122 -0
  670. data/packages/gsutil/gslib/commands/getwebcfg.pyc +0 -0
  671. data/packages/gsutil/gslib/commands/help.py +218 -0
  672. data/packages/gsutil/gslib/commands/help.pyc +0 -0
  673. data/packages/gsutil/gslib/commands/ls.py +578 -0
  674. data/packages/gsutil/gslib/commands/ls.pyc +0 -0
  675. data/packages/gsutil/gslib/commands/mb.py +172 -0
  676. data/packages/gsutil/gslib/commands/mb.pyc +0 -0
  677. data/packages/gsutil/gslib/commands/mv.py +159 -0
  678. data/packages/gsutil/gslib/commands/mv.pyc +0 -0
  679. data/packages/gsutil/gslib/commands/perfdiag.py +903 -0
  680. data/packages/gsutil/gslib/commands/perfdiag.pyc +0 -0
  681. data/packages/gsutil/gslib/commands/rb.py +113 -0
  682. data/packages/gsutil/gslib/commands/rb.pyc +0 -0
  683. data/packages/gsutil/gslib/commands/rm.py +239 -0
  684. data/packages/gsutil/gslib/commands/rm.pyc +0 -0
  685. data/packages/gsutil/gslib/commands/setacl.py +138 -0
  686. data/packages/gsutil/gslib/commands/setacl.pyc +0 -0
  687. data/packages/gsutil/gslib/commands/setcors.py +145 -0
  688. data/packages/gsutil/gslib/commands/setcors.pyc +0 -0
  689. data/packages/gsutil/gslib/commands/setdefacl.py +105 -0
  690. data/packages/gsutil/gslib/commands/setdefacl.pyc +0 -0
  691. data/packages/gsutil/gslib/commands/setmeta.py +428 -0
  692. data/packages/gsutil/gslib/commands/setmeta.pyc +0 -0
  693. data/packages/gsutil/gslib/commands/setversioning.py +114 -0
  694. data/packages/gsutil/gslib/commands/setversioning.pyc +0 -0
  695. data/packages/gsutil/gslib/commands/setwebcfg.py +190 -0
  696. data/packages/gsutil/gslib/commands/setwebcfg.pyc +0 -0
  697. data/packages/gsutil/gslib/commands/test.py +228 -0
  698. data/packages/gsutil/gslib/commands/test.pyc +0 -0
  699. data/packages/gsutil/gslib/commands/update.py +305 -0
  700. data/packages/gsutil/gslib/commands/update.pyc +0 -0
  701. data/packages/gsutil/gslib/commands/version.py +150 -0
  702. data/packages/gsutil/gslib/commands/version.pyc +0 -0
  703. data/packages/gsutil/gslib/exception.py +76 -0
  704. data/packages/gsutil/gslib/exception.pyc +0 -0
  705. data/packages/gsutil/gslib/help_provider.py +81 -0
  706. data/packages/gsutil/gslib/help_provider.pyc +0 -0
  707. data/packages/gsutil/gslib/name_expansion.py +550 -0
  708. data/packages/gsutil/gslib/name_expansion.pyc +0 -0
  709. data/packages/gsutil/gslib/no_op_auth_plugin.py +30 -0
  710. data/packages/gsutil/gslib/plurality_checkable_iterator.py +56 -0
  711. data/packages/gsutil/gslib/plurality_checkable_iterator.pyc +0 -0
  712. data/packages/gsutil/gslib/project_id.py +67 -0
  713. data/packages/gsutil/gslib/project_id.pyc +0 -0
  714. data/packages/gsutil/gslib/storage_uri_builder.py +56 -0
  715. data/packages/gsutil/gslib/storage_uri_builder.pyc +0 -0
  716. data/packages/gsutil/gslib/tests/__init__.py +15 -0
  717. data/packages/gsutil/gslib/tests/__init__.pyc +0 -0
  718. data/packages/gsutil/gslib/tests/test_chacl.py +236 -0
  719. data/packages/gsutil/gslib/tests/test_cp.py +267 -0
  720. data/packages/gsutil/gslib/tests/test_data/test.gif +0 -0
  721. data/packages/gsutil/gslib/tests/test_data/test.mp3 +0 -0
  722. data/packages/gsutil/gslib/tests/test_ls.py +66 -0
  723. data/packages/gsutil/gslib/tests/test_mv.py +69 -0
  724. data/packages/gsutil/gslib/tests/test_naming.py +989 -0
  725. data/packages/gsutil/gslib/tests/test_perfdiag.py +41 -0
  726. data/packages/gsutil/gslib/tests/test_plurality_checkable_iterator.py +67 -0
  727. data/packages/gsutil/gslib/tests/test_rm.py +143 -0
  728. data/packages/gsutil/gslib/tests/test_setacl.py +152 -0
  729. data/packages/gsutil/gslib/tests/test_setcors.py +168 -0
  730. data/packages/gsutil/gslib/tests/test_setmeta.py +91 -0
  731. data/packages/gsutil/gslib/tests/test_setversioning.py +44 -0
  732. data/packages/gsutil/gslib/tests/test_setwebcfg.py +63 -0
  733. data/packages/gsutil/gslib/tests/test_thread_pool.py +92 -0
  734. data/packages/gsutil/gslib/tests/test_wildcard_iterator.py +364 -0
  735. data/packages/gsutil/gslib/tests/testcase/__init__.py +18 -0
  736. data/packages/gsutil/gslib/tests/testcase/base.py +89 -0
  737. data/packages/gsutil/gslib/tests/testcase/integration_testcase.py +197 -0
  738. data/packages/gsutil/gslib/tests/testcase/unit_testcase.py +230 -0
  739. data/packages/gsutil/gslib/tests/util.py +125 -0
  740. data/packages/gsutil/gslib/tests/util.pyc +0 -0
  741. data/packages/gsutil/gslib/thread_pool.py +79 -0
  742. data/packages/gsutil/gslib/thread_pool.pyc +0 -0
  743. data/packages/gsutil/gslib/util.py +151 -0
  744. data/packages/gsutil/gslib/util.pyc +0 -0
  745. data/packages/gsutil/gslib/wildcard_iterator.py +492 -0
  746. data/packages/gsutil/gslib/wildcard_iterator.pyc +0 -0
  747. data/packages/gsutil/gsutil +377 -0
  748. data/packages/gsutil/gsutil.spec.in +75 -0
  749. data/packages/gsutil/oauth2_plugin/__init__.py +22 -0
  750. data/packages/gsutil/oauth2_plugin/__init__.pyc +0 -0
  751. data/packages/gsutil/oauth2_plugin/oauth2_client.py +642 -0
  752. data/packages/gsutil/oauth2_plugin/oauth2_client.pyc +0 -0
  753. data/packages/gsutil/oauth2_plugin/oauth2_client_test.py +374 -0
  754. data/packages/gsutil/oauth2_plugin/oauth2_helper.py +110 -0
  755. data/packages/gsutil/oauth2_plugin/oauth2_helper.pyc +0 -0
  756. data/packages/gsutil/oauth2_plugin/oauth2_plugin.py +24 -0
  757. data/packages/gsutil/oauth2_plugin/oauth2_plugin.pyc +0 -0
  758. data/packages/gsutil/pkg_gen.sh +54 -0
  759. data/packages/gsutil/pkg_util.py +60 -0
  760. data/packages/gsutil/setup.py +141 -0
  761. data/packages/gsutil/third_party/__init__.py +0 -0
  762. data/packages/gsutil/third_party/__init__.pyc +0 -0
  763. data/packages/gsutil/third_party/fancy_urllib/README +21 -0
  764. data/packages/gsutil/third_party/fancy_urllib/__init__.py +398 -0
  765. data/packages/gsutil/third_party/fancy_urllib/__init__.pyc +0 -0
  766. data/tasks/rubygem.rake +3 -1
  767. metadata +770 -7
  768. metadata.gz.sig +0 -0
@@ -0,0 +1,172 @@
1
+ # Copyright 2011 Google Inc. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from gslib.command import Command
16
+ from gslib.command import COMMAND_NAME
17
+ from gslib.command import COMMAND_NAME_ALIASES
18
+ from gslib.command import CONFIG_REQUIRED
19
+ from gslib.command import FILE_URIS_OK
20
+ from gslib.command import MAX_ARGS
21
+ from gslib.command import MIN_ARGS
22
+ from gslib.command import PROVIDER_URIS_OK
23
+ from gslib.command import SUPPORTED_SUB_ARGS
24
+ from gslib.command import URIS_START_ARG
25
+ from gslib.exception import CommandException
26
+ from gslib.help_provider import HELP_NAME
27
+ from gslib.help_provider import HELP_NAME_ALIASES
28
+ from gslib.help_provider import HELP_ONE_LINE_SUMMARY
29
+ from gslib.help_provider import HELP_TEXT
30
+ from gslib.help_provider import HelpType
31
+ from gslib.help_provider import HELP_TYPE
32
+ from gslib.util import NO_MAX
33
+
34
+ _detailed_help_text = ("""
35
+ <B>SYNOPSIS</B>
36
+ gsutil mb [-c storage_class] [-l location] [-p proj_id] uri...
37
+
38
+
39
+ <B>DESCRIPTION</B>
40
+ The mb command creates a new bucket. Google Cloud Storage has a single
41
+ namespace, so you will not be allowed to create a bucket with a name already
42
+ in use by another user. You can, however, carve out parts of the bucket name
43
+ space corresponding to your company's domain name (see "gsutil help naming").
44
+
45
+ If you don't specify a project ID using the -p option, the bucket
46
+ will be created using the default project ID specified in your gsutil
47
+ configuration file (see "gsutil help config"). For more details about
48
+ projects see "gsutil help projects".
49
+
50
+ The -c and -l options specify the storage class and location, respectively,
51
+ for the bucket. Once a bucket is created in a given location and with a
52
+ given storage class, it cannot be moved to a different location, and the
53
+ storage class cannot be changed. Instead, you would need to create a new
54
+ bucket and move the data over and then delete the original bucket.
55
+
56
+
57
+ <B>BUCKET STORAGE CLASSES</B>
58
+ If you don't specify a -c option, the bucket will be created with the default
59
+ (standard) storage class.
60
+
61
+ If you specify -c DURABLE_REDUCED_AVAILABILITY (or -c DRA), it causes the data
62
+ stored in the bucket to use durable reduced availability storage. Buckets
63
+ created with this storage class have lower availability than standard storage
64
+ class buckets, but durability equal to that of buckets created with standard
65
+ storage class. This option allows users to reduce costs for data for which
66
+ lower availability is acceptable. Durable Reduced Availability storage would
67
+ not be appropriate for "hot" objects (i.e., objects being accessed frequently)
68
+ or for interactive workloads; however, it might be appropriate for other types
69
+ of applications. See the online documentation for pricing and SLA details.
70
+
71
+
72
+ <B>BUCKET LOCATIONS</B>
73
+ If you don't specify a -l option, the bucket will be created in the default
74
+ location (US). Otherwise, you can specify one of the available locations:
75
+ US (United States) or EU (Europe).
76
+
77
+
78
+ <B>OPTIONS</B>
79
+ -c storage_class Can be DRA (or DURABLE_REDUCED_AVAILABILITY) or S (or
80
+ STANDARD). Default is STANDARD.
81
+
82
+ -l location Can be US or EU. Default is US. Locations are case
83
+ insensitive.
84
+
85
+ -p proj_id Specifies the project ID under which to create the bucket.
86
+ """)
87
+
88
+
89
+ class MbCommand(Command):
90
+ """Implementation of gsutil mb command."""
91
+
92
+ # Command specification (processed by parent class).
93
+ command_spec = {
94
+ # Name of command.
95
+ COMMAND_NAME : 'mb',
96
+ # List of command name aliases.
97
+ COMMAND_NAME_ALIASES : ['makebucket', 'createbucket', 'md', 'mkdir'],
98
+ # Min number of args required by this command.
99
+ MIN_ARGS : 1,
100
+ # Max number of args required by this command, or NO_MAX.
101
+ MAX_ARGS : NO_MAX,
102
+ # Getopt-style string specifying acceptable sub args.
103
+ SUPPORTED_SUB_ARGS : 'c:l:p:',
104
+ # True if file URIs acceptable for this command.
105
+ FILE_URIS_OK : False,
106
+ # True if provider-only URIs acceptable for this command.
107
+ PROVIDER_URIS_OK : False,
108
+ # Index in args of first URI arg.
109
+ URIS_START_ARG : 0,
110
+ # True if must configure gsutil before running command.
111
+ CONFIG_REQUIRED : True,
112
+ }
113
+ help_spec = {
114
+ # Name of command or auxiliary help info for which this help applies.
115
+ HELP_NAME : 'mb',
116
+ # List of help name aliases.
117
+ HELP_NAME_ALIASES : ['createbucket', 'makebucket', 'md', 'mkdir',
118
+ 'location', 'dra', 'dras', 'reduced_availability',
119
+ 'durable_reduced_availability',
120
+ 'rr', 'reduced_redundancy',
121
+ 'standard', 'storage class' ],
122
+ # Type of help:
123
+ HELP_TYPE : HelpType.COMMAND_HELP,
124
+ # One line summary of this help.
125
+ HELP_ONE_LINE_SUMMARY : 'Make buckets',
126
+ # The full help text.
127
+ HELP_TEXT : _detailed_help_text,
128
+ }
129
+
130
+ # Command entry point.
131
+ def RunCommand(self):
132
+ location = ''
133
+ storage_class = ''
134
+ if self.sub_opts:
135
+ for o, a in self.sub_opts:
136
+ if o == '-l':
137
+ location = a
138
+ elif o == '-p':
139
+ self.proj_id_handler.SetProjectId(a)
140
+ elif o == '-c':
141
+ storage_class = self._Normalize_Storage_Class(a)
142
+
143
+ if not self.headers:
144
+ headers = {}
145
+ else:
146
+ headers = self.headers.copy()
147
+
148
+ for bucket_uri_str in self.args:
149
+ bucket_uri = self.suri_builder.StorageUri(bucket_uri_str)
150
+ if not bucket_uri.names_bucket():
151
+ raise CommandException('The mb command requires a URI that specifies a '
152
+ 'bucket.\n"%s" is not valid.' % bucket_uri)
153
+ self.proj_id_handler.FillInProjectHeaderIfNeeded('mb', bucket_uri,
154
+ headers)
155
+ print 'Creating %s...' % bucket_uri
156
+ # Pass storage_class param only if this is a GCS bucket. (In S3 the
157
+ # storage class is specified on the key object.)
158
+ if bucket_uri.scheme == 'gs':
159
+ bucket_uri.create_bucket(headers=headers, location=location,
160
+ storage_class=storage_class)
161
+ else:
162
+ bucket_uri.create_bucket(headers=headers, location=location)
163
+
164
+ return 0
165
+
166
+ def _Normalize_Storage_Class(self, sc):
167
+ sc = sc.upper()
168
+ if sc in ('DRA', 'DURABLE_REDUCED_AVAILABILITY'):
169
+ return 'DURABLE_REDUCED_AVAILABILITY'
170
+ if sc in ('S', 'STD', 'STANDARD'):
171
+ return 'STANDARD'
172
+ return sc
@@ -0,0 +1,159 @@
1
+ # Copyright 2011 Google Inc. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ from gslib.command import Command
16
+ from gslib.command import COMMAND_NAME
17
+ from gslib.command import COMMAND_NAME_ALIASES
18
+ from gslib.command import CONFIG_REQUIRED
19
+ from gslib.command import FILE_URIS_OK
20
+ from gslib.command import MAX_ARGS
21
+ from gslib.command import MIN_ARGS
22
+ from gslib.command import PROVIDER_URIS_OK
23
+ from gslib.command import SUPPORTED_SUB_ARGS
24
+ from gslib.command import URIS_START_ARG
25
+ from gslib.exception import CommandException
26
+ from gslib.help_provider import HELP_NAME
27
+ from gslib.help_provider import HELP_NAME_ALIASES
28
+ from gslib.help_provider import HELP_ONE_LINE_SUMMARY
29
+ from gslib.help_provider import HELP_TEXT
30
+ from gslib.help_provider import HelpType
31
+ from gslib.help_provider import HELP_TYPE
32
+ from gslib.util import NO_MAX
33
+
34
+ _detailed_help_text = ("""
35
+ <B>SYNOPSIS</B>
36
+ gsutil mv [-p] src_uri dst_uri
37
+ - or -
38
+ gsutil mv [-p] uri... dst_uri
39
+
40
+
41
+ <B>DESCRIPTION</B>
42
+ The gsutil mv command allows you to move data between your local file
43
+ system and the cloud, move data within the cloud, and move data between
44
+ cloud storage providers. For example, to move all objects from a
45
+ bucket to a local directory you could use:
46
+
47
+ gsutil mv gs://my_bucket dir
48
+
49
+ Similarly, to move all objects from a local directory to a bucket you could
50
+ use:
51
+
52
+ gsutil mv ./dir gs://my_bucket
53
+
54
+
55
+ <B>RENAMING BUCKET SUBDIRECTORIES</B>
56
+ You can use the gsutil mv command to rename subdirectories. For example,
57
+ the command:
58
+
59
+ gsutil mv gs://my_bucket/olddir gs://my_bucket/newdir
60
+
61
+ would rename all objects and subdirectories under gs://my_bucket/olddir to be
62
+ under gs://my_bucket/newdir, otherwise preserving the subdirectory structure.
63
+
64
+ If you do a rename as specified above and you want to preserve ACLs, you
65
+ should use the -p option (see OPTIONS).
66
+
67
+ Note that when using mv to rename bucket subdirectories you cannot specify
68
+ the source URI using wildcards. You need to spell out the complete name:
69
+
70
+ gsutil mv gs://my_bucket/olddir gs://my_bucket/newdir
71
+
72
+ If you have a large number of files to move you might want to use the
73
+ gsutil -m option, to perform a multi-threaded/multi-processing move:
74
+
75
+ gsutil -m mv gs://my_bucket/olddir gs://my_bucket/newdir
76
+
77
+
78
+ <B>NON-ATOMIC OPERATION</B>
79
+ Unlike the case with many file systems, the gsutil mv command does not
80
+ perform a single atomic operation. Rather, it performs a copy from source
81
+ to destination followed by removing the source for each object.
82
+
83
+
84
+ <B>OPTIONS</B>
85
+ -p Causes ACL to be preserved when moving in the cloud. Note that
86
+ this option has performance and cost implications, because it
87
+ is essentially performing three requests (getacl, cp, setacl).
88
+ (The performance issue can be mitigated to some degree by
89
+ using gsutil -m cp to cause multi-threaded/multi-processing
90
+ copying.)
91
+ """)
92
+
93
+
94
+ class MvCommand(Command):
95
+ """Implementation of gsutil mv command.
96
+ Note that there is no atomic rename operation - this command is simply
97
+ a shorthand for 'cp' followed by 'rm'.
98
+ """
99
+
100
+ # Command specification (processed by parent class).
101
+ command_spec = {
102
+ # Name of command.
103
+ COMMAND_NAME : 'mv',
104
+ # List of command name aliases.
105
+ COMMAND_NAME_ALIASES : ['move', 'ren', 'rename'],
106
+ # Min number of args required by this command.
107
+ MIN_ARGS : 2,
108
+ # Max number of args required by this command, or NO_MAX.
109
+ MAX_ARGS : NO_MAX,
110
+ # Getopt-style string specifying acceptable sub args.
111
+ SUPPORTED_SUB_ARGS : 'pv',
112
+ # True if file URIs acceptable for this command.
113
+ FILE_URIS_OK : True,
114
+ # True if provider-only URIs acceptable for this command.
115
+ PROVIDER_URIS_OK : False,
116
+ # Index in args of first URI arg.
117
+ URIS_START_ARG : 0,
118
+ # True if must configure gsutil before running command.
119
+ CONFIG_REQUIRED : True,
120
+ }
121
+ help_spec = {
122
+ # Name of command or auxiliary help info for which this help applies.
123
+ HELP_NAME : 'mv',
124
+ # List of help name aliases.
125
+ HELP_NAME_ALIASES : ['move', 'rename'],
126
+ # Type of help:
127
+ HELP_TYPE : HelpType.COMMAND_HELP,
128
+ # One line summary of this help.
129
+ HELP_ONE_LINE_SUMMARY : 'Move/rename objects and/or subdirectories',
130
+ # The full help text.
131
+ HELP_TEXT : _detailed_help_text,
132
+ }
133
+
134
+ # Command entry point.
135
+ def RunCommand(self):
136
+ # Check each source arg up, refusing to delete a bucket src URI (force users
137
+ # to explicitly do that as a separate operation).
138
+ for arg_to_check in self.args[0:-1]:
139
+ if self.suri_builder.StorageUri(arg_to_check).names_bucket():
140
+ raise CommandException('You cannot move a source bucket using the mv '
141
+ 'command. If you meant to move\nall objects in '
142
+ 'the bucket, you can use a command like:\n'
143
+ '\tgsutil mv %s/* %s' %
144
+ (arg_to_check, self.args[-1]))
145
+
146
+ # Insert command-line opts in front of args so they'll be picked up by cp
147
+ # and rm commands (e.g., for -p option). Use undocumented (internal
148
+ # use-only) cp -M option, which causes each original object to be deleted
149
+ # after successfully copying to its destination, and also causes naming
150
+ # behavior consistent with Unix mv naming behavior (see comments in
151
+ # _ConstructDstUri in cp.py).
152
+ unparsed_args = ['-M']
153
+ if self.recursion_requested:
154
+ unparsed_args.append('-R')
155
+ unparsed_args.extend(self.unparsed_args)
156
+ self.command_runner.RunNamedCommand('cp', unparsed_args, self.headers,
157
+ self.debug, self.parallel_operations)
158
+
159
+ return 0
@@ -0,0 +1,903 @@
1
+ # Copyright 2012 Google Inc. All Rights Reserved.
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # http://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+ """Contains the perfdiag gsutil command."""
16
+
17
+ import calendar
18
+ from collections import defaultdict
19
+ import contextlib
20
+ import datetime
21
+ import json
22
+ import math
23
+ import multiprocessing
24
+ import os
25
+ import re
26
+ import socket
27
+ import string
28
+ import subprocess
29
+ import tempfile
30
+ import time
31
+
32
+ import boto.gs.connection
33
+
34
+ from gslib.command import Command
35
+ from gslib.command import COMMAND_NAME
36
+ from gslib.command import COMMAND_NAME_ALIASES
37
+ from gslib.command import CONFIG_REQUIRED
38
+ from gslib.command import FILE_URIS_OK
39
+ from gslib.command import MAX_ARGS
40
+ from gslib.command import MIN_ARGS
41
+ from gslib.command import PROVIDER_URIS_OK
42
+ from gslib.command import SUPPORTED_SUB_ARGS
43
+ from gslib.command import URIS_START_ARG
44
+ from gslib.commands import config
45
+ from gslib.exception import CommandException
46
+ from gslib.help_provider import HELP_NAME
47
+ from gslib.help_provider import HELP_NAME_ALIASES
48
+ from gslib.help_provider import HELP_ONE_LINE_SUMMARY
49
+ from gslib.help_provider import HELP_TEXT
50
+ from gslib.help_provider import HELP_TYPE
51
+ from gslib.help_provider import HelpType
52
+ from gslib.util import IS_LINUX
53
+ from gslib.util import MakeBitsHumanReadable
54
+ from gslib.util import MakeHumanReadable
55
+ from gslib.util import Percentile
56
+
57
+ _detailed_help_text = ("""
58
+ <B>SYNOPSIS</B>
59
+ gsutil perfdiag [-i in.json] [-o out.json]
60
+ [-n iterations] [-c concurrency] [-s size] [-t tests] uri...
61
+
62
+
63
+ <B>DESCRIPTION</B>
64
+ The perfdiag command runs a suite of diagnostic tests for a given Google
65
+ Storage bucket.
66
+
67
+ The 'uri' parameter must name an existing bucket (e.g. gs://foo) to which
68
+ the user has write permission. Several test files will be uploaded to and
69
+ downloaded from this bucket. All test files will be deleted at the completion
70
+ of the diagnostic if it finishes successfully.
71
+
72
+ gsutil performance can be impacted by many factors at the client, server,
73
+ and in-between, such as: CPU speed; available memory; the access path to the
74
+ local disk; network bandwidth; contention and error rates along the path
75
+ between gsutil and Google; operating system buffering configuration; and
76
+ firewalls and other network elements. The perfdiag command is provided so
77
+ that customers can run a known measurement suite when troubleshooting
78
+ performance problems.
79
+
80
+
81
+ <B>PROVIDING DIAGNOSTIC OUTPUT TO GOOGLE CLOUD STORAGE TEAM</B>
82
+ If the Google Cloud Storage Team asks you to run a performance diagnostic
83
+ please use the following command, and email the output file (output.json)
84
+ to gs-team@google.com:
85
+
86
+ gsutil perfdiag -o output.json gs://your-bucket
87
+
88
+
89
+ <B>OPTIONS</B>
90
+ -n Sets the number of iterations performed when downloading and
91
+ uploading files during latency and throughput tests. Defaults to
92
+ 5.
93
+
94
+ -c Sets the level of concurrency to use while running throughput
95
+ experiments. The default value of 1 will only run a single read
96
+ or write operation concurrently.
97
+
98
+ -s Sets the size (in bytes) of the test file used to perform read
99
+ and write throughput tests. The default is 1 MiB.
100
+
101
+ -t Sets the list of diagnostic tests to perform. The default is to
102
+ run all diagnostic tests. Must be a comma-separated list
103
+ containing one or more of the following:
104
+
105
+ lat: Runs N iterations (set with -n) of writing the file,
106
+ retrieving its metadata, reading the file, and deleting
107
+ the file. Records the latency of each operation.
108
+
109
+ rthru: Runs N (set with -n) read operations, with at most C
110
+ (set with -c) reads outstanding at any given time.
111
+
112
+ wthru: Runs N (set with -n) write operations, with at most C
113
+ (set with -c) writes outstanding at any given time.
114
+
115
+ -o Writes the results of the diagnostic to an output file. The output
116
+ is a JSON file containing system information and performance
117
+ diagnostic results. The file can be read and reported later using
118
+ the -i option.
119
+
120
+ -i Reads the JSON output file created using the -o command and prints
121
+ a formatted description of the results.
122
+
123
+
124
+ <B>NOTE</B>
125
+ The perfdiag command collects system information. It collects your IP address,
126
+ executes DNS queries to Google servers and collects the results, and collects
127
+ network statistics information from the output of netstat -s. None of this
128
+ information will be sent to Google unless you choose to send it.
129
+ """)
130
+
131
+
132
+ class PerfDiagCommand(Command):
133
+ """Implementation of gsutil perfdiag command."""
134
+
135
+ # Command specification (processed by parent class).
136
+ command_spec = {
137
+ # Name of command.
138
+ COMMAND_NAME: 'perfdiag',
139
+ # List of command name aliases.
140
+ COMMAND_NAME_ALIASES: ['diag', 'diagnostic', 'perf', 'performance'],
141
+ # Min number of args required by this command.
142
+ MIN_ARGS: 0,
143
+ # Max number of args required by this command, or NO_MAX.
144
+ MAX_ARGS: 1,
145
+ # Getopt-style string specifying acceptable sub args.
146
+ SUPPORTED_SUB_ARGS: 'n:c:s:t:i:o:',
147
+ # True if file URIs acceptable for this command.
148
+ FILE_URIS_OK: False,
149
+ # True if provider-only URIs acceptable for this command.
150
+ PROVIDER_URIS_OK: False,
151
+ # Index in args of first URI arg.
152
+ URIS_START_ARG: 0,
153
+ # True if must configure gsutil before running command.
154
+ CONFIG_REQUIRED: True,
155
+ }
156
+ help_spec = {
157
+ # Name of command or auxiliary help info for which this help applies.
158
+ HELP_NAME: 'perfdiag',
159
+ # List of help name aliases.
160
+ HELP_NAME_ALIASES: [],
161
+ # Type of help:
162
+ HELP_TYPE: HelpType.COMMAND_HELP,
163
+ # One line summary of this help.
164
+ HELP_ONE_LINE_SUMMARY: 'Run performance diagnostic',
165
+ # The full help text.
166
+ HELP_TEXT: _detailed_help_text,
167
+ }
168
+
169
+ # Byte sizes to use for testing files.
170
+ # TODO: Consider letting the user specify these sizes with a configuration
171
+ # parameter.
172
+ test_file_sizes = (
173
+ 0, # 0 bytes
174
+ 1024, # 1 KB
175
+ 102400, # 100 KB
176
+ 1048576, # 1MB
177
+ )
178
+
179
+ # List of all diagnostic tests.
180
+ ALL_DIAG_TESTS = ('rthru', 'wthru', 'lat')
181
+
182
+ # Google Cloud Storage API endpoint host.
183
+ GOOGLE_API_HOST = boto.gs.connection.GSConnection.DefaultHost
184
+
185
+ def _WindowedExec(self, cmd, n, w, raise_on_error=True):
186
+ """Executes a command n times with a window size of w.
187
+
188
+ Up to w instances of the command will be executed and left outstanding at a
189
+ time until n instances of the command have completed.
190
+
191
+ Args:
192
+ cmd: List containing the command to execute.
193
+ n: Number of times the command will be executed.
194
+ w: Window size of outstanding commands being executed.
195
+ raise_on_error: See _Exec.
196
+
197
+ Raises:
198
+ Exception: If raise_on_error is set to True and any process exits with a
199
+ non-zero return code.
200
+ """
201
+ if self.debug:
202
+ print 'Running command:', cmd
203
+ devnull_f = open(os.devnull, 'w')
204
+ num_finished = 0
205
+ running = []
206
+ while len(running) or num_finished < n:
207
+ # Fires off new commands that can be executed.
208
+ while len(running) < w and num_finished + len(running) < n:
209
+ print 'Starting concurrent command: %s' % (' '.join(cmd))
210
+ p = subprocess.Popen(cmd, stdout=devnull_f, stderr=devnull_f)
211
+ running.append(p)
212
+
213
+ # Checks for finished commands.
214
+ prev_running = running
215
+ running = []
216
+ for p in prev_running:
217
+ retcode = p.poll()
218
+ if retcode is None:
219
+ running.append(p)
220
+ elif raise_on_error and retcode:
221
+ raise CommandException("Received non-zero return code (%d) from "
222
+ "subprocess '%s'." % (retcode, ' '.join(cmd)))
223
+ else:
224
+ num_finished += 1
225
+
226
+ def _Exec(self, cmd, raise_on_error=True, return_output=False,
227
+ mute_stderr=False):
228
+ """Executes a command in a subprocess.
229
+
230
+ Args:
231
+ cmd: List containing the command to execute.
232
+ raise_on_error: Whether or not to raise an exception when a process exits
233
+ with a non-zero return code.
234
+ return_output: If set to True, the return value of the function is the
235
+ stdout of the process.
236
+ mute_stderr: If set to True, the stderr of the process is not printed to
237
+ the console.
238
+
239
+ Returns:
240
+ The return code of the process or the stdout if return_output is set.
241
+
242
+ Raises:
243
+ Exception: If raise_on_error is set to True and any process exits with a
244
+ non-zero return code.
245
+ """
246
+ if self.debug:
247
+ print 'Running command:', cmd
248
+ stderr = subprocess.PIPE if mute_stderr else None
249
+ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr)
250
+ (stdoutdata, stderrdata) = p.communicate()
251
+ if raise_on_error and p.returncode:
252
+ raise CommandException("Received non-zero return code (%d) from "
253
+ "subprocess '%s'." % (p.returncode, ' '.join(cmd)))
254
+ return stdoutdata if return_output else p.returncode
255
+
256
+ def _GsUtil(self, cmd, raise_on_error=True, return_output=False,
257
+ mute_stderr=False):
258
+ """Executes a gsutil command in a subprocess.
259
+
260
+ Args:
261
+ cmd: A list containing the arguments to the gsutil program, e.g. ['ls',
262
+ 'gs://foo'].
263
+ raise_on_error: see _Exec.
264
+ return_output: see _Exec.
265
+ mute_stderr: see _Exec.
266
+
267
+ Returns:
268
+ The return code of the process or the stdout if return_output is set.
269
+ """
270
+ cmd = self.gsutil_exec_list + cmd
271
+ return self._Exec(cmd, raise_on_error=raise_on_error,
272
+ return_output=return_output, mute_stderr=mute_stderr)
273
+
274
+ def _SetUp(self):
275
+ """Performs setup operations needed before diagnostics can be run."""
276
+
277
+ # Stores test result data.
278
+ self.results = {}
279
+ # List of test files in a temporary location on disk for latency ops.
280
+ self.latency_files = []
281
+ # Maps each test file path to its size in bytes.
282
+ self.file_sizes = {}
283
+ # Maps each test file to its contents as a string.
284
+ self.file_contents = {}
285
+
286
+ def _MakeFile(file_size):
287
+ """Creates a temporary file of the given size and returns its path."""
288
+ fd, fpath = tempfile.mkstemp(suffix='.bin', prefix='gsutil_test_file',
289
+ text=False)
290
+ self.file_sizes[fpath] = file_size
291
+ f = os.fdopen(fd, 'wb')
292
+ f.write(os.urandom(file_size))
293
+ f.close()
294
+ f = open(fpath, 'rb')
295
+ self.file_contents[fpath] = f.read()
296
+ f.close()
297
+ return fpath
298
+
299
+ # Create files for latency tests.
300
+ for file_size in self.test_file_sizes:
301
+ fpath = _MakeFile(file_size)
302
+ self.latency_files.append(fpath)
303
+
304
+ # Local file on disk for write throughput tests.
305
+ self.thru_local_file = _MakeFile(self.thru_filesize)
306
+ # Remote file to write/read from during throughput tests.
307
+ self.thru_remote_file = (str(self.bucket_uri) +
308
+ os.path.basename(self.thru_local_file))
309
+
310
+ def _TearDown(self):
311
+ """Performs operations to clean things up after performing diagnostics."""
312
+ for fpath in self.latency_files + [self.thru_local_file]:
313
+ try:
314
+ os.remove(fpath)
315
+ except OSError:
316
+ pass
317
+
318
+ self._GsUtil(['rm', self.thru_remote_file], raise_on_error=False,
319
+ mute_stderr=True)
320
+
321
+ @contextlib.contextmanager
322
+ def _Time(self, key, bucket):
323
+ """A context manager that measures time.
324
+
325
+ A context manager that prints a status message before and after executing
326
+ the inner command and times how long the inner command takes. Keeps track of
327
+ the timing, aggregated by the given key.
328
+
329
+ Args:
330
+ key: The key to insert the timing value into a dictionary bucket.
331
+ bucket: A dictionary to place the timing value in.
332
+
333
+ Yields:
334
+ For the context manager.
335
+ """
336
+ print key, 'starting...'
337
+ t0 = time.time()
338
+ yield
339
+ t1 = time.time()
340
+ bucket[key].append(t1 - t0)
341
+ print key, 'done.'
342
+
343
+ def _RunLatencyTests(self):
344
+ """Runs latency tests."""
345
+ # Stores timing information for each category of operation.
346
+ self.results['latency'] = defaultdict(list)
347
+
348
+ for i in range(self.num_iterations):
349
+ print
350
+ print 'Running latency iteration %d...' % (i+1)
351
+ for fpath in self.latency_files:
352
+ basename = os.path.basename(fpath)
353
+ gsbucket = str(self.bucket_uri)
354
+ gsuri = gsbucket + basename
355
+ file_size = self.file_sizes[fpath]
356
+ readable_file_size = MakeHumanReadable(file_size)
357
+
358
+ print
359
+ print ("File of size %(size)s located on disk at '%(fpath)s' being "
360
+ "diagnosed in the cloud at '%(gsuri)s'."
361
+ % {'size': readable_file_size,
362
+ 'fpath': fpath,
363
+ 'gsuri': gsuri})
364
+
365
+ k = self.bucket.key_class(self.bucket)
366
+ k.key = basename
367
+
368
+ with self._Time('UPLOAD_%d' % file_size, self.results['latency']):
369
+ k.set_contents_from_string(self.file_contents[fpath])
370
+ with self._Time('METADATA_%d' % file_size, self.results['latency']):
371
+ k.exists()
372
+ with self._Time('DOWNLOAD_%d' % file_size, self.results['latency']):
373
+ k.get_contents_as_string()
374
+ with self._Time('DELETE_%d' % file_size, self.results['latency']):
375
+ k.delete()
376
+
377
+ def _RunReadThruTests(self):
378
+ """Runs read throughput tests."""
379
+ self.results['read_throughput'] = {'file_size': self.thru_filesize,
380
+ 'num_times': self.num_iterations,
381
+ 'concurrency': self.concurrency}
382
+
383
+ # Copy the file to remote location before reading.
384
+ self._GsUtil(['cp', self.thru_local_file, self.thru_remote_file])
385
+
386
+ if self.concurrency == 1:
387
+ k = self.bucket.key_class(self.bucket)
388
+ k.key = os.path.basename(self.thru_local_file)
389
+ # Warm up the TCP connection by transferring a couple times first.
390
+ for i in range(2):
391
+ k.get_contents_as_string()
392
+ t0 = time.time()
393
+ for i in range(self.num_iterations):
394
+ k.get_contents_as_string()
395
+ t1 = time.time()
396
+ else:
397
+ cmd = self.gsutil_exec_list + ['cp', self.thru_remote_file, os.devnull]
398
+ t0 = time.time()
399
+ self._WindowedExec(cmd, self.num_iterations, self.concurrency)
400
+ t1 = time.time()
401
+
402
+ time_took = t1 - t0
403
+ total_bytes_copied = self.thru_filesize * self.num_iterations
404
+ bytes_per_second = total_bytes_copied / time_took
405
+
406
+ self.results['read_throughput']['time_took'] = time_took
407
+ self.results['read_throughput']['total_bytes_copied'] = total_bytes_copied
408
+ self.results['read_throughput']['bytes_per_second'] = bytes_per_second
409
+
410
+ def _RunWriteThruTests(self):
411
+ """Runs write throughput tests."""
412
+ self.results['write_throughput'] = {'file_size': self.thru_filesize,
413
+ 'num_copies': self.num_iterations,
414
+ 'concurrency': self.concurrency}
415
+
416
+ if self.concurrency == 1:
417
+ k = self.bucket.key_class(self.bucket)
418
+ k.key = os.path.basename(self.thru_local_file)
419
+ # Warm up the TCP connection by transferring a couple times first.
420
+ for i in range(2):
421
+ k.set_contents_from_string(self.file_contents[self.thru_local_file])
422
+ t0 = time.time()
423
+ for i in range(self.num_iterations):
424
+ k.set_contents_from_string(self.file_contents[self.thru_local_file])
425
+ t1 = time.time()
426
+ else:
427
+ cmd = self.gsutil_exec_list + ['cp', self.thru_local_file,
428
+ self.thru_remote_file]
429
+ t0 = time.time()
430
+ self._WindowedExec(cmd, self.num_iterations, self.concurrency)
431
+ t1 = time.time()
432
+
433
+ time_took = t1 - t0
434
+ total_bytes_copied = self.thru_filesize * self.num_iterations
435
+ bytes_per_second = total_bytes_copied / time_took
436
+
437
+ self.results['write_throughput']['time_took'] = time_took
438
+ self.results['write_throughput']['total_bytes_copied'] = total_bytes_copied
439
+ self.results['write_throughput']['bytes_per_second'] = bytes_per_second
440
+
441
+ def _GetDiskCounters(self):
442
+ """Retrieves disk I/O statistics for all disks.
443
+
444
+ Adapted from the psutil module's psutil._pslinux.disk_io_counters:
445
+ http://code.google.com/p/psutil/source/browse/trunk/psutil/_pslinux.py
446
+
447
+ Originally distributed under under a BSD license.
448
+ Original Copyright (c) 2009, Jay Loden, Dave Daeschler, Giampaolo Rodola.
449
+
450
+ Returns:
451
+ A dictionary containing disk names mapped to the disk counters from
452
+ /disk/diskstats.
453
+ """
454
+ # iostat documentation states that sectors are equivalent with blocks and
455
+ # have a size of 512 bytes since 2.4 kernels. This value is needed to
456
+ # calculate the amount of disk I/O in bytes.
457
+ sector_size = 512
458
+
459
+ partitions = []
460
+ with open('/proc/partitions', 'r') as f:
461
+ lines = f.readlines()[2:]
462
+ for line in lines:
463
+ _, _, _, name = line.split()
464
+ if name[-1].isdigit():
465
+ partitions.append(name)
466
+
467
+ retdict = {}
468
+ with open('/proc/diskstats', 'r') as f:
469
+ for line in f:
470
+ values = line.split()[:11]
471
+ _, _, name, reads, _, rbytes, rtime, writes, _, wbytes, wtime = values
472
+ if name in partitions:
473
+ rbytes = int(rbytes) * sector_size
474
+ wbytes = int(wbytes) * sector_size
475
+ reads = int(reads)
476
+ writes = int(writes)
477
+ rtime = int(rtime)
478
+ wtime = int(wtime)
479
+ retdict[name] = (reads, writes, rbytes, wbytes, rtime, wtime)
480
+ return retdict
481
+
482
+ def _GetTcpStats(self):
483
+ """Tries to parse out TCP packet information from netstat output.
484
+
485
+ Returns:
486
+ A dictionary containing TCP information
487
+ """
488
+ # netstat return code is non-zero for -s on Linux, so don't raise on error.
489
+ netstat_output = self._Exec(['netstat', '-s'], return_output=True,
490
+ raise_on_error=False)
491
+ netstat_output = netstat_output.strip().lower()
492
+ found_tcp = False
493
+ tcp_retransmit = None
494
+ tcp_received = None
495
+ tcp_sent = None
496
+ for line in netstat_output.split('\n'):
497
+ # Header for TCP section is "Tcp:" in Linux/Mac and
498
+ # "TCP Statistics for" in Windows.
499
+ if 'tcp:' in line or 'tcp statistics' in line:
500
+ found_tcp = True
501
+
502
+ # Linux == "segments retransmited" (sic), Mac == "retransmit timeouts"
503
+ # Windows == "segments retransmitted".
504
+ if (found_tcp and tcp_retransmit is None and
505
+ ('segments retransmited' in line or 'retransmit timeouts' in line or
506
+ 'segments retransmitted' in line)):
507
+ tcp_retransmit = ''.join(c for c in line if c in string.digits)
508
+
509
+ # Linux+Windows == "segments received", Mac == "packets received".
510
+ if (found_tcp and tcp_received is None and
511
+ ('segments received' in line or 'packets received' in line)):
512
+ tcp_received = ''.join(c for c in line if c in string.digits)
513
+
514
+ # Linux == "segments send out" (sic), Mac+Windows == "packets sent".
515
+ if (found_tcp and tcp_sent is None and
516
+ ('segments send out' in line or 'packets sent' in line or
517
+ 'segments sent' in line)):
518
+ tcp_sent = ''.join(c for c in line if c in string.digits)
519
+
520
+ result = {}
521
+ try:
522
+ result['tcp_retransmit'] = int(tcp_retransmit)
523
+ result['tcp_received'] = int(tcp_received)
524
+ result['tcp_sent'] = int(tcp_sent)
525
+ except (ValueError, TypeError):
526
+ result['tcp_retransmit'] = None
527
+ result['tcp_received'] = None
528
+ result['tcp_sent'] = None
529
+
530
+ return result
531
+
532
+ def _CollectSysInfo(self):
533
+ """Collects system information."""
534
+ sysinfo = {}
535
+
536
+ # Get the local IP address from socket lib.
537
+ sysinfo['ip_address'] = socket.gethostbyname(socket.gethostname())
538
+ # Record the temporary directory used since it can affect performance, e.g.
539
+ # when on a networked filesystem.
540
+ sysinfo['tempdir'] = tempfile.gettempdir()
541
+
542
+ # Produces an RFC 2822 compliant GMT timestamp.
543
+ sysinfo['gmt_timestamp'] = time.strftime('%a, %d %b %Y %H:%M:%S +0000',
544
+ time.gmtime())
545
+
546
+ # Execute a CNAME lookup on Google DNS to find what Google server
547
+ # it's routing to.
548
+ cmd = ['nslookup', '-type=CNAME', self.GOOGLE_API_HOST]
549
+ nslookup_cname_output = self._Exec(cmd, return_output=True)
550
+ m = re.search(r' = (?P<googserv>[^.]+)\.', nslookup_cname_output)
551
+ sysinfo['googserv_route'] = m.group('googserv') if m else None
552
+
553
+ # Look up IP addresses for Google Server.
554
+ (hostname, aliaslist, ipaddrlist) = socket.gethostbyname_ex(
555
+ self.GOOGLE_API_HOST)
556
+ sysinfo['googserv_ips'] = ipaddrlist
557
+
558
+ # Reverse lookup the hostnames for the Google Server IPs.
559
+ sysinfo['googserv_hostnames'] = []
560
+ for googserv_ip in ipaddrlist:
561
+ (hostname, aliaslist, ipaddrlist) = socket.gethostbyaddr(googserv_ip)
562
+ sysinfo['googserv_hostnames'].append(hostname)
563
+
564
+ # Query o-o to find out what the Google DNS thinks is the user's IP.
565
+ cmd = ['nslookup', '-type=TXT', 'o-o.myaddr.google.com.']
566
+ nslookup_txt_output = self._Exec(cmd, return_output=True)
567
+ m = re.search(r'text\s+=\s+"(?P<dnsip>[\.\d]+)"', nslookup_txt_output)
568
+ sysinfo['dns_o-o_ip'] = m.group('dnsip') if m else None
569
+
570
+ # Try and find the number of CPUs in the system if available.
571
+ try:
572
+ sysinfo['cpu_count'] = multiprocessing.cpu_count()
573
+ except NotImplementedError:
574
+ sysinfo['cpu_count'] = None
575
+
576
+ # For *nix platforms, obtain the CPU load.
577
+ try:
578
+ sysinfo['load_avg'] = list(os.getloadavg())
579
+ except (AttributeError, OSError):
580
+ sysinfo['load_avg'] = None
581
+
582
+ # Try and collect memory information from /proc/meminfo if possible.
583
+ mem_total = None
584
+ mem_free = None
585
+ mem_buffers = None
586
+ mem_cached = None
587
+
588
+ try:
589
+ with open('/proc/meminfo', 'r') as f:
590
+ for line in f:
591
+ if line.startswith('MemTotal'):
592
+ mem_total = (int(''.join(c for c in line if c in string.digits))
593
+ * 1000)
594
+ elif line.startswith('MemFree'):
595
+ mem_free = (int(''.join(c for c in line if c in string.digits))
596
+ * 1000)
597
+ elif line.startswith('Buffers'):
598
+ mem_buffers = (int(''.join(c for c in line if c in string.digits))
599
+ * 1000)
600
+ elif line.startswith('Cached'):
601
+ mem_cached = (int(''.join(c for c in line if c in string.digits))
602
+ * 1000)
603
+ except (IOError, ValueError):
604
+ pass
605
+
606
+ sysinfo['meminfo'] = {'mem_total': mem_total,
607
+ 'mem_free': mem_free,
608
+ 'mem_buffers': mem_buffers,
609
+ 'mem_cached': mem_cached}
610
+
611
+ # Get configuration attributes from config module.
612
+ sysinfo['gsutil_config'] = {}
613
+ for attr in dir(config):
614
+ attr_value = getattr(config, attr)
615
+ # Filter out multiline strings that are not useful.
616
+ if attr.isupper() and not (isinstance(attr_value, basestring) and
617
+ '\n' in attr_value):
618
+ sysinfo['gsutil_config'][attr] = attr_value
619
+
620
+ self.results['sysinfo'] = sysinfo
621
+
622
+ def _DisplayStats(self, trials):
623
+ """Prints out mean, standard deviation, median, and 90th percentile."""
624
+ n = len(trials)
625
+ mean = float(sum(trials)) / n
626
+ stdev = math.sqrt(sum((x - mean)**2 for x in trials) / n)
627
+
628
+ print str(n).rjust(6), '',
629
+ print ('%.1f' % (mean * 1000)).rjust(9), '',
630
+ print ('%.1f' % (stdev * 1000)).rjust(12), '',
631
+ print ('%.1f' % (Percentile(trials, 0.5) * 1000)).rjust(11), '',
632
+ print ('%.1f' % (Percentile(trials, 0.9) * 1000)).rjust(11), ''
633
+
634
+ def _DisplayResults(self):
635
+ """Displays results collected from diagnostic run."""
636
+ print
637
+ print '=' * 78
638
+ print 'DIAGNOSTIC RESULTS'.center(78)
639
+ print '=' * 78
640
+
641
+ if 'latency' in self.results:
642
+ print
643
+ print '-' * 78
644
+ print 'Latency'.center(78)
645
+ print '-' * 78
646
+ print ('Operation Size Trials Mean (ms) Std Dev (ms) '
647
+ 'Median (ms) 90th % (ms)')
648
+ print ('========= ========= ====== ========= ============ '
649
+ '=========== ===========')
650
+ for key in sorted(self.results['latency']):
651
+ trials = sorted(self.results['latency'][key])
652
+ op, numbytes = key.split('_')
653
+ numbytes = int(numbytes)
654
+ if op == 'METADATA':
655
+ print 'Metadata'.rjust(9), '',
656
+ print MakeHumanReadable(numbytes).rjust(9), '',
657
+ self._DisplayStats(trials)
658
+ if op == 'DOWNLOAD':
659
+ print 'Download'.rjust(9), '',
660
+ print MakeHumanReadable(numbytes).rjust(9), '',
661
+ self._DisplayStats(trials)
662
+ if op == 'UPLOAD':
663
+ print 'Upload'.rjust(9), '',
664
+ print MakeHumanReadable(numbytes).rjust(9), '',
665
+ self._DisplayStats(trials)
666
+ if op == 'DELETE':
667
+ print 'Delete'.rjust(9), '',
668
+ print MakeHumanReadable(numbytes).rjust(9), '',
669
+ self._DisplayStats(trials)
670
+
671
+ if 'write_throughput' in self.results:
672
+ print
673
+ print '-' * 78
674
+ print 'Write Throughput'.center(78)
675
+ print '-' * 78
676
+ write_thru = self.results['write_throughput']
677
+ print 'Copied a %s file %d times for a total transfer size of %s.' % (
678
+ MakeHumanReadable(write_thru['file_size']),
679
+ write_thru['num_copies'],
680
+ MakeHumanReadable(write_thru['total_bytes_copied']))
681
+ print 'Write throughput: %s/s.' % (
682
+ MakeBitsHumanReadable(write_thru['bytes_per_second'] * 8))
683
+
684
+ if 'read_throughput' in self.results:
685
+ print
686
+ print '-' * 78
687
+ print 'Read Throughput'.center(78)
688
+ print '-' * 78
689
+ read_thru = self.results['read_throughput']
690
+ print 'Copied a %s file %d times for a total transfer size of %s.' % (
691
+ MakeHumanReadable(read_thru['file_size']),
692
+ read_thru['num_times'],
693
+ MakeHumanReadable(read_thru['total_bytes_copied']))
694
+ print 'Read throughput: %s/s.' % (
695
+ MakeBitsHumanReadable(read_thru['bytes_per_second'] * 8))
696
+
697
+ if 'sysinfo' in self.results:
698
+ print
699
+ print '-' * 78
700
+ print 'System Information'.center(78)
701
+ print '-' * 78
702
+ info = self.results['sysinfo']
703
+ print 'IP Address: \n %s' % info['ip_address']
704
+ print 'Temporary Directory: \n %s' % info['tempdir']
705
+ print 'Bucket URI: \n %s' % self.results['bucket_uri']
706
+
707
+ if 'gmt_timestamp' in info:
708
+ ts_string = info['gmt_timestamp']
709
+ timetuple = None
710
+ try:
711
+ # Convert RFC 2822 string to Linux timestamp.
712
+ timetuple = time.strptime(ts_string, '%a, %d %b %Y %H:%M:%S +0000')
713
+ except ValueError:
714
+ pass
715
+
716
+ if timetuple:
717
+ # Converts the GMT time tuple to local Linux timestamp.
718
+ localtime = calendar.timegm(timetuple)
719
+ localdt = datetime.datetime.fromtimestamp(localtime)
720
+ print 'Measurement time: \n %s' % localdt.strftime(
721
+ '%Y-%m-%d %I-%M-%S %p %Z')
722
+
723
+ print 'Google Server: \n %s' % info['googserv_route']
724
+ print ('Google Server IP Addresses: \n %s' %
725
+ ('\n '.join(info['googserv_ips'])))
726
+ print ('Google Server Hostnames: \n %s' %
727
+ ('\n '.join(info['googserv_hostnames'])))
728
+ print 'Google DNS thinks your IP is: \n %s' % info['dns_o-o_ip']
729
+ print 'CPU Count: \n %s' % info['cpu_count']
730
+ print 'CPU Load Average: \n %s' % info['load_avg']
731
+ try:
732
+ print ('Total Memory: \n %s' %
733
+ MakeHumanReadable(info['meminfo']['mem_total']))
734
+ # Free memory is really MemFree + Buffers + Cached.
735
+ print 'Free Memory: \n %s' % MakeHumanReadable(
736
+ info['meminfo']['mem_free'] +
737
+ info['meminfo']['mem_buffers'] +
738
+ info['meminfo']['mem_cached'])
739
+ except TypeError:
740
+ pass
741
+
742
+ netstat_after = info['netstat_end']
743
+ netstat_before = info['netstat_start']
744
+ for tcp_type in ('sent', 'received', 'retransmit'):
745
+ try:
746
+ delta = (netstat_after['tcp_%s' % tcp_type] -
747
+ netstat_before['tcp_%s' % tcp_type])
748
+ print 'TCP segments %s during test:\n %d' % (tcp_type, delta)
749
+ except TypeError:
750
+ pass
751
+
752
+ if 'disk_counters_end' in info and 'disk_counters_start' in info:
753
+ print 'Disk Counter Deltas:\n',
754
+ disk_after = info['disk_counters_end']
755
+ disk_before = info['disk_counters_start']
756
+ print '', 'disk'.rjust(6),
757
+ for colname in ['reads', 'writes', 'rbytes', 'wbytes', 'rtime',
758
+ 'wtime']:
759
+ print colname.rjust(8),
760
+ print
761
+ for diskname in sorted(disk_after):
762
+ before = disk_before[diskname]
763
+ after = disk_after[diskname]
764
+ (reads1, writes1, rbytes1, wbytes1, rtime1, wtime1) = before
765
+ (reads2, writes2, rbytes2, wbytes2, rtime2, wtime2) = after
766
+ print '', diskname.rjust(6),
767
+ deltas = [reads2-reads1, writes2-writes1, rbytes2-rbytes1,
768
+ wbytes2-wbytes1, rtime2-rtime1, wtime2-wtime1]
769
+ for delta in deltas:
770
+ print str(delta).rjust(8),
771
+ print
772
+
773
+ if self.output_file:
774
+ with open(self.output_file, 'w') as f:
775
+ json.dump(self.results, f, indent=2)
776
+ print
777
+ print "Output file written to '%s'." % self.output_file
778
+
779
+ print
780
+
781
+ def _ParsePositiveInteger(self, val, msg):
782
+ """Tries to convert val argument to a positive integer.
783
+
784
+ Args:
785
+ val: The value (as a string) to convert to a positive integer.
786
+ msg: The error message to place in the CommandException on an error.
787
+
788
+ Returns:
789
+ A valid positive integer.
790
+
791
+ Raises:
792
+ CommandException: If the supplied value is not a valid positive integer.
793
+ """
794
+ try:
795
+ val = int(val)
796
+ if val < 1:
797
+ raise CommandException(msg)
798
+ return val
799
+ except ValueError:
800
+ raise CommandException(msg)
801
+
802
+ def _ParseArgs(self):
803
+ """Parses arguments for perfdiag command."""
804
+ # From -n.
805
+ self.num_iterations = 5
806
+ # From -c.
807
+ self.concurrency = 1
808
+ # From -s.
809
+ self.thru_filesize = 1048576
810
+ # From -t.
811
+ self.diag_tests = self.ALL_DIAG_TESTS
812
+ # From -o.
813
+ self.output_file = None
814
+ # From -i.
815
+ self.input_file = None
816
+
817
+ if self.sub_opts:
818
+ for o, a in self.sub_opts:
819
+ if o == '-n':
820
+ self.num_iterations = self._ParsePositiveInteger(
821
+ a, 'The -n parameter must be a positive integer.')
822
+ if o == '-c':
823
+ self.concurrency = self._ParsePositiveInteger(
824
+ a, 'The -c parameter must be a positive integer.')
825
+ if o == '-s':
826
+ self.thru_filesize = self._ParsePositiveInteger(
827
+ a, 'The -s parameter must be a positive integer.')
828
+ if o == '-t':
829
+ self.diag_tests = []
830
+ for test_name in a.strip().split(','):
831
+ if test_name.lower() not in self.ALL_DIAG_TESTS:
832
+ raise CommandException("List of test names (-t) contains invalid "
833
+ "test name '%s'." % test_name)
834
+ self.diag_tests.append(test_name)
835
+ if o == '-o':
836
+ self.output_file = os.path.abspath(a)
837
+ if o == '-i':
838
+ self.input_file = os.path.abspath(a)
839
+ if not os.path.isfile(self.input_file):
840
+ raise CommandException("Invalid input file (-i): '%s'." % a)
841
+ try:
842
+ with open(self.input_file, 'r') as f:
843
+ self.results = json.load(f)
844
+ print "Read input file: '%s'." % self.input_file
845
+ except ValueError:
846
+ raise CommandException("Could not decode input file (-i): '%s'." %
847
+ a)
848
+ return
849
+
850
+ if not self.args:
851
+ raise CommandException('Wrong number of arguments for "perfdiag" '
852
+ 'command.')
853
+ self.bucket_uri = self.suri_builder.StorageUri(self.args[0])
854
+ if not self.bucket_uri.names_bucket():
855
+ raise CommandException('The perfdiag command requires a URI that '
856
+ 'specifies a bucket.\n"%s" is not '
857
+ 'valid.' % self.bucket_uri)
858
+ self.bucket = self.bucket_uri.get_bucket()
859
+
860
+ # Command entry point.
861
+ def RunCommand(self):
862
+ """Called by gsutil when the command is being invoked."""
863
+ self._ParseArgs()
864
+
865
+ if self.input_file:
866
+ self._DisplayResults()
867
+ return 0
868
+
869
+ print 'Number of iterations to run: %d' % self.num_iterations
870
+ print 'Base bucket URI: %s' % self.bucket_uri
871
+ print 'Concurrency level: %d' % self.concurrency
872
+ print 'Throughput file size: %s' % MakeHumanReadable(self.thru_filesize)
873
+ print 'Diagnostics to run: %s' % (', '.join(self.diag_tests))
874
+
875
+ try:
876
+ self._SetUp()
877
+
878
+ # Collect generic system info.
879
+ self._CollectSysInfo()
880
+ # Collect netstat info and disk counters before tests (and again later).
881
+ self.results['sysinfo']['netstat_start'] = self._GetTcpStats()
882
+ if IS_LINUX:
883
+ self.results['sysinfo']['disk_counters_start'] = self._GetDiskCounters()
884
+ # Record bucket URI.
885
+ self.results['bucket_uri'] = str(self.bucket_uri)
886
+
887
+ if 'lat' in self.diag_tests:
888
+ self._RunLatencyTests()
889
+ if 'rthru' in self.diag_tests:
890
+ self._RunReadThruTests()
891
+ if 'wthru' in self.diag_tests:
892
+ self._RunWriteThruTests()
893
+
894
+ # Collect netstat info and disk counters after tests.
895
+ self.results['sysinfo']['netstat_end'] = self._GetTcpStats()
896
+ if IS_LINUX:
897
+ self.results['sysinfo']['disk_counters_end'] = self._GetDiskCounters()
898
+
899
+ self._DisplayResults()
900
+ finally:
901
+ self._TearDown()
902
+
903
+ return 0