gcloud 0.0.7 → 0.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (998) hide show
  1. checksums.yaml +7 -0
  2. data/{packages/gcutil-1.7.1/lib/google_api_python_client/apiclient/ext/__init__.py → .gemtest} +0 -0
  3. data/.rubocop.yml +17 -0
  4. data/CHANGELOG.md +5 -0
  5. data/CONTRIBUTING.md +93 -0
  6. data/LICENSE +201 -674
  7. data/Manifest.txt +66 -0
  8. data/README.md +110 -12
  9. data/Rakefile +34 -3
  10. data/gcloud.gemspec +48 -17
  11. data/lib/gcloud.rb +8 -0
  12. data/lib/gcloud/backoff.rb +115 -0
  13. data/lib/gcloud/credentials.rb +124 -0
  14. data/lib/gcloud/datastore.rb +66 -0
  15. data/lib/gcloud/datastore/connection.rb +177 -0
  16. data/lib/gcloud/datastore/credentials.rb +41 -0
  17. data/lib/gcloud/datastore/dataset.rb +259 -0
  18. data/lib/gcloud/datastore/dataset/lookup_results.rb +60 -0
  19. data/lib/gcloud/datastore/dataset/query_results.rb +87 -0
  20. data/lib/gcloud/datastore/entity.rb +257 -0
  21. data/lib/gcloud/datastore/errors.rb +65 -0
  22. data/lib/gcloud/datastore/key.rb +151 -0
  23. data/lib/gcloud/datastore/properties.rb +102 -0
  24. data/lib/gcloud/datastore/proto.rb +253 -0
  25. data/lib/gcloud/datastore/query.rb +189 -0
  26. data/lib/gcloud/datastore/transaction.rb +116 -0
  27. data/lib/gcloud/proto/datastore_v1.pb.rb +377 -0
  28. data/lib/gcloud/storage.rb +78 -0
  29. data/lib/gcloud/storage/bucket.rb +254 -0
  30. data/lib/gcloud/storage/bucket/acl.rb +311 -0
  31. data/lib/gcloud/storage/bucket/list.rb +45 -0
  32. data/lib/gcloud/storage/connection.rb +358 -0
  33. data/{packages/gsutil/gslib/commands/__init__.py → lib/gcloud/storage/credentials.rb} +13 -2
  34. data/lib/gcloud/storage/errors.rb +78 -0
  35. data/lib/gcloud/storage/file.rb +287 -0
  36. data/lib/gcloud/storage/file/acl.rb +172 -0
  37. data/lib/gcloud/storage/file/list.rb +52 -0
  38. data/lib/gcloud/storage/file/verifier.rb +65 -0
  39. data/lib/gcloud/storage/project.rb +119 -0
  40. data/lib/gcloud/version.rb +5 -0
  41. data/{packages/gsutil/gslib/tests/testcase/__init__.py → rakelib/console.rake} +14 -4
  42. data/{packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/version.py → rakelib/manifest.rake} +11 -4
  43. data/{packages/gsutil/gslib/addlhelp/__init__.py → rakelib/proto.rake} +4 -2
  44. data/{packages/gsutil/gslib/tests/__init__.py → rakelib/rubocop.rake} +4 -2
  45. data/rakelib/test.rake +144 -0
  46. data/test/gcloud/datastore/proto/test_cursor.rb +36 -0
  47. data/test/gcloud/datastore/proto/test_direction.rb +60 -0
  48. data/test/gcloud/datastore/proto/test_operator.rb +76 -0
  49. data/test/gcloud/datastore/proto/test_value.rb +231 -0
  50. data/test/gcloud/datastore/test_connection.rb +93 -0
  51. data/test/gcloud/datastore/test_credentials.rb +38 -0
  52. data/test/gcloud/datastore/test_dataset.rb +413 -0
  53. data/test/gcloud/datastore/test_entity.rb +161 -0
  54. data/test/gcloud/datastore/test_entity_exclude.rb +225 -0
  55. data/test/gcloud/datastore/test_key.rb +189 -0
  56. data/test/gcloud/datastore/test_query.rb +271 -0
  57. data/test/gcloud/datastore/test_transaction.rb +121 -0
  58. data/test/gcloud/storage/test_backoff.rb +127 -0
  59. data/test/gcloud/storage/test_bucket.rb +270 -0
  60. data/test/gcloud/storage/test_bucket_acl.rb +253 -0
  61. data/test/gcloud/storage/test_default_acl.rb +256 -0
  62. data/test/gcloud/storage/test_file.rb +221 -0
  63. data/test/gcloud/storage/test_file_acl.rb +367 -0
  64. data/test/gcloud/storage/test_project.rb +180 -0
  65. data/{packages/gsutil/gslib/no_op_auth_plugin.py → test/gcloud/storage/test_storage.rb} +14 -15
  66. data/test/gcloud/storage/test_verifier.rb +62 -0
  67. data/test/gcloud/test_version.rb +8 -0
  68. data/test/helper.rb +91 -0
  69. metadata +259 -1020
  70. data.tar.gz.sig +0 -2
  71. data/CHANGELOG +0 -14
  72. data/Makefile +0 -13
  73. data/Manifest +0 -937
  74. data/TODO +0 -2
  75. data/bin/gcloud +0 -7
  76. data/bin/gcutil-symlink +0 -53
  77. data/bin/gsutil-symlink +0 -377
  78. data/init.rb +0 -3
  79. data/packages/README +0 -6
  80. data/packages/gcutil-1.7.1.tar.gz +0 -0
  81. data/packages/gcutil-1.7.1/CHANGELOG +0 -197
  82. data/packages/gcutil-1.7.1/LICENSE +0 -202
  83. data/packages/gcutil-1.7.1/VERSION +0 -1
  84. data/packages/gcutil-1.7.1/gcutil +0 -53
  85. data/packages/gcutil-1.7.1/lib/google_api_python_client/LICENSE +0 -23
  86. data/packages/gcutil-1.7.1/lib/google_api_python_client/apiclient/__init__.py +0 -1
  87. data/packages/gcutil-1.7.1/lib/google_api_python_client/apiclient/__init__.pyc +0 -0
  88. data/packages/gcutil-1.7.1/lib/google_api_python_client/apiclient/discovery.py +0 -743
  89. data/packages/gcutil-1.7.1/lib/google_api_python_client/apiclient/discovery.pyc +0 -0
  90. data/packages/gcutil-1.7.1/lib/google_api_python_client/apiclient/errors.py +0 -123
  91. data/packages/gcutil-1.7.1/lib/google_api_python_client/apiclient/errors.pyc +0 -0
  92. data/packages/gcutil-1.7.1/lib/google_api_python_client/apiclient/http.py +0 -1443
  93. data/packages/gcutil-1.7.1/lib/google_api_python_client/apiclient/http.pyc +0 -0
  94. data/packages/gcutil-1.7.1/lib/google_api_python_client/apiclient/mimeparse.py +0 -172
  95. data/packages/gcutil-1.7.1/lib/google_api_python_client/apiclient/mimeparse.pyc +0 -0
  96. data/packages/gcutil-1.7.1/lib/google_api_python_client/apiclient/model.py +0 -385
  97. data/packages/gcutil-1.7.1/lib/google_api_python_client/apiclient/model.pyc +0 -0
  98. data/packages/gcutil-1.7.1/lib/google_api_python_client/apiclient/schema.py +0 -303
  99. data/packages/gcutil-1.7.1/lib/google_api_python_client/apiclient/schema.pyc +0 -0
  100. data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/__init__.py +0 -1
  101. data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/__init__.pyc +0 -0
  102. data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/anyjson.py +0 -32
  103. data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/anyjson.pyc +0 -0
  104. data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/appengine.py +0 -528
  105. data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/client.py +0 -1139
  106. data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/client.pyc +0 -0
  107. data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/clientsecrets.py +0 -105
  108. data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/clientsecrets.pyc +0 -0
  109. data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/crypt.py +0 -244
  110. data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/crypt.pyc +0 -0
  111. data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/django_orm.py +0 -124
  112. data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/file.py +0 -107
  113. data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/locked_file.py +0 -343
  114. data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/locked_file.pyc +0 -0
  115. data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/multistore_file.py +0 -379
  116. data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/multistore_file.pyc +0 -0
  117. data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/tools.py +0 -174
  118. data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/tools.pyc +0 -0
  119. data/packages/gcutil-1.7.1/lib/google_api_python_client/uritemplate/__init__.py +0 -147
  120. data/packages/gcutil-1.7.1/lib/google_api_python_client/uritemplate/__init__.pyc +0 -0
  121. data/packages/gcutil-1.7.1/lib/google_apputils/LICENSE +0 -202
  122. data/packages/gcutil-1.7.1/lib/google_apputils/google/__init__.py +0 -3
  123. data/packages/gcutil-1.7.1/lib/google_apputils/google/__init__.pyc +0 -0
  124. data/packages/gcutil-1.7.1/lib/google_apputils/google/apputils/__init__.py +0 -3
  125. data/packages/gcutil-1.7.1/lib/google_apputils/google/apputils/__init__.pyc +0 -0
  126. data/packages/gcutil-1.7.1/lib/google_apputils/google/apputils/app.py +0 -356
  127. data/packages/gcutil-1.7.1/lib/google_apputils/google/apputils/app.pyc +0 -0
  128. data/packages/gcutil-1.7.1/lib/google_apputils/google/apputils/appcommands.py +0 -783
  129. data/packages/gcutil-1.7.1/lib/google_apputils/google/apputils/appcommands.pyc +0 -0
  130. data/packages/gcutil-1.7.1/lib/google_apputils/google/apputils/basetest.py +0 -1260
  131. data/packages/gcutil-1.7.1/lib/google_apputils/google/apputils/datelib.py +0 -421
  132. data/packages/gcutil-1.7.1/lib/google_apputils/google/apputils/debug.py +0 -60
  133. data/packages/gcutil-1.7.1/lib/google_apputils/google/apputils/file_util.py +0 -181
  134. data/packages/gcutil-1.7.1/lib/google_apputils/google/apputils/resources.py +0 -67
  135. data/packages/gcutil-1.7.1/lib/google_apputils/google/apputils/run_script_module.py +0 -217
  136. data/packages/gcutil-1.7.1/lib/google_apputils/google/apputils/setup_command.py +0 -159
  137. data/packages/gcutil-1.7.1/lib/google_apputils/google/apputils/shellutil.py +0 -49
  138. data/packages/gcutil-1.7.1/lib/google_apputils/google/apputils/stopwatch.py +0 -204
  139. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/__init__.py +0 -0
  140. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/__init__.pyc +0 -0
  141. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/auth_helper.py +0 -140
  142. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/auth_helper.pyc +0 -0
  143. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/auth_helper_test.py +0 -149
  144. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/auto_auth.py +0 -130
  145. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/auto_auth.pyc +0 -0
  146. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/auto_auth_test.py +0 -75
  147. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/basic_cmds.py +0 -128
  148. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/basic_cmds.pyc +0 -0
  149. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/basic_cmds_test.py +0 -111
  150. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/command_base.py +0 -1808
  151. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/command_base.pyc +0 -0
  152. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/command_base_test.py +0 -1651
  153. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/compute/v1beta13.json +0 -2851
  154. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/compute/v1beta14.json +0 -3361
  155. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/disk_cmds.py +0 -342
  156. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/disk_cmds.pyc +0 -0
  157. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/disk_cmds_test.py +0 -474
  158. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/firewall_cmds.py +0 -344
  159. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/firewall_cmds.pyc +0 -0
  160. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/firewall_cmds_test.py +0 -231
  161. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/flags_cache.py +0 -274
  162. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/flags_cache.pyc +0 -0
  163. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/gcutil +0 -89
  164. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/gcutil_logging.py +0 -69
  165. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/gcutil_logging.pyc +0 -0
  166. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/image_cmds.py +0 -262
  167. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/image_cmds.pyc +0 -0
  168. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/image_cmds_test.py +0 -172
  169. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/instance_cmds.py +0 -1506
  170. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/instance_cmds.pyc +0 -0
  171. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/instance_cmds_test.py +0 -1904
  172. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/kernel_cmds.py +0 -91
  173. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/kernel_cmds.pyc +0 -0
  174. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/kernel_cmds_test.py +0 -56
  175. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/machine_type_cmds.py +0 -106
  176. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/machine_type_cmds.pyc +0 -0
  177. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/machine_type_cmds_test.py +0 -59
  178. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/metadata.py +0 -96
  179. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/metadata.pyc +0 -0
  180. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/metadata_lib.py +0 -357
  181. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/metadata_lib.pyc +0 -0
  182. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/metadata_test.py +0 -84
  183. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/mock_api.py +0 -420
  184. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/mock_metadata.py +0 -58
  185. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/move_cmds.py +0 -824
  186. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/move_cmds.pyc +0 -0
  187. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/move_cmds_test.py +0 -307
  188. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/network_cmds.py +0 -178
  189. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/network_cmds.pyc +0 -0
  190. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/network_cmds_test.py +0 -133
  191. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/operation_cmds.py +0 -181
  192. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/operation_cmds.pyc +0 -0
  193. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/operation_cmds_test.py +0 -196
  194. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/path_initializer.py +0 -38
  195. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/path_initializer.pyc +0 -0
  196. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/project_cmds.py +0 -173
  197. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/project_cmds.pyc +0 -0
  198. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/project_cmds_test.py +0 -111
  199. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/scopes.py +0 -61
  200. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/scopes.pyc +0 -0
  201. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/scopes_test.py +0 -50
  202. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/snapshot_cmds.py +0 -276
  203. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/snapshot_cmds.pyc +0 -0
  204. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/snapshot_cmds_test.py +0 -260
  205. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/ssh_keys.py +0 -266
  206. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/ssh_keys.pyc +0 -0
  207. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/ssh_keys_test.py +0 -128
  208. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/table_formatter.py +0 -563
  209. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/table_formatter.pyc +0 -0
  210. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/thread_pool.py +0 -188
  211. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/thread_pool.pyc +0 -0
  212. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/thread_pool_test.py +0 -88
  213. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/utils.py +0 -208
  214. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/utils.pyc +0 -0
  215. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/utils_test.py +0 -193
  216. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/version.pyc +0 -0
  217. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/version_checker.py +0 -246
  218. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/version_checker.pyc +0 -0
  219. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/version_checker_test.py +0 -271
  220. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/zone_cmds.py +0 -151
  221. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/zone_cmds.pyc +0 -0
  222. data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/zone_cmds_test.py +0 -60
  223. data/packages/gcutil-1.7.1/lib/httplib2/LICENSE +0 -21
  224. data/packages/gcutil-1.7.1/lib/httplib2/httplib2/__init__.py +0 -1630
  225. data/packages/gcutil-1.7.1/lib/httplib2/httplib2/__init__.pyc +0 -0
  226. data/packages/gcutil-1.7.1/lib/httplib2/httplib2/cacerts.txt +0 -714
  227. data/packages/gcutil-1.7.1/lib/httplib2/httplib2/iri2uri.py +0 -110
  228. data/packages/gcutil-1.7.1/lib/httplib2/httplib2/iri2uri.pyc +0 -0
  229. data/packages/gcutil-1.7.1/lib/httplib2/httplib2/socks.py +0 -438
  230. data/packages/gcutil-1.7.1/lib/httplib2/httplib2/socks.pyc +0 -0
  231. data/packages/gcutil-1.7.1/lib/iso8601/LICENSE +0 -20
  232. data/packages/gcutil-1.7.1/lib/iso8601/iso8601/__init__.py +0 -1
  233. data/packages/gcutil-1.7.1/lib/iso8601/iso8601/__init__.pyc +0 -0
  234. data/packages/gcutil-1.7.1/lib/iso8601/iso8601/iso8601.py +0 -102
  235. data/packages/gcutil-1.7.1/lib/iso8601/iso8601/iso8601.pyc +0 -0
  236. data/packages/gcutil-1.7.1/lib/iso8601/iso8601/test_iso8601.py +0 -111
  237. data/packages/gcutil-1.7.1/lib/python_gflags/AUTHORS +0 -2
  238. data/packages/gcutil-1.7.1/lib/python_gflags/LICENSE +0 -28
  239. data/packages/gcutil-1.7.1/lib/python_gflags/gflags.py +0 -2862
  240. data/packages/gcutil-1.7.1/lib/python_gflags/gflags.pyc +0 -0
  241. data/packages/gcutil-1.7.1/lib/python_gflags/gflags2man.py +0 -544
  242. data/packages/gcutil-1.7.1/lib/python_gflags/gflags_validators.py +0 -187
  243. data/packages/gcutil-1.7.1/lib/python_gflags/gflags_validators.pyc +0 -0
  244. data/packages/gsutil.tar.gz +0 -0
  245. data/packages/gsutil/CHECKSUM +0 -1
  246. data/packages/gsutil/COPYING +0 -202
  247. data/packages/gsutil/LICENSE.third_party +0 -295
  248. data/packages/gsutil/MANIFEST.in +0 -5
  249. data/packages/gsutil/README +0 -38
  250. data/packages/gsutil/README.pkg +0 -49
  251. data/packages/gsutil/ReleaseNotes.txt +0 -780
  252. data/packages/gsutil/VERSION +0 -1
  253. data/packages/gsutil/boto/Changelog.rst +0 -35
  254. data/packages/gsutil/boto/LICENSE +0 -18
  255. data/packages/gsutil/boto/MANIFEST.in +0 -12
  256. data/packages/gsutil/boto/README.rst +0 -163
  257. data/packages/gsutil/boto/bin/asadmin +0 -290
  258. data/packages/gsutil/boto/bin/bundle_image +0 -27
  259. data/packages/gsutil/boto/bin/cfadmin +0 -108
  260. data/packages/gsutil/boto/bin/cq +0 -89
  261. data/packages/gsutil/boto/bin/cwutil +0 -140
  262. data/packages/gsutil/boto/bin/elbadmin +0 -284
  263. data/packages/gsutil/boto/bin/fetch_file +0 -43
  264. data/packages/gsutil/boto/bin/glacier +0 -154
  265. data/packages/gsutil/boto/bin/instance_events +0 -145
  266. data/packages/gsutil/boto/bin/kill_instance +0 -35
  267. data/packages/gsutil/boto/bin/launch_instance +0 -252
  268. data/packages/gsutil/boto/bin/list_instances +0 -90
  269. data/packages/gsutil/boto/bin/lss3 +0 -77
  270. data/packages/gsutil/boto/bin/mturk +0 -465
  271. data/packages/gsutil/boto/bin/pyami_sendmail +0 -52
  272. data/packages/gsutil/boto/bin/route53 +0 -205
  273. data/packages/gsutil/boto/bin/s3put +0 -374
  274. data/packages/gsutil/boto/bin/sdbadmin +0 -194
  275. data/packages/gsutil/boto/bin/taskadmin +0 -116
  276. data/packages/gsutil/boto/boto/__init__.py +0 -793
  277. data/packages/gsutil/boto/boto/__init__.pyc +0 -0
  278. data/packages/gsutil/boto/boto/auth.py +0 -682
  279. data/packages/gsutil/boto/boto/auth.pyc +0 -0
  280. data/packages/gsutil/boto/boto/auth_handler.py +0 -58
  281. data/packages/gsutil/boto/boto/auth_handler.pyc +0 -0
  282. data/packages/gsutil/boto/boto/beanstalk/__init__.py +0 -65
  283. data/packages/gsutil/boto/boto/beanstalk/exception.py +0 -64
  284. data/packages/gsutil/boto/boto/beanstalk/layer1.py +0 -1153
  285. data/packages/gsutil/boto/boto/beanstalk/response.py +0 -703
  286. data/packages/gsutil/boto/boto/beanstalk/wrapper.py +0 -29
  287. data/packages/gsutil/boto/boto/cacerts/__init__.py +0 -22
  288. data/packages/gsutil/boto/boto/cacerts/__init__.pyc +0 -0
  289. data/packages/gsutil/boto/boto/cacerts/cacerts.txt +0 -633
  290. data/packages/gsutil/boto/boto/cloudformation/__init__.py +0 -68
  291. data/packages/gsutil/boto/boto/cloudformation/connection.py +0 -364
  292. data/packages/gsutil/boto/boto/cloudformation/stack.py +0 -360
  293. data/packages/gsutil/boto/boto/cloudformation/template.py +0 -43
  294. data/packages/gsutil/boto/boto/cloudfront/__init__.py +0 -324
  295. data/packages/gsutil/boto/boto/cloudfront/distribution.py +0 -745
  296. data/packages/gsutil/boto/boto/cloudfront/exception.py +0 -26
  297. data/packages/gsutil/boto/boto/cloudfront/identity.py +0 -122
  298. data/packages/gsutil/boto/boto/cloudfront/invalidation.py +0 -216
  299. data/packages/gsutil/boto/boto/cloudfront/logging.py +0 -38
  300. data/packages/gsutil/boto/boto/cloudfront/object.py +0 -48
  301. data/packages/gsutil/boto/boto/cloudfront/origin.py +0 -150
  302. data/packages/gsutil/boto/boto/cloudfront/signers.py +0 -60
  303. data/packages/gsutil/boto/boto/cloudsearch/__init__.py +0 -45
  304. data/packages/gsutil/boto/boto/cloudsearch/document.py +0 -265
  305. data/packages/gsutil/boto/boto/cloudsearch/domain.py +0 -394
  306. data/packages/gsutil/boto/boto/cloudsearch/layer1.py +0 -738
  307. data/packages/gsutil/boto/boto/cloudsearch/layer2.py +0 -67
  308. data/packages/gsutil/boto/boto/cloudsearch/optionstatus.py +0 -248
  309. data/packages/gsutil/boto/boto/cloudsearch/search.py +0 -365
  310. data/packages/gsutil/boto/boto/cloudsearch/sourceattribute.py +0 -75
  311. data/packages/gsutil/boto/boto/compat.py +0 -28
  312. data/packages/gsutil/boto/boto/compat.pyc +0 -0
  313. data/packages/gsutil/boto/boto/connection.py +0 -1081
  314. data/packages/gsutil/boto/boto/connection.pyc +0 -0
  315. data/packages/gsutil/boto/boto/contrib/__init__.py +0 -22
  316. data/packages/gsutil/boto/boto/contrib/ymlmessage.py +0 -52
  317. data/packages/gsutil/boto/boto/core/README +0 -58
  318. data/packages/gsutil/boto/boto/core/__init__.py +0 -23
  319. data/packages/gsutil/boto/boto/core/auth.py +0 -78
  320. data/packages/gsutil/boto/boto/core/credentials.py +0 -154
  321. data/packages/gsutil/boto/boto/core/dictresponse.py +0 -178
  322. data/packages/gsutil/boto/boto/core/service.py +0 -67
  323. data/packages/gsutil/boto/boto/datapipeline/__init__.py +0 -0
  324. data/packages/gsutil/boto/boto/datapipeline/exceptions.py +0 -42
  325. data/packages/gsutil/boto/boto/datapipeline/layer1.py +0 -546
  326. data/packages/gsutil/boto/boto/dynamodb/__init__.py +0 -66
  327. data/packages/gsutil/boto/boto/dynamodb/batch.py +0 -262
  328. data/packages/gsutil/boto/boto/dynamodb/condition.py +0 -170
  329. data/packages/gsutil/boto/boto/dynamodb/exceptions.py +0 -64
  330. data/packages/gsutil/boto/boto/dynamodb/item.py +0 -196
  331. data/packages/gsutil/boto/boto/dynamodb/layer1.py +0 -575
  332. data/packages/gsutil/boto/boto/dynamodb/layer2.py +0 -798
  333. data/packages/gsutil/boto/boto/dynamodb/schema.py +0 -112
  334. data/packages/gsutil/boto/boto/dynamodb/table.py +0 -540
  335. data/packages/gsutil/boto/boto/dynamodb/types.py +0 -326
  336. data/packages/gsutil/boto/boto/ec2/__init__.py +0 -96
  337. data/packages/gsutil/boto/boto/ec2/address.py +0 -103
  338. data/packages/gsutil/boto/boto/ec2/autoscale/__init__.py +0 -781
  339. data/packages/gsutil/boto/boto/ec2/autoscale/activity.py +0 -74
  340. data/packages/gsutil/boto/boto/ec2/autoscale/group.py +0 -337
  341. data/packages/gsutil/boto/boto/ec2/autoscale/instance.py +0 -60
  342. data/packages/gsutil/boto/boto/ec2/autoscale/launchconfig.py +0 -209
  343. data/packages/gsutil/boto/boto/ec2/autoscale/policy.py +0 -166
  344. data/packages/gsutil/boto/boto/ec2/autoscale/request.py +0 -38
  345. data/packages/gsutil/boto/boto/ec2/autoscale/scheduled.py +0 -60
  346. data/packages/gsutil/boto/boto/ec2/autoscale/tag.py +0 -84
  347. data/packages/gsutil/boto/boto/ec2/blockdevicemapping.py +0 -141
  348. data/packages/gsutil/boto/boto/ec2/bundleinstance.py +0 -78
  349. data/packages/gsutil/boto/boto/ec2/buyreservation.py +0 -84
  350. data/packages/gsutil/boto/boto/ec2/cloudwatch/__init__.py +0 -603
  351. data/packages/gsutil/boto/boto/ec2/cloudwatch/alarm.py +0 -316
  352. data/packages/gsutil/boto/boto/ec2/cloudwatch/datapoint.py +0 -40
  353. data/packages/gsutil/boto/boto/ec2/cloudwatch/dimension.py +0 -38
  354. data/packages/gsutil/boto/boto/ec2/cloudwatch/listelement.py +0 -31
  355. data/packages/gsutil/boto/boto/ec2/cloudwatch/metric.py +0 -175
  356. data/packages/gsutil/boto/boto/ec2/connection.py +0 -3409
  357. data/packages/gsutil/boto/boto/ec2/ec2object.py +0 -107
  358. data/packages/gsutil/boto/boto/ec2/elb/__init__.py +0 -553
  359. data/packages/gsutil/boto/boto/ec2/elb/healthcheck.py +0 -89
  360. data/packages/gsutil/boto/boto/ec2/elb/instancestate.py +0 -62
  361. data/packages/gsutil/boto/boto/ec2/elb/listelement.py +0 -36
  362. data/packages/gsutil/boto/boto/ec2/elb/listener.py +0 -75
  363. data/packages/gsutil/boto/boto/ec2/elb/loadbalancer.py +0 -324
  364. data/packages/gsutil/boto/boto/ec2/elb/policies.py +0 -89
  365. data/packages/gsutil/boto/boto/ec2/elb/securitygroup.py +0 -38
  366. data/packages/gsutil/boto/boto/ec2/group.py +0 -39
  367. data/packages/gsutil/boto/boto/ec2/image.py +0 -350
  368. data/packages/gsutil/boto/boto/ec2/instance.py +0 -661
  369. data/packages/gsutil/boto/boto/ec2/instanceinfo.py +0 -51
  370. data/packages/gsutil/boto/boto/ec2/instancestatus.py +0 -207
  371. data/packages/gsutil/boto/boto/ec2/keypair.py +0 -113
  372. data/packages/gsutil/boto/boto/ec2/launchspecification.py +0 -105
  373. data/packages/gsutil/boto/boto/ec2/networkinterface.py +0 -247
  374. data/packages/gsutil/boto/boto/ec2/placementgroup.py +0 -51
  375. data/packages/gsutil/boto/boto/ec2/regioninfo.py +0 -34
  376. data/packages/gsutil/boto/boto/ec2/reservedinstance.py +0 -227
  377. data/packages/gsutil/boto/boto/ec2/securitygroup.py +0 -357
  378. data/packages/gsutil/boto/boto/ec2/snapshot.py +0 -170
  379. data/packages/gsutil/boto/boto/ec2/spotdatafeedsubscription.py +0 -63
  380. data/packages/gsutil/boto/boto/ec2/spotinstancerequest.py +0 -188
  381. data/packages/gsutil/boto/boto/ec2/spotpricehistory.py +0 -55
  382. data/packages/gsutil/boto/boto/ec2/tag.py +0 -87
  383. data/packages/gsutil/boto/boto/ec2/vmtype.py +0 -58
  384. data/packages/gsutil/boto/boto/ec2/volume.py +0 -293
  385. data/packages/gsutil/boto/boto/ec2/volumestatus.py +0 -200
  386. data/packages/gsutil/boto/boto/ec2/zone.py +0 -80
  387. data/packages/gsutil/boto/boto/ecs/__init__.py +0 -90
  388. data/packages/gsutil/boto/boto/ecs/item.py +0 -153
  389. data/packages/gsutil/boto/boto/elasticache/__init__.py +0 -62
  390. data/packages/gsutil/boto/boto/elasticache/layer1.py +0 -1252
  391. data/packages/gsutil/boto/boto/elastictranscoder/__init__.py +0 -62
  392. data/packages/gsutil/boto/boto/elastictranscoder/exceptions.py +0 -46
  393. data/packages/gsutil/boto/boto/elastictranscoder/layer1.py +0 -509
  394. data/packages/gsutil/boto/boto/emr/__init__.py +0 -73
  395. data/packages/gsutil/boto/boto/emr/bootstrap_action.py +0 -44
  396. data/packages/gsutil/boto/boto/emr/connection.py +0 -531
  397. data/packages/gsutil/boto/boto/emr/emrobject.py +0 -176
  398. data/packages/gsutil/boto/boto/emr/instance_group.py +0 -43
  399. data/packages/gsutil/boto/boto/emr/step.py +0 -281
  400. data/packages/gsutil/boto/boto/exception.py +0 -476
  401. data/packages/gsutil/boto/boto/exception.pyc +0 -0
  402. data/packages/gsutil/boto/boto/file/README +0 -49
  403. data/packages/gsutil/boto/boto/file/__init__.py +0 -28
  404. data/packages/gsutil/boto/boto/file/bucket.py +0 -112
  405. data/packages/gsutil/boto/boto/file/connection.py +0 -33
  406. data/packages/gsutil/boto/boto/file/key.py +0 -199
  407. data/packages/gsutil/boto/boto/file/simpleresultset.py +0 -30
  408. data/packages/gsutil/boto/boto/fps/__init__.py +0 -21
  409. data/packages/gsutil/boto/boto/fps/connection.py +0 -369
  410. data/packages/gsutil/boto/boto/fps/exception.py +0 -344
  411. data/packages/gsutil/boto/boto/fps/response.py +0 -175
  412. data/packages/gsutil/boto/boto/glacier/__init__.py +0 -57
  413. data/packages/gsutil/boto/boto/glacier/concurrent.py +0 -409
  414. data/packages/gsutil/boto/boto/glacier/exceptions.py +0 -58
  415. data/packages/gsutil/boto/boto/glacier/job.py +0 -152
  416. data/packages/gsutil/boto/boto/glacier/layer1.py +0 -637
  417. data/packages/gsutil/boto/boto/glacier/layer2.py +0 -93
  418. data/packages/gsutil/boto/boto/glacier/response.py +0 -48
  419. data/packages/gsutil/boto/boto/glacier/utils.py +0 -163
  420. data/packages/gsutil/boto/boto/glacier/vault.py +0 -387
  421. data/packages/gsutil/boto/boto/glacier/writer.py +0 -242
  422. data/packages/gsutil/boto/boto/gs/__init__.py +0 -22
  423. data/packages/gsutil/boto/boto/gs/__init__.pyc +0 -0
  424. data/packages/gsutil/boto/boto/gs/acl.py +0 -304
  425. data/packages/gsutil/boto/boto/gs/acl.pyc +0 -0
  426. data/packages/gsutil/boto/boto/gs/bucket.py +0 -870
  427. data/packages/gsutil/boto/boto/gs/bucket.pyc +0 -0
  428. data/packages/gsutil/boto/boto/gs/bucketlistresultset.py +0 -64
  429. data/packages/gsutil/boto/boto/gs/bucketlistresultset.pyc +0 -0
  430. data/packages/gsutil/boto/boto/gs/connection.py +0 -103
  431. data/packages/gsutil/boto/boto/gs/connection.pyc +0 -0
  432. data/packages/gsutil/boto/boto/gs/cors.py +0 -169
  433. data/packages/gsutil/boto/boto/gs/cors.pyc +0 -0
  434. data/packages/gsutil/boto/boto/gs/key.py +0 -704
  435. data/packages/gsutil/boto/boto/gs/key.pyc +0 -0
  436. data/packages/gsutil/boto/boto/gs/resumable_upload_handler.py +0 -659
  437. data/packages/gsutil/boto/boto/gs/resumable_upload_handler.pyc +0 -0
  438. data/packages/gsutil/boto/boto/gs/user.py +0 -54
  439. data/packages/gsutil/boto/boto/gs/user.pyc +0 -0
  440. data/packages/gsutil/boto/boto/handler.py +0 -44
  441. data/packages/gsutil/boto/boto/handler.pyc +0 -0
  442. data/packages/gsutil/boto/boto/https_connection.py +0 -124
  443. data/packages/gsutil/boto/boto/https_connection.pyc +0 -0
  444. data/packages/gsutil/boto/boto/iam/__init__.py +0 -74
  445. data/packages/gsutil/boto/boto/iam/connection.py +0 -1317
  446. data/packages/gsutil/boto/boto/iam/summarymap.py +0 -42
  447. data/packages/gsutil/boto/boto/jsonresponse.py +0 -163
  448. data/packages/gsutil/boto/boto/jsonresponse.pyc +0 -0
  449. data/packages/gsutil/boto/boto/manage/__init__.py +0 -23
  450. data/packages/gsutil/boto/boto/manage/cmdshell.py +0 -241
  451. data/packages/gsutil/boto/boto/manage/propget.py +0 -64
  452. data/packages/gsutil/boto/boto/manage/server.py +0 -556
  453. data/packages/gsutil/boto/boto/manage/task.py +0 -175
  454. data/packages/gsutil/boto/boto/manage/test_manage.py +0 -34
  455. data/packages/gsutil/boto/boto/manage/volume.py +0 -420
  456. data/packages/gsutil/boto/boto/mashups/__init__.py +0 -23
  457. data/packages/gsutil/boto/boto/mashups/interactive.py +0 -97
  458. data/packages/gsutil/boto/boto/mashups/iobject.py +0 -115
  459. data/packages/gsutil/boto/boto/mashups/order.py +0 -211
  460. data/packages/gsutil/boto/boto/mashups/server.py +0 -395
  461. data/packages/gsutil/boto/boto/mturk/__init__.py +0 -23
  462. data/packages/gsutil/boto/boto/mturk/connection.py +0 -1027
  463. data/packages/gsutil/boto/boto/mturk/layoutparam.py +0 -55
  464. data/packages/gsutil/boto/boto/mturk/notification.py +0 -103
  465. data/packages/gsutil/boto/boto/mturk/price.py +0 -48
  466. data/packages/gsutil/boto/boto/mturk/qualification.py +0 -137
  467. data/packages/gsutil/boto/boto/mturk/question.py +0 -455
  468. data/packages/gsutil/boto/boto/mws/__init__.py +0 -21
  469. data/packages/gsutil/boto/boto/mws/connection.py +0 -813
  470. data/packages/gsutil/boto/boto/mws/exception.py +0 -75
  471. data/packages/gsutil/boto/boto/mws/response.py +0 -655
  472. data/packages/gsutil/boto/boto/plugin.py +0 -90
  473. data/packages/gsutil/boto/boto/plugin.pyc +0 -0
  474. data/packages/gsutil/boto/boto/provider.py +0 -337
  475. data/packages/gsutil/boto/boto/provider.pyc +0 -0
  476. data/packages/gsutil/boto/boto/pyami/__init__.py +0 -22
  477. data/packages/gsutil/boto/boto/pyami/__init__.pyc +0 -0
  478. data/packages/gsutil/boto/boto/pyami/bootstrap.py +0 -134
  479. data/packages/gsutil/boto/boto/pyami/config.py +0 -229
  480. data/packages/gsutil/boto/boto/pyami/config.pyc +0 -0
  481. data/packages/gsutil/boto/boto/pyami/copybot.cfg +0 -60
  482. data/packages/gsutil/boto/boto/pyami/copybot.py +0 -97
  483. data/packages/gsutil/boto/boto/pyami/helloworld.py +0 -28
  484. data/packages/gsutil/boto/boto/pyami/installers/__init__.py +0 -64
  485. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/__init__.py +0 -22
  486. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/apache.py +0 -43
  487. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/ebs.py +0 -238
  488. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/installer.py +0 -96
  489. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/mysql.py +0 -109
  490. data/packages/gsutil/boto/boto/pyami/installers/ubuntu/trac.py +0 -139
  491. data/packages/gsutil/boto/boto/pyami/launch_ami.py +0 -178
  492. data/packages/gsutil/boto/boto/pyami/scriptbase.py +0 -44
  493. data/packages/gsutil/boto/boto/pyami/startup.py +0 -60
  494. data/packages/gsutil/boto/boto/rds/__init__.py +0 -1194
  495. data/packages/gsutil/boto/boto/rds/dbinstance.py +0 -357
  496. data/packages/gsutil/boto/boto/rds/dbsecuritygroup.py +0 -177
  497. data/packages/gsutil/boto/boto/rds/dbsnapshot.py +0 -108
  498. data/packages/gsutil/boto/boto/rds/event.py +0 -49
  499. data/packages/gsutil/boto/boto/rds/parametergroup.py +0 -201
  500. data/packages/gsutil/boto/boto/rds/regioninfo.py +0 -32
  501. data/packages/gsutil/boto/boto/regioninfo.py +0 -63
  502. data/packages/gsutil/boto/boto/regioninfo.pyc +0 -0
  503. data/packages/gsutil/boto/boto/resultset.py +0 -169
  504. data/packages/gsutil/boto/boto/resultset.pyc +0 -0
  505. data/packages/gsutil/boto/boto/roboto/__init__.py +0 -1
  506. data/packages/gsutil/boto/boto/roboto/awsqueryrequest.py +0 -504
  507. data/packages/gsutil/boto/boto/roboto/awsqueryservice.py +0 -121
  508. data/packages/gsutil/boto/boto/roboto/param.py +0 -147
  509. data/packages/gsutil/boto/boto/route53/__init__.py +0 -75
  510. data/packages/gsutil/boto/boto/route53/connection.py +0 -403
  511. data/packages/gsutil/boto/boto/route53/exception.py +0 -27
  512. data/packages/gsutil/boto/boto/route53/hostedzone.py +0 -56
  513. data/packages/gsutil/boto/boto/route53/record.py +0 -306
  514. data/packages/gsutil/boto/boto/route53/status.py +0 -42
  515. data/packages/gsutil/boto/boto/route53/zone.py +0 -412
  516. data/packages/gsutil/boto/boto/s3/__init__.py +0 -84
  517. data/packages/gsutil/boto/boto/s3/__init__.pyc +0 -0
  518. data/packages/gsutil/boto/boto/s3/acl.py +0 -164
  519. data/packages/gsutil/boto/boto/s3/acl.pyc +0 -0
  520. data/packages/gsutil/boto/boto/s3/bucket.py +0 -1634
  521. data/packages/gsutil/boto/boto/s3/bucket.pyc +0 -0
  522. data/packages/gsutil/boto/boto/s3/bucketlistresultset.py +0 -139
  523. data/packages/gsutil/boto/boto/s3/bucketlistresultset.pyc +0 -0
  524. data/packages/gsutil/boto/boto/s3/bucketlogging.py +0 -83
  525. data/packages/gsutil/boto/boto/s3/bucketlogging.pyc +0 -0
  526. data/packages/gsutil/boto/boto/s3/connection.py +0 -540
  527. data/packages/gsutil/boto/boto/s3/connection.pyc +0 -0
  528. data/packages/gsutil/boto/boto/s3/cors.py +0 -210
  529. data/packages/gsutil/boto/boto/s3/cors.pyc +0 -0
  530. data/packages/gsutil/boto/boto/s3/deletemarker.py +0 -55
  531. data/packages/gsutil/boto/boto/s3/deletemarker.pyc +0 -0
  532. data/packages/gsutil/boto/boto/s3/key.py +0 -1712
  533. data/packages/gsutil/boto/boto/s3/key.pyc +0 -0
  534. data/packages/gsutil/boto/boto/s3/keyfile.py +0 -134
  535. data/packages/gsutil/boto/boto/s3/keyfile.pyc +0 -0
  536. data/packages/gsutil/boto/boto/s3/lifecycle.py +0 -231
  537. data/packages/gsutil/boto/boto/s3/lifecycle.pyc +0 -0
  538. data/packages/gsutil/boto/boto/s3/multidelete.py +0 -138
  539. data/packages/gsutil/boto/boto/s3/multidelete.pyc +0 -0
  540. data/packages/gsutil/boto/boto/s3/multipart.py +0 -315
  541. data/packages/gsutil/boto/boto/s3/multipart.pyc +0 -0
  542. data/packages/gsutil/boto/boto/s3/prefix.py +0 -42
  543. data/packages/gsutil/boto/boto/s3/prefix.pyc +0 -0
  544. data/packages/gsutil/boto/boto/s3/resumable_download_handler.py +0 -339
  545. data/packages/gsutil/boto/boto/s3/resumable_download_handler.pyc +0 -0
  546. data/packages/gsutil/boto/boto/s3/tagging.py +0 -71
  547. data/packages/gsutil/boto/boto/s3/tagging.pyc +0 -0
  548. data/packages/gsutil/boto/boto/s3/user.py +0 -49
  549. data/packages/gsutil/boto/boto/s3/user.pyc +0 -0
  550. data/packages/gsutil/boto/boto/s3/website.py +0 -237
  551. data/packages/gsutil/boto/boto/s3/website.pyc +0 -0
  552. data/packages/gsutil/boto/boto/sdb/__init__.py +0 -67
  553. data/packages/gsutil/boto/boto/sdb/connection.py +0 -617
  554. data/packages/gsutil/boto/boto/sdb/db/__init__.py +0 -20
  555. data/packages/gsutil/boto/boto/sdb/db/blob.py +0 -75
  556. data/packages/gsutil/boto/boto/sdb/db/key.py +0 -59
  557. data/packages/gsutil/boto/boto/sdb/db/manager/__init__.py +0 -85
  558. data/packages/gsutil/boto/boto/sdb/db/manager/sdbmanager.py +0 -732
  559. data/packages/gsutil/boto/boto/sdb/db/manager/xmlmanager.py +0 -517
  560. data/packages/gsutil/boto/boto/sdb/db/model.py +0 -294
  561. data/packages/gsutil/boto/boto/sdb/db/property.py +0 -703
  562. data/packages/gsutil/boto/boto/sdb/db/query.py +0 -85
  563. data/packages/gsutil/boto/boto/sdb/db/sequence.py +0 -226
  564. data/packages/gsutil/boto/boto/sdb/db/test_db.py +0 -231
  565. data/packages/gsutil/boto/boto/sdb/domain.py +0 -377
  566. data/packages/gsutil/boto/boto/sdb/item.py +0 -181
  567. data/packages/gsutil/boto/boto/sdb/queryresultset.py +0 -92
  568. data/packages/gsutil/boto/boto/sdb/regioninfo.py +0 -32
  569. data/packages/gsutil/boto/boto/services/__init__.py +0 -23
  570. data/packages/gsutil/boto/boto/services/bs.py +0 -179
  571. data/packages/gsutil/boto/boto/services/message.py +0 -58
  572. data/packages/gsutil/boto/boto/services/result.py +0 -136
  573. data/packages/gsutil/boto/boto/services/service.py +0 -161
  574. data/packages/gsutil/boto/boto/services/servicedef.py +0 -91
  575. data/packages/gsutil/boto/boto/services/sonofmmm.cfg +0 -43
  576. data/packages/gsutil/boto/boto/services/sonofmmm.py +0 -81
  577. data/packages/gsutil/boto/boto/services/submit.py +0 -88
  578. data/packages/gsutil/boto/boto/ses/__init__.py +0 -54
  579. data/packages/gsutil/boto/boto/ses/connection.py +0 -521
  580. data/packages/gsutil/boto/boto/ses/exceptions.py +0 -77
  581. data/packages/gsutil/boto/boto/sns/__init__.py +0 -78
  582. data/packages/gsutil/boto/boto/sns/connection.py +0 -431
  583. data/packages/gsutil/boto/boto/sqs/__init__.py +0 -56
  584. data/packages/gsutil/boto/boto/sqs/attributes.py +0 -46
  585. data/packages/gsutil/boto/boto/sqs/batchresults.py +0 -95
  586. data/packages/gsutil/boto/boto/sqs/connection.py +0 -417
  587. data/packages/gsutil/boto/boto/sqs/jsonmessage.py +0 -43
  588. data/packages/gsutil/boto/boto/sqs/message.py +0 -253
  589. data/packages/gsutil/boto/boto/sqs/queue.py +0 -478
  590. data/packages/gsutil/boto/boto/sqs/regioninfo.py +0 -32
  591. data/packages/gsutil/boto/boto/storage_uri.py +0 -835
  592. data/packages/gsutil/boto/boto/storage_uri.pyc +0 -0
  593. data/packages/gsutil/boto/boto/sts/__init__.py +0 -55
  594. data/packages/gsutil/boto/boto/sts/connection.py +0 -207
  595. data/packages/gsutil/boto/boto/sts/credentials.py +0 -215
  596. data/packages/gsutil/boto/boto/swf/__init__.py +0 -60
  597. data/packages/gsutil/boto/boto/swf/exceptions.py +0 -44
  598. data/packages/gsutil/boto/boto/swf/layer1.py +0 -1512
  599. data/packages/gsutil/boto/boto/swf/layer1_decisions.py +0 -287
  600. data/packages/gsutil/boto/boto/swf/layer2.py +0 -342
  601. data/packages/gsutil/boto/boto/utils.py +0 -927
  602. data/packages/gsutil/boto/boto/utils.pyc +0 -0
  603. data/packages/gsutil/boto/boto/vpc/__init__.py +0 -910
  604. data/packages/gsutil/boto/boto/vpc/customergateway.py +0 -54
  605. data/packages/gsutil/boto/boto/vpc/dhcpoptions.py +0 -72
  606. data/packages/gsutil/boto/boto/vpc/internetgateway.py +0 -72
  607. data/packages/gsutil/boto/boto/vpc/routetable.py +0 -109
  608. data/packages/gsutil/boto/boto/vpc/subnet.py +0 -57
  609. data/packages/gsutil/boto/boto/vpc/vpc.py +0 -54
  610. data/packages/gsutil/boto/boto/vpc/vpnconnection.py +0 -60
  611. data/packages/gsutil/boto/boto/vpc/vpngateway.py +0 -83
  612. data/packages/gsutil/boto/docs/BotoCheatSheet.pdf +0 -0
  613. data/packages/gsutil/boto/docs/Makefile +0 -89
  614. data/packages/gsutil/boto/docs/make.bat +0 -113
  615. data/packages/gsutil/boto/docs/source/_templates/layout.html +0 -3
  616. data/packages/gsutil/boto/docs/source/autoscale_tut.rst +0 -220
  617. data/packages/gsutil/boto/docs/source/boto_config_tut.rst +0 -125
  618. data/packages/gsutil/boto/docs/source/boto_theme/static/boto.css_t +0 -239
  619. data/packages/gsutil/boto/docs/source/boto_theme/static/pygments.css +0 -61
  620. data/packages/gsutil/boto/docs/source/boto_theme/theme.conf +0 -3
  621. data/packages/gsutil/boto/docs/source/cloudfront_tut.rst +0 -196
  622. data/packages/gsutil/boto/docs/source/cloudsearch_tut.rst +0 -411
  623. data/packages/gsutil/boto/docs/source/cloudwatch_tut.rst +0 -116
  624. data/packages/gsutil/boto/docs/source/conf.py +0 -32
  625. data/packages/gsutil/boto/docs/source/contributing.rst +0 -204
  626. data/packages/gsutil/boto/docs/source/documentation.rst +0 -59
  627. data/packages/gsutil/boto/docs/source/dynamodb_tut.rst +0 -339
  628. data/packages/gsutil/boto/docs/source/ec2_tut.rst +0 -86
  629. data/packages/gsutil/boto/docs/source/elb_tut.rst +0 -257
  630. data/packages/gsutil/boto/docs/source/emr_tut.rst +0 -108
  631. data/packages/gsutil/boto/docs/source/index.rst +0 -146
  632. data/packages/gsutil/boto/docs/source/rds_tut.rst +0 -108
  633. data/packages/gsutil/boto/docs/source/ref/autoscale.rst +0 -62
  634. data/packages/gsutil/boto/docs/source/ref/beanstalk.rst +0 -26
  635. data/packages/gsutil/boto/docs/source/ref/boto.rst +0 -47
  636. data/packages/gsutil/boto/docs/source/ref/cloudformation.rst +0 -34
  637. data/packages/gsutil/boto/docs/source/ref/cloudfront.rst +0 -68
  638. data/packages/gsutil/boto/docs/source/ref/cloudsearch.rst +0 -59
  639. data/packages/gsutil/boto/docs/source/ref/cloudwatch.rst +0 -27
  640. data/packages/gsutil/boto/docs/source/ref/contrib.rst +0 -32
  641. data/packages/gsutil/boto/docs/source/ref/datapipeline.rst +0 -26
  642. data/packages/gsutil/boto/docs/source/ref/dynamodb.rst +0 -61
  643. data/packages/gsutil/boto/docs/source/ref/ec2.rst +0 -140
  644. data/packages/gsutil/boto/docs/source/ref/ecs.rst +0 -19
  645. data/packages/gsutil/boto/docs/source/ref/elasticache.rst +0 -19
  646. data/packages/gsutil/boto/docs/source/ref/elastictranscoder.rst +0 -26
  647. data/packages/gsutil/boto/docs/source/ref/elb.rst +0 -47
  648. data/packages/gsutil/boto/docs/source/ref/emr.rst +0 -34
  649. data/packages/gsutil/boto/docs/source/ref/file.rst +0 -34
  650. data/packages/gsutil/boto/docs/source/ref/fps.rst +0 -19
  651. data/packages/gsutil/boto/docs/source/ref/glacier.rst +0 -63
  652. data/packages/gsutil/boto/docs/source/ref/gs.rst +0 -72
  653. data/packages/gsutil/boto/docs/source/ref/iam.rst +0 -27
  654. data/packages/gsutil/boto/docs/source/ref/index.rst +0 -40
  655. data/packages/gsutil/boto/docs/source/ref/manage.rst +0 -47
  656. data/packages/gsutil/boto/docs/source/ref/mturk.rst +0 -54
  657. data/packages/gsutil/boto/docs/source/ref/mws.rst +0 -33
  658. data/packages/gsutil/boto/docs/source/ref/pyami.rst +0 -103
  659. data/packages/gsutil/boto/docs/source/ref/rds.rst +0 -47
  660. data/packages/gsutil/boto/docs/source/ref/route53.rst +0 -34
  661. data/packages/gsutil/boto/docs/source/ref/s3.rst +0 -111
  662. data/packages/gsutil/boto/docs/source/ref/sdb.rst +0 -45
  663. data/packages/gsutil/boto/docs/source/ref/sdb_db.rst +0 -83
  664. data/packages/gsutil/boto/docs/source/ref/services.rst +0 -61
  665. data/packages/gsutil/boto/docs/source/ref/ses.rst +0 -21
  666. data/packages/gsutil/boto/docs/source/ref/sns.rst +0 -17
  667. data/packages/gsutil/boto/docs/source/ref/sqs.rst +0 -61
  668. data/packages/gsutil/boto/docs/source/ref/sts.rst +0 -25
  669. data/packages/gsutil/boto/docs/source/ref/swf.rst +0 -22
  670. data/packages/gsutil/boto/docs/source/ref/vpc.rst +0 -54
  671. data/packages/gsutil/boto/docs/source/s3_tut.rst +0 -450
  672. data/packages/gsutil/boto/docs/source/security_groups.rst +0 -82
  673. data/packages/gsutil/boto/docs/source/ses_tut.rst +0 -171
  674. data/packages/gsutil/boto/docs/source/simpledb_tut.rst +0 -188
  675. data/packages/gsutil/boto/docs/source/sqs_tut.rst +0 -246
  676. data/packages/gsutil/boto/docs/source/vpc_tut.rst +0 -100
  677. data/packages/gsutil/boto/pylintrc +0 -305
  678. data/packages/gsutil/boto/requirements.txt +0 -10
  679. data/packages/gsutil/boto/setup.py +0 -89
  680. data/packages/gsutil/boto/tests/__init__.py +0 -20
  681. data/packages/gsutil/boto/tests/db/test_lists.py +0 -96
  682. data/packages/gsutil/boto/tests/db/test_password.py +0 -128
  683. data/packages/gsutil/boto/tests/db/test_query.py +0 -152
  684. data/packages/gsutil/boto/tests/db/test_sequence.py +0 -109
  685. data/packages/gsutil/boto/tests/devpay/__init__.py +0 -0
  686. data/packages/gsutil/boto/tests/devpay/test_s3.py +0 -181
  687. data/packages/gsutil/boto/tests/fps/__init__.py +0 -0
  688. data/packages/gsutil/boto/tests/fps/test.py +0 -100
  689. data/packages/gsutil/boto/tests/fps/test_verify_signature.py +0 -12
  690. data/packages/gsutil/boto/tests/integration/__init__.py +0 -0
  691. data/packages/gsutil/boto/tests/integration/beanstalk/test_wrapper.py +0 -209
  692. data/packages/gsutil/boto/tests/integration/cloudformation/__init__.py +0 -21
  693. data/packages/gsutil/boto/tests/integration/cloudformation/test_cert_verification.py +0 -40
  694. data/packages/gsutil/boto/tests/integration/cloudformation/test_connection.py +0 -110
  695. data/packages/gsutil/boto/tests/integration/cloudsearch/__init__.py +0 -21
  696. data/packages/gsutil/boto/tests/integration/cloudsearch/test_cert_verification.py +0 -40
  697. data/packages/gsutil/boto/tests/integration/datapipeline/test_layer1.py +0 -122
  698. data/packages/gsutil/boto/tests/integration/dynamodb/__init__.py +0 -20
  699. data/packages/gsutil/boto/tests/integration/dynamodb/test_cert_verification.py +0 -40
  700. data/packages/gsutil/boto/tests/integration/dynamodb/test_layer1.py +0 -266
  701. data/packages/gsutil/boto/tests/integration/dynamodb/test_layer2.py +0 -484
  702. data/packages/gsutil/boto/tests/integration/dynamodb/test_table.py +0 -84
  703. data/packages/gsutil/boto/tests/integration/ec2/__init__.py +0 -20
  704. data/packages/gsutil/boto/tests/integration/ec2/autoscale/__init__.py +0 -21
  705. data/packages/gsutil/boto/tests/integration/ec2/autoscale/test_cert_verification.py +0 -40
  706. data/packages/gsutil/boto/tests/integration/ec2/autoscale/test_connection.py +0 -167
  707. data/packages/gsutil/boto/tests/integration/ec2/cloudwatch/__init__.py +0 -20
  708. data/packages/gsutil/boto/tests/integration/ec2/cloudwatch/test_cert_verification.py +0 -40
  709. data/packages/gsutil/boto/tests/integration/ec2/cloudwatch/test_connection.py +0 -277
  710. data/packages/gsutil/boto/tests/integration/ec2/elb/__init__.py +0 -20
  711. data/packages/gsutil/boto/tests/integration/ec2/elb/test_cert_verification.py +0 -40
  712. data/packages/gsutil/boto/tests/integration/ec2/elb/test_connection.py +0 -130
  713. data/packages/gsutil/boto/tests/integration/ec2/test_cert_verification.py +0 -40
  714. data/packages/gsutil/boto/tests/integration/ec2/test_connection.py +0 -192
  715. data/packages/gsutil/boto/tests/integration/ec2/vpc/__init__.py +0 -0
  716. data/packages/gsutil/boto/tests/integration/ec2/vpc/test_connection.py +0 -95
  717. data/packages/gsutil/boto/tests/integration/elasticache/__init__.py +0 -0
  718. data/packages/gsutil/boto/tests/integration/elasticache/test_layer1.py +0 -67
  719. data/packages/gsutil/boto/tests/integration/elastictranscoder/__init__.py +0 -0
  720. data/packages/gsutil/boto/tests/integration/elastictranscoder/test_cert_verification.py +0 -35
  721. data/packages/gsutil/boto/tests/integration/elastictranscoder/test_layer1.py +0 -115
  722. data/packages/gsutil/boto/tests/integration/emr/__init__.py +0 -20
  723. data/packages/gsutil/boto/tests/integration/emr/test_cert_verification.py +0 -40
  724. data/packages/gsutil/boto/tests/integration/glacier/__init__.py +0 -22
  725. data/packages/gsutil/boto/tests/integration/glacier/test_cert_verification.py +0 -40
  726. data/packages/gsutil/boto/tests/integration/glacier/test_layer1.py +0 -44
  727. data/packages/gsutil/boto/tests/integration/glacier/test_layer2.py +0 -45
  728. data/packages/gsutil/boto/tests/integration/gs/__init__.py +0 -0
  729. data/packages/gsutil/boto/tests/integration/gs/cb_test_harness.py +0 -71
  730. data/packages/gsutil/boto/tests/integration/gs/test_basic.py +0 -379
  731. data/packages/gsutil/boto/tests/integration/gs/test_generation_conditionals.py +0 -399
  732. data/packages/gsutil/boto/tests/integration/gs/test_resumable_downloads.py +0 -358
  733. data/packages/gsutil/boto/tests/integration/gs/test_resumable_uploads.py +0 -525
  734. data/packages/gsutil/boto/tests/integration/gs/test_storage_uri.py +0 -125
  735. data/packages/gsutil/boto/tests/integration/gs/test_versioning.py +0 -268
  736. data/packages/gsutil/boto/tests/integration/gs/testcase.py +0 -116
  737. data/packages/gsutil/boto/tests/integration/gs/util.py +0 -63
  738. data/packages/gsutil/boto/tests/integration/iam/__init__.py +0 -20
  739. data/packages/gsutil/boto/tests/integration/iam/test_cert_verification.py +0 -40
  740. data/packages/gsutil/boto/tests/integration/mws/__init__.py +0 -0
  741. data/packages/gsutil/boto/tests/integration/mws/test.py +0 -100
  742. data/packages/gsutil/boto/tests/integration/rds/__init__.py +0 -21
  743. data/packages/gsutil/boto/tests/integration/rds/test_cert_verification.py +0 -40
  744. data/packages/gsutil/boto/tests/integration/route53/__init__.py +0 -20
  745. data/packages/gsutil/boto/tests/integration/route53/test_cert_verification.py +0 -40
  746. data/packages/gsutil/boto/tests/integration/route53/test_zone.py +0 -132
  747. data/packages/gsutil/boto/tests/integration/s3/__init__.py +0 -20
  748. data/packages/gsutil/boto/tests/integration/s3/mock_storage_service.py +0 -589
  749. data/packages/gsutil/boto/tests/integration/s3/other_cacerts.txt +0 -70
  750. data/packages/gsutil/boto/tests/integration/s3/test_bucket.py +0 -263
  751. data/packages/gsutil/boto/tests/integration/s3/test_cert_verification.py +0 -40
  752. data/packages/gsutil/boto/tests/integration/s3/test_connection.py +0 -245
  753. data/packages/gsutil/boto/tests/integration/s3/test_cors.py +0 -78
  754. data/packages/gsutil/boto/tests/integration/s3/test_encryption.py +0 -115
  755. data/packages/gsutil/boto/tests/integration/s3/test_https_cert_validation.py +0 -141
  756. data/packages/gsutil/boto/tests/integration/s3/test_key.py +0 -375
  757. data/packages/gsutil/boto/tests/integration/s3/test_mfa.py +0 -95
  758. data/packages/gsutil/boto/tests/integration/s3/test_multidelete.py +0 -181
  759. data/packages/gsutil/boto/tests/integration/s3/test_multipart.py +0 -139
  760. data/packages/gsutil/boto/tests/integration/s3/test_pool.py +0 -246
  761. data/packages/gsutil/boto/tests/integration/s3/test_versioning.py +0 -158
  762. data/packages/gsutil/boto/tests/integration/sdb/__init__.py +0 -20
  763. data/packages/gsutil/boto/tests/integration/sdb/test_cert_verification.py +0 -40
  764. data/packages/gsutil/boto/tests/integration/sdb/test_connection.py +0 -119
  765. data/packages/gsutil/boto/tests/integration/ses/__init__.py +0 -0
  766. data/packages/gsutil/boto/tests/integration/ses/test_cert_verification.py +0 -40
  767. data/packages/gsutil/boto/tests/integration/ses/test_connection.py +0 -38
  768. data/packages/gsutil/boto/tests/integration/sns/__init__.py +0 -20
  769. data/packages/gsutil/boto/tests/integration/sns/test_cert_verification.py +0 -40
  770. data/packages/gsutil/boto/tests/integration/sqs/__init__.py +0 -20
  771. data/packages/gsutil/boto/tests/integration/sqs/test_cert_verification.py +0 -40
  772. data/packages/gsutil/boto/tests/integration/sqs/test_connection.py +0 -217
  773. data/packages/gsutil/boto/tests/integration/sts/__init__.py +0 -20
  774. data/packages/gsutil/boto/tests/integration/sts/test_cert_verification.py +0 -40
  775. data/packages/gsutil/boto/tests/integration/sts/test_session_token.py +0 -65
  776. data/packages/gsutil/boto/tests/integration/swf/__init__.py +0 -0
  777. data/packages/gsutil/boto/tests/integration/swf/test_cert_verification.py +0 -40
  778. data/packages/gsutil/boto/tests/integration/swf/test_layer1.py +0 -246
  779. data/packages/gsutil/boto/tests/integration/swf/test_layer1_workflow_execution.py +0 -173
  780. data/packages/gsutil/boto/tests/mturk/__init__.py +0 -0
  781. data/packages/gsutil/boto/tests/mturk/_init_environment.py +0 -28
  782. data/packages/gsutil/boto/tests/mturk/all_tests.py +0 -24
  783. data/packages/gsutil/boto/tests/mturk/cleanup_tests.py +0 -47
  784. data/packages/gsutil/boto/tests/mturk/common.py +0 -45
  785. data/packages/gsutil/boto/tests/mturk/create_free_text_question_regex.doctest +0 -100
  786. data/packages/gsutil/boto/tests/mturk/create_hit.doctest +0 -92
  787. data/packages/gsutil/boto/tests/mturk/create_hit_binary.doctest +0 -94
  788. data/packages/gsutil/boto/tests/mturk/create_hit_external.py +0 -21
  789. data/packages/gsutil/boto/tests/mturk/create_hit_from_hit_type.doctest +0 -103
  790. data/packages/gsutil/boto/tests/mturk/create_hit_test.py +0 -21
  791. data/packages/gsutil/boto/tests/mturk/create_hit_with_qualifications.py +0 -16
  792. data/packages/gsutil/boto/tests/mturk/hit_persistence.py +0 -27
  793. data/packages/gsutil/boto/tests/mturk/mocks.py +0 -11
  794. data/packages/gsutil/boto/tests/mturk/reviewable_hits.doctest +0 -129
  795. data/packages/gsutil/boto/tests/mturk/run-doctest.py +0 -13
  796. data/packages/gsutil/boto/tests/mturk/search_hits.doctest +0 -16
  797. data/packages/gsutil/boto/tests/mturk/selenium_support.py +0 -61
  798. data/packages/gsutil/boto/tests/mturk/support.py +0 -7
  799. data/packages/gsutil/boto/tests/mturk/test_disable_hit.py +0 -11
  800. data/packages/gsutil/boto/tests/test.py +0 -59
  801. data/packages/gsutil/boto/tests/unit/__init__.py +0 -79
  802. data/packages/gsutil/boto/tests/unit/auth/__init__.py +0 -0
  803. data/packages/gsutil/boto/tests/unit/auth/test_sigv4.py +0 -73
  804. data/packages/gsutil/boto/tests/unit/beanstalk/__init__.py +0 -0
  805. data/packages/gsutil/boto/tests/unit/beanstalk/test_layer1.py +0 -128
  806. data/packages/gsutil/boto/tests/unit/cloudformation/__init__.py +0 -0
  807. data/packages/gsutil/boto/tests/unit/cloudformation/test_connection.py +0 -605
  808. data/packages/gsutil/boto/tests/unit/cloudformation/test_stack.py +0 -63
  809. data/packages/gsutil/boto/tests/unit/cloudfront/__init__.py +0 -0
  810. data/packages/gsutil/boto/tests/unit/cloudfront/test_invalidation_list.py +0 -113
  811. data/packages/gsutil/boto/tests/unit/cloudfront/test_signed_urls.py +0 -354
  812. data/packages/gsutil/boto/tests/unit/cloudsearch/__init__.py +0 -1
  813. data/packages/gsutil/boto/tests/unit/cloudsearch/test_connection.py +0 -241
  814. data/packages/gsutil/boto/tests/unit/cloudsearch/test_document.py +0 -324
  815. data/packages/gsutil/boto/tests/unit/cloudsearch/test_search.py +0 -325
  816. data/packages/gsutil/boto/tests/unit/dynamodb/__init__.py +0 -0
  817. data/packages/gsutil/boto/tests/unit/dynamodb/test_batch.py +0 -103
  818. data/packages/gsutil/boto/tests/unit/dynamodb/test_layer2.py +0 -119
  819. data/packages/gsutil/boto/tests/unit/dynamodb/test_types.py +0 -82
  820. data/packages/gsutil/boto/tests/unit/ec2/__init__.py +0 -0
  821. data/packages/gsutil/boto/tests/unit/ec2/autoscale/__init__.py +0 -0
  822. data/packages/gsutil/boto/tests/unit/ec2/autoscale/test_group.py +0 -162
  823. data/packages/gsutil/boto/tests/unit/ec2/test_address.py +0 -39
  824. data/packages/gsutil/boto/tests/unit/ec2/test_blockdevicemapping.py +0 -79
  825. data/packages/gsutil/boto/tests/unit/ec2/test_connection.py +0 -480
  826. data/packages/gsutil/boto/tests/unit/ec2/test_instance.py +0 -243
  827. data/packages/gsutil/boto/tests/unit/ec2/test_networkinterface.py +0 -140
  828. data/packages/gsutil/boto/tests/unit/ec2/test_volume.py +0 -248
  829. data/packages/gsutil/boto/tests/unit/emr/test_emr_responses.py +0 -373
  830. data/packages/gsutil/boto/tests/unit/glacier/__init__.py +0 -0
  831. data/packages/gsutil/boto/tests/unit/glacier/test_concurrent.py +0 -120
  832. data/packages/gsutil/boto/tests/unit/glacier/test_job.py +0 -60
  833. data/packages/gsutil/boto/tests/unit/glacier/test_layer1.py +0 -98
  834. data/packages/gsutil/boto/tests/unit/glacier/test_layer2.py +0 -266
  835. data/packages/gsutil/boto/tests/unit/glacier/test_utils.py +0 -116
  836. data/packages/gsutil/boto/tests/unit/glacier/test_vault.py +0 -100
  837. data/packages/gsutil/boto/tests/unit/glacier/test_writer.py +0 -185
  838. data/packages/gsutil/boto/tests/unit/provider/__init__.py +0 -0
  839. data/packages/gsutil/boto/tests/unit/provider/test_provider.py +0 -176
  840. data/packages/gsutil/boto/tests/unit/rds/__init__.py +0 -0
  841. data/packages/gsutil/boto/tests/unit/rds/test_connection.py +0 -131
  842. data/packages/gsutil/boto/tests/unit/s3/__init__.py +0 -0
  843. data/packages/gsutil/boto/tests/unit/s3/test_cors_configuration.py +0 -77
  844. data/packages/gsutil/boto/tests/unit/s3/test_key.py +0 -75
  845. data/packages/gsutil/boto/tests/unit/s3/test_keyfile.py +0 -101
  846. data/packages/gsutil/boto/tests/unit/s3/test_lifecycle.py +0 -97
  847. data/packages/gsutil/boto/tests/unit/s3/test_tagging.py +0 -47
  848. data/packages/gsutil/boto/tests/unit/s3/test_uri.py +0 -257
  849. data/packages/gsutil/boto/tests/unit/s3/test_website.py +0 -188
  850. data/packages/gsutil/boto/tests/unit/sns/__init__.py +0 -0
  851. data/packages/gsutil/boto/tests/unit/sns/test_connection.py +0 -99
  852. data/packages/gsutil/boto/tests/unit/sqs/__init__.py +0 -0
  853. data/packages/gsutil/boto/tests/unit/sqs/test_connection.py +0 -98
  854. data/packages/gsutil/boto/tests/unit/sqs/test_queue.py +0 -40
  855. data/packages/gsutil/boto/tests/unit/sts/test_connection.py +0 -74
  856. data/packages/gsutil/boto/tests/unit/test_connection.py +0 -60
  857. data/packages/gsutil/boto/tests/unit/utils/test_utils.py +0 -109
  858. data/packages/gsutil/boto/tox.ini +0 -8
  859. data/packages/gsutil/gslib/README +0 -5
  860. data/packages/gsutil/gslib/__init__.py +0 -22
  861. data/packages/gsutil/gslib/__init__.pyc +0 -0
  862. data/packages/gsutil/gslib/addlhelp/acls.py +0 -234
  863. data/packages/gsutil/gslib/addlhelp/anon.py +0 -57
  864. data/packages/gsutil/gslib/addlhelp/command_opts.py +0 -116
  865. data/packages/gsutil/gslib/addlhelp/dev.py +0 -139
  866. data/packages/gsutil/gslib/addlhelp/metadata.py +0 -186
  867. data/packages/gsutil/gslib/addlhelp/naming.py +0 -173
  868. data/packages/gsutil/gslib/addlhelp/prod.py +0 -160
  869. data/packages/gsutil/gslib/addlhelp/projects.py +0 -130
  870. data/packages/gsutil/gslib/addlhelp/subdirs.py +0 -110
  871. data/packages/gsutil/gslib/addlhelp/support.py +0 -86
  872. data/packages/gsutil/gslib/addlhelp/versioning.py +0 -242
  873. data/packages/gsutil/gslib/addlhelp/wildcards.py +0 -170
  874. data/packages/gsutil/gslib/bucket_listing_ref.py +0 -175
  875. data/packages/gsutil/gslib/bucket_listing_ref.pyc +0 -0
  876. data/packages/gsutil/gslib/command.py +0 -722
  877. data/packages/gsutil/gslib/command.pyc +0 -0
  878. data/packages/gsutil/gslib/command_runner.py +0 -101
  879. data/packages/gsutil/gslib/command_runner.pyc +0 -0
  880. data/packages/gsutil/gslib/commands/__init__.pyc +0 -0
  881. data/packages/gsutil/gslib/commands/cat.py +0 -131
  882. data/packages/gsutil/gslib/commands/cat.pyc +0 -0
  883. data/packages/gsutil/gslib/commands/chacl.py +0 -532
  884. data/packages/gsutil/gslib/commands/chacl.pyc +0 -0
  885. data/packages/gsutil/gslib/commands/config.py +0 -694
  886. data/packages/gsutil/gslib/commands/config.pyc +0 -0
  887. data/packages/gsutil/gslib/commands/cp.py +0 -1818
  888. data/packages/gsutil/gslib/commands/cp.pyc +0 -0
  889. data/packages/gsutil/gslib/commands/disablelogging.py +0 -101
  890. data/packages/gsutil/gslib/commands/disablelogging.pyc +0 -0
  891. data/packages/gsutil/gslib/commands/enablelogging.py +0 -149
  892. data/packages/gsutil/gslib/commands/enablelogging.pyc +0 -0
  893. data/packages/gsutil/gslib/commands/getacl.py +0 -82
  894. data/packages/gsutil/gslib/commands/getacl.pyc +0 -0
  895. data/packages/gsutil/gslib/commands/getcors.py +0 -121
  896. data/packages/gsutil/gslib/commands/getcors.pyc +0 -0
  897. data/packages/gsutil/gslib/commands/getdefacl.py +0 -86
  898. data/packages/gsutil/gslib/commands/getdefacl.pyc +0 -0
  899. data/packages/gsutil/gslib/commands/getlogging.py +0 -137
  900. data/packages/gsutil/gslib/commands/getlogging.pyc +0 -0
  901. data/packages/gsutil/gslib/commands/getversioning.py +0 -116
  902. data/packages/gsutil/gslib/commands/getversioning.pyc +0 -0
  903. data/packages/gsutil/gslib/commands/getwebcfg.py +0 -122
  904. data/packages/gsutil/gslib/commands/getwebcfg.pyc +0 -0
  905. data/packages/gsutil/gslib/commands/help.py +0 -218
  906. data/packages/gsutil/gslib/commands/help.pyc +0 -0
  907. data/packages/gsutil/gslib/commands/ls.py +0 -578
  908. data/packages/gsutil/gslib/commands/ls.pyc +0 -0
  909. data/packages/gsutil/gslib/commands/mb.py +0 -172
  910. data/packages/gsutil/gslib/commands/mb.pyc +0 -0
  911. data/packages/gsutil/gslib/commands/mv.py +0 -159
  912. data/packages/gsutil/gslib/commands/mv.pyc +0 -0
  913. data/packages/gsutil/gslib/commands/perfdiag.py +0 -903
  914. data/packages/gsutil/gslib/commands/perfdiag.pyc +0 -0
  915. data/packages/gsutil/gslib/commands/rb.py +0 -113
  916. data/packages/gsutil/gslib/commands/rb.pyc +0 -0
  917. data/packages/gsutil/gslib/commands/rm.py +0 -239
  918. data/packages/gsutil/gslib/commands/rm.pyc +0 -0
  919. data/packages/gsutil/gslib/commands/setacl.py +0 -138
  920. data/packages/gsutil/gslib/commands/setacl.pyc +0 -0
  921. data/packages/gsutil/gslib/commands/setcors.py +0 -145
  922. data/packages/gsutil/gslib/commands/setcors.pyc +0 -0
  923. data/packages/gsutil/gslib/commands/setdefacl.py +0 -105
  924. data/packages/gsutil/gslib/commands/setdefacl.pyc +0 -0
  925. data/packages/gsutil/gslib/commands/setmeta.py +0 -428
  926. data/packages/gsutil/gslib/commands/setmeta.pyc +0 -0
  927. data/packages/gsutil/gslib/commands/setversioning.py +0 -114
  928. data/packages/gsutil/gslib/commands/setversioning.pyc +0 -0
  929. data/packages/gsutil/gslib/commands/setwebcfg.py +0 -190
  930. data/packages/gsutil/gslib/commands/setwebcfg.pyc +0 -0
  931. data/packages/gsutil/gslib/commands/test.py +0 -228
  932. data/packages/gsutil/gslib/commands/test.pyc +0 -0
  933. data/packages/gsutil/gslib/commands/update.py +0 -305
  934. data/packages/gsutil/gslib/commands/update.pyc +0 -0
  935. data/packages/gsutil/gslib/commands/version.py +0 -150
  936. data/packages/gsutil/gslib/commands/version.pyc +0 -0
  937. data/packages/gsutil/gslib/exception.py +0 -76
  938. data/packages/gsutil/gslib/exception.pyc +0 -0
  939. data/packages/gsutil/gslib/help_provider.py +0 -81
  940. data/packages/gsutil/gslib/help_provider.pyc +0 -0
  941. data/packages/gsutil/gslib/name_expansion.py +0 -550
  942. data/packages/gsutil/gslib/name_expansion.pyc +0 -0
  943. data/packages/gsutil/gslib/plurality_checkable_iterator.py +0 -56
  944. data/packages/gsutil/gslib/plurality_checkable_iterator.pyc +0 -0
  945. data/packages/gsutil/gslib/project_id.py +0 -67
  946. data/packages/gsutil/gslib/project_id.pyc +0 -0
  947. data/packages/gsutil/gslib/storage_uri_builder.py +0 -56
  948. data/packages/gsutil/gslib/storage_uri_builder.pyc +0 -0
  949. data/packages/gsutil/gslib/tests/__init__.pyc +0 -0
  950. data/packages/gsutil/gslib/tests/test_chacl.py +0 -236
  951. data/packages/gsutil/gslib/tests/test_cp.py +0 -267
  952. data/packages/gsutil/gslib/tests/test_data/test.gif +0 -0
  953. data/packages/gsutil/gslib/tests/test_data/test.mp3 +0 -0
  954. data/packages/gsutil/gslib/tests/test_ls.py +0 -66
  955. data/packages/gsutil/gslib/tests/test_mv.py +0 -69
  956. data/packages/gsutil/gslib/tests/test_naming.py +0 -989
  957. data/packages/gsutil/gslib/tests/test_perfdiag.py +0 -41
  958. data/packages/gsutil/gslib/tests/test_plurality_checkable_iterator.py +0 -67
  959. data/packages/gsutil/gslib/tests/test_rm.py +0 -143
  960. data/packages/gsutil/gslib/tests/test_setacl.py +0 -152
  961. data/packages/gsutil/gslib/tests/test_setcors.py +0 -168
  962. data/packages/gsutil/gslib/tests/test_setmeta.py +0 -91
  963. data/packages/gsutil/gslib/tests/test_setversioning.py +0 -44
  964. data/packages/gsutil/gslib/tests/test_setwebcfg.py +0 -63
  965. data/packages/gsutil/gslib/tests/test_thread_pool.py +0 -92
  966. data/packages/gsutil/gslib/tests/test_wildcard_iterator.py +0 -364
  967. data/packages/gsutil/gslib/tests/testcase/base.py +0 -89
  968. data/packages/gsutil/gslib/tests/testcase/integration_testcase.py +0 -197
  969. data/packages/gsutil/gslib/tests/testcase/unit_testcase.py +0 -230
  970. data/packages/gsutil/gslib/tests/util.py +0 -125
  971. data/packages/gsutil/gslib/tests/util.pyc +0 -0
  972. data/packages/gsutil/gslib/thread_pool.py +0 -79
  973. data/packages/gsutil/gslib/thread_pool.pyc +0 -0
  974. data/packages/gsutil/gslib/util.py +0 -151
  975. data/packages/gsutil/gslib/util.pyc +0 -0
  976. data/packages/gsutil/gslib/wildcard_iterator.py +0 -492
  977. data/packages/gsutil/gslib/wildcard_iterator.pyc +0 -0
  978. data/packages/gsutil/gsutil +0 -377
  979. data/packages/gsutil/gsutil.spec.in +0 -75
  980. data/packages/gsutil/oauth2_plugin/__init__.py +0 -22
  981. data/packages/gsutil/oauth2_plugin/__init__.pyc +0 -0
  982. data/packages/gsutil/oauth2_plugin/oauth2_client.py +0 -642
  983. data/packages/gsutil/oauth2_plugin/oauth2_client.pyc +0 -0
  984. data/packages/gsutil/oauth2_plugin/oauth2_client_test.py +0 -374
  985. data/packages/gsutil/oauth2_plugin/oauth2_helper.py +0 -110
  986. data/packages/gsutil/oauth2_plugin/oauth2_helper.pyc +0 -0
  987. data/packages/gsutil/oauth2_plugin/oauth2_plugin.py +0 -24
  988. data/packages/gsutil/oauth2_plugin/oauth2_plugin.pyc +0 -0
  989. data/packages/gsutil/pkg_gen.sh +0 -54
  990. data/packages/gsutil/pkg_util.py +0 -60
  991. data/packages/gsutil/setup.py +0 -141
  992. data/packages/gsutil/third_party/__init__.py +0 -0
  993. data/packages/gsutil/third_party/__init__.pyc +0 -0
  994. data/packages/gsutil/third_party/fancy_urllib/README +0 -21
  995. data/packages/gsutil/third_party/fancy_urllib/__init__.py +0 -398
  996. data/packages/gsutil/third_party/fancy_urllib/__init__.pyc +0 -0
  997. data/tasks/rubygem.rake +0 -31
  998. metadata.gz.sig +0 -0
@@ -1,1818 +0,0 @@
1
- # Copyright 2011 Google Inc. All Rights Reserved.
2
- # Copyright 2011, Nexenta Systems Inc.
3
- #
4
- # Licensed under the Apache License, Version 2.0 (the "License");
5
- # you may not use this file except in compliance with the License.
6
- # You may obtain a copy of the License at
7
- #
8
- # http://www.apache.org/licenses/LICENSE-2.0
9
- #
10
- # Unless required by applicable law or agreed to in writing, software
11
- # distributed under the License is distributed on an "AS IS" BASIS,
12
- # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
- # See the License for the specific language governing permissions and
14
- # limitations under the License.
15
-
16
- import boto
17
- import errno
18
- import gzip
19
- import hashlib
20
- import mimetypes
21
- import os
22
- import platform
23
- import re
24
- import subprocess
25
- import stat
26
- import sys
27
- import tempfile
28
- import threading
29
- import time
30
-
31
- from boto import config
32
- from boto.exception import GSResponseError
33
- from boto.exception import ResumableUploadException
34
- from boto.gs.resumable_upload_handler import ResumableUploadHandler
35
- from boto.s3.keyfile import KeyFile
36
- from boto.s3.resumable_download_handler import ResumableDownloadHandler
37
- from boto.storage_uri import BucketStorageUri
38
- from gslib.command import COMMAND_NAME
39
- from gslib.command import COMMAND_NAME_ALIASES
40
- from gslib.command import CONFIG_REQUIRED
41
- from gslib.command import Command
42
- from gslib.command import FILE_URIS_OK
43
- from gslib.command import MAX_ARGS
44
- from gslib.command import MIN_ARGS
45
- from gslib.command import PROVIDER_URIS_OK
46
- from gslib.command import SUPPORTED_SUB_ARGS
47
- from gslib.command import URIS_START_ARG
48
- from gslib.exception import CommandException
49
- from gslib.help_provider import HELP_NAME
50
- from gslib.help_provider import HELP_NAME_ALIASES
51
- from gslib.help_provider import HELP_ONE_LINE_SUMMARY
52
- from gslib.help_provider import HELP_TEXT
53
- from gslib.help_provider import HELP_TYPE
54
- from gslib.help_provider import HelpType
55
- from gslib.name_expansion import NameExpansionIterator
56
- from gslib.util import ExtractErrorDetail
57
- from gslib.util import IS_WINDOWS
58
- from gslib.util import MakeHumanReadable
59
- from gslib.util import NO_MAX
60
- from gslib.util import TWO_MB
61
- from gslib.wildcard_iterator import ContainsWildcard
62
-
63
- _detailed_help_text = ("""
64
- <B>SYNOPSIS</B>
65
- gsutil cp [OPTION]... src_uri dst_uri
66
- - or -
67
- gsutil cp [OPTION]... src_uri... dst_uri
68
- - or -
69
- gsutil cp [OPTION]... -I dst_uri
70
-
71
-
72
- <B>DESCRIPTION</B>
73
- The gsutil cp command allows you to copy data between your local file
74
- system and the cloud, copy data within the cloud, and copy data between
75
- cloud storage providers. For example, to copy all text files from the
76
- local directory to a bucket you could do:
77
-
78
- gsutil cp *.txt gs://my_bucket
79
-
80
- Similarly, you can download text files from a bucket by doing:
81
-
82
- gsutil cp gs://my_bucket/*.txt .
83
-
84
- If you want to copy an entire directory tree you need to use the -R option:
85
-
86
- gsutil cp -R dir gs://my_bucket
87
-
88
- If you have a large number of files to upload you might want to use the
89
- gsutil -m option, to perform a parallel (multi-threaded/multi-processing)
90
- copy:
91
-
92
- gsutil -m cp -R dir gs://my_bucket
93
-
94
- You can pass a list of URIs to copy on STDIN instead of as command line
95
- arguments by using the -I option. This allows you to use gsutil in a
96
- pipeline to copy files and objects as generated by a program, such as:
97
-
98
- some_program | gsutil -m cp -I gs://my_bucket
99
-
100
- The contents of STDIN can name files, cloud URIs, and wildcards of files
101
- and cloud URIs.
102
-
103
-
104
- <B>HOW NAMES ARE CONSTRUCTED</B>
105
- The gsutil cp command strives to name objects in a way consistent with how
106
- Linux cp works, which causes names to be constructed in varying ways depending
107
- on whether you're performing a recursive directory copy or copying
108
- individually named objects; and whether you're copying to an existing or
109
- non-existent directory.
110
-
111
- When performing recursive directory copies, object names are constructed
112
- that mirror the source directory structure starting at the point of
113
- recursive processing. For example, the command:
114
-
115
- gsutil cp -R dir1/dir2 gs://my_bucket
116
-
117
- will create objects named like gs://my_bucket/dir2/a/b/c, assuming
118
- dir1/dir2 contains the file a/b/c.
119
-
120
- In contrast, copying individually named files will result in objects named
121
- by the final path component of the source files. For example, the command:
122
-
123
- gsutil cp dir1/dir2/** gs://my_bucket
124
-
125
- will create objects named like gs://my_bucket/c.
126
-
127
- The same rules apply for downloads: recursive copies of buckets and
128
- bucket subdirectories produce a mirrored filename structure, while copying
129
- individually (or wildcard) named objects produce flatly named files.
130
-
131
- Note that in the above example the '**' wildcard matches all names
132
- anywhere under dir. The wildcard '*' will match names just one level deep. For
133
- more details see 'gsutil help wildcards'.
134
-
135
- There's an additional wrinkle when working with subdirectories: the resulting
136
- names depend on whether the destination subdirectory exists. For example,
137
- if gs://my_bucket/subdir exists as a subdirectory, the command:
138
-
139
- gsutil cp -R dir1/dir2 gs://my_bucket/subdir
140
-
141
- will create objects named like gs://my_bucket/subdir/dir2/a/b/c. In contrast,
142
- if gs://my_bucket/subdir does not exist, this same gsutil cp command will
143
- create objects named like gs://my_bucket/subdir/a/b/c.
144
-
145
-
146
- <B>COPYING TO/FROM SUBDIRECTORIES; DISTRIBUTING TRANSFERS ACROSS MACHINES</B>
147
- You can use gsutil to copy to and from subdirectories by using a command like:
148
-
149
- gsutil cp -R dir gs://my_bucket/data
150
-
151
- This will cause dir and all of its files and nested subdirectories to be
152
- copied under the specified destination, resulting in objects with names like
153
- gs://my_bucket/data/dir/a/b/c. Similarly you can download from bucket
154
- subdirectories by using a command like:
155
-
156
- gsutil cp -R gs://my_bucket/data dir
157
-
158
- This will cause everything nested under gs://my_bucket/data to be downloaded
159
- into dir, resulting in files with names like dir/data/a/b/c.
160
-
161
- Copying subdirectories is useful if you want to add data to an existing
162
- bucket directory structure over time. It's also useful if you want
163
- to parallelize uploads and downloads across multiple machines (often
164
- reducing overall transfer time compared with simply running gsutil -m
165
- cp on one machine). For example, if your bucket contains this structure:
166
-
167
- gs://my_bucket/data/result_set_01/
168
- gs://my_bucket/data/result_set_02/
169
- ...
170
- gs://my_bucket/data/result_set_99/
171
-
172
- you could perform concurrent downloads across 3 machines by running these
173
- commands on each machine, respectively:
174
-
175
- gsutil -m cp -R gs://my_bucket/data/result_set_[0-3]* dir
176
- gsutil -m cp -R gs://my_bucket/data/result_set_[4-6]* dir
177
- gsutil -m cp -R gs://my_bucket/data/result_set_[7-9]* dir
178
-
179
- Note that dir could be a local directory on each machine, or it could
180
- be a directory mounted off of a shared file server; whether the latter
181
- performs acceptably may depend on a number of things, so we recommend
182
- you experiment and find out what works best for you.
183
-
184
-
185
- <B>COPYING IN THE CLOUD AND METADATA PRESERVATION</B>
186
- If both the source and destination URI are cloud URIs from the same
187
- provider, gsutil copies data "in the cloud" (i.e., without downloading
188
- to and uploading from the machine where you run gsutil). In addition to
189
- the performance and cost advantages of doing this, copying in the cloud
190
- preserves metadata (like Content-Type and Cache-Control). In contrast,
191
- when you download data from the cloud it ends up in a file, which has
192
- no associated metadata. Thus, unless you have some way to hold on to
193
- or re-create that metadata, downloading to a file will not retain the
194
- metadata.
195
-
196
- Note that by default, the gsutil cp command does not copy the object
197
- ACL to the new object, and instead will use the default bucket ACL (see
198
- "gsutil help setdefacl"). You can override this behavior with the -p
199
- option (see OPTIONS below).
200
-
201
- gsutil does not preserve metadata when copying objects between providers.
202
-
203
-
204
- <B>RESUMABLE TRANSFERS</B>
205
- gsutil automatically uses the Google Cloud Storage resumable upload
206
- feature whenever you use the cp command to upload an object that is larger
207
- than 2 MB. You do not need to specify any special command line options
208
- to make this happen. If your upload is interrupted you can restart the
209
- upload by running the same cp command that you ran to start the upload.
210
-
211
- Similarly, gsutil automatically performs resumable downloads (using HTTP
212
- standard Range GET operations) whenever you use the cp command to download an
213
- object larger than 2 MB.
214
-
215
- Resumable uploads and downloads store some state information in a file
216
- in ~/.gsutil named by the destination object or file. If you attempt to
217
- resume a transfer from a machine with a different directory, the transfer
218
- will start over from scratch.
219
-
220
- See also "gsutil help prod" for details on using resumable transfers
221
- in production.
222
-
223
-
224
- <B>STREAMING TRANSFERS</B>
225
- Use '-' in place of src_uri or dst_uri to perform a streaming
226
- transfer. For example:
227
- long_running_computation | gsutil cp - gs://my_bucket/obj
228
-
229
- Streaming transfers do not support resumable uploads/downloads.
230
- (The Google resumable transfer protocol has a way to support streaming
231
- transers, but gsutil doesn't currently implement support for this.)
232
-
233
-
234
- <B>CHANGING TEMP DIRECTORIES</B>
235
- gsutil writes data to a temporary directory in several cases:
236
- - when compressing data to be uploaded (see the -z option)
237
- - when decompressing data being downloaded (when the data has
238
- Content-Encoding:gzip, e.g., as happens when uploaded using gsutil cp -z)
239
- - when running integration tests (using the gsutil test command)
240
-
241
- In these cases it's possible the temp file location on your system that
242
- gsutil selects by default may not have enough space. If you find that
243
- gsutil runs out of space during one of these operations (e.g., raising
244
- "CommandException: Inadequate temp space available to compress <your file>"
245
- during a gsutil cp -z operation), you can change where it writes these
246
- temp files by setting the TMPDIR environment variable. On Linux and MacOS
247
- you can do this either by running gsutil this way:
248
-
249
- TMPDIR=/some/directory gsutil cp ...
250
-
251
- or by adding this line to your ~/.bashrc file and then restarting the shell
252
- before running gsutil:
253
-
254
- export TMPDIR=/some/directory
255
-
256
- On Windows 7 you can change the TMPDIR environment variable from Start ->
257
- Computer -> System -> Advanced System Settings -> Environment Variables.
258
- You need to reboot after making this change for it to take effect. (Rebooting
259
- is not necessary after running the export command on Linux and MacOS.)
260
-
261
-
262
- <B>OPTIONS</B>
263
- -a canned_acl Sets named canned_acl when uploaded objects created. See
264
- 'gsutil help acls' for further details.
265
-
266
- -c If an error occurrs, continue to attempt to copy the remaining
267
- files.
268
-
269
- -D Copy in "daisy chain" mode, i.e., copying between two buckets by
270
- hooking a download to an upload, via the machine where gsutil is
271
- run. By default, data are copied between two buckets "in the
272
- cloud", i.e., without needing to copy via the machine where
273
- gsutil runs. However, copy-in-the-cloud is not supported when
274
- copying between different locations (like US and EU) or between
275
- different storage classes (like STANDARD and
276
- DURABLE_REDUCED_AVAILABILITY). For these cases, you can use the
277
- -D option to copy data between buckets.
278
- Note: Daisy chain mode is automatically used when copying
279
- between providers (e.g., to copy data from Google Cloud Storage
280
- to another provider).
281
-
282
- -e Exclude symlinks. When specified, symbolic links will not be
283
- copied.
284
-
285
- -n No-clobber. When specified, existing files or objects at the
286
- destination will not be overwritten. Any items that are skipped
287
- by this option will be reported as being skipped. This option
288
- will perform an additional HEAD request to check if an item
289
- exists before attempting to upload the data. This will save
290
- retransmitting data, but the additional HTTP requests may make
291
- small object transfers slower and more expensive.
292
-
293
- This option can be combined with the -c option to build a script
294
- that copies a large number of objects, allowing retries when
295
- some failures occur from which gsutil doesn't automatically
296
- recover, using a bash script like the following:
297
-
298
- status=1
299
- while [ $status -ne 0 ] ; do
300
- gsutil cp -c -n -R ./dir gs://bucket
301
- status=$?
302
- done
303
-
304
- The -c option will cause copying to continue after failures
305
- occur, and the -n option will cause objects already copied to be
306
- skipped on subsequent iterations. The loop will continue running
307
- as long as gsutil exits with a non-zero status (such a status
308
- indicates there was at least one failure during the gsutil run).
309
-
310
- -p Causes ACLs to be preserved when copying in the cloud. Note that
311
- this option has performance and cost implications, because it
312
- is essentially performing three requests (getacl, cp, setacl).
313
- (The performance issue can be mitigated to some degree by
314
- using gsutil -m cp to cause parallel copying.)
315
-
316
- You can avoid the additional performance and cost of using cp -p
317
- if you want all objects in the destination bucket to end up with
318
- the same ACL by setting a default ACL on that bucket instead of
319
- using cp -p. See "help gsutil setdefacl".
320
-
321
- Note that it's not valid to specify both the -a and -p options
322
- together.
323
-
324
- -q Causes copies to be performed quietly, i.e., without reporting
325
- progress indicators of files being copied. Errors are still
326
- reported. This option can be useful for running gsutil from a
327
- cron job that logs its output to a file, for which the only
328
- information desired in the log is failures.
329
-
330
- -R, -r Causes directories, buckets, and bucket subdirectories to be
331
- copied recursively. If you neglect to use this option for
332
- an upload, gsutil will copy any files it finds and skip any
333
- directories. Similarly, neglecting to specify -R for a download
334
- will cause gsutil to copy any objects at the current bucket
335
- directory level, and skip any subdirectories.
336
-
337
- -v Requests that the version-specific URI for each uploaded object
338
- be printed. Given this URI you can make future upload requests
339
- that are safe in the face of concurrent updates, because Google
340
- Cloud Storage will refuse to perform the update if the current
341
- object version doesn't match the version-specific URI. See
342
- 'gsutil help versioning' for more details. Note: at present this
343
- option does not work correctly for objects copied "in the cloud"
344
- (e.g., gsutil cp gs://bucket/obj1 gs://bucket/obj2).
345
-
346
- -z ext1,... Compresses file uploads with the given extensions. If you are
347
- uploading a large file with compressible content, such as
348
- a .js, .css, or .html file, you can gzip-compress the file
349
- during the upload process by specifying the -z <extensions>
350
- option. Compressing data before upload saves on usage charges
351
- because you are uploading a smaller amount of data.
352
-
353
- When you specify the -z option, the data from your files is
354
- compressed before it is uploaded, but your actual files are left
355
- uncompressed on the local disk. The uploaded objects retain the
356
- original content type and name as the original files but are
357
- given a Content-Encoding header with the value "gzip" to
358
- indicate that the object data stored are compressed on the
359
- Google Cloud Storage servers.
360
-
361
- For example, the following command:
362
-
363
- gsutil cp -z html -a public-read cattypes.html gs://mycats
364
-
365
- will do all of the following:
366
- - Upload as the object gs://mycats/cattypes.html (cp command)
367
- - Set the Content-Type to text/html (based on file extension)
368
- - Compress the data in the file cattypes.html (-z option)
369
- - Set the Content-Encoding to gzip (-z option)
370
- - Set the ACL to public-read (-a option)
371
- - If a user tries to view cattypes.html in a browser, the
372
- browser will know to uncompress the data based on the
373
- Content-Encoding header, and to render it as HTML based on
374
- the Content-Type header.
375
- """)
376
-
377
- class CpCommand(Command):
378
- """
379
- Implementation of gsutil cp command.
380
-
381
- Note that CpCommand is run for both gsutil cp and gsutil mv. The latter
382
- happens by MvCommand calling CpCommand and passing the hidden (undocumented)
383
- -M option. This allows the copy and remove needed for each mv to run
384
- together (rather than first running all the cp's and then all the rm's, as
385
- we originally had implemented), which in turn avoids the following problem
386
- with removing the wrong objects: starting with a bucket containing only
387
- the object gs://bucket/obj, say the user does:
388
- gsutil mv gs://bucket/* gs://bucket/d.txt
389
- If we ran all the cp's and then all the rm's and we didn't expand the wildcard
390
- first, the cp command would first copy gs://bucket/obj to gs://bucket/d.txt,
391
- and the rm command would then remove that object. In the implementation
392
- prior to gsutil release 3.12 we avoided this by building a list of objects
393
- to process and then running the copies and then the removes; but building
394
- the list up front limits scalability (compared with the current approach
395
- of processing the bucket listing iterator on the fly).
396
- """
397
-
398
- # Set default Content-Type type.
399
- DEFAULT_CONTENT_TYPE = 'application/octet-stream'
400
- USE_MAGICFILE = boto.config.getbool('GSUtil', 'use_magicfile', False)
401
-
402
- # Command specification (processed by parent class).
403
- command_spec = {
404
- # Name of command.
405
- COMMAND_NAME : 'cp',
406
- # List of command name aliases.
407
- COMMAND_NAME_ALIASES : ['copy'],
408
- # Min number of args required by this command.
409
- MIN_ARGS : 1,
410
- # Max number of args required by this command, or NO_MAX.
411
- MAX_ARGS : NO_MAX,
412
- # Getopt-style string specifying acceptable sub args.
413
- # -t is deprecated but leave intact for now to avoid breakage.
414
- SUPPORTED_SUB_ARGS : 'a:cDeIMNnpqrRtvz:',
415
- # True if file URIs acceptable for this command.
416
- FILE_URIS_OK : True,
417
- # True if provider-only URIs acceptable for this command.
418
- PROVIDER_URIS_OK : False,
419
- # Index in args of first URI arg.
420
- URIS_START_ARG : 0,
421
- # True if must configure gsutil before running command.
422
- CONFIG_REQUIRED : True,
423
- }
424
- help_spec = {
425
- # Name of command or auxiliary help info for which this help applies.
426
- HELP_NAME : 'cp',
427
- # List of help name aliases.
428
- HELP_NAME_ALIASES : ['copy'],
429
- # Type of help:
430
- HELP_TYPE : HelpType.COMMAND_HELP,
431
- # One line summary of this help.
432
- HELP_ONE_LINE_SUMMARY : 'Copy files and objects',
433
- # The full help text.
434
- HELP_TEXT : _detailed_help_text,
435
- }
436
-
437
- def _CheckFinalMd5(self, key, file_name):
438
- """
439
- Checks that etag from server agrees with md5 computed after the
440
- download completes.
441
- """
442
- obj_md5 = key.etag.strip('"\'')
443
- file_md5 = None
444
-
445
- if hasattr(key, 'md5') and key.md5:
446
- file_md5 = key.md5
447
- else:
448
- print 'Computing MD5 from scratch for resumed download'
449
-
450
- # Open file in binary mode to avoid surprises in Windows.
451
- fp = open(file_name, 'rb')
452
- try:
453
- file_md5 = key.compute_md5(fp)[0]
454
- finally:
455
- fp.close()
456
-
457
- if self.debug:
458
- print 'Checking file md5 against etag. (%s/%s)' % (file_md5, obj_md5)
459
- if file_md5 != obj_md5:
460
- # Checksums don't match - remove file and raise exception.
461
- os.unlink(file_name)
462
- raise CommandException(
463
- 'File changed during download: md5 signature doesn\'t match '
464
- 'etag (incorrect downloaded file deleted)')
465
-
466
- def _CheckForDirFileConflict(self, exp_src_uri, dst_uri):
467
- """Checks whether copying exp_src_uri into dst_uri is not possible.
468
-
469
- This happens if a directory exists in local file system where a file
470
- needs to go or vice versa. In that case we print an error message and
471
- exits. Example: if the file "./x" exists and you try to do:
472
- gsutil cp gs://mybucket/x/y .
473
- the request can't succeed because it requires a directory where
474
- the file x exists.
475
-
476
- Note that we don't enforce any corresponding restrictions for buckets,
477
- because the flat namespace semantics for buckets doesn't prohibit such
478
- cases the way hierarchical file systems do. For example, if a bucket
479
- contains an object called gs://bucket/dir and then you run the command:
480
- gsutil cp file1 file2 gs://bucket/dir
481
- you'll end up with objects gs://bucket/dir, gs://bucket/dir/file1, and
482
- gs://bucket/dir/file2.
483
-
484
- Args:
485
- exp_src_uri: Expanded source StorageUri of copy.
486
- dst_uri: Destination URI.
487
-
488
- Raises:
489
- CommandException: if errors encountered.
490
- """
491
- if dst_uri.is_cloud_uri():
492
- # The problem can only happen for file destination URIs.
493
- return
494
- dst_path = dst_uri.object_name
495
- final_dir = os.path.dirname(dst_path)
496
- if os.path.isfile(final_dir):
497
- raise CommandException('Cannot retrieve %s because a file exists '
498
- 'where a directory needs to be created (%s).' %
499
- (exp_src_uri, final_dir))
500
- if os.path.isdir(dst_path):
501
- raise CommandException('Cannot retrieve %s because a directory exists '
502
- '(%s) where the file needs to be created.' %
503
- (exp_src_uri, dst_path))
504
-
505
- def _InsistDstUriNamesContainer(self, exp_dst_uri,
506
- have_existing_dst_container, command_name):
507
- """
508
- Raises an exception if URI doesn't name a directory, bucket, or bucket
509
- subdir, with special exception for cp -R (see comments below).
510
-
511
- Args:
512
- exp_dst_uri: Wildcard-expanding dst_uri.
513
- have_existing_dst_container: bool indicator of whether exp_dst_uri
514
- names a container (directory, bucket, or existing bucket subdir).
515
- command_name: Name of command making call. May not be the same as
516
- self.command_name in the case of commands implemented atop other
517
- commands (like mv command).
518
-
519
- Raises:
520
- CommandException: if the URI being checked does not name a container.
521
- """
522
- if exp_dst_uri.is_file_uri():
523
- ok = exp_dst_uri.names_directory()
524
- else:
525
- if have_existing_dst_container:
526
- ok = True
527
- else:
528
- # It's ok to specify a non-existing bucket subdir, for example:
529
- # gsutil cp -R dir gs://bucket/abc
530
- # where gs://bucket/abc isn't an existing subdir.
531
- ok = exp_dst_uri.names_object()
532
- if not ok:
533
- raise CommandException('Destination URI must name a directory, bucket, '
534
- 'or bucket\nsubdirectory for the multiple '
535
- 'source form of the %s command.' % command_name)
536
-
537
- class _FileCopyCallbackHandler(object):
538
- """Outputs progress info for large copy requests."""
539
-
540
- def __init__(self, upload):
541
- if upload:
542
- self.announce_text = 'Uploading'
543
- else:
544
- self.announce_text = 'Downloading'
545
-
546
- def call(self, total_bytes_transferred, total_size):
547
- sys.stderr.write('%s: %s/%s \r' % (
548
- self.announce_text,
549
- MakeHumanReadable(total_bytes_transferred),
550
- MakeHumanReadable(total_size)))
551
- if total_bytes_transferred == total_size:
552
- sys.stderr.write('\n')
553
-
554
- class _StreamCopyCallbackHandler(object):
555
- """Outputs progress info for Stream copy to cloud.
556
- Total Size of the stream is not known, so we output
557
- only the bytes transferred.
558
- """
559
-
560
- def call(self, total_bytes_transferred, total_size):
561
- sys.stderr.write('Uploading: %s \r' % (
562
- MakeHumanReadable(total_bytes_transferred)))
563
- if total_size and total_bytes_transferred == total_size:
564
- sys.stderr.write('\n')
565
-
566
- def _GetTransferHandlers(self, dst_uri, size, upload):
567
- """
568
- Selects upload/download and callback handlers.
569
-
570
- We use a callback handler that shows a simple textual progress indicator
571
- if size is above the configurable threshold.
572
-
573
- We use a resumable transfer handler if size is >= the configurable
574
- threshold and resumable transfers are supported by the given provider.
575
- boto supports resumable downloads for all providers, but resumable
576
- uploads are currently only supported by GS.
577
-
578
- Args:
579
- dst_uri: the destination URI.
580
- size: size of file (object) being uploaded (downloaded).
581
- upload: bool indication of whether transfer is an upload.
582
- """
583
- config = boto.config
584
- resumable_threshold = config.getint('GSUtil', 'resumable_threshold', TWO_MB)
585
- transfer_handler = None
586
- cb = None
587
- num_cb = None
588
-
589
- # Checks whether the destination file is a "special" file, like /dev/null on
590
- # Linux platforms or null on Windows platforms, so we can disable resumable
591
- # download support since the file size of the destination won't ever be
592
- # correct.
593
- dst_is_special = False
594
- if dst_uri.is_file_uri():
595
- # Check explicitly first because os.stat doesn't work on 'nul' in Windows.
596
- if dst_uri.object_name == os.devnull:
597
- dst_is_special = True
598
- try:
599
- mode = os.stat(dst_uri.object_name).st_mode
600
- if stat.S_ISCHR(mode):
601
- dst_is_special = True
602
- except OSError:
603
- pass
604
-
605
- if size >= resumable_threshold and not dst_is_special:
606
- if not self.quiet:
607
- cb = self._FileCopyCallbackHandler(upload).call
608
- num_cb = int(size / TWO_MB)
609
-
610
- resumable_tracker_dir = config.get(
611
- 'GSUtil', 'resumable_tracker_dir',
612
- os.path.expanduser('~' + os.sep + '.gsutil'))
613
- if not os.path.exists(resumable_tracker_dir):
614
- os.makedirs(resumable_tracker_dir)
615
-
616
- if upload:
617
- # Encode the dest bucket and object name into the tracker file name.
618
- res_tracker_file_name = (
619
- re.sub('[/\\\\]', '_', 'resumable_upload__%s__%s.url' %
620
- (dst_uri.bucket_name, dst_uri.object_name)))
621
- else:
622
- # Encode the fully-qualified dest file name into the tracker file name.
623
- res_tracker_file_name = (
624
- re.sub('[/\\\\]', '_', 'resumable_download__%s.etag' %
625
- (os.path.realpath(dst_uri.object_name))))
626
-
627
- res_tracker_file_name = _hash_filename(res_tracker_file_name)
628
- tracker_file = '%s%s%s' % (resumable_tracker_dir, os.sep,
629
- res_tracker_file_name)
630
- if upload:
631
- if dst_uri.scheme == 'gs':
632
- transfer_handler = ResumableUploadHandler(tracker_file)
633
- else:
634
- transfer_handler = ResumableDownloadHandler(tracker_file)
635
-
636
- return (cb, num_cb, transfer_handler)
637
-
638
- def _LogCopyOperation(self, src_uri, dst_uri, headers):
639
- """
640
- Logs copy operation being performed, including Content-Type if appropriate.
641
- """
642
- if self.quiet:
643
- return
644
- if 'Content-Type' in headers and dst_uri.is_cloud_uri():
645
- content_type_msg = ' [Content-Type=%s]' % headers['Content-Type']
646
- else:
647
- content_type_msg = ''
648
- if src_uri.is_stream():
649
- self.THREADED_LOGGER.info('Copying from <STDIN>%s...', content_type_msg)
650
- else:
651
- self.THREADED_LOGGER.info('Copying %s%s...', src_uri, content_type_msg)
652
-
653
- # We pass the headers explicitly to this call instead of using self.headers
654
- # so we can set different metadata (like Content-Type type) for each object.
655
- def _CopyObjToObjInTheCloud(self, src_key, src_uri, dst_uri, headers):
656
- """Performs copy-in-the cloud from specified src to dest object.
657
-
658
- Args:
659
- src_key: Source Key.
660
- src_uri: Source StorageUri.
661
- dst_uri: Destination StorageUri.
662
- headers: A copy of the headers dictionary.
663
-
664
- Returns:
665
- (elapsed_time, bytes_transferred, dst_uri) excluding overhead like initial
666
- HEAD. Note: At present copy-in-the-cloud doesn't return the generation of
667
- the created object, so the returned URI is actually not version-specific
668
- (unlike other cp cases).
669
-
670
- Raises:
671
- CommandException: if errors encountered.
672
- """
673
- self._SetContentTypeHeader(src_uri, headers)
674
- self._LogCopyOperation(src_uri, dst_uri, headers)
675
- # Do Object -> object copy within same provider (uses
676
- # x-<provider>-copy-source metadata HTTP header to request copying at the
677
- # server).
678
- src_bucket = src_uri.get_bucket(False, headers)
679
- preserve_acl = False
680
- canned_acl = None
681
- if self.sub_opts:
682
- for o, a in self.sub_opts:
683
- if o == '-a':
684
- canned_acls = dst_uri.canned_acls()
685
- if a not in canned_acls:
686
- raise CommandException('Invalid canned ACL "%s".' % a)
687
- canned_acl = a
688
- headers[dst_uri.get_provider().acl_header] = canned_acl
689
- if o == '-p':
690
- preserve_acl = True
691
- if preserve_acl and canned_acl:
692
- raise CommandException(
693
- 'Specifying both the -p and -a options together is invalid.')
694
- start_time = time.time()
695
- # Pass headers in headers param not metadata param, so boto will copy
696
- # existing key's metadata and just set the additional headers specified
697
- # in the headers param (rather than using the headers to override existing
698
- # metadata). In particular this allows us to copy the existing key's
699
- # Content-Type and other metadata users need while still being able to
700
- # set headers the API needs (like x-goog-project-id). Note that this means
701
- # you can't do something like:
702
- # gsutil cp -t Content-Type text/html gs://bucket/* gs://bucket2
703
- # to change the Content-Type while copying.
704
-
705
- try:
706
- dst_key = dst_uri.copy_key(
707
- src_bucket.name, src_uri.object_name, preserve_acl=False,
708
- headers=headers, src_version_id=src_uri.version_id,
709
- src_generation=src_uri.generation)
710
- except GSResponseError as e:
711
- exc_name, error_detail = ExtractErrorDetail(e)
712
- if (exc_name == 'GSResponseError'
713
- and ('Copy-in-the-cloud disallowed' in error_detail)):
714
- raise CommandException('%s.\nNote: you can copy between locations '
715
- 'and between storage classes by using the '
716
- 'gsutil cp -D option.' % error_detail)
717
- else:
718
- raise
719
- end_time = time.time()
720
- return (end_time - start_time, src_key.size,
721
- dst_uri.clone_replace_key(dst_key))
722
-
723
- def _CheckFreeSpace(self, path):
724
- """Return path/drive free space (in bytes)."""
725
- if platform.system() == 'Windows':
726
- from ctypes import c_int, c_uint64, c_wchar_p, windll, POINTER, WINFUNCTYPE, WinError
727
- try:
728
- GetDiskFreeSpaceEx = WINFUNCTYPE(c_int, c_wchar_p, POINTER(c_uint64),
729
- POINTER(c_uint64), POINTER(c_uint64))
730
- GetDiskFreeSpaceEx = GetDiskFreeSpaceEx(
731
- ('GetDiskFreeSpaceExW', windll.kernel32), (
732
- (1, 'lpszPathName'),
733
- (2, 'lpFreeUserSpace'),
734
- (2, 'lpTotalSpace'),
735
- (2, 'lpFreeSpace'),))
736
- except AttributeError:
737
- GetDiskFreeSpaceEx = WINFUNCTYPE(c_int, c_char_p, POINTER(c_uint64),
738
- POINTER(c_uint64), POINTER(c_uint64))
739
- GetDiskFreeSpaceEx = GetDiskFreeSpaceEx(
740
- ('GetDiskFreeSpaceExA', windll.kernel32), (
741
- (1, 'lpszPathName'),
742
- (2, 'lpFreeUserSpace'),
743
- (2, 'lpTotalSpace'),
744
- (2, 'lpFreeSpace'),))
745
-
746
- def GetDiskFreeSpaceEx_errcheck(result, func, args):
747
- if not result:
748
- raise WinError()
749
- return args[1].value
750
- GetDiskFreeSpaceEx.errcheck = GetDiskFreeSpaceEx_errcheck
751
-
752
- return GetDiskFreeSpaceEx(os.getenv('SystemDrive'))
753
- else:
754
- (_, f_frsize, _, _, f_bavail, _, _, _, _, _) = os.statvfs(path)
755
- return f_frsize * f_bavail
756
-
757
- def _PerformResumableUploadIfApplies(self, fp, dst_uri, canned_acl, headers):
758
- """
759
- Performs resumable upload if supported by provider and file is above
760
- threshold, else performs non-resumable upload.
761
-
762
- Returns (elapsed_time, bytes_transferred, version-specific dst_uri).
763
- """
764
- start_time = time.time()
765
- # Determine file size different ways for case where fp is actually a wrapper
766
- # around a Key vs an actual file.
767
- if isinstance(fp, KeyFile):
768
- file_size = fp.getkey().size
769
- else:
770
- file_size = os.path.getsize(fp.name)
771
- (cb, num_cb, res_upload_handler) = self._GetTransferHandlers(
772
- dst_uri, file_size, True)
773
- if dst_uri.scheme == 'gs':
774
- # Resumable upload protocol is Google Cloud Storage-specific.
775
- dst_uri.set_contents_from_file(fp, headers, policy=canned_acl,
776
- cb=cb, num_cb=num_cb,
777
- res_upload_handler=res_upload_handler)
778
- else:
779
- dst_uri.set_contents_from_file(fp, headers, policy=canned_acl,
780
- cb=cb, num_cb=num_cb)
781
- if res_upload_handler:
782
- # ResumableUploadHandler does not update upload_start_point from its
783
- # initial value of -1 if transferring the whole file, so clamp at 0
784
- bytes_transferred = file_size - max(
785
- res_upload_handler.upload_start_point, 0)
786
- else:
787
- bytes_transferred = file_size
788
- end_time = time.time()
789
- return (end_time - start_time, bytes_transferred, dst_uri)
790
-
791
- def _PerformStreamingUpload(self, fp, dst_uri, headers, canned_acl=None):
792
- """
793
- Performs a streaming upload to the cloud.
794
-
795
- Args:
796
- fp: The file whose contents to upload.
797
- dst_uri: Destination StorageUri.
798
- headers: A copy of the headers dictionary.
799
- canned_acl: Optional canned ACL to set on the object.
800
-
801
- Returns (elapsed_time, bytes_transferred, version-specific dst_uri).
802
- """
803
- start_time = time.time()
804
-
805
- if self.quiet:
806
- cb = None
807
- else:
808
- cb = self._StreamCopyCallbackHandler().call
809
- dst_uri.set_contents_from_stream(
810
- fp, headers, policy=canned_acl, cb=cb)
811
- try:
812
- bytes_transferred = fp.tell()
813
- except:
814
- bytes_transferred = 0
815
-
816
- end_time = time.time()
817
- return (end_time - start_time, bytes_transferred, dst_uri)
818
-
819
- def _SetContentTypeHeader(self, src_uri, headers):
820
- """
821
- Sets content type header to value specified in '-h Content-Type' option (if
822
- specified); else sets using Content-Type detection.
823
- """
824
- if 'Content-Type' in headers:
825
- # If empty string specified (i.e., -h "Content-Type:") set header to None,
826
- # which will inhibit boto from sending the CT header. Otherwise, boto will
827
- # pass through the user specified CT header.
828
- if not headers['Content-Type']:
829
- headers['Content-Type'] = None
830
- # else we'll keep the value passed in via -h option (not performing
831
- # content type detection).
832
- else:
833
- # Only do content type recognition is src_uri is a file. Object-to-object
834
- # copies with no -h Content-Type specified re-use the content type of the
835
- # source object.
836
- if src_uri.is_file_uri():
837
- object_name = src_uri.object_name
838
- content_type = None
839
- # Streams (denoted by '-') are expected to be 'application/octet-stream'
840
- # and 'file' would partially consume them.
841
- if object_name != '-':
842
- if self.USE_MAGICFILE:
843
- p = subprocess.Popen(['file', '--mime-type', object_name],
844
- stdout=subprocess.PIPE, stderr=subprocess.PIPE)
845
- output, error = p.communicate()
846
- if p.returncode != 0 or error:
847
- raise CommandException(
848
- 'Encountered error running "file --mime-type %s" '
849
- '(returncode=%d).\n%s' % (object_name, p.returncode, error))
850
- # Parse output by removing line delimiter and splitting on last ":
851
- content_type = output.rstrip().rpartition(': ')[2]
852
- else:
853
- content_type = mimetypes.guess_type(object_name)[0]
854
- if not content_type:
855
- content_type = self.DEFAULT_CONTENT_TYPE
856
- headers['Content-Type'] = content_type
857
-
858
- def _UploadFileToObject(self, src_key, src_uri, dst_uri, headers,
859
- should_log=True):
860
- """Uploads a local file to an object.
861
-
862
- Args:
863
- src_key: Source StorageUri. Must be a file URI.
864
- src_uri: Source StorageUri.
865
- dst_uri: Destination StorageUri.
866
- headers: The headers dictionary.
867
- should_log: bool indicator whether we should log this operation.
868
- Returns:
869
- (elapsed_time, bytes_transferred, version-specific dst_uri), excluding
870
- overhead like initial HEAD.
871
-
872
- Raises:
873
- CommandException: if errors encountered.
874
- """
875
- gzip_exts = []
876
- canned_acl = None
877
- if self.sub_opts:
878
- for o, a in self.sub_opts:
879
- if o == '-a':
880
- canned_acls = dst_uri.canned_acls()
881
- if a not in canned_acls:
882
- raise CommandException('Invalid canned ACL "%s".' % a)
883
- canned_acl = a
884
- elif o == '-t':
885
- print('Warning: -t is deprecated, and will be removed in the future. '
886
- 'Content type\ndetection is '
887
- 'now performed by default, unless inhibited by specifying '
888
- 'a\nContent-Type header via the -h option.')
889
- elif o == '-z':
890
- gzip_exts = a.split(',')
891
-
892
- self._SetContentTypeHeader(src_uri, headers)
893
- if should_log:
894
- self._LogCopyOperation(src_uri, dst_uri, headers)
895
-
896
- if 'Content-Language' not in headers:
897
- content_language = config.get_value('GSUtil', 'content_language')
898
- if content_language:
899
- headers['Content-Language'] = content_language
900
-
901
- fname_parts = src_uri.object_name.split('.')
902
- if len(fname_parts) > 1 and fname_parts[-1] in gzip_exts:
903
- if self.debug:
904
- print 'Compressing %s (to tmp)...' % src_key
905
- (gzip_fh, gzip_path) = tempfile.mkstemp()
906
- gzip_fp = None
907
- try:
908
- # Check for temp space. Assume the compressed object is at most 2x
909
- # the size of the object (normally should compress to smaller than
910
- # the object)
911
- if (self._CheckFreeSpace(gzip_path)
912
- < 2*int(os.path.getsize(src_key.name))):
913
- raise CommandException('Inadequate temp space available to compress '
914
- '%s' % src_key.name)
915
- gzip_fp = gzip.open(gzip_path, 'wb')
916
- gzip_fp.writelines(src_key.fp)
917
- finally:
918
- if gzip_fp:
919
- gzip_fp.close()
920
- os.close(gzip_fh)
921
- headers['Content-Encoding'] = 'gzip'
922
- gzip_fp = open(gzip_path, 'rb')
923
- try:
924
- (elapsed_time, bytes_transferred, result_uri) = (
925
- self._PerformResumableUploadIfApplies(gzip_fp, dst_uri,
926
- canned_acl, headers))
927
- finally:
928
- gzip_fp.close()
929
- try:
930
- os.unlink(gzip_path)
931
- # Windows sometimes complains the temp file is locked when you try to
932
- # delete it.
933
- except Exception, e:
934
- pass
935
- elif (src_key.is_stream()
936
- and dst_uri.get_provider().supports_chunked_transfer()):
937
- (elapsed_time, bytes_transferred, result_uri) = (
938
- self._PerformStreamingUpload(src_key.fp, dst_uri, headers,
939
- canned_acl))
940
- else:
941
- if src_key.is_stream():
942
- # For Providers that doesn't support chunked Transfers
943
- tmp = tempfile.NamedTemporaryFile()
944
- file_uri = self.suri_builder.StorageUri('file://%s' % tmp.name)
945
- try:
946
- file_uri.new_key(False, headers).set_contents_from_file(
947
- src_key.fp, headers)
948
- src_key = file_uri.get_key()
949
- finally:
950
- file_uri.close()
951
- try:
952
- (elapsed_time, bytes_transferred, result_uri) = (
953
- self._PerformResumableUploadIfApplies(src_key.fp, dst_uri,
954
- canned_acl, headers))
955
- finally:
956
- if src_key.is_stream():
957
- tmp.close()
958
- else:
959
- src_key.close()
960
-
961
- return (elapsed_time, bytes_transferred, result_uri)
962
-
963
- def _DownloadObjectToFile(self, src_key, src_uri, dst_uri, headers,
964
- should_log=True):
965
- """Downloads an object to a local file.
966
-
967
- Args:
968
- src_key: Source StorageUri. Must be a file URI.
969
- src_uri: Source StorageUri.
970
- dst_uri: Destination StorageUri.
971
- headers: The headers dictionary.
972
- should_log: bool indicator whether we should log this operation.
973
- Returns:
974
- (elapsed_time, bytes_transferred, dst_uri), excluding overhead like
975
- initial HEAD.
976
-
977
- Raises:
978
- CommandException: if errors encountered.
979
- """
980
- if should_log:
981
- self._LogCopyOperation(src_uri, dst_uri, headers)
982
- (cb, num_cb, res_download_handler) = self._GetTransferHandlers(
983
- dst_uri, src_key.size, False)
984
- file_name = dst_uri.object_name
985
- dir_name = os.path.dirname(file_name)
986
- if dir_name and not os.path.exists(dir_name):
987
- # Do dir creation in try block so can ignore case where dir already
988
- # exists. This is needed to avoid a race condition when running gsutil
989
- # -m cp.
990
- try:
991
- os.makedirs(dir_name)
992
- except OSError, e:
993
- if e.errno != errno.EEXIST:
994
- raise
995
- # For gzipped objects not named *.gz download to a temp file and unzip.
996
- if (hasattr(src_key, 'content_encoding')
997
- and src_key.content_encoding == 'gzip'
998
- and not file_name.endswith('.gz')):
999
- # We can't use tempfile.mkstemp() here because we need a predictable
1000
- # filename for resumable downloads.
1001
- download_file_name = '%s_.gztmp' % file_name
1002
- need_to_unzip = True
1003
- else:
1004
- download_file_name = file_name
1005
- need_to_unzip = False
1006
- fp = None
1007
- try:
1008
- if res_download_handler:
1009
- fp = open(download_file_name, 'ab')
1010
- else:
1011
- fp = open(download_file_name, 'wb')
1012
- start_time = time.time()
1013
- src_key.get_contents_to_file(fp, headers, cb=cb, num_cb=num_cb,
1014
- res_download_handler=res_download_handler)
1015
- # If a custom test method is defined, call it here. For the copy command,
1016
- # test methods are expected to take one argument: an open file pointer,
1017
- # and are used to perturb the open file during download to exercise
1018
- # download error detection.
1019
- if self.test_method:
1020
- self.test_method(fp)
1021
- end_time = time.time()
1022
- finally:
1023
- if fp:
1024
- fp.close()
1025
-
1026
- # Discard the md5 if we are resuming a partial download.
1027
- if res_download_handler and res_download_handler.download_start_point:
1028
- src_key.md5 = None
1029
-
1030
- # Verify downloaded file checksum matched source object's checksum.
1031
- self._CheckFinalMd5(src_key, download_file_name)
1032
-
1033
- if res_download_handler:
1034
- bytes_transferred = (
1035
- src_key.size - res_download_handler.download_start_point)
1036
- else:
1037
- bytes_transferred = src_key.size
1038
- if need_to_unzip:
1039
- # Log that we're uncompressing if the file is big enough that
1040
- # decompressing would make it look like the transfer "stalled" at the end.
1041
- if not self.quiet and bytes_transferred > 10 * 1024 * 1024:
1042
- self.THREADED_LOGGER.info('Uncompressing downloaded tmp file to %s...',
1043
- file_name)
1044
- # Downloaded gzipped file to a filename w/o .gz extension, so unzip.
1045
- f_in = gzip.open(download_file_name, 'rb')
1046
- f_out = open(file_name, 'wb')
1047
- try:
1048
- while True:
1049
- data = f_in.read(8192)
1050
- if not data:
1051
- break
1052
- f_out.write(data)
1053
- finally:
1054
- f_out.close()
1055
- f_in.close()
1056
- os.unlink(download_file_name)
1057
- return (end_time - start_time, bytes_transferred, dst_uri)
1058
-
1059
- def _PerformDownloadToStream(self, src_key, src_uri, str_fp, headers):
1060
- (cb, num_cb, res_download_handler) = self._GetTransferHandlers(
1061
- src_uri, src_key.size, False)
1062
- start_time = time.time()
1063
- src_key.get_contents_to_file(str_fp, headers, cb=cb, num_cb=num_cb)
1064
- end_time = time.time()
1065
- bytes_transferred = src_key.size
1066
- end_time = time.time()
1067
- return (end_time - start_time, bytes_transferred)
1068
-
1069
- def _CopyFileToFile(self, src_key, src_uri, dst_uri, headers):
1070
- """Copies a local file to a local file.
1071
-
1072
- Args:
1073
- src_key: Source StorageUri. Must be a file URI.
1074
- src_uri: Source StorageUri.
1075
- dst_uri: Destination StorageUri.
1076
- headers: The headers dictionary.
1077
- Returns:
1078
- (elapsed_time, bytes_transferred, dst_uri), excluding
1079
- overhead like initial HEAD.
1080
-
1081
- Raises:
1082
- CommandException: if errors encountered.
1083
- """
1084
- self._LogCopyOperation(src_uri, dst_uri, headers)
1085
- dst_key = dst_uri.new_key(False, headers)
1086
- start_time = time.time()
1087
- dst_key.set_contents_from_file(src_key.fp, headers)
1088
- end_time = time.time()
1089
- return (end_time - start_time, os.path.getsize(src_key.fp.name), dst_uri)
1090
-
1091
- def _CopyObjToObjDaisyChainMode(self, src_key, src_uri, dst_uri, headers):
1092
- """Copies from src_uri to dst_uri in "daisy chain" mode.
1093
- See -D OPTION documentation about what daisy chain mode is.
1094
-
1095
- Args:
1096
- src_key: Source Key.
1097
- src_uri: Source StorageUri.
1098
- dst_uri: Destination StorageUri.
1099
- headers: A copy of the headers dictionary.
1100
-
1101
- Returns:
1102
- (elapsed_time, bytes_transferred, version-specific dst_uri) excluding
1103
- overhead like initial HEAD.
1104
-
1105
- Raises:
1106
- CommandException: if errors encountered.
1107
- """
1108
- self._SetContentTypeHeader(src_uri, headers)
1109
- self._LogCopyOperation(src_uri, dst_uri, headers)
1110
- canned_acl = None
1111
- if self.sub_opts:
1112
- for o, a in self.sub_opts:
1113
- if o == '-a':
1114
- canned_acls = dst_uri.canned_acls()
1115
- if a not in canned_acls:
1116
- raise CommandException('Invalid canned ACL "%s".' % a)
1117
- canned_acl = a
1118
- elif o == '-p':
1119
- # We don't attempt to preserve ACLs across providers because
1120
- # GCS and S3 support different ACLs and disjoint principals.
1121
- raise NotImplementedError('Cross-provider cp -p not supported')
1122
- return self._PerformResumableUploadIfApplies(KeyFile(src_key), dst_uri,
1123
- canned_acl, headers)
1124
-
1125
- def _PerformCopy(self, src_uri, dst_uri):
1126
- """Performs copy from src_uri to dst_uri, handling various special cases.
1127
-
1128
- Args:
1129
- src_uri: Source StorageUri.
1130
- dst_uri: Destination StorageUri.
1131
-
1132
- Returns:
1133
- (elapsed_time, bytes_transferred, version-specific dst_uri) excluding
1134
- overhead like initial HEAD.
1135
-
1136
- Raises:
1137
- CommandException: if errors encountered.
1138
- """
1139
- # Make a copy of the input headers each time so we can set a different
1140
- # content type for each object.
1141
- if self.headers:
1142
- headers = self.headers.copy()
1143
- else:
1144
- headers = {}
1145
-
1146
- src_key = src_uri.get_key(False, headers)
1147
- if not src_key:
1148
- raise CommandException('"%s" does not exist.' % src_uri)
1149
-
1150
- # On Windows, stdin is opened as text mode instead of binary which causes
1151
- # problems when piping a binary file, so this switches it to binary mode.
1152
- if IS_WINDOWS and src_uri.is_file_uri() and src_key.is_stream():
1153
- import msvcrt
1154
- msvcrt.setmode(src_key.fp.fileno(), os.O_BINARY)
1155
-
1156
- if self.no_clobber:
1157
- # There are two checks to prevent clobbering:
1158
- # 1) The first check is to see if the item
1159
- # already exists at the destination and prevent the upload/download
1160
- # from happening. This is done by the exists() call.
1161
- # 2) The second check is only relevant if we are writing to gs. We can
1162
- # enforce that the server only writes the object if it doesn't exist
1163
- # by specifying the header below. This check only happens at the
1164
- # server after the complete file has been uploaded. We specify this
1165
- # header to prevent a race condition where a destination file may
1166
- # be created after the first check and before the file is fully
1167
- # uploaded.
1168
- # In order to save on unnecessary uploads/downloads we perform both
1169
- # checks. However, this may come at the cost of additional HTTP calls.
1170
- if dst_uri.exists(headers):
1171
- if not self.quiet:
1172
- self.THREADED_LOGGER.info('Skipping existing item: %s' %
1173
- dst_uri.uri)
1174
- return (0, 0, None)
1175
- if dst_uri.is_cloud_uri() and dst_uri.scheme == 'gs':
1176
- headers['x-goog-if-generation-match'] = '0'
1177
-
1178
- if src_uri.is_cloud_uri() and dst_uri.is_cloud_uri():
1179
- if src_uri.scheme == dst_uri.scheme and not self.daisy_chain:
1180
- return self._CopyObjToObjInTheCloud(src_key, src_uri, dst_uri, headers)
1181
- else:
1182
- return self._CopyObjToObjDaisyChainMode(src_key, src_uri, dst_uri,
1183
- headers)
1184
- elif src_uri.is_file_uri() and dst_uri.is_cloud_uri():
1185
- return self._UploadFileToObject(src_key, src_uri, dst_uri, headers)
1186
- elif src_uri.is_cloud_uri() and dst_uri.is_file_uri():
1187
- return self._DownloadObjectToFile(src_key, src_uri, dst_uri, headers)
1188
- elif src_uri.is_file_uri() and dst_uri.is_file_uri():
1189
- return self._CopyFileToFile(src_key, src_uri, dst_uri, headers)
1190
- else:
1191
- raise CommandException('Unexpected src/dest case')
1192
-
1193
- def _ExpandDstUri(self, dst_uri_str):
1194
- """
1195
- Expands wildcard if present in dst_uri_str.
1196
-
1197
- Args:
1198
- dst_uri_str: String representation of requested dst_uri.
1199
-
1200
- Returns:
1201
- (exp_dst_uri, have_existing_dst_container)
1202
- where have_existing_dst_container is a bool indicating whether
1203
- exp_dst_uri names an existing directory, bucket, or bucket subdirectory.
1204
-
1205
- Raises:
1206
- CommandException: if dst_uri_str matched more than 1 URI.
1207
- """
1208
- dst_uri = self.suri_builder.StorageUri(dst_uri_str)
1209
-
1210
- # Handle wildcarded dst_uri case.
1211
- if ContainsWildcard(dst_uri):
1212
- blr_expansion = list(self.WildcardIterator(dst_uri))
1213
- if len(blr_expansion) != 1:
1214
- raise CommandException('Destination (%s) must match exactly 1 URI' %
1215
- dst_uri_str)
1216
- blr = blr_expansion[0]
1217
- uri = blr.GetUri()
1218
- if uri.is_cloud_uri():
1219
- return (uri, uri.names_bucket() or blr.HasPrefix()
1220
- or blr.GetKey().endswith('/'))
1221
- else:
1222
- return (uri, uri.names_directory())
1223
-
1224
- # Handle non-wildcarded dst_uri:
1225
- if dst_uri.is_file_uri():
1226
- return (dst_uri, dst_uri.names_directory())
1227
- if dst_uri.names_bucket():
1228
- return (dst_uri, True)
1229
- # For object URIs check 3 cases: (a) if the name ends with '/' treat as a
1230
- # subdir; else, perform a wildcard expansion with dst_uri + "*" and then
1231
- # find if (b) there's a Prefix matching dst_uri, or (c) name is of form
1232
- # dir_$folder$ (and in both these cases also treat dir as a subdir).
1233
- if dst_uri.is_cloud_uri() and dst_uri_str.endswith('/'):
1234
- return (dst_uri, True)
1235
- blr_expansion = list(self.WildcardIterator(
1236
- '%s*' % dst_uri_str.rstrip(dst_uri.delim)))
1237
- for blr in blr_expansion:
1238
- if blr.GetRStrippedUriString().endswith('_$folder$'):
1239
- return (dst_uri, True)
1240
- if blr.GetRStrippedUriString() == dst_uri_str.rstrip(dst_uri.delim):
1241
- return (dst_uri, blr.HasPrefix())
1242
- return (dst_uri, False)
1243
-
1244
- def _ConstructDstUri(self, src_uri, exp_src_uri,
1245
- src_uri_names_container, src_uri_expands_to_multi,
1246
- have_multiple_srcs, exp_dst_uri,
1247
- have_existing_dest_subdir):
1248
- """
1249
- Constructs the destination URI for a given exp_src_uri/exp_dst_uri pair,
1250
- using context-dependent naming rules that mimic Linux cp and mv behavior.
1251
-
1252
- Args:
1253
- src_uri: src_uri to be copied.
1254
- exp_src_uri: Single StorageUri from wildcard expansion of src_uri.
1255
- src_uri_names_container: True if src_uri names a container (including the
1256
- case of a wildcard-named bucket subdir (like gs://bucket/abc,
1257
- where gs://bucket/abc/* matched some objects). Note that this is
1258
- additional semantics tha src_uri.names_container() doesn't understand
1259
- because the latter only understands StorageUris, not wildcards.
1260
- src_uri_expands_to_multi: True if src_uri expanded to multiple URIs.
1261
- have_multiple_srcs: True if this is a multi-source request. This can be
1262
- true if src_uri wildcard-expanded to multiple URIs or if there were
1263
- multiple source URIs in the request.
1264
- exp_dst_uri: the expanded StorageUri requested for the cp destination.
1265
- Final written path is constructed from this plus a context-dependent
1266
- variant of src_uri.
1267
- have_existing_dest_subdir: bool indicator whether dest is an existing
1268
- subdirectory.
1269
-
1270
- Returns:
1271
- StorageUri to use for copy.
1272
-
1273
- Raises:
1274
- CommandException if destination object name not specified for
1275
- source and source is a stream.
1276
- """
1277
- if self._ShouldTreatDstUriAsSingleton(
1278
- have_multiple_srcs, have_existing_dest_subdir, exp_dst_uri):
1279
- # We're copying one file or object to one file or object.
1280
- return exp_dst_uri
1281
-
1282
- if exp_src_uri.is_stream():
1283
- if exp_dst_uri.names_container():
1284
- raise CommandException('Destination object name needed when '
1285
- 'source is a stream')
1286
- return exp_dst_uri
1287
-
1288
- if not self.recursion_requested and not have_multiple_srcs:
1289
- # We're copying one file or object to a subdirectory. Append final comp
1290
- # of exp_src_uri to exp_dst_uri.
1291
- src_final_comp = exp_src_uri.object_name.rpartition(src_uri.delim)[-1]
1292
- return self.suri_builder.StorageUri('%s%s%s' % (
1293
- exp_dst_uri.uri.rstrip(exp_dst_uri.delim), exp_dst_uri.delim,
1294
- src_final_comp))
1295
-
1296
- # Else we're copying multiple sources to a directory, bucket, or a bucket
1297
- # "sub-directory".
1298
-
1299
- # Ensure exp_dst_uri ends in delim char if we're doing a multi-src copy or
1300
- # a copy to a directory. (The check for copying to a directory needs
1301
- # special-case handling so that the command:
1302
- # gsutil cp gs://bucket/obj dir
1303
- # will turn into file://dir/ instead of file://dir -- the latter would cause
1304
- # the file "dirobj" to be created.)
1305
- # Note: need to check have_multiple_srcs or src_uri.names_container()
1306
- # because src_uri could be a bucket containing a single object, named
1307
- # as gs://bucket.
1308
- if ((have_multiple_srcs or src_uri.names_container()
1309
- or os.path.isdir(exp_dst_uri.object_name))
1310
- and not exp_dst_uri.uri.endswith(exp_dst_uri.delim)):
1311
- exp_dst_uri = exp_dst_uri.clone_replace_name(
1312
- '%s%s' % (exp_dst_uri.object_name, exp_dst_uri.delim)
1313
- )
1314
-
1315
- # Making naming behavior match how things work with local Linux cp and mv
1316
- # operations depends on many factors, including whether the destination is a
1317
- # container, the plurality of the source(s), and whether the mv command is
1318
- # being used:
1319
- # 1. For the "mv" command that specifies a non-existent destination subdir,
1320
- # renaming should occur at the level of the src subdir, vs appending that
1321
- # subdir beneath the dst subdir like is done for copying. For example:
1322
- # gsutil rm -R gs://bucket
1323
- # gsutil cp -R dir1 gs://bucket
1324
- # gsutil cp -R dir2 gs://bucket/subdir1
1325
- # gsutil mv gs://bucket/subdir1 gs://bucket/subdir2
1326
- # would (if using cp naming behavior) end up with paths like:
1327
- # gs://bucket/subdir2/subdir1/dir2/.svn/all-wcprops
1328
- # whereas mv naming behavior should result in:
1329
- # gs://bucket/subdir2/dir2/.svn/all-wcprops
1330
- # 2. Copying from directories, buckets, or bucket subdirs should result in
1331
- # objects/files mirroring the source directory hierarchy. For example:
1332
- # gsutil cp dir1/dir2 gs://bucket
1333
- # should create the object gs://bucket/dir2/file2, assuming dir1/dir2
1334
- # contains file2).
1335
- # To be consistent with Linux cp behavior, there's one more wrinkle when
1336
- # working with subdirs: The resulting object names depend on whether the
1337
- # destination subdirectory exists. For example, if gs://bucket/subdir
1338
- # exists, the command:
1339
- # gsutil cp -R dir1/dir2 gs://bucket/subdir
1340
- # should create objects named like gs://bucket/subdir/dir2/a/b/c. In
1341
- # contrast, if gs://bucket/subdir does not exist, this same command
1342
- # should create objects named like gs://bucket/subdir/a/b/c.
1343
- # 3. Copying individual files or objects to dirs, buckets or bucket subdirs
1344
- # should result in objects/files named by the final source file name
1345
- # component. Example:
1346
- # gsutil cp dir1/*.txt gs://bucket
1347
- # should create the objects gs://bucket/f1.txt and gs://bucket/f2.txt,
1348
- # assuming dir1 contains f1.txt and f2.txt.
1349
-
1350
- if (self.perform_mv and self.recursion_requested
1351
- and src_uri_expands_to_multi and not have_existing_dest_subdir):
1352
- # Case 1. Handle naming rules for bucket subdir mv. Here we want to
1353
- # line up the src_uri against its expansion, to find the base to build
1354
- # the new name. For example, running the command:
1355
- # gsutil mv gs://bucket/abcd gs://bucket/xyz
1356
- # when processing exp_src_uri=gs://bucket/abcd/123
1357
- # exp_src_uri_tail should become /123
1358
- # Note: mv.py code disallows wildcard specification of source URI.
1359
- exp_src_uri_tail = exp_src_uri.uri[len(src_uri.uri):]
1360
- dst_key_name = '%s/%s' % (exp_dst_uri.object_name.rstrip('/'),
1361
- exp_src_uri_tail.strip('/'))
1362
- return exp_dst_uri.clone_replace_name(dst_key_name)
1363
-
1364
- if src_uri_names_container and not exp_dst_uri.names_file():
1365
- # Case 2. Build dst_key_name from subpath of exp_src_uri past
1366
- # where src_uri ends. For example, for src_uri=gs://bucket/ and
1367
- # exp_src_uri=gs://bucket/src_subdir/obj, dst_key_name should be
1368
- # src_subdir/obj.
1369
- src_uri_path_sans_final_dir = _GetPathBeforeFinalDir(src_uri)
1370
- dst_key_name = exp_src_uri.uri[
1371
- len(src_uri_path_sans_final_dir):].lstrip(src_uri.delim)
1372
- # Handle case where dst_uri is a non-existent subdir.
1373
- if not have_existing_dest_subdir:
1374
- dst_key_name = dst_key_name.partition(src_uri.delim)[-1]
1375
- # Handle special case where src_uri was a directory named with '.' or
1376
- # './', so that running a command like:
1377
- # gsutil cp -r . gs://dest
1378
- # will produce obj names of the form gs://dest/abc instead of
1379
- # gs://dest/./abc.
1380
- if dst_key_name.startswith('.%s' % os.sep):
1381
- dst_key_name = dst_key_name[2:]
1382
-
1383
- else:
1384
- # Case 3.
1385
- dst_key_name = exp_src_uri.object_name.rpartition(src_uri.delim)[-1]
1386
-
1387
- if (exp_dst_uri.is_file_uri()
1388
- or self._ShouldTreatDstUriAsBucketSubDir(
1389
- have_multiple_srcs, exp_dst_uri, have_existing_dest_subdir)):
1390
- if exp_dst_uri.object_name.endswith(exp_dst_uri.delim):
1391
- dst_key_name = '%s%s%s' % (
1392
- exp_dst_uri.object_name.rstrip(exp_dst_uri.delim),
1393
- exp_dst_uri.delim, dst_key_name)
1394
- else:
1395
- delim = exp_dst_uri.delim if exp_dst_uri.object_name else ''
1396
- dst_key_name = '%s%s%s' % (exp_dst_uri.object_name, delim, dst_key_name)
1397
-
1398
- return exp_dst_uri.clone_replace_name(dst_key_name)
1399
-
1400
- def _FixWindowsNaming(self, src_uri, dst_uri):
1401
- """
1402
- Rewrites the destination URI built by _ConstructDstUri() to translate
1403
- Windows pathnames to cloud pathnames if needed.
1404
-
1405
- Args:
1406
- src_uri: Source URI to be copied.
1407
- dst_uri: The destination URI built by _ConstructDstUri().
1408
-
1409
- Returns:
1410
- StorageUri to use for copy.
1411
- """
1412
- if (src_uri.is_file_uri() and src_uri.delim == '\\'
1413
- and dst_uri.is_cloud_uri()):
1414
- trans_uri_str = re.sub(r'\\', '/', dst_uri.uri)
1415
- dst_uri = self.suri_builder.StorageUri(trans_uri_str)
1416
- return dst_uri
1417
-
1418
- # Command entry point.
1419
- def RunCommand(self):
1420
-
1421
- # Inner funcs.
1422
- def _CopyExceptionHandler(e):
1423
- """Simple exception handler to allow post-completion status."""
1424
- self.THREADED_LOGGER.error(str(e))
1425
- self.copy_failure_count += 1
1426
-
1427
- def _CopyFunc(name_expansion_result):
1428
- """Worker function for performing the actual copy (and rm, for mv)."""
1429
- if self.perform_mv:
1430
- cmd_name = 'mv'
1431
- else:
1432
- cmd_name = self.command_name
1433
- src_uri = self.suri_builder.StorageUri(
1434
- name_expansion_result.GetSrcUriStr())
1435
- exp_src_uri = self.suri_builder.StorageUri(
1436
- name_expansion_result.GetExpandedUriStr())
1437
- src_uri_names_container = name_expansion_result.NamesContainer()
1438
- src_uri_expands_to_multi = name_expansion_result.NamesContainer()
1439
- have_multiple_srcs = name_expansion_result.IsMultiSrcRequest()
1440
- have_existing_dest_subdir = (
1441
- name_expansion_result.HaveExistingDstContainer())
1442
-
1443
- if src_uri.names_provider():
1444
- raise CommandException(
1445
- 'The %s command does not allow provider-only source URIs (%s)' %
1446
- (cmd_name, src_uri))
1447
- if have_multiple_srcs:
1448
- self._InsistDstUriNamesContainer(exp_dst_uri,
1449
- have_existing_dst_container,
1450
- cmd_name)
1451
-
1452
- if self.perform_mv:
1453
- if name_expansion_result.NamesContainer():
1454
- # Use recursion_requested when performing name expansion for the
1455
- # directory mv case so we can determine if any of the source URIs are
1456
- # directories (and then use cp -R and rm -R to perform the move, to
1457
- # match the behavior of Linux mv (which when moving a directory moves
1458
- # all the contained files).
1459
- self.recursion_requested = True
1460
- # Disallow wildcard src URIs when moving directories, as supporting it
1461
- # would make the name transformation too complex and would also be
1462
- # dangerous (e.g., someone could accidentally move many objects to the
1463
- # wrong name, or accidentally overwrite many objects).
1464
- if ContainsWildcard(src_uri):
1465
- raise CommandException('The mv command disallows naming source '
1466
- 'directories using wildcards')
1467
-
1468
- if (exp_dst_uri.is_file_uri()
1469
- and not os.path.exists(exp_dst_uri.object_name)
1470
- and have_multiple_srcs):
1471
- os.makedirs(exp_dst_uri.object_name)
1472
-
1473
- dst_uri = self._ConstructDstUri(src_uri, exp_src_uri,
1474
- src_uri_names_container,
1475
- src_uri_expands_to_multi,
1476
- have_multiple_srcs, exp_dst_uri,
1477
- have_existing_dest_subdir)
1478
- dst_uri = self._FixWindowsNaming(src_uri, dst_uri)
1479
-
1480
- self._CheckForDirFileConflict(exp_src_uri, dst_uri)
1481
- if self._SrcDstSame(exp_src_uri, dst_uri):
1482
- raise CommandException('%s: "%s" and "%s" are the same file - '
1483
- 'abort.' % (cmd_name, exp_src_uri, dst_uri))
1484
-
1485
- if dst_uri.is_cloud_uri() and dst_uri.is_version_specific:
1486
- raise CommandException('%s: a version-specific URI\n(%s)\ncannot be '
1487
- 'the destination for gsutil cp - abort.'
1488
- % (cmd_name, dst_uri))
1489
-
1490
- elapsed_time = bytes_transferred = 0
1491
- try:
1492
- (elapsed_time, bytes_transferred, result_uri) = (
1493
- self._PerformCopy(exp_src_uri, dst_uri))
1494
- except Exception, e:
1495
- if self._IsNoClobberServerException(e):
1496
- if not self.quiet:
1497
- self.THREADED_LOGGER.info('Rejected (noclobber): %s' % dst_uri.uri)
1498
- elif self.continue_on_error:
1499
- if not self.quiet:
1500
- self.THREADED_LOGGER.error('Error copying %s: %s' % (src_uri.uri,
1501
- str(e)))
1502
- self.copy_failure_count += 1
1503
- else:
1504
- raise
1505
- if self.print_ver:
1506
- # Some cases don't return a version-specific URI (e.g., if destination
1507
- # is a file).
1508
- if hasattr(result_uri, 'version_specific_uri'):
1509
- self.THREADED_LOGGER.info('Created: %s' %
1510
- result_uri.version_specific_uri)
1511
- else:
1512
- self.THREADED_LOGGER.info('Created: %s' % result_uri.uri)
1513
-
1514
- # TODO: If we ever use -n (noclobber) with -M (move) (not possible today
1515
- # since we call copy internally from move and don't specify the -n flag)
1516
- # we'll need to only remove the source when we have not skipped the
1517
- # destination.
1518
- if self.perform_mv:
1519
- if not self.quiet:
1520
- self.THREADED_LOGGER.info('Removing %s...', exp_src_uri)
1521
- exp_src_uri.delete_key(validate=False, headers=self.headers)
1522
- stats_lock.acquire()
1523
- self.total_elapsed_time += elapsed_time
1524
- self.total_bytes_transferred += bytes_transferred
1525
- stats_lock.release()
1526
-
1527
- # Start of RunCommand code.
1528
- self._ParseArgs()
1529
-
1530
- self.total_elapsed_time = self.total_bytes_transferred = 0
1531
- if self.args[-1] == '-' or self.args[-1] == 'file://-':
1532
- self._HandleStreamingDownload()
1533
- return 0
1534
-
1535
- if self.read_args_from_stdin:
1536
- if len(self.args) != 1:
1537
- raise CommandException('Source URIs cannot be specified with -I option')
1538
- uri_strs = self._StdinIterator()
1539
- else:
1540
- if len(self.args) < 2:
1541
- raise CommandException('Wrong number of arguments for "cp" command.')
1542
- uri_strs = self.args[0:len(self.args)-1]
1543
-
1544
- (exp_dst_uri, have_existing_dst_container) = self._ExpandDstUri(
1545
- self.args[-1])
1546
- name_expansion_iterator = NameExpansionIterator(
1547
- self.command_name, self.proj_id_handler, self.headers, self.debug,
1548
- self.bucket_storage_uri_class, uri_strs,
1549
- self.recursion_requested or self.perform_mv,
1550
- have_existing_dst_container)
1551
-
1552
- # Use a lock to ensure accurate statistics in the face of
1553
- # multi-threading/multi-processing.
1554
- stats_lock = threading.Lock()
1555
-
1556
- # Tracks if any copies failed.
1557
- self.copy_failure_count = 0
1558
-
1559
- # Start the clock.
1560
- start_time = time.time()
1561
-
1562
- # Tuple of attributes to share/manage across multiple processes in
1563
- # parallel (-m) mode.
1564
- shared_attrs = ('copy_failure_count', 'total_bytes_transferred')
1565
-
1566
- # Perform copy requests in parallel (-m) mode, if requested, using
1567
- # configured number of parallel processes and threads. Otherwise,
1568
- # perform requests with sequential function calls in current process.
1569
- self.Apply(_CopyFunc, name_expansion_iterator, _CopyExceptionHandler,
1570
- shared_attrs)
1571
- if self.debug:
1572
- print 'total_bytes_transferred:' + str(self.total_bytes_transferred)
1573
-
1574
- end_time = time.time()
1575
- self.total_elapsed_time = end_time - start_time
1576
-
1577
- # Sometimes, particularly when running unit tests, the total elapsed time
1578
- # is really small. On Windows, the timer resolution is too small and
1579
- # causes total_elapsed_time to be zero.
1580
- try:
1581
- float(self.total_bytes_transferred) / float(self.total_elapsed_time)
1582
- except ZeroDivisionError:
1583
- self.total_elapsed_time = 0.01
1584
-
1585
- self.total_bytes_per_second = (float(self.total_bytes_transferred) /
1586
- float(self.total_elapsed_time))
1587
-
1588
- if self.debug == 3:
1589
- # Note that this only counts the actual GET and PUT bytes for the copy
1590
- # - not any transfers for doing wildcard expansion, the initial HEAD
1591
- # request boto performs when doing a bucket.get_key() operation, etc.
1592
- if self.total_bytes_transferred != 0:
1593
- self.THREADED_LOGGER.info(
1594
- 'Total bytes copied=%d, total elapsed time=%5.3f secs (%sps)',
1595
- self.total_bytes_transferred, self.total_elapsed_time,
1596
- MakeHumanReadable(self.total_bytes_per_second))
1597
- if self.copy_failure_count:
1598
- plural_str = ''
1599
- if self.copy_failure_count > 1:
1600
- plural_str = 's'
1601
- raise CommandException('%d file%s/object%s could not be transferred.' % (
1602
- self.copy_failure_count, plural_str, plural_str))
1603
-
1604
- return 0
1605
-
1606
- def _ParseArgs(self):
1607
- self.perform_mv = False
1608
- self.exclude_symlinks = False
1609
- self.quiet = False
1610
- self.no_clobber = False
1611
- self.continue_on_error = False
1612
- self.daisy_chain = False
1613
- self.read_args_from_stdin = False
1614
- self.print_ver = False
1615
- # self.recursion_requested initialized in command.py (so can be checked
1616
- # in parent class for all commands).
1617
- if self.sub_opts:
1618
- for o, unused_a in self.sub_opts:
1619
- if o == '-c':
1620
- self.continue_on_error = True
1621
- elif o == '-D':
1622
- self.daisy_chain = True
1623
- elif o == '-e':
1624
- self.exclude_symlinks = True
1625
- elif o == '-I':
1626
- self.read_args_from_stdin = True
1627
- elif o == '-M':
1628
- # Note that we signal to the cp command to perform a move (copy
1629
- # followed by remove) and use directory-move naming rules by passing
1630
- # the undocumented (for internal use) -M option when running the cp
1631
- # command from mv.py.
1632
- self.perform_mv = True
1633
- elif o == '-n':
1634
- self.no_clobber = True
1635
- elif o == '-q':
1636
- self.quiet = True
1637
- elif o == '-r' or o == '-R':
1638
- self.recursion_requested = True
1639
- elif o == '-v':
1640
- self.print_ver = True
1641
-
1642
- def _HandleStreamingDownload(self):
1643
- # Destination is <STDOUT>. Manipulate sys.stdout so as to redirect all
1644
- # debug messages to <STDERR>.
1645
- stdout_fp = sys.stdout
1646
- sys.stdout = sys.stderr
1647
- did_some_work = False
1648
- for uri_str in self.args[0:len(self.args)-1]:
1649
- for uri in self.WildcardIterator(uri_str).IterUris():
1650
- did_some_work = True
1651
- key = uri.get_key(False, self.headers)
1652
- (elapsed_time, bytes_transferred) = self._PerformDownloadToStream(
1653
- key, uri, stdout_fp, self.headers)
1654
- self.total_elapsed_time += elapsed_time
1655
- self.total_bytes_transferred += bytes_transferred
1656
- if not did_some_work:
1657
- raise CommandException('No URIs matched')
1658
- if self.debug == 3:
1659
- if self.total_bytes_transferred != 0:
1660
- self.THREADED_LOGGER.info(
1661
- 'Total bytes copied=%d, total elapsed time=%5.3f secs (%sps)',
1662
- self.total_bytes_transferred, self.total_elapsed_time,
1663
- MakeHumanReadable(float(self.total_bytes_transferred) /
1664
- float(self.total_elapsed_time)))
1665
-
1666
- def _StdinIterator(self):
1667
- """A generator function that returns lines from stdin."""
1668
- for line in sys.stdin:
1669
- # Strip CRLF.
1670
- yield line.rstrip()
1671
-
1672
- def _SrcDstSame(self, src_uri, dst_uri):
1673
- """Checks if src_uri and dst_uri represent the same object or file.
1674
-
1675
- We don't handle anything about hard or symbolic links.
1676
-
1677
- Args:
1678
- src_uri: Source StorageUri.
1679
- dst_uri: Destination StorageUri.
1680
-
1681
- Returns:
1682
- Bool indicator.
1683
- """
1684
- if src_uri.is_file_uri() and dst_uri.is_file_uri():
1685
- # Translate a/b/./c to a/b/c, so src=dst comparison below works.
1686
- new_src_path = os.path.normpath(src_uri.object_name)
1687
- new_dst_path = os.path.normpath(dst_uri.object_name)
1688
- return (src_uri.clone_replace_name(new_src_path).uri ==
1689
- dst_uri.clone_replace_name(new_dst_path).uri)
1690
- else:
1691
- return (src_uri.uri == dst_uri.uri and
1692
- src_uri.generation == dst_uri.generation and
1693
- src_uri.version_id == dst_uri.version_id)
1694
-
1695
- def _ShouldTreatDstUriAsBucketSubDir(self, have_multiple_srcs, dst_uri,
1696
- have_existing_dest_subdir):
1697
- """
1698
- Checks whether dst_uri should be treated as a bucket "sub-directory". The
1699
- decision about whether something constitutes a bucket "sub-directory"
1700
- depends on whether there are multiple sources in this request and whether
1701
- there is an existing bucket subdirectory. For example, when running the
1702
- command:
1703
- gsutil cp file gs://bucket/abc
1704
- if there's no existing gs://bucket/abc bucket subdirectory we should copy
1705
- file to the object gs://bucket/abc. In contrast, if
1706
- there's an existing gs://bucket/abc bucket subdirectory we should copy
1707
- file to gs://bucket/abc/file. And regardless of whether gs://bucket/abc
1708
- exists, when running the command:
1709
- gsutil cp file1 file2 gs://bucket/abc
1710
- we should copy file1 to gs://bucket/abc/file1 (and similarly for file2).
1711
-
1712
- Note that we don't disallow naming a bucket "sub-directory" where there's
1713
- already an object at that URI. For example it's legitimate (albeit
1714
- confusing) to have an object called gs://bucket/dir and
1715
- then run the command
1716
- gsutil cp file1 file2 gs://bucket/dir
1717
- Doing so will end up with objects gs://bucket/dir, gs://bucket/dir/file1,
1718
- and gs://bucket/dir/file2.
1719
-
1720
- Args:
1721
- have_multiple_srcs: Bool indicator of whether this is a multi-source
1722
- operation.
1723
- dst_uri: StorageUri to check.
1724
- have_existing_dest_subdir: bool indicator whether dest is an existing
1725
- subdirectory.
1726
-
1727
- Returns:
1728
- bool indicator.
1729
- """
1730
- return ((have_multiple_srcs and dst_uri.is_cloud_uri())
1731
- or (have_existing_dest_subdir))
1732
-
1733
- def _ShouldTreatDstUriAsSingleton(self, have_multiple_srcs,
1734
- have_existing_dest_subdir, dst_uri):
1735
- """
1736
- Checks that dst_uri names a singleton (file or object) after
1737
- dir/wildcard expansion. The decision is more nuanced than simply
1738
- dst_uri.names_singleton()) because of the possibility that an object path
1739
- might name a bucket sub-directory.
1740
-
1741
- Args:
1742
- have_multiple_srcs: Bool indicator of whether this is a multi-source
1743
- operation.
1744
- have_existing_dest_subdir: bool indicator whether dest is an existing
1745
- subdirectory.
1746
- dst_uri: StorageUri to check.
1747
-
1748
- Returns:
1749
- bool indicator.
1750
- """
1751
- if have_multiple_srcs:
1752
- # Only a file meets the criteria in this case.
1753
- return dst_uri.names_file()
1754
- return not have_existing_dest_subdir and dst_uri.names_singleton()
1755
-
1756
- def _IsNoClobberServerException(self, e):
1757
- """
1758
- Checks to see if the server attempted to clobber a file after we specified
1759
- in the header that we didn't want the file clobbered.
1760
-
1761
- Args:
1762
- e: The Exception that was generated by a failed copy operation
1763
-
1764
- Returns:
1765
- bool indicator - True indicates that the server did attempt to clobber
1766
- an existing file.
1767
- """
1768
- return self.no_clobber and (
1769
- (isinstance(e, GSResponseError) and e.status==412) or
1770
- (isinstance(e, ResumableUploadException) and 'code 412' in e.message))
1771
-
1772
- def _GetPathBeforeFinalDir(uri):
1773
- """
1774
- Returns the part of the path before the final directory component for the
1775
- given URI, handling cases for file system directories, bucket, and bucket
1776
- subdirectories. Example: for gs://bucket/dir/ we'll return 'gs://bucket',
1777
- and for file://dir we'll return file://
1778
-
1779
- Args:
1780
- uri: StorageUri.
1781
-
1782
- Returns:
1783
- String name of above-described path, sans final path separator.
1784
- """
1785
- sep = uri.delim
1786
- assert not uri.names_file()
1787
- if uri.names_directory():
1788
- past_scheme = uri.uri[len('file://'):]
1789
- if past_scheme.find(sep) == -1:
1790
- return 'file://'
1791
- else:
1792
- return 'file://%s' % past_scheme.rstrip(sep).rpartition(sep)[0]
1793
- if uri.names_bucket():
1794
- return '%s://' % uri.scheme
1795
- # Else it names a bucket subdir.
1796
- return uri.uri.rstrip(sep).rpartition(sep)[0]
1797
-
1798
- def _hash_filename(filename):
1799
- """
1800
- Apply a hash function (SHA1) to shorten the passed file name. The spec
1801
- for the hashed file name is as follows:
1802
-
1803
- TRACKER_<hash>_<trailing>
1804
-
1805
- where hash is a SHA1 hash on the original file name and trailing is
1806
- the last 16 chars from the original file name. Max file name lengths
1807
- vary by operating system so the goal of this function is to ensure
1808
- the hashed version takes fewer than 100 characters.
1809
-
1810
- Args:
1811
- filename: file name to be hashed.
1812
-
1813
- Returns:
1814
- shorter, hashed version of passed file name
1815
- """
1816
- m = hashlib.sha1(unicode(filename, 'utf8').encode('utf-8'))
1817
- hashed_name = ("TRACKER_" + m.hexdigest() + '.' + filename[-16:])
1818
- return hashed_name