googlecloud 0.0.4 → 0.0.6
Sign up to get free protection for your applications and to get access to all the features.
- data.tar.gz.sig +0 -0
- data/CHANGELOG +4 -0
- data/Makefile +4 -0
- data/Manifest +760 -1
- data/bin/gcloud +7 -0
- data/bin/{gcutil → gcutil.symlink} +0 -0
- data/bin/gsutil-symlink +377 -0
- data/googlecloud.gemspec +5 -5
- data/packages/gsutil/CHECKSUM +1 -0
- data/packages/gsutil/COPYING +202 -0
- data/packages/gsutil/LICENSE.third_party +295 -0
- data/packages/gsutil/MANIFEST.in +5 -0
- data/packages/gsutil/README +38 -0
- data/packages/gsutil/README.pkg +49 -0
- data/packages/gsutil/ReleaseNotes.txt +780 -0
- data/packages/gsutil/VERSION +1 -0
- data/packages/gsutil/boto/Changelog.rst +35 -0
- data/packages/gsutil/boto/LICENSE +18 -0
- data/packages/gsutil/boto/MANIFEST.in +12 -0
- data/packages/gsutil/boto/README.rst +163 -0
- data/packages/gsutil/boto/bin/asadmin +290 -0
- data/packages/gsutil/boto/bin/bundle_image +27 -0
- data/packages/gsutil/boto/bin/cfadmin +108 -0
- data/packages/gsutil/boto/bin/cq +89 -0
- data/packages/gsutil/boto/bin/cwutil +140 -0
- data/packages/gsutil/boto/bin/elbadmin +284 -0
- data/packages/gsutil/boto/bin/fetch_file +43 -0
- data/packages/gsutil/boto/bin/glacier +154 -0
- data/packages/gsutil/boto/bin/instance_events +145 -0
- data/packages/gsutil/boto/bin/kill_instance +35 -0
- data/packages/gsutil/boto/bin/launch_instance +252 -0
- data/packages/gsutil/boto/bin/list_instances +90 -0
- data/packages/gsutil/boto/bin/lss3 +77 -0
- data/packages/gsutil/boto/bin/mturk +465 -0
- data/packages/gsutil/boto/bin/pyami_sendmail +52 -0
- data/packages/gsutil/boto/bin/route53 +205 -0
- data/packages/gsutil/boto/bin/s3put +374 -0
- data/packages/gsutil/boto/bin/sdbadmin +194 -0
- data/packages/gsutil/boto/bin/taskadmin +116 -0
- data/packages/gsutil/boto/boto/__init__.py +793 -0
- data/packages/gsutil/boto/boto/__init__.pyc +0 -0
- data/packages/gsutil/boto/boto/auth.py +682 -0
- data/packages/gsutil/boto/boto/auth.pyc +0 -0
- data/packages/gsutil/boto/boto/auth_handler.py +58 -0
- data/packages/gsutil/boto/boto/auth_handler.pyc +0 -0
- data/packages/gsutil/boto/boto/beanstalk/__init__.py +65 -0
- data/packages/gsutil/boto/boto/beanstalk/exception.py +64 -0
- data/packages/gsutil/boto/boto/beanstalk/layer1.py +1153 -0
- data/packages/gsutil/boto/boto/beanstalk/response.py +703 -0
- data/packages/gsutil/boto/boto/beanstalk/wrapper.py +29 -0
- data/packages/gsutil/boto/boto/cacerts/__init__.py +22 -0
- data/packages/gsutil/boto/boto/cacerts/__init__.pyc +0 -0
- data/packages/gsutil/boto/boto/cacerts/cacerts.txt +633 -0
- data/packages/gsutil/boto/boto/cloudformation/__init__.py +68 -0
- data/packages/gsutil/boto/boto/cloudformation/connection.py +364 -0
- data/packages/gsutil/boto/boto/cloudformation/stack.py +360 -0
- data/packages/gsutil/boto/boto/cloudformation/template.py +43 -0
- data/packages/gsutil/boto/boto/cloudfront/__init__.py +324 -0
- data/packages/gsutil/boto/boto/cloudfront/distribution.py +745 -0
- data/packages/gsutil/boto/boto/cloudfront/exception.py +26 -0
- data/packages/gsutil/boto/boto/cloudfront/identity.py +122 -0
- data/packages/gsutil/boto/boto/cloudfront/invalidation.py +216 -0
- data/packages/gsutil/boto/boto/cloudfront/logging.py +38 -0
- data/packages/gsutil/boto/boto/cloudfront/object.py +48 -0
- data/packages/gsutil/boto/boto/cloudfront/origin.py +150 -0
- data/packages/gsutil/boto/boto/cloudfront/signers.py +60 -0
- data/packages/gsutil/boto/boto/cloudsearch/__init__.py +45 -0
- data/packages/gsutil/boto/boto/cloudsearch/document.py +265 -0
- data/packages/gsutil/boto/boto/cloudsearch/domain.py +394 -0
- data/packages/gsutil/boto/boto/cloudsearch/layer1.py +738 -0
- data/packages/gsutil/boto/boto/cloudsearch/layer2.py +67 -0
- data/packages/gsutil/boto/boto/cloudsearch/optionstatus.py +248 -0
- data/packages/gsutil/boto/boto/cloudsearch/search.py +365 -0
- data/packages/gsutil/boto/boto/cloudsearch/sourceattribute.py +75 -0
- data/packages/gsutil/boto/boto/compat.py +28 -0
- data/packages/gsutil/boto/boto/compat.pyc +0 -0
- data/packages/gsutil/boto/boto/connection.py +1081 -0
- data/packages/gsutil/boto/boto/connection.pyc +0 -0
- data/packages/gsutil/boto/boto/contrib/__init__.py +22 -0
- data/packages/gsutil/boto/boto/contrib/ymlmessage.py +52 -0
- data/packages/gsutil/boto/boto/core/README +58 -0
- data/packages/gsutil/boto/boto/core/__init__.py +23 -0
- data/packages/gsutil/boto/boto/core/auth.py +78 -0
- data/packages/gsutil/boto/boto/core/credentials.py +154 -0
- data/packages/gsutil/boto/boto/core/dictresponse.py +178 -0
- data/packages/gsutil/boto/boto/core/service.py +67 -0
- data/packages/gsutil/boto/boto/datapipeline/__init__.py +0 -0
- data/packages/gsutil/boto/boto/datapipeline/exceptions.py +42 -0
- data/packages/gsutil/boto/boto/datapipeline/layer1.py +546 -0
- data/packages/gsutil/boto/boto/dynamodb/__init__.py +66 -0
- data/packages/gsutil/boto/boto/dynamodb/batch.py +262 -0
- data/packages/gsutil/boto/boto/dynamodb/condition.py +170 -0
- data/packages/gsutil/boto/boto/dynamodb/exceptions.py +64 -0
- data/packages/gsutil/boto/boto/dynamodb/item.py +196 -0
- data/packages/gsutil/boto/boto/dynamodb/layer1.py +575 -0
- data/packages/gsutil/boto/boto/dynamodb/layer2.py +798 -0
- data/packages/gsutil/boto/boto/dynamodb/schema.py +112 -0
- data/packages/gsutil/boto/boto/dynamodb/table.py +540 -0
- data/packages/gsutil/boto/boto/dynamodb/types.py +326 -0
- data/packages/gsutil/boto/boto/ec2/__init__.py +96 -0
- data/packages/gsutil/boto/boto/ec2/address.py +103 -0
- data/packages/gsutil/boto/boto/ec2/autoscale/__init__.py +781 -0
- data/packages/gsutil/boto/boto/ec2/autoscale/activity.py +74 -0
- data/packages/gsutil/boto/boto/ec2/autoscale/group.py +337 -0
- data/packages/gsutil/boto/boto/ec2/autoscale/instance.py +60 -0
- data/packages/gsutil/boto/boto/ec2/autoscale/launchconfig.py +209 -0
- data/packages/gsutil/boto/boto/ec2/autoscale/policy.py +166 -0
- data/packages/gsutil/boto/boto/ec2/autoscale/request.py +38 -0
- data/packages/gsutil/boto/boto/ec2/autoscale/scheduled.py +60 -0
- data/packages/gsutil/boto/boto/ec2/autoscale/tag.py +84 -0
- data/packages/gsutil/boto/boto/ec2/blockdevicemapping.py +141 -0
- data/packages/gsutil/boto/boto/ec2/bundleinstance.py +78 -0
- data/packages/gsutil/boto/boto/ec2/buyreservation.py +84 -0
- data/packages/gsutil/boto/boto/ec2/cloudwatch/__init__.py +603 -0
- data/packages/gsutil/boto/boto/ec2/cloudwatch/alarm.py +316 -0
- data/packages/gsutil/boto/boto/ec2/cloudwatch/datapoint.py +40 -0
- data/packages/gsutil/boto/boto/ec2/cloudwatch/dimension.py +38 -0
- data/packages/gsutil/boto/boto/ec2/cloudwatch/listelement.py +31 -0
- data/packages/gsutil/boto/boto/ec2/cloudwatch/metric.py +175 -0
- data/packages/gsutil/boto/boto/ec2/connection.py +3409 -0
- data/packages/gsutil/boto/boto/ec2/ec2object.py +107 -0
- data/packages/gsutil/boto/boto/ec2/elb/__init__.py +553 -0
- data/packages/gsutil/boto/boto/ec2/elb/healthcheck.py +89 -0
- data/packages/gsutil/boto/boto/ec2/elb/instancestate.py +62 -0
- data/packages/gsutil/boto/boto/ec2/elb/listelement.py +36 -0
- data/packages/gsutil/boto/boto/ec2/elb/listener.py +75 -0
- data/packages/gsutil/boto/boto/ec2/elb/loadbalancer.py +324 -0
- data/packages/gsutil/boto/boto/ec2/elb/policies.py +89 -0
- data/packages/gsutil/boto/boto/ec2/elb/securitygroup.py +38 -0
- data/packages/gsutil/boto/boto/ec2/group.py +39 -0
- data/packages/gsutil/boto/boto/ec2/image.py +350 -0
- data/packages/gsutil/boto/boto/ec2/instance.py +661 -0
- data/packages/gsutil/boto/boto/ec2/instanceinfo.py +51 -0
- data/packages/gsutil/boto/boto/ec2/instancestatus.py +207 -0
- data/packages/gsutil/boto/boto/ec2/keypair.py +113 -0
- data/packages/gsutil/boto/boto/ec2/launchspecification.py +105 -0
- data/packages/gsutil/boto/boto/ec2/networkinterface.py +247 -0
- data/packages/gsutil/boto/boto/ec2/placementgroup.py +51 -0
- data/packages/gsutil/boto/boto/ec2/regioninfo.py +34 -0
- data/packages/gsutil/boto/boto/ec2/reservedinstance.py +227 -0
- data/packages/gsutil/boto/boto/ec2/securitygroup.py +357 -0
- data/packages/gsutil/boto/boto/ec2/snapshot.py +170 -0
- data/packages/gsutil/boto/boto/ec2/spotdatafeedsubscription.py +63 -0
- data/packages/gsutil/boto/boto/ec2/spotinstancerequest.py +188 -0
- data/packages/gsutil/boto/boto/ec2/spotpricehistory.py +55 -0
- data/packages/gsutil/boto/boto/ec2/tag.py +87 -0
- data/packages/gsutil/boto/boto/ec2/vmtype.py +58 -0
- data/packages/gsutil/boto/boto/ec2/volume.py +293 -0
- data/packages/gsutil/boto/boto/ec2/volumestatus.py +200 -0
- data/packages/gsutil/boto/boto/ec2/zone.py +80 -0
- data/packages/gsutil/boto/boto/ecs/__init__.py +90 -0
- data/packages/gsutil/boto/boto/ecs/item.py +153 -0
- data/packages/gsutil/boto/boto/elasticache/__init__.py +62 -0
- data/packages/gsutil/boto/boto/elasticache/layer1.py +1252 -0
- data/packages/gsutil/boto/boto/elastictranscoder/__init__.py +62 -0
- data/packages/gsutil/boto/boto/elastictranscoder/exceptions.py +46 -0
- data/packages/gsutil/boto/boto/elastictranscoder/layer1.py +509 -0
- data/packages/gsutil/boto/boto/emr/__init__.py +73 -0
- data/packages/gsutil/boto/boto/emr/bootstrap_action.py +44 -0
- data/packages/gsutil/boto/boto/emr/connection.py +531 -0
- data/packages/gsutil/boto/boto/emr/emrobject.py +176 -0
- data/packages/gsutil/boto/boto/emr/instance_group.py +43 -0
- data/packages/gsutil/boto/boto/emr/step.py +281 -0
- data/packages/gsutil/boto/boto/exception.py +476 -0
- data/packages/gsutil/boto/boto/exception.pyc +0 -0
- data/packages/gsutil/boto/boto/file/README +49 -0
- data/packages/gsutil/boto/boto/file/__init__.py +28 -0
- data/packages/gsutil/boto/boto/file/bucket.py +112 -0
- data/packages/gsutil/boto/boto/file/connection.py +33 -0
- data/packages/gsutil/boto/boto/file/key.py +199 -0
- data/packages/gsutil/boto/boto/file/simpleresultset.py +30 -0
- data/packages/gsutil/boto/boto/fps/__init__.py +21 -0
- data/packages/gsutil/boto/boto/fps/connection.py +369 -0
- data/packages/gsutil/boto/boto/fps/exception.py +344 -0
- data/packages/gsutil/boto/boto/fps/response.py +175 -0
- data/packages/gsutil/boto/boto/glacier/__init__.py +57 -0
- data/packages/gsutil/boto/boto/glacier/concurrent.py +409 -0
- data/packages/gsutil/boto/boto/glacier/exceptions.py +58 -0
- data/packages/gsutil/boto/boto/glacier/job.py +152 -0
- data/packages/gsutil/boto/boto/glacier/layer1.py +637 -0
- data/packages/gsutil/boto/boto/glacier/layer2.py +93 -0
- data/packages/gsutil/boto/boto/glacier/response.py +48 -0
- data/packages/gsutil/boto/boto/glacier/utils.py +163 -0
- data/packages/gsutil/boto/boto/glacier/vault.py +387 -0
- data/packages/gsutil/boto/boto/glacier/writer.py +242 -0
- data/packages/gsutil/boto/boto/gs/__init__.py +22 -0
- data/packages/gsutil/boto/boto/gs/__init__.pyc +0 -0
- data/packages/gsutil/boto/boto/gs/acl.py +304 -0
- data/packages/gsutil/boto/boto/gs/acl.pyc +0 -0
- data/packages/gsutil/boto/boto/gs/bucket.py +870 -0
- data/packages/gsutil/boto/boto/gs/bucket.pyc +0 -0
- data/packages/gsutil/boto/boto/gs/bucketlistresultset.py +64 -0
- data/packages/gsutil/boto/boto/gs/bucketlistresultset.pyc +0 -0
- data/packages/gsutil/boto/boto/gs/connection.py +103 -0
- data/packages/gsutil/boto/boto/gs/connection.pyc +0 -0
- data/packages/gsutil/boto/boto/gs/cors.py +169 -0
- data/packages/gsutil/boto/boto/gs/cors.pyc +0 -0
- data/packages/gsutil/boto/boto/gs/key.py +704 -0
- data/packages/gsutil/boto/boto/gs/key.pyc +0 -0
- data/packages/gsutil/boto/boto/gs/resumable_upload_handler.py +659 -0
- data/packages/gsutil/boto/boto/gs/resumable_upload_handler.pyc +0 -0
- data/packages/gsutil/boto/boto/gs/user.py +54 -0
- data/packages/gsutil/boto/boto/gs/user.pyc +0 -0
- data/packages/gsutil/boto/boto/handler.py +44 -0
- data/packages/gsutil/boto/boto/handler.pyc +0 -0
- data/packages/gsutil/boto/boto/https_connection.py +124 -0
- data/packages/gsutil/boto/boto/https_connection.pyc +0 -0
- data/packages/gsutil/boto/boto/iam/__init__.py +74 -0
- data/packages/gsutil/boto/boto/iam/connection.py +1317 -0
- data/packages/gsutil/boto/boto/iam/summarymap.py +42 -0
- data/packages/gsutil/boto/boto/jsonresponse.py +163 -0
- data/packages/gsutil/boto/boto/jsonresponse.pyc +0 -0
- data/packages/gsutil/boto/boto/manage/__init__.py +23 -0
- data/packages/gsutil/boto/boto/manage/cmdshell.py +241 -0
- data/packages/gsutil/boto/boto/manage/propget.py +64 -0
- data/packages/gsutil/boto/boto/manage/server.py +556 -0
- data/packages/gsutil/boto/boto/manage/task.py +175 -0
- data/packages/gsutil/boto/boto/manage/test_manage.py +34 -0
- data/packages/gsutil/boto/boto/manage/volume.py +420 -0
- data/packages/gsutil/boto/boto/mashups/__init__.py +23 -0
- data/packages/gsutil/boto/boto/mashups/interactive.py +97 -0
- data/packages/gsutil/boto/boto/mashups/iobject.py +115 -0
- data/packages/gsutil/boto/boto/mashups/order.py +211 -0
- data/packages/gsutil/boto/boto/mashups/server.py +395 -0
- data/packages/gsutil/boto/boto/mturk/__init__.py +23 -0
- data/packages/gsutil/boto/boto/mturk/connection.py +1027 -0
- data/packages/gsutil/boto/boto/mturk/layoutparam.py +55 -0
- data/packages/gsutil/boto/boto/mturk/notification.py +103 -0
- data/packages/gsutil/boto/boto/mturk/price.py +48 -0
- data/packages/gsutil/boto/boto/mturk/qualification.py +137 -0
- data/packages/gsutil/boto/boto/mturk/question.py +455 -0
- data/packages/gsutil/boto/boto/mws/__init__.py +21 -0
- data/packages/gsutil/boto/boto/mws/connection.py +813 -0
- data/packages/gsutil/boto/boto/mws/exception.py +75 -0
- data/packages/gsutil/boto/boto/mws/response.py +655 -0
- data/packages/gsutil/boto/boto/plugin.py +90 -0
- data/packages/gsutil/boto/boto/plugin.pyc +0 -0
- data/packages/gsutil/boto/boto/provider.py +337 -0
- data/packages/gsutil/boto/boto/provider.pyc +0 -0
- data/packages/gsutil/boto/boto/pyami/__init__.py +22 -0
- data/packages/gsutil/boto/boto/pyami/__init__.pyc +0 -0
- data/packages/gsutil/boto/boto/pyami/bootstrap.py +134 -0
- data/packages/gsutil/boto/boto/pyami/config.py +229 -0
- data/packages/gsutil/boto/boto/pyami/config.pyc +0 -0
- data/packages/gsutil/boto/boto/pyami/copybot.cfg +60 -0
- data/packages/gsutil/boto/boto/pyami/copybot.py +97 -0
- data/packages/gsutil/boto/boto/pyami/helloworld.py +28 -0
- data/packages/gsutil/boto/boto/pyami/installers/__init__.py +64 -0
- data/packages/gsutil/boto/boto/pyami/installers/ubuntu/__init__.py +22 -0
- data/packages/gsutil/boto/boto/pyami/installers/ubuntu/apache.py +43 -0
- data/packages/gsutil/boto/boto/pyami/installers/ubuntu/ebs.py +238 -0
- data/packages/gsutil/boto/boto/pyami/installers/ubuntu/installer.py +96 -0
- data/packages/gsutil/boto/boto/pyami/installers/ubuntu/mysql.py +109 -0
- data/packages/gsutil/boto/boto/pyami/installers/ubuntu/trac.py +139 -0
- data/packages/gsutil/boto/boto/pyami/launch_ami.py +178 -0
- data/packages/gsutil/boto/boto/pyami/scriptbase.py +44 -0
- data/packages/gsutil/boto/boto/pyami/startup.py +60 -0
- data/packages/gsutil/boto/boto/rds/__init__.py +1194 -0
- data/packages/gsutil/boto/boto/rds/dbinstance.py +357 -0
- data/packages/gsutil/boto/boto/rds/dbsecuritygroup.py +177 -0
- data/packages/gsutil/boto/boto/rds/dbsnapshot.py +108 -0
- data/packages/gsutil/boto/boto/rds/event.py +49 -0
- data/packages/gsutil/boto/boto/rds/parametergroup.py +201 -0
- data/packages/gsutil/boto/boto/rds/regioninfo.py +32 -0
- data/packages/gsutil/boto/boto/regioninfo.py +63 -0
- data/packages/gsutil/boto/boto/regioninfo.pyc +0 -0
- data/packages/gsutil/boto/boto/resultset.py +169 -0
- data/packages/gsutil/boto/boto/resultset.pyc +0 -0
- data/packages/gsutil/boto/boto/roboto/__init__.py +1 -0
- data/packages/gsutil/boto/boto/roboto/awsqueryrequest.py +504 -0
- data/packages/gsutil/boto/boto/roboto/awsqueryservice.py +121 -0
- data/packages/gsutil/boto/boto/roboto/param.py +147 -0
- data/packages/gsutil/boto/boto/route53/__init__.py +75 -0
- data/packages/gsutil/boto/boto/route53/connection.py +403 -0
- data/packages/gsutil/boto/boto/route53/exception.py +27 -0
- data/packages/gsutil/boto/boto/route53/hostedzone.py +56 -0
- data/packages/gsutil/boto/boto/route53/record.py +306 -0
- data/packages/gsutil/boto/boto/route53/status.py +42 -0
- data/packages/gsutil/boto/boto/route53/zone.py +412 -0
- data/packages/gsutil/boto/boto/s3/__init__.py +84 -0
- data/packages/gsutil/boto/boto/s3/__init__.pyc +0 -0
- data/packages/gsutil/boto/boto/s3/acl.py +164 -0
- data/packages/gsutil/boto/boto/s3/acl.pyc +0 -0
- data/packages/gsutil/boto/boto/s3/bucket.py +1634 -0
- data/packages/gsutil/boto/boto/s3/bucket.pyc +0 -0
- data/packages/gsutil/boto/boto/s3/bucketlistresultset.py +139 -0
- data/packages/gsutil/boto/boto/s3/bucketlistresultset.pyc +0 -0
- data/packages/gsutil/boto/boto/s3/bucketlogging.py +83 -0
- data/packages/gsutil/boto/boto/s3/bucketlogging.pyc +0 -0
- data/packages/gsutil/boto/boto/s3/connection.py +540 -0
- data/packages/gsutil/boto/boto/s3/connection.pyc +0 -0
- data/packages/gsutil/boto/boto/s3/cors.py +210 -0
- data/packages/gsutil/boto/boto/s3/cors.pyc +0 -0
- data/packages/gsutil/boto/boto/s3/deletemarker.py +55 -0
- data/packages/gsutil/boto/boto/s3/deletemarker.pyc +0 -0
- data/packages/gsutil/boto/boto/s3/key.py +1712 -0
- data/packages/gsutil/boto/boto/s3/key.pyc +0 -0
- data/packages/gsutil/boto/boto/s3/keyfile.py +134 -0
- data/packages/gsutil/boto/boto/s3/keyfile.pyc +0 -0
- data/packages/gsutil/boto/boto/s3/lifecycle.py +231 -0
- data/packages/gsutil/boto/boto/s3/lifecycle.pyc +0 -0
- data/packages/gsutil/boto/boto/s3/multidelete.py +138 -0
- data/packages/gsutil/boto/boto/s3/multidelete.pyc +0 -0
- data/packages/gsutil/boto/boto/s3/multipart.py +315 -0
- data/packages/gsutil/boto/boto/s3/multipart.pyc +0 -0
- data/packages/gsutil/boto/boto/s3/prefix.py +42 -0
- data/packages/gsutil/boto/boto/s3/prefix.pyc +0 -0
- data/packages/gsutil/boto/boto/s3/resumable_download_handler.py +339 -0
- data/packages/gsutil/boto/boto/s3/resumable_download_handler.pyc +0 -0
- data/packages/gsutil/boto/boto/s3/tagging.py +71 -0
- data/packages/gsutil/boto/boto/s3/tagging.pyc +0 -0
- data/packages/gsutil/boto/boto/s3/user.py +49 -0
- data/packages/gsutil/boto/boto/s3/user.pyc +0 -0
- data/packages/gsutil/boto/boto/s3/website.py +237 -0
- data/packages/gsutil/boto/boto/s3/website.pyc +0 -0
- data/packages/gsutil/boto/boto/sdb/__init__.py +67 -0
- data/packages/gsutil/boto/boto/sdb/connection.py +617 -0
- data/packages/gsutil/boto/boto/sdb/db/__init__.py +20 -0
- data/packages/gsutil/boto/boto/sdb/db/blob.py +75 -0
- data/packages/gsutil/boto/boto/sdb/db/key.py +59 -0
- data/packages/gsutil/boto/boto/sdb/db/manager/__init__.py +85 -0
- data/packages/gsutil/boto/boto/sdb/db/manager/sdbmanager.py +732 -0
- data/packages/gsutil/boto/boto/sdb/db/manager/xmlmanager.py +517 -0
- data/packages/gsutil/boto/boto/sdb/db/model.py +294 -0
- data/packages/gsutil/boto/boto/sdb/db/property.py +703 -0
- data/packages/gsutil/boto/boto/sdb/db/query.py +85 -0
- data/packages/gsutil/boto/boto/sdb/db/sequence.py +226 -0
- data/packages/gsutil/boto/boto/sdb/db/test_db.py +231 -0
- data/packages/gsutil/boto/boto/sdb/domain.py +377 -0
- data/packages/gsutil/boto/boto/sdb/item.py +181 -0
- data/packages/gsutil/boto/boto/sdb/queryresultset.py +92 -0
- data/packages/gsutil/boto/boto/sdb/regioninfo.py +32 -0
- data/packages/gsutil/boto/boto/services/__init__.py +23 -0
- data/packages/gsutil/boto/boto/services/bs.py +179 -0
- data/packages/gsutil/boto/boto/services/message.py +58 -0
- data/packages/gsutil/boto/boto/services/result.py +136 -0
- data/packages/gsutil/boto/boto/services/service.py +161 -0
- data/packages/gsutil/boto/boto/services/servicedef.py +91 -0
- data/packages/gsutil/boto/boto/services/sonofmmm.cfg +43 -0
- data/packages/gsutil/boto/boto/services/sonofmmm.py +81 -0
- data/packages/gsutil/boto/boto/services/submit.py +88 -0
- data/packages/gsutil/boto/boto/ses/__init__.py +54 -0
- data/packages/gsutil/boto/boto/ses/connection.py +521 -0
- data/packages/gsutil/boto/boto/ses/exceptions.py +77 -0
- data/packages/gsutil/boto/boto/sns/__init__.py +78 -0
- data/packages/gsutil/boto/boto/sns/connection.py +431 -0
- data/packages/gsutil/boto/boto/sqs/__init__.py +56 -0
- data/packages/gsutil/boto/boto/sqs/attributes.py +46 -0
- data/packages/gsutil/boto/boto/sqs/batchresults.py +95 -0
- data/packages/gsutil/boto/boto/sqs/connection.py +417 -0
- data/packages/gsutil/boto/boto/sqs/jsonmessage.py +43 -0
- data/packages/gsutil/boto/boto/sqs/message.py +253 -0
- data/packages/gsutil/boto/boto/sqs/queue.py +478 -0
- data/packages/gsutil/boto/boto/sqs/regioninfo.py +32 -0
- data/packages/gsutil/boto/boto/storage_uri.py +835 -0
- data/packages/gsutil/boto/boto/storage_uri.pyc +0 -0
- data/packages/gsutil/boto/boto/sts/__init__.py +55 -0
- data/packages/gsutil/boto/boto/sts/connection.py +207 -0
- data/packages/gsutil/boto/boto/sts/credentials.py +215 -0
- data/packages/gsutil/boto/boto/swf/__init__.py +60 -0
- data/packages/gsutil/boto/boto/swf/exceptions.py +44 -0
- data/packages/gsutil/boto/boto/swf/layer1.py +1512 -0
- data/packages/gsutil/boto/boto/swf/layer1_decisions.py +287 -0
- data/packages/gsutil/boto/boto/swf/layer2.py +342 -0
- data/packages/gsutil/boto/boto/utils.py +927 -0
- data/packages/gsutil/boto/boto/utils.pyc +0 -0
- data/packages/gsutil/boto/boto/vpc/__init__.py +910 -0
- data/packages/gsutil/boto/boto/vpc/customergateway.py +54 -0
- data/packages/gsutil/boto/boto/vpc/dhcpoptions.py +72 -0
- data/packages/gsutil/boto/boto/vpc/internetgateway.py +72 -0
- data/packages/gsutil/boto/boto/vpc/routetable.py +109 -0
- data/packages/gsutil/boto/boto/vpc/subnet.py +57 -0
- data/packages/gsutil/boto/boto/vpc/vpc.py +54 -0
- data/packages/gsutil/boto/boto/vpc/vpnconnection.py +60 -0
- data/packages/gsutil/boto/boto/vpc/vpngateway.py +83 -0
- data/packages/gsutil/boto/docs/BotoCheatSheet.pdf +0 -0
- data/packages/gsutil/boto/docs/Makefile +89 -0
- data/packages/gsutil/boto/docs/make.bat +113 -0
- data/packages/gsutil/boto/docs/source/_templates/layout.html +3 -0
- data/packages/gsutil/boto/docs/source/autoscale_tut.rst +220 -0
- data/packages/gsutil/boto/docs/source/boto_config_tut.rst +125 -0
- data/packages/gsutil/boto/docs/source/boto_theme/static/boto.css_t +239 -0
- data/packages/gsutil/boto/docs/source/boto_theme/static/pygments.css +61 -0
- data/packages/gsutil/boto/docs/source/boto_theme/theme.conf +3 -0
- data/packages/gsutil/boto/docs/source/cloudfront_tut.rst +196 -0
- data/packages/gsutil/boto/docs/source/cloudsearch_tut.rst +411 -0
- data/packages/gsutil/boto/docs/source/cloudwatch_tut.rst +116 -0
- data/packages/gsutil/boto/docs/source/conf.py +32 -0
- data/packages/gsutil/boto/docs/source/contributing.rst +204 -0
- data/packages/gsutil/boto/docs/source/documentation.rst +59 -0
- data/packages/gsutil/boto/docs/source/dynamodb_tut.rst +339 -0
- data/packages/gsutil/boto/docs/source/ec2_tut.rst +86 -0
- data/packages/gsutil/boto/docs/source/elb_tut.rst +257 -0
- data/packages/gsutil/boto/docs/source/emr_tut.rst +108 -0
- data/packages/gsutil/boto/docs/source/index.rst +146 -0
- data/packages/gsutil/boto/docs/source/rds_tut.rst +108 -0
- data/packages/gsutil/boto/docs/source/ref/autoscale.rst +62 -0
- data/packages/gsutil/boto/docs/source/ref/beanstalk.rst +26 -0
- data/packages/gsutil/boto/docs/source/ref/boto.rst +47 -0
- data/packages/gsutil/boto/docs/source/ref/cloudformation.rst +34 -0
- data/packages/gsutil/boto/docs/source/ref/cloudfront.rst +68 -0
- data/packages/gsutil/boto/docs/source/ref/cloudsearch.rst +59 -0
- data/packages/gsutil/boto/docs/source/ref/cloudwatch.rst +27 -0
- data/packages/gsutil/boto/docs/source/ref/contrib.rst +32 -0
- data/packages/gsutil/boto/docs/source/ref/datapipeline.rst +26 -0
- data/packages/gsutil/boto/docs/source/ref/dynamodb.rst +61 -0
- data/packages/gsutil/boto/docs/source/ref/ec2.rst +140 -0
- data/packages/gsutil/boto/docs/source/ref/ecs.rst +19 -0
- data/packages/gsutil/boto/docs/source/ref/elasticache.rst +19 -0
- data/packages/gsutil/boto/docs/source/ref/elastictranscoder.rst +26 -0
- data/packages/gsutil/boto/docs/source/ref/elb.rst +47 -0
- data/packages/gsutil/boto/docs/source/ref/emr.rst +34 -0
- data/packages/gsutil/boto/docs/source/ref/file.rst +34 -0
- data/packages/gsutil/boto/docs/source/ref/fps.rst +19 -0
- data/packages/gsutil/boto/docs/source/ref/glacier.rst +63 -0
- data/packages/gsutil/boto/docs/source/ref/gs.rst +72 -0
- data/packages/gsutil/boto/docs/source/ref/iam.rst +27 -0
- data/packages/gsutil/boto/docs/source/ref/index.rst +40 -0
- data/packages/gsutil/boto/docs/source/ref/manage.rst +47 -0
- data/packages/gsutil/boto/docs/source/ref/mturk.rst +54 -0
- data/packages/gsutil/boto/docs/source/ref/mws.rst +33 -0
- data/packages/gsutil/boto/docs/source/ref/pyami.rst +103 -0
- data/packages/gsutil/boto/docs/source/ref/rds.rst +47 -0
- data/packages/gsutil/boto/docs/source/ref/route53.rst +34 -0
- data/packages/gsutil/boto/docs/source/ref/s3.rst +111 -0
- data/packages/gsutil/boto/docs/source/ref/sdb.rst +45 -0
- data/packages/gsutil/boto/docs/source/ref/sdb_db.rst +83 -0
- data/packages/gsutil/boto/docs/source/ref/services.rst +61 -0
- data/packages/gsutil/boto/docs/source/ref/ses.rst +21 -0
- data/packages/gsutil/boto/docs/source/ref/sns.rst +17 -0
- data/packages/gsutil/boto/docs/source/ref/sqs.rst +61 -0
- data/packages/gsutil/boto/docs/source/ref/sts.rst +25 -0
- data/packages/gsutil/boto/docs/source/ref/swf.rst +22 -0
- data/packages/gsutil/boto/docs/source/ref/vpc.rst +54 -0
- data/packages/gsutil/boto/docs/source/s3_tut.rst +450 -0
- data/packages/gsutil/boto/docs/source/security_groups.rst +82 -0
- data/packages/gsutil/boto/docs/source/ses_tut.rst +171 -0
- data/packages/gsutil/boto/docs/source/simpledb_tut.rst +188 -0
- data/packages/gsutil/boto/docs/source/sqs_tut.rst +246 -0
- data/packages/gsutil/boto/docs/source/vpc_tut.rst +100 -0
- data/packages/gsutil/boto/pylintrc +305 -0
- data/packages/gsutil/boto/requirements.txt +10 -0
- data/packages/gsutil/boto/setup.py +89 -0
- data/packages/gsutil/boto/tests/__init__.py +20 -0
- data/packages/gsutil/boto/tests/db/test_lists.py +96 -0
- data/packages/gsutil/boto/tests/db/test_password.py +128 -0
- data/packages/gsutil/boto/tests/db/test_query.py +152 -0
- data/packages/gsutil/boto/tests/db/test_sequence.py +109 -0
- data/packages/gsutil/boto/tests/devpay/__init__.py +0 -0
- data/packages/gsutil/boto/tests/devpay/test_s3.py +181 -0
- data/packages/gsutil/boto/tests/fps/__init__.py +0 -0
- data/packages/gsutil/boto/tests/fps/test.py +100 -0
- data/packages/gsutil/boto/tests/fps/test_verify_signature.py +12 -0
- data/packages/gsutil/boto/tests/integration/__init__.py +0 -0
- data/packages/gsutil/boto/tests/integration/beanstalk/test_wrapper.py +209 -0
- data/packages/gsutil/boto/tests/integration/cloudformation/__init__.py +21 -0
- data/packages/gsutil/boto/tests/integration/cloudformation/test_cert_verification.py +40 -0
- data/packages/gsutil/boto/tests/integration/cloudformation/test_connection.py +110 -0
- data/packages/gsutil/boto/tests/integration/cloudsearch/__init__.py +21 -0
- data/packages/gsutil/boto/tests/integration/cloudsearch/test_cert_verification.py +40 -0
- data/packages/gsutil/boto/tests/integration/datapipeline/test_layer1.py +122 -0
- data/packages/gsutil/boto/tests/integration/dynamodb/__init__.py +20 -0
- data/packages/gsutil/boto/tests/integration/dynamodb/test_cert_verification.py +40 -0
- data/packages/gsutil/boto/tests/integration/dynamodb/test_layer1.py +266 -0
- data/packages/gsutil/boto/tests/integration/dynamodb/test_layer2.py +484 -0
- data/packages/gsutil/boto/tests/integration/dynamodb/test_table.py +84 -0
- data/packages/gsutil/boto/tests/integration/ec2/__init__.py +20 -0
- data/packages/gsutil/boto/tests/integration/ec2/autoscale/__init__.py +21 -0
- data/packages/gsutil/boto/tests/integration/ec2/autoscale/test_cert_verification.py +40 -0
- data/packages/gsutil/boto/tests/integration/ec2/autoscale/test_connection.py +167 -0
- data/packages/gsutil/boto/tests/integration/ec2/cloudwatch/__init__.py +20 -0
- data/packages/gsutil/boto/tests/integration/ec2/cloudwatch/test_cert_verification.py +40 -0
- data/packages/gsutil/boto/tests/integration/ec2/cloudwatch/test_connection.py +277 -0
- data/packages/gsutil/boto/tests/integration/ec2/elb/__init__.py +20 -0
- data/packages/gsutil/boto/tests/integration/ec2/elb/test_cert_verification.py +40 -0
- data/packages/gsutil/boto/tests/integration/ec2/elb/test_connection.py +130 -0
- data/packages/gsutil/boto/tests/integration/ec2/test_cert_verification.py +40 -0
- data/packages/gsutil/boto/tests/integration/ec2/test_connection.py +192 -0
- data/packages/gsutil/boto/tests/integration/ec2/vpc/__init__.py +0 -0
- data/packages/gsutil/boto/tests/integration/ec2/vpc/test_connection.py +95 -0
- data/packages/gsutil/boto/tests/integration/elasticache/__init__.py +0 -0
- data/packages/gsutil/boto/tests/integration/elasticache/test_layer1.py +67 -0
- data/packages/gsutil/boto/tests/integration/elastictranscoder/__init__.py +0 -0
- data/packages/gsutil/boto/tests/integration/elastictranscoder/test_cert_verification.py +35 -0
- data/packages/gsutil/boto/tests/integration/elastictranscoder/test_layer1.py +115 -0
- data/packages/gsutil/boto/tests/integration/emr/__init__.py +20 -0
- data/packages/gsutil/boto/tests/integration/emr/test_cert_verification.py +40 -0
- data/packages/gsutil/boto/tests/integration/glacier/__init__.py +22 -0
- data/packages/gsutil/boto/tests/integration/glacier/test_cert_verification.py +40 -0
- data/packages/gsutil/boto/tests/integration/glacier/test_layer1.py +44 -0
- data/packages/gsutil/boto/tests/integration/glacier/test_layer2.py +45 -0
- data/packages/gsutil/boto/tests/integration/gs/__init__.py +0 -0
- data/packages/gsutil/boto/tests/integration/gs/cb_test_harness.py +71 -0
- data/packages/gsutil/boto/tests/integration/gs/test_basic.py +379 -0
- data/packages/gsutil/boto/tests/integration/gs/test_generation_conditionals.py +399 -0
- data/packages/gsutil/boto/tests/integration/gs/test_resumable_downloads.py +358 -0
- data/packages/gsutil/boto/tests/integration/gs/test_resumable_uploads.py +525 -0
- data/packages/gsutil/boto/tests/integration/gs/test_storage_uri.py +125 -0
- data/packages/gsutil/boto/tests/integration/gs/test_versioning.py +268 -0
- data/packages/gsutil/boto/tests/integration/gs/testcase.py +116 -0
- data/packages/gsutil/boto/tests/integration/gs/util.py +63 -0
- data/packages/gsutil/boto/tests/integration/iam/__init__.py +20 -0
- data/packages/gsutil/boto/tests/integration/iam/test_cert_verification.py +40 -0
- data/packages/gsutil/boto/tests/integration/mws/__init__.py +0 -0
- data/packages/gsutil/boto/tests/integration/mws/test.py +100 -0
- data/packages/gsutil/boto/tests/integration/rds/__init__.py +21 -0
- data/packages/gsutil/boto/tests/integration/rds/test_cert_verification.py +40 -0
- data/packages/gsutil/boto/tests/integration/route53/__init__.py +20 -0
- data/packages/gsutil/boto/tests/integration/route53/test_cert_verification.py +40 -0
- data/packages/gsutil/boto/tests/integration/route53/test_zone.py +132 -0
- data/packages/gsutil/boto/tests/integration/s3/__init__.py +20 -0
- data/packages/gsutil/boto/tests/integration/s3/mock_storage_service.py +589 -0
- data/packages/gsutil/boto/tests/integration/s3/other_cacerts.txt +70 -0
- data/packages/gsutil/boto/tests/integration/s3/test_bucket.py +263 -0
- data/packages/gsutil/boto/tests/integration/s3/test_cert_verification.py +40 -0
- data/packages/gsutil/boto/tests/integration/s3/test_connection.py +245 -0
- data/packages/gsutil/boto/tests/integration/s3/test_cors.py +78 -0
- data/packages/gsutil/boto/tests/integration/s3/test_encryption.py +115 -0
- data/packages/gsutil/boto/tests/integration/s3/test_https_cert_validation.py +141 -0
- data/packages/gsutil/boto/tests/integration/s3/test_key.py +375 -0
- data/packages/gsutil/boto/tests/integration/s3/test_mfa.py +95 -0
- data/packages/gsutil/boto/tests/integration/s3/test_multidelete.py +181 -0
- data/packages/gsutil/boto/tests/integration/s3/test_multipart.py +139 -0
- data/packages/gsutil/boto/tests/integration/s3/test_pool.py +246 -0
- data/packages/gsutil/boto/tests/integration/s3/test_versioning.py +158 -0
- data/packages/gsutil/boto/tests/integration/sdb/__init__.py +20 -0
- data/packages/gsutil/boto/tests/integration/sdb/test_cert_verification.py +40 -0
- data/packages/gsutil/boto/tests/integration/sdb/test_connection.py +119 -0
- data/packages/gsutil/boto/tests/integration/ses/__init__.py +0 -0
- data/packages/gsutil/boto/tests/integration/ses/test_cert_verification.py +40 -0
- data/packages/gsutil/boto/tests/integration/ses/test_connection.py +38 -0
- data/packages/gsutil/boto/tests/integration/sns/__init__.py +20 -0
- data/packages/gsutil/boto/tests/integration/sns/test_cert_verification.py +40 -0
- data/packages/gsutil/boto/tests/integration/sqs/__init__.py +20 -0
- data/packages/gsutil/boto/tests/integration/sqs/test_cert_verification.py +40 -0
- data/packages/gsutil/boto/tests/integration/sqs/test_connection.py +217 -0
- data/packages/gsutil/boto/tests/integration/sts/__init__.py +20 -0
- data/packages/gsutil/boto/tests/integration/sts/test_cert_verification.py +40 -0
- data/packages/gsutil/boto/tests/integration/sts/test_session_token.py +65 -0
- data/packages/gsutil/boto/tests/integration/swf/__init__.py +0 -0
- data/packages/gsutil/boto/tests/integration/swf/test_cert_verification.py +40 -0
- data/packages/gsutil/boto/tests/integration/swf/test_layer1.py +246 -0
- data/packages/gsutil/boto/tests/integration/swf/test_layer1_workflow_execution.py +173 -0
- data/packages/gsutil/boto/tests/mturk/__init__.py +0 -0
- data/packages/gsutil/boto/tests/mturk/_init_environment.py +28 -0
- data/packages/gsutil/boto/tests/mturk/all_tests.py +24 -0
- data/packages/gsutil/boto/tests/mturk/cleanup_tests.py +47 -0
- data/packages/gsutil/boto/tests/mturk/common.py +45 -0
- data/packages/gsutil/boto/tests/mturk/create_free_text_question_regex.doctest +100 -0
- data/packages/gsutil/boto/tests/mturk/create_hit.doctest +92 -0
- data/packages/gsutil/boto/tests/mturk/create_hit_binary.doctest +94 -0
- data/packages/gsutil/boto/tests/mturk/create_hit_external.py +21 -0
- data/packages/gsutil/boto/tests/mturk/create_hit_from_hit_type.doctest +103 -0
- data/packages/gsutil/boto/tests/mturk/create_hit_test.py +21 -0
- data/packages/gsutil/boto/tests/mturk/create_hit_with_qualifications.py +16 -0
- data/packages/gsutil/boto/tests/mturk/hit_persistence.py +27 -0
- data/packages/gsutil/boto/tests/mturk/mocks.py +11 -0
- data/packages/gsutil/boto/tests/mturk/reviewable_hits.doctest +129 -0
- data/packages/gsutil/boto/tests/mturk/run-doctest.py +13 -0
- data/packages/gsutil/boto/tests/mturk/search_hits.doctest +16 -0
- data/packages/gsutil/boto/tests/mturk/selenium_support.py +61 -0
- data/packages/gsutil/boto/tests/mturk/support.py +7 -0
- data/packages/gsutil/boto/tests/mturk/test_disable_hit.py +11 -0
- data/packages/gsutil/boto/tests/test.py +59 -0
- data/packages/gsutil/boto/tests/unit/__init__.py +79 -0
- data/packages/gsutil/boto/tests/unit/auth/__init__.py +0 -0
- data/packages/gsutil/boto/tests/unit/auth/test_sigv4.py +73 -0
- data/packages/gsutil/boto/tests/unit/beanstalk/__init__.py +0 -0
- data/packages/gsutil/boto/tests/unit/beanstalk/test_layer1.py +128 -0
- data/packages/gsutil/boto/tests/unit/cloudformation/__init__.py +0 -0
- data/packages/gsutil/boto/tests/unit/cloudformation/test_connection.py +605 -0
- data/packages/gsutil/boto/tests/unit/cloudformation/test_stack.py +63 -0
- data/packages/gsutil/boto/tests/unit/cloudfront/__init__.py +0 -0
- data/packages/gsutil/boto/tests/unit/cloudfront/test_invalidation_list.py +113 -0
- data/packages/gsutil/boto/tests/unit/cloudfront/test_signed_urls.py +354 -0
- data/packages/gsutil/boto/tests/unit/cloudsearch/__init__.py +1 -0
- data/packages/gsutil/boto/tests/unit/cloudsearch/test_connection.py +241 -0
- data/packages/gsutil/boto/tests/unit/cloudsearch/test_document.py +324 -0
- data/packages/gsutil/boto/tests/unit/cloudsearch/test_search.py +325 -0
- data/packages/gsutil/boto/tests/unit/dynamodb/__init__.py +0 -0
- data/packages/gsutil/boto/tests/unit/dynamodb/test_batch.py +103 -0
- data/packages/gsutil/boto/tests/unit/dynamodb/test_layer2.py +119 -0
- data/packages/gsutil/boto/tests/unit/dynamodb/test_types.py +82 -0
- data/packages/gsutil/boto/tests/unit/ec2/__init__.py +0 -0
- data/packages/gsutil/boto/tests/unit/ec2/autoscale/__init__.py +0 -0
- data/packages/gsutil/boto/tests/unit/ec2/autoscale/test_group.py +162 -0
- data/packages/gsutil/boto/tests/unit/ec2/test_address.py +39 -0
- data/packages/gsutil/boto/tests/unit/ec2/test_blockdevicemapping.py +79 -0
- data/packages/gsutil/boto/tests/unit/ec2/test_connection.py +480 -0
- data/packages/gsutil/boto/tests/unit/ec2/test_instance.py +243 -0
- data/packages/gsutil/boto/tests/unit/ec2/test_networkinterface.py +140 -0
- data/packages/gsutil/boto/tests/unit/ec2/test_volume.py +248 -0
- data/packages/gsutil/boto/tests/unit/emr/test_emr_responses.py +373 -0
- data/packages/gsutil/boto/tests/unit/glacier/__init__.py +0 -0
- data/packages/gsutil/boto/tests/unit/glacier/test_concurrent.py +120 -0
- data/packages/gsutil/boto/tests/unit/glacier/test_job.py +60 -0
- data/packages/gsutil/boto/tests/unit/glacier/test_layer1.py +98 -0
- data/packages/gsutil/boto/tests/unit/glacier/test_layer2.py +266 -0
- data/packages/gsutil/boto/tests/unit/glacier/test_utils.py +116 -0
- data/packages/gsutil/boto/tests/unit/glacier/test_vault.py +100 -0
- data/packages/gsutil/boto/tests/unit/glacier/test_writer.py +185 -0
- data/packages/gsutil/boto/tests/unit/provider/__init__.py +0 -0
- data/packages/gsutil/boto/tests/unit/provider/test_provider.py +176 -0
- data/packages/gsutil/boto/tests/unit/rds/__init__.py +0 -0
- data/packages/gsutil/boto/tests/unit/rds/test_connection.py +131 -0
- data/packages/gsutil/boto/tests/unit/s3/__init__.py +0 -0
- data/packages/gsutil/boto/tests/unit/s3/test_cors_configuration.py +77 -0
- data/packages/gsutil/boto/tests/unit/s3/test_key.py +75 -0
- data/packages/gsutil/boto/tests/unit/s3/test_keyfile.py +101 -0
- data/packages/gsutil/boto/tests/unit/s3/test_lifecycle.py +97 -0
- data/packages/gsutil/boto/tests/unit/s3/test_tagging.py +47 -0
- data/packages/gsutil/boto/tests/unit/s3/test_uri.py +257 -0
- data/packages/gsutil/boto/tests/unit/s3/test_website.py +188 -0
- data/packages/gsutil/boto/tests/unit/sns/__init__.py +0 -0
- data/packages/gsutil/boto/tests/unit/sns/test_connection.py +99 -0
- data/packages/gsutil/boto/tests/unit/sqs/__init__.py +0 -0
- data/packages/gsutil/boto/tests/unit/sqs/test_connection.py +98 -0
- data/packages/gsutil/boto/tests/unit/sqs/test_queue.py +40 -0
- data/packages/gsutil/boto/tests/unit/sts/test_connection.py +74 -0
- data/packages/gsutil/boto/tests/unit/test_connection.py +60 -0
- data/packages/gsutil/boto/tests/unit/utils/test_utils.py +109 -0
- data/packages/gsutil/boto/tox.ini +8 -0
- data/packages/gsutil/gslib/README +5 -0
- data/packages/gsutil/gslib/__init__.py +22 -0
- data/packages/gsutil/gslib/__init__.pyc +0 -0
- data/packages/gsutil/gslib/addlhelp/__init__.py +15 -0
- data/packages/gsutil/gslib/addlhelp/acls.py +234 -0
- data/packages/gsutil/gslib/addlhelp/anon.py +57 -0
- data/packages/gsutil/gslib/addlhelp/command_opts.py +116 -0
- data/packages/gsutil/gslib/addlhelp/dev.py +139 -0
- data/packages/gsutil/gslib/addlhelp/metadata.py +186 -0
- data/packages/gsutil/gslib/addlhelp/naming.py +173 -0
- data/packages/gsutil/gslib/addlhelp/prod.py +160 -0
- data/packages/gsutil/gslib/addlhelp/projects.py +130 -0
- data/packages/gsutil/gslib/addlhelp/subdirs.py +110 -0
- data/packages/gsutil/gslib/addlhelp/support.py +86 -0
- data/packages/gsutil/gslib/addlhelp/versioning.py +242 -0
- data/packages/gsutil/gslib/addlhelp/wildcards.py +170 -0
- data/packages/gsutil/gslib/bucket_listing_ref.py +175 -0
- data/packages/gsutil/gslib/bucket_listing_ref.pyc +0 -0
- data/packages/gsutil/gslib/command.py +722 -0
- data/packages/gsutil/gslib/command.pyc +0 -0
- data/packages/gsutil/gslib/command_runner.py +101 -0
- data/packages/gsutil/gslib/command_runner.pyc +0 -0
- data/packages/gsutil/gslib/commands/__init__.py +15 -0
- data/packages/gsutil/gslib/commands/__init__.pyc +0 -0
- data/packages/gsutil/gslib/commands/cat.py +131 -0
- data/packages/gsutil/gslib/commands/cat.pyc +0 -0
- data/packages/gsutil/gslib/commands/chacl.py +532 -0
- data/packages/gsutil/gslib/commands/chacl.pyc +0 -0
- data/packages/gsutil/gslib/commands/config.py +694 -0
- data/packages/gsutil/gslib/commands/config.pyc +0 -0
- data/packages/gsutil/gslib/commands/cp.py +1818 -0
- data/packages/gsutil/gslib/commands/cp.pyc +0 -0
- data/packages/gsutil/gslib/commands/disablelogging.py +101 -0
- data/packages/gsutil/gslib/commands/disablelogging.pyc +0 -0
- data/packages/gsutil/gslib/commands/enablelogging.py +149 -0
- data/packages/gsutil/gslib/commands/enablelogging.pyc +0 -0
- data/packages/gsutil/gslib/commands/getacl.py +82 -0
- data/packages/gsutil/gslib/commands/getacl.pyc +0 -0
- data/packages/gsutil/gslib/commands/getcors.py +121 -0
- data/packages/gsutil/gslib/commands/getcors.pyc +0 -0
- data/packages/gsutil/gslib/commands/getdefacl.py +86 -0
- data/packages/gsutil/gslib/commands/getdefacl.pyc +0 -0
- data/packages/gsutil/gslib/commands/getlogging.py +137 -0
- data/packages/gsutil/gslib/commands/getlogging.pyc +0 -0
- data/packages/gsutil/gslib/commands/getversioning.py +116 -0
- data/packages/gsutil/gslib/commands/getversioning.pyc +0 -0
- data/packages/gsutil/gslib/commands/getwebcfg.py +122 -0
- data/packages/gsutil/gslib/commands/getwebcfg.pyc +0 -0
- data/packages/gsutil/gslib/commands/help.py +218 -0
- data/packages/gsutil/gslib/commands/help.pyc +0 -0
- data/packages/gsutil/gslib/commands/ls.py +578 -0
- data/packages/gsutil/gslib/commands/ls.pyc +0 -0
- data/packages/gsutil/gslib/commands/mb.py +172 -0
- data/packages/gsutil/gslib/commands/mb.pyc +0 -0
- data/packages/gsutil/gslib/commands/mv.py +159 -0
- data/packages/gsutil/gslib/commands/mv.pyc +0 -0
- data/packages/gsutil/gslib/commands/perfdiag.py +903 -0
- data/packages/gsutil/gslib/commands/perfdiag.pyc +0 -0
- data/packages/gsutil/gslib/commands/rb.py +113 -0
- data/packages/gsutil/gslib/commands/rb.pyc +0 -0
- data/packages/gsutil/gslib/commands/rm.py +239 -0
- data/packages/gsutil/gslib/commands/rm.pyc +0 -0
- data/packages/gsutil/gslib/commands/setacl.py +138 -0
- data/packages/gsutil/gslib/commands/setacl.pyc +0 -0
- data/packages/gsutil/gslib/commands/setcors.py +145 -0
- data/packages/gsutil/gslib/commands/setcors.pyc +0 -0
- data/packages/gsutil/gslib/commands/setdefacl.py +105 -0
- data/packages/gsutil/gslib/commands/setdefacl.pyc +0 -0
- data/packages/gsutil/gslib/commands/setmeta.py +428 -0
- data/packages/gsutil/gslib/commands/setmeta.pyc +0 -0
- data/packages/gsutil/gslib/commands/setversioning.py +114 -0
- data/packages/gsutil/gslib/commands/setversioning.pyc +0 -0
- data/packages/gsutil/gslib/commands/setwebcfg.py +190 -0
- data/packages/gsutil/gslib/commands/setwebcfg.pyc +0 -0
- data/packages/gsutil/gslib/commands/test.py +228 -0
- data/packages/gsutil/gslib/commands/test.pyc +0 -0
- data/packages/gsutil/gslib/commands/update.py +305 -0
- data/packages/gsutil/gslib/commands/update.pyc +0 -0
- data/packages/gsutil/gslib/commands/version.py +150 -0
- data/packages/gsutil/gslib/commands/version.pyc +0 -0
- data/packages/gsutil/gslib/exception.py +76 -0
- data/packages/gsutil/gslib/exception.pyc +0 -0
- data/packages/gsutil/gslib/help_provider.py +81 -0
- data/packages/gsutil/gslib/help_provider.pyc +0 -0
- data/packages/gsutil/gslib/name_expansion.py +550 -0
- data/packages/gsutil/gslib/name_expansion.pyc +0 -0
- data/packages/gsutil/gslib/no_op_auth_plugin.py +30 -0
- data/packages/gsutil/gslib/plurality_checkable_iterator.py +56 -0
- data/packages/gsutil/gslib/plurality_checkable_iterator.pyc +0 -0
- data/packages/gsutil/gslib/project_id.py +67 -0
- data/packages/gsutil/gslib/project_id.pyc +0 -0
- data/packages/gsutil/gslib/storage_uri_builder.py +56 -0
- data/packages/gsutil/gslib/storage_uri_builder.pyc +0 -0
- data/packages/gsutil/gslib/tests/__init__.py +15 -0
- data/packages/gsutil/gslib/tests/__init__.pyc +0 -0
- data/packages/gsutil/gslib/tests/test_chacl.py +236 -0
- data/packages/gsutil/gslib/tests/test_cp.py +267 -0
- data/packages/gsutil/gslib/tests/test_data/test.gif +0 -0
- data/packages/gsutil/gslib/tests/test_data/test.mp3 +0 -0
- data/packages/gsutil/gslib/tests/test_ls.py +66 -0
- data/packages/gsutil/gslib/tests/test_mv.py +69 -0
- data/packages/gsutil/gslib/tests/test_naming.py +989 -0
- data/packages/gsutil/gslib/tests/test_perfdiag.py +41 -0
- data/packages/gsutil/gslib/tests/test_plurality_checkable_iterator.py +67 -0
- data/packages/gsutil/gslib/tests/test_rm.py +143 -0
- data/packages/gsutil/gslib/tests/test_setacl.py +152 -0
- data/packages/gsutil/gslib/tests/test_setcors.py +168 -0
- data/packages/gsutil/gslib/tests/test_setmeta.py +91 -0
- data/packages/gsutil/gslib/tests/test_setversioning.py +44 -0
- data/packages/gsutil/gslib/tests/test_setwebcfg.py +63 -0
- data/packages/gsutil/gslib/tests/test_thread_pool.py +92 -0
- data/packages/gsutil/gslib/tests/test_wildcard_iterator.py +364 -0
- data/packages/gsutil/gslib/tests/testcase/__init__.py +18 -0
- data/packages/gsutil/gslib/tests/testcase/base.py +89 -0
- data/packages/gsutil/gslib/tests/testcase/integration_testcase.py +197 -0
- data/packages/gsutil/gslib/tests/testcase/unit_testcase.py +230 -0
- data/packages/gsutil/gslib/tests/util.py +125 -0
- data/packages/gsutil/gslib/tests/util.pyc +0 -0
- data/packages/gsutil/gslib/thread_pool.py +79 -0
- data/packages/gsutil/gslib/thread_pool.pyc +0 -0
- data/packages/gsutil/gslib/util.py +151 -0
- data/packages/gsutil/gslib/util.pyc +0 -0
- data/packages/gsutil/gslib/wildcard_iterator.py +492 -0
- data/packages/gsutil/gslib/wildcard_iterator.pyc +0 -0
- data/packages/gsutil/gsutil +377 -0
- data/packages/gsutil/gsutil.spec.in +75 -0
- data/packages/gsutil/oauth2_plugin/__init__.py +22 -0
- data/packages/gsutil/oauth2_plugin/__init__.pyc +0 -0
- data/packages/gsutil/oauth2_plugin/oauth2_client.py +642 -0
- data/packages/gsutil/oauth2_plugin/oauth2_client.pyc +0 -0
- data/packages/gsutil/oauth2_plugin/oauth2_client_test.py +374 -0
- data/packages/gsutil/oauth2_plugin/oauth2_helper.py +110 -0
- data/packages/gsutil/oauth2_plugin/oauth2_helper.pyc +0 -0
- data/packages/gsutil/oauth2_plugin/oauth2_plugin.py +24 -0
- data/packages/gsutil/oauth2_plugin/oauth2_plugin.pyc +0 -0
- data/packages/gsutil/pkg_gen.sh +54 -0
- data/packages/gsutil/pkg_util.py +60 -0
- data/packages/gsutil/setup.py +141 -0
- data/packages/gsutil/third_party/__init__.py +0 -0
- data/packages/gsutil/third_party/__init__.pyc +0 -0
- data/packages/gsutil/third_party/fancy_urllib/README +21 -0
- data/packages/gsutil/third_party/fancy_urllib/__init__.py +398 -0
- data/packages/gsutil/third_party/fancy_urllib/__init__.pyc +0 -0
- data/tasks/rubygem.rake +3 -1
- metadata +770 -7
- metadata.gz.sig +0 -0
Binary file
|
@@ -0,0 +1,659 @@
|
|
1
|
+
# Copyright 2010 Google Inc.
|
2
|
+
#
|
3
|
+
# Permission is hereby granted, free of charge, to any person obtaining a
|
4
|
+
# copy of this software and associated documentation files (the
|
5
|
+
# "Software"), to deal in the Software without restriction, including
|
6
|
+
# without limitation the rights to use, copy, modify, merge, publish, dis-
|
7
|
+
# tribute, sublicense, and/or sell copies of the Software, and to permit
|
8
|
+
# persons to whom the Software is furnished to do so, subject to the fol-
|
9
|
+
# lowing conditions:
|
10
|
+
#
|
11
|
+
# The above copyright notice and this permission notice shall be included
|
12
|
+
# in all copies or substantial portions of the Software.
|
13
|
+
#
|
14
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
|
15
|
+
# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
|
16
|
+
# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
|
17
|
+
# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
|
18
|
+
# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
19
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
|
20
|
+
# IN THE SOFTWARE.
|
21
|
+
|
22
|
+
import cgi
|
23
|
+
import errno
|
24
|
+
import httplib
|
25
|
+
import os
|
26
|
+
import random
|
27
|
+
import re
|
28
|
+
import socket
|
29
|
+
import time
|
30
|
+
import urlparse
|
31
|
+
import boto
|
32
|
+
from boto import config, UserAgent
|
33
|
+
from boto.connection import AWSAuthConnection
|
34
|
+
from boto.exception import InvalidUriError
|
35
|
+
from boto.exception import ResumableTransferDisposition
|
36
|
+
from boto.exception import ResumableUploadException
|
37
|
+
from boto.s3.keyfile import KeyFile
|
38
|
+
try:
|
39
|
+
from hashlib import md5
|
40
|
+
except ImportError:
|
41
|
+
from md5 import md5
|
42
|
+
|
43
|
+
"""
|
44
|
+
Handler for Google Cloud Storage resumable uploads. See
|
45
|
+
http://code.google.com/apis/storage/docs/developer-guide.html#resumable
|
46
|
+
for details.
|
47
|
+
|
48
|
+
Resumable uploads will retry failed uploads, resuming at the byte
|
49
|
+
count completed by the last upload attempt. If too many retries happen with
|
50
|
+
no progress (per configurable num_retries param), the upload will be
|
51
|
+
aborted in the current process.
|
52
|
+
|
53
|
+
The caller can optionally specify a tracker_file_name param in the
|
54
|
+
ResumableUploadHandler constructor. If you do this, that file will
|
55
|
+
save the state needed to allow retrying later, in a separate process
|
56
|
+
(e.g., in a later run of gsutil).
|
57
|
+
"""
|
58
|
+
|
59
|
+
|
60
|
+
class ResumableUploadHandler(object):
|
61
|
+
|
62
|
+
BUFFER_SIZE = 8192
|
63
|
+
RETRYABLE_EXCEPTIONS = (httplib.HTTPException, IOError, socket.error,
|
64
|
+
socket.gaierror)
|
65
|
+
|
66
|
+
# (start, end) response indicating server has nothing (upload protocol uses
|
67
|
+
# inclusive numbering).
|
68
|
+
SERVER_HAS_NOTHING = (0, -1)
|
69
|
+
|
70
|
+
def __init__(self, tracker_file_name=None, num_retries=None):
|
71
|
+
"""
|
72
|
+
Constructor. Instantiate once for each uploaded file.
|
73
|
+
|
74
|
+
:type tracker_file_name: string
|
75
|
+
:param tracker_file_name: optional file name to save tracker URI.
|
76
|
+
If supplied and the current process fails the upload, it can be
|
77
|
+
retried in a new process. If called with an existing file containing
|
78
|
+
a valid tracker URI, we'll resume the upload from this URI; else
|
79
|
+
we'll start a new resumable upload (and write the URI to this
|
80
|
+
tracker file).
|
81
|
+
|
82
|
+
:type num_retries: int
|
83
|
+
:param num_retries: the number of times we'll re-try a resumable upload
|
84
|
+
making no progress. (Count resets every time we get progress, so
|
85
|
+
upload can span many more than this number of retries.)
|
86
|
+
"""
|
87
|
+
self.tracker_file_name = tracker_file_name
|
88
|
+
self.num_retries = num_retries
|
89
|
+
self.server_has_bytes = 0 # Byte count at last server check.
|
90
|
+
self.tracker_uri = None
|
91
|
+
if tracker_file_name:
|
92
|
+
self._load_tracker_uri_from_file()
|
93
|
+
# Save upload_start_point in instance state so caller can find how
|
94
|
+
# much was transferred by this ResumableUploadHandler (across retries).
|
95
|
+
self.upload_start_point = None
|
96
|
+
|
97
|
+
def _load_tracker_uri_from_file(self):
|
98
|
+
f = None
|
99
|
+
try:
|
100
|
+
f = open(self.tracker_file_name, 'r')
|
101
|
+
uri = f.readline().strip()
|
102
|
+
self._set_tracker_uri(uri)
|
103
|
+
except IOError, e:
|
104
|
+
# Ignore non-existent file (happens first time an upload
|
105
|
+
# is attempted on a file), but warn user for other errors.
|
106
|
+
if e.errno != errno.ENOENT:
|
107
|
+
# Will restart because self.tracker_uri == None.
|
108
|
+
print('Couldn\'t read URI tracker file (%s): %s. Restarting '
|
109
|
+
'upload from scratch.' %
|
110
|
+
(self.tracker_file_name, e.strerror))
|
111
|
+
except InvalidUriError, e:
|
112
|
+
# Warn user, but proceed (will restart because
|
113
|
+
# self.tracker_uri == None).
|
114
|
+
print('Invalid tracker URI (%s) found in URI tracker file '
|
115
|
+
'(%s). Restarting upload from scratch.' %
|
116
|
+
(uri, self.tracker_file_name))
|
117
|
+
finally:
|
118
|
+
if f:
|
119
|
+
f.close()
|
120
|
+
|
121
|
+
def _save_tracker_uri_to_file(self):
|
122
|
+
"""
|
123
|
+
Saves URI to tracker file if one was passed to constructor.
|
124
|
+
"""
|
125
|
+
if not self.tracker_file_name:
|
126
|
+
return
|
127
|
+
f = None
|
128
|
+
try:
|
129
|
+
f = open(self.tracker_file_name, 'w')
|
130
|
+
f.write(self.tracker_uri)
|
131
|
+
except IOError, e:
|
132
|
+
raise ResumableUploadException(
|
133
|
+
'Couldn\'t write URI tracker file (%s): %s.\nThis can happen'
|
134
|
+
'if you\'re using an incorrectly configured upload tool\n'
|
135
|
+
'(e.g., gsutil configured to save tracker files to an '
|
136
|
+
'unwritable directory)' %
|
137
|
+
(self.tracker_file_name, e.strerror),
|
138
|
+
ResumableTransferDisposition.ABORT)
|
139
|
+
finally:
|
140
|
+
if f:
|
141
|
+
f.close()
|
142
|
+
|
143
|
+
def _set_tracker_uri(self, uri):
|
144
|
+
"""
|
145
|
+
Called when we start a new resumable upload or get a new tracker
|
146
|
+
URI for the upload. Saves URI and resets upload state.
|
147
|
+
|
148
|
+
Raises InvalidUriError if URI is syntactically invalid.
|
149
|
+
"""
|
150
|
+
parse_result = urlparse.urlparse(uri)
|
151
|
+
if (parse_result.scheme.lower() not in ['http', 'https'] or
|
152
|
+
not parse_result.netloc):
|
153
|
+
raise InvalidUriError('Invalid tracker URI (%s)' % uri)
|
154
|
+
self.tracker_uri = uri
|
155
|
+
self.tracker_uri_host = parse_result.netloc
|
156
|
+
self.tracker_uri_path = '%s?%s' % (
|
157
|
+
parse_result.path, parse_result.query)
|
158
|
+
self.server_has_bytes = 0
|
159
|
+
|
160
|
+
def get_tracker_uri(self):
|
161
|
+
"""
|
162
|
+
Returns upload tracker URI, or None if the upload has not yet started.
|
163
|
+
"""
|
164
|
+
return self.tracker_uri
|
165
|
+
|
166
|
+
def _remove_tracker_file(self):
|
167
|
+
if (self.tracker_file_name and
|
168
|
+
os.path.exists(self.tracker_file_name)):
|
169
|
+
os.unlink(self.tracker_file_name)
|
170
|
+
|
171
|
+
def _build_content_range_header(self, range_spec='*', length_spec='*'):
|
172
|
+
return 'bytes %s/%s' % (range_spec, length_spec)
|
173
|
+
|
174
|
+
def _query_server_state(self, conn, file_length):
|
175
|
+
"""
|
176
|
+
Queries server to find out state of given upload.
|
177
|
+
|
178
|
+
Note that this method really just makes special case use of the
|
179
|
+
fact that the upload server always returns the current start/end
|
180
|
+
state whenever a PUT doesn't complete.
|
181
|
+
|
182
|
+
Returns HTTP response from sending request.
|
183
|
+
|
184
|
+
Raises ResumableUploadException if problem querying server.
|
185
|
+
"""
|
186
|
+
# Send an empty PUT so that server replies with this resumable
|
187
|
+
# transfer's state.
|
188
|
+
put_headers = {}
|
189
|
+
put_headers['Content-Range'] = (
|
190
|
+
self._build_content_range_header('*', file_length))
|
191
|
+
put_headers['Content-Length'] = '0'
|
192
|
+
return AWSAuthConnection.make_request(conn, 'PUT',
|
193
|
+
path=self.tracker_uri_path,
|
194
|
+
auth_path=self.tracker_uri_path,
|
195
|
+
headers=put_headers,
|
196
|
+
host=self.tracker_uri_host)
|
197
|
+
|
198
|
+
def _query_server_pos(self, conn, file_length):
|
199
|
+
"""
|
200
|
+
Queries server to find out what bytes it currently has.
|
201
|
+
|
202
|
+
Returns (server_start, server_end), where the values are inclusive.
|
203
|
+
For example, (0, 2) would mean that the server has bytes 0, 1, *and* 2.
|
204
|
+
|
205
|
+
Raises ResumableUploadException if problem querying server.
|
206
|
+
"""
|
207
|
+
resp = self._query_server_state(conn, file_length)
|
208
|
+
if resp.status == 200:
|
209
|
+
# To handle the boundary condition where the server has the complete
|
210
|
+
# file, we return (server_start, file_length-1). That way the
|
211
|
+
# calling code can always simply read up through server_end. (If we
|
212
|
+
# didn't handle this boundary condition here, the caller would have
|
213
|
+
# to check whether server_end == file_length and read one fewer byte
|
214
|
+
# in that case.)
|
215
|
+
return (0, file_length - 1) # Completed upload.
|
216
|
+
if resp.status != 308:
|
217
|
+
# This means the server didn't have any state for the given
|
218
|
+
# upload ID, which can happen (for example) if the caller saved
|
219
|
+
# the tracker URI to a file and then tried to restart the transfer
|
220
|
+
# after that upload ID has gone stale. In that case we need to
|
221
|
+
# start a new transfer (and the caller will then save the new
|
222
|
+
# tracker URI to the tracker file).
|
223
|
+
raise ResumableUploadException(
|
224
|
+
'Got non-308 response (%s) from server state query' %
|
225
|
+
resp.status, ResumableTransferDisposition.START_OVER)
|
226
|
+
got_valid_response = False
|
227
|
+
range_spec = resp.getheader('range')
|
228
|
+
if range_spec:
|
229
|
+
# Parse 'bytes=<from>-<to>' range_spec.
|
230
|
+
m = re.search('bytes=(\d+)-(\d+)', range_spec)
|
231
|
+
if m:
|
232
|
+
server_start = long(m.group(1))
|
233
|
+
server_end = long(m.group(2))
|
234
|
+
got_valid_response = True
|
235
|
+
else:
|
236
|
+
# No Range header, which means the server does not yet have
|
237
|
+
# any bytes. Note that the Range header uses inclusive 'from'
|
238
|
+
# and 'to' values. Since Range 0-0 would mean that the server
|
239
|
+
# has byte 0, omitting the Range header is used to indicate that
|
240
|
+
# the server doesn't have any bytes.
|
241
|
+
return self.SERVER_HAS_NOTHING
|
242
|
+
if not got_valid_response:
|
243
|
+
raise ResumableUploadException(
|
244
|
+
'Couldn\'t parse upload server state query response (%s)' %
|
245
|
+
str(resp.getheaders()), ResumableTransferDisposition.START_OVER)
|
246
|
+
if conn.debug >= 1:
|
247
|
+
print 'Server has: Range: %d - %d.' % (server_start, server_end)
|
248
|
+
return (server_start, server_end)
|
249
|
+
|
250
|
+
def _start_new_resumable_upload(self, key, headers=None):
|
251
|
+
"""
|
252
|
+
Starts a new resumable upload.
|
253
|
+
|
254
|
+
Raises ResumableUploadException if any errors occur.
|
255
|
+
"""
|
256
|
+
conn = key.bucket.connection
|
257
|
+
if conn.debug >= 1:
|
258
|
+
print 'Starting new resumable upload.'
|
259
|
+
self.server_has_bytes = 0
|
260
|
+
|
261
|
+
# Start a new resumable upload by sending a POST request with an
|
262
|
+
# empty body and the "X-Goog-Resumable: start" header. Include any
|
263
|
+
# caller-provided headers (e.g., Content-Type) EXCEPT Content-Length
|
264
|
+
# (and raise an exception if they tried to pass one, since it's
|
265
|
+
# a semantic error to specify it at this point, and if we were to
|
266
|
+
# include one now it would cause the server to expect that many
|
267
|
+
# bytes; the POST doesn't include the actual file bytes We set
|
268
|
+
# the Content-Length in the subsequent PUT, based on the uploaded
|
269
|
+
# file size.
|
270
|
+
post_headers = {}
|
271
|
+
for k in headers:
|
272
|
+
if k.lower() == 'content-length':
|
273
|
+
raise ResumableUploadException(
|
274
|
+
'Attempt to specify Content-Length header (disallowed)',
|
275
|
+
ResumableTransferDisposition.ABORT)
|
276
|
+
post_headers[k] = headers[k]
|
277
|
+
post_headers[conn.provider.resumable_upload_header] = 'start'
|
278
|
+
|
279
|
+
resp = conn.make_request(
|
280
|
+
'POST', key.bucket.name, key.name, post_headers)
|
281
|
+
# Get tracker URI from response 'Location' header.
|
282
|
+
body = resp.read()
|
283
|
+
|
284
|
+
# Check for various status conditions.
|
285
|
+
if resp.status in [500, 503]:
|
286
|
+
# Retry status 500 and 503 errors after a delay.
|
287
|
+
raise ResumableUploadException(
|
288
|
+
'Got status %d from attempt to start resumable upload. '
|
289
|
+
'Will wait/retry' % resp.status,
|
290
|
+
ResumableTransferDisposition.WAIT_BEFORE_RETRY)
|
291
|
+
elif resp.status != 200 and resp.status != 201:
|
292
|
+
raise ResumableUploadException(
|
293
|
+
'Got status %d from attempt to start resumable upload. '
|
294
|
+
'Aborting' % resp.status,
|
295
|
+
ResumableTransferDisposition.ABORT)
|
296
|
+
|
297
|
+
# Else we got 200 or 201 response code, indicating the resumable
|
298
|
+
# upload was created.
|
299
|
+
tracker_uri = resp.getheader('Location')
|
300
|
+
if not tracker_uri:
|
301
|
+
raise ResumableUploadException(
|
302
|
+
'No resumable tracker URI found in resumable initiation '
|
303
|
+
'POST response (%s)' % body,
|
304
|
+
ResumableTransferDisposition.WAIT_BEFORE_RETRY)
|
305
|
+
self._set_tracker_uri(tracker_uri)
|
306
|
+
self._save_tracker_uri_to_file()
|
307
|
+
|
308
|
+
def _upload_file_bytes(self, conn, http_conn, fp, file_length,
|
309
|
+
total_bytes_uploaded, cb, num_cb, md5sum, headers):
|
310
|
+
"""
|
311
|
+
Makes one attempt to upload file bytes, using an existing resumable
|
312
|
+
upload connection.
|
313
|
+
|
314
|
+
Returns (etag, generation, meta_generation) from server upon success.
|
315
|
+
|
316
|
+
Raises ResumableUploadException if any problems occur.
|
317
|
+
"""
|
318
|
+
buf = fp.read(self.BUFFER_SIZE)
|
319
|
+
if cb:
|
320
|
+
# The cb_count represents the number of full buffers to send between
|
321
|
+
# cb executions.
|
322
|
+
if num_cb > 2:
|
323
|
+
cb_count = file_length / self.BUFFER_SIZE / (num_cb-2)
|
324
|
+
elif num_cb < 0:
|
325
|
+
cb_count = -1
|
326
|
+
else:
|
327
|
+
cb_count = 0
|
328
|
+
i = 0
|
329
|
+
cb(total_bytes_uploaded, file_length)
|
330
|
+
|
331
|
+
# Build resumable upload headers for the transfer. Don't send a
|
332
|
+
# Content-Range header if the file is 0 bytes long, because the
|
333
|
+
# resumable upload protocol uses an *inclusive* end-range (so, sending
|
334
|
+
# 'bytes 0-0/1' would actually mean you're sending a 1-byte file).
|
335
|
+
if not headers:
|
336
|
+
put_headers = {}
|
337
|
+
else:
|
338
|
+
put_headers = headers.copy()
|
339
|
+
if file_length:
|
340
|
+
if total_bytes_uploaded == file_length:
|
341
|
+
range_header = self._build_content_range_header(
|
342
|
+
'*', file_length)
|
343
|
+
else:
|
344
|
+
range_header = self._build_content_range_header(
|
345
|
+
'%d-%d' % (total_bytes_uploaded, file_length - 1),
|
346
|
+
file_length)
|
347
|
+
put_headers['Content-Range'] = range_header
|
348
|
+
# Set Content-Length to the total bytes we'll send with this PUT.
|
349
|
+
put_headers['Content-Length'] = str(file_length - total_bytes_uploaded)
|
350
|
+
http_request = AWSAuthConnection.build_base_http_request(
|
351
|
+
conn, 'PUT', path=self.tracker_uri_path, auth_path=None,
|
352
|
+
headers=put_headers, host=self.tracker_uri_host)
|
353
|
+
http_conn.putrequest('PUT', http_request.path)
|
354
|
+
for k in put_headers:
|
355
|
+
http_conn.putheader(k, put_headers[k])
|
356
|
+
http_conn.endheaders()
|
357
|
+
|
358
|
+
# Turn off debug on http connection so upload content isn't included
|
359
|
+
# in debug stream.
|
360
|
+
http_conn.set_debuglevel(0)
|
361
|
+
while buf:
|
362
|
+
http_conn.send(buf)
|
363
|
+
md5sum.update(buf)
|
364
|
+
total_bytes_uploaded += len(buf)
|
365
|
+
if cb:
|
366
|
+
i += 1
|
367
|
+
if i == cb_count or cb_count == -1:
|
368
|
+
cb(total_bytes_uploaded, file_length)
|
369
|
+
i = 0
|
370
|
+
buf = fp.read(self.BUFFER_SIZE)
|
371
|
+
http_conn.set_debuglevel(conn.debug)
|
372
|
+
if cb:
|
373
|
+
cb(total_bytes_uploaded, file_length)
|
374
|
+
if total_bytes_uploaded != file_length:
|
375
|
+
# Abort (and delete the tracker file) so if the user retries
|
376
|
+
# they'll start a new resumable upload rather than potentially
|
377
|
+
# attempting to pick back up later where we left off.
|
378
|
+
raise ResumableUploadException(
|
379
|
+
'File changed during upload: EOF at %d bytes of %d byte file.' %
|
380
|
+
(total_bytes_uploaded, file_length),
|
381
|
+
ResumableTransferDisposition.ABORT)
|
382
|
+
resp = http_conn.getresponse()
|
383
|
+
body = resp.read()
|
384
|
+
# Restore http connection debug level.
|
385
|
+
http_conn.set_debuglevel(conn.debug)
|
386
|
+
|
387
|
+
if resp.status == 200:
|
388
|
+
# Success.
|
389
|
+
return (resp.getheader('etag'),
|
390
|
+
resp.getheader('x-goog-generation'),
|
391
|
+
resp.getheader('x-goog-metageneration'))
|
392
|
+
# Retry timeout (408) and status 500 and 503 errors after a delay.
|
393
|
+
elif resp.status in [408, 500, 503]:
|
394
|
+
disposition = ResumableTransferDisposition.WAIT_BEFORE_RETRY
|
395
|
+
else:
|
396
|
+
# Catch all for any other error codes.
|
397
|
+
disposition = ResumableTransferDisposition.ABORT
|
398
|
+
raise ResumableUploadException('Got response code %d while attempting '
|
399
|
+
'upload (%s)' %
|
400
|
+
(resp.status, resp.reason), disposition)
|
401
|
+
|
402
|
+
def _attempt_resumable_upload(self, key, fp, file_length, headers, cb,
|
403
|
+
num_cb, md5sum):
|
404
|
+
"""
|
405
|
+
Attempts a resumable upload.
|
406
|
+
|
407
|
+
Returns (etag, generation, meta_generation) from server upon success.
|
408
|
+
|
409
|
+
Raises ResumableUploadException if any problems occur.
|
410
|
+
"""
|
411
|
+
(server_start, server_end) = self.SERVER_HAS_NOTHING
|
412
|
+
conn = key.bucket.connection
|
413
|
+
if self.tracker_uri:
|
414
|
+
# Try to resume existing resumable upload.
|
415
|
+
try:
|
416
|
+
(server_start, server_end) = (
|
417
|
+
self._query_server_pos(conn, file_length))
|
418
|
+
self.server_has_bytes = server_start
|
419
|
+
|
420
|
+
if server_end:
|
421
|
+
# If the server already has some of the content, we need to
|
422
|
+
# update the md5 with the bytes that have already been
|
423
|
+
# uploaded to ensure we get a complete hash in the end.
|
424
|
+
print 'Catching up md5 for resumed upload'
|
425
|
+
fp.seek(0)
|
426
|
+
# Read local file's bytes through position server has. For
|
427
|
+
# example, if server has (0, 3) we want to read 3-0+1=4 bytes.
|
428
|
+
bytes_to_go = server_end + 1
|
429
|
+
while bytes_to_go:
|
430
|
+
chunk = fp.read(min(key.BufferSize, bytes_to_go))
|
431
|
+
if not chunk:
|
432
|
+
raise ResumableUploadException(
|
433
|
+
'Hit end of file during resumable upload md5 '
|
434
|
+
'catchup. This should not happen under\n'
|
435
|
+
'normal circumstances, as it indicates the '
|
436
|
+
'server has more bytes of this transfer\nthan'
|
437
|
+
' the current file size. Restarting upload.',
|
438
|
+
ResumableTransferDisposition.START_OVER)
|
439
|
+
md5sum.update(chunk)
|
440
|
+
bytes_to_go -= len(chunk)
|
441
|
+
|
442
|
+
if conn.debug >= 1:
|
443
|
+
print 'Resuming transfer.'
|
444
|
+
except ResumableUploadException, e:
|
445
|
+
if conn.debug >= 1:
|
446
|
+
print 'Unable to resume transfer (%s).' % e.message
|
447
|
+
self._start_new_resumable_upload(key, headers)
|
448
|
+
else:
|
449
|
+
self._start_new_resumable_upload(key, headers)
|
450
|
+
|
451
|
+
# upload_start_point allows the code that instantiated the
|
452
|
+
# ResumableUploadHandler to find out the point from which it started
|
453
|
+
# uploading (e.g., so it can correctly compute throughput).
|
454
|
+
if self.upload_start_point is None:
|
455
|
+
self.upload_start_point = server_end
|
456
|
+
|
457
|
+
total_bytes_uploaded = server_end + 1
|
458
|
+
# Corner case: Don't attempt to seek if we've already uploaded the
|
459
|
+
# entire file, because if the file is a stream (e.g., the KeyFile
|
460
|
+
# wrapper around input key when copying between providers), attempting
|
461
|
+
# to seek to the end of file would result in an InvalidRange error.
|
462
|
+
if file_length < total_bytes_uploaded:
|
463
|
+
fp.seek(total_bytes_uploaded)
|
464
|
+
conn = key.bucket.connection
|
465
|
+
|
466
|
+
# Get a new HTTP connection (vs conn.get_http_connection(), which reuses
|
467
|
+
# pool connections) because httplib requires a new HTTP connection per
|
468
|
+
# transaction. (Without this, calling http_conn.getresponse() would get
|
469
|
+
# "ResponseNotReady".)
|
470
|
+
http_conn = conn.new_http_connection(self.tracker_uri_host,
|
471
|
+
conn.is_secure)
|
472
|
+
http_conn.set_debuglevel(conn.debug)
|
473
|
+
|
474
|
+
# Make sure to close http_conn at end so if a local file read
|
475
|
+
# failure occurs partway through server will terminate current upload
|
476
|
+
# and can report that progress on next attempt.
|
477
|
+
try:
|
478
|
+
return self._upload_file_bytes(conn, http_conn, fp, file_length,
|
479
|
+
total_bytes_uploaded, cb, num_cb, md5sum,
|
480
|
+
headers)
|
481
|
+
except (ResumableUploadException, socket.error):
|
482
|
+
resp = self._query_server_state(conn, file_length)
|
483
|
+
if resp.status == 400:
|
484
|
+
raise ResumableUploadException('Got 400 response from server '
|
485
|
+
'state query after failed resumable upload attempt. This '
|
486
|
+
'can happen for various reasons, including specifying an '
|
487
|
+
'invalid request (e.g., an invalid canned ACL) or if the '
|
488
|
+
'file size changed between upload attempts',
|
489
|
+
ResumableTransferDisposition.ABORT)
|
490
|
+
else:
|
491
|
+
raise
|
492
|
+
finally:
|
493
|
+
http_conn.close()
|
494
|
+
|
495
|
+
def _check_final_md5(self, key, etag):
|
496
|
+
"""
|
497
|
+
Checks that etag from server agrees with md5 computed before upload.
|
498
|
+
This is important, since the upload could have spanned a number of
|
499
|
+
hours and multiple processes (e.g., gsutil runs), and the user could
|
500
|
+
change some of the file and not realize they have inconsistent data.
|
501
|
+
"""
|
502
|
+
if key.bucket.connection.debug >= 1:
|
503
|
+
print 'Checking md5 against etag.'
|
504
|
+
if key.md5 != etag.strip('"\''):
|
505
|
+
# Call key.open_read() before attempting to delete the
|
506
|
+
# (incorrect-content) key, so we perform that request on a
|
507
|
+
# different HTTP connection. This is neededb because httplib
|
508
|
+
# will return a "Response not ready" error if you try to perform
|
509
|
+
# a second transaction on the connection.
|
510
|
+
key.open_read()
|
511
|
+
key.close()
|
512
|
+
key.delete()
|
513
|
+
raise ResumableUploadException(
|
514
|
+
'File changed during upload: md5 signature doesn\'t match etag '
|
515
|
+
'(incorrect uploaded object deleted)',
|
516
|
+
ResumableTransferDisposition.ABORT)
|
517
|
+
|
518
|
+
def handle_resumable_upload_exception(self, e, debug):
|
519
|
+
if (e.disposition == ResumableTransferDisposition.ABORT_CUR_PROCESS):
|
520
|
+
if debug >= 1:
|
521
|
+
print('Caught non-retryable ResumableUploadException (%s); '
|
522
|
+
'aborting but retaining tracker file' % e.message)
|
523
|
+
raise
|
524
|
+
elif (e.disposition == ResumableTransferDisposition.ABORT):
|
525
|
+
if debug >= 1:
|
526
|
+
print('Caught non-retryable ResumableUploadException (%s); '
|
527
|
+
'aborting and removing tracker file' % e.message)
|
528
|
+
self._remove_tracker_file()
|
529
|
+
raise
|
530
|
+
else:
|
531
|
+
if debug >= 1:
|
532
|
+
print('Caught ResumableUploadException (%s) - will retry' %
|
533
|
+
e.message)
|
534
|
+
|
535
|
+
def track_progress_less_iterations(self, server_had_bytes_before_attempt,
|
536
|
+
roll_back_md5=True, debug=0):
|
537
|
+
# At this point we had a re-tryable failure; see if made progress.
|
538
|
+
if self.server_has_bytes > server_had_bytes_before_attempt:
|
539
|
+
self.progress_less_iterations = 0 # If progress, reset counter.
|
540
|
+
else:
|
541
|
+
self.progress_less_iterations += 1
|
542
|
+
if roll_back_md5:
|
543
|
+
# Rollback any potential md5sum updates, as we did not
|
544
|
+
# make any progress in this iteration.
|
545
|
+
self.md5sum = self.md5sum_before_attempt
|
546
|
+
|
547
|
+
if self.progress_less_iterations > self.num_retries:
|
548
|
+
# Don't retry any longer in the current process.
|
549
|
+
raise ResumableUploadException(
|
550
|
+
'Too many resumable upload attempts failed without '
|
551
|
+
'progress. You might try this upload again later',
|
552
|
+
ResumableTransferDisposition.ABORT_CUR_PROCESS)
|
553
|
+
|
554
|
+
# Use binary exponential backoff to desynchronize client requests.
|
555
|
+
sleep_time_secs = random.random() * (2**self.progress_less_iterations)
|
556
|
+
if debug >= 1:
|
557
|
+
print ('Got retryable failure (%d progress-less in a row).\n'
|
558
|
+
'Sleeping %3.1f seconds before re-trying' %
|
559
|
+
(self.progress_less_iterations, sleep_time_secs))
|
560
|
+
time.sleep(sleep_time_secs)
|
561
|
+
|
562
|
+
def send_file(self, key, fp, headers, cb=None, num_cb=10):
|
563
|
+
"""
|
564
|
+
Upload a file to a key into a bucket on GS, using GS resumable upload
|
565
|
+
protocol.
|
566
|
+
|
567
|
+
:type key: :class:`boto.s3.key.Key` or subclass
|
568
|
+
:param key: The Key object to which data is to be uploaded
|
569
|
+
|
570
|
+
:type fp: file-like object
|
571
|
+
:param fp: The file pointer to upload
|
572
|
+
|
573
|
+
:type headers: dict
|
574
|
+
:param headers: The headers to pass along with the PUT request
|
575
|
+
|
576
|
+
:type cb: function
|
577
|
+
:param cb: a callback function that will be called to report progress on
|
578
|
+
the upload. The callback should accept two integer parameters, the
|
579
|
+
first representing the number of bytes that have been successfully
|
580
|
+
transmitted to GS, and the second representing the total number of
|
581
|
+
bytes that need to be transmitted.
|
582
|
+
|
583
|
+
:type num_cb: int
|
584
|
+
:param num_cb: (optional) If a callback is specified with the cb
|
585
|
+
parameter, this parameter determines the granularity of the callback
|
586
|
+
by defining the maximum number of times the callback will be called
|
587
|
+
during the file transfer. Providing a negative integer will cause
|
588
|
+
your callback to be called with each buffer read.
|
589
|
+
|
590
|
+
Raises ResumableUploadException if a problem occurs during the transfer.
|
591
|
+
"""
|
592
|
+
|
593
|
+
if not headers:
|
594
|
+
headers = {}
|
595
|
+
# If Content-Type header is present and set to None, remove it.
|
596
|
+
# This is gsutil's way of asking boto to refrain from auto-generating
|
597
|
+
# that header.
|
598
|
+
CT = 'Content-Type'
|
599
|
+
if CT in headers and headers[CT] is None:
|
600
|
+
del headers[CT]
|
601
|
+
|
602
|
+
headers['User-Agent'] = UserAgent
|
603
|
+
|
604
|
+
# Determine file size different ways for case where fp is actually a
|
605
|
+
# wrapper around a Key vs an actual file.
|
606
|
+
if isinstance(fp, KeyFile):
|
607
|
+
file_length = fp.getkey().size
|
608
|
+
else:
|
609
|
+
fp.seek(0, os.SEEK_END)
|
610
|
+
file_length = fp.tell()
|
611
|
+
fp.seek(0)
|
612
|
+
debug = key.bucket.connection.debug
|
613
|
+
|
614
|
+
# Compute the MD5 checksum on the fly.
|
615
|
+
self.md5sum = md5()
|
616
|
+
|
617
|
+
# Use num-retries from constructor if one was provided; else check
|
618
|
+
# for a value specified in the boto config file; else default to 5.
|
619
|
+
if self.num_retries is None:
|
620
|
+
self.num_retries = config.getint('Boto', 'num_retries', 6)
|
621
|
+
self.progress_less_iterations = 0
|
622
|
+
|
623
|
+
while True: # Retry as long as we're making progress.
|
624
|
+
server_had_bytes_before_attempt = self.server_has_bytes
|
625
|
+
self.md5sum_before_attempt = self.md5sum.copy()
|
626
|
+
try:
|
627
|
+
# Save generation and meta_generation in class state so caller
|
628
|
+
# can find these values, for use in preconditions of future
|
629
|
+
# operations on the uploaded object.
|
630
|
+
(etag, self.generation, self.meta_generation) = (
|
631
|
+
self._attempt_resumable_upload(key, fp, file_length,
|
632
|
+
headers, cb, num_cb,
|
633
|
+
self.md5sum))
|
634
|
+
|
635
|
+
# Get the final md5 for the uploaded content.
|
636
|
+
hd = self.md5sum.hexdigest()
|
637
|
+
key.md5, key.base64md5 = key.get_md5_from_hexdigest(hd)
|
638
|
+
|
639
|
+
# Upload succceded, so remove the tracker file (if have one).
|
640
|
+
self._remove_tracker_file()
|
641
|
+
self._check_final_md5(key, etag)
|
642
|
+
if debug >= 1:
|
643
|
+
print 'Resumable upload complete.'
|
644
|
+
return
|
645
|
+
except self.RETRYABLE_EXCEPTIONS, e:
|
646
|
+
if debug >= 1:
|
647
|
+
print('Caught exception (%s)' % e.__repr__())
|
648
|
+
if isinstance(e, IOError) and e.errno == errno.EPIPE:
|
649
|
+
# Broken pipe error causes httplib to immediately
|
650
|
+
# close the socket (http://bugs.python.org/issue5542),
|
651
|
+
# so we need to close the connection before we resume
|
652
|
+
# the upload (which will cause a new connection to be
|
653
|
+
# opened the next time an HTTP request is sent).
|
654
|
+
key.bucket.connection.connection.close()
|
655
|
+
except ResumableUploadException, e:
|
656
|
+
self.handle_resumable_upload_exception(e, debug)
|
657
|
+
|
658
|
+
self.track_progress_less_iterations(server_had_bytes_before_attempt,
|
659
|
+
True, debug)
|