s3_cmd_bin 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. data/.gitignore +17 -0
  2. data/Gemfile +3 -0
  3. data/LICENSE.txt +22 -0
  4. data/README.md +28 -0
  5. data/Rakefile +1 -0
  6. data/lib/s3_cmd_bin/version.rb +3 -0
  7. data/lib/s3_cmd_bin.rb +15 -0
  8. data/resources/ChangeLog +1462 -0
  9. data/resources/INSTALL +97 -0
  10. data/resources/LICENSE +339 -0
  11. data/resources/MANIFEST.in +2 -0
  12. data/resources/Makefile +4 -0
  13. data/resources/NEWS +234 -0
  14. data/resources/README +342 -0
  15. data/resources/S3/ACL.py +224 -0
  16. data/resources/S3/ACL.pyc +0 -0
  17. data/resources/S3/AccessLog.py +92 -0
  18. data/resources/S3/AccessLog.pyc +0 -0
  19. data/resources/S3/BidirMap.py +42 -0
  20. data/resources/S3/BidirMap.pyc +0 -0
  21. data/resources/S3/CloudFront.py +773 -0
  22. data/resources/S3/CloudFront.pyc +0 -0
  23. data/resources/S3/Config.py +294 -0
  24. data/resources/S3/Config.pyc +0 -0
  25. data/resources/S3/ConnMan.py +71 -0
  26. data/resources/S3/ConnMan.pyc +0 -0
  27. data/resources/S3/Exceptions.py +88 -0
  28. data/resources/S3/Exceptions.pyc +0 -0
  29. data/resources/S3/FileDict.py +53 -0
  30. data/resources/S3/FileDict.pyc +0 -0
  31. data/resources/S3/FileLists.py +517 -0
  32. data/resources/S3/FileLists.pyc +0 -0
  33. data/resources/S3/HashCache.py +53 -0
  34. data/resources/S3/HashCache.pyc +0 -0
  35. data/resources/S3/MultiPart.py +137 -0
  36. data/resources/S3/MultiPart.pyc +0 -0
  37. data/resources/S3/PkgInfo.py +14 -0
  38. data/resources/S3/PkgInfo.pyc +0 -0
  39. data/resources/S3/Progress.py +173 -0
  40. data/resources/S3/Progress.pyc +0 -0
  41. data/resources/S3/S3.py +979 -0
  42. data/resources/S3/S3.pyc +0 -0
  43. data/resources/S3/S3Uri.py +223 -0
  44. data/resources/S3/S3Uri.pyc +0 -0
  45. data/resources/S3/SimpleDB.py +178 -0
  46. data/resources/S3/SortedDict.py +66 -0
  47. data/resources/S3/SortedDict.pyc +0 -0
  48. data/resources/S3/Utils.py +462 -0
  49. data/resources/S3/Utils.pyc +0 -0
  50. data/resources/S3/__init__.py +0 -0
  51. data/resources/S3/__init__.pyc +0 -0
  52. data/resources/TODO +52 -0
  53. data/resources/artwork/AtomicClockRadio.ttf +0 -0
  54. data/resources/artwork/TypeRa.ttf +0 -0
  55. data/resources/artwork/site-top-full-size.xcf +0 -0
  56. data/resources/artwork/site-top-label-download.png +0 -0
  57. data/resources/artwork/site-top-label-s3cmd.png +0 -0
  58. data/resources/artwork/site-top-label-s3sync.png +0 -0
  59. data/resources/artwork/site-top-s3tools-logo.png +0 -0
  60. data/resources/artwork/site-top.jpg +0 -0
  61. data/resources/artwork/site-top.png +0 -0
  62. data/resources/artwork/site-top.xcf +0 -0
  63. data/resources/format-manpage.pl +196 -0
  64. data/resources/magic +63 -0
  65. data/resources/run-tests.py +537 -0
  66. data/resources/s3cmd +2116 -0
  67. data/resources/s3cmd.1 +435 -0
  68. data/resources/s3db +55 -0
  69. data/resources/setup.cfg +2 -0
  70. data/resources/setup.py +80 -0
  71. data/resources/testsuite.tar.gz +0 -0
  72. data/resources/upload-to-sf.sh +7 -0
  73. data/s3_cmd_bin.gemspec +23 -0
  74. metadata +152 -0
Binary file
@@ -0,0 +1,223 @@
1
+ ## Amazon S3 manager
2
+ ## Author: Michal Ludvig <michal@logix.cz>
3
+ ## http://www.logix.cz/michal
4
+ ## License: GPL Version 2
5
+
6
+ import os
7
+ import re
8
+ import sys
9
+ from BidirMap import BidirMap
10
+ from logging import debug
11
+ import S3
12
+ from Utils import unicodise, check_bucket_name_dns_conformity
13
+ import Config
14
+
15
+ class S3Uri(object):
16
+ type = None
17
+ _subclasses = None
18
+
19
+ def __new__(self, string):
20
+ if not self._subclasses:
21
+ ## Generate a list of all subclasses of S3Uri
22
+ self._subclasses = []
23
+ dict = sys.modules[__name__].__dict__
24
+ for something in dict:
25
+ if type(dict[something]) is not type(self):
26
+ continue
27
+ if issubclass(dict[something], self) and dict[something] != self:
28
+ self._subclasses.append(dict[something])
29
+ for subclass in self._subclasses:
30
+ try:
31
+ instance = object.__new__(subclass)
32
+ instance.__init__(string)
33
+ return instance
34
+ except ValueError, e:
35
+ continue
36
+ raise ValueError("%s: not a recognized URI" % string)
37
+
38
+ def __str__(self):
39
+ return self.uri()
40
+
41
+ def __unicode__(self):
42
+ return self.uri()
43
+
44
+ def __repr__(self):
45
+ return "<%s: %s>" % (self.__class__.__name__, self.__unicode__())
46
+
47
+ def public_url(self):
48
+ raise ValueError("This S3 URI does not have Anonymous URL representation")
49
+
50
+ def basename(self):
51
+ return self.__unicode__().split("/")[-1]
52
+
53
+ class S3UriS3(S3Uri):
54
+ type = "s3"
55
+ _re = re.compile("^s3://([^/]+)/?(.*)", re.IGNORECASE)
56
+ def __init__(self, string):
57
+ match = self._re.match(string)
58
+ if not match:
59
+ raise ValueError("%s: not a S3 URI" % string)
60
+ groups = match.groups()
61
+ self._bucket = groups[0]
62
+ self._object = unicodise(groups[1])
63
+
64
+ def bucket(self):
65
+ return self._bucket
66
+
67
+ def object(self):
68
+ return self._object
69
+
70
+ def has_bucket(self):
71
+ return bool(self._bucket)
72
+
73
+ def has_object(self):
74
+ return bool(self._object)
75
+
76
+ def uri(self):
77
+ return "/".join(["s3:/", self._bucket, self._object])
78
+
79
+ def is_dns_compatible(self):
80
+ return check_bucket_name_dns_conformity(self._bucket)
81
+
82
+ def public_url(self):
83
+ if self.is_dns_compatible():
84
+ return "http://%s.%s/%s" % (self._bucket, Config.Config().host_base, self._object)
85
+ else:
86
+ return "http://%s/%s/%s" % (self._bucket, Config.Config().host_base, self._object)
87
+
88
+ def host_name(self):
89
+ if self.is_dns_compatible():
90
+ return "%s.s3.amazonaws.com" % (self._bucket)
91
+ else:
92
+ return "s3.amazonaws.com"
93
+
94
+ @staticmethod
95
+ def compose_uri(bucket, object = ""):
96
+ return "s3://%s/%s" % (bucket, object)
97
+
98
+ @staticmethod
99
+ def httpurl_to_s3uri(http_url):
100
+ m=re.match("(https?://)?([^/]+)/?(.*)", http_url, re.IGNORECASE)
101
+ hostname, object = m.groups()[1:]
102
+ hostname = hostname.lower()
103
+ if hostname == "s3.amazonaws.com":
104
+ ## old-style url: http://s3.amazonaws.com/bucket/object
105
+ if object.count("/") == 0:
106
+ ## no object given
107
+ bucket = object
108
+ object = ""
109
+ else:
110
+ ## bucket/object
111
+ bucket, object = object.split("/", 1)
112
+ elif hostname.endswith(".s3.amazonaws.com"):
113
+ ## new-style url: http://bucket.s3.amazonaws.com/object
114
+ bucket = hostname[:-(len(".s3.amazonaws.com"))]
115
+ else:
116
+ raise ValueError("Unable to parse URL: %s" % http_url)
117
+ return S3Uri("s3://%(bucket)s/%(object)s" % {
118
+ 'bucket' : bucket,
119
+ 'object' : object })
120
+
121
+ class S3UriS3FS(S3Uri):
122
+ type = "s3fs"
123
+ _re = re.compile("^s3fs://([^/]*)/?(.*)", re.IGNORECASE)
124
+ def __init__(self, string):
125
+ match = self._re.match(string)
126
+ if not match:
127
+ raise ValueError("%s: not a S3fs URI" % string)
128
+ groups = match.groups()
129
+ self._fsname = groups[0]
130
+ self._path = unicodise(groups[1]).split("/")
131
+
132
+ def fsname(self):
133
+ return self._fsname
134
+
135
+ def path(self):
136
+ return "/".join(self._path)
137
+
138
+ def uri(self):
139
+ return "/".join(["s3fs:/", self._fsname, self.path()])
140
+
141
+ class S3UriFile(S3Uri):
142
+ type = "file"
143
+ _re = re.compile("^(\w+://)?(.*)")
144
+ def __init__(self, string):
145
+ match = self._re.match(string)
146
+ groups = match.groups()
147
+ if groups[0] not in (None, "file://"):
148
+ raise ValueError("%s: not a file:// URI" % string)
149
+ self._path = unicodise(groups[1]).split("/")
150
+
151
+ def path(self):
152
+ return "/".join(self._path)
153
+
154
+ def uri(self):
155
+ return "/".join(["file:/", self.path()])
156
+
157
+ def isdir(self):
158
+ return os.path.isdir(self.path())
159
+
160
+ def dirname(self):
161
+ return os.path.dirname(self.path())
162
+
163
+ class S3UriCloudFront(S3Uri):
164
+ type = "cf"
165
+ _re = re.compile("^cf://([^/]*)/*(.*)", re.IGNORECASE)
166
+ def __init__(self, string):
167
+ match = self._re.match(string)
168
+ if not match:
169
+ raise ValueError("%s: not a CloudFront URI" % string)
170
+ groups = match.groups()
171
+ self._dist_id = groups[0]
172
+ self._request_id = groups[1] != "/" and groups[1] or None
173
+
174
+ def dist_id(self):
175
+ return self._dist_id
176
+
177
+ def request_id(self):
178
+ return self._request_id
179
+
180
+ def uri(self):
181
+ uri = "cf://" + self.dist_id()
182
+ if self.request_id():
183
+ uri += "/" + self.request_id()
184
+ return uri
185
+
186
+ if __name__ == "__main__":
187
+ uri = S3Uri("s3://bucket/object")
188
+ print "type() =", type(uri)
189
+ print "uri =", uri
190
+ print "uri.type=", uri.type
191
+ print "bucket =", uri.bucket()
192
+ print "object =", uri.object()
193
+ print
194
+
195
+ uri = S3Uri("s3://bucket")
196
+ print "type() =", type(uri)
197
+ print "uri =", uri
198
+ print "uri.type=", uri.type
199
+ print "bucket =", uri.bucket()
200
+ print
201
+
202
+ uri = S3Uri("s3fs://filesystem1/path/to/remote/file.txt")
203
+ print "type() =", type(uri)
204
+ print "uri =", uri
205
+ print "uri.type=", uri.type
206
+ print "path =", uri.path()
207
+ print
208
+
209
+ uri = S3Uri("/path/to/local/file.txt")
210
+ print "type() =", type(uri)
211
+ print "uri =", uri
212
+ print "uri.type=", uri.type
213
+ print "path =", uri.path()
214
+ print
215
+
216
+ uri = S3Uri("cf://1234567890ABCD/")
217
+ print "type() =", type(uri)
218
+ print "uri =", uri
219
+ print "uri.type=", uri.type
220
+ print "dist_id =", uri.dist_id()
221
+ print
222
+
223
+ # vim:et:ts=4:sts=4:ai
Binary file
@@ -0,0 +1,178 @@
1
+ ## Amazon SimpleDB library
2
+ ## Author: Michal Ludvig <michal@logix.cz>
3
+ ## http://www.logix.cz/michal
4
+ ## License: GPL Version 2
5
+
6
+ """
7
+ Low-level class for working with Amazon SimpleDB
8
+ """
9
+
10
+ import time
11
+ import urllib
12
+ import base64
13
+ import hmac
14
+ import sha
15
+ import httplib
16
+ from logging import debug, info, warning, error
17
+
18
+ from Utils import convertTupleListToDict
19
+ from SortedDict import SortedDict
20
+ from Exceptions import *
21
+
22
+ class SimpleDB(object):
23
+ # API Version
24
+ # See http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/
25
+ Version = "2007-11-07"
26
+ SignatureVersion = 1
27
+
28
+ def __init__(self, config):
29
+ self.config = config
30
+
31
+ ## ------------------------------------------------
32
+ ## Methods implementing SimpleDB API
33
+ ## ------------------------------------------------
34
+
35
+ def ListDomains(self, MaxNumberOfDomains = 100):
36
+ '''
37
+ Lists all domains associated with our Access Key. Returns
38
+ domain names up to the limit set by MaxNumberOfDomains.
39
+ '''
40
+ parameters = SortedDict()
41
+ parameters['MaxNumberOfDomains'] = MaxNumberOfDomains
42
+ return self.send_request("ListDomains", DomainName = None, parameters = parameters)
43
+
44
+ def CreateDomain(self, DomainName):
45
+ return self.send_request("CreateDomain", DomainName = DomainName)
46
+
47
+ def DeleteDomain(self, DomainName):
48
+ return self.send_request("DeleteDomain", DomainName = DomainName)
49
+
50
+ def PutAttributes(self, DomainName, ItemName, Attributes):
51
+ parameters = SortedDict()
52
+ parameters['ItemName'] = ItemName
53
+ seq = 0
54
+ for attrib in Attributes:
55
+ if type(Attributes[attrib]) == type(list()):
56
+ for value in Attributes[attrib]:
57
+ parameters['Attribute.%d.Name' % seq] = attrib
58
+ parameters['Attribute.%d.Value' % seq] = unicode(value)
59
+ seq += 1
60
+ else:
61
+ parameters['Attribute.%d.Name' % seq] = attrib
62
+ parameters['Attribute.%d.Value' % seq] = unicode(Attributes[attrib])
63
+ seq += 1
64
+ ## TODO:
65
+ ## - support for Attribute.N.Replace
66
+ ## - support for multiple values for one attribute
67
+ return self.send_request("PutAttributes", DomainName = DomainName, parameters = parameters)
68
+
69
+ def GetAttributes(self, DomainName, ItemName, Attributes = []):
70
+ parameters = SortedDict()
71
+ parameters['ItemName'] = ItemName
72
+ seq = 0
73
+ for attrib in Attributes:
74
+ parameters['AttributeName.%d' % seq] = attrib
75
+ seq += 1
76
+ return self.send_request("GetAttributes", DomainName = DomainName, parameters = parameters)
77
+
78
+ def DeleteAttributes(self, DomainName, ItemName, Attributes = {}):
79
+ """
80
+ Remove specified Attributes from ItemName.
81
+ Attributes parameter can be either:
82
+ - not specified, in which case the whole Item is removed
83
+ - list, e.g. ['Attr1', 'Attr2'] in which case these parameters are removed
84
+ - dict, e.g. {'Attr' : 'One', 'Attr' : 'Two'} in which case the
85
+ specified values are removed from multi-value attributes.
86
+ """
87
+ parameters = SortedDict()
88
+ parameters['ItemName'] = ItemName
89
+ seq = 0
90
+ for attrib in Attributes:
91
+ parameters['Attribute.%d.Name' % seq] = attrib
92
+ if type(Attributes) == type(dict()):
93
+ parameters['Attribute.%d.Value' % seq] = unicode(Attributes[attrib])
94
+ seq += 1
95
+ return self.send_request("DeleteAttributes", DomainName = DomainName, parameters = parameters)
96
+
97
+ def Query(self, DomainName, QueryExpression = None, MaxNumberOfItems = None, NextToken = None):
98
+ parameters = SortedDict()
99
+ if QueryExpression:
100
+ parameters['QueryExpression'] = QueryExpression
101
+ if MaxNumberOfItems:
102
+ parameters['MaxNumberOfItems'] = MaxNumberOfItems
103
+ if NextToken:
104
+ parameters['NextToken'] = NextToken
105
+ return self.send_request("Query", DomainName = DomainName, parameters = parameters)
106
+ ## Handle NextToken? Or maybe not - let the upper level do it
107
+
108
+ ## ------------------------------------------------
109
+ ## Low-level methods for handling SimpleDB requests
110
+ ## ------------------------------------------------
111
+
112
+ def send_request(self, *args, **kwargs):
113
+ request = self.create_request(*args, **kwargs)
114
+ #debug("Request: %s" % repr(request))
115
+ conn = self.get_connection()
116
+ conn.request("GET", self.format_uri(request['uri_params']))
117
+ http_response = conn.getresponse()
118
+ response = {}
119
+ response["status"] = http_response.status
120
+ response["reason"] = http_response.reason
121
+ response["headers"] = convertTupleListToDict(http_response.getheaders())
122
+ response["data"] = http_response.read()
123
+ conn.close()
124
+
125
+ if response["status"] < 200 or response["status"] > 299:
126
+ debug("Response: " + str(response))
127
+ raise S3Error(response)
128
+
129
+ return response
130
+
131
+ def create_request(self, Action, DomainName, parameters = None):
132
+ if not parameters:
133
+ parameters = SortedDict()
134
+ if len(self.config.access_token) > 0:
135
+ self.config.refresh_role()
136
+ parameters['Signature']=self.config.access_token
137
+ parameters['AWSAccessKeyId'] = self.config.access_key
138
+ parameters['Version'] = self.Version
139
+ parameters['SignatureVersion'] = self.SignatureVersion
140
+ parameters['Action'] = Action
141
+ parameters['Timestamp'] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
142
+ if DomainName:
143
+ parameters['DomainName'] = DomainName
144
+ parameters['Signature'] = self.sign_request(parameters)
145
+ parameters.keys_return_lowercase = False
146
+ uri_params = urllib.urlencode(parameters)
147
+ request = {}
148
+ request['uri_params'] = uri_params
149
+ request['parameters'] = parameters
150
+ return request
151
+
152
+ def sign_request(self, parameters):
153
+ h = ""
154
+ parameters.keys_sort_lowercase = True
155
+ parameters.keys_return_lowercase = False
156
+ for key in parameters:
157
+ h += "%s%s" % (key, parameters[key])
158
+ #debug("SignRequest: %s" % h)
159
+ return base64.encodestring(hmac.new(self.config.secret_key, h, sha).digest()).strip()
160
+
161
+ def get_connection(self):
162
+ if self.config.proxy_host != "":
163
+ return httplib.HTTPConnection(self.config.proxy_host, self.config.proxy_port)
164
+ else:
165
+ if self.config.use_https:
166
+ return httplib.HTTPSConnection(self.config.simpledb_host)
167
+ else:
168
+ return httplib.HTTPConnection(self.config.simpledb_host)
169
+
170
+ def format_uri(self, uri_params):
171
+ if self.config.proxy_host != "":
172
+ uri = "http://%s/?%s" % (self.config.simpledb_host, uri_params)
173
+ else:
174
+ uri = "/?%s" % uri_params
175
+ #debug('format_uri(): ' + uri)
176
+ return uri
177
+
178
+ # vim:et:ts=4:sts=4:ai
@@ -0,0 +1,66 @@
1
+ ## Amazon S3 manager
2
+ ## Author: Michal Ludvig <michal@logix.cz>
3
+ ## http://www.logix.cz/michal
4
+ ## License: GPL Version 2
5
+
6
+ from BidirMap import BidirMap
7
+ import Utils
8
+
9
+ class SortedDictIterator(object):
10
+ def __init__(self, sorted_dict, keys):
11
+ self.sorted_dict = sorted_dict
12
+ self.keys = keys
13
+
14
+ def next(self):
15
+ try:
16
+ return self.keys.pop(0)
17
+ except IndexError:
18
+ raise StopIteration
19
+
20
+ class SortedDict(dict):
21
+ def __init__(self, mapping = {}, ignore_case = True, **kwargs):
22
+ """
23
+ WARNING: SortedDict() with ignore_case==True will
24
+ drop entries differing only in capitalisation!
25
+ Eg: SortedDict({'auckland':1, 'Auckland':2}).keys() => ['Auckland']
26
+ With ignore_case==False it's all right
27
+ """
28
+ dict.__init__(self, mapping, **kwargs)
29
+ self.ignore_case = ignore_case
30
+
31
+ def keys(self):
32
+ keys = dict.keys(self)
33
+ if self.ignore_case:
34
+ # Translation map
35
+ xlat_map = BidirMap()
36
+ for key in keys:
37
+ xlat_map[key.lower()] = key
38
+ # Lowercase keys
39
+ lc_keys = xlat_map.keys()
40
+ lc_keys.sort()
41
+ return [xlat_map[k] for k in lc_keys]
42
+ else:
43
+ keys.sort()
44
+ return keys
45
+
46
+ def __iter__(self):
47
+ return SortedDictIterator(self, self.keys())
48
+
49
+
50
+
51
+ if __name__ == "__main__":
52
+ d = { 'AWS' : 1, 'Action' : 2, 'america' : 3, 'Auckland' : 4, 'America' : 5 }
53
+ sd = SortedDict(d)
54
+ print "Wanted: Action, america, Auckland, AWS, [ignore case]"
55
+ print "Got: ",
56
+ for key in sd:
57
+ print "%s," % key,
58
+ print " [used: __iter__()]"
59
+ d = SortedDict(d, ignore_case = False)
60
+ print "Wanted: AWS, Action, Auckland, america, [case sensitive]"
61
+ print "Got: ",
62
+ for key in d.keys():
63
+ print "%s," % key,
64
+ print " [used: keys()]"
65
+
66
+ # vim:et:ts=4:sts=4:ai
Binary file