s3_cmd_bin 0.0.1

data/resources/S3/S3Uri.py

@@ -0,0 +1,223 @@
+## Amazon S3 manager
+## Author: Michal Ludvig <michal@logix.cz>
+##         http://www.logix.cz/michal
+## License: GPL Version 2
+
+import os
+import re
+import sys
+from BidirMap import BidirMap
+from logging import debug
+import S3
+from Utils import unicodise, check_bucket_name_dns_conformity
+import Config
+
+class S3Uri(object):
+    type = None
+    _subclasses = None
+
+    def __new__(self, string):
+        if not self._subclasses:
+            ## Generate a list of all subclasses of S3Uri
+            self._subclasses = []
+            dict = sys.modules[__name__].__dict__
+            for something in dict:
+                if type(dict[something]) is not type(self):
+                    continue
+                if issubclass(dict[something], self) and dict[something] != self:
+                    self._subclasses.append(dict[something])
+        for subclass in self._subclasses:
+            try:
+                instance = object.__new__(subclass)
+                instance.__init__(string)
+                return instance
+            except ValueError, e:
+                continue
+        raise ValueError("%s: not a recognized URI" % string)
+
+    def __str__(self):
+        return self.uri()
+
+    def __unicode__(self):
+        return self.uri()
+
+    def __repr__(self):
+        return "<%s: %s>" % (self.__class__.__name__, self.__unicode__())
+
+    def public_url(self):
+        raise ValueError("This S3 URI does not have Anonymous URL representation")
+
+    def basename(self):
+        return self.__unicode__().split("/")[-1]
+
+class S3UriS3(S3Uri):
+    type = "s3"
+    _re = re.compile("^s3://([^/]+)/?(.*)", re.IGNORECASE)
+    def __init__(self, string):
+        match = self._re.match(string)
+        if not match:
+            raise ValueError("%s: not a S3 URI" % string)
+        groups = match.groups()
+        self._bucket = groups[0]
+        self._object = unicodise(groups[1])
+
+    def bucket(self):
+        return self._bucket
+
+    def object(self):
+        return self._object
+
+    def has_bucket(self):
+        return bool(self._bucket)
+
+    def has_object(self):
+        return bool(self._object)
+
+    def uri(self):
+        return "/".join(["s3:/", self._bucket, self._object])
+
+    def is_dns_compatible(self):
+        return check_bucket_name_dns_conformity(self._bucket)
+
+    def public_url(self):
+        if self.is_dns_compatible():
+            return "http://%s.%s/%s" % (self._bucket, Config.Config().host_base, self._object)
+        else:
+            return "http://%s/%s/%s" % (self._bucket, Config.Config().host_base, self._object)
+
+    def host_name(self):
+        if self.is_dns_compatible():
+            return "%s.s3.amazonaws.com" % (self._bucket)
+        else:
+            return "s3.amazonaws.com"
+
+    @staticmethod
+    def compose_uri(bucket, object = ""):
+        return "s3://%s/%s" % (bucket, object)
+
+    @staticmethod
+    def httpurl_to_s3uri(http_url):
+        m=re.match("(https?://)?([^/]+)/?(.*)", http_url, re.IGNORECASE)
+        hostname, object = m.groups()[1:]
+        hostname = hostname.lower()
+        if hostname == "s3.amazonaws.com":
+            ## old-style url: http://s3.amazonaws.com/bucket/object
+            if object.count("/") == 0:
+                ## no object given
+                bucket = object
+                object = ""
+            else:
+                ## bucket/object
+                bucket, object = object.split("/", 1)
+        elif hostname.endswith(".s3.amazonaws.com"):
+            ## new-style url: http://bucket.s3.amazonaws.com/object
+            bucket = hostname[:-(len(".s3.amazonaws.com"))]
+        else:
+            raise ValueError("Unable to parse URL: %s" % http_url)
+        return S3Uri("s3://%(bucket)s/%(object)s" % {
+            'bucket' : bucket,
+            'object' : object })
+
+class S3UriS3FS(S3Uri):
+    type = "s3fs"
+    _re = re.compile("^s3fs://([^/]*)/?(.*)", re.IGNORECASE)
+    def __init__(self, string):
+        match = self._re.match(string)
+        if not match:
+            raise ValueError("%s: not a S3fs URI" % string)
+        groups = match.groups()
+        self._fsname = groups[0]
+        self._path = unicodise(groups[1]).split("/")
+
+    def fsname(self):
+        return self._fsname
+
+    def path(self):
+        return "/".join(self._path)
+
+    def uri(self):
+        return "/".join(["s3fs:/", self._fsname, self.path()])
+
+class S3UriFile(S3Uri):
+    type = "file"
+    _re = re.compile("^(\w+://)?(.*)")
+    def __init__(self, string):
+        match = self._re.match(string)
+        groups = match.groups()
+        if groups[0] not in (None, "file://"):
+            raise ValueError("%s: not a file:// URI" % string)
+        self._path = unicodise(groups[1]).split("/")
+
+    def path(self):
+        return "/".join(self._path)
+
+    def uri(self):
+        return "/".join(["file:/", self.path()])
+
+    def isdir(self):
+        return os.path.isdir(self.path())
+
+    def dirname(self):
+        return os.path.dirname(self.path())
+
+class S3UriCloudFront(S3Uri):
+    type = "cf"
+    _re = re.compile("^cf://([^/]*)/*(.*)", re.IGNORECASE)
+    def __init__(self, string):
+        match = self._re.match(string)
+        if not match:
+            raise ValueError("%s: not a CloudFront URI" % string)
+        groups = match.groups()
+        self._dist_id = groups[0]
+        self._request_id = groups[1] != "/" and groups[1] or None
+
+    def dist_id(self):
+        return self._dist_id
+
+    def request_id(self):
+        return self._request_id
+
+    def uri(self):
+        uri = "cf://" + self.dist_id()
+        if self.request_id():
+            uri += "/" + self.request_id()
+        return uri
+
+if __name__ == "__main__":
+    uri = S3Uri("s3://bucket/object")
+    print "type()  =", type(uri)
+    print "uri     =", uri
+    print "uri.type=", uri.type
+    print "bucket  =", uri.bucket()
+    print "object  =", uri.object()
+    print
+
+    uri = S3Uri("s3://bucket")
+    print "type()  =", type(uri)
+    print "uri     =", uri
+    print "uri.type=", uri.type
+    print "bucket  =", uri.bucket()
+    print
+
+    uri = S3Uri("s3fs://filesystem1/path/to/remote/file.txt")
+    print "type()  =", type(uri)
+    print "uri     =", uri
+    print "uri.type=", uri.type
+    print "path    =", uri.path()
+    print
+
+    uri = S3Uri("/path/to/local/file.txt")
+    print "type()  =", type(uri)
+    print "uri     =", uri
+    print "uri.type=", uri.type
+    print "path    =", uri.path()
+    print
+
+    uri = S3Uri("cf://1234567890ABCD/")
+    print "type()  =", type(uri)
+    print "uri     =", uri
+    print "uri.type=", uri.type
+    print "dist_id =", uri.dist_id()
+    print
+
+# vim:et:ts=4:sts=4:ai

data/resources/S3/SimpleDB.py

@@ -0,0 +1,178 @@
+## Amazon SimpleDB library
+## Author: Michal Ludvig <michal@logix.cz>
+##         http://www.logix.cz/michal
+## License: GPL Version 2
+
+"""
+Low-level class for working with Amazon SimpleDB
+"""
+
+import time
+import urllib
+import base64
+import hmac
+import sha
+import httplib
+from logging import debug, info, warning, error
+
+from Utils import convertTupleListToDict
+from SortedDict import SortedDict
+from Exceptions import *
+
+class SimpleDB(object):
+    # API Version
+    # See http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/
+    Version = "2007-11-07"
+    SignatureVersion = 1
+
+    def __init__(self, config):
+        self.config = config
+
+    ## ------------------------------------------------
+    ## Methods implementing SimpleDB API
+    ## ------------------------------------------------
+
+    def ListDomains(self, MaxNumberOfDomains = 100):
+        '''
+        Lists all domains associated with our Access Key. Returns
+        domain names up to the limit set by MaxNumberOfDomains.
+        '''
+        parameters = SortedDict()
+        parameters['MaxNumberOfDomains'] = MaxNumberOfDomains
+        return self.send_request("ListDomains", DomainName = None, parameters = parameters)
+
+    def CreateDomain(self, DomainName):
+        return self.send_request("CreateDomain", DomainName = DomainName)
+
+    def DeleteDomain(self, DomainName):
+        return self.send_request("DeleteDomain", DomainName = DomainName)
+
+    def PutAttributes(self, DomainName, ItemName, Attributes):
+        parameters = SortedDict()
+        parameters['ItemName'] = ItemName
+        seq = 0
+        for attrib in Attributes:
+            if type(Attributes[attrib]) == type(list()):
+                for value in Attributes[attrib]:
+                    parameters['Attribute.%d.Name' % seq] = attrib
+                    parameters['Attribute.%d.Value' % seq] = unicode(value)
+                    seq += 1
+            else:
+                parameters['Attribute.%d.Name' % seq] = attrib
+                parameters['Attribute.%d.Value' % seq] = unicode(Attributes[attrib])
+                seq += 1
+        ## TODO:
+        ## - support for Attribute.N.Replace
+        ## - support for multiple values for one attribute
+        return self.send_request("PutAttributes", DomainName = DomainName, parameters = parameters)
+
+    def GetAttributes(self, DomainName, ItemName, Attributes = []):
+        parameters = SortedDict()
+        parameters['ItemName'] = ItemName
+        seq = 0
+        for attrib in Attributes:
+            parameters['AttributeName.%d' % seq] = attrib
+            seq += 1
+        return self.send_request("GetAttributes", DomainName = DomainName, parameters = parameters)
+
+    def DeleteAttributes(self, DomainName, ItemName, Attributes = {}):
+        """
+        Remove specified Attributes from ItemName.
+        Attributes parameter can be either:
+        - not specified, in which case the whole Item is removed
+        - list, e.g. ['Attr1', 'Attr2'] in which case these parameters are removed
+        - dict, e.g. {'Attr' : 'One', 'Attr' : 'Two'} in which case the
+          specified values are removed from multi-value attributes.
+        """
+        parameters = SortedDict()
+        parameters['ItemName'] = ItemName
+        seq = 0
+        for attrib in Attributes:
+            parameters['Attribute.%d.Name' % seq] = attrib
+            if type(Attributes) == type(dict()):
+                parameters['Attribute.%d.Value' % seq] = unicode(Attributes[attrib])
+            seq += 1
+        return self.send_request("DeleteAttributes", DomainName = DomainName, parameters = parameters)
+
+    def Query(self, DomainName, QueryExpression = None, MaxNumberOfItems = None, NextToken = None):
+        parameters = SortedDict()
+        if QueryExpression:
+            parameters['QueryExpression'] = QueryExpression
+        if MaxNumberOfItems:
+            parameters['MaxNumberOfItems'] = MaxNumberOfItems
+        if NextToken:
+            parameters['NextToken'] = NextToken
+        return self.send_request("Query", DomainName = DomainName, parameters = parameters)
+        ## Handle NextToken? Or maybe not - let the upper level do it
+
+    ## ------------------------------------------------
+    ## Low-level methods for handling SimpleDB requests
+    ## ------------------------------------------------
+
+    def send_request(self, *args, **kwargs):
+        request = self.create_request(*args, **kwargs)
+        #debug("Request: %s" % repr(request))
+        conn = self.get_connection()
+        conn.request("GET", self.format_uri(request['uri_params']))
+        http_response = conn.getresponse()
+        response = {}
+        response["status"] = http_response.status
+        response["reason"] = http_response.reason
+        response["headers"] = convertTupleListToDict(http_response.getheaders())
+        response["data"] =  http_response.read()
+        conn.close()
+
+        if response["status"] < 200 or response["status"] > 299:
+            debug("Response: " + str(response))
+            raise S3Error(response)
+
+        return response
+
+    def create_request(self, Action, DomainName, parameters = None):
+        if not parameters:
+            parameters = SortedDict()
+        if len(self.config.access_token) > 0:
+            self.config.refresh_role()
+            parameters['Signature']=self.config.access_token
+        parameters['AWSAccessKeyId'] = self.config.access_key
+        parameters['Version'] = self.Version
+        parameters['SignatureVersion'] = self.SignatureVersion
+        parameters['Action'] = Action
+        parameters['Timestamp'] = time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
+        if DomainName:
+            parameters['DomainName'] = DomainName
+        parameters['Signature'] = self.sign_request(parameters)
+        parameters.keys_return_lowercase = False
+        uri_params = urllib.urlencode(parameters)
+        request = {}
+        request['uri_params'] = uri_params
+        request['parameters'] = parameters
+        return request
+
+    def sign_request(self, parameters):
+        h = ""
+        parameters.keys_sort_lowercase = True
+        parameters.keys_return_lowercase = False
+        for key in parameters:
+            h += "%s%s" % (key, parameters[key])
+        #debug("SignRequest: %s" % h)
+        return base64.encodestring(hmac.new(self.config.secret_key, h, sha).digest()).strip()
+
+    def get_connection(self):
+        if self.config.proxy_host != "":
+            return httplib.HTTPConnection(self.config.proxy_host, self.config.proxy_port)
+        else:
+            if self.config.use_https:
+                return httplib.HTTPSConnection(self.config.simpledb_host)
+            else:
+                return httplib.HTTPConnection(self.config.simpledb_host)
+
+    def format_uri(self, uri_params):
+        if self.config.proxy_host != "":
+            uri = "http://%s/?%s" % (self.config.simpledb_host, uri_params)
+        else:
+            uri = "/?%s" % uri_params
+        #debug('format_uri(): ' + uri)
+        return uri
+
+# vim:et:ts=4:sts=4:ai

data/resources/S3/SortedDict.py

@@ -0,0 +1,66 @@
+## Amazon S3 manager
+## Author: Michal Ludvig <michal@logix.cz>
+##         http://www.logix.cz/michal
+## License: GPL Version 2
+
+from BidirMap import BidirMap
+import Utils
+
+class SortedDictIterator(object):
+    def __init__(self, sorted_dict, keys):
+        self.sorted_dict = sorted_dict
+        self.keys = keys
+
+    def next(self):
+        try:
+            return self.keys.pop(0)
+        except IndexError:
+            raise StopIteration
+
+class SortedDict(dict):
+    def __init__(self, mapping = {}, ignore_case = True, **kwargs):
+        """
+        WARNING: SortedDict() with ignore_case==True will
+                 drop entries differing only in capitalisation!
+                 Eg: SortedDict({'auckland':1, 'Auckland':2}).keys() => ['Auckland']
+                 With ignore_case==False it's all right
+        """
+        dict.__init__(self, mapping, **kwargs)
+        self.ignore_case = ignore_case
+
+    def keys(self):
+        keys = dict.keys(self)
+        if self.ignore_case:
+            # Translation map
+            xlat_map = BidirMap()
+            for key in keys:
+                xlat_map[key.lower()] = key
+            # Lowercase keys
+            lc_keys = xlat_map.keys()
+            lc_keys.sort()
+            return [xlat_map[k] for k in lc_keys]
+        else:
+            keys.sort()
+            return keys
+
+    def __iter__(self):
+        return SortedDictIterator(self, self.keys())
+
+
+
+if __name__ == "__main__":
+    d = { 'AWS' : 1, 'Action' : 2, 'america' : 3, 'Auckland' : 4, 'America' : 5 }
+    sd = SortedDict(d)
+    print "Wanted: Action, america, Auckland, AWS,    [ignore case]"
+    print "Got:   ",
+    for key in sd:
+        print "%s," % key,
+    print "   [used: __iter__()]"
+    d = SortedDict(d, ignore_case = False)
+    print "Wanted: AWS, Action, Auckland, america,    [case sensitive]"
+    print "Got:   ",
+    for key in d.keys():
+        print "%s," % key,
+    print "   [used: keys()]"
+
+# vim:et:ts=4:sts=4:ai