s3_cmd_bin 0.0.1

data/resources/S3/Config.py

@@ -0,0 +1,294 @@
+## Amazon S3 manager
+## Author: Michal Ludvig <michal@logix.cz>
+##         http://www.logix.cz/michal
+## License: GPL Version 2
+
+import logging
+from logging import debug, info, warning, error
+import re
+import os
+import sys
+import Progress
+from SortedDict import SortedDict
+import httplib
+import json
+
+class Config(object):
+    _instance = None
+    _parsed_files = []
+    _doc = {}
+    access_key = ""
+    secret_key = ""
+    access_token = ""
+    host_base = "s3.amazonaws.com"
+    host_bucket = "%(bucket)s.s3.amazonaws.com"
+    simpledb_host = "sdb.amazonaws.com"
+    cloudfront_host = "cloudfront.amazonaws.com"
+    verbosity = logging.WARNING
+    progress_meter = True
+    progress_class = Progress.ProgressCR
+    send_chunk = 4096
+    recv_chunk = 4096
+    list_md5 = False
+    human_readable_sizes = False
+    extra_headers = SortedDict(ignore_case = True)
+    force = False
+    enable = None
+    get_continue = False
+    skip_existing = False
+    recursive = False
+    acl_public = None
+    acl_grants = []
+    acl_revokes = []
+    proxy_host = ""
+    proxy_port = 3128
+    encrypt = False
+    dry_run = False
+    add_encoding_exts = ""
+    preserve_attrs = True
+    preserve_attrs_list = [
+        'uname',    # Verbose owner Name (e.g. 'root')
+        'uid',      # Numeric user ID (e.g. 0)
+        'gname',    # Group name (e.g. 'users')
+        'gid',      # Numeric group ID (e.g. 100)
+        'atime',    # Last access timestamp
+        'mtime',    # Modification timestamp
+        'ctime',    # Creation timestamp
+        'mode',     # File mode (e.g. rwxr-xr-x = 755)
+        'md5',      # File MD5 (if known)
+        #'acl',     # Full ACL (not yet supported)
+    ]
+    delete_removed = False
+    delete_after = False
+    delete_after_fetch = False
+    _doc['delete_removed'] = "[sync] Remove remote S3 objects when local file has been deleted"
+    delay_updates = False
+    gpg_passphrase = ""
+    gpg_command = ""
+    gpg_encrypt = "%(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s"
+    gpg_decrypt = "%(gpg_command)s -d --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s"
+    use_https = False
+    bucket_location = "US"
+    default_mime_type = "binary/octet-stream"
+    guess_mime_type = True
+    mime_type = ""
+    enable_multipart = True
+    multipart_chunk_size_mb = 15    # MB
+    # List of checks to be performed for 'sync'
+    sync_checks = ['size', 'md5']   # 'weak-timestamp'
+    # List of compiled REGEXPs
+    exclude = []
+    include = []
+    # Dict mapping compiled REGEXPs back to their textual form
+    debug_exclude = {}
+    debug_include = {}
+    encoding = "utf-8"
+    urlencoding_mode = "normal"
+    log_target_prefix = ""
+    reduced_redundancy = False
+    follow_symlinks = False
+    socket_timeout = 300
+    invalidate_on_cf = False
+    # joseprio: new flags for default index invalidation
+    invalidate_default_index_on_cf = False
+    invalidate_default_index_root_on_cf = True
+    website_index = "index.html"
+    website_error = ""
+    website_endpoint = "http://%(bucket)s.s3-website-%(location)s.amazonaws.com/"
+    additional_destinations = []
+    cache_file = ""
+    add_headers = ""
+
+    ## Creating a singleton
+    def __new__(self, configfile = None):
+        if self._instance is None:
+            self._instance = object.__new__(self)
+        return self._instance
+
+    def __init__(self, configfile = None):
+        if configfile:
+            try:
+                self.read_config_file(configfile)
+            except IOError, e:
+                if 'AWS_CREDENTIAL_FILE' in os.environ:
+                    self.env_config()
+            if len(self.access_key)==0:
+                self.role_config()  
+
+    def role_config(self):
+        conn = httplib.HTTPConnection(host='169.254.169.254',timeout=0.1)
+        try:
+            conn.request('GET', "/latest/meta-data/iam/security-credentials/")
+            resp = conn.getresponse()
+            files = resp.read()
+            if resp.status == 200 and len(files)>1:               
+                conn.request('GET', "/latest/meta-data/iam/security-credentials/%s"%files)
+                resp=conn.getresponse()
+                if resp.status == 200:
+                    creds=json.load(resp)
+                    Config().update_option('access_key', creds['AccessKeyId'].encode('ascii'))
+                    Config().update_option('secret_key', creds['SecretAccessKey'].encode('ascii'))
+                    Config().update_option('access_token', creds['Token'].encode('ascii'))
+                else:
+                    raise IOError
+            else:
+                raise IOError
+        except:
+            raise
+
+    def role_refresh(self):
+        try:
+            self.role_config()
+        except:
+            warning("Could not refresh role")
+
+    def env_config(self):
+        cred_content = ""
+        try:
+            cred_file = open(os.environ['AWS_CREDENTIAL_FILE'],'r')
+            cred_content = cred_file.read()
+        except IOError, e:
+            debug("Error %d accessing credentials file %s" % (e.errno,os.environ['AWS_CREDENTIAL_FILE']))
+        r_data = re.compile("^\s*(?P<orig_key>\w+)\s*=\s*(?P<value>.*)")
+        r_quotes = re.compile("^\"(.*)\"\s*$")
+        if len(cred_content)>0:
+            for line in cred_content.splitlines():
+                is_data = r_data.match(line)
+                is_data = r_data.match(line)
+                if is_data:
+                    data = is_data.groupdict()
+                    if r_quotes.match(data["value"]):
+                        data["value"] = data["value"][1:-1]
+                    if data["orig_key"]=="AWSAccessKeyId":
+                        data["key"] = "access_key"
+                    elif data["orig_key"]=="AWSSecretKey":
+                        data["key"] = "secret_key"
+                    else:
+                        del data["key"]                    
+                    if "key" in data:
+                        Config().update_option(data["key"], data["value"])
+                        if data["key"] in ("access_key", "secret_key", "gpg_passphrase"):
+                            print_value = (data["value"][:2]+"...%d_chars..."+data["value"][-1:]) % (len(data["value"]) - 3)
+                        else:
+                            print_value = data["value"]
+                        debug("env_Config: %s->%s" % (data["key"], print_value))
+                
+        
+
+    def option_list(self):
+        retval = []
+        for option in dir(self):
+            ## Skip attributes that start with underscore or are not string, int or bool
+            option_type = type(getattr(Config, option))
+            if option.startswith("_") or \
+               not (option_type in (
+                    type("string"), # str
+                        type(42),   # int
+                    type(True))):   # bool
+                continue
+            retval.append(option)
+        return retval
+
+    def read_config_file(self, configfile):
+        cp = ConfigParser(configfile)
+        for option in self.option_list():
+            self.update_option(option, cp.get(option))
+
+        if cp.get('add_headers'):
+            for option in cp.get('add_headers').split(","):
+                (key, value) = option.split(':')
+                self.extra_headers[key.replace('_', '-').strip()] = value.strip()
+
+        self._parsed_files.append(configfile)
+
+    def dump_config(self, stream):
+        ConfigDumper(stream).dump("default", self)
+
+    def update_option(self, option, value):
+        if value is None:
+            return
+        #### Handle environment reference
+        if str(value).startswith("$"):
+            return self.update_option(option, os.getenv(str(value)[1:]))
+        #### Special treatment of some options
+        ## verbosity must be known to "logging" module
+        if option == "verbosity":
+            try:
+                setattr(Config, "verbosity", logging._levelNames[value])
+            except KeyError:
+                error("Config: verbosity level '%s' is not valid" % value)
+        ## allow yes/no, true/false, on/off and 1/0 for boolean options
+        elif type(getattr(Config, option)) is type(True):   # bool
+            if str(value).lower() in ("true", "yes", "on", "1"):
+                setattr(Config, option, True)
+            elif str(value).lower() in ("false", "no", "off", "0"):
+                setattr(Config, option, False)
+            else:
+                error("Config: value of option '%s' must be Yes or No, not '%s'" % (option, value))
+        elif type(getattr(Config, option)) is type(42):     # int
+            try:
+                setattr(Config, option, int(value))
+            except ValueError, e:
+                error("Config: value of option '%s' must be an integer, not '%s'" % (option, value))
+        else:                           # string
+            setattr(Config, option, value)
+
+class ConfigParser(object):
+    def __init__(self, file, sections = []):
+        self.cfg = {}
+        self.parse_file(file, sections)
+
+    def parse_file(self, file, sections = []):
+        debug("ConfigParser: Reading file '%s'" % file)
+        if type(sections) != type([]):
+            sections = [sections]
+        in_our_section = True
+        f = open(file, "r")
+        r_comment = re.compile("^\s*#.*")
+        r_empty = re.compile("^\s*$")
+        r_section = re.compile("^\[([^\]]+)\]")
+        r_data = re.compile("^\s*(?P<key>\w+)\s*=\s*(?P<value>.*)")
+        r_quotes = re.compile("^\"(.*)\"\s*$")
+        for line in f:
+            if r_comment.match(line) or r_empty.match(line):
+                continue
+            is_section = r_section.match(line)
+            if is_section:
+                section = is_section.groups()[0]
+                in_our_section = (section in sections) or (len(sections) == 0)
+                continue
+            is_data = r_data.match(line)
+            if is_data and in_our_section:
+                data = is_data.groupdict()
+                if r_quotes.match(data["value"]):
+                    data["value"] = data["value"][1:-1]
+                self.__setitem__(data["key"], data["value"])
+                if data["key"] in ("access_key", "secret_key", "gpg_passphrase"):
+                    print_value = (data["value"][:2]+"...%d_chars..."+data["value"][-1:]) % (len(data["value"]) - 3)
+                else:
+                    print_value = data["value"]
+                debug("ConfigParser: %s->%s" % (data["key"], print_value))
+                continue
+            warning("Ignoring invalid line in '%s': %s" % (file, line))
+
+    def __getitem__(self, name):
+        return self.cfg[name]
+
+    def __setitem__(self, name, value):
+        self.cfg[name] = value
+
+    def get(self, name, default = None):
+        if self.cfg.has_key(name):
+            return self.cfg[name]
+        return default
+
+class ConfigDumper(object):
+    def __init__(self, stream):
+        self.stream = stream
+
+    def dump(self, section, config):
+        self.stream.write("[%s]\n" % section)
+        for option in config.option_list():
+            self.stream.write("%s = %s\n" % (option, getattr(config, option)))
+
+# vim:et:ts=4:sts=4:ai

data/resources/S3/ConnMan.py

@@ -0,0 +1,71 @@
+import httplib
+from urlparse import urlparse
+from threading import Semaphore
+from logging import debug, info, warning, error
+
+from Config import Config
+from Exceptions import ParameterError
+
+__all__ = [ "ConnMan" ]
+
+class http_connection(object):
+    def __init__(self, id, hostname, ssl, cfg):
+        self.hostname = hostname
+        self.ssl = ssl
+        self.id = id
+        self.counter = 0
+        if cfg.proxy_host != "":
+            self.c = httplib.HTTPConnection(cfg.proxy_host, cfg.proxy_port)
+        elif not ssl:
+            self.c = httplib.HTTPConnection(hostname)
+        else:
+            self.c = httplib.HTTPSConnection(hostname)
+
+class ConnMan(object):
+    conn_pool_sem = Semaphore()
+    conn_pool = {}
+    conn_max_counter = 800    ## AWS closes connection after some ~90 requests
+
+    @staticmethod
+    def get(hostname, ssl = None):
+        cfg = Config()
+        if ssl == None:
+            ssl = cfg.use_https
+        conn = None
+        if cfg.proxy_host != "":
+            if ssl:
+                raise ParameterError("use_ssl=True can't be used with proxy")
+            conn_id = "proxy://%s:%s" % (cfg.proxy_host, cfg.proxy_port)
+        else:
+            conn_id = "http%s://%s" % (ssl and "s" or "", hostname)
+        ConnMan.conn_pool_sem.acquire()
+        if not ConnMan.conn_pool.has_key(conn_id):
+            ConnMan.conn_pool[conn_id] = []
+        if len(ConnMan.conn_pool[conn_id]):
+            conn = ConnMan.conn_pool[conn_id].pop()
+            debug("ConnMan.get(): re-using connection: %s#%d" % (conn.id, conn.counter))
+        ConnMan.conn_pool_sem.release()
+        if not conn:
+            debug("ConnMan.get(): creating new connection: %s" % conn_id)
+            conn = http_connection(conn_id, hostname, ssl, cfg)
+            conn.c.connect()
+        conn.counter += 1
+        return conn
+
+    @staticmethod
+    def put(conn):
+        if conn.id.startswith("proxy://"):
+            conn.c.close()
+            debug("ConnMan.put(): closing proxy connection (keep-alive not yet supported)")
+            return
+
+        if conn.counter >= ConnMan.conn_max_counter:
+            conn.c.close()
+            debug("ConnMan.put(): closing over-used connection")
+            return
+
+        ConnMan.conn_pool_sem.acquire()
+        ConnMan.conn_pool[conn.id].append(conn)
+        ConnMan.conn_pool_sem.release()
+        debug("ConnMan.put(): connection put back to pool (%s#%d)" % (conn.id, conn.counter))
+

data/resources/S3/Exceptions.py

@@ -0,0 +1,88 @@
+## Amazon S3 manager - Exceptions library
+## Author: Michal Ludvig <michal@logix.cz>
+##         http://www.logix.cz/michal
+## License: GPL Version 2
+
+from Utils import getTreeFromXml, unicodise, deunicodise
+from logging import debug, info, warning, error
+
+try:
+    import xml.etree.ElementTree as ET
+except ImportError:
+    import elementtree.ElementTree as ET
+
+class S3Exception(Exception):
+    def __init__(self, message = ""):
+        self.message = unicodise(message)
+
+    def __str__(self):
+        ## Call unicode(self) instead of self.message because
+        ## __unicode__() method could be overriden in subclasses!
+        return deunicodise(unicode(self))
+
+    def __unicode__(self):
+        return self.message
+
+    ## (Base)Exception.message has been deprecated in Python 2.6
+    def _get_message(self):
+        return self._message
+    def _set_message(self, message):
+        self._message = message
+    message = property(_get_message, _set_message)
+
+
+class S3Error (S3Exception):
+    def __init__(self, response):
+        self.status = response["status"]
+        self.reason = response["reason"]
+        self.info = {
+            "Code" : "",
+            "Message" : "",
+            "Resource" : ""
+        }
+        debug("S3Error: %s (%s)" % (self.status, self.reason))
+        if response.has_key("headers"):
+            for header in response["headers"]:
+                debug("HttpHeader: %s: %s" % (header, response["headers"][header]))
+        if response.has_key("data") and response["data"]:
+            tree = getTreeFromXml(response["data"])
+            error_node = tree
+            if not error_node.tag == "Error":
+                error_node = tree.find(".//Error")
+            for child in error_node.getchildren():
+                if child.text != "":
+                    debug("ErrorXML: " + child.tag + ": " + repr(child.text))
+                    self.info[child.tag] = child.text
+        self.code = self.info["Code"]
+        self.message = self.info["Message"]
+        self.resource = self.info["Resource"]
+
+    def __unicode__(self):
+        retval = u"%d " % (self.status)
+        retval += (u"(%s)" % (self.info.has_key("Code") and self.info["Code"] or self.reason))
+        if self.info.has_key("Message"):
+            retval += (u": %s" % self.info["Message"])
+        return retval
+
+class CloudFrontError(S3Error):
+    pass
+
+class S3UploadError(S3Exception):
+    pass
+
+class S3DownloadError(S3Exception):
+    pass
+
+class S3RequestError(S3Exception):
+    pass
+
+class S3ResponseError(S3Exception):
+    pass
+
+class InvalidFileError(S3Exception):
+    pass
+
+class ParameterError(S3Exception):
+    pass
+
+# vim:et:ts=4:sts=4:ai

data/resources/S3/FileDict.py

@@ -0,0 +1,53 @@
+## Amazon S3 manager
+## Author: Michal Ludvig <michal@logix.cz>
+##         http://www.logix.cz/michal
+## License: GPL Version 2
+
+from SortedDict import SortedDict
+import Utils
+
+class FileDict(SortedDict):
+    def __init__(self, mapping = {}, ignore_case = True, **kwargs):
+        SortedDict.__init__(self, mapping = mapping, ignore_case = ignore_case, **kwargs)
+        self.hardlinks = dict() # { dev: { inode : {'md5':, 'relative_files':}}}
+        self.by_md5 = dict() # {md5: set(relative_files)}
+
+    def record_md5(self, relative_file, md5):
+        if md5 not in self.by_md5:
+            self.by_md5[md5] = set()
+        self.by_md5[md5].add(relative_file)
+
+    def find_md5_one(self, md5):
+        try:
+            return list(self.by_md5.get(md5, set()))[0]
+        except:
+            return None
+
+    def get_md5(self, relative_file):
+        """returns md5 if it can, or raises IOError if file is unreadable"""
+        md5 = None
+        if 'md5' in self[relative_file]:
+            return self[relative_file]['md5']
+        md5 = self.get_hardlink_md5(relative_file)
+        if md5 is None:
+            md5 = Utils.hash_file_md5(self[relative_file]['full_name'])
+        self.record_md5(relative_file, md5)
+        self[relative_file]['md5'] = md5
+        return md5
+
+    def record_hardlink(self, relative_file, dev, inode, md5):
+        if dev not in self.hardlinks:
+            self.hardlinks[dev] = dict()
+        if inode not in self.hardlinks[dev]:
+            self.hardlinks[dev][inode] = dict(md5=md5, relative_files=set())
+        self.hardlinks[dev][inode]['relative_files'].add(relative_file)
+
+    def get_hardlink_md5(self, relative_file):
+        md5 = None
+        dev = self[relative_file]['dev']
+        inode = self[relative_file]['inode']
+        try:
+            md5 = self.hardlinks[dev][inode]['md5']
+        except:
+            pass
+        return md5