s3_cmd_bin 0.0.1 → 0.0.2

data/lib/s3_cmd_bin/version.rb

@@ -1,3 +1,3 @@
 module S3CmdBin
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/resources/MANIFEST.in

@@ -1,2 +1,3 @@
 include INSTALL README NEWS
 include s3cmd.1
+include S3/*.py

data/resources/NEWS

@@ -1,43 +1,4 @@
-s3cmd 1.5.0-alpha3   -   2013-03-11
-==================
-* Persistent HTTP/HTTPS connections for massive speedup (Michal Ludvig)
-* New switch --quiet for suppressing all output (Siddarth Prakash)
-* Honour "umask" on file downloads (Jason Dalton)
-* Various bugfixes from many contributors
-
-s3cmd 1.5.0-alpha2   -   2013-03-04
-==================
-* IAM roles support (David Kohen, Eric Dowd)
-* Manage bucket policies (Kota Uenishi)
-* Various bugfixes from many contributors
-
-s3cmd 1.5.0-alpha1   -   2013-02-19
-==================
-* Server-side copy for hardlinks/softlinks to improve performance
-  (Matt Domsch)
-* New [signurl] command (Craig Ringer)
-* Improved symlink-loop detection (Michal Ludvig)
-* Add --delete-after option for sync (Matt Domsch)
-* Handle empty return bodies when processing S3 errors.
-  (Kelly McLaughlin)
-* Upload from STDIN (Eric Connell)
-* Updated bucket locations (Stefhen Hovland)
-* Support custom HTTP headers (Brendan O'Connor, Karl Matthias)
-* Improved MIME support (Karsten Sperling, Christopher Noyes)
-* Added support for --acl-grant/--acl-revoke to 'sync' command
-  (Michael Tyson)
-* CloudFront: Support default index and default root invalidation
-  (Josep del Rio)
-* Command line options for access/secret keys (Matt Sweeney)
-* Support [setpolicy] for setting bucket policies (Joe Fiorini)
-* Respect the $TZ environment variable (James Brown)
-* Reduce memory consumption for [s3cmd du] (Charlie Schluting)
-* Rate limit progress updates (Steven Noonan)
-* Download from S3 to a temp file first (Sumit Kumar)
-* Reuse a single connection when doing a bucket list (Kelly McLaughlin)
-* Delete empty files if object_get() failed (Oren Held)
-
-s3cmd 1.1.0   -   (never released)
+s3cmd 1.1.0   -   ???
 ===========
 * MultiPart upload enabled for both [put] and [sync]. Default chunk
   size is 15MB.

data/resources/S3/CloudFront.py

@@ -133,7 +133,7 @@ class DistributionConfig(object):
     ##  </Logging>
     ## </DistributionConfig>
 
-    EMPTY_CONFIG = "<DistributionConfig><S3Origin><DNSName/></S3Origin><CallerReference/><Enabled>true</Enabled></DistributionConfig>"
+    EMPTY_CONFIG = "<DistributionConfig><Origin/><CallerReference/><Enabled>true</Enabled></DistributionConfig>"
     xmlns = "http://cloudfront.amazonaws.com/doc/%(api_ver)s/" % { 'api_ver' : cloudfront_api_version }
     def __init__(self, xml = None, tree = None):
         if xml is None:
@@ -174,8 +174,7 @@ class DistributionConfig(object):
         tree.attrib['xmlns'] = DistributionConfig.xmlns
 
         ## Retain the order of the following calls!
-        s3org = appendXmlTextNode("S3Origin", '', tree)
-        appendXmlTextNode("DNSName", self.info['S3Origin']['DNSName'], s3org)
+        appendXmlTextNode("Origin", self.info['Origin'], tree)
         appendXmlTextNode("CallerReference", self.info['CallerReference'], tree)
         for cname in self.info['CNAME']:
             appendXmlTextNode("CNAME", cname.lower(), tree)
@@ -282,7 +281,7 @@ class InvalidationBatch(object):
         tree = ET.Element("InvalidationBatch")
 
         for path in self.paths:
-            if len(path) < 1 or path[0] != "/":
+            if path[0] != "/":
                 path = "/" + path
             appendXmlTextNode("Path", path, tree)
         appendXmlTextNode("CallerReference", self.reference, tree)
@@ -323,7 +322,7 @@ class CloudFront(object):
     def CreateDistribution(self, uri, cnames_add = [], comment = None, logging = None, default_root_object = None):
         dist_config = DistributionConfig()
         dist_config.info['Enabled'] = True
-        dist_config.info['S3Origin']['DNSName'] = uri.host_name()
+        dist_config.info['Origin'] = uri.host_name()
         dist_config.info['CallerReference'] = str(uri)
         dist_config.info['DefaultRootObject'] = default_root_object
         if comment == None:
@@ -424,23 +423,7 @@ class CloudFront(object):
                                      body = request_body, headers = headers)
         return response
 
-    def InvalidateObjects(self, uri, paths, default_index_file, invalidate_default_index_on_cf, invalidate_default_index_root_on_cf):
-        # joseprio: if the user doesn't want to invalidate the default index
-        # path, or if the user wants to invalidate the root of the default
-        # index, we need to process those paths
-        if default_index_file is not None and (not invalidate_default_index_on_cf or invalidate_default_index_root_on_cf):
-            new_paths = []
-            default_index_suffix = '/' + default_index_file
-            for path in paths:
-                if path.endswith(default_index_suffix) or path ==  default_index_file:
-                    if invalidate_default_index_on_cf:
-                        new_paths.append(path)
-                    if invalidate_default_index_root_on_cf:
-                        new_paths.append(path[:-len(default_index_file)])
-                else:
-                    new_paths.append(path)
-            paths = new_paths
-
+    def InvalidateObjects(self, uri, paths):
         # uri could be either cf:// or s3:// uri
         cfuri = self.get_dist_name_for_bucket(uri)
         if len(paths) > 999:
@@ -534,10 +517,6 @@ class CloudFront(object):
         if not headers.has_key("x-amz-date"):
             headers["x-amz-date"] = time.strftime("%a, %d %b %Y %H:%M:%S +0000", time.gmtime())
 
-        if len(self.config.access_token)>0:
-            self.config.refresh_role()
-            headers['x-amz-security-token']=self.config.access_token
-
         signature = self.sign_request(headers)
         headers["Authorization"] = "AWS "+self.config.access_key+":"+signature
 
@@ -576,16 +555,8 @@ class CloudFront(object):
             for d in response['dist_list'].dist_summs:
                 if d.info.has_key("S3Origin"):
                     CloudFront.dist_list[getBucketFromHostname(d.info['S3Origin']['DNSName'])[0]] = d.uri()
-                elif d.info.has_key("CustomOrigin"):
-                    # Aral: This used to skip over distributions with CustomOrigin, however, we mustn't
-                    #       do this since S3 buckets that are set up as websites use custom origins.
-                    #       Thankfully, the custom origin URLs they use start with the URL of the
-                    #       S3 bucket. Here, we make use this naming convention to support this use case.
-                    distListIndex = getBucketFromHostname(d.info['CustomOrigin']['DNSName'])[0];
-                    distListIndex = distListIndex[:len(uri.bucket())]
-                    CloudFront.dist_list[distListIndex] = d.uri()
                 else:
-                    # Aral: I'm not sure when this condition will be reached, but keeping it in there.
+                    # Skip over distributions with CustomOrigin
                     continue
             debug("dist_list: %s" % CloudFront.dist_list)
         try:
@@ -692,7 +663,7 @@ class Cmd(object):
             d = response['distribution']
             dc = d.info['DistributionConfig']
             output("Distribution created:")
-            pretty_output("Origin", S3UriS3.httpurl_to_s3uri(dc.info['S3Origin']['DNSName']))
+            pretty_output("Origin", S3UriS3.httpurl_to_s3uri(dc.info['Origin']))
             pretty_output("DistId", d.uri())
             pretty_output("DomainName", d.info['DomainName'])
             pretty_output("CNAMEs", ", ".join(dc.info['CNAME']))
@@ -734,7 +705,7 @@ class Cmd(object):
         response = cf.GetDistInfo(cfuri)
         d = response['distribution']
         dc = d.info['DistributionConfig']
-        pretty_output("Origin", S3UriS3.httpurl_to_s3uri(dc.info['S3Origin']['DNSName']))
+        pretty_output("Origin", S3UriS3.httpurl_to_s3uri(dc.info['Origin']))
         pretty_output("DistId", d.uri())
         pretty_output("DomainName", d.info['DomainName'])
         pretty_output("Status", d.info['Status'])

data/resources/S3/Config.py

@@ -7,11 +7,8 @@ import logging
 from logging import debug, info, warning, error
 import re
 import os
-import sys
 import Progress
 from SortedDict import SortedDict
-import httplib
-import json
 
 class Config(object):
     _instance = None
@@ -19,7 +16,6 @@ class Config(object):
     _doc = {}
     access_key = ""
     secret_key = ""
-    access_token = ""
     host_base = "s3.amazonaws.com"
     host_bucket = "%(bucket)s.s3.amazonaws.com"
     simpledb_host = "sdb.amazonaws.com"
@@ -44,7 +40,6 @@ class Config(object):
     proxy_port = 3128
     encrypt = False
     dry_run = False
-    add_encoding_exts = ""
     preserve_attrs = True
     preserve_attrs_list = [
         'uname',    # Verbose owner Name (e.g. 'root')
@@ -55,14 +50,10 @@ class Config(object):
         'mtime',    # Modification timestamp
         'ctime',    # Creation timestamp
         'mode',     # File mode (e.g. rwxr-xr-x = 755)
-        'md5',      # File MD5 (if known)
         #'acl',     # Full ACL (not yet supported)
     ]
     delete_removed = False
-    delete_after = False
-    delete_after_fetch = False
     _doc['delete_removed'] = "[sync] Remove remote S3 objects when local file has been deleted"
-    delay_updates = False
     gpg_passphrase = ""
     gpg_command = ""
     gpg_encrypt = "%(gpg_command)s -c --verbose --no-use-agent --batch --yes --passphrase-fd %(passphrase_fd)s -o %(output_file)s %(input_file)s"
@@ -89,15 +80,9 @@ class Config(object):
     follow_symlinks = False
     socket_timeout = 300
     invalidate_on_cf = False
-    # joseprio: new flags for default index invalidation
-    invalidate_default_index_on_cf = False
-    invalidate_default_index_root_on_cf = True
     website_index = "index.html"
     website_error = ""
     website_endpoint = "http://%(bucket)s.s3-website-%(location)s.amazonaws.com/"
-    additional_destinations = []
-    cache_file = ""
-    add_headers = ""
 
     ## Creating a singleton
     def __new__(self, configfile = None):
@@ -107,73 +92,7 @@ class Config(object):
 
     def __init__(self, configfile = None):
         if configfile:
-            try:
-                self.read_config_file(configfile)
-            except IOError, e:
-                if 'AWS_CREDENTIAL_FILE' in os.environ:
-                    self.env_config()
-            if len(self.access_key)==0:
-                self.role_config()  
-
-    def role_config(self):
-        conn = httplib.HTTPConnection(host='169.254.169.254',timeout=0.1)
-        try:
-            conn.request('GET', "/latest/meta-data/iam/security-credentials/")
-            resp = conn.getresponse()
-            files = resp.read()
-            if resp.status == 200 and len(files)>1:               
-                conn.request('GET', "/latest/meta-data/iam/security-credentials/%s"%files)
-                resp=conn.getresponse()
-                if resp.status == 200:
-                    creds=json.load(resp)
-                    Config().update_option('access_key', creds['AccessKeyId'].encode('ascii'))
-                    Config().update_option('secret_key', creds['SecretAccessKey'].encode('ascii'))
-                    Config().update_option('access_token', creds['Token'].encode('ascii'))
-                else:
-                    raise IOError
-            else:
-                raise IOError
-        except:
-            raise
-
-    def role_refresh(self):
-        try:
-            self.role_config()
-        except:
-            warning("Could not refresh role")
-
-    def env_config(self):
-        cred_content = ""
-        try:
-            cred_file = open(os.environ['AWS_CREDENTIAL_FILE'],'r')
-            cred_content = cred_file.read()
-        except IOError, e:
-            debug("Error %d accessing credentials file %s" % (e.errno,os.environ['AWS_CREDENTIAL_FILE']))
-        r_data = re.compile("^\s*(?P<orig_key>\w+)\s*=\s*(?P<value>.*)")
-        r_quotes = re.compile("^\"(.*)\"\s*$")
-        if len(cred_content)>0:
-            for line in cred_content.splitlines():
-                is_data = r_data.match(line)
-                is_data = r_data.match(line)
-                if is_data:
-                    data = is_data.groupdict()
-                    if r_quotes.match(data["value"]):
-                        data["value"] = data["value"][1:-1]
-                    if data["orig_key"]=="AWSAccessKeyId":
-                        data["key"] = "access_key"
-                    elif data["orig_key"]=="AWSSecretKey":
-                        data["key"] = "secret_key"
-                    else:
-                        del data["key"]                    
-                    if "key" in data:
-                        Config().update_option(data["key"], data["value"])
-                        if data["key"] in ("access_key", "secret_key", "gpg_passphrase"):
-                            print_value = (data["value"][:2]+"...%d_chars..."+data["value"][-1:]) % (len(data["value"]) - 3)
-                        else:
-                            print_value = data["value"]
-                        debug("env_Config: %s->%s" % (data["key"], print_value))
-                
-        
+            self.read_config_file(configfile)
 
     def option_list(self):
         retval = []
@@ -193,12 +112,6 @@ class Config(object):
         cp = ConfigParser(configfile)
         for option in self.option_list():
             self.update_option(option, cp.get(option))
-
-        if cp.get('add_headers'):
-            for option in cp.get('add_headers').split(","):
-                (key, value) = option.split(':')
-                self.extra_headers[key.replace('_', '-').strip()] = value.strip()
-
         self._parsed_files.append(configfile)
 
     def dump_config(self, stream):

data/resources/S3/Exceptions.py

@@ -44,7 +44,7 @@ class S3Error (S3Exception):
         if response.has_key("headers"):
             for header in response["headers"]:
                 debug("HttpHeader: %s: %s" % (header, response["headers"][header]))
-        if response.has_key("data") and response["data"]:
+        if response.has_key("data"):
             tree = getTreeFromXml(response["data"])
             error_node = tree
             if not error_node.tag == "Error":

data/resources/S3/FileLists.py

@@ -6,59 +6,56 @@
 from S3 import S3
 from Config import Config
 from S3Uri import S3Uri
-from FileDict import FileDict
+from SortedDict import SortedDict
 from Utils import *
 from Exceptions import ParameterError
-from HashCache import HashCache
 
 from logging import debug, info, warning, error
 
 import os
 import glob
-import copy
 
-__all__ = ["fetch_local_list", "fetch_remote_list", "compare_filelists", "filter_exclude_include", "parse_attrs_header"]
+__all__ = ["fetch_local_list", "fetch_remote_list", "compare_filelists", "filter_exclude_include"]
 
 def _fswalk_follow_symlinks(path):
-    '''
-    Walk filesystem, following symbolic links (but without recursion), on python2.4 and later
-
-    If a symlink directory loop is detected, emit a warning and skip.
-    E.g.: dir1/dir2/sym-dir -> ../dir2
-    '''
-    assert os.path.isdir(path) # only designed for directory argument
-    walkdirs = set([path])
-    for dirpath, dirnames, filenames in os.walk(path):
-        handle_exclude_include_walk(dirpath, dirnames, [])
-        real_dirpath = os.path.realpath(dirpath)
-        for dirname in dirnames:
-            current = os.path.join(dirpath, dirname)
-            real_current = os.path.realpath(current)
-            if os.path.islink(current):
-                if (real_dirpath == real_current or
-                    real_dirpath.startswith(real_current + os.path.sep)):
-                    warning("Skipping recursively symlinked directory %s" % dirname)
-                else:
-                    walkdirs.add(current)
-    for walkdir in walkdirs:
-        for dirpath, dirnames, filenames in os.walk(walkdir):
-            handle_exclude_include_walk(dirpath, dirnames, [])
-            yield (dirpath, dirnames, filenames)
-
-def _fswalk_no_symlinks(path):
-    '''
-    Directory tree generator
-
-    path (str) is the root of the directory tree to walk
-    '''
-    for dirpath, dirnames, filenames in os.walk(path):
-        handle_exclude_include_walk(dirpath, dirnames, filenames)
-        yield (dirpath, dirnames, filenames)
+        '''
+        Walk filesystem, following symbolic links (but without recursion), on python2.4 and later
+
+        If a recursive directory link is detected, emit a warning and skip.
+        '''
+        assert os.path.isdir(path) # only designed for directory argument
+        walkdirs = set([path])
+        targets = set()
+        for dirpath, dirnames, filenames in os.walk(path):
+                for dirname in dirnames:
+                        current = os.path.join(dirpath, dirname)
+                        target = os.path.realpath(current)
+                        if os.path.islink(current):
+                                if target in targets:
+                                        warning("Skipping recursively symlinked directory %s" % dirname)
+                                else:
+                                        walkdirs.add(current)
+                        targets.add(target)
+        for walkdir in walkdirs:
+                for value in os.walk(walkdir):
+                        yield value
+
+def _fswalk(path, follow_symlinks):
+        '''
+        Directory tree generator
+
+        path (str) is the root of the directory tree to walk
+
+        follow_symlinks (bool) indicates whether to descend into symbolically linked directories
+        '''
+        if follow_symlinks:
+                return _fswalk_follow_symlinks(path)
+        return os.walk(path)
 
 def filter_exclude_include(src_list):
     info(u"Applying --exclude/--include")
     cfg = Config()
-    exclude_list = FileDict(ignore_case = False)
+    exclude_list = SortedDict(ignore_case = False)
     for file in src_list.keys():
         debug(u"CHECK: %s" % file)
         excluded = False
@@ -81,90 +78,23 @@ def filter_exclude_include(src_list):
             del(src_list[file])
             continue
         else:
-            debug(u"PASS: %r" % (file))
+            debug(u"PASS: %s" % (file))
     return src_list, exclude_list
 
-def handle_exclude_include_walk(root, dirs, files):
-    cfg = Config()
-    copydirs = copy.copy(dirs)
-    copyfiles = copy.copy(files)
-
-    # exclude dir matches in the current directory
-    # this prevents us from recursing down trees we know we want to ignore
-    for x in copydirs:
-        d = os.path.join(root, x, '')
-        debug(u"CHECK: %r" % d)
-        excluded = False
-        for r in cfg.exclude:
-            if r.search(d):
-                excluded = True
-                debug(u"EXCL-MATCH: '%s'" % (cfg.debug_exclude[r]))
-                break
-        if excluded:
-            ## No need to check for --include if not excluded
-            for r in cfg.include:
-                if r.search(d):
-                    excluded = False
-                    debug(u"INCL-MATCH: '%s'" % (cfg.debug_include[r]))
-                    break
-        if excluded:
-            ## Still excluded - ok, action it
-            debug(u"EXCLUDE: %r" % d)
-            dirs.remove(x)
-            continue
-        else:
-            debug(u"PASS: %r" % (d))
-
-    # exclude file matches in the current directory
-    for x in copyfiles:
-        file = os.path.join(root, x)
-        debug(u"CHECK: %r" % file)
-        excluded = False
-        for r in cfg.exclude:
-            if r.search(file):
-                excluded = True
-                debug(u"EXCL-MATCH: '%s'" % (cfg.debug_exclude[r]))
-                break
-        if excluded:
-            ## No need to check for --include if not excluded
-            for r in cfg.include:
-                if r.search(file):
-                    excluded = False
-                    debug(u"INCL-MATCH: '%s'" % (cfg.debug_include[r]))
-                    break
-        if excluded:
-            ## Still excluded - ok, action it
-            debug(u"EXCLUDE: %s" % file)
-            files.remove(x)
-            continue
-        else:
-            debug(u"PASS: %r" % (file))
-
 def fetch_local_list(args, recursive = None):
-    def _get_filelist_local(loc_list, local_uri, cache):
+    def _get_filelist_local(local_uri):
         info(u"Compiling list of local files...")
-
-        if deunicodise(local_uri.basename()) == "-":
-            loc_list["-"] = {
-                'full_name_unicode' : '-',
-                'full_name' : '-',
-                'size' : -1,
-                'mtime' : -1,
-            }
-            return loc_list, True
         if local_uri.isdir():
             local_base = deunicodise(local_uri.basename())
             local_path = deunicodise(local_uri.path())
-            if cfg.follow_symlinks:
-                filelist = _fswalk_follow_symlinks(local_path)
-            else:
-                filelist = _fswalk_no_symlinks(local_path)
+            filelist = _fswalk(local_path, cfg.follow_symlinks)
             single_file = False
         else:
             local_base = ""
             local_path = deunicodise(local_uri.dirname())
             filelist = [( local_path, [], [deunicodise(local_uri.basename())] )]
             single_file = True
+        loc_list = SortedDict(ignore_case = False)
         for root, dirs, files in filelist:
             rel_root = root.replace(local_path, local_base, 1)
             for f in files:
@@ -188,43 +118,13 @@ def fetch_local_list(args, recursive = None):
                     'full_name' : full_name,
                     'size' : sr.st_size,
                     'mtime' : sr.st_mtime,
-                    'dev'   : sr.st_dev,
-                    'inode' : sr.st_ino,
-                    'uid' : sr.st_uid,
-                    'gid' : sr.st_gid,
-                    'sr': sr # save it all, may need it in preserve_attrs_list
                     ## TODO: Possibly more to save here...
                 }
-                if 'md5' in cfg.sync_checks:
-                    md5 = cache.md5(sr.st_dev, sr.st_ino, sr.st_mtime, sr.st_size)
-                    if md5 is None:
-                            try:
-                                md5 = loc_list.get_md5(relative_file) # this does the file I/O
-                            except IOError:
-                                continue
-                            cache.add(sr.st_dev, sr.st_ino, sr.st_mtime, sr.st_size, md5)
-                    loc_list.record_hardlink(relative_file, sr.st_dev, sr.st_ino, md5)
         return loc_list, single_file
 
-    def _maintain_cache(cache, local_list):
-        if cfg.cache_file:
-            cache.mark_all_for_purge()
-            for i in local_list.keys():
-                cache.unmark_for_purge(local_list[i]['dev'], local_list[i]['inode'], local_list[i]['mtime'], local_list[i]['size'])
-            cache.purge()
-            cache.save(cfg.cache_file)
-
     cfg = Config()
-
-    cache = HashCache()
-    if cfg.cache_file:
-        try:
-            cache.load(cfg.cache_file)
-        except IOError:
-            info(u"No cache file found, creating it.")
-
     local_uris = []
-    local_list = FileDict(ignore_case = False)
+    local_list = SortedDict(ignore_case = False)
     single_file = False
 
     if type(args) not in (list, tuple):
@@ -242,7 +142,8 @@ def fetch_local_list(args, recursive = None):
         local_uris.append(uri)
 
     for uri in local_uris:
-        list_for_uri, single_file = _get_filelist_local(local_list, uri, cache)
+        list_for_uri, single_file = _get_filelist_local(uri)
+        local_list.update(list_for_uri)
 
     ## Single file is True if and only if the user
     ## specified one local URI and that URI represents
@@ -252,8 +153,6 @@ def fetch_local_list(args, recursive = None):
     if len(local_list) > 1:
         single_file = False
 
-    _maintain_cache(cache, local_list)
-
     return local_list, single_file
 
 def fetch_remote_list(args, require_attribs = False, recursive = None):
@@ -284,7 +183,7 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):
             rem_base = rem_base[:rem_base.rfind('/')+1]
             remote_uri = S3Uri("s3://%s/%s" % (remote_uri.bucket(), rem_base))
         rem_base_len = len(rem_base)
-        rem_list = FileDict(ignore_case = False)
+        rem_list = SortedDict(ignore_case = False)
         break_now = False
         for object in response['list']:
             if object['Key'] == rem_base_original and object['Key'][-1] != os.path.sep:
@@ -292,7 +191,7 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):
                 key = os.path.basename(object['Key'])
                 object_uri_str = remote_uri_original.uri()
                 break_now = True
-                rem_list = FileDict(ignore_case = False)   ## Remove whatever has already been put to rem_list
+                rem_list = {}   ## Remove whatever has already been put to rem_list
             else:
                 key = object['Key'][rem_base_len:]      ## Beware - this may be '' if object['Key']==rem_base !!
                 object_uri_str = remote_uri.uri() + key
@@ -303,18 +202,14 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):
                 'object_key' : object['Key'],
                 'object_uri_str' : object_uri_str,
                 'base_uri' : remote_uri,
-                'dev' : None,
-                'inode' : None,
             }
-            md5 = object['ETag'][1:-1]
-            rem_list.record_md5(key, md5)
             if break_now:
                 break
         return rem_list
 
     cfg = Config()
     remote_uris = []
-    remote_list = FileDict(ignore_case = False)
+    remote_list = SortedDict(ignore_case = False)
 
     if type(args) not in (list, tuple):
         args = [args]
@@ -333,7 +228,6 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):
             objectlist = _get_filelist_remote(uri)
             for key in objectlist:
                 remote_list[key] = objectlist[key]
-                remote_list.record_md5(key, objectlist.get_md5(key))
     else:
         for uri in remote_uris:
             uri_str = str(uri)
@@ -371,147 +265,81 @@ def fetch_remote_list(args, require_attribs = False, recursive = None):
                     'md5': response['headers']['etag'].strip('"\''),
                     'timestamp' : dateRFC822toUnix(response['headers']['date'])
                     })
-                    # get md5 from header if it's present.  We would have set that during upload
-                    if response['headers'].has_key('x-amz-meta-s3cmd-attrs'):
-                        attrs = parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs'])
-                        if attrs.has_key('md5'):
-                            remote_item.update({'md5': attrs['md5']})
-
                 remote_list[key] = remote_item
     return remote_list
 
-def parse_attrs_header(attrs_header):
-    attrs = {}
-    for attr in attrs_header.split("/"):
-        key, val = attr.split(":")
-        attrs[key] = val
-    return attrs
-
-
-def compare_filelists(src_list, dst_list, src_remote, dst_remote, delay_updates = False):
+def compare_filelists(src_list, dst_list, src_remote, dst_remote):
     def __direction_str(is_remote):
         return is_remote and "remote" or "local"
 
-    def _compare(src_list, dst_lst, src_remote, dst_remote, file):
-        """Return True if src_list[file] matches dst_list[file], else False"""
-        attribs_match = True
-        if not (src_list.has_key(file) and dst_list.has_key(file)):
-            info(u"%s: does not exist in one side or the other: src_list=%s, dst_list=%s" % (file, src_list.has_key(file), dst_list.has_key(file)))
-            return False
-
-        ## check size first
-        if 'size' in cfg.sync_checks and dst_list[file]['size'] != src_list[file]['size']:
-            debug(u"xfer: %s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size']))
-            attribs_match = False
-
-        ## check md5
-        compare_md5 = 'md5' in cfg.sync_checks
-        # Multipart-uploaded files don't have a valid md5 sum - it ends with "...-nn"
-        if compare_md5:
-            if (src_remote == True and src_list[file]['md5'].find("-") >= 0) or (dst_remote == True and dst_list[file]['md5'].find("-") >= 0):
-                compare_md5 = False
-                info(u"disabled md5 check for %s" % file)
-        if attribs_match and compare_md5:
-            try:
-                src_md5 = src_list.get_md5(file)
-                dst_md5 = dst_list.get_md5(file)
-            except (IOError,OSError), e:
-                # md5 sum verification failed - ignore that file altogether
-                debug(u"IGNR: %s (disappeared)" % (file))
-                warning(u"%s: file disappeared, ignoring." % (file))
-                raise
-
-            if src_md5 != dst_md5:
-                ## checksums are different.
-                attribs_match = False
-                debug(u"XFER: %s (md5 mismatch: src=%s dst=%s)" % (file, src_md5, dst_md5))
-
-        return attribs_match
-
-    # we don't support local->local sync, use 'rsync' or something like that instead ;-)
+    # We don't support local->local sync, use 'rsync' or something like that instead ;-)
     assert(not(src_remote == False and dst_remote == False))
 
     info(u"Verifying attributes...")
     cfg = Config()
-    ## Items left on src_list will be transferred
-    ## Items left on update_list will be transferred after src_list
-    ## Items left on copy_pairs will be copied from dst1 to dst2
-    update_list = FileDict(ignore_case = False)
-    ## Items left on dst_list will be deleted
-    copy_pairs = []
+    exists_list = SortedDict(ignore_case = False)
 
     debug("Comparing filelists (direction: %s -> %s)" % (__direction_str(src_remote), __direction_str(dst_remote)))
+    debug("src_list.keys: %s" % src_list.keys())
+    debug("dst_list.keys: %s" % dst_list.keys())
 
-    for relative_file in src_list.keys():
-        debug(u"CHECK: %s" % (relative_file))
-
-        if dst_list.has_key(relative_file):
+    for file in src_list.keys():
+        debug(u"CHECK: %s" % file)
+        if dst_list.has_key(file):
             ## Was --skip-existing requested?
             if cfg.skip_existing:
-                debug(u"IGNR: %s (used --skip-existing)" % (relative_file))
-                del(src_list[relative_file])
-                del(dst_list[relative_file])
-                continue
-
-            try:
-                same_file = _compare(src_list, dst_list, src_remote, dst_remote, relative_file)
-            except (IOError,OSError), e:
-                debug(u"IGNR: %s (disappeared)" % (relative_file))
-                warning(u"%s: file disappeared, ignoring." % (relative_file))
-                del(src_list[relative_file])
-                del(dst_list[relative_file])
+                debug(u"IGNR: %s (used --skip-existing)" % (file))
+                exists_list[file] = src_list[file]
+                del(src_list[file])
+                ## Remove from destination-list, all that is left there will be deleted
+                del(dst_list[file])
                 continue
 
-            if same_file:
-                debug(u"IGNR: %s (transfer not needed)" % relative_file)
-                del(src_list[relative_file])
-                del(dst_list[relative_file])
+            attribs_match = True
+            ## Check size first
+            if 'size' in cfg.sync_checks and dst_list[file]['size'] != src_list[file]['size']:
+                debug(u"XFER: %s (size mismatch: src=%s dst=%s)" % (file, src_list[file]['size'], dst_list[file]['size']))
+                attribs_match = False
 
-            else:
-                # look for matching file in src
+            ## Check MD5
+            compare_md5 = 'md5' in cfg.sync_checks
+            # Multipart-uploaded files don't have a valid MD5 sum - it ends with "...-NN"
+            if compare_md5 and (src_remote == True and src_list[file]['md5'].find("-") >= 0) or (dst_remote == True and dst_list[file]['md5'].find("-") >= 0):
+                compare_md5 = False
+                info(u"Disabled MD5 check for %s" % file)
+            if attribs_match and compare_md5:
                 try:
-                    md5 = src_list.get_md5(relative_file)
-                except IOError:
-                    md5 = None
-                if md5 is not None and dst_list.by_md5.has_key(md5):
-                    # Found one, we want to copy
-                    dst1 = list(dst_list.by_md5[md5])[0]
-                    debug(u"DST COPY src: %s -> %s" % (dst1, relative_file))
-                    copy_pairs.append((src_list[relative_file], dst1, relative_file))
-                    del(src_list[relative_file])
-                    del(dst_list[relative_file])
-                else:
-                    # record that we will get this file transferred to us (before all the copies), so if we come across it later again,
-                    # we can copy from _this_ copy (e.g. we only upload it once, and copy thereafter).
-                    dst_list.record_md5(relative_file, md5)
-                    update_list[relative_file] = src_list[relative_file]
-                    del src_list[relative_file]
-                    del dst_list[relative_file]
+                    if src_remote == False and dst_remote == True:
+                        src_md5 = hash_file_md5(src_list[file]['full_name'])
+                        dst_md5 = dst_list[file]['md5']
+                    elif src_remote == True and dst_remote == False:
+                        src_md5 = src_list[file]['md5']
+                        dst_md5 = hash_file_md5(dst_list[file]['full_name'])
+                    elif src_remote == True and dst_remote == True:
+                        src_md5 = src_list[file]['md5']
+                        dst_md5 = dst_list[file]['md5']
+                except (IOError,OSError), e:
+                    # MD5 sum verification failed - ignore that file altogether
+                    debug(u"IGNR: %s (disappeared)" % (file))
+                    warning(u"%s: file disappeared, ignoring." % (file))
+                    del(src_list[file])
+                    del(dst_list[file])
+                    continue
 
-        else:
-            # dst doesn't have this file
-            # look for matching file elsewhere in dst
-            try:
-                md5 = src_list.get_md5(relative_file)
-            except IOError:
-               md5 = None
-            dst1 = dst_list.find_md5_one(md5)
-            if dst1 is not None:
-                # Found one, we want to copy
-                debug(u"DST COPY dst: %s -> %s" % (dst1, relative_file))
-                copy_pairs.append((src_list[relative_file], dst1, relative_file))
-                del(src_list[relative_file])
-            else:
-                # we don't have this file, and we don't have a copy of this file elsewhere.  Get it.
-                # record that we will get this file transferred to us (before all the copies), so if we come across it later again,
-                # we can copy from _this_ copy (e.g. we only upload it once, and copy thereafter).
-                dst_list.record_md5(relative_file, md5)
+                if src_md5 != dst_md5:
+                    ## Checksums are different.
+                    attribs_match = False
+                    debug(u"XFER: %s (md5 mismatch: src=%s dst=%s)" % (file, src_md5, dst_md5))
+
+            if attribs_match:
+                ## Remove from source-list, all that is left there will be transferred
+                debug(u"IGNR: %s (transfer not needed)" % file)
+                exists_list[file] = src_list[file]
+                del(src_list[file])
 
-    for f in dst_list.keys():
-        if src_list.has_key(f) or update_list.has_key(f):
-            # leave only those not on src_list + update_list
-            del dst_list[f]
+            ## Remove from destination-list, all that is left there will be deleted
+            del(dst_list[file])
 
-    return src_list, dst_list, update_list, copy_pairs
+    return src_list, dst_list, exists_list
 
 # vim:et:ts=4:sts=4:ai