s3_cmd_bin 0.0.1 → 0.0.2

data/resources/S3/S3Uri.py

@@ -10,7 +10,6 @@ from BidirMap import BidirMap
 from logging import debug
 import S3
 from Utils import unicodise, check_bucket_name_dns_conformity
-import Config
 
 class S3Uri(object):
     type = None
@@ -81,9 +80,9 @@ class S3UriS3(S3Uri):
 
     def public_url(self):
         if self.is_dns_compatible():
-            return "http://%s.%s/%s" % (self._bucket, Config.Config().host_base, self._object)
+            return "http://%s.s3.amazonaws.com/%s" % (self._bucket, self._object)
         else:
-            return "http://%s/%s/%s" % (self._bucket, Config.Config().host_base, self._object)
+            return "http://s3.amazonaws.com/%s/%s" % (self._bucket, self._object)
 
     def host_name(self):
         if self.is_dns_compatible():

data/resources/S3/SimpleDB.py

@@ -131,9 +131,6 @@ class SimpleDB(object):
     def create_request(self, Action, DomainName, parameters = None):
         if not parameters:
             parameters = SortedDict()
-        if len(self.config.access_token) > 0:
-            self.config.refresh_role()
-            parameters['Signature']=self.config.access_token
         parameters['AWSAccessKeyId'] = self.config.access_key
         parameters['Version'] = self.Version
         parameters['SignatureVersion'] = self.SignatureVersion

data/resources/S3/SortedDict.py

@@ -4,7 +4,6 @@
 ## License: GPL Version 2
 
 from BidirMap import BidirMap
-import Utils
 
 class SortedDictIterator(object):
     def __init__(self, sorted_dict, keys):
@@ -46,8 +45,6 @@ class SortedDict(dict):
     def __iter__(self):
         return SortedDictIterator(self, self.keys())
 
-
-
 if __name__ == "__main__":
     d = { 'AWS' : 1, 'Action' : 2, 'america' : 3, 'Auckland' : 4, 'America' : 5 }
     sd = SortedDict(d)

data/resources/S3/Utils.py

@@ -3,7 +3,6 @@
 ##         http://www.logix.cz/michal
 ## License: GPL Version 2
 
-import datetime
 import os
 import sys
 import time
@@ -14,11 +13,9 @@ import rfc822
 import hmac
 import base64
 import errno
-import urllib
 
 from logging import debug, info, warning, error
 
-
 import Config
 import Exceptions
 
@@ -166,20 +163,7 @@ def formatSize(size, human_readable = False, floating_point = False):
 __all__.append("formatSize")
 
 def formatDateTime(s3timestamp):
-    try:
-        import pytz
-        timezone = pytz.timezone(os.environ.get('TZ', 'UTC'))
-        tz = pytz.timezone('UTC')
-        ## Can't unpack args and follow that with kwargs in python 2.5
-        ## So we pass them all as kwargs
-        params = zip(('year', 'month', 'day', 'hour', 'minute', 'second', 'tzinfo'),
-                     dateS3toPython(s3timestamp)[0:6] + (tz,))
-        params = dict(params)
-        utc_dt = datetime.datetime(**params)
-        dt_object = utc_dt.astimezone(timezone)
-    except ImportError:
-        dt_object = datetime.datetime(*dateS3toPython(s3timestamp)[0:6])
-    return dt_object.strftime("%Y-%m-%d %H:%M")
+    return time.strftime("%Y-%m-%d %H:%M", dateS3toPython(s3timestamp))
 __all__.append("formatDateTime")
 
 def convertTupleListToDict(list):
@@ -335,73 +319,12 @@ def replace_nonprintables(string):
 __all__.append("replace_nonprintables")
 
 def sign_string(string_to_sign):
-    """Sign a string with the secret key, returning base64 encoded results.
-    By default the configured secret key is used, but may be overridden as
-    an argument.
-
-    Useful for REST authentication. See http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html
-    """
+    #debug("string_to_sign: %s" % string_to_sign)
     signature = base64.encodestring(hmac.new(Config.Config().secret_key, string_to_sign, sha1).digest()).strip()
+    #debug("signature: %s" % signature)
     return signature
 __all__.append("sign_string")
 
-def sign_url(url_to_sign, expiry):
-    """Sign a URL in s3://bucket/object form with the given expiry
-    time. The object will be accessible via the signed URL until the
-    AWS key and secret are revoked or the expiry time is reached, even
-    if the object is otherwise private.
-
-    See: http://s3.amazonaws.com/doc/s3-developer-guide/RESTAuthentication.html
-    """
-    return sign_url_base(
-        bucket = url_to_sign.bucket(),
-        object = url_to_sign.object(),
-        expiry = expiry
-    )
-__all__.append("sign_url")
-
-def sign_url_base(**parms):
-    """Shared implementation of sign_url methods. Takes a hash of 'bucket', 'object' and 'expiry' as args."""
-    parms['expiry']=time_to_epoch(parms['expiry'])
-    parms['access_key']=Config.Config().access_key
-    debug("Expiry interpreted as epoch time %s", parms['expiry'])
-    signtext = 'GET\n\n\n%(expiry)d\n/%(bucket)s/%(object)s' % parms
-    debug("Signing plaintext: %r", signtext)
-    parms['sig'] = urllib.quote_plus(sign_string(signtext))
-    debug("Urlencoded signature: %s", parms['sig'])
-    return "http://%(bucket)s.s3.amazonaws.com/%(object)s?AWSAccessKeyId=%(access_key)s&Expires=%(expiry)d&Signature=%(sig)s" % parms
-
-def time_to_epoch(t):
-    """Convert time specified in a variety of forms into UNIX epoch time.
-    Accepts datetime.datetime, int, anything that has a strftime() method, and standard time 9-tuples
-    """
-    if isinstance(t, int):
-        # Already an int
-        return t
-    elif isinstance(t, tuple) or isinstance(t, time.struct_time):
-        # Assume it's a time 9-tuple
-        return int(time.mktime(t))
-    elif hasattr(t, 'timetuple'):
-        # Looks like a datetime object or compatible
-        return int(time.mktime(ex.timetuple()))
-    elif hasattr(t, 'strftime'):
-        # Looks like the object supports standard srftime()
-        return int(t.strftime('%s'))
-    elif isinstance(t, str) or isinstance(t, unicode):
-        # See if it's a string representation of an epoch
-        try:
-            return int(t)
-        except ValueError:
-            # Try to parse it as a timestamp string
-            try:
-                return time.strptime(t)
-            except ValueError, ex:
-                # Will fall through
-                debug("Failed to parse date with strptime: %s", ex)
-                pass
-    raise Exceptions.ParameterError('Unable to convert %r to an epoch time. Pass an epoch time. Try `date -d \'now + 1 year\' +%%s` (shell) or time.mktime (Python).' % t)
-
-
 def check_bucket_name(bucket, dns_strict = True):
     if dns_strict:
         invalid = re.search("([^a-z0-9\.-])", bucket)

data/resources/run-tests.py

@@ -471,7 +471,7 @@ test_s3cmd("Rename within S3", ['mv', '%s/copy/etc2/Logo.PNG' % pbucket(2), '%s/
 ## ====== Sync between buckets
 test_s3cmd("Sync remote2remote", ['sync', '%s/xyz/' % pbucket(1), '%s/copy/' % pbucket(2), '--delete-removed', '--exclude', 'non-printables*'],
     must_find = [ "File %s/xyz/demo/dir1/file1-1.txt copied to %s/copy/demo/dir1/file1-1.txt" % (pbucket(1), pbucket(2)),
-                  "remote copy: etc/logo.png -> etc2/Logo.PNG",
+                  "File %s/xyz/etc2/Logo.PNG copied to %s/copy/etc2/Logo.PNG" % (pbucket(1), pbucket(2)),
                   "deleted: '%s/copy/etc/logo.png'" % pbucket(2) ],
     must_not_find = [ "blah.txt" ])
 
@@ -485,7 +485,7 @@ test_s3cmd("Put symbolic links", ['put', 'testsuite/etc/linked1.png', 's3://%s/x
 
 ## ====== Sync symbolic links
 test_s3cmd("Sync symbolic links", ['sync', 'testsuite/', 's3://%s/xyz/' % bucket(1), '--no-encrypt', '--follow-symlinks' ],
-    must_find = ["remote copy: etc2/Logo.PNG -> etc/linked.png"],
+    must_find = ["File 'testsuite/etc/linked.png' stored as '%s/xyz/etc/linked.png'" % pbucket(1)],
            # Don't want to recursively copy linked directories!
            must_not_find_re = ["etc/more/linked-dir/more/give-me-more.txt",
                                "etc/brokenlink.png"],

data/resources/s3cmd

@@ -23,9 +23,6 @@ import locale
 import subprocess
 import htmlentitydefs
 import socket
-import shutil
-import tempfile
-import S3.Exceptions
 
 from copy import copy
 from optparse import OptionParser, Option, OptionValueError, IndentedHelpFormatter
@@ -34,7 +31,6 @@ from distutils.spawn import find_executable
 
 def output(message):
     sys.stdout.write(message + "\n")
-    sys.stdout.flush()
 
 def check_args_type(args, type, verbose_type):
     for arg in args:
@@ -69,28 +65,18 @@ def subcmd_bucket_usage(s3, uri):
 
     if object.endswith('*'):
         object = object[:-1]
-
+    try:
+        response = s3.bucket_list(bucket, prefix = object, recursive = True)
+    except S3Error, e:
+        if S3.codes.has_key(e.info["Code"]):
+            error(S3.codes[e.info["Code"]] % bucket)
+            return
+        else:
+            raise
     bucket_size = 0
-    # iterate and store directories to traverse, while summing objects:
-    dirs = [object]
-    while dirs:
-        try:
-            response = s3.bucket_list(bucket, prefix=dirs.pop())
-        except S3Error, e:
-            if S3.codes.has_key(e.info["Code"]):
-                error(S3.codes[e.info["Code"]] % bucket)
-                return
-            else:
-                raise
-
-        # objects in the current scope:
-        for obj in response["list"]:
-            bucket_size += int(obj["Size"])
-
-        # directories found in current scope:
-        for obj in response["common_prefixes"]:
-            dirs.append(obj["Prefix"])
-
+    for object in response["list"]:
+        size, size_coeff = formatSize(object["Size"], False)
+        bucket_size += size
     total_size, size_coeff = formatSize(bucket_size, Config().human_readable_sizes)
     total_size_str = str(total_size) + size_coeff
     output(u"%s %s" % (total_size_str.ljust(8), uri))
@@ -280,11 +266,7 @@ def cmd_object_put(args):
     info(u"Summary: %d local files to upload" % local_count)
 
     if local_count > 0:
-        if not single_file_local and '-' in local_list.keys():
-            raise ParameterError("Cannot specify multiple local files if uploading from '-' (ie stdin)")
-        elif single_file_local and local_list.keys()[0] == "-" and destination_base.endswith("/"):
-            raise ParameterError("Destination S3 URI must not end with '/' when uploading from stdin.")
-        elif not destination_base.endswith("/"):
+        if not destination_base.endswith("/"):
             if not single_file_local:
                 raise ParameterError("Destination S3 URI must end with '/' (ie must refer to a directory on the remote side).")
             local_list[local_list.keys()[0]]['remote_uri'] = unicodise(destination_base)
@@ -296,13 +278,9 @@ def cmd_object_put(args):
         for key in exclude_list:
             output(u"exclude: %s" % unicodise(key))
         for key in local_list:
-            if key != "-":
-                nicekey = local_list[key]['full_name_unicode']
-            else:
-                nicekey = "<stdin>"
-            output(u"upload: %s -> %s" % (nicekey, local_list[key]['remote_uri']))
+            output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], local_list[key]['remote_uri']))
 
-        warning(u"Exiting now because of --dry-run")
+        warning(u"Exitting now because of --dry-run")
         return
 
     seq = 0
@@ -415,7 +393,7 @@ def cmd_object_get(args):
         for key in remote_list:
             output(u"download: %s -> %s" % (remote_list[key]['object_uri_str'], remote_list[key]['local_filename']))
 
-        warning(u"Exiting now because of --dry-run")
+        warning(u"Exitting now because of --dry-run")
         return
 
     seq = 0
@@ -462,14 +440,7 @@ def cmd_object_get(args):
             except IOError, e:
                 error(u"Skipping %s: %s" % (destination, e.strerror))
                 continue
-        try:
-            response = s3.object_get(uri, dst_stream, start_position = start_position, extra_label = seq_label)
-        except S3Error, e:
-            if not file_exists: # Delete, only if file didn't exist before!
-                debug(u"object_get failed for '%s', deleting..." % (destination,))
-                os.unlink(destination)
-            raise
-
+        response = s3.object_get(uri, dst_stream, start_position = start_position, extra_label = seq_label)
         if response["headers"].has_key("x-amz-meta-s3tools-gpgenc"):
             gpg_decrypt(destination, response["headers"]["x-amz-meta-s3tools-gpgenc"])
             response["size"] = os.stat(destination)[6]
@@ -477,9 +448,6 @@ def cmd_object_get(args):
             speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)
             output(u"File %s saved as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s)" %
                 (uri, destination, response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1]))
-        if Config().delete_after_fetch:
-            s3.object_delete(uri)
-            output(u"File %s removed after fetch" % (uri))
 
 def cmd_object_del(args):
     for uri_str in args:
@@ -512,7 +480,7 @@ def subcmd_object_del_uri(uri_str, recursive = None):
         for key in remote_list:
             output(u"delete: %s" % remote_list[key]['object_uri_str'])
 
-        warning(u"Exiting now because of --dry-run")
+        warning(u"Exitting now because of --dry-run")
         return
 
     for key in remote_list:
@@ -553,7 +521,7 @@ def subcmd_cp_mv(args, process_fce, action_str, message):
         for key in remote_list:
             output(u"%s: %s -> %s" % (action_str, remote_list[key]['object_uri_str'], remote_list[key]['dest_name']))
 
-        warning(u"Exiting now because of --dry-run")
+        warning(u"Exitting now because of --dry-run")
         return
 
     seq = 0
@@ -602,18 +570,10 @@ def cmd_info(args):
                 output(u"   Location:  %s" % info['bucket-location'])
             acl = s3.get_acl(uri)
             acl_grant_list = acl.getGrantList()
-
-            try:
-                policy = s3.get_policy(uri)
-                output(u"   policy: %s" % policy)
-            except:
-                output(u"   policy: none")
-            
             for grant in acl_grant_list:
                 output(u"   ACL:       %s: %s" % (grant['grantee'], grant['permission']))
             if acl.isAnonRead():
                 output(u"   URL:       %s" % uri.public_url())
-
         except S3Error, e:
             if S3.codes.has_key(e.info["Code"]):
                 error(S3.codes[e.info["Code"]] % uri.bucket())
@@ -622,17 +582,6 @@ def cmd_info(args):
                 raise
 
 def cmd_sync_remote2remote(args):
-    def _do_deletes(s3, dst_list):
-        # Delete items in destination that are not in source
-        if cfg.dry_run:
-            for key in dst_list:
-                output(u"delete: %s" % dst_list[key]['object_uri_str'])
-        else:
-            for key in dst_list:
-                uri = S3Uri(dst_list[key]['object_uri_str'])
-                s3.object_delete(uri)
-                output(u"deleted: '%s'" % uri)
-
     s3 = S3(Config())
 
     # Normalise s3://uri (e.g. assert trailing slash)
@@ -648,19 +597,17 @@ def cmd_sync_remote2remote(args):
 
     src_list, exclude_list = filter_exclude_include(src_list)
 
-    src_list, dst_list, update_list, copy_pairs = compare_filelists(src_list, dst_list, src_remote = True, dst_remote = True, delay_updates = cfg.delay_updates)
+    src_list, dst_list, existing_list = compare_filelists(src_list, dst_list, src_remote = True, dst_remote = True)
 
     src_count = len(src_list)
-    update_count = len(update_list)
     dst_count = len(dst_list)
 
     print(u"Summary: %d source files to copy, %d files at destination to delete" % (src_count, dst_count))
 
-    ### Populate 'target_uri' only if we've got something to sync from src to dst
-    for key in src_list:
-        src_list[key]['target_uri'] = destination_base + key
-    for key in update_list:
-        update_list[key]['target_uri'] = destination_base + key
+    if src_count > 0:
+        ### Populate 'remote_uri' only if we've got something to sync from src to dst
+        for key in src_list:
+            src_list[key]['target_uri'] = destination_base + key
 
     if cfg.dry_run:
         for key in exclude_list:
@@ -670,42 +617,37 @@ def cmd_sync_remote2remote(args):
                 output(u"delete: %s" % dst_list[key]['object_uri_str'])
         for key in src_list:
             output(u"Sync: %s -> %s" % (src_list[key]['object_uri_str'], src_list[key]['target_uri']))
-        warning(u"Exiting now because of --dry-run")
+        warning(u"Exitting now because of --dry-run")
         return
 
-    # if there are copy pairs, we can't do delete_before, on the chance
-    # we need one of the to-be-deleted files as a copy source.
-    if len(copy_pairs) > 0:
-        cfg.delete_after = True
-
     # Delete items in destination that are not in source
-    if cfg.delete_removed and not cfg.delete_after:
-        _do_deletes(s3, dst_list)
-
-    def _upload(src_list, seq, src_count):
-        file_list = src_list.keys()
-        file_list.sort()
-        for file in file_list:
-            seq += 1
-            item = src_list[file]
-            src_uri = S3Uri(item['object_uri_str'])
-            dst_uri = S3Uri(item['target_uri'])
-            seq_label = "[%d of %d]" % (seq, src_count)
-            extra_headers = copy(cfg.extra_headers)
-            try:
-                response = s3.object_copy(src_uri, dst_uri, extra_headers)
-                output("File %(src)s copied to %(dst)s" % { "src" : src_uri, "dst" : dst_uri })
-            except S3Error, e:
-                error("File %(src)s could not be copied: %(e)s" % { "src" : src_uri, "e" : e })
-        return seq
+    if cfg.delete_removed:
+        if cfg.dry_run:
+            for key in dst_list:
+                output(u"delete: %s" % dst_list[key]['object_uri_str'])
+        else:
+            for key in dst_list:
+                uri = S3Uri(dst_list[key]['object_uri_str'])
+                s3.object_delete(uri)
+                output(u"deleted: '%s'" % uri)
 
     # Perform the synchronization of files
     timestamp_start = time.time()
     seq = 0
-    seq = _upload(src_list, seq, src_count + update_count)
-    seq = _upload(update_list, seq, src_count + update_count)
-    n_copied, bytes_saved = remote_copy(s3, copy_pairs, destination_base)
-
+    file_list = src_list.keys()
+    file_list.sort()
+    for file in file_list:
+        seq += 1
+        item = src_list[file]
+        src_uri = S3Uri(item['object_uri_str'])
+        dst_uri = S3Uri(item['target_uri'])
+        seq_label = "[%d of %d]" % (seq, src_count)
+        extra_headers = copy(cfg.extra_headers)
+        try:
+            response = s3.object_copy(src_uri, dst_uri, extra_headers)
+            output("File %(src)s copied to %(dst)s" % { "src" : src_uri, "dst" : dst_uri })
+        except S3Error, e:
+            error("File %(src)s could not be copied: %(e)s" % { "src" : src_uri, "e" : e })
     total_elapsed = time.time() - timestamp_start
     outstr = "Done. Copied %d files in %0.1f seconds, %0.2f files/s" % (seq, total_elapsed, seq/total_elapsed)
     if seq > 0:
@@ -713,15 +655,13 @@ def cmd_sync_remote2remote(args):
     else:
         info(outstr)
 
-    # Delete items in destination that are not in source
-    if cfg.delete_removed and cfg.delete_after:
-        _do_deletes(s3, dst_list)
-
 def cmd_sync_remote2local(args):
-    def _do_deletes(local_list):
-        for key in local_list:
-            os.unlink(local_list[key]['full_name'])
-            output(u"deleted: %s" % local_list[key]['full_name_unicode'])
+    def _parse_attrs_header(attrs_header):
+        attrs = {}
+        for attr in attrs_header.split("/"):
+            key, val = attr.split(":")
+            attrs[key] = val
+        return attrs
 
     s3 = S3(Config())
 
@@ -736,34 +676,26 @@ def cmd_sync_remote2local(args):
 
     remote_list, exclude_list = filter_exclude_include(remote_list)
 
-    remote_list, local_list, update_list, copy_pairs = compare_filelists(remote_list, local_list, src_remote = True, dst_remote = False, delay_updates = cfg.delay_updates)
+    remote_list, local_list, existing_list = compare_filelists(remote_list, local_list, src_remote = True, dst_remote = False)
 
     local_count = len(local_list)
     remote_count = len(remote_list)
-    update_count = len(update_list)
-    copy_pairs_count = len(copy_pairs)
-
-    info(u"Summary: %d remote files to download, %d local files to delete, %d local files to hardlink" % (remote_count + update_count, local_count, copy_pairs_count))
 
-    def _set_local_filename(remote_list, destination_base):
-        if len(remote_list) == 0:
-            return
-        if not os.path.isdir(destination_base):
-            ## We were either given a file name (existing or not) or want STDOUT
-            if len(remote_list) > 1:
-                raise ParameterError("Destination must be a directory when downloading multiple sources.")
-            remote_list[remote_list.keys()[0]]['local_filename'] = deunicodise(destination_base)
-        else:
-            if destination_base[-1] != os.path.sep:
-                destination_base += os.path.sep
-            for key in remote_list:
-                local_filename = destination_base + key
-                if os.path.sep != "/":
-                    local_filename = os.path.sep.join(local_filename.split("/"))
-                remote_list[key]['local_filename'] = deunicodise(local_filename)
+    info(u"Summary: %d remote files to download, %d local files to delete" % (remote_count, local_count))
 
-    _set_local_filename(remote_list, destination_base)
-    _set_local_filename(update_list, destination_base)
+    if not os.path.isdir(destination_base):
+        ## We were either given a file name (existing or not) or want STDOUT
+        if remote_count > 1:
+            raise ParameterError("Destination must be a directory when downloading multiple sources.")
+        remote_list[remote_list.keys()[0]]['local_filename'] = deunicodise(destination_base)
+    else:
+        if destination_base[-1] != os.path.sep:
+            destination_base += os.path.sep
+        for key in remote_list:
+            local_filename = destination_base + key
+            if os.path.sep != "/":
+                local_filename = os.path.sep.join(local_filename.split("/"))
+            remote_list[key]['local_filename'] = deunicodise(local_filename)
 
     if cfg.dry_run:
         for key in exclude_list:
@@ -772,128 +704,94 @@ def cmd_sync_remote2local(args):
             for key in local_list:
                 output(u"delete: %s" % local_list[key]['full_name_unicode'])
         for key in remote_list:
-            output(u"download: %s -> %s" % (unicodise(remote_list[key]['object_uri_str']), unicodise(remote_list[key]['local_filename'])))
-        for key in update_list:
-            output(u"download: %s -> %s" % (update_list[key]['object_uri_str'], update_list[key]['local_filename']))
+            output(u"download: %s -> %s" % (remote_list[key]['object_uri_str'], remote_list[key]['local_filename']))
 
-        warning(u"Exiting now because of --dry-run")
+        warning(u"Exitting now because of --dry-run")
         return
 
-    # if there are copy pairs, we can't do delete_before, on the chance
-    # we need one of the to-be-deleted files as a copy source.
-    if len(copy_pairs) > 0:
-        cfg.delete_after = True
-
-    if cfg.delete_removed and not cfg.delete_after:
-        _do_deletes(local_list)
-
-    def _download(remote_list, seq, total, total_size, dir_cache):
-        file_list = remote_list.keys()
-        file_list.sort()
-        for file in file_list:
-            seq += 1
-            item = remote_list[file]
-            uri = S3Uri(item['object_uri_str'])
-            dst_file = item['local_filename']
-            seq_label = "[%d of %d]" % (seq, total)
-            try:
-                dst_dir = os.path.dirname(dst_file)
-                if not dir_cache.has_key(dst_dir):
-                    dir_cache[dst_dir] = Utils.mkdir_with_parents(dst_dir)
-                if dir_cache[dst_dir] == False:
-                    warning(u"%s: destination directory not writable: %s" % (file, dst_dir))
-                    continue
-                try:
-                    debug(u"dst_file=%s" % unicodise(dst_file))
-                    # create temporary files (of type .s3cmd.XXXX.tmp) in the same directory
-                    # for downloading and then rename once downloaded
-                    chkptfd, chkptfname = tempfile.mkstemp(".tmp",".s3cmd.",os.path.dirname(dst_file))
-                    debug(u"created chkptfname=%s" % unicodise(chkptfname))
-                    dst_stream = os.fdopen(chkptfd, "wb")
-                    response = s3.object_get(uri, dst_stream, extra_label = seq_label)
-                    dst_stream.close()
-                    # download completed, rename the file to destination
-                    os.rename(chkptfname, dst_file)
-
-                    # set permissions on destination file
-                    original_umask = os.umask(0);
-                    os.umask(original_umask);
-                    mode = 0777 - original_umask;
-                    debug(u"mode=%s" % oct(mode))
-                    
-                    os.chmod(dst_file, mode);
-                    
-                    debug(u"renamed chkptfname=%s to dst_file=%s" % (unicodise(chkptfname), unicodise(dst_file)))
-                    if response['headers'].has_key('x-amz-meta-s3cmd-attrs') and cfg.preserve_attrs:
-                        attrs = parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs'])
-                        if attrs.has_key('mode'):
-                            os.chmod(dst_file, int(attrs['mode']))
-                        if attrs.has_key('mtime') or attrs.has_key('atime'):
-                            mtime = attrs.has_key('mtime') and int(attrs['mtime']) or int(time.time())
-                            atime = attrs.has_key('atime') and int(attrs['atime']) or int(time.time())
-                            os.utime(dst_file, (atime, mtime))
-                        ## FIXME: uid/gid / uname/gname handling comes here! TODO
-                except OSError, e:
-                    try: 
-                        dst_stream.close() 
-                        os.remove(chkptfname)
-                    except: pass
-                    if e.errno == errno.EEXIST:
-                        warning(u"%s exists - not overwriting" % (dst_file))
-                        continue
-                    if e.errno in (errno.EPERM, errno.EACCES):
-                        warning(u"%s not writable: %s" % (dst_file, e.strerror))
-                        continue
-                    if e.errno == errno.EISDIR:
-                        warning(u"%s is a directory - skipping over" % dst_file)
-                        continue
-                    raise e
-                except KeyboardInterrupt:
-                    try: 
-                        dst_stream.close()
-                        os.remove(chkptfname)
-                    except: pass
-                    warning(u"Exiting after keyboard interrupt")
-                    return
-                except Exception, e:
-                    try: 
-                        dst_stream.close()
-                        os.remove(chkptfname)
-                    except: pass
-                    error(u"%s: %s" % (file, e))
-                    continue
-                # We have to keep repeating this call because
-                # Python 2.4 doesn't support try/except/finally
-                # construction :-(
-                try: 
-                    dst_stream.close()
-                    os.remove(chkptfname)
-                except: pass
-            except S3DownloadError, e:
-                error(u"%s: download failed too many times. Skipping that file." % file)
-                continue
-            speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)
-            if not Config().progress_meter:
-                output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" %
-                    (uri, unicodise(dst_file), response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1],
-                    seq_label))
-            total_size += response["size"]
-            if Config().delete_after_fetch:
-                s3.object_delete(uri)
-                output(u"File '%s' removed after syncing" % (uri))
-        return seq, total_size
+    if cfg.delete_removed:
+        for key in local_list:
+            os.unlink(local_list[key]['full_name'])
+            output(u"deleted: %s" % local_list[key]['full_name_unicode'])
 
     total_size = 0
     total_elapsed = 0.0
     timestamp_start = time.time()
-    dir_cache = {}
     seq = 0
-    seq, total_size = _download(remote_list, seq, remote_count + update_count, total_size, dir_cache)
-    seq, total_size = _download(update_list, seq, remote_count + update_count, total_size, dir_cache)
-
-    failed_copy_list = local_copy(copy_pairs, destination_base)
-    _set_local_filename(failed_copy_list, destination_base)
-    seq, total_size = _download(failed_copy_list, seq, len(failed_copy_list) + remote_count + update_count, total_size, dir_cache)
+    dir_cache = {}
+    file_list = remote_list.keys()
+    file_list.sort()
+    for file in file_list:
+        seq += 1
+        item = remote_list[file]
+        uri = S3Uri(item['object_uri_str'])
+        dst_file = item['local_filename']
+        seq_label = "[%d of %d]" % (seq, remote_count)
+        try:
+            dst_dir = os.path.dirname(dst_file)
+            if not dir_cache.has_key(dst_dir):
+                dir_cache[dst_dir] = Utils.mkdir_with_parents(dst_dir)
+            if dir_cache[dst_dir] == False:
+                warning(u"%s: destination directory not writable: %s" % (file, dst_dir))
+                continue
+            try:
+                open_flags = os.O_CREAT
+                open_flags |= os.O_TRUNC
+                # open_flags |= os.O_EXCL
+
+                debug(u"dst_file=%s" % unicodise(dst_file))
+                # This will have failed should the file exist
+                os.close(os.open(dst_file, open_flags))
+                # Yeah I know there is a race condition here. Sadly I don't know how to open() in exclusive mode.
+                dst_stream = open(dst_file, "wb")
+                response = s3.object_get(uri, dst_stream, extra_label = seq_label)
+                dst_stream.close()
+                if response['headers'].has_key('x-amz-meta-s3cmd-attrs') and cfg.preserve_attrs:
+                    attrs = _parse_attrs_header(response['headers']['x-amz-meta-s3cmd-attrs'])
+                    if attrs.has_key('mode'):
+                        os.chmod(dst_file, int(attrs['mode']))
+                    if attrs.has_key('mtime') or attrs.has_key('atime'):
+                        mtime = attrs.has_key('mtime') and int(attrs['mtime']) or int(time.time())
+                        atime = attrs.has_key('atime') and int(attrs['atime']) or int(time.time())
+                        os.utime(dst_file, (atime, mtime))
+                    ## FIXME: uid/gid / uname/gname handling comes here! TODO
+            except OSError, e:
+                try: dst_stream.close()
+                except: pass
+                if e.errno == errno.EEXIST:
+                    warning(u"%s exists - not overwriting" % (dst_file))
+                    continue
+                if e.errno in (errno.EPERM, errno.EACCES):
+                    warning(u"%s not writable: %s" % (dst_file, e.strerror))
+                    continue
+                if e.errno == errno.EISDIR:
+                    warning(u"%s is a directory - skipping over" % dst_file)
+                    continue
+                raise e
+            except KeyboardInterrupt:
+                try: dst_stream.close()
+                except: pass
+                warning(u"Exiting after keyboard interrupt")
+                return
+            except Exception, e:
+                try: dst_stream.close()
+                except: pass
+                error(u"%s: %s" % (file, e))
+                continue
+            # We have to keep repeating this call because
+            # Python 2.4 doesn't support try/except/finally
+            # construction :-(
+            try: dst_stream.close()
+            except: pass
+        except S3DownloadError, e:
+            error(u"%s: download failed too many times. Skipping that file." % file)
+            continue
+        speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)
+        if not Config().progress_meter:
+            output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" %
+                (uri, unicodise(dst_file), response["size"], response["elapsed"], speed_fmt[0], speed_fmt[1],
+                seq_label))
+        total_size += response["size"]
 
     total_elapsed = time.time() - timestamp_start
     speed_fmt = formatSize(total_size/total_elapsed, human_readable = True, floating_point = True)
@@ -906,282 +804,158 @@ def cmd_sync_remote2local(args):
     else:
         info(outstr)
 
-    if cfg.delete_removed and cfg.delete_after:
-        _do_deletes(local_list)
-
-def local_copy(copy_pairs, destination_base):
-    # Do NOT hardlink local files by default, that'd be silly
-    # For instance all empty files would become hardlinked together!
-
-    failed_copy_list = FileDict()
-    for (src_obj, dst1, relative_file) in copy_pairs:
-        src_file = os.path.join(destination_base, dst1)
-        dst_file = os.path.join(destination_base, relative_file)
-        dst_dir = os.path.dirname(dst_file)
-        try:
-            if not os.path.isdir(dst_dir):
-                debug("MKDIR %s" % dst_dir)
-                os.makedirs(dst_dir)
-            debug(u"Copying %s to %s" % (src_file, dst_file))
-            shutil.copy2(src_file, dst_file)
-        except (IOError, OSError), e:
-            warning(u'Unable to hardlink or copy files %s -> %s: %s' % (src_file, dst_file, e))
-            failed_copy_list[relative_file] = src_obj
-    return failed_copy_list
-
-def remote_copy(s3, copy_pairs, destination_base):
-    saved_bytes = 0
-    for (src_obj, dst1, dst2) in copy_pairs:
-        debug(u"Remote Copying from %s to %s" % (dst1, dst2))
-        dst1_uri = S3Uri(destination_base + dst1)
-        dst2_uri = S3Uri(destination_base + dst2)
-        extra_headers = copy(cfg.extra_headers)
-        try:
-            s3.object_copy(dst1_uri, dst2_uri, extra_headers)
-            info = s3.object_info(dst2_uri)
-            saved_bytes = saved_bytes + int(info['headers']['content-length'])
-            output(u"remote copy: %s -> %s" % (dst1, dst2))
-        except:
-            raise
-    return (len(copy_pairs), saved_bytes)
-
-
 def cmd_sync_local2remote(args):
-    def _build_attr_header(local_list, src):
+    def _build_attr_header(src):
         import pwd, grp
         attrs = {}
+        src = deunicodise(src)
+        try:
+            st = os.stat_result(os.stat(src))
+        except OSError, e:
+            raise InvalidFileError(u"%s: %s" % (unicodise(src), e.strerror))
         for attr in cfg.preserve_attrs_list:
             if attr == 'uname':
                 try:
-                    val = pwd.getpwuid(local_list[src]['uid']).pw_name
+                    val = pwd.getpwuid(st.st_uid).pw_name
                 except KeyError:
                     attr = "uid"
-                    val = local_list[src].get('uid')
-                    warning(u"%s: Owner username not known. Storing UID=%d instead." % (src, val))
+                    val = st.st_uid
+                    warning(u"%s: Owner username not known. Storing UID=%d instead." % (unicodise(src), val))
             elif attr == 'gname':
                 try:
-                    val = grp.getgrgid(local_list[src].get('gid')).gr_name
+                    val = grp.getgrgid(st.st_gid).gr_name
                 except KeyError:
                     attr = "gid"
-                    val = local_list[src].get('gid')
-                    warning(u"%s: Owner groupname not known. Storing GID=%d instead." % (src, val))
-            elif attr == 'md5':
-                try:
-                    val = local_list.get_md5(src)
-                except IOError:
-                    val = None
+                    val = st.st_gid
+                    warning(u"%s: Owner groupname not known. Storing GID=%d instead." % (unicodise(src), val))
             else:
-                val = getattr(local_list[src]['sr'], 'st_' + attr)
+                val = getattr(st, 'st_' + attr)
             attrs[attr] = val
-
-        if 'md5' in attrs and attrs['md5'] is None:
-            del attrs['md5']
-
         result = ""
         for k in attrs: result += "%s:%s/" % (k, attrs[k])
         return { 'x-amz-meta-s3cmd-attrs' : result[:-1] }
 
-    def _do_deletes(s3, remote_list):
-        for key in remote_list:
-            uri = S3Uri(remote_list[key]['object_uri_str'])
-            s3.object_delete(uri)
-            output(u"deleted: '%s'" % uri)
-
-    def _single_process(local_list):
-        for dest in destinations:
-            ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash)
-            destination_base_uri = S3Uri(dest)
-            if destination_base_uri.type != 's3':
-                raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri)
-            destination_base = str(destination_base_uri)
-            _child(destination_base, local_list)
-            return destination_base_uri
-
-    def _parent():
-        # Now that we've done all the disk I/O to look at the local file system and
-        # calculate the md5 for each file, fork for each destination to upload to them separately
-        # and in parallel
-        child_pids = []
-
-        for dest in destinations:
-            ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash)
-            destination_base_uri = S3Uri(dest)
-            if destination_base_uri.type != 's3':
-                raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri)
-            destination_base = str(destination_base_uri)
-            child_pid = os.fork()
-            if child_pid == 0:
-                _child(destination_base, local_list)
-                os._exit(0)
-            else:
-                child_pids.append(child_pid)
-
-        while len(child_pids):
-            (pid, status) = os.wait()
-            child_pids.remove(pid)
-
-        return
+    s3 = S3(cfg)
 
-    def _child(destination_base, local_list):
-        def _set_remote_uri(local_list, destination_base, single_file_local):
-            if len(local_list) > 0:
-                ## Populate 'remote_uri' only if we've got something to upload
-                if not destination_base.endswith("/"):
-                    if not single_file_local:
-                        raise ParameterError("Destination S3 URI must end with '/' (ie must refer to a directory on the remote side).")
-                    local_list[local_list.keys()[0]]['remote_uri'] = unicodise(destination_base)
-                else:
-                    for key in local_list:
-                        local_list[key]['remote_uri'] = unicodise(destination_base + key)
-
-        def _upload(local_list, seq, total, total_size):
-            file_list = local_list.keys()
-            file_list.sort()
-            for file in file_list:
-                seq += 1
-                item = local_list[file]
-                src = item['full_name']
-                uri = S3Uri(item['remote_uri'])
-                seq_label = "[%d of %d]" % (seq, total)
-                extra_headers = copy(cfg.extra_headers)
-                try:
-                    if cfg.preserve_attrs:
-                        attr_header = _build_attr_header(local_list, file)
-                        debug(u"attr_header: %s" % attr_header)
-                        extra_headers.update(attr_header)
-                    response = s3.object_put(src, uri, extra_headers, extra_label = seq_label)
-                except InvalidFileError, e:
-                    warning(u"File can not be uploaded: %s" % e)
-                    continue
-                except S3UploadError, e:
-                    error(u"%s: upload failed too many times. Skipping that file." % item['full_name_unicode'])
-                    continue
-                speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)
-                if not cfg.progress_meter:
-                    output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" %
-                        (item['full_name_unicode'], uri, response["size"], response["elapsed"],
-                        speed_fmt[0], speed_fmt[1], seq_label))
-                total_size += response["size"]
-                uploaded_objects_list.append(uri.object())
-            return seq, total_size
+    if cfg.encrypt:
+        error(u"S3cmd 'sync' doesn't yet support GPG encryption, sorry.")
+        error(u"Either use unconditional 's3cmd put --recursive'")
+        error(u"or disable encryption with --no-encrypt parameter.")
+        sys.exit(1)
 
-        remote_list = fetch_remote_list(destination_base, recursive = True, require_attribs = True)
+    ## Normalize URI to convert s3://bkt to s3://bkt/ (trailing slash)
+    destination_base_uri = S3Uri(args[-1])
+    if destination_base_uri.type != 's3':
+        raise ParameterError("Destination must be S3Uri. Got: %s" % destination_base_uri)
+    destination_base = str(destination_base_uri)
 
-        local_count = len(local_list)
-        remote_count = len(remote_list)
+    local_list, single_file_local = fetch_local_list(args[:-1], recursive = True)
+    remote_list = fetch_remote_list(destination_base, recursive = True, require_attribs = True)
 
-        info(u"Found %d local files, %d remote files" % (local_count, remote_count))
+    local_count = len(local_list)
+    remote_count = len(remote_list)
 
-        local_list, exclude_list = filter_exclude_include(local_list)
+    info(u"Found %d local files, %d remote files" % (local_count, remote_count))
 
-        if single_file_local and len(local_list) == 1 and len(remote_list) == 1:
-            ## Make remote_key same as local_key for comparison if we're dealing with only one file
-            remote_list_entry = remote_list[remote_list.keys()[0]]
-            # Flush remote_list, by the way
-            remote_list = FileDict()
-            remote_list[local_list.keys()[0]] =  remote_list_entry
+    local_list, exclude_list = filter_exclude_include(local_list)
 
-        local_list, remote_list, update_list, copy_pairs = compare_filelists(local_list, remote_list, src_remote = False, dst_remote = True, delay_updates = cfg.delay_updates)
+    if single_file_local and len(local_list) == 1 and len(remote_list) == 1:
+        ## Make remote_key same as local_key for comparison if we're dealing with only one file
+        remote_list_entry = remote_list[remote_list.keys()[0]]
+        # Flush remote_list, by the way
+        remote_list = { local_list.keys()[0] : remote_list_entry }
 
-        local_count = len(local_list)
-        update_count = len(update_list)
-        copy_count = len(copy_pairs)
-        remote_count = len(remote_list)
+    local_list, remote_list, existing_list = compare_filelists(local_list, remote_list, src_remote = False, dst_remote = True)
 
-        info(u"Summary: %d local files to upload, %d files to remote copy, %d remote files to delete" % (local_count + update_count, copy_count, remote_count))
+    local_count = len(local_list)
+    remote_count = len(remote_list)
 
-        _set_remote_uri(local_list, destination_base, single_file_local)
-        _set_remote_uri(update_list, destination_base, single_file_local)
+    info(u"Summary: %d local files to upload, %d remote files to delete" % (local_count, remote_count))
 
-        if cfg.dry_run:
-            for key in exclude_list:
-                output(u"exclude: %s" % unicodise(key))
+    if local_count > 0:
+        ## Populate 'remote_uri' only if we've got something to upload
+        if not destination_base.endswith("/"):
+            if not single_file_local:
+                raise ParameterError("Destination S3 URI must end with '/' (ie must refer to a directory on the remote side).")
+            local_list[local_list.keys()[0]]['remote_uri'] = unicodise(destination_base)
+        else:
             for key in local_list:
-                output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], local_list[key]['remote_uri']))
-            for key in update_list:
-                output(u"upload: %s -> %s" % (update_list[key]['full_name_unicode'], update_list[key]['remote_uri']))
-            for (src_obj, dst1, dst2) in copy_pairs:
-                output(u"remote copy: %s -> %s" % (dst1, dst2))
-            if cfg.delete_removed:
-                for key in remote_list:
-                    output(u"delete: %s" % remote_list[key]['object_uri_str'])
-
-            warning(u"Exiting now because of --dry-run")
-            return
+                local_list[key]['remote_uri'] = unicodise(destination_base + key)
 
-        # if there are copy pairs, we can't do delete_before, on the chance
-        # we need one of the to-be-deleted files as a copy source.
-        if len(copy_pairs) > 0:
-            cfg.delete_after = True
-
-        if cfg.delete_removed and not cfg.delete_after:
-            _do_deletes(s3, remote_list)
-
-        total_size = 0
-        total_elapsed = 0.0
-        timestamp_start = time.time()
-        n, total_size = _upload(local_list, 0, local_count, total_size)
-        n, total_size = _upload(update_list, n, local_count, total_size)
-        n_copies, saved_bytes = remote_copy(s3, copy_pairs, destination_base)
-        if cfg.delete_removed and cfg.delete_after:
-            _do_deletes(s3, remote_list)
-        total_elapsed = time.time() - timestamp_start
-        total_speed = total_elapsed and total_size/total_elapsed or 0.0
-        speed_fmt = formatSize(total_speed, human_readable = True, floating_point = True)
-
-        # Only print out the result if any work has been done or
-        # if the user asked for verbose output
-        outstr = "Done. Uploaded %d bytes in %0.1f seconds, %0.2f %sB/s.  Copied %d files saving %d bytes transfer." % (total_size, total_elapsed, speed_fmt[0], speed_fmt[1], n_copies, saved_bytes)
-        if total_size + saved_bytes > 0:
-            output(outstr)
-        else:
-            info(outstr)
+    if cfg.dry_run:
+        for key in exclude_list:
+            output(u"exclude: %s" % unicodise(key))
+        if cfg.delete_removed:
+            for key in remote_list:
+                output(u"delete: %s" % remote_list[key]['object_uri_str'])
+        for key in local_list:
+            output(u"upload: %s -> %s" % (local_list[key]['full_name_unicode'], local_list[key]['remote_uri']))
 
+        warning(u"Exitting now because of --dry-run")
         return
 
-    def _invalidate_on_cf(destination_base_uri):
-        cf = CloudFront(cfg)
-        default_index_file = None
-        if cfg.invalidate_default_index_on_cf or cfg.invalidate_default_index_root_on_cf:
-            info_response = s3.website_info(destination_base_uri, cfg.bucket_location)
-            if info_response:
-              default_index_file = info_response['index_document']
-              if len(default_index_file) < 1:
-                  default_index_file = None
-
-        result = cf.InvalidateObjects(destination_base_uri, uploaded_objects_list, default_index_file, cfg.invalidate_default_index_on_cf, cfg.invalidate_default_index_root_on_cf)
-        if result['status'] == 201:
-            output("Created invalidation request for %d paths" % len(uploaded_objects_list))
-            output("Check progress with: s3cmd cfinvalinfo cf://%s/%s" % (result['dist_id'], result['request_id']))
-
+    if cfg.delete_removed:
+        for key in remote_list:
+            uri = S3Uri(remote_list[key]['object_uri_str'])
+            s3.object_delete(uri)
+            output(u"deleted: '%s'" % uri)
 
-    # main execution
-    s3 = S3(cfg)
     uploaded_objects_list = []
+    total_size = 0
+    total_elapsed = 0.0
+    timestamp_start = time.time()
+    seq = 0
+    file_list = local_list.keys()
+    file_list.sort()
+    for file in file_list:
+        seq += 1
+        item = local_list[file]
+        src = item['full_name']
+        uri = S3Uri(item['remote_uri'])
+        seq_label = "[%d of %d]" % (seq, local_count)
+        extra_headers = copy(cfg.extra_headers)
+        try:
+            if cfg.preserve_attrs:
+                attr_header = _build_attr_header(src)
+                debug(u"attr_header: %s" % attr_header)
+                extra_headers.update(attr_header)
+            response = s3.object_put(src, uri, extra_headers, extra_label = seq_label)
+        except InvalidFileError, e:
+            warning(u"File can not be uploaded: %s" % e)
+            continue
+        except S3UploadError, e:
+            error(u"%s: upload failed too many times. Skipping that file." % item['full_name_unicode'])
+            continue
+        speed_fmt = formatSize(response["speed"], human_readable = True, floating_point = True)
+        if not cfg.progress_meter:
+            output(u"File '%s' stored as '%s' (%d bytes in %0.1f seconds, %0.2f %sB/s) %s" %
+                (item['full_name_unicode'], uri, response["size"], response["elapsed"],
+                speed_fmt[0], speed_fmt[1], seq_label))
+        total_size += response["size"]
+        uploaded_objects_list.append(uri.object())
 
-    if cfg.encrypt:
-        error(u"S3cmd 'sync' doesn't yet support GPG encryption, sorry.")
-        error(u"Either use unconditional 's3cmd put --recursive'")
-        error(u"or disable encryption with --no-encrypt parameter.")
-        sys.exit(1)
-
-    local_list, single_file_local = fetch_local_list(args[:-1], recursive = True)
-
-    destinations = [args[-1]]
-    if cfg.additional_destinations:
-        destinations = destinations + cfg.additional_destinations
+    total_elapsed = time.time() - timestamp_start
+    total_speed = total_elapsed and total_size/total_elapsed or 0.0
+    speed_fmt = formatSize(total_speed, human_readable = True, floating_point = True)
 
-    if 'fork' not in os.__all__ or len(destinations) < 2:
-        destination_base_uri = _single_process(local_list)
-        if cfg.invalidate_on_cf:
-            if len(uploaded_objects_list) == 0:
-                info("Nothing to invalidate in CloudFront")
-            else:
-                _invalidate_on_cf(destination_base_uri)
+    # Only print out the result if any work has been done or
+    # if the user asked for verbose output
+    outstr = "Done. Uploaded %d bytes in %0.1f seconds, %0.2f %sB/s" % (total_size, total_elapsed, speed_fmt[0], speed_fmt[1])
+    if total_size > 0:
+        output(outstr)
     else:
-        _parent()
-        if cfg.invalidate_on_cf:
-            error(u"You cannot use both --cf-invalidate and --add-destination.")
+        info(outstr)
+
+    if cfg.invalidate_on_cf:
+        if len(uploaded_objects_list) == 0:
+            info("Nothing to invalidate in CloudFront")
+        else:
+            # 'uri' from the last iteration is still valid at this point
+            cf = CloudFront(cfg)
+            result = cf.InvalidateObjects(uri, uploaded_objects_list)
+            if result['status'] == 201:
+                output("Created invalidation request for %d paths" % len(uploaded_objects_list))
+                output("Check progress with: s3cmd cfinvalinfo cf://%s/%s" % (result['dist_id'], result['request_id']))
 
 def cmd_sync(args):
     if (len(args) < 2):
@@ -1196,6 +970,45 @@ def cmd_sync(args):
     raise ParameterError("Invalid source/destination: '%s'" % "' '".join(args))
 
 def cmd_setacl(args):
+    def _update_acl(uri, seq_label = ""):
+        something_changed = False
+        acl = s3.get_acl(uri)
+        debug(u"acl: %s - %r" % (uri, acl.grantees))
+        if cfg.acl_public == True:
+            if acl.isAnonRead():
+                info(u"%s: already Public, skipping %s" % (uri, seq_label))
+            else:
+                acl.grantAnonRead()
+                something_changed = True
+        elif cfg.acl_public == False: # we explicitely check for False, because it could be None
+            if not acl.isAnonRead():
+                info(u"%s: already Private, skipping %s" % (uri, seq_label))
+            else:
+                acl.revokeAnonRead()
+                something_changed = True
+
+        # update acl with arguments
+        # grant first and revoke later, because revoke has priority
+        if cfg.acl_grants:
+            something_changed = True
+            for grant in cfg.acl_grants:
+                acl.grant(**grant);
+
+        if cfg.acl_revokes:
+            something_changed = True
+            for revoke in cfg.acl_revokes:
+                acl.revoke(**revoke);
+
+        if not something_changed:
+            return
+
+        retsponse = s3.set_acl(uri, acl)
+        if retsponse['status'] == 200:
+            if cfg.acl_public in (True, False):
+                output(u"%s: ACL set to %s  %s" % (uri, set_to_acl, seq_label))
+            else:
+                output(u"%s: ACL updated" % uri)
+
     s3 = S3(cfg)
 
     set_to_acl = cfg.acl_public and "Public" or "Private"
@@ -1211,7 +1024,7 @@ def cmd_setacl(args):
                 else:
                     info("Setting bucket-level ACL for %s" % (uri.uri()))
                 if not cfg.dry_run:
-                    update_acl(s3, uri)
+                    _update_acl(uri)
             else:
                 args.append(arg)
 
@@ -1228,7 +1041,7 @@ def cmd_setacl(args):
         for key in remote_list:
             output(u"setacl: %s" % remote_list[key]['object_uri_str'])
 
-        warning(u"Exiting now because of --dry-run")
+        warning(u"Exitting now because of --dry-run")
         return
 
     seq = 0
@@ -1236,34 +1049,7 @@ def cmd_setacl(args):
         seq += 1
         seq_label = "[%d of %d]" % (seq, remote_count)
         uri = S3Uri(remote_list[key]['object_uri_str'])
-        update_acl(s3, uri, seq_label)
-
-def cmd_setpolicy(args):
-    s3 = S3(cfg)
-    uri = S3Uri(args[1])
-    policy_file = args[0]
-    policy = open(policy_file, 'r').read()
-
-    if cfg.dry_run: return
-
-    response = s3.set_policy(uri, policy)
-
-    #if retsponse['status'] == 200:
-    debug(u"response - %s" % response['status'])
-    if response['status'] == 204:
-        output(u"%s: Policy updated" % uri)
-
-def cmd_delpolicy(args):
-    s3 = S3(cfg)
-    uri = S3Uri(args[0])
-    if cfg.dry_run: return
-
-    response = s3.delete_policy(uri)
-
-    #if retsponse['status'] == 200:
-    debug(u"response - %s" % response['status'])
-    output(u"%s: Policy deleted" % uri)
-
+        _update_acl(uri, seq_label)
 
 def cmd_accesslog(args):
     s3 = S3(cfg)
@@ -1292,15 +1078,6 @@ def cmd_sign(args):
     signature = Utils.sign_string(string_to_sign)
     output("Signature: %s" % signature)
 
-def cmd_signurl(args):
-    expiry = args.pop()
-    url_to_sign = S3Uri(args.pop())
-    if url_to_sign.type != 's3':
-        raise ParameterError("Must be S3Uri. Got: %s" % url_to_sign)
-    debug("url to sign: %r" % url_to_sign)
-    signed_url = Utils.sign_url(url_to_sign, expiry)
-    output(signed_url)
-
 def cmd_fixbucket(args):
     def _unescape(text):
         ##
@@ -1615,13 +1392,8 @@ def get_commands_list():
     {"cmd":"cp", "label":"Copy object", "param":"s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func":cmd_cp, "argc":2},
     {"cmd":"mv", "label":"Move object", "param":"s3://BUCKET1/OBJECT1 s3://BUCKET2[/OBJECT2]", "func":cmd_mv, "argc":2},
     {"cmd":"setacl", "label":"Modify Access control list for Bucket or Files", "param":"s3://BUCKET[/OBJECT]", "func":cmd_setacl, "argc":1},
-
-    {"cmd":"setpolicy", "label":"Modify Bucket Policy", "param":"FILE s3://BUCKET", "func":cmd_setpolicy, "argc":2},
-    {"cmd":"delpolicy", "label":"Delete Bucket Policy", "param":"s3://BUCKET", "func":cmd_delpolicy, "argc":1},
-
     {"cmd":"accesslog", "label":"Enable/disable bucket access logging", "param":"s3://BUCKET", "func":cmd_accesslog, "argc":1},
     {"cmd":"sign", "label":"Sign arbitrary string using the secret key", "param":"STRING-TO-SIGN", "func":cmd_sign, "argc":1},
-    {"cmd":"signurl", "label":"Sign an S3 URL to provide limited public access with expiry", "param":"s3://BUCKET/OBJECT expiry_epoch", "func":cmd_signurl, "argc":2},
     {"cmd":"fixbucket", "label":"Fix invalid file names in a bucket", "param":"s3://BUCKET[/PREFIX]", "func":cmd_fixbucket, "argc":1},
 
     ## Website commands
@@ -1645,47 +1417,6 @@ def format_commands(progname, commands_list):
         help += "  %s\n      %s %s %s\n" % (cmd["label"], progname, cmd["cmd"], cmd["param"])
     return help
 
-
-def update_acl(s3, uri, seq_label=""):
-    something_changed = False
-    acl = s3.get_acl(uri)
-    debug(u"acl: %s - %r" % (uri, acl.grantees))
-    if cfg.acl_public == True:
-        if acl.isAnonRead():
-            info(u"%s: already Public, skipping %s" % (uri, seq_label))
-        else:
-            acl.grantAnonRead()
-            something_changed = True
-    elif cfg.acl_public == False:  # we explicitely check for False, because it could be None
-        if not acl.isAnonRead():
-            info(u"%s: already Private, skipping %s" % (uri, seq_label))
-        else:
-            acl.revokeAnonRead()
-            something_changed = True
-
-    # update acl with arguments
-    # grant first and revoke later, because revoke has priority
-    if cfg.acl_grants:
-        something_changed = True
-        for grant in cfg.acl_grants:
-            acl.grant(**grant)
-
-    if cfg.acl_revokes:
-        something_changed = True
-        for revoke in cfg.acl_revokes:
-            acl.revoke(**revoke)
-
-    if not something_changed:
-        return
-
-    retsponse = s3.set_acl(uri, acl)
-    if retsponse['status'] == 200:
-        if cfg.acl_public in (True, False):
-            set_to_acl = cfg.acl_public and "Public" or "Private"
-            output(u"%s: ACL set to %s  %s" % (uri, set_to_acl, seq_label))
-        else:
-            output(u"%s: ACL updated" % uri)
-
 class OptionMimeType(Option):
     def check_mimetype(option, opt, value):
         if re.compile("^[a-z0-9]+/[a-z0-9+\.-]+(;.*)?$", re.IGNORECASE).match(value):
@@ -1750,8 +1481,6 @@ def main():
     optparser.add_option(      "--configure", dest="run_configure", action="store_true", help="Invoke interactive (re)configuration tool. Optionally use as '--configure s3://come-bucket' to test access to a specific bucket instead of attempting to list them all.")
     optparser.add_option("-c", "--config", dest="config", metavar="FILE", help="Config file name. Defaults to %default")
     optparser.add_option(      "--dump-config", dest="dump_config", action="store_true", help="Dump current configuration after parsing config files and command line options and exit.")
-    optparser.add_option(      "--access_key", dest="access_key", help="AWS Access Key")
-    optparser.add_option(      "--secret_key", dest="secret_key", help="AWS Secret Key")
 
     optparser.add_option("-n", "--dry-run", dest="dry_run", action="store_true", help="Only show what should be uploaded or downloaded but don't actually do it. May still perform S3 requests to get bucket listings and other information though (only for file transfer commands)")
 
@@ -1770,10 +1499,6 @@ def main():
 
     optparser.add_option(      "--delete-removed", dest="delete_removed", action="store_true", help="Delete remote objects with no corresponding local file [sync]")
     optparser.add_option(      "--no-delete-removed", dest="delete_removed", action="store_false", help="Don't delete remote objects.")
-    optparser.add_option(      "--delete-after", dest="delete_after", action="store_true", help="Perform deletes after new uploads [sync]")
-    optparser.add_option(      "--delay-updates", dest="delay_updates", action="store_true", help="Put all updated files into place at end [sync]")
-    optparser.add_option(      "--add-destination", dest="additional_destinations", action="append", help="Additional destination for parallel uploads, in addition to last arg.  May be repeated.")
-    optparser.add_option(      "--delete-after-fetch", dest="delete_after_fetch", action="store_true", help="Delete remote objects after fetching to local file (only for [get] and [sync] commands).")
     optparser.add_option("-p", "--preserve", dest="preserve_attrs", action="store_true", help="Preserve filesystem attributes (mode, ownership, timestamps). Default for [sync] command.")
     optparser.add_option(      "--no-preserve", dest="preserve_attrs", action="store_false", help="Don't store FS attributes")
     optparser.add_option(      "--exclude", dest="exclude", action="append", metavar="GLOB", help="Filenames and paths matching GLOB will be excluded from sync")
@@ -1785,21 +1510,20 @@ def main():
     optparser.add_option(      "--rinclude", dest="rinclude", action="append", metavar="REGEXP", help="Same as --include but uses REGEXP (regular expression) instead of GLOB")
     optparser.add_option(      "--rinclude-from", dest="rinclude_from", action="append", metavar="FILE", help="Read --rinclude REGEXPs from FILE")
 
-    optparser.add_option(      "--bucket-location", dest="bucket_location", help="Datacentre to create bucket in. As of now the datacenters are: US (default), EU, ap-northeast-1, ap-southeast-1, sa-east-1, us-west-1 and us-west-2")
+    optparser.add_option(      "--bucket-location", dest="bucket_location", help="Datacentre to create bucket in. As of now the datacenters are: US (default), EU, us-west-1, and ap-southeast-1")
     optparser.add_option(      "--reduced-redundancy", "--rr", dest="reduced_redundancy", action="store_true", help="Store object with 'Reduced redundancy'. Lower per-GB price. [put, cp, mv]")
 
     optparser.add_option(      "--access-logging-target-prefix", dest="log_target_prefix", help="Target prefix for access logs (S3 URI) (for [cfmodify] and [accesslog] commands)")
     optparser.add_option(      "--no-access-logging", dest="log_target_prefix", action="store_false", help="Disable access logging (for [cfmodify] and [accesslog] commands)")
 
     optparser.add_option(      "--default-mime-type", dest="default_mime_type", action="store_true", help="Default MIME-type for stored objects. Application default is binary/octet-stream.")
-    optparser.add_option("-M", "--guess-mime-type", dest="guess_mime_type", action="store_true", help="Guess MIME-type of files by their extension or mime magic. Fall back to default MIME-Type as specified by --default-mime-type option")
+    optparser.add_option(      "--guess-mime-type", dest="guess_mime_type", action="store_true", help="Guess MIME-type of files by their extension or mime magic. Fall back to default MIME-Type as specified by --default-mime-type option")
     optparser.add_option(      "--no-guess-mime-type", dest="guess_mime_type", action="store_false", help="Don't guess MIME-type and use the default type instead.")
     optparser.add_option("-m", "--mime-type", dest="mime_type", type="mimetype", metavar="MIME/TYPE", help="Force MIME-type. Override both --default-mime-type and --guess-mime-type.")
 
     optparser.add_option(      "--add-header", dest="add_header", action="append", metavar="NAME:VALUE", help="Add a given HTTP header to the upload request. Can be used multiple times. For instance set 'Expires' or 'Cache-Control' headers (or both) using this options if you like.")
 
     optparser.add_option(      "--encoding", dest="encoding", metavar="ENCODING", help="Override autodetected terminal and filesystem encoding (character set). Autodetected: %s" % preferred_encoding)
-    optparser.add_option(      "--add-encoding-exts", dest="add_encoding_exts", metavar="EXTENSIONs", help="Add encoding to these comma delimited extensions i.e. (css,js,html) when uploading to S3 )")
     optparser.add_option(      "--verbatim", dest="urlencoding_mode", action="store_const", const="verbatim", help="Use the S3 name as given on the command line. No pre-processing, encoding, etc. Use with caution!")
 
     optparser.add_option(      "--disable-multipart", dest="enable_multipart", action="store_false", help="Disable multipart upload on files bigger than --multipart-chunk-size-mb")
@@ -1816,10 +1540,6 @@ def main():
     optparser.add_option(      "--enable", dest="enable", action="store_true", help="Enable given CloudFront distribution (only for [cfmodify] command)")
     optparser.add_option(      "--disable", dest="enable", action="store_false", help="Enable given CloudFront distribution (only for [cfmodify] command)")
     optparser.add_option(      "--cf-invalidate", dest="invalidate_on_cf", action="store_true", help="Invalidate the uploaded filed in CloudFront. Also see [cfinval] command.")
-    # joseprio: adding options to invalidate the default index and the default
-    # index root
-    optparser.add_option(      "--cf-invalidate-default-index", dest="invalidate_default_index_on_cf", action="store_true", help="When using Custom Origin and S3 static website, invalidate the default index file.")
-    optparser.add_option(      "--cf-no-invalidate-default-index-root", dest="invalidate_default_index_root_on_cf", action="store_false", help="When using Custom Origin and S3 static website, don't invalidate the path to the default index file.")
     optparser.add_option(      "--cf-add-cname", dest="cf_cnames_add", action="append", metavar="CNAME", help="Add given CNAME to a CloudFront distribution (only for [cfcreate] and [cfmodify] commands)")
     optparser.add_option(      "--cf-remove-cname", dest="cf_cnames_remove", action="append", metavar="CNAME", help="Remove given CNAME from a CloudFront distribution (only for [cfmodify] command)")
     optparser.add_option(      "--cf-comment", dest="cf_comment", action="store", metavar="COMMENT", help="Set COMMENT for a given CloudFront distribution (only for [cfcreate] and [cfmodify] commands)")
@@ -1828,8 +1548,6 @@ def main():
     optparser.add_option("-d", "--debug", dest="verbosity", action="store_const", const=logging.DEBUG, help="Enable debug output.")
     optparser.add_option(      "--version", dest="show_version", action="store_true", help="Show s3cmd version (%s) and exit." % (PkgInfo.version))
     optparser.add_option("-F", "--follow-symlinks", dest="follow_symlinks", action="store_true", default=False, help="Follow symbolic links as if they are regular files")
-    optparser.add_option(      "--cache-file", dest="cache_file", action="store", default="",  metavar="FILE", help="Cache FILE containing local source MD5 values")
-    optparser.add_option("-q", "--quiet", dest="quiet", action="store_true", default=False, help="Silence output on stdout")
 
     optparser.set_usage(optparser.usage + " COMMAND [parameters]")
     optparser.set_description('S3cmd is a tool for managing objects in '+
@@ -1852,14 +1570,6 @@ def main():
         output(u"s3cmd version %s" % PkgInfo.version)
         sys.exit(0)
 
-    if options.quiet:
-        try:
-            f = open("/dev/null", "w")
-            sys.stdout.close()
-            sys.stdout = f
-        except IOError:
-            warning(u"Unable to open /dev/null: --quiet disabled.")
-
     ## Now finally parse the config file
     if not options.config:
         error(u"Can't find a config file. Please use --config option.")
@@ -1964,9 +1674,6 @@ def main():
             ## Some CloudFront.Cmd.Options() options are not settable from command line
             pass
 
-    if options.additional_destinations:
-        cfg.additional_destinations = options.additional_destinations
-
     ## Set output and filesystem encoding for printing out filenames.
     sys.stdout = codecs.getwriter(cfg.encoding)(sys.stdout, "replace")
     sys.stderr = codecs.getwriter(cfg.encoding)(sys.stderr, "replace")
@@ -2027,7 +1734,7 @@ def main():
         sys.exit(1)
 
     if len(args) < commands[command]["argc"]:
-        error(u"Not enough parameters for command '%s'" % command)
+        error(u"Not enough paramters for command '%s'" % command)
         sys.exit(1)
 
     try:
@@ -2081,7 +1788,6 @@ if __name__ == '__main__':
         from S3.S3 import S3
         from S3.Config import Config
         from S3.SortedDict import SortedDict
-        from S3.FileDict import FileDict
         from S3.S3Uri import S3Uri
         from S3 import Utils
         from S3.Utils import *