s3_cmd_bin 0.0.1

data/resources/S3/MultiPart.py

@@ -0,0 +1,137 @@
+## Amazon S3 Multipart upload support
+## Author: Jerome Leclanche <jerome.leclanche@gmail.com>
+## License: GPL Version 2
+
+import os
+from stat import ST_SIZE
+from logging import debug, info, warning, error
+from Utils import getTextFromXml, formatSize, unicodise
+from Exceptions import S3UploadError
+
+class MultiPartUpload(object):
+
+    MIN_CHUNK_SIZE_MB = 5       # 5MB
+    MAX_CHUNK_SIZE_MB = 5120    # 5GB
+    MAX_FILE_SIZE = 42949672960 # 5TB
+
+    def __init__(self, s3, file, uri, headers_baseline = {}):
+        self.s3 = s3
+        self.file = file
+        self.uri = uri
+        self.parts = {}
+        self.headers_baseline = headers_baseline
+        self.upload_id = self.initiate_multipart_upload()
+
+    def initiate_multipart_upload(self):
+        """
+        Begin a multipart upload
+        http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadInitiate.html
+        """
+        request = self.s3.create_request("OBJECT_POST", uri = self.uri, headers = self.headers_baseline, extra = "?uploads")
+        response = self.s3.send_request(request)
+        data = response["data"]
+        self.upload_id = getTextFromXml(data, "UploadId")
+        return self.upload_id
+
+    def upload_all_parts(self):
+        """
+        Execute a full multipart upload on a file
+        Returns the seq/etag dict
+        TODO use num_processes to thread it
+        """
+        if not self.upload_id:
+            raise RuntimeError("Attempting to use a multipart upload that has not been initiated.")
+
+        self.chunk_size = self.s3.config.multipart_chunk_size_mb * 1024 * 1024
+
+        if self.file.name != "<stdin>":
+                size_left = file_size = os.stat(self.file.name)[ST_SIZE]
+                nr_parts = file_size / self.chunk_size + (file_size % self.chunk_size and 1)
+                debug("MultiPart: Uploading %s in %d parts" % (self.file.name, nr_parts))
+        else:
+            debug("MultiPart: Uploading from %s" % (self.file.name))
+
+        seq = 1
+        if self.file.name != "<stdin>":
+            while size_left > 0:
+                offset = self.chunk_size * (seq - 1)
+                current_chunk_size = min(file_size - offset, self.chunk_size)
+                size_left -= current_chunk_size
+                labels = {
+                    'source' : unicodise(self.file.name),
+                    'destination' : unicodise(self.uri.uri()),
+                    'extra' : "[part %d of %d, %s]" % (seq, nr_parts, "%d%sB" % formatSize(current_chunk_size, human_readable = True))
+                }
+                try:
+                    self.upload_part(seq, offset, current_chunk_size, labels)
+                except:
+                    error(u"Upload of '%s' part %d failed. Aborting multipart upload." % (self.file.name, seq))
+                    self.abort_upload()
+                    raise
+                seq += 1
+        else:
+            while True:
+                buffer = self.file.read(self.chunk_size)
+                offset = self.chunk_size * (seq - 1)
+                current_chunk_size = len(buffer)
+                labels = {
+                    'source' : unicodise(self.file.name),
+                    'destination' : unicodise(self.uri.uri()),
+                    'extra' : "[part %d, %s]" % (seq, "%d%sB" % formatSize(current_chunk_size, human_readable = True))
+                }
+                if len(buffer) == 0: # EOF
+                    break
+                try:
+                    self.upload_part(seq, offset, current_chunk_size, labels, buffer)
+                except:
+                    error(u"Upload of '%s' part %d failed. Aborting multipart upload." % (self.file.name, seq))
+                    self.abort_upload()
+                    raise
+                seq += 1
+
+        debug("MultiPart: Upload finished: %d parts", seq - 1)
+
+    def upload_part(self, seq, offset, chunk_size, labels, buffer = ''):
+        """
+        Upload a file chunk
+        http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadUploadPart.html
+        """
+        # TODO implement Content-MD5
+        debug("Uploading part %i of %r (%s bytes)" % (seq, self.upload_id, chunk_size))
+        headers = { "content-length": chunk_size }
+        query_string = "?partNumber=%i&uploadId=%s" % (seq, self.upload_id)
+        request = self.s3.create_request("OBJECT_PUT", uri = self.uri, headers = headers, extra = query_string)
+        response = self.s3.send_file(request, self.file, labels, buffer, offset = offset, chunk_size = chunk_size)
+        self.parts[seq] = response["headers"]["etag"]
+        return response
+
+    def complete_multipart_upload(self):
+        """
+        Finish a multipart upload
+        http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadComplete.html
+        """
+        debug("MultiPart: Completing upload: %s" % self.upload_id)
+
+        parts_xml = []
+        part_xml = "<Part><PartNumber>%i</PartNumber><ETag>%s</ETag></Part>"
+        for seq, etag in self.parts.items():
+            parts_xml.append(part_xml % (seq, etag))
+        body = "<CompleteMultipartUpload>%s</CompleteMultipartUpload>" % ("".join(parts_xml))
+
+        headers = { "content-length": len(body) }
+        request = self.s3.create_request("OBJECT_POST", uri = self.uri, headers = headers, extra = "?uploadId=%s" % (self.upload_id))
+        response = self.s3.send_request(request, body = body)
+
+        return response
+
+    def abort_upload(self):
+        """
+        Abort multipart upload
+        http://docs.amazonwebservices.com/AmazonS3/latest/API/index.html?mpUploadAbort.html
+        """
+        debug("MultiPart: Aborting upload: %s" % self.upload_id)
+        request = self.s3.create_request("OBJECT_DELETE", uri = self.uri, extra = "?uploadId=%s" % (self.upload_id))
+        response = self.s3.send_request(request)
+        return response
+
+# vim:et:ts=4:sts=4:ai

data/resources/S3/PkgInfo.py

@@ -0,0 +1,14 @@
+package = "s3cmd"
+version = "1.5.0-alpha3"
+url = "http://s3tools.org"
+license = "GPL version 2"
+short_description = "Command line tool for managing Amazon S3 and CloudFront services"
+long_description = """
+S3cmd lets you copy files from/to Amazon S3
+(Simple Storage Service) using a simple to use
+command line client. Supports rsync-like backup,
+GPG encryption, and more. Also supports management
+of Amazon's CloudFront content delivery network.
+"""
+
+# vim:et:ts=4:sts=4:ai

data/resources/S3/Progress.py

@@ -0,0 +1,173 @@
+## Amazon S3 manager
+## Author: Michal Ludvig <michal@logix.cz>
+##         http://www.logix.cz/michal
+## License: GPL Version 2
+
+import sys
+import datetime
+import time
+import Utils
+
+class Progress(object):
+    _stdout = sys.stdout
+    _last_display = 0
+
+    def __init__(self, labels, total_size):
+        self._stdout = sys.stdout
+        self.new_file(labels, total_size)
+
+    def new_file(self, labels, total_size):
+        self.labels = labels
+        self.total_size = total_size
+        # Set initial_position to something in the
+        # case we're not counting from 0. For instance
+        # when appending to a partially downloaded file.
+        # Setting initial_position will let the speed
+        # be computed right.
+        self.initial_position = 0
+        self.current_position = self.initial_position
+        self.time_start = datetime.datetime.now()
+        self.time_last = self.time_start
+        self.time_current = self.time_start
+
+        self.display(new_file = True)
+
+    def update(self, current_position = -1, delta_position = -1):
+        self.time_last = self.time_current
+        self.time_current = datetime.datetime.now()
+        if current_position > -1:
+            self.current_position = current_position
+        elif delta_position > -1:
+            self.current_position += delta_position
+        #else:
+        #   no update, just call display()
+        self.display()
+
+    def done(self, message):
+        self.display(done_message = message)
+
+    def output_labels(self):
+        self._stdout.write(u"%(source)s -> %(destination)s  %(extra)s\n" % self.labels)
+        self._stdout.flush()
+
+    def _display_needed(self):
+        # We only need to update the display every so often.
+        if time.time() - self._last_display > 1:
+            self._last_display = time.time()
+            return True
+        return False
+
+    def display(self, new_file = False, done_message = None):
+        """
+        display(new_file = False[/True], done = False[/True])
+
+        Override this method to provide a nicer output.
+        """
+        if new_file:
+            self.output_labels()
+            self.last_milestone = 0
+            return
+
+        if self.current_position == self.total_size:
+            print_size = Utils.formatSize(self.current_position, True)
+            if print_size[1] != "": print_size[1] += "B"
+            timedelta = self.time_current - self.time_start
+            sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0
+            print_speed = Utils.formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True)
+            self._stdout.write("100%%  %s%s in %.2fs (%.2f %sB/s)\n" %
+                (print_size[0], print_size[1], sec_elapsed, print_speed[0], print_speed[1]))
+            self._stdout.flush()
+            return
+
+        rel_position = selfself.current_position * 100 / self.total_size
+        if rel_position >= self.last_milestone:
+            self.last_milestone = (int(rel_position) / 5) * 5
+            self._stdout.write("%d%% ", self.last_milestone)
+            self._stdout.flush()
+            return
+
+class ProgressANSI(Progress):
+    ## http://en.wikipedia.org/wiki/ANSI_escape_code
+    SCI = '\x1b['
+    ANSI_hide_cursor = SCI + "?25l"
+    ANSI_show_cursor = SCI + "?25h"
+    ANSI_save_cursor_pos = SCI + "s"
+    ANSI_restore_cursor_pos = SCI + "u"
+    ANSI_move_cursor_to_column = SCI + "%uG"
+    ANSI_erase_to_eol = SCI + "0K"
+    ANSI_erase_current_line = SCI + "2K"
+
+    def display(self, new_file = False, done_message = None):
+        """
+        display(new_file = False[/True], done_message = None)
+        """
+        if new_file:
+            self.output_labels()
+            self._stdout.write(self.ANSI_save_cursor_pos)
+            self._stdout.flush()
+            return
+
+        # Only display progress every so often
+        if not (new_file or done_message) and not self._display_needed():
+            return
+
+        timedelta = self.time_current - self.time_start
+        sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0
+        if (sec_elapsed > 0):
+            print_speed = Utils.formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True)
+        else:
+            print_speed = (0, "")
+        self._stdout.write(self.ANSI_restore_cursor_pos)
+        self._stdout.write(self.ANSI_erase_to_eol)
+        self._stdout.write("%(current)s of %(total)s   %(percent)3d%% in %(elapsed)ds  %(speed).2f %(speed_coeff)sB/s" % {
+            "current" : str(self.current_position).rjust(len(str(self.total_size))),
+            "total" : self.total_size,
+            "percent" : self.total_size and (self.current_position * 100 / self.total_size) or 0,
+            "elapsed" : sec_elapsed,
+            "speed" : print_speed[0],
+            "speed_coeff" : print_speed[1]
+        })
+
+        if done_message:
+            self._stdout.write("  %s\n" % done_message)
+
+        self._stdout.flush()
+
+class ProgressCR(Progress):
+    ## Uses CR char (Carriage Return) just like other progress bars do.
+    CR_char = chr(13)
+
+    def display(self, new_file = False, done_message = None):
+        """
+        display(new_file = False[/True], done_message = None)
+        """
+        if new_file:
+            self.output_labels()
+            return
+
+        # Only display progress every so often
+        if not (new_file or done_message) and not self._display_needed():
+            return
+
+        timedelta = self.time_current - self.time_start
+        sec_elapsed = timedelta.days * 86400 + timedelta.seconds + float(timedelta.microseconds)/1000000.0
+        if (sec_elapsed > 0):
+            print_speed = Utils.formatSize((self.current_position - self.initial_position) / sec_elapsed, True, True)
+        else:
+            print_speed = (0, "")
+        self._stdout.write(self.CR_char)
+        output = " %(current)s of %(total)s   %(percent)3d%% in %(elapsed)4ds  %(speed)7.2f %(speed_coeff)sB/s" % {
+            "current" : str(self.current_position).rjust(len(str(self.total_size))),
+            "total" : self.total_size,
+            "percent" : self.total_size and (self.current_position * 100 / self.total_size) or 0,
+            "elapsed" : sec_elapsed,
+            "speed" : print_speed[0],
+            "speed_coeff" : print_speed[1]
+        }
+        self._stdout.write(output)
+        if done_message:
+            self._stdout.write("  %s\n" % done_message)
+
+        self._stdout.flush()
+
+# vim:et:ts=4:sts=4:ai