gcloud 0.0.7 → 0.1.0

data/packages/gsutil/boto/boto/gs/resumable_upload_handler.py

@@ -1,659 +0,0 @@
-# Copyright 2010 Google Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish, dis-
-# tribute, sublicense, and/or sell copies of the Software, and to permit
-# persons to whom the Software is furnished to do so, subject to the fol-
-# lowing conditions:
-#
-# The above copyright notice and this permission notice shall be included
-# in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
-# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
-# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-
-import cgi
-import errno
-import httplib
-import os
-import random
-import re
-import socket
-import time
-import urlparse
-import boto
-from boto import config, UserAgent
-from boto.connection import AWSAuthConnection
-from boto.exception import InvalidUriError
-from boto.exception import ResumableTransferDisposition
-from boto.exception import ResumableUploadException
-from boto.s3.keyfile import KeyFile
-try:
-    from hashlib import md5
-except ImportError:
-    from md5 import md5
-
-"""
-Handler for Google Cloud Storage resumable uploads. See
-http://code.google.com/apis/storage/docs/developer-guide.html#resumable
-for details.
-
-Resumable uploads will retry failed uploads, resuming at the byte
-count completed by the last upload attempt. If too many retries happen with
-no progress (per configurable num_retries param), the upload will be
-aborted in the current process.
-
-The caller can optionally specify a tracker_file_name param in the
-ResumableUploadHandler constructor. If you do this, that file will
-save the state needed to allow retrying later, in a separate process
-(e.g., in a later run of gsutil).
-"""
-
-
-class ResumableUploadHandler(object):
-
-    BUFFER_SIZE = 8192
-    RETRYABLE_EXCEPTIONS = (httplib.HTTPException, IOError, socket.error,
-                            socket.gaierror)
-
-    # (start, end) response indicating server has nothing (upload protocol uses
-    # inclusive numbering).
-    SERVER_HAS_NOTHING = (0, -1)
-
-    def __init__(self, tracker_file_name=None, num_retries=None):
-        """
-        Constructor. Instantiate once for each uploaded file.
-
-        :type tracker_file_name: string
-        :param tracker_file_name: optional file name to save tracker URI.
-            If supplied and the current process fails the upload, it can be
-            retried in a new process. If called with an existing file containing
-            a valid tracker URI, we'll resume the upload from this URI; else
-            we'll start a new resumable upload (and write the URI to this
-            tracker file).
-
-        :type num_retries: int
-        :param num_retries: the number of times we'll re-try a resumable upload
-            making no progress. (Count resets every time we get progress, so
-            upload can span many more than this number of retries.)
-        """
-        self.tracker_file_name = tracker_file_name
-        self.num_retries = num_retries
-        self.server_has_bytes = 0  # Byte count at last server check.
-        self.tracker_uri = None
-        if tracker_file_name:
-            self._load_tracker_uri_from_file()
-        # Save upload_start_point in instance state so caller can find how
-        # much was transferred by this ResumableUploadHandler (across retries).
-        self.upload_start_point = None
-
-    def _load_tracker_uri_from_file(self):
-        f = None
-        try:
-            f = open(self.tracker_file_name, 'r')
-            uri = f.readline().strip()
-            self._set_tracker_uri(uri)
-        except IOError, e:
-            # Ignore non-existent file (happens first time an upload
-            # is attempted on a file), but warn user for other errors.
-            if e.errno != errno.ENOENT:
-                # Will restart because self.tracker_uri == None.
-                print('Couldn\'t read URI tracker file (%s): %s. Restarting '
-                      'upload from scratch.' %
-                      (self.tracker_file_name, e.strerror))
-        except InvalidUriError, e:
-            # Warn user, but proceed (will restart because
-            # self.tracker_uri == None).
-            print('Invalid tracker URI (%s) found in URI tracker file '
-                  '(%s). Restarting upload from scratch.' %
-                  (uri, self.tracker_file_name))
-        finally:
-            if f:
-                f.close()
-
-    def _save_tracker_uri_to_file(self):
-        """
-        Saves URI to tracker file if one was passed to constructor.
-        """
-        if not self.tracker_file_name:
-            return
-        f = None
-        try:
-            f = open(self.tracker_file_name, 'w')
-            f.write(self.tracker_uri)
-        except IOError, e:
-            raise ResumableUploadException(
-                'Couldn\'t write URI tracker file (%s): %s.\nThis can happen'
-                'if you\'re using an incorrectly configured upload tool\n'
-                '(e.g., gsutil configured to save tracker files to an '
-                'unwritable directory)' %
-                (self.tracker_file_name, e.strerror),
-                ResumableTransferDisposition.ABORT)
-        finally:
-            if f:
-                f.close()
-
-    def _set_tracker_uri(self, uri):
-        """
-        Called when we start a new resumable upload or get a new tracker
-        URI for the upload. Saves URI and resets upload state.
-
-        Raises InvalidUriError if URI is syntactically invalid.
-        """
-        parse_result = urlparse.urlparse(uri)
-        if (parse_result.scheme.lower() not in ['http', 'https'] or
-            not parse_result.netloc):
-            raise InvalidUriError('Invalid tracker URI (%s)' % uri)
-        self.tracker_uri = uri
-        self.tracker_uri_host = parse_result.netloc
-        self.tracker_uri_path = '%s?%s' % (
-            parse_result.path, parse_result.query)
-        self.server_has_bytes = 0
-
-    def get_tracker_uri(self):
-        """
-        Returns upload tracker URI, or None if the upload has not yet started.
-        """
-        return self.tracker_uri
-
-    def _remove_tracker_file(self):
-        if (self.tracker_file_name and
-            os.path.exists(self.tracker_file_name)):
-                os.unlink(self.tracker_file_name)
-
-    def _build_content_range_header(self, range_spec='*', length_spec='*'):
-        return 'bytes %s/%s' % (range_spec, length_spec)
-
-    def _query_server_state(self, conn, file_length):
-        """
-        Queries server to find out state of given upload.
-
-        Note that this method really just makes special case use of the
-        fact that the upload server always returns the current start/end
-        state whenever a PUT doesn't complete.
-
-        Returns HTTP response from sending request.
-
-        Raises ResumableUploadException if problem querying server.
-        """
-        # Send an empty PUT so that server replies with this resumable
-        # transfer's state.
-        put_headers = {}
-        put_headers['Content-Range'] = (
-            self._build_content_range_header('*', file_length))
-        put_headers['Content-Length'] = '0'
-        return AWSAuthConnection.make_request(conn, 'PUT',
-                                              path=self.tracker_uri_path,
-                                              auth_path=self.tracker_uri_path,
-                                              headers=put_headers,
-                                              host=self.tracker_uri_host)
-
-    def _query_server_pos(self, conn, file_length):
-        """
-        Queries server to find out what bytes it currently has.
-
-        Returns (server_start, server_end), where the values are inclusive.
-        For example, (0, 2) would mean that the server has bytes 0, 1, *and* 2.
-
-        Raises ResumableUploadException if problem querying server.
-        """
-        resp = self._query_server_state(conn, file_length)
-        if resp.status == 200:
-            # To handle the boundary condition where the server has the complete
-            # file, we return (server_start, file_length-1). That way the
-            # calling code can always simply read up through server_end. (If we
-            # didn't handle this boundary condition here, the caller would have
-            # to check whether server_end == file_length and read one fewer byte
-            # in that case.)
-            return (0, file_length - 1)  # Completed upload.
-        if resp.status != 308:
-            # This means the server didn't have any state for the given
-            # upload ID, which can happen (for example) if the caller saved
-            # the tracker URI to a file and then tried to restart the transfer
-            # after that upload ID has gone stale. In that case we need to
-            # start a new transfer (and the caller will then save the new
-            # tracker URI to the tracker file).
-            raise ResumableUploadException(
-                'Got non-308 response (%s) from server state query' %
-                resp.status, ResumableTransferDisposition.START_OVER)
-        got_valid_response = False
-        range_spec = resp.getheader('range')
-        if range_spec:
-            # Parse 'bytes=<from>-<to>' range_spec.
-            m = re.search('bytes=(\d+)-(\d+)', range_spec)
-            if m:
-                server_start = long(m.group(1))
-                server_end = long(m.group(2))
-                got_valid_response = True
-        else:
-            # No Range header, which means the server does not yet have
-            # any bytes. Note that the Range header uses inclusive 'from'
-            # and 'to' values. Since Range 0-0 would mean that the server
-            # has byte 0, omitting the Range header is used to indicate that
-            # the server doesn't have any bytes.
-            return self.SERVER_HAS_NOTHING
-        if not got_valid_response:
-            raise ResumableUploadException(
-                'Couldn\'t parse upload server state query response (%s)' %
-                str(resp.getheaders()), ResumableTransferDisposition.START_OVER)
-        if conn.debug >= 1:
-            print 'Server has: Range: %d - %d.' % (server_start, server_end)
-        return (server_start, server_end)
-
-    def _start_new_resumable_upload(self, key, headers=None):
-        """
-        Starts a new resumable upload.
-
-        Raises ResumableUploadException if any errors occur.
-        """
-        conn = key.bucket.connection
-        if conn.debug >= 1:
-            print 'Starting new resumable upload.'
-        self.server_has_bytes = 0
-
-        # Start a new resumable upload by sending a POST request with an
-        # empty body and the "X-Goog-Resumable: start" header. Include any
-        # caller-provided headers (e.g., Content-Type) EXCEPT Content-Length
-        # (and raise an exception if they tried to pass one, since it's
-        # a semantic error to specify it at this point, and if we were to
-        # include one now it would cause the server to expect that many
-        # bytes; the POST doesn't include the actual file bytes  We set
-        # the Content-Length in the subsequent PUT, based on the uploaded
-        # file size.
-        post_headers = {}
-        for k in headers:
-            if k.lower() == 'content-length':
-                raise ResumableUploadException(
-                    'Attempt to specify Content-Length header (disallowed)',
-                    ResumableTransferDisposition.ABORT)
-            post_headers[k] = headers[k]
-        post_headers[conn.provider.resumable_upload_header] = 'start'
-
-        resp = conn.make_request(
-            'POST', key.bucket.name, key.name, post_headers)
-        # Get tracker URI from response 'Location' header.
-        body = resp.read()
-
-        # Check for various status conditions.
-        if resp.status in [500, 503]:
-            # Retry status 500 and 503 errors after a delay.
-            raise ResumableUploadException(
-                'Got status %d from attempt to start resumable upload. '
-                'Will wait/retry' % resp.status,
-                ResumableTransferDisposition.WAIT_BEFORE_RETRY)
-        elif resp.status != 200 and resp.status != 201:
-            raise ResumableUploadException(
-                'Got status %d from attempt to start resumable upload. '
-                'Aborting' % resp.status,
-                ResumableTransferDisposition.ABORT)
-
-        # Else we got 200 or 201 response code, indicating the resumable
-        # upload was created.
-        tracker_uri = resp.getheader('Location')
-        if not tracker_uri:
-            raise ResumableUploadException(
-                'No resumable tracker URI found in resumable initiation '
-                'POST response (%s)' % body,
-                ResumableTransferDisposition.WAIT_BEFORE_RETRY)
-        self._set_tracker_uri(tracker_uri)
-        self._save_tracker_uri_to_file()
-
-    def _upload_file_bytes(self, conn, http_conn, fp, file_length,
-                           total_bytes_uploaded, cb, num_cb, md5sum, headers):
-        """
-        Makes one attempt to upload file bytes, using an existing resumable
-        upload connection.
-
-        Returns (etag, generation, meta_generation) from server upon success.
-
-        Raises ResumableUploadException if any problems occur.
-        """
-        buf = fp.read(self.BUFFER_SIZE)
-        if cb:
-            # The cb_count represents the number of full buffers to send between
-            # cb executions.
-            if num_cb > 2:
-                cb_count = file_length / self.BUFFER_SIZE / (num_cb-2)
-            elif num_cb < 0:
-                cb_count = -1
-            else:
-                cb_count = 0
-            i = 0
-            cb(total_bytes_uploaded, file_length)
-
-        # Build resumable upload headers for the transfer. Don't send a
-        # Content-Range header if the file is 0 bytes long, because the
-        # resumable upload protocol uses an *inclusive* end-range (so, sending
-        # 'bytes 0-0/1' would actually mean you're sending a 1-byte file).
-        if not headers:
-          put_headers = {}
-        else:
-          put_headers = headers.copy()
-        if file_length:
-            if total_bytes_uploaded == file_length:
-                range_header = self._build_content_range_header(
-                    '*', file_length)
-            else:
-                range_header = self._build_content_range_header(
-                    '%d-%d' % (total_bytes_uploaded, file_length - 1),
-                    file_length)
-            put_headers['Content-Range'] = range_header
-        # Set Content-Length to the total bytes we'll send with this PUT.
-        put_headers['Content-Length'] = str(file_length - total_bytes_uploaded)
-        http_request = AWSAuthConnection.build_base_http_request(
-            conn, 'PUT', path=self.tracker_uri_path, auth_path=None,
-            headers=put_headers, host=self.tracker_uri_host)
-        http_conn.putrequest('PUT', http_request.path)
-        for k in put_headers:
-            http_conn.putheader(k, put_headers[k])
-        http_conn.endheaders()
-
-        # Turn off debug on http connection so upload content isn't included
-        # in debug stream.
-        http_conn.set_debuglevel(0)
-        while buf:
-            http_conn.send(buf)
-            md5sum.update(buf)
-            total_bytes_uploaded += len(buf)
-            if cb:
-                i += 1
-                if i == cb_count or cb_count == -1:
-                    cb(total_bytes_uploaded, file_length)
-                    i = 0
-            buf = fp.read(self.BUFFER_SIZE)
-        http_conn.set_debuglevel(conn.debug)
-        if cb:
-            cb(total_bytes_uploaded, file_length)
-        if total_bytes_uploaded != file_length:
-            # Abort (and delete the tracker file) so if the user retries
-            # they'll start a new resumable upload rather than potentially
-            # attempting to pick back up later where we left off.
-            raise ResumableUploadException(
-                'File changed during upload: EOF at %d bytes of %d byte file.' %
-                (total_bytes_uploaded, file_length),
-                ResumableTransferDisposition.ABORT)
-        resp = http_conn.getresponse()
-        body = resp.read()
-        # Restore http connection debug level.
-        http_conn.set_debuglevel(conn.debug)
-
-        if resp.status == 200:
-            # Success.
-            return (resp.getheader('etag'),
-                    resp.getheader('x-goog-generation'),
-                    resp.getheader('x-goog-metageneration'))
-        # Retry timeout (408) and status 500 and 503 errors after a delay.
-        elif resp.status in [408, 500, 503]:
-            disposition = ResumableTransferDisposition.WAIT_BEFORE_RETRY
-        else:
-            # Catch all for any other error codes.
-            disposition = ResumableTransferDisposition.ABORT
-        raise ResumableUploadException('Got response code %d while attempting '
-                                       'upload (%s)' %
-                                       (resp.status, resp.reason), disposition)
-
-    def _attempt_resumable_upload(self, key, fp, file_length, headers, cb,
-                                  num_cb, md5sum):
-        """
-        Attempts a resumable upload.
-
-        Returns (etag, generation, meta_generation) from server upon success.
-
-        Raises ResumableUploadException if any problems occur.
-        """
-        (server_start, server_end) = self.SERVER_HAS_NOTHING
-        conn = key.bucket.connection
-        if self.tracker_uri:
-            # Try to resume existing resumable upload.
-            try:
-                (server_start, server_end) = (
-                    self._query_server_pos(conn, file_length))
-                self.server_has_bytes = server_start
-
-                if server_end:
-                  # If the server already has some of the content, we need to
-                  # update the md5 with the bytes that have already been
-                  # uploaded to ensure we get a complete hash in the end.
-                  print 'Catching up md5 for resumed upload'
-                  fp.seek(0)
-                  # Read local file's bytes through position server has. For
-                  # example, if server has (0, 3) we want to read 3-0+1=4 bytes.
-                  bytes_to_go = server_end + 1
-                  while bytes_to_go:
-                      chunk = fp.read(min(key.BufferSize, bytes_to_go))
-                      if not chunk:
-                          raise ResumableUploadException(
-                              'Hit end of file during resumable upload md5 '
-                              'catchup. This should not happen under\n'
-                              'normal circumstances, as it indicates the '
-                              'server has more bytes of this transfer\nthan'
-                              ' the current file size. Restarting upload.',
-                              ResumableTransferDisposition.START_OVER)
-                      md5sum.update(chunk)
-                      bytes_to_go -= len(chunk)
-
-                if conn.debug >= 1:
-                    print 'Resuming transfer.'
-            except ResumableUploadException, e:
-                if conn.debug >= 1:
-                    print 'Unable to resume transfer (%s).' % e.message
-                self._start_new_resumable_upload(key, headers)
-        else:
-            self._start_new_resumable_upload(key, headers)
-
-        # upload_start_point allows the code that instantiated the
-        # ResumableUploadHandler to find out the point from which it started
-        # uploading (e.g., so it can correctly compute throughput).
-        if self.upload_start_point is None:
-            self.upload_start_point = server_end
-
-        total_bytes_uploaded = server_end + 1
-        # Corner case: Don't attempt to seek if we've already uploaded the
-        # entire file, because if the file is a stream (e.g., the KeyFile
-        # wrapper around input key when copying between providers), attempting
-        # to seek to the end of file would result in an InvalidRange error.
-        if file_length < total_bytes_uploaded:
-          fp.seek(total_bytes_uploaded)
-        conn = key.bucket.connection
-
-        # Get a new HTTP connection (vs conn.get_http_connection(), which reuses
-        # pool connections) because httplib requires a new HTTP connection per
-        # transaction. (Without this, calling http_conn.getresponse() would get
-        # "ResponseNotReady".)
-        http_conn = conn.new_http_connection(self.tracker_uri_host,
-                                             conn.is_secure)
-        http_conn.set_debuglevel(conn.debug)
-
-        # Make sure to close http_conn at end so if a local file read
-        # failure occurs partway through server will terminate current upload
-        # and can report that progress on next attempt.
-        try:
-            return self._upload_file_bytes(conn, http_conn, fp, file_length,
-                                           total_bytes_uploaded, cb, num_cb, md5sum,
-                                           headers)
-        except (ResumableUploadException, socket.error):
-            resp = self._query_server_state(conn, file_length)
-            if resp.status == 400:
-                raise ResumableUploadException('Got 400 response from server '
-                    'state query after failed resumable upload attempt. This '
-                    'can happen for various reasons, including specifying an '
-                    'invalid request (e.g., an invalid canned ACL) or if the '
-                    'file size changed between upload attempts',
-                    ResumableTransferDisposition.ABORT)
-            else:
-                raise
-        finally:
-            http_conn.close()
-
-    def _check_final_md5(self, key, etag):
-        """
-        Checks that etag from server agrees with md5 computed before upload.
-        This is important, since the upload could have spanned a number of
-        hours and multiple processes (e.g., gsutil runs), and the user could
-        change some of the file and not realize they have inconsistent data.
-        """
-        if key.bucket.connection.debug >= 1:
-            print 'Checking md5 against etag.'
-        if key.md5 != etag.strip('"\''):
-            # Call key.open_read() before attempting to delete the
-            # (incorrect-content) key, so we perform that request on a
-            # different HTTP connection. This is neededb because httplib
-            # will return a "Response not ready" error if you try to perform
-            # a second transaction on the connection.
-            key.open_read()
-            key.close()
-            key.delete()
-            raise ResumableUploadException(
-                'File changed during upload: md5 signature doesn\'t match etag '
-                '(incorrect uploaded object deleted)',
-                ResumableTransferDisposition.ABORT)
-
-    def handle_resumable_upload_exception(self, e, debug):
-        if (e.disposition == ResumableTransferDisposition.ABORT_CUR_PROCESS):
-            if debug >= 1:
-                print('Caught non-retryable ResumableUploadException (%s); '
-                      'aborting but retaining tracker file' % e.message)
-            raise
-        elif (e.disposition == ResumableTransferDisposition.ABORT):
-            if debug >= 1:
-                print('Caught non-retryable ResumableUploadException (%s); '
-                      'aborting and removing tracker file' % e.message)
-            self._remove_tracker_file()
-            raise
-        else:
-            if debug >= 1:
-                print('Caught ResumableUploadException (%s) - will retry' %
-                      e.message)
-
-    def track_progress_less_iterations(self, server_had_bytes_before_attempt,
-                                       roll_back_md5=True, debug=0):
-        # At this point we had a re-tryable failure; see if made progress.
-        if self.server_has_bytes > server_had_bytes_before_attempt:
-            self.progress_less_iterations = 0   # If progress, reset counter.
-        else:
-            self.progress_less_iterations += 1
-            if roll_back_md5:
-                # Rollback any potential md5sum updates, as we did not
-                # make any progress in this iteration.
-                self.md5sum = self.md5sum_before_attempt
-
-        if self.progress_less_iterations > self.num_retries:
-            # Don't retry any longer in the current process.
-            raise ResumableUploadException(
-                    'Too many resumable upload attempts failed without '
-                    'progress. You might try this upload again later',
-                    ResumableTransferDisposition.ABORT_CUR_PROCESS)
-
-        # Use binary exponential backoff to desynchronize client requests.
-        sleep_time_secs = random.random() * (2**self.progress_less_iterations)
-        if debug >= 1:
-            print ('Got retryable failure (%d progress-less in a row).\n'
-                   'Sleeping %3.1f seconds before re-trying' %
-                   (self.progress_less_iterations, sleep_time_secs))
-        time.sleep(sleep_time_secs)
-
-    def send_file(self, key, fp, headers, cb=None, num_cb=10):
-        """
-        Upload a file to a key into a bucket on GS, using GS resumable upload
-        protocol.
-
-        :type key: :class:`boto.s3.key.Key` or subclass
-        :param key: The Key object to which data is to be uploaded
-
-        :type fp: file-like object
-        :param fp: The file pointer to upload
-
-        :type headers: dict
-        :param headers: The headers to pass along with the PUT request
-
-        :type cb: function
-        :param cb: a callback function that will be called to report progress on
-            the upload.  The callback should accept two integer parameters, the
-            first representing the number of bytes that have been successfully
-            transmitted to GS, and the second representing the total number of
-            bytes that need to be transmitted.
-
-        :type num_cb: int
-        :param num_cb: (optional) If a callback is specified with the cb
-            parameter, this parameter determines the granularity of the callback
-            by defining the maximum number of times the callback will be called
-            during the file transfer. Providing a negative integer will cause
-            your callback to be called with each buffer read.
-
-        Raises ResumableUploadException if a problem occurs during the transfer.
-        """
-
-        if not headers:
-            headers = {}
-        # If Content-Type header is present and set to None, remove it.
-        # This is gsutil's way of asking boto to refrain from auto-generating
-        # that header.
-        CT = 'Content-Type'
-        if CT in headers and headers[CT] is None:
-          del headers[CT]
-
-        headers['User-Agent'] = UserAgent
-
-        # Determine file size different ways for case where fp is actually a
-        # wrapper around a Key vs an actual file.
-        if isinstance(fp, KeyFile):
-          file_length = fp.getkey().size
-        else:
-          fp.seek(0, os.SEEK_END)
-          file_length = fp.tell()
-          fp.seek(0)
-        debug = key.bucket.connection.debug
-
-        # Compute the MD5 checksum on the fly.
-        self.md5sum = md5()
-
-        # Use num-retries from constructor if one was provided; else check
-        # for a value specified in the boto config file; else default to 5.
-        if self.num_retries is None:
-            self.num_retries = config.getint('Boto', 'num_retries', 6)
-        self.progress_less_iterations = 0
-
-        while True:  # Retry as long as we're making progress.
-            server_had_bytes_before_attempt = self.server_has_bytes
-            self.md5sum_before_attempt = self.md5sum.copy()
-            try:
-                # Save generation and meta_generation in class state so caller
-                # can find these values, for use in preconditions of future
-                # operations on the uploaded object.
-                (etag, self.generation, self.meta_generation) = (
-                    self._attempt_resumable_upload(key, fp, file_length,
-                                                   headers, cb, num_cb,
-                                                   self.md5sum))
-
-                # Get the final md5 for the uploaded content.
-                hd = self.md5sum.hexdigest()
-                key.md5, key.base64md5 = key.get_md5_from_hexdigest(hd)
-
-                # Upload succceded, so remove the tracker file (if have one).
-                self._remove_tracker_file()
-                self._check_final_md5(key, etag)
-                if debug >= 1:
-                    print 'Resumable upload complete.'
-                return
-            except self.RETRYABLE_EXCEPTIONS, e:
-                if debug >= 1:
-                    print('Caught exception (%s)' % e.__repr__())
-                if isinstance(e, IOError) and e.errno == errno.EPIPE:
-                    # Broken pipe error causes httplib to immediately
-                    # close the socket (http://bugs.python.org/issue5542),
-                    # so we need to close the connection before we resume
-                    # the upload (which will cause a new connection to be
-                    # opened the next time an HTTP request is sent).
-                    key.bucket.connection.connection.close()
-            except ResumableUploadException, e:
-                self.handle_resumable_upload_exception(e, debug)
-
-            self.track_progress_less_iterations(server_had_bytes_before_attempt,
-                                                True, debug)