gcloud 0.0.7 → 0.1.0

data/packages/gsutil/boto/boto/glacier/writer.py

@@ -1,242 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright (c) 2012 Thomas Parslow http://almostobsolete.net/
-# Copyright (c) 2012 Robie Basak <robie@justgohome.co.uk>
-# Tree hash implementation from Aaron Brady bradya@gmail.com
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish, dis-
-# tribute, sublicense, and/or sell copies of the Software, and to permit
-# persons to whom the Software is furnished to do so, subject to the fol-
-# lowing conditions:
-#
-# The above copyright notice and this permission notice shall be included
-# in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
-# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
-# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-#
-import hashlib
-
-from boto.glacier.utils import chunk_hashes, tree_hash, bytes_to_hex
-# This import is provided for backwards compatibility.  This function is
-# now in boto.glacier.utils, but any existing code can still import
-# this directly from this module.
-from boto.glacier.utils import compute_hashes_from_fileobj
-
-
-_ONE_MEGABYTE = 1024 * 1024
-
-
-class _Partitioner(object):
-    """Convert variable-size writes into part-sized writes
-
-    Call write(data) with variable sized data as needed to write all data. Call
-    flush() after all data is written.
-
-    This instance will call send_fn(part_data) as needed in part_size pieces,
-    except for the final part which may be shorter than part_size. Make sure to
-    call flush() to ensure that a short final part results in a final send_fn
-    call.
-
-    """
-    def __init__(self, part_size, send_fn):
-        self.part_size = part_size
-        self.send_fn = send_fn
-        self._buffer = []
-        self._buffer_size = 0
-
-    def write(self, data):
-        if data == '':
-            return
-        self._buffer.append(data)
-        self._buffer_size += len(data)
-        while self._buffer_size > self.part_size:
-            self._send_part()
-
-    def _send_part(self):
-        data = ''.join(self._buffer)
-        # Put back any data remaining over the part size into the
-        # buffer
-        if len(data) > self.part_size:
-            self._buffer = [data[self.part_size:]]
-            self._buffer_size = len(self._buffer[0])
-        else:
-            self._buffer = []
-            self._buffer_size = 0
-        # The part we will send
-        part = data[:self.part_size]
-        self.send_fn(part)
-
-    def flush(self):
-        if self._buffer_size > 0:
-            self._send_part()
-
-
-class _Uploader(object):
-    """Upload to a Glacier upload_id.
-
-    Call upload_part for each part (in any order) and then close to complete
-    the upload.
-
-    """
-    def __init__(self, vault, upload_id, part_size, chunk_size=_ONE_MEGABYTE):
-        self.vault = vault
-        self.upload_id = upload_id
-        self.part_size = part_size
-        self.chunk_size = chunk_size
-        self.archive_id = None
-
-        self._uploaded_size = 0
-        self._tree_hashes = []
-
-        self.closed = False
-
-    def _insert_tree_hash(self, index, raw_tree_hash):
-        list_length = len(self._tree_hashes)
-        if index >= list_length:
-            self._tree_hashes.extend([None] * (list_length - index + 1))
-        self._tree_hashes[index] = raw_tree_hash
-
-    def upload_part(self, part_index, part_data):
-        """Upload a part to Glacier.
-
-        :param part_index: part number where 0 is the first part
-        :param part_data: data to upload corresponding to this part
-
-        """
-        if self.closed:
-            raise ValueError("I/O operation on closed file")
-        # Create a request and sign it
-        part_tree_hash = tree_hash(chunk_hashes(part_data, self.chunk_size))
-        self._insert_tree_hash(part_index, part_tree_hash)
-
-        hex_tree_hash = bytes_to_hex(part_tree_hash)
-        linear_hash = hashlib.sha256(part_data).hexdigest()
-        start = self.part_size * part_index
-        content_range = (start,
-                         (start + len(part_data)) - 1)
-        response = self.vault.layer1.upload_part(self.vault.name,
-                                                 self.upload_id,
-                                                 linear_hash,
-                                                 hex_tree_hash,
-                                                 content_range, part_data)
-        response.read()
-        self._uploaded_size += len(part_data)
-
-    def skip_part(self, part_index, part_tree_hash, part_length):
-        """Skip uploading of a part.
-
-        The final close call needs to calculate the tree hash and total size
-        of all uploaded data, so this is the mechanism for resume
-        functionality to provide it without actually uploading the data again.
-
-        :param part_index: part number where 0 is the first part
-        :param part_tree_hash: binary tree_hash of part being skipped
-        :param part_length: length of part being skipped
-
-        """
-        if self.closed:
-            raise ValueError("I/O operation on closed file")
-        self._insert_tree_hash(part_index, part_tree_hash)
-        self._uploaded_size += part_length
-
-    def close(self):
-        if self.closed:
-            return
-        if None in self._tree_hashes:
-            raise RuntimeError("Some parts were not uploaded.")
-        # Complete the multiplart glacier upload
-        hex_tree_hash = bytes_to_hex(tree_hash(self._tree_hashes))
-        response = self.vault.layer1.complete_multipart_upload(
-            self.vault.name, self.upload_id, hex_tree_hash,
-            self._uploaded_size)
-        self.archive_id = response['ArchiveId']
-        self.closed = True
-
-
-def generate_parts_from_fobj(fobj, part_size):
-    data = fobj.read(part_size)
-    while data:
-        yield data
-        data = fobj.read(part_size)
-
-
-def resume_file_upload(vault, upload_id, part_size, fobj, part_hash_map,
-                       chunk_size=_ONE_MEGABYTE):
-    """Resume upload of a file already part-uploaded to Glacier.
-
-    The resumption of an upload where the part-uploaded section is empty is a
-    valid degenerate case that this function can handle. In this case,
-    part_hash_map should be an empty dict.
-
-    :param vault: boto.glacier.vault.Vault object.
-    :param upload_id: existing Glacier upload id of upload being resumed.
-    :param part_size: part size of existing upload.
-    :param fobj: file object containing local data to resume. This must read
-        from the start of the entire upload, not just from the point being
-        resumed. Use fobj.seek(0) to achieve this if necessary.
-    :param part_hash_map: {part_index: part_tree_hash, ...} of data already
-        uploaded. Each supplied part_tree_hash will be verified and the part
-        re-uploaded if there is a mismatch.
-    :param chunk_size: chunk size of tree hash calculation. This must be
-        1 MiB for Amazon.
-
-    """
-    uploader = _Uploader(vault, upload_id, part_size, chunk_size)
-    for part_index, part_data in enumerate(
-            generate_parts_from_fobj(fobj, part_size)):
-        part_tree_hash = tree_hash(chunk_hashes(part_data, chunk_size))
-        if (part_index not in part_hash_map or
-                part_hash_map[part_index] != part_tree_hash):
-            uploader.upload_part(part_index, part_data)
-        else:
-            uploader.skip_part(part_index, part_tree_hash, len(part_data))
-    uploader.close()
-    return uploader.archive_id
-
-
-class Writer(object):
-    """
-    Presents a file-like object for writing to a Amazon Glacier
-    Archive. The data is written using the multi-part upload API.
-    """
-    def __init__(self, vault, upload_id, part_size, chunk_size=_ONE_MEGABYTE):
-        self.uploader = _Uploader(vault, upload_id, part_size, chunk_size)
-        self.partitioner = _Partitioner(part_size, self._upload_part)
-        self.closed = False
-        self.next_part_index = 0
-
-    def write(self, data):
-        if self.closed:
-            raise ValueError("I/O operation on closed file")
-        self.partitioner.write(data)
-
-    def _upload_part(self, part_data):
-        self.uploader.upload_part(self.next_part_index, part_data)
-        self.next_part_index += 1
-
-    def close(self):
-        if self.closed:
-            return
-        self.partitioner.flush()
-        self.uploader.close()
-        self.closed = True
-
-    def get_archive_id(self):
-        self.close()
-        return self.uploader.archive_id
-
-    @property
-    def upload_id(self):
-        return self.uploader.upload_id
-
-    @property
-    def vault(self):
-        return self.uploader.vault

data/packages/gsutil/boto/boto/gs/__init__.py

@@ -1,22 +0,0 @@
-# Copyright 2010 Google Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish, dis-
-# tribute, sublicense, and/or sell copies of the Software, and to permit
-# persons to whom the Software is furnished to do so, subject to the fol-
-# lowing conditions:
-#
-# The above copyright notice and this permission notice shall be included
-# in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
-# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
-# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
-# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-#
-

data/packages/gsutil/boto/boto/gs/acl.py

@@ -1,304 +0,0 @@
-# Copyright 2010 Google Inc.
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish, dis-
-# tribute, sublicense, and/or sell copies of the Software, and to permit
-# persons to whom the Software is furnished to do so, subject to the fol-
-# lowing conditions:
-#
-# The above copyright notice and this permission notice shall be included
-# in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
-# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
-# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
-# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-
-from boto.gs.user import User
-from boto.exception import InvalidAclError
-
-ACCESS_CONTROL_LIST = 'AccessControlList'
-ALL_AUTHENTICATED_USERS = 'AllAuthenticatedUsers'
-ALL_USERS = 'AllUsers'
-DISPLAY_NAME = 'DisplayName'
-DOMAIN = 'Domain'
-EMAIL_ADDRESS = 'EmailAddress'
-ENTRY = 'Entry'
-ENTRIES = 'Entries'
-GROUP_BY_DOMAIN = 'GroupByDomain'
-GROUP_BY_EMAIL = 'GroupByEmail'
-GROUP_BY_ID = 'GroupById'
-ID = 'ID'
-NAME = 'Name'
-OWNER = 'Owner'
-PERMISSION = 'Permission'
-SCOPE = 'Scope'
-TYPE = 'type'
-USER_BY_EMAIL = 'UserByEmail'
-USER_BY_ID = 'UserById'
-
-
-CannedACLStrings = ['private', 'public-read', 'project-private',
-                    'public-read-write', 'authenticated-read',
-                    'bucket-owner-read', 'bucket-owner-full-control']
-"""A list of Google Cloud Storage predefined (canned) ACL strings."""
-
-SupportedPermissions = ['READ', 'WRITE', 'FULL_CONTROL']
-"""A list of supported ACL permissions."""
-
-class ACL:
-
-    def __init__(self, parent=None):
-        self.parent = parent
-        self.entries = []
-
-    @property
-    def acl(self):
-        return self
-
-    def __repr__(self):
-        # Owner is optional in GS ACLs.
-        if hasattr(self, 'owner'):
-            entries_repr = ['Owner:%s' % self.owner.__repr__()]
-        else:
-            entries_repr = ['']
-        acl_entries = self.entries
-        if acl_entries:
-            for e in acl_entries.entry_list:
-                entries_repr.append(e.__repr__())
-        return '<%s>' % ', '.join(entries_repr)
-
-    # Method with same signature as boto.s3.acl.ACL.add_email_grant(), to allow
-    # polymorphic treatment at application layer.
-    def add_email_grant(self, permission, email_address):
-        entry = Entry(type=USER_BY_EMAIL, email_address=email_address,
-                      permission=permission)
-        self.entries.entry_list.append(entry)
-
-    # Method with same signature as boto.s3.acl.ACL.add_user_grant(), to allow
-    # polymorphic treatment at application layer.
-    def add_user_grant(self, permission, user_id):
-        entry = Entry(permission=permission, type=USER_BY_ID, id=user_id)
-        self.entries.entry_list.append(entry)
-
-    def add_group_email_grant(self, permission, email_address):
-        entry = Entry(type=GROUP_BY_EMAIL, email_address=email_address,
-                      permission=permission)
-        self.entries.entry_list.append(entry)
-
-    def add_group_grant(self, permission, group_id):
-        entry = Entry(type=GROUP_BY_ID, id=group_id, permission=permission)
-        self.entries.entry_list.append(entry)
-
-    def startElement(self, name, attrs, connection):
-        if name.lower() == OWNER.lower():
-            self.owner = User(self)
-            return self.owner
-        elif name.lower() == ENTRIES.lower():
-            self.entries = Entries(self)
-            return self.entries
-        else:
-            return None
-
-    def endElement(self, name, value, connection):
-        if name.lower() == OWNER.lower():
-            pass
-        elif name.lower() == ENTRIES.lower():
-            pass
-        else:
-            setattr(self, name, value)
-
-    def to_xml(self):
-        s = '<%s>' % ACCESS_CONTROL_LIST
-        # Owner is optional in GS ACLs.
-        if hasattr(self, 'owner'):
-            s += self.owner.to_xml()
-        acl_entries = self.entries
-        if acl_entries:
-            s += acl_entries.to_xml()
-        s += '</%s>' % ACCESS_CONTROL_LIST
-        return s
-
-
-class Entries:
-
-    def __init__(self, parent=None):
-        self.parent = parent
-        # Entries is the class that represents the same-named XML
-        # element. entry_list is the list within this class that holds the data.
-        self.entry_list = []
-
-    def __repr__(self):
-        entries_repr = []
-        for e in self.entry_list:
-            entries_repr.append(e.__repr__())
-        return '<Entries: %s>' % ', '.join(entries_repr)
-
-    def startElement(self, name, attrs, connection):
-        if name.lower() == ENTRY.lower():
-            entry = Entry(self)
-            self.entry_list.append(entry)
-            return entry
-        else:
-            return None
-
-    def endElement(self, name, value, connection):
-        if name.lower() == ENTRY.lower():
-            pass
-        else:
-            setattr(self, name, value)
-
-    def to_xml(self):
-        s = '<%s>' % ENTRIES
-        for entry in self.entry_list:
-            s += entry.to_xml()
-        s += '</%s>' % ENTRIES
-        return s
-        
-
-# Class that represents a single (Scope, Permission) entry in an ACL.
-class Entry:
-
-    def __init__(self, scope=None, type=None, id=None, name=None,
-                 email_address=None, domain=None, permission=None):
-        if not scope:
-            scope = Scope(self, type, id, name, email_address, domain)
-        self.scope = scope
-        self.permission = permission
-
-    def __repr__(self):
-        return '<%s: %s>' % (self.scope.__repr__(), self.permission.__repr__())
-
-    def startElement(self, name, attrs, connection):
-        if name.lower() == SCOPE.lower():
-            # The following if statement used to look like this: 
-            #   if not TYPE in attrs:
-            # which caused problems because older versions of the 
-            # AttributesImpl class in the xml.sax library neglected to include 
-            # a __contains__() method (which Python calls to implement the 
-            # 'in' operator). So when you use the in operator, like the if
-            # statement above, Python invokes the __getiter__() method with
-            # index 0, which raises an exception. More recent versions of 
-            # xml.sax include the __contains__() method, rendering the in 
-            # operator functional. The work-around here is to formulate the
-            # if statement as below, which is the legal way to query 
-            # AttributesImpl for containment (and is also how the added
-            # __contains__() method works). At one time gsutil disallowed
-            # xmlplus-based parsers, until this more specific problem was 
-            # determined.
-            if TYPE not in attrs:
-                raise InvalidAclError('Missing "%s" in "%s" part of ACL' %
-                                      (TYPE, SCOPE))
-            self.scope = Scope(self, attrs[TYPE])
-            return self.scope
-        elif name.lower() == PERMISSION.lower():
-            pass
-        else:
-            return None
-
-    def endElement(self, name, value, connection):
-        if name.lower() == SCOPE.lower():
-            pass
-        elif name.lower() == PERMISSION.lower():
-            value = value.strip()
-            if not value in SupportedPermissions:
-                raise InvalidAclError('Invalid Permission "%s"' % value)
-            self.permission = value
-        else:
-            setattr(self, name, value)
-
-    def to_xml(self):
-        s = '<%s>' % ENTRY
-        s += self.scope.to_xml()
-        s += '<%s>%s</%s>' % (PERMISSION, self.permission, PERMISSION)
-        s += '</%s>' % ENTRY
-        return s
-
-class Scope:
-
-    # Map from Scope type.lower() to lower-cased list of allowed sub-elems.
-    ALLOWED_SCOPE_TYPE_SUB_ELEMS = {
-        ALL_AUTHENTICATED_USERS.lower() : [],
-        ALL_USERS.lower() : [],
-        GROUP_BY_DOMAIN.lower() : [DOMAIN.lower()],
-        GROUP_BY_EMAIL.lower() : [
-            DISPLAY_NAME.lower(), EMAIL_ADDRESS.lower(), NAME.lower()],
-        GROUP_BY_ID.lower() : [DISPLAY_NAME.lower(), ID.lower(), NAME.lower()],
-        USER_BY_EMAIL.lower() : [
-            DISPLAY_NAME.lower(), EMAIL_ADDRESS.lower(), NAME.lower()],
-        USER_BY_ID.lower() : [DISPLAY_NAME.lower(), ID.lower(), NAME.lower()]
-    }
-
-    def __init__(self, parent, type=None, id=None, name=None,
-                 email_address=None, domain=None):
-        self.parent = parent
-        self.type = type
-        self.name = name
-        self.id = id
-        self.domain = domain
-        self.email_address = email_address
-        if self.type.lower() not in self.ALLOWED_SCOPE_TYPE_SUB_ELEMS:
-            raise InvalidAclError('Invalid %s %s "%s" ' %
-                                  (SCOPE, TYPE, self.type))
-
-    def __repr__(self):
-        named_entity = None
-        if self.id:
-            named_entity = self.id
-        elif self.email_address:
-            named_entity = self.email_address
-        elif self.domain:
-            named_entity = self.domain
-        if named_entity:
-            return '<%s: %s>' % (self.type, named_entity)
-        else:
-            return '<%s>' % self.type
-
-    def startElement(self, name, attrs, connection):
-        if (not name.lower() in
-            self.ALLOWED_SCOPE_TYPE_SUB_ELEMS[self.type.lower()]):
-            raise InvalidAclError('Element "%s" not allowed in %s %s "%s" ' %
-                                   (name, SCOPE, TYPE, self.type))
-        return None
-
-    def endElement(self, name, value, connection):
-        value = value.strip()
-        if name.lower() == DOMAIN.lower():
-            self.domain = value
-        elif name.lower() == EMAIL_ADDRESS.lower():
-            self.email_address = value
-        elif name.lower() == ID.lower():
-            self.id = value
-        elif name.lower() == NAME.lower():
-            self.name = value
-        else:
-            setattr(self, name, value)
-
-    def to_xml(self):
-        s = '<%s type="%s">' % (SCOPE, self.type)
-        if (self.type.lower() == ALL_AUTHENTICATED_USERS.lower()
-            or self.type.lower() == ALL_USERS.lower()):
-            pass
-        elif self.type.lower() == GROUP_BY_DOMAIN.lower():
-            s += '<%s>%s</%s>' % (DOMAIN, self.domain, DOMAIN)
-        elif (self.type.lower() == GROUP_BY_EMAIL.lower()
-              or self.type.lower() == USER_BY_EMAIL.lower()):
-            s += '<%s>%s</%s>' % (EMAIL_ADDRESS, self.email_address,
-                                  EMAIL_ADDRESS)
-            if self.name:
-              s += '<%s>%s</%s>' % (NAME, self.name, NAME)
-        elif (self.type.lower() == GROUP_BY_ID.lower()
-              or self.type.lower() == USER_BY_ID.lower()):
-            s += '<%s>%s</%s>' % (ID, self.id, ID)
-            if self.name:
-              s += '<%s>%s</%s>' % (NAME, self.name, NAME)
-        else:
-            raise InvalidAclError('Invalid scope type "%s" ', self.type)
-
-        s += '</%s>' % SCOPE
-        return s