gcloud 0.0.4 → 0.0.6

data/packages/gsutil/boto/boto/file/README

@@ -0,0 +1,49 @@
+Handling of file:// URIs:
+
+This directory contains code to map basic boto connection, bucket, and key
+operations onto files in the local filesystem, in support of file://
+URI operations.
+
+Bucket storage operations cannot be mapped completely onto a file system
+because of the different naming semantics in these types of systems: the
+former have a flat name space of objects within each named bucket; the
+latter have a hierarchical name space of files, and nothing corresponding to
+the notion of a bucket. The mapping we selected was guided by the desire
+to achieve meaningful semantics for a useful subset of operations that can
+be implemented polymorphically across both types of systems. We considered
+several possibilities for mapping path names to bucket + object name:
+
+1) bucket = the file system root or local directory (for absolute vs
+relative file:// URIs, respectively) and object = remainder of path.
+We discarded this choice because the get_all_keys() method doesn't make
+sense under this approach: Enumerating all files under the root or current
+directory could include more than the caller intended. For example,
+StorageUri("file:///usr/bin/X11/vim").get_all_keys() would enumerate all
+files in the file system.
+
+2) bucket is treated mostly as an anonymous placeholder, with the object
+name holding the URI path (minus the "file://" part). Two sub-options,
+for object enumeration (the get_all_keys() call):
+  a) disallow get_all_keys(). This isn't great, as then the caller must
+     know the URI type before deciding whether to make this call.
+  b) return the single key for which this "bucket" was defined.
+     Note that this option means the app cannot use this API for listing
+     contents of the file system. While that makes the API less generally
+     useful, it avoids the potentially dangerous/unintended consequences
+     noted in option (1) above.
+
+We selected 2b, resulting in a class hierarchy where StorageUri is an abstract
+class, with FileStorageUri and BucketStorageUri subclasses.
+
+Some additional notes:
+
+BucketStorageUri and FileStorageUri each implement these methods:
+  - clone_replace_name() creates a same-type URI with a
+    different object name - which is useful for various enumeration cases
+    (e.g., implementing wildcarding in a command line utility).
+  - names_container() determines if the given URI names a container for
+    multiple objects/files - i.e., a bucket or directory.
+  - names_singleton() determines if the given URI names an individual object
+    or file.
+  - is_file_uri() and is_cloud_uri() determine if the given URI is a
+    FileStorageUri or BucketStorageUri, respectively

data/packages/gsutil/boto/boto/file/__init__.py

@@ -0,0 +1,28 @@
+# Copyright 2010 Google Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+import boto
+
+from connection import FileConnection as Connection
+from key import Key
+from bucket import Bucket
+
+__all__ = ['Connection', 'Key', 'Bucket']

data/packages/gsutil/boto/boto/file/bucket.py

@@ -0,0 +1,112 @@
+# Copyright 2010 Google Inc.
+# Copyright (c) 2011, Nexenta Systems Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+# File representation of bucket, for use with "file://" URIs.
+
+import os
+from key import Key
+from boto.file.simpleresultset import SimpleResultSet
+from boto.s3.bucketlistresultset import BucketListResultSet
+
+class Bucket(object):
+    def __init__(self, name, contained_key):
+        """Instantiate an anonymous file-based Bucket around a single key.
+        """
+        self.name = name
+        self.contained_key = contained_key
+
+    def __iter__(self):
+        return iter(BucketListResultSet(self))
+
+    def __str__(self):
+        return 'anonymous bucket for file://' + self.contained_key
+
+    def delete_key(self, key_name, headers=None,
+                   version_id=None, mfa_token=None):
+        """
+        Deletes a key from the bucket.
+
+        :type key_name: string
+        :param key_name: The key name to delete
+
+        :type version_id: string
+        :param version_id: Unused in this subclass.
+
+        :type mfa_token: tuple or list of strings
+        :param mfa_token: Unused in this subclass.
+        """
+        os.remove(key_name)
+
+    def get_all_keys(self, headers=None, **params):
+        """
+        This method returns the single key around which this anonymous Bucket
+        was instantiated.
+
+        :rtype: SimpleResultSet
+        :return: The result from file system listing the keys requested
+
+        """
+        key = Key(self.name, self.contained_key)
+        return SimpleResultSet([key])
+
+    def get_key(self, key_name, headers=None, version_id=None,
+                                            key_type=Key.KEY_REGULAR_FILE):
+        """
+        Check to see if a particular key exists within the bucket.
+        Returns: An instance of a Key object or None
+
+        :type key_name: string
+        :param key_name: The name of the key to retrieve
+
+        :type version_id: string
+        :param version_id: Unused in this subclass.
+
+        :type stream_type: integer
+        :param stream_type: Type of the Key - Regular File or input/output Stream
+
+        :rtype: :class:`boto.file.key.Key`
+        :returns: A Key object from this bucket.
+        """
+        if key_name == '-':
+            return Key(self.name, '-', key_type=Key.KEY_STREAM_READABLE)
+        else:
+            fp = open(key_name, 'rb')
+            return Key(self.name, key_name, fp)
+
+    def new_key(self, key_name=None, key_type=Key.KEY_REGULAR_FILE):
+        """
+        Creates a new key
+
+        :type key_name: string
+        :param key_name: The name of the key to create
+
+        :rtype: :class:`boto.file.key.Key`
+        :returns: An instance of the newly created key object
+        """
+        if key_name == '-':
+            return Key(self.name, '-', key_type=Key.KEY_STREAM_WRITABLE)
+        else:
+            dir_name = os.path.dirname(key_name)
+            if dir_name and not os.path.exists(dir_name):
+                os.makedirs(dir_name)
+            fp = open(key_name, 'wb')
+            return Key(self.name, key_name, fp)

data/packages/gsutil/boto/boto/file/connection.py

@@ -0,0 +1,33 @@
+# Copyright 2010 Google Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+# File representation of connection, for use with "file://" URIs.
+
+from bucket import Bucket
+
+class FileConnection(object):
+
+    def __init__(self, file_storage_uri):
+        # FileConnections are per-file storage URI.
+        self.file_storage_uri = file_storage_uri
+
+    def get_bucket(self, bucket_name, validate=True, headers=None):
+        return Bucket(bucket_name, self.file_storage_uri.object_name)

data/packages/gsutil/boto/boto/file/key.py

@@ -0,0 +1,199 @@
+# Copyright 2010 Google Inc.
+# Copyright (c) 2011, Nexenta Systems Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+# File representation of key, for use with "file://" URIs.
+
+import os, shutil, StringIO
+import sys
+
+class Key(object):
+
+    KEY_STREAM_READABLE = 0x01
+    KEY_STREAM_WRITABLE = 0x02
+    KEY_STREAM          = (KEY_STREAM_READABLE | KEY_STREAM_WRITABLE)
+    KEY_REGULAR_FILE    = 0x00
+
+    def __init__(self, bucket, name, fp=None, key_type=KEY_REGULAR_FILE):
+        self.bucket = bucket
+        self.full_path = name
+        if name == '-':
+            self.name = None
+            self.size = None
+        else:
+            self.name = name
+            self.size = os.stat(name).st_size
+        self.key_type = key_type
+        if key_type == self.KEY_STREAM_READABLE:
+            self.fp = sys.stdin
+            self.full_path = '<STDIN>'
+        elif key_type == self.KEY_STREAM_WRITABLE:
+            self.fp = sys.stdout
+            self.full_path = '<STDOUT>'
+        else:
+            self.fp = fp
+
+    def __str__(self):
+        return 'file://' + self.full_path
+
+    def get_file(self, fp, headers=None, cb=None, num_cb=10, torrent=False):
+        """
+        Retrieves a file from a Key
+
+        :type fp: file
+        :param fp: File pointer to put the data into
+
+        :type headers: string
+        :param: ignored in this subclass.
+
+        :type cb: function
+        :param cb: ignored in this subclass.
+
+        :type cb: int
+        :param num_cb: ignored in this subclass.
+        """
+        if self.key_type & self.KEY_STREAM_WRITABLE:
+            raise BotoClientError('Stream is not readable')
+        elif self.key_type & self.KEY_STREAM_READABLE:
+            key_file = self.fp
+        else:
+            key_file = open(self.full_path, 'rb')
+        try:
+            shutil.copyfileobj(key_file, fp)
+        finally:
+            key_file.close()
+
+    def set_contents_from_file(self, fp, headers=None, replace=True, cb=None,
+                               num_cb=10, policy=None, md5=None):
+        """
+        Store an object in a file using the name of the Key object as the
+        key in file URI and the contents of the file pointed to by 'fp' as the
+        contents.
+
+        :type fp: file
+        :param fp: the file whose contents to upload
+
+        :type headers: dict
+        :param headers: ignored in this subclass.
+
+        :type replace: bool
+        :param replace: If this parameter is False, the method
+                        will first check to see if an object exists in the
+                        bucket with the same key.  If it does, it won't
+                        overwrite it.  The default value is True which will
+                        overwrite the object.
+
+        :type cb: function
+        :param cb: ignored in this subclass.
+
+        :type cb: int
+        :param num_cb: ignored in this subclass.
+
+        :type policy: :class:`boto.s3.acl.CannedACLStrings`
+        :param policy: ignored in this subclass.
+
+        :type md5: A tuple containing the hexdigest version of the MD5 checksum
+                   of the file as the first element and the Base64-encoded
+                   version of the plain checksum as the second element.
+                   This is the same format returned by the compute_md5 method.
+        :param md5: ignored in this subclass.
+        """
+        if self.key_type & self.KEY_STREAM_READABLE:
+            raise BotoClientError('Stream is not writable')
+        elif self.key_type & self.KEY_STREAM_WRITABLE:
+            key_file = self.fp
+        else:
+            if not replace and os.path.exists(self.full_path):
+                return
+            key_file = open(self.full_path, 'wb')
+        try:
+            shutil.copyfileobj(fp, key_file)
+        finally:
+            key_file.close()
+
+    def get_contents_to_file(self, fp, headers=None, cb=None, num_cb=None,
+                             torrent=False, version_id=None,
+                             res_download_handler=None, response_headers=None):
+        """
+        Copy contents from the current file to the file pointed to by 'fp'.
+
+        :type fp: File-like object
+        :param fp:
+
+        :type headers: dict
+        :param headers: Unused in this subclass.
+
+        :type cb: function
+        :param cb: Unused in this subclass.
+
+        :type cb: int
+        :param num_cb: Unused in this subclass.
+
+        :type torrent: bool
+        :param torrent: Unused in this subclass.
+
+        :type res_upload_handler: ResumableDownloadHandler
+        :param res_download_handler: Unused in this subclass.
+
+        :type response_headers: dict
+        :param response_headers: Unused in this subclass.
+        """
+        shutil.copyfileobj(self.fp, fp)
+
+    def get_contents_as_string(self, headers=None, cb=None, num_cb=10,
+                               torrent=False):
+        """
+        Retrieve file data from the Key, and return contents as a string.
+
+        :type headers: dict
+        :param headers: ignored in this subclass.
+
+        :type cb: function
+        :param cb: ignored in this subclass.
+
+        :type cb: int
+        :param num_cb: ignored in this subclass.
+
+        :type cb: int
+        :param num_cb: ignored in this subclass.
+
+        :type torrent: bool
+        :param torrent: ignored in this subclass.
+
+        :rtype: string
+        :returns: The contents of the file as a string
+        """
+
+        fp = StringIO.StringIO()
+        self.get_contents_to_file(fp)
+        return fp.getvalue()
+
+    def is_stream(self):
+        return (self.key_type & self.KEY_STREAM)
+
+    def close(self):
+        """
+        Closes fp associated with underlying file.
+        Caller should call this method when done with this class, to avoid
+        using up OS resources (e.g., when iterating over a large number
+        of files).
+        """
+        self.fp.close()

data/packages/gsutil/boto/boto/file/simpleresultset.py

@@ -0,0 +1,30 @@
+# Copyright 2010 Google Inc.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+class SimpleResultSet(list):
+    """
+    ResultSet facade built from a simple list, rather than via XML parsing.
+    """
+
+    def __init__(self, input_list):
+        for x in input_list:
+            self.append(x)
+        self.is_truncated = False