googlecloud 0.0.4 → 0.0.6

data/packages/gsutil/gslib/addlhelp/versioning.py

@@ -0,0 +1,242 @@
+# Copyright 2012 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from gslib.help_provider import HELP_NAME
+from gslib.help_provider import HELP_NAME_ALIASES
+from gslib.help_provider import HELP_ONE_LINE_SUMMARY
+from gslib.help_provider import HelpProvider
+from gslib.help_provider import HELP_TEXT
+from gslib.help_provider import HelpType
+from gslib.help_provider import HELP_TYPE
+
+_detailed_help_text = ("""
+<B>OVERVIEW</B>
+  Versioning-enabled buckets maintain an archive of objects, providing a way to
+  un-delete data that you accidentally deleted, or to retrieve older versions of
+  your data. You can turn versioning on or off for a bucket at any time. Turning
+  versioning off leaves existing object versions in place, and simply causes the
+  bucket to stop accumulating new object versions. In this case, if you upload
+  to an existing object the current version is overwritten instead of creating
+  a new version.
+
+  Regardless of whether you have enabled versioning on a bucket, every object
+  has two associated positive integer fields:
+    - the generation, which is updated when the content of an object is
+      overwritten.
+    - the meta-generation, which identifies the metadata generation. It starts
+      at 1; is updated every time the metadata (e.g., ACL or Content-Type) for a
+      given content generation is updated; and gets reset when the generation
+      number changes.
+
+  Of these two integers, only the generation is used when working with versioned
+  data. Both generation and meta-generation can be used with concurrency control
+  (discussed in a later section).
+
+  To work with object versioning in gsutil, you can use a flavor of storage URIs
+  that that embed the object generation, which we refer to as version-specific URIs.
+  For example, the version-less object URI:
+
+    gs://bucket/object
+
+  might have have two versions, with these version-specific URIs:
+
+    gs://bucket/object#1360383693690000
+    gs://bucket/object#1360383802725000
+  
+  The following sections discuss how to work with versioning and concurrency
+  control.
+
+
+<B>OBJECT VERSIONING</B>
+  You can view, enable, and disable object versioning on a bucket using
+  the getversioning and setversioning commands. For example:
+
+    gsutil setversioning on gs://bucket
+
+  will enable versioning for the named bucket. See 'gsutil help getversioning'
+  and 'gsutil help setversioning' for additional details.
+
+  To see all object versions in a versioning-enabled bucket along with
+  their generation.meta-generation information, use gsutil ls -a:
+
+    gsutil ls -a gs://bucket
+
+  You can also specify particular objects for which you want to find the
+  version-specific URI(s), or you can use wildcards:
+
+    gsutil ls -a gs://bucket/object1 gs://bucket/images/*.jpg
+
+  The generation values form a monotonically increasing sequence as you create
+  additional object versions.  Because of this, the latest object version is
+  always the last one listed in the gsutil ls output for a particular object.
+  For example, if a bucket contains these three versions of gs://bucket/object:
+
+    gs://bucket/object#1360035307075000
+    gs://bucket/object#1360101007329000
+    gs://bucket/object#1360102216114000
+
+  then gs://bucket/object#1360102216114000 is the latest version and
+  gs://bucket/object#1360035307075000 is the oldest available version.
+
+  If you specify version-less URIs with gsutil, you will operate on the
+  latest not-deleted version of an object, for example:
+
+    gsutil cp gs://bucket/object ./dir
+
+  or
+
+    gsutil rm gs://bucket/object
+
+  To operate on a specific object version, use a version-specific URI.
+  For example, suppose the output of the above gsutil ls -a command is:
+
+    gs://bucket/object#1360035307075000
+    gs://bucket/object#1360101007329000
+
+  In this case, the command:
+
+    gsutil cp gs://bucket/object#1360035307075000 ./dir
+
+  will retrieve the second most recent version of the object.
+
+  Note that version-specific URIs cannot be the target of the gsutil cp
+  command (trying to do so will result in an error), because writing to a
+  versioned object always creates a new version.
+
+  If an object has been deleted, it will not show up in a normal gsutil ls
+  listing (i.e., ls without the -a option). You can restore a deleted object by
+  running gsutil ls -a to find the available versions, and then copying one of
+  the version-specific URIs to the version-less URI, for example:
+
+    gsutil cp gs://bucket/object#1360101007329000 gs://bucket/object
+
+  Note that when you do this it creates a new object version, which will incur
+  additional charges. You can get rid of the extra copy by deleting the older
+  version-specfic object:
+
+    gsutil rm gs://bucket/object#1360101007329000
+
+  Or you can combine the two steps by using the gsutil mv command:
+
+    gsutil mv gs://bucket/object#1360101007329000 gs://bucket/object
+
+  If you want to remove all versions of an object use the gsutil rm -a option:
+
+    gsutil rm -a gs://bucket/object
+
+  Note that there is no limit to the number of older versions of an object you
+  will create if you continue to upload to the same object in a versioning-
+  enabled bucket. It is your responsibility to delete versions beyond the ones
+  you want to retain.
+
+
+<B>CONCURRENCY CONTROL</B>
+  If you are building an application using Google Cloud Storage, you may need to
+  be careful about concurrency control. Normally gsutil itself isn't used for
+  this purpose, but it's possible to write scripts around gsutil that perform
+  concurrency control.
+
+  For example, suppose you want to implement a "rolling update" system using
+  gsutil, where a periodic job computes some data and uploads it to the cloud.
+  On each run, the job starts with the data that it computed from last run, and
+  computes a new value. To make this system robust, you need to have multiple
+  machines on which the job can run, which raises the possibility that two
+  simultaneous runs could attempt to update an object at the same time. This
+  leads to the following potential race condition:
+    - job 1 computes the new value to be written
+    - job 2 computes the new value to be written
+    - job 2 writes the new value
+    - job 1 writes the new value
+
+  In this case, the value that job 1 read is no longer current by the time
+  it goes to write the updated object, and writing at this point would result
+  in stale (or, depending on the application, corrupt) data.
+
+  To prevent this, you can find the version-specific name of the object that was
+  created, and then use the information contained in that URI to specify an
+  x-goog-if-generation-match header on a subsequent gsutil cp command. You can
+  do this in two steps. First, use the gsutil cp -v option at upload time to get
+  the version-specific name of the object that was created, for example:
+
+    gsutil cp -v file gs://bucket/object
+
+  might output:
+
+    Created: gs://bucket/object#1360432179236000
+
+  You can extract the generation value from this object and then construct a
+  subsequent gsutil command like this:
+
+    gsutil -h x-goog-if-generation-match:1360432179236000 cp newfile \\
+        gs://bucket/object
+
+  This command requests Google Cloud Storage to attempt to upload newfile
+  but to fail the request if the generation of newfile that is live at the
+  time of the upload does not match that specified.
+
+  If the command you use updates object metadata, you will need to find the
+  current meta_generation for an object. To do this, use the gsutil ls -a and
+  -l options. For example, the command:
+
+    gsutil ls -l -a gs://bucket/object
+
+  will output something like:
+
+      64  2013-02-12T19:59:13  gs://bucket/object#1360699153986000  meta_generation=3
+    1521  2013-02-13T02:04:08  gs://bucket/object#1360721048778000  meta_generation=2
+
+  Given this information, you could use the following command to request setting
+  the ACL on the older version of the object, such that the command will fail
+  unless that is the current version of the data+metadata:
+
+    gsutil -h x-goog-if-generation-match:1360699153986000 -h \\
+      x-goog-if-metageneration-match:3 setacl public-read \\
+      gs://bucket/object#1360699153986000
+
+  Without adding these headers, the update would simply overwrite the existing
+  ACL. Note that in contrast, the gsutil chacl command uses these headers
+  automatically, because it performs a read-modify-write cycle in order to edit
+  ACLs.
+
+  If you want to experiment with how generations and metagenerations work, try
+  the following. First, upload an object; then use gsutil ls -l -a to list all
+  versions of the object, along with each version's meta_generation; then re-
+  upload the object and repeat the gsutil ls -l -a. You should see two object
+  versions, each with meta_generation=1. Now try setting the ACL, and rerun the
+  gsutil ls -l -a. You should see the most recent object generation now has
+  meta_generation=2.
+
+
+<B>FOR MORE INFORMATION</B>
+  For more details on how to use versioning and preconditions, see
+  https://developers.google.com/storage/docs/object-versioning
+""")
+
+
+class CommandOptions(HelpProvider):
+  """Additional help about object versioning."""
+
+  help_spec = {
+    # Name of command or auxiliary help info for which this help applies.
+    HELP_NAME : 'versioning',
+    # List of help name aliases.
+    HELP_NAME_ALIASES : ['concurrency', 'concurrency control', 'versioning',
+                         'versions'],
+    # Type of help:
+    HELP_TYPE : HelpType.ADDITIONAL_HELP,
+    # One line summary of this help.
+    HELP_ONE_LINE_SUMMARY : 'Working with object versions; concurrency control',
+    # The full help text.
+    HELP_TEXT : _detailed_help_text,
+  }

data/packages/gsutil/gslib/addlhelp/wildcards.py

@@ -0,0 +1,170 @@
+# Copyright 2012 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from gslib.help_provider import HELP_NAME
+from gslib.help_provider import HELP_NAME_ALIASES
+from gslib.help_provider import HELP_ONE_LINE_SUMMARY
+from gslib.help_provider import HelpProvider
+from gslib.help_provider import HELP_TEXT
+from gslib.help_provider import HelpType
+from gslib.help_provider import HELP_TYPE
+
+_detailed_help_text = ("""
+<B>DESCRIPTION</B>
+  gsutil supports URI wildcards. For example, the command:
+
+    gsutil cp gs://bucket/data/abc* .
+
+  will copy all objects that start with gs://bucket/data/abc followed by any
+  number of characters within that subdirectory.
+
+
+<B>DIRECTORY BY DIRECTORY VS RECURSIVE WILDCARDS</B>
+  The "*" wildcard only matches up to the end of a path within
+  a subdirectory. For example, if bucket contains objects
+  named gs://bucket/data/abcd, gs://bucket/data/abcdef,
+  and gs://bucket/data/abcxyx, as well as an object in a sub-directory
+  (gs://bucket/data/abc/def) the above gsutil cp command would match the
+  first 3 object names but not the last one.
+
+  If you want matches to span directory boundaries, use a '**' wildcard:
+
+    gsutil cp gs://bucket/data/abc** .
+
+  will match all four objects above.
+
+  Note that gsutil supports the same wildcards for both objects and file names.
+  Thus, for example:
+
+    gsutil cp data/abc* gs://bucket
+
+  will match all names in the local file system. Most command shells also
+  support wildcarding, so if you run the above command probably your shell
+  is expanding the matches before running gsutil. However, most shells do not
+  support recursive wildcards ('**'), and you can cause gsutil's wildcarding
+  support to work for such shells by single-quoting the arguments so they
+  don't get interpreted by the shell before being passed to gsutil:
+
+    gsutil cp 'data/abc**' gs://bucket
+
+
+<B>BUCKET WILDCARDS</B>
+  You can specify wildcards for bucket names. For example:
+
+    gsutil ls gs://data*.example.com
+
+  will list the contents of all buckets whose name starts with "data" and
+  ends with ".example.com".
+
+  You can also combine bucket and object name wildcards. For example this
+  command will remove all ".txt" files in any of your Google Cloud Storage
+  buckets:
+
+    gsutil rm gs://*/**.txt
+
+
+<B>OTHER WILDCARD CHARACTERS</B>
+  In addition to '*', you can use these wildcards:
+
+    ? Matches a single character. For example "gs://bucket/??.txt"
+      only matches objects with two characters followed by .txt.
+
+    [chars] Match any of the specified characters. For example
+      "gs://bucket/[aeiou].txt" matches objects that contain a single vowel
+      character followed by .txt
+
+    [char range] Match any of the range of characters. For example
+      "gs://bucket/[a-m].txt" matches objects that contain letters
+      a, b, c, ... or m, and end with .txt.
+
+    You can combine wildcards to provide more powerful matches, for example:
+      gs://bucket/[a-m]??.j*g
+
+
+<B>EFFICIENCY CONSIDERATION: USING WILDCARDS OVER MANY OBJECTS</B>
+  It is more efficient, faster, and less network traffic-intensive
+  to use wildcards that have a non-wildcard object-name prefix, like:
+
+    gs://bucket/abc*.txt
+
+  than it is to use wildcards as the first part of the object name, like:
+
+    gs://bucket/*abc.txt
+
+  This is because the request for "gs://bucket/abc*.txt" asks the server
+  to send back the subset of results whose object names start with "abc",
+  and then gsutil filters the result list for objects whose name ends with
+  ".txt". In contrast, "gs://bucket/*abc.txt" asks the server for the complete
+  list of objects in the bucket and then filters for those objects whose name
+  ends with "abc.txt". This efficiency consideration becomes increasingly
+  noticeable when you use buckets containing thousands or more objects. It is
+  sometimes possible to set up the names of your objects to fit with expected
+  wildcard matching patterns, to take advantage of the efficiency of doing
+  server-side prefix requests. See, for example "gsutil help prod" for a
+  concrete use case example.
+
+
+<B>EFFICIENCY CONSIDERATION: USING MID-PATH WILDCARDS</B>
+  Suppose you have a bucket with these objects:
+    gs://bucket/obj1
+    gs://bucket/obj2
+    gs://bucket/obj3
+    gs://bucket/obj4
+    gs://bucket/dir1/obj5
+    gs://bucket/dir2/obj6
+
+  If you run the command:
+    gsutil ls gs://bucket/*/obj5
+  gsutil will perform a /-delimited top-level bucket listing and then one bucket
+  listing for each subdirectory, for a total of 3 bucket listings:
+    GET /bucket/?delimiter=/
+    GET /bucket/?prefix=dir1/obj5&delimiter=/
+    GET /bucket/?prefix=dir2/obj5&delimiter=/
+
+  The more bucket listings your wildcard requires, the slower and more expensive
+  it will be. The number of bucket listings required grows as:
+    - the number of wildcard components (e.g., "gs://bucket/a??b/c*/*/d"
+      has 3 wildcard components);
+    - the number of subdirectories that match each component; and
+    - the number of results (pagination is implemented using one GET
+      request per 1000 results, specifying markers for each).
+
+  If you want to use a mid-path wildcard, you might try instead using a
+  recursive wildcard, for example:
+
+    gsutil ls gs://bucket/**/obj5
+
+  This will match more objects than gs://bucket/*/obj5 (since it spans
+  directories), but is implemented using a delimiter-less bucket listing
+  request (which means fewer bucket requests, though it will list the entire
+  bucket and filter locally, so that could require a non-trivial amount of
+  network traffic).
+""")
+
+
+class CommandOptions(HelpProvider):
+  """Additional help about wildcards."""
+
+  help_spec = {
+    # Name of command or auxiliary help info for which this help applies.
+    HELP_NAME : 'wildcards',
+    # List of help name aliases.
+    HELP_NAME_ALIASES : ['wildcard', '*', '**'],
+    # Type of help:
+    HELP_TYPE : HelpType.ADDITIONAL_HELP,
+    # One line summary of this help.
+    HELP_ONE_LINE_SUMMARY : 'Wildcard support',
+    # The full help text.
+    HELP_TEXT : _detailed_help_text,
+  }

data/packages/gsutil/gslib/bucket_listing_ref.py

@@ -0,0 +1,175 @@
+# Copyright 2012 Google Inc. All Rights Reserved.
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+import time
+
+
+class BucketListingRef(object):
+  """
+  Container that holds a reference to one result from a bucket listing, allowing
+  polymorphic iteration over wildcard-iterated URIs, Keys, or Prefixes. At a
+  minimum, every reference contains a StorageUri. If the reference came from a
+  bucket listing (as opposed to a manually instantiated ref that might populate
+  only the StorageUri), it will additionally contain either a Key or a Prefix,
+  depending on whether it was a reference to an object or was just a prefix of a
+  path (i.e., bucket subdirectory). The latter happens when the bucket was
+  listed using delimiter='/'.
+
+  Note that Keys are shallow-populated, based on the contents extracted from
+  parsing a bucket listing. This includes name, length, and other fields
+  (basically, the info listed by gsutil ls -l), but does not include information
+  like ACL and location (which require separate server requests, which is why
+  there's a separate gsutil ls -L option to get this more detailed info).
+  """
+
+  def __init__(self, uri, key=None, prefix=None, headers=None):
+    """Instantiate BucketListingRef from uri and (if available) key or prefix.
+
+    Args:
+      uri: StorageUri for the object (required).
+      key: Key for the object, or None if not available.
+      prefix: Prefix for the subdir, or None if not available.
+      headers: Dictionary containing optional HTTP headers to pass to boto
+          (which happens when GetKey() is called on an BucketListingRef which
+          has no constructor-populated Key), or None if not available.
+
+      At most one of key and prefix can be populated.
+    """
+    assert key is None or prefix is None
+    self.uri = uri
+    self.key = key
+    self.prefix = prefix
+    self.headers = headers or {}
+
+  def GetUri(self):
+    """Get URI form of listed URI.
+
+    Returns:
+      StorageUri.
+    """
+    return self.uri
+
+  def GetUriString(self):
+    """Get string URI form of listed URI.
+
+    Returns:
+      String.
+    """
+    return self.uri.uri
+
+  def NamesBucket(self):
+    """Determines if this BucketListingRef names a bucket.
+
+    Returns:
+      bool indicator.
+    """
+    return self.key is None and self.prefix is None and self.uri.names_bucket()
+
+  def IsLatest(self):
+    """Determines if this BucketListingRef names the latest version of an
+    object.
+
+    Returns:
+      bool indicator.
+    """
+    return hasattr(self.uri, 'is_latest') and self.uri.is_latest
+
+  def GetRStrippedUriString(self):
+    """Get string URI form of listed URI, stripped of any right trailing
+    delims, and without version string.
+
+    Returns:
+      String.
+    """
+    return self.uri.versionless_uri.rstrip('/')
+
+  def HasKey(self):
+    """Return bool indicator of whether this BucketListingRef has a Key."""
+    return bool(self.key)
+
+  def HasPrefix(self):
+    """Return bool indicator of whether this BucketListingRef has a Prefix."""
+    return bool(self.prefix)
+
+  def GetKey(self):
+    """Get Key form of listed URI.
+
+    Returns:
+      Subclass of boto.s3.key.Key.
+
+    Raises:
+      BucketListingRefException: for bucket-only uri.
+    """
+    # For gsutil ls -l gs://bucket self.key will be populated from (boto)
+    # parsing the bucket listing. But as noted and handled below there are
+    # cases where self.key isn't populated.
+    if not self.key:
+      if not self.uri.names_object():
+        raise BucketListingRefException(
+            'Attempt to call GetKey() on Key-less BucketListingRef (uri=%s) ' %
+            self.uri)
+      # This case happens when we do gsutil ls -l on a object name-ful
+      # StorageUri with no object-name wildcard. Since the ls command
+      # implementation only reads bucket info we need to read the object
+      # for this case.
+      self.key = self.uri.get_key(validate=False, headers=self.headers)
+      # When we retrieve the object this way its last_modified timestamp
+      # is formatted in RFC 1123 format, which is different from when we
+      # retrieve from the bucket listing (which uses ISO 8601 format), so
+      # convert so we consistently return ISO 8601 format.
+      tuple_time = (time.strptime(self.key.last_modified,
+                                  '%a, %d %b %Y %H:%M:%S %Z'))
+      self.key.last_modified = time.strftime('%Y-%m-%dT%H:%M:%S', tuple_time)
+    return self.key
+
+  def GetPrefix(self):
+    """Get Prefix form of listed URI.
+
+    Returns:
+      boto.s3.prefix.Prefix.
+
+    Raises:
+      BucketListingRefException: if this object has no Prefix.
+    """
+    if not self.prefix:
+      raise BucketListingRefException(
+          'Attempt to call GetPrefix() on Prefix-less BucketListingRef '
+          '(uri=%s)' % self.uri)
+    return self.prefix
+
+  def __repr__(self):
+    """Returns string representation of BucketListingRef."""
+    return 'BucketListingRef(%s, HasKey=%s, HasPrefix=%s)' % (
+        self.uri, self.HasKey(), self.HasPrefix())
+
+
+class BucketListingRefException(StandardError):
+  """Exception thrown for invalid BucketListingRef requests."""
+
+  def __init__(self, reason):
+    StandardError.__init__(self)
+    self.reason = reason
+
+  def __repr__(self):
+    return 'BucketListingRefException: %s' % self.reason
+
+  def __str__(self):
+    return 'BucketListingRefException: %s' % self.reason