gcloud 0.0.7 → 0.1.0

data/packages/gsutil/gslib/tests/util.py

@@ -1,125 +0,0 @@
-# Copyright 2013 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import posixpath
-import os.path
-import time
-import urllib
-import urlparse
-
-import unittest
-if not hasattr(unittest.TestCase, 'assertIsNone'):
-  # external dependency unittest2 required for Python <= 2.6
-  import unittest2 as unittest
-
-# Flags for running different types of tests.
-RUN_INTEGRATION_TESTS = True
-RUN_UNIT_TESTS = True
-
-# Whether the tests are running verbose or not.
-VERBOSE_OUTPUT = False
-
-
-def _NormalizeURI(uri):
-  """Normalizes the path component of a URI.
-
-  Examples:
-    gs://foo//bar -> gs://foo/bar
-    gs://foo/./bar -> gs://foo/bar
-  """
-  # Note: we have to do this dance of changing gs:// to file:// because on
-  # Windows, the urlparse function won't work with URL schemes that are not
-  # known. urlparse('gs://foo/bar') on Windows turns into:
-  #     scheme='gs', netloc='', path='//foo/bar'
-  # while on non-Windows platforms, it turns into:
-  #     scheme='gs', netloc='foo', path='/bar'
-  uri = uri.replace('gs://', 'file://')
-  parsed = list(urlparse.urlparse(uri))
-  parsed[2] = posixpath.normpath(parsed[2])
-  if parsed[2].startswith('//'):
-    # The normpath function doesn't change '//foo' -> '/foo' by design.
-    parsed[2] = parsed[2][1:]
-  unparsed = urlparse.urlunparse(parsed)
-  unparsed = unparsed.replace('file://', 'gs://')
-  return unparsed
-
-
-def ObjectToURI(obj, *suffixes):
-  """Returns the storage URI string for a given StorageUri or file object.
-
-  Args:
-    obj: The object to get the URI from. Can be a file object, a subclass of
-         boto.storage_uri.StorageURI, or a string. If a string, it is assumed to
-         be a local on-disk path.
-    suffixes: Suffixes to append. For example, ObjectToUri(bucketuri, 'foo')
-              would return the URI for a key name 'foo' inside the given bucket.
-  """
-  if isinstance(obj, file):
-    return 'file://%s' % os.path.abspath(os.path.join(obj.name, *suffixes))
-  if isinstance(obj, basestring):
-    return 'file://%s' % os.path.join(obj, *suffixes)
-  uri = obj.uri
-  if suffixes:
-    uri = _NormalizeURI('/'.join([uri] + list(suffixes)))
-
-  # Storage URIs shouldn't contain a trailing slash.
-  if uri.endswith('/'):
-    uri = uri[:-1]
-  return uri
-
-
-def Retry(ExceptionToCheck, tries=4, delay=3, backoff=2, logger=None):
-  """Retry calling the decorated function using an exponential backoff.
-
-  Taken from:
-    https://github.com/saltycrane/retry-decorator
-  Licensed under BSD:
-    https://github.com/saltycrane/retry-decorator/blob/master/LICENSE
-
-  :param ExceptionToCheck: the exception to check. may be a tuple of
-      exceptions to check
-  :type ExceptionToCheck: Exception or tuple
-  :param tries: number of times to try (not retry) before giving up
-  :type tries: int
-  :param delay: initial delay between retries in seconds
-  :type delay: int
-  :param backoff: backoff multiplier e.g. value of 2 will double the delay
-      each retry
-  :type backoff: int
-  :param logger: logger to use. If None, print
-  :type logger: logging.Logger instance
-  """
-  def deco_retry(f):
-    def f_retry(*args, **kwargs):
-      mtries, mdelay = tries, delay
-      try_one_last_time = True
-      while mtries > 1:
-        try:
-          return f(*args, **kwargs)
-          try_one_last_time = False
-          break
-        except ExceptionToCheck, e:
-          msg = "%s, Retrying in %d seconds..." % (str(e), mdelay)
-          if logger:
-              logger.warning(msg)
-          else:
-              print msg
-          time.sleep(mdelay)
-          mtries -= 1
-          mdelay *= backoff
-      if try_one_last_time:
-        return f(*args, **kwargs)
-      return
-    return f_retry  # true decorator
-  return deco_retry

data/packages/gsutil/gslib/thread_pool.py

@@ -1,79 +0,0 @@
-# Copyright 2011 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Basic thread pool with exception handler."""
-
-import logging
-import Queue
-import threading
-
-
-# Magic values used to cleanly bring down threads.
-_THREAD_EXIT_MAGIC = ('Clean', 'Thread', 'Exit')
-
-
-def _DefaultExceptionHandler(e):
-  logging.exception(e)
-
-
-class Worker(threading.Thread):
-  """Thread executing tasks from a given task's queue."""
-
-  def __init__(self, tasks, exception_handler):
-    threading.Thread.__init__(self)
-    self.tasks = tasks
-    self.daemon = True
-    self.exception_handler = exception_handler
-    self.start()
-
-  def run(self):
-    while True:
-      func, args, kargs = self.tasks.get()
-
-      # Listen for magic value indicating thread exit.
-      if (func, args, kargs) == _THREAD_EXIT_MAGIC:
-        break
-
-      try:
-        func(*args, **kargs)
-      except Exception, e:
-        self.exception_handler(e)
-      finally:
-        self.tasks.task_done()
-
-
-class ThreadPool(object):
-  """Pool of threads consuming tasks from a queue."""
-
-  def __init__(self, num_threads, exception_handler=_DefaultExceptionHandler):
-    self.tasks = Queue.Queue(num_threads)
-    self.threads = []
-    for _ in range(num_threads):
-      self.threads.append(Worker(self.tasks, exception_handler))
-
-  def AddTask(self, func, *args, **kargs):
-    """Add a task to the queue."""
-    self.tasks.put((func, args, kargs))
-
-  def WaitCompletion(self):
-    """Wait for completion of all the tasks in the queue."""
-    self.tasks.join()
-
-  def Shutdown(self):
-    """Shutdown the thread pool."""
-    for thread in self.threads:
-      self.tasks.put(_THREAD_EXIT_MAGIC)
-
-    for thread in self.threads:
-      thread.join()

data/packages/gsutil/gslib/util.py

@@ -1,151 +0,0 @@
-# Copyright 2010 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Static data and helper functions."""
-
-import math
-import re
-import sys
-
-import boto
-
-# We don't use the oauth2 authentication plugin directly; importing it here
-# ensures that it's loaded and available by default. Note: we made this static
-# state instead of Command instance state because the top-level gsutil code
-# needs to check it.
-HAVE_OAUTH2 = False
-try:
-  from oauth2_plugin import oauth2_helper
-  HAVE_OAUTH2 = True
-except ImportError:
-  pass
-
-TWO_MB = 2 * 1024 * 1024
-
-NO_MAX = sys.maxint
-
-# Binary exponentiation strings.
-_EXP_STRINGS = [
-  (0, 'B', 'bit'),
-  (10, 'KB', 'kbit'),
-  (20, 'MB', 'Mbit'),
-  (30, 'GB', 'Gbit'),
-  (40, 'TB', 'Tbit'),
-  (50, 'PB', 'Pbit'),
-]
-
-# Detect platform types.
-IS_WINDOWS = 'win32' in str(sys.platform).lower()
-IS_LINUX = 'linux' in str(sys.platform).lower()
-IS_OSX = 'darwin' in str(sys.platform).lower()
-
-# Enum class for specifying listing style.
-class ListingStyle(object):
-  SHORT = 'SHORT'
-  LONG = 'LONG'
-  LONG_LONG = 'LONG_LONG'
-
-
-def HasConfiguredCredentials():
-  """Determines if boto credential/config file exists."""
-  config = boto.config
-  has_goog_creds = (config.has_option('Credentials', 'gs_access_key_id') and
-                    config.has_option('Credentials', 'gs_secret_access_key'))
-  has_amzn_creds = (config.has_option('Credentials', 'aws_access_key_id') and
-                    config.has_option('Credentials', 'aws_secret_access_key'))
-  has_oauth_creds = (HAVE_OAUTH2 and
-      config.has_option('Credentials', 'gs_oauth2_refresh_token'))
-  has_auth_plugins = config.has_option('Plugin', 'plugin_directory')
-  return (has_goog_creds or has_amzn_creds or has_oauth_creds
-          or has_auth_plugins)
-
-
-def _RoundToNearestExponent(num):
-  i = 0
-  while i+1 < len(_EXP_STRINGS) and num >= (2 ** _EXP_STRINGS[i+1][0]):
-    i += 1
-  return i, round(float(num) / 2 ** _EXP_STRINGS[i][0], 2)
-
-def MakeHumanReadable(num):
-  """Generates human readable string for a number of bytes.
-
-  Args:
-    num: The number, in bytes.
-
-  Returns:
-    A string form of the number using size abbreviations (KB, MB, etc.).
-  """
-  i, rounded_val = _RoundToNearestExponent(num)
-  return '%s %s' % (rounded_val, _EXP_STRINGS[i][1])
-
-def MakeBitsHumanReadable(num):
-  """Generates human readable string for a number of bits.
-
-  Args:
-    num: The number, in bits.
-
-  Returns:
-    A string form of the number using bit size abbreviations (kbit, Mbit, etc.)
-  """
-  i, rounded_val = _RoundToNearestExponent(num)
-  return '%s %s' % (rounded_val, _EXP_STRINGS[i][2])
-
-def Percentile(values, percent, key=lambda x:x):
-  """Find the percentile of a list of values.
-
-  Taken from: http://code.activestate.com/recipes/511478/
-
-  Args:
-    values: a list of numeric values. Note that the values MUST BE already
-            sorted.
-    percent: a float value from 0.0 to 1.0.
-    key: optional key function to compute value from each element of the list
-         of values.
-
-  Returns:
-    The percentile of the values.
-  """
-  if not values:
-    return None
-  k = (len(values) - 1) * percent
-  f = math.floor(k)
-  c = math.ceil(k)
-  if f == c:
-    return key(values[int(k)])
-  d0 = key(values[int(f)]) * (c-k)
-  d1 = key(values[int(c)]) * (k-f)
-  return d0 + d1
-
-def ExtractErrorDetail(e):
-  """Extract <Details> text from XML content.
-
-  Args:
-    e: The GSResponseError that includes XML to be parsed.
-
-  Returns:
-    (exception_name, d), where d is <Details> text or None if not found.
-  """
-  exc_name_parts = re.split("[\.']", str(type(e)))
-  if len(exc_name_parts) < 2:
-    # Shouldn't happen, but have fallback in case.
-    exc_name = str(type(e))
-  else:
-    exc_name = exc_name_parts[-2]
-  if not hasattr(e, 'body'):
-    return (exc_name, None)
-  detail_start = e.body.find('<Details>')
-  detail_end = e.body.find('</Details>')
-  if detail_start != -1 and detail_end != -1:
-    return (exc_name, e.body[detail_start+9:detail_end])
-  return (exc_name, None)

data/packages/gsutil/gslib/wildcard_iterator.py

@@ -1,492 +0,0 @@
-# Copyright 2010 Google Inc. All Rights Reserved.
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish, dis-
-# tribute, sublicense, and/or sell copies of the Software, and to permit
-# persons to whom the Software is furnished to do so, subject to the fol-
-# lowing conditions:
-#
-# The above copyright notice and this permission notice shall be included
-# in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
-# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
-# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-
-"""Implementation of wildcarding over StorageUris.
-
-StorageUri is an abstraction that Google introduced in the boto library,
-for representing storage provider-independent bucket and object names with
-a shorthand URI-like syntax (see boto/boto/storage_uri.py) The current
-class provides wildcarding support for StorageUri objects (including both
-bucket and file system objects), allowing one to express collections of
-objects with syntax like the following:
-  gs://mybucket/images/*.png
-  file:///tmp/???abc???
-
-We provide wildcarding support as part of gsutil rather than as part
-of boto because wildcarding is really part of shell command-like
-functionality.
-
-A comment about wildcard semantics: We support both single path component
-wildcards (e.g., using '*') and recursive wildcards (using '**'), for both
-file and cloud URIs. For example,
-  gs://bucket/doc/*/*.html
-would enumerate HTML files one directory down from gs://bucket/doc, while
-  gs://bucket/**/*.html
-would enumerate HTML files in all objects contained in the bucket.
-
-Note also that if you use file system wildcards it's likely your shell
-interprets the wildcarding before passing the command to gsutil. For example:
-  % gsutil cp /opt/eclipse/*/*.html gs://bucket/eclipse
-would likely be expanded by the shell into the following before running gsutil:
-  % gsutil cp /opt/eclipse/RUNNING.html gs://bucket/eclipse
-
-Note also that most shells don't support '**' wildcarding (I think only
-zsh does). If you want to use '**' wildcarding with such a shell you can
-single quote each wildcarded string, so it gets passed uninterpreted by the
-shell to gsutil (at which point gsutil will perform the wildcarding expansion):
-  % gsutil cp '/opt/eclipse/**/*.html' gs://bucket/eclipse
-"""
-
-import boto
-import fnmatch
-import glob
-import os
-import re
-import sys
-import urllib
-
-from boto.s3.prefix import Prefix
-from boto.storage_uri import BucketStorageUri
-from bucket_listing_ref import BucketListingRef
-
-# Regex to determine if a string contains any wildcards.
-WILDCARD_REGEX = re.compile('[*?\[\]]')
-
-WILDCARD_OBJECT_ITERATOR = 'wildcard_object_iterator'
-WILDCARD_BUCKET_ITERATOR = 'wildcard_bucket_iterator'
-
-
-class WildcardIterator(object):
-  """Base class for wildcarding over StorageUris.
-
-  This class implements support for iterating over StorageUris that
-  contain wildcards.
-
-  The base class is abstract; you should instantiate using the
-  wildcard_iterator() static factory method, which chooses the right
-  implementation depending on the StorageUri.
-  """
-
-  def __repr__(self):
-    """Returns string representation of WildcardIterator."""
-    return 'WildcardIterator(%s)' % self.wildcard_uri
-
-
-class CloudWildcardIterator(WildcardIterator):
-  """WildcardIterator subclass for buckets and objects.
-
-  Iterates over BucketListingRef matching the StorageUri wildcard. It's
-  much more efficient to request the Key from the BucketListingRef (via
-  GetKey()) than to request the StorageUri and then call uri.get_key()
-  to retrieve the key, for cases where you want to get metadata that's
-  available in the Bucket (for example to get the name and size of
-  each object), because that information is available in the bucket GET
-  results. If you were to iterate over URIs for such cases and then get
-  the name and size info from each resulting StorageUri, it would cause
-  an additional object GET request for each of the result URIs.
-  """
-
-  def __init__(self, wildcard_uri, proj_id_handler,
-               bucket_storage_uri_class=BucketStorageUri, all_versions=False,
-               headers=None, debug=0):
-    """
-    Instantiates an iterator over BucketListingRef matching given wildcard URI.
-
-    Args:
-      wildcard_uri: StorageUri that contains the wildcard to iterate.
-      proj_id_handler: ProjectIdHandler to use for current command.
-      bucket_storage_uri_class: BucketStorageUri interface.
-                                Settable for testing/mocking.
-      headers: Dictionary containing optional HTTP headers to pass to boto.
-      debug: Debug level to pass in to boto connection (range 0..3).
-    """
-    self.wildcard_uri = wildcard_uri
-    # Make a copy of the headers so any updates we make during wildcard
-    # expansion aren't left in the input params (specifically, so we don't
-    # include the x-goog-project-id header needed by a subset of cases, in
-    # the data returned to caller, which could then be used in other cases
-    # where that header must not be passed).
-    if headers is None:
-      self.headers = {}
-    else:
-      self.headers = headers.copy()
-    self.proj_id_handler = proj_id_handler
-    self.bucket_storage_uri_class = bucket_storage_uri_class
-    self.all_versions = all_versions
-    self.debug = debug
-
-  def __iter__(self):
-    """Python iterator that gets called when iterating over cloud wildcard.
-
-    Yields:
-      BucketListingRef, or empty iterator if no matches.
-    """
-    # First handle bucket wildcarding, if any.
-    if ContainsWildcard(self.wildcard_uri.bucket_name):
-      regex = fnmatch.translate(self.wildcard_uri.bucket_name)
-      bucket_uris = []
-      prog = re.compile(regex)
-      self.proj_id_handler.FillInProjectHeaderIfNeeded(WILDCARD_BUCKET_ITERATOR,
-                                                       self.wildcard_uri,
-                                                       self.headers)
-      for b in self.wildcard_uri.get_all_buckets(headers=self.headers):
-        if prog.match(b.name):
-          # Use str(b.name) because get_all_buckets() returns Unicode
-          # string, which when used to construct x-goog-copy-src metadata
-          # requests for object-to-object copies causes pathname '/' chars
-          # to be entity-encoded (bucket%2Fdir instead of bucket/dir),
-          # which causes the request to fail.
-          uri_str = '%s://%s' % (self.wildcard_uri.scheme,
-                                 urllib.quote_plus(str(b.name)))
-          bucket_uris.append(
-              boto.storage_uri(
-                  uri_str, debug=self.debug,
-                  bucket_storage_uri_class=self.bucket_storage_uri_class,
-                  suppress_consec_slashes=False))
-    else:
-      bucket_uris = [self.wildcard_uri.clone_replace_name('')]
-
-    # Now iterate over bucket(s), and handle object wildcarding, if any.
-    self.proj_id_handler.FillInProjectHeaderIfNeeded(WILDCARD_OBJECT_ITERATOR,
-                                                     self.wildcard_uri,
-                                                     self.headers)
-    for bucket_uri in bucket_uris:
-      if self.wildcard_uri.names_bucket():
-        # Bucket-only URI.
-        yield BucketListingRef(bucket_uri, key=None, prefix=None,
-                               headers=self.headers)
-      else:
-        # URI contains an object name. If there's no wildcard just yield
-        # the needed URI.
-        if not ContainsWildcard(self.wildcard_uri.object_name):
-          uri_to_yield = bucket_uri.clone_replace_name(
-              self.wildcard_uri.object_name)
-          yield BucketListingRef(uri_to_yield, key=None, prefix=None,
-                          headers=self.headers)
-        else:
-          # URI contains a wildcard. Expand iteratively by building
-          # prefix/delimiter bucket listing request, filtering the results per
-          # the current level's wildcard, and continuing with the next component
-          # of the wildcard. See _BuildBucketFilterStrings() documentation
-          # for details.
-          #
-          # Initialize the iteration with bucket name from bucket_uri but
-          # object name from self.wildcard_uri. This is needed to handle cases
-          # where both the bucket and object names contain wildcards.
-          uris_needing_expansion = [
-              bucket_uri.clone_replace_name(self.wildcard_uri.object_name)]
-          while len(uris_needing_expansion) > 0:
-            uri = uris_needing_expansion.pop(0)
-            (prefix, delimiter, prefix_wildcard, suffix_wildcard) = (
-                self._BuildBucketFilterStrings(uri.object_name))
-            prog = re.compile(fnmatch.translate(prefix_wildcard))
-            # List bucket for objects matching prefix up to delimiter.
-            for key in bucket_uri.list_bucket(prefix=prefix,
-                                              delimiter=delimiter,
-                                              headers=self.headers,
-                                              all_versions=self.all_versions):
-              # Check that the prefix regex matches rstripped key.name (to
-              # correspond with the rstripped prefix_wildcard from
-              # _BuildBucketFilterStrings()).
-              if prog.match(key.name.rstrip('/')):
-                if suffix_wildcard and key.name.rstrip('/') != suffix_wildcard:
-                  if isinstance(key, Prefix):
-                    # There's more wildcard left to expand.
-                    uris_needing_expansion.append(
-                        uri.clone_replace_name(key.name.rstrip('/') + '/'
-                        + suffix_wildcard))
-                else:
-                  # Done expanding.
-                  expanded_uri = uri.clone_replace_key(key)
-
-                  if isinstance(key, Prefix):
-                    yield BucketListingRef(expanded_uri, key=None, prefix=key,
-                                           headers=self.headers)
-                  else:
-                    yield BucketListingRef(expanded_uri, key=key, prefix=None,
-                                           headers=self.headers)
-
-  def _BuildBucketFilterStrings(self, wildcard):
-    """
-    Builds strings needed for querying a bucket and filtering results to
-    implement wildcard object name matching.
-
-    Args:
-      wildcard: The wildcard string to match to objects.
-
-    Returns:
-      (prefix, delimiter, prefix_wildcard, suffix_wildcard)
-      where:
-        prefix is the prefix to be sent in bucket GET request.
-        delimiter is the delimiter to be sent in bucket GET request.
-        prefix_wildcard is the wildcard to be used to filter bucket GET results.
-        suffix_wildcard is wildcard to be appended to filtered bucket GET
-          results for next wildcard expansion iteration.
-      For example, given the wildcard gs://bucket/abc/d*e/f*.txt we
-      would build prefix= abc/d, delimiter=/, prefix_wildcard=d*e, and
-      suffix_wildcard=f*.txt. Using this prefix and delimiter for a bucket
-      listing request will then produce a listing result set that can be
-      filtered using this prefix_wildcard; and we'd use this suffix_wildcard
-      to feed into the next call(s) to _BuildBucketFilterStrings(), for the
-      next iteration of listing/filtering.
-
-    Raises:
-      AssertionError if wildcard doesn't contain any wildcard chars.
-    """
-    # Generate a request prefix if the object name part of the wildcard starts
-    # with a non-wildcard string (e.g., that's true for 'gs://bucket/abc*xyz').
-    match = WILDCARD_REGEX.search(wildcard)
-    if not match:
-      # Input "wildcard" has no wildcard chars, so just return tuple that will
-      # cause a bucket listing to match the given input wildcard. Example: if
-      # previous iteration yielded gs://bucket/dir/ with suffix_wildcard abc,
-      # the next iteration will call _BuildBucketFilterStrings() with
-      # gs://bucket/dir/abc, and we will return prefix ='dir/abc',
-      # delimiter='/', prefix_wildcard='dir/abc', and suffix_wildcard=''.
-      prefix = wildcard
-      delimiter = '/'
-      prefix_wildcard = wildcard
-      suffix_wildcard = ''
-    else:
-      if match.start() > 0:
-        # Wildcard does not occur at beginning of object name, so construct a
-        # prefix string to send to server.
-        prefix = wildcard[:match.start()]
-        wildcard_part = wildcard[match.start():]
-      else:
-        prefix = None
-        wildcard_part = wildcard
-      end = wildcard_part.find('/')
-      if end != -1:
-        wildcard_part = wildcard_part[:end+1]
-      # Remove trailing '/' so we will match gs://bucket/abc* as well as
-      # gs://bucket/abc*/ with the same wildcard regex.
-      prefix_wildcard = ((prefix or '') + wildcard_part).rstrip('/')
-      suffix_wildcard = wildcard[match.end():]
-      end = suffix_wildcard.find('/')
-      if end == -1:
-        suffix_wildcard = ''
-      else:
-        suffix_wildcard = suffix_wildcard[end+1:]
-      # To implement recursive (**) wildcarding, if prefix_wildcard
-      # suffix_wildcard starts with '**' don't send a delimiter, and combine
-      # suffix_wildcard at end of prefix_wildcard.
-      if prefix_wildcard.find('**') != -1:
-        delimiter = None
-        prefix_wildcard = prefix_wildcard + suffix_wildcard
-        suffix_wildcard = ''
-      else:
-        delimiter = '/'
-        delim_pos = suffix_wildcard.find(delimiter)
-    # The following debug output is useful for tracing how the algorithm
-    # walks through a multi-part wildcard like gs://bucket/abc/d*e/f*.txt
-    if self.debug > 1:
-      sys.stderr.write(
-          'DEBUG: wildcard=%s, prefix=%s, delimiter=%s, '
-          'prefix_wildcard=%s, suffix_wildcard=%s\n' %
-          (wildcard, prefix, delimiter, prefix_wildcard, suffix_wildcard))
-    return (prefix, delimiter, prefix_wildcard, suffix_wildcard)
-
-  def IterKeys(self):
-    """
-    Convenience iterator that runs underlying iterator and returns Key for each
-    iteration.
-
-    Yields:
-      Subclass of boto.s3.key.Key, or empty iterator if no matches.
-
-    Raises:
-      WildcardException: for bucket-only uri.
-    """
-    for bucket_listing_ref in self. __iter__():
-      if bucket_listing_ref.HasKey():
-        yield bucket_listing_ref.GetKey()
-
-  def IterUris(self):
-    """
-    Convenience iterator that runs underlying iterator and returns StorageUri
-    for each iteration.
-
-    Yields:
-      StorageUri, or empty iterator if no matches.
-    """
-    for bucket_listing_ref in self. __iter__():
-      yield bucket_listing_ref.GetUri()
-
-  def IterUrisForKeys(self):
-    """
-    Convenience iterator that runs underlying iterator and returns the
-    StorageUri for each iterated BucketListingRef that has a Key.
-
-    Yields:
-      StorageUri, or empty iterator if no matches.
-    """
-    for bucket_listing_ref in self. __iter__():
-      if bucket_listing_ref.HasKey():
-        yield bucket_listing_ref.GetUri()
-
-
-class FileWildcardIterator(WildcardIterator):
-  """WildcardIterator subclass for files and directories.
-
-  If you use recursive wildcards ('**') only a single such wildcard is
-  supported. For example you could use the wildcard '**/*.txt' to list all .txt
-  files in any subdirectory of the current directory, but you couldn't use a
-  wildcard like '**/abc/**/*.txt' (which would, if supported, let you find .txt
-  files in any subdirectory named 'abc').
-  """
-
-  def __init__(self, wildcard_uri, headers=None, debug=0):
-    """
-    Instantiate an iterator over BucketListingRefs matching given wildcard URI.
-
-    Args:
-      wildcard_uri: StorageUri that contains the wildcard to iterate.
-      headers: Dictionary containing optional HTTP headers to pass to boto.
-      debug: Debug level to pass in to boto connection (range 0..3).
-    """
-    self.wildcard_uri = wildcard_uri
-    self.headers = headers
-    self.debug = debug
-
-  def __iter__(self):
-    wildcard = self.wildcard_uri.object_name
-    match = re.search('\*\*', wildcard)
-    if match:
-      # Recursive wildcarding request ('.../**/...').
-      # Example input: wildcard = '/tmp/tmp2pQJAX/**/*'
-      base_dir = wildcard[:match.start()-1]
-      remaining_wildcard = wildcard[match.start()+2:]
-      # At this point for the above example base_dir = '/tmp/tmp2pQJAX' and
-      # remaining_wildcard = '/*'
-      if remaining_wildcard.startswith('*'):
-        raise WildcardException('Invalid wildcard with more than 2 consecutive '
-                                '*s (%s)' % wildcard)
-      # If there was no remaining wildcard past the recursive wildcard,
-      # treat it as if it were a '*'. For example, file://tmp/** is equivalent
-      # to file://tmp/**/*
-      if not remaining_wildcard:
-        remaining_wildcard = '*'
-      # Skip slash(es).
-      remaining_wildcard = remaining_wildcard.lstrip(os.sep)
-      filepaths = []
-      for dirpath, unused_dirnames, filenames in os.walk(base_dir):
-        filepaths.extend(
-            os.path.join(dirpath, f) for f in fnmatch.filter(filenames,
-                                                             remaining_wildcard)
-        )
-    else:
-      # Not a recursive wildcarding request.
-      filepaths = glob.glob(wildcard)
-    for filepath in filepaths:
-      expanded_uri = self.wildcard_uri.clone_replace_name(filepath)
-      yield BucketListingRef(expanded_uri)
-
-  def IterKeys(self):
-    """
-    Placeholder to allow polymorphic use of WildcardIterator.
-
-    Raises:
-      WildcardException: in all cases.
-    """
-    raise WildcardException(
-        'Iterating over Keys not possible for file wildcards')
-
-  def IterUris(self):
-    """
-    Convenience iterator that runs underlying iterator and returns StorageUri
-    for each iteration.
-
-    Yields:
-      StorageUri, or empty iterator if no matches.
-    """
-    for bucket_listing_ref in self. __iter__():
-      yield bucket_listing_ref.GetUri()
-
-
-class WildcardException(StandardError):
-  """Exception thrown for invalid wildcard URIs."""
-
-  def __init__(self, reason):
-    StandardError.__init__(self)
-    self.reason = reason
-
-  def __repr__(self):
-    return 'WildcardException: %s' % self.reason
-
-  def __str__(self):
-    return 'WildcardException: %s' % self.reason
-
-
-def wildcard_iterator(uri_or_str, proj_id_handler,
-                      bucket_storage_uri_class=BucketStorageUri,
-                      all_versions=False,
-                      headers=None, debug=0):
-  """Instantiate a WildCardIterator for the given StorageUri.
-
-  Args:
-    uri_or_str: StorageUri or URI string naming wildcard objects to iterate.
-    proj_id_handler: ProjectIdHandler to use for current command.
-    bucket_storage_uri_class: BucketStorageUri interface.
-        Settable for testing/mocking.
-    headers: Dictionary containing optional HTTP headers to pass to boto.
-    debug: Debug level to pass in to boto connection (range 0..3).
-
-  Returns:
-    A WildcardIterator that handles the requested iteration.
-  """
-
-  if isinstance(uri_or_str, basestring):
-    # Disable enforce_bucket_naming, to allow bucket names containing wildcard
-    # chars.
-    uri = boto.storage_uri(
-        uri_or_str, debug=debug, validate=False,
-        bucket_storage_uri_class=bucket_storage_uri_class,
-        suppress_consec_slashes=False)
-  else:
-    uri = uri_or_str
-
-  if uri.is_cloud_uri():
-    return CloudWildcardIterator(
-        uri, proj_id_handler,
-        bucket_storage_uri_class=bucket_storage_uri_class,
-        all_versions=all_versions,
-        headers=headers,
-        debug=debug)
-  elif uri.is_file_uri():
-    return FileWildcardIterator(uri, headers=headers, debug=debug)
-  else:
-    raise WildcardException('Unexpected type of StorageUri (%s)' % uri)
-
-
-def ContainsWildcard(uri_or_str):
-  """Checks whether uri_or_str contains a wildcard.
-
-  Args:
-    uri_or_str: StorageUri or URI string to check.
-
-  Returns:
-    bool indicator.
-  """
-  if isinstance(uri_or_str, basestring):
-    return bool(WILDCARD_REGEX.search(uri_or_str))
-  else:
-    return bool(WILDCARD_REGEX.search(uri_or_str.uri))