gcloud 0.0.7 → 0.1.0

data/packages/gsutil/gslib/exception.py

@@ -1,76 +0,0 @@
-# Copyright 2010 Google Inc. All Rights Reserved.
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish, dis-
-# tribute, sublicense, and/or sell copies of the Software, and to permit
-# persons to whom the Software is furnished to do so, subject to the fol-
-# lowing conditions:
-#
-# The above copyright notice and this permission notice shall be included
-# in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
-# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
-# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-
-"""gsutil exceptions."""
-
-
-class AbortException(StandardError):
-  """Exception raised when a user aborts a command that needs to do cleanup."""
-
-  def __init__(self, reason):
-    StandardError.__init__(self)
-    self.reason = reason
-
-  def __repr__(self):
-    return 'AbortException: %s' % self.reason
-
-  def __str__(self):
-    return 'AbortException: %s' % self.reason
-
-
-class CommandException(StandardError):
-  """Exception raised when a problem is encountered running a gsutil command.
-
-  This exception should be used to signal user errors or system failures
-  (like timeouts), not bugs (like an incorrect param value). For the
-  latter you should raise Exception so we can see where/how it happened
-  via gsutil -D (which will include a stack trace for raised Exceptions).
-  """
-
-  def __init__(self, reason, informational=False):
-    """Instantiate a CommandException.
-
-    Args:
-      reason: Text describing the problem.
-      informational: Indicates reason should be printed as FYI, not a failure.
-    """
-    StandardError.__init__(self)
-    self.reason = reason
-    self.informational = informational
-
-  def __repr__(self):
-    return 'CommandException: %s' % self.reason
-
-  def __str__(self):
-    return 'CommandException: %s' % self.reason
-
-
-class ProjectIdException(StandardError):
-
-  def __init__(self, reason):
-    StandardError.__init__(self)
-    self.reason = reason
-
-  def __repr__(self):
-    return 'ProjectIdException: %s' % self.reason
-
-  def __str__(self):
-    return 'ProjectIdException: %s' % self.reason

data/packages/gsutil/gslib/help_provider.py

@@ -1,81 +0,0 @@
-# Copyright 2012 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from gslib.exception import CommandException
-
-class HelpType(object):
-  COMMAND_HELP = 'command_help'
-  ADDITIONAL_HELP = 'additional_help'
-
-ALL_HELP_TYPES = [HelpType.COMMAND_HELP, HelpType.ADDITIONAL_HELP]
-
-# help_spec key constants.
-HELP_NAME = 'help_name'
-HELP_NAME_ALIASES = 'help_name_aliases'
-HELP_TYPE = 'help_type'
-HELP_ONE_LINE_SUMMARY = 'help_one_line_summary'
-HELP_TEXT = 'help_text'
-
-# Constants enforced by SanityCheck
-MAX_HELP_NAME_LEN = 15
-MIN_ONE_LINE_SUMMARY_LEN = 10
-MAX_ONE_LINE_SUMMARY_LEN = 80 - MAX_HELP_NAME_LEN
-
-REQUIRED_SPEC_KEYS = [HELP_NAME, HELP_NAME_ALIASES, HELP_TYPE,
-                      HELP_ONE_LINE_SUMMARY, HELP_TEXT]
-
-class HelpProvider(object):
-  """Interface for providing help."""
-
-  # Each subclass must define the following map.
-  help_spec = {
-    # Name of command or auxiliary help info for which this help applies.
-    HELP_NAME : None,
-    # List of help name aliases.
-    HELP_NAME_ALIASES : None,
-    # HelpType.
-    HELP_TYPE : None,
-    # One line summary of this help.
-    HELP_ONE_LINE_SUMMARY : None,
-    # The full help text.
-    HELP_TEXT : None,
-  }
-
-# This is a static helper instead of a class method because the help loader
-# (gslib.commands.help._LoadHelpMaps()) operates on classes not instances.
-def SanityCheck(help_provider, help_name_map):
-  """Helper for checking that a HelpProvider has minimally adequate content."""
-  for k in REQUIRED_SPEC_KEYS:
-    if k not in help_provider.help_spec or help_provider.help_spec[k] is None:
-      raise CommandException('"%s" help implementation is missing %s '
-                             'specification' % (help_provider.help_name, k))
-  # Sanity check the content.
-  assert (len(help_provider.help_spec[HELP_NAME]) > 1
-          and len(help_provider.help_spec[HELP_NAME]) < MAX_HELP_NAME_LEN)
-  for hna in help_provider.help_spec[HELP_NAME_ALIASES]:
-    assert len(hna) > 0
-  one_line_summary_len = len(help_provider.help_spec[HELP_ONE_LINE_SUMMARY])
-  assert (one_line_summary_len > MIN_ONE_LINE_SUMMARY_LEN
-          and one_line_summary_len < MAX_ONE_LINE_SUMMARY_LEN)
-  assert len(help_provider.help_spec[HELP_TEXT]) > 10
-
-  # Ensure there are no dupe help names or aliases across commands.
-  name_check_list = [help_provider.help_spec[HELP_NAME]]
-  name_check_list.extend(help_provider.help_spec[HELP_NAME_ALIASES])
-  for name_or_alias in name_check_list:
-    if help_name_map.has_key(name_or_alias):
-      raise CommandException(
-          'Duplicate help name/alias "%s" found while loading help from %s. '
-          'That name/alias was already taken by %s' % (name_or_alias,
-          help_provider.__module__, help_name_map[name_or_alias].__module__))

data/packages/gsutil/gslib/name_expansion.py

@@ -1,550 +0,0 @@
-# Copyright 2012 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import copy
-import threading
-import wildcard_iterator
-
-from bucket_listing_ref import BucketListingRef
-from gslib.exception import CommandException
-from gslib.plurality_checkable_iterator import PluralityCheckableIterator
-from gslib.storage_uri_builder import StorageUriBuilder
-from wildcard_iterator import ContainsWildcard
-
-"""
-Name expansion support for the various ways gsutil lets users refer to
-collections of data (via explicit wildcarding as well as directory,
-bucket, and bucket subdir implicit wildcarding). This class encapsulates
-the various rules for determining how these expansions are done.
-"""
-
-
-class NameExpansionResult(object):
-  """
-  Holds one fully expanded result from iterating over NameExpansionIterator.
-
-  The member data in this class need to be pickleable because
-  NameExpansionResult instances are passed through Multiprocessing.Queue. In
-  particular, don't include any boto state like StorageUri, since that pulls
-  in a big tree of objects, some of which aren't pickleable (and even if
-  they were, pickling/unpickling such a large object tree would result in
-  significant overhead).
-
-  The state held in this object is needed for handling the various naming cases
-  (e.g., copying from a single source URI to a directory generates different
-  dest URI names than copying multiple URIs to a directory, to be consistent
-  with naming rules used by the Unix cp command). For more details see comments
-  in _NameExpansionIterator.
-  """
-
-  def __init__(self, src_uri_str, is_multi_src_request,
-               src_uri_expands_to_multi, names_container, expanded_uri_str,
-               have_existing_dst_container=None, is_latest=False):
-    """
-    Args:
-      src_uri_str: string representation of StorageUri that was expanded.
-      is_multi_src_request: bool indicator whether src_uri_str expanded to more
-          than 1 BucketListingRef.
-      src_uri_expands_to_multi: bool indicator whether the current src_uri
-          expanded to more than 1 BucketListingRef.
-      names_container: Bool indicator whether src_uri names a container.
-      expanded_uri_str: string representation of StorageUri to which src_uri_str
-          expands.
-      have_existing_dst_container: bool indicator whether this is a copy
-          request to an existing bucket, bucket subdir, or directory. Default
-          None value should be used in cases where this is not needed (commands
-          other than cp).
-      is_latest: Bool indicating that the result represents the object's current
-          version.
-    """
-    self.src_uri_str = src_uri_str
-    self.is_multi_src_request = is_multi_src_request
-    self.src_uri_expands_to_multi = src_uri_expands_to_multi
-    self.names_container = names_container
-    self.expanded_uri_str = expanded_uri_str
-    self.have_existing_dst_container = have_existing_dst_container
-    self.is_latest = is_latest
-
-  def __repr__(self):
-    return '%s' % self.expanded_uri_str
-
-  def IsEmpty(self):
-    """Returns True if name expansion yielded no matches."""
-    return self.expanded_blr is None
-
-  def GetSrcUriStr(self):
-    """Returns the string representation of the StorageUri that was expanded."""
-    return self.src_uri_str
-
-  def IsMultiSrcRequest(self):
-    """
-    Returns bool indicator whether name expansion resulted in more than 0
-    BucketListingRef.
-    """
-    return self.is_multi_src_request
-
-  def SrcUriExpandsToMulti(self):
-    """
-    Returns bool indicator whether the current src_uri expanded to more than
-    1 BucketListingRef
-    """
-    return self.src_uri_expands_to_multi
-
-  def NamesContainer(self):
-    """
-    Returns bool indicator of whether src_uri names a directory, bucket, or
-    bucket subdir.
-    """
-    return self.names_container
-
-  def GetExpandedUriStr(self):
-    """
-    Returns the string representation of StorageUri to which src_uri_str
-    expands.
-    """
-    return self.expanded_uri_str
-
-  def HaveExistingDstContainer(self):
-    """Returns bool indicator whether this is a copy request to an
-       existing bucket, bucket subdir, or directory, or None if not
-       relevant."""
-    return self.have_existing_dst_container
-
-
-class _NameExpansionIterator(object):
-  """
-  Iterates over all src_uris, expanding wildcards, object-less bucket names,
-  subdir bucket names, and directory names, generating a flat listing of all
-  the matching objects/files.
-
-  You should instantiate this object using the static factory function
-  NameExpansionIterator, because consumers of this iterator need the
-  PluralityCheckableIterator wrapper built by that function.
-
-  Yields:
-    gslib.name_expansion.NameExpansionResult.
-
-  Raises:
-    CommandException: if errors encountered.
-  """
-
-  def __init__(self, command_name, proj_id_handler, headers, debug,
-               bucket_storage_uri_class, uri_strs, recursion_requested,
-               have_existing_dst_container=None, flat=True,
-               all_versions=False, for_all_version_delete=False):
-    """
-    Args:
-      command_name: name of command being run.
-      proj_id_handler: ProjectIdHandler to use for current command.
-      headers: Dictionary containing optional HTTP headers to pass to boto.
-      debug: Debug level to pass in to boto connection (range 0..3).
-      bucket_storage_uri_class: Class to instantiate for cloud StorageUris.
-          Settable for testing/mocking.
-      uri_strs: PluralityCheckableIterator of URI strings needing expansion.
-      recursion_requested: True if -R specified on command-line.
-      have_existing_dst_container: Bool indicator whether this is a copy
-          request to an existing bucket, bucket subdir, or directory. Default
-          None value should be used in cases where this is not needed (commands
-          other than cp).
-      flat: Bool indicating whether bucket listings should be flattened, i.e.,
-          so the mapped-to results contain objects spanning subdirectories.
-      all_versions: Bool indicating whether to iterate over all object versions.
-      for_all_version_delete: Bool indicating whether this is for an all-version
-          delete.
-
-    Examples of _NameExpansionIterator with flat=True:
-      - Calling with one of the uri_strs being 'gs://bucket' will enumerate all
-        top-level objects, as will 'gs://bucket/' and 'gs://bucket/*'.
-      - 'gs://bucket/**' will enumerate all objects in the bucket.
-      - 'gs://bucket/abc' will enumerate all next-level objects under directory
-        abc (i.e., not including subdirectories of abc) if gs://bucket/abc/*
-        matches any objects; otherwise it will enumerate the single name
-        gs://bucket/abc
-      - 'gs://bucket/abc/**' will enumerate all objects under abc or any of its
-        subdirectories.
-      - 'file:///tmp' will enumerate all files under /tmp, as will
-        'file:///tmp/*'
-      - 'file:///tmp/**' will enumerate all files under /tmp or any of its
-        subdirectories.
-
-    Example if flat=False: calling with gs://bucket/abc/* lists matching objects
-    or subdirs, but not sub-subdirs or objects beneath subdirs.
-
-    Note: In step-by-step comments below we give examples assuming there's a
-    gs://bucket with object paths:
-      abcd/o1.txt
-      abcd/o2.txt
-      xyz/o1.txt
-      xyz/o2.txt
-    and a directory file://dir with file paths:
-      dir/a.txt
-      dir/b.txt
-      dir/c/
-    """
-    self.command_name = command_name
-    self.proj_id_handler = proj_id_handler
-    self.headers = headers
-    self.debug = debug
-    self.bucket_storage_uri_class = bucket_storage_uri_class
-    self.suri_builder = StorageUriBuilder(debug, bucket_storage_uri_class)
-    self.uri_strs = uri_strs
-    self.recursion_requested = recursion_requested
-    self.have_existing_dst_container = have_existing_dst_container
-    self.flat = flat
-    self.all_versions = all_versions
-
-    # Map holding wildcard strings to use for flat vs subdir-by-subdir listings.
-    # (A flat listing means show all objects expanded all the way down.)
-    self._flatness_wildcard = {True: '**', False: '*'}
-
-  def __iter__(self):
-    for uri_str in self.uri_strs:
-      # Step 1: Expand any explicitly specified wildcards. The output from this
-      # step is an iterator of BucketListingRef.
-      # Starting with gs://buck*/abc* this step would expand to gs://bucket/abcd
-      if ContainsWildcard(uri_str):
-        post_step1_iter = self._WildcardIterator(uri_str)
-      else:
-        suri = self.suri_builder.StorageUri(uri_str)
-        post_step1_iter = iter([BucketListingRef(suri)])
-      post_step1_iter = PluralityCheckableIterator(post_step1_iter)
-
-      # Step 2: Expand bucket subdirs and versions. The output from this
-      # step is an iterator of (names_container, BucketListingRef).
-      # Starting with gs://bucket/abcd this step would expand to:
-      #   iter([(True, abcd/o1.txt), (True, abcd/o2.txt)]).
-      if self.flat and self.recursion_requested:
-        post_step2_iter = _ImplicitBucketSubdirIterator(self,
-            post_step1_iter, self.flat)
-      elif self.all_versions:
-        post_step2_iter = _AllVersionIterator(self, post_step1_iter,
-                                              headers=self.headers)
-      else:
-        post_step2_iter = _NonContainerTuplifyIterator(post_step1_iter)
-      post_step2_iter = PluralityCheckableIterator(post_step2_iter)
-
-      # Step 3. Expand directories and buckets. This step yields the iterated
-      # values. Starting with gs://bucket this step would expand to:
-      #  [abcd/o1.txt, abcd/o2.txt, xyz/o1.txt, xyz/o2.txt]
-      # Starting with file://dir this step would expand to:
-      #  [dir/a.txt, dir/b.txt, dir/c/]
-      exp_src_bucket_listing_refs = []
-      wc = self._flatness_wildcard[self.flat]
-      src_uri_expands_to_multi = (post_step1_iter.has_plurality()
-                                  or post_step2_iter.has_plurality())
-      is_multi_src_request = (self.uri_strs.has_plurality()
-                              or src_uri_expands_to_multi)
-
-      if post_step2_iter.is_empty():
-        raise CommandException('No URIs matched: %s' % uri_str)
-      for (names_container, blr) in post_step2_iter:
-        if (not blr.GetUri().names_container()
-            and (self.flat or not blr.HasPrefix())):
-          yield NameExpansionResult(uri_str, is_multi_src_request,
-                                    src_uri_expands_to_multi, names_container,
-                                    blr.GetUriString(),
-                                    self.have_existing_dst_container,
-                                    is_latest=blr.IsLatest())
-          continue
-        if not self.recursion_requested:
-          if blr.GetUri().is_file_uri():
-            desc = 'directory'
-          else:
-            desc = 'bucket'
-          print 'Omitting %s "%s". (Did you mean to do %s -R?)' % (
-              desc, blr.GetUri(), self.command_name)
-          continue
-        if blr.GetUri().is_file_uri():
-          # Convert dir to implicit recursive wildcard.
-          uri_to_iterate = '%s/%s' % (blr.GetUriString(), wc)
-        else:
-          # Convert bucket to implicit recursive wildcard.
-          uri_to_iterate = blr.GetUri().clone_replace_name(wc)
-        wc_iter = PluralityCheckableIterator(
-            self._WildcardIterator(uri_to_iterate))
-        src_uri_expands_to_multi = (src_uri_expands_to_multi
-                                    or wc_iter.has_plurality())
-        is_multi_src_request = (self.uri_strs.has_plurality()
-                                or src_uri_expands_to_multi)
-        for blr in wc_iter:
-          yield NameExpansionResult(uri_str, is_multi_src_request,
-                                    src_uri_expands_to_multi, True,
-                                    blr.GetUriString(),
-                                    self.have_existing_dst_container,
-                                    is_latest=blr.IsLatest())
-
-  def _WildcardIterator(self, uri_or_str):
-    """
-    Helper to instantiate gslib.WildcardIterator. Args are same as
-    gslib.WildcardIterator interface, but this method fills in most of the
-    values from instance state.
-
-    Args:
-      uri_or_str: StorageUri or URI string naming wildcard objects to iterate.
-    """
-    return wildcard_iterator.wildcard_iterator(
-        uri_or_str, self.proj_id_handler,
-        bucket_storage_uri_class=self.bucket_storage_uri_class,
-        headers=self.headers, debug=self.debug,
-        all_versions=self.all_versions)
-
-
-def NameExpansionIterator(command_name, proj_id_handler, headers, debug,
-                          bucket_storage_uri_class, uri_strs,
-                          recursion_requested,
-                          have_existing_dst_container=None, flat=True,
-                          all_versions=False,
-                          for_all_version_delete=False):
-  """
-  Static factory function for instantiating _NameExpansionIterator, which
-  wraps the resulting iterator in a PluralityCheckableIterator and checks
-  that it is non-empty. Also, allows uri_strs can be either an array or an
-  iterator.
-
-  Args:
-    command_name: name of command being run.
-    proj_id_handler: ProjectIdHandler to use for current command.
-    headers: Dictionary containing optional HTTP headers to pass to boto.
-    debug: Debug level to pass in to boto connection (range 0..3).
-    bucket_storage_uri_class: Class to instantiate for cloud StorageUris.
-        Settable for testing/mocking.
-    uri_strs: PluralityCheckableIterator of URI strings needing expansion.
-    recursion_requested: True if -R specified on command-line.
-    have_existing_dst_container: Bool indicator whether this is a copy
-        request to an existing bucket, bucket subdir, or directory. Default
-        None value should be used in cases where this is not needed (commands
-        other than cp).
-    flat: Bool indicating whether bucket listings should be flattened, i.e.,
-        so the mapped-to results contain objects spanning subdirectories.
-    all_versions: Bool indicating whether to iterate over all object versions.
-    for_all_version_delete: Bool indicating whether this is for an all-version
-        delete.
-
-  Examples of ExpandWildcardsAndContainers with flat=True:
-    - Calling with one of the uri_strs being 'gs://bucket' will enumerate all
-      top-level objects, as will 'gs://bucket/' and 'gs://bucket/*'.
-    - 'gs://bucket/**' will enumerate all objects in the bucket.
-    - 'gs://bucket/abc' will enumerate all next-level objects under directory
-      abc (i.e., not including subdirectories of abc) if gs://bucket/abc/*
-      matches any objects; otherwise it will enumerate the single name
-      gs://bucket/abc
-    - 'gs://bucket/abc/**' will enumerate all objects under abc or any of its
-      subdirectories.
-    - 'file:///tmp' will enumerate all files under /tmp, as will
-      'file:///tmp/*'
-    - 'file:///tmp/**' will enumerate all files under /tmp or any of its
-      subdirectories.
-
-  Example if flat=False: calling with gs://bucket/abc/* lists matching objects
-  or subdirs, but not sub-subdirs or objects beneath subdirs.
-
-  Note: In step-by-step comments below we give examples assuming there's a
-  gs://bucket with object paths:
-    abcd/o1.txt
-    abcd/o2.txt
-    xyz/o1.txt
-    xyz/o2.txt
-  and a directory file://dir with file paths:
-    dir/a.txt
-    dir/b.txt
-    dir/c/
-  """
-  uri_strs = PluralityCheckableIterator(uri_strs)
-  name_expansion_iterator = _NameExpansionIterator(
-      command_name, proj_id_handler, headers, debug, bucket_storage_uri_class,
-      uri_strs, recursion_requested, have_existing_dst_container, flat,
-      all_versions=all_versions, for_all_version_delete=for_all_version_delete)
-  name_expansion_iterator = PluralityCheckableIterator(name_expansion_iterator)
-  if name_expansion_iterator.is_empty():
-    raise CommandException('No URIs matched')
-  return name_expansion_iterator
-
-
-class NameExpansionIteratorQueue(object):
-  """
-  Wrapper around NameExpansionIterator that provides a Multiprocessing.Queue
-  facade.
-
-  Only a blocking get() function can be called, and the block and timeout
-  params on that function are ignored. All other class functions raise
-  NotImplementedError.
-
-  This class is thread safe.
-  """
-
-  def __init__(self, name_expansion_iterator, final_value):
-    self.name_expansion_iterator = name_expansion_iterator
-    self.final_value = final_value
-    self.lock = threading.Lock()
-
-  def qsize(self):
-    raise NotImplementedError(
-        "NameExpansionIteratorQueue.qsize() not implemented")
-
-  def empty(self):
-    raise NotImplementedError(
-        "NameExpansionIteratorQueue.empty() not implemented")
-
-  def full(self):
-    raise NotImplementedError(
-        "NameExpansionIteratorQueue.full() not implemented")
-
-  def put(self, obj=None, block=None, timeout=None):
-    raise NotImplementedError(
-        "NameExpansionIteratorQueue.put() not implemented")
-
-  def put_nowait(self, obj):
-    raise NotImplementedError(
-        "NameExpansionIteratorQueue.put_nowait() not implemented")
-
-  def get(self, block=None, timeout=None):
-    self.lock.acquire()
-    try:
-      if self.name_expansion_iterator.is_empty():
-        return self.final_value
-      return self.name_expansion_iterator.next()
-    finally:
-      self.lock.release()
-
-  def get_nowait(self):
-    raise NotImplementedError(
-        "NameExpansionIteratorQueue.get_nowait() not implemented")
-
-  def get_no_wait(self):
-    raise NotImplementedError(
-        "NameExpansionIteratorQueue.get_no_wait() not implemented")
-
-  def close(self):
-     raise NotImplementedError(
-         "NameExpansionIteratorQueue.close() not implemented")
-
-  def join_thread(self):
-    raise NotImplementedError(
-        "NameExpansionIteratorQueue.join_thread() not implemented")
-
-  def cancel_join_thread(self):
-    raise NotImplementedError(
-        "NameExpansionIteratorQueue.cancel_join_thread() not implemented")
-
-
-class _NonContainerTuplifyIterator(object):
-  """
-  Iterator that produces the tuple (False, blr) for each iteration
-  of blr_iter. Used for cases where blr_iter iterates over a set of
-  BucketListingRefs known not to name containers.
-  """
-
-  def __init__(self, blr_iter):
-    """
-    Args:
-      blr_iter: iterator of BucketListingRef.
-    """
-    self.blr_iter = blr_iter
-
-  def __iter__(self):
-    for blr in self.blr_iter:
-      yield (False, blr)
-
-
-class _ImplicitBucketSubdirIterator(object):
-
-  """
-  Iterator wrapper that iterates over blr_iter, performing implicit bucket
-  subdir expansion.
-
-  Each iteration yields tuple (names_container, expanded BucketListingRefs)
-    where names_container is true if URI names a directory, bucket,
-    or bucket subdir (vs how StorageUri.names_container() doesn't
-    handle latter case).
-
-  For example, iterating over [BucketListingRef("gs://abc")] would expand to:
-    [BucketListingRef("gs://abc/o1"), BucketListingRef("gs://abc/o2")]
-  if those subdir objects exist, and [BucketListingRef("gs://abc") otherwise.
-  """
-
-  def __init__(self, name_expansion_instance, blr_iter, flat):
-    """
-    Args:
-      name_expansion_instance: calling instance of NameExpansion class.
-      blr_iter: iterator of BucketListingRef.
-      flat: bool indicating whether bucket listings should be flattened, i.e.,
-          so the mapped-to results contain objects spanning subdirectories.
-    """
-    self.blr_iter = blr_iter
-    self.name_expansion_instance = name_expansion_instance
-    self.flat = flat
-
-  def __iter__(self):
-    for blr in self.blr_iter:
-      uri = blr.GetUri()
-      if uri.names_object():
-        # URI could be a bucket subdir.
-        implicit_subdir_iterator = PluralityCheckableIterator(
-            self.name_expansion_instance._WildcardIterator(
-                self.name_expansion_instance.suri_builder.StorageUri(
-                    '%s/%s' % (uri.uri.rstrip('/'),
-                    self.name_expansion_instance._flatness_wildcard[
-                        self.flat]))))
-        if not implicit_subdir_iterator.is_empty():
-          for exp_blr in implicit_subdir_iterator:
-            yield (True, exp_blr)
-        else:
-          yield (False, blr)
-      else:
-        yield (False, blr)
-
-class _AllVersionIterator(object):
-  """
-  Iterator wrapper that iterates over blr_iter, performing implicit version
-  expansion.
-
-  Output behavior is identical to that in _ImplicitBucketSubdirIterator above.
-
-  For example, iterating over [BucketListingRef("gs://abc/o1")] would expand to:
-    [BucketListingRef("gs://abc/o1#1234"), BucketListingRef("gs://abc/o1#1235")]
-  """
-
-  def __init__(self, name_expansion_instance, blr_iter, headers=None):
-    """
-    Args:
-      name_expansion_instance: calling instance of NameExpansion class.
-      blr_iter: iterator of BucketListingRef.
-      flat: bool indicating whether bucket listings should be flattened, i.e.,
-          so the mapped-to results contain objects spanning subdirectories.
-    """
-    self.blr_iter = blr_iter
-    self.name_expansion_instance = name_expansion_instance
-    self.headers = headers
-
-  def __iter__(self):
-    empty = True
-    for blr in self.blr_iter:
-      uri = blr.GetUri()
-      if not uri.names_object():
-        empty = False
-        yield (True, blr)
-        break
-      for key in uri.list_bucket(
-          prefix=uri.object_name, headers=self.headers, all_versions=True):
-        if key.name != uri.object_name:
-          # The desired entries will be alphabetically first in this listing.
-          break
-        version_blr = BucketListingRef(uri.clone_replace_key(key), key=key)
-        empty = False
-        yield (False, version_blr)
-      # If no version exists, yield the unversioned blr, and let the consuming
-      # operation fail. This mirrors behavior in _ImplicitBucketSubdirIterator.
-      if empty:
-        yield (False, blr)
-