gcloud 0.0.7 → 0.1.0

data/packages/gsutil/gslib/tests/testcase/base.py

@@ -1,89 +0,0 @@
-import os.path
-import random
-import shutil
-import tempfile
-
-from gslib.tests.util import unittest
-
-
-MAX_BUCKET_LENGTH = 63
-
-
-class GsUtilTestCase(unittest.TestCase):
-  """Base test case class for unit and integration tests."""
-
-  def setUp(self):
-    self.tempdirs = []
-
-  def tearDown(self):
-    while self.tempdirs:
-      tmpdir = self.tempdirs.pop()
-      shutil.rmtree(tmpdir, ignore_errors=True)
-
-  def assertNumLines(self, text, numlines):
-    self.assertEqual(text.count('\n'), numlines)
-
-  def MakeTempName(self, kind):
-    """Creates a temporary name that is most-likely unique.
-
-    Args:
-      kind: A string indicating what kind of test name this is.
-
-    Returns:
-      The temporary name.
-    """
-    name = 'gsutil-test-%s-%s' % (self._testMethodName, kind)
-    name = name[:MAX_BUCKET_LENGTH-9]
-    name = '%s-%08x' % (name, random.randrange(256**4))
-    return name
-
-  def CreateTempDir(self, test_files=0):
-    """Creates a temporary directory on disk.
-
-    The directory and all of its contents will be deleted after the test.
-
-    Args:
-      test_files: The number of test files to place in the directory or a list
-                  of test file names.
-
-    Returns:
-      The path to the new temporary directory.
-    """
-    tmpdir = tempfile.mkdtemp(prefix=self.MakeTempName('directory'))
-    self.tempdirs.append(tmpdir)
-    try:
-      iter(test_files)
-    except TypeError:
-      test_files = [self.MakeTempName('file') for _ in range(test_files)]
-    for i, name in enumerate(test_files):
-      self.CreateTempFile(tmpdir=tmpdir, file_name=name, contents='test %d' % i)
-    return tmpdir
-
-  def CreateTempFile(self, tmpdir=None, contents=None, file_name=None):
-    """Creates a temporary file on disk.
-
-    Args:
-      tmpdir: The temporary directory to place the file in. If not specified, a
-              new temporary directory is created.
-      file_name: The name to use for the file. If not specified, a temporary
-                 test file name is constructed. This can also be a tuple, where
-                 ('dir', 'foo') means to create a file named 'foo' inside a
-                 subdirectory named 'dir'.
-      contents: The contents to write to the file. If not specified, a test
-                string is constructed and written to the file.
-
-    Returns:
-      The path to the new temporary file.
-    """
-    tmpdir = tmpdir or self.CreateTempDir()
-    file_name = file_name or self.MakeTempName('file')
-    if isinstance(file_name, basestring):
-      fpath = os.path.join(tmpdir, file_name)
-    else:
-      fpath = os.path.join(tmpdir, *file_name)
-    if not os.path.isdir(os.path.dirname(fpath)):
-      os.makedirs(os.path.dirname(fpath))
-    with open(fpath, 'w') as f:
-      contents = contents or self.MakeTempName('contents')
-      f.write(contents)
-    return fpath

data/packages/gsutil/gslib/tests/testcase/integration_testcase.py

@@ -1,197 +0,0 @@
-# Copyright 2013 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Contains gsutil base integration test case class."""
-
-import logging
-import os.path
-import subprocess
-import sys
-
-import boto
-from boto.exception import GSResponseError
-
-from gslib.project_id import ProjectIdHandler
-import gslib.tests.util as util
-from gslib.tests.util import Retry
-from gslib.tests.util import unittest
-from gslib.util import IS_WINDOWS
-import base
-
-
-CURDIR = os.path.abspath(os.path.dirname(__file__))
-TESTS_DIR = os.path.split(CURDIR)[0]
-GSLIB_DIR = os.path.split(TESTS_DIR)[0]
-GSUTIL_DIR = os.path.split(GSLIB_DIR)[0]
-GSUTIL_PATH = os.path.join(GSUTIL_DIR, 'gsutil')
-LOGGER = logging.getLogger('integration-test')
-
-
-@unittest.skipUnless(util.RUN_INTEGRATION_TESTS,
-                     'Not running integration tests.')
-class GsUtilIntegrationTestCase(base.GsUtilTestCase):
-  """Base class for gsutil integration tests."""
-  GROUP_TEST_ADDRESS = 'gs-discussion@googlegroups.com'
-  GROUP_TEST_ID = '00b4903a97d097895ab58ef505d535916a712215b79c3e54932c2eb502ad97f5'
-  USER_TEST_ADDRESS = 'gs-team@google.com'
-  USER_TEST_ID = '00b4903a9703325c6bfc98992d72e75600387a64b3b6bee9ef74613ef8842080'
-  DOMAIN_TEST = 'google.com'
-
-  def setUp(self):
-    super(GsUtilIntegrationTestCase, self).setUp()
-    self.bucket_uris = []
-
-    # Set up API version and project ID handler.
-    self.api_version = boto.config.get_value(
-        'GSUtil', 'default_api_version', '1')
-    self.proj_id_handler = ProjectIdHandler()
-
-  # Retry with an exponential backoff if a server error is received. This
-  # ensures that we try *really* hard to clean up after ourselves.
-  @Retry(GSResponseError, logger=LOGGER)
-  def tearDown(self):
-    super(GsUtilIntegrationTestCase, self).tearDown()
-
-    while self.bucket_uris:
-      bucket_uri = self.bucket_uris[-1]
-      bucket_list = list(bucket_uri.list_bucket(all_versions=True))
-      while bucket_list:
-        for k in bucket_list:
-          k.delete()
-        bucket_list = list(bucket_uri.list_bucket(all_versions=True))
-      bucket_uri.delete_bucket()
-      self.bucket_uris.pop()
-
-  def CreateBucket(self, bucket_name=None, test_objects=0, storage_class=None):
-    """Creates a test bucket.
-
-    The bucket and all of its contents will be deleted after the test.
-
-    Args:
-      bucket_name: Create the bucket with this name. If not provided, a
-                   temporary test bucket name is constructed.
-      test_objects: The number of objects that should be placed in the bucket.
-                    Defaults to 0.
-      storage_class: storage class to use. If not provided we us standard.
-
-    Returns:
-      StorageUri for the created bucket.
-    """
-    bucket_name = bucket_name or self.MakeTempName('bucket')
-
-    bucket_uri = boto.storage_uri('gs://%s' % bucket_name.lower(),
-                                  suppress_consec_slashes=False)
-
-    # Apply API version and project ID headers if necessary.
-    headers = {'x-goog-api-version': self.api_version}
-    self.proj_id_handler.FillInProjectHeaderIfNeeded(
-        'test', bucket_uri, headers)
-
-    bucket_uri.create_bucket(storage_class=storage_class, headers=headers)
-    self.bucket_uris.append(bucket_uri)
-    for i in range(test_objects):
-      self.CreateObject(bucket_uri=bucket_uri,
-                        object_name=self.MakeTempName('obj'),
-                        contents='test %d' % i)
-    return bucket_uri
-
-  def CreateVersionedBucket(self, bucket_name=None, test_objects=0):
-    """Creates a versioned test bucket.
-
-    The bucket and all of its contents will be deleted after the test.
-
-    Args:
-      bucket_name: Create the bucket with this name. If not provided, a
-                   temporary test bucket name is constructed.
-      test_objects: The number of objects that should be placed in the bucket.
-                    Defaults to 0.
-
-    Returns:
-      StorageUri for the created bucket with versioning enabled.
-    """
-    bucket_uri = self.CreateBucket(bucket_name=bucket_name,
-                               test_objects=test_objects)
-    bucket_uri.configure_versioning(True)
-    return bucket_uri
-
-  def CreateObject(self, bucket_uri=None, object_name=None, contents=None):
-    """Creates a test object.
-
-    Args:
-      bucket: The URI of the bucket to place the object in. If not specified, a
-              new temporary bucket is created.
-      object_name: The name to use for the object. If not specified, a temporary
-                   test object name is constructed.
-      contents: The contents to write to the object. If not specified, the key
-                is not written to, which means that it isn't actually created
-                yet on the server.
-
-    Returns:
-      A StorageUri for the created object.
-    """
-    bucket_uri = bucket_uri or self.CreateBucket()
-    object_name = object_name or self.MakeTempName('obj')
-    key_uri = bucket_uri.clone_replace_name(object_name)
-    if contents is not None:
-      key_uri.set_contents_from_string(contents)
-    return key_uri
-
-  def RunGsUtil(self, cmd, return_status=False, return_stdout=False,
-                return_stderr=False, expected_status=0, stdin=None):
-    """Runs the gsutil command.
-
-    Args:
-      cmd: The command to run, as a list, e.g. ['cp', 'foo', 'bar']
-      return_status: If True, the exit status code is returned.
-      return_stdout: If True, the standard output of the command is returned.
-      return_stderr: If True, the standard error of the command is returned.
-      expected_status: The expected return code. If not specified, defaults to
-                       0. If the return code is a different value, an exception
-                       is raised.
-      stdin: A string of data to pipe to the process as standard input.
-
-    Returns:
-      A tuple containing the desired return values specified by the return_*
-      arguments.
-    """
-    cmd = [GSUTIL_PATH] + cmd
-    if IS_WINDOWS:
-      cmd = [sys.executable] + cmd
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                         stdin=subprocess.PIPE)
-    (stdout, stderr) = p.communicate(stdin)
-    status = p.returncode
-
-    if expected_status is not None:
-      self.assertEqual(
-          status, expected_status,
-          msg='Expected status %d, got %d.\nCommand:\n%s\n\nstderr:\n%s' % (
-              expected_status, status, ' '.join(cmd), stderr))
-
-    toreturn = []
-    if return_status:
-      toreturn.append(status)
-    if return_stdout:
-      if IS_WINDOWS:
-        stdout = stdout.replace('\r\n', '\n')
-      toreturn.append(stdout)
-    if return_stderr:
-      if IS_WINDOWS:
-        stderr = stderr.replace('\r\n', '\n')
-      toreturn.append(stderr)
-
-    if len(toreturn) == 1:
-      return toreturn[0]
-    elif toreturn:
-      return tuple(toreturn)

data/packages/gsutil/gslib/tests/testcase/unit_testcase.py

@@ -1,230 +0,0 @@
-# Copyright 2013 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Contains gsutil base unit test case class."""
-
-import os.path
-import sys
-import tempfile
-
-import boto
-
-from gslib import wildcard_iterator
-from gslib.command_runner import CommandRunner
-from gslib.project_id import ProjectIdHandler
-import gslib.tests.util as util
-from gslib.tests.util import unittest
-from tests.integration.s3 import mock_storage_service
-import base
-
-
-CURDIR = os.path.abspath(os.path.dirname(__file__))
-TESTS_DIR = os.path.split(CURDIR)[0]
-GSLIB_DIR = os.path.split(TESTS_DIR)[0]
-GSUTIL_DIR = os.path.split(GSLIB_DIR)[0]
-BOTO_DIR = os.path.join(GSUTIL_DIR, 'boto')
-
-
-class GSMockConnection(mock_storage_service.MockConnection):
-
-  def __init__(self, *args, **kwargs):
-    kwargs['provider'] = 'gs'
-    super(GSMockConnection, self).__init__(*args, **kwargs)
-
-mock_connection = GSMockConnection()
-
-
-class GSMockBucketStorageUri(mock_storage_service.MockBucketStorageUri):
-
-  def connect(self, access_key_id=None, secret_access_key=None):
-    return mock_connection
-
-
-@unittest.skipUnless(util.RUN_UNIT_TESTS,
-                     'Not running integration tests.')
-class GsUtilUnitTestCase(base.GsUtilTestCase):
-  """Base class for gsutil unit tests."""
-
-  @classmethod
-  def setUpClass(cls):
-    base.GsUtilTestCase.setUpClass()
-    cls.mock_bucket_storage_uri = GSMockBucketStorageUri
-    cls.proj_id_handler = ProjectIdHandler()
-    config_file_list = boto.pyami.config.BotoConfigLocations
-    # Use "gsutil_test_commands" as a fake UserAgent. This value will never be
-    # sent via HTTP because we're using MockStorageService here.
-    cls.command_runner = CommandRunner(GSUTIL_DIR, BOTO_DIR,
-                                       config_file_list, 'gsutil_test_commands',
-                                       cls.mock_bucket_storage_uri)
-
-  def setUp(self):
-    super(GsUtilUnitTestCase, self).setUp()
-    self.bucket_uris = []
-
-  def RunCommand(self, command_name, args=None, headers=None, debug=0,
-                 test_method=None, return_stdout=False, cwd=None):
-    """
-    Method for calling gslib.command_runner.CommandRunner, passing
-    parallel_operations=False for all tests, optionally saving/returning stdout
-    output. We run all tests multi-threaded, to exercise those more complicated
-    code paths.
-    TODO: change to run with parallel_operations=True for all tests. At
-    present when you do this it causes many test failures.
-
-    Args:
-      command_name: The name of the command being run.
-      args: Command-line args (arg0 = actual arg, not command name ala bash).
-      headers: Dictionary containing optional HTTP headers to pass to boto.
-      debug: Debug level to pass in to boto connection (range 0..3).
-      parallel_operations: Should command operations be executed in parallel?
-      test_method: Optional general purpose method for testing purposes.
-                   Application and semantics of this method will vary by
-                   command and test type.
-      cwd: The working directory that should be switched to before running the
-           command. The working directory will be reset back to its original
-           value after running the command. If not specified, the working
-           directory is left unchanged.
-      return_stdout: If true will save and return stdout produced by command.
-    """
-    if util.VERBOSE_OUTPUT:
-      sys.stderr.write('\nRunning test of %s %s\n' %
-                       (command_name, ' '.join(args)))
-    if return_stdout:
-      # Redirect stdout temporarily, to save output to a file.
-      fh, outfile = tempfile.mkstemp()
-      os.close(fh)
-    elif not util.VERBOSE_OUTPUT:
-      outfile = os.devnull
-    else:
-      outfile = None
-
-    stdout_sav = sys.stdout
-    output = None
-    cwd_sav = None
-    try:
-      cwd_sav = os.getcwd()
-    except OSError:
-      # This can happen if the current working directory no longer exists.
-      pass
-    try:
-      if outfile:
-        fp = open(outfile, 'w')
-        sys.stdout = fp
-      if cwd:
-        os.chdir(cwd)
-      self.command_runner.RunNamedCommand(
-          command_name, args=args, headers=headers, debug=debug,
-          parallel_operations=False, test_method=test_method)
-    finally:
-      if cwd and cwd_sav:
-        os.chdir(cwd_sav)
-      if outfile:
-        fp.close()
-        sys.stdout = stdout_sav
-        with open(outfile, 'r') as f:
-          output = f.read()
-        if return_stdout:
-          os.unlink(outfile)
-
-    if output is not None and return_stdout:
-      return output
-
-  @classmethod
-  def _test_wildcard_iterator(cls, uri_or_str, debug=0):
-    """
-    Convenience method for instantiating a testing instance of
-    WildCardIterator, without having to specify all the params of that class
-    (like bucket_storage_uri_class=mock_storage_service.MockBucketStorageUri).
-    Also naming the factory method this way makes it clearer in the test code
-    that WildcardIterator needs to be set up for testing.
-
-    Args are same as for wildcard_iterator.wildcard_iterator(), except there's
-    no bucket_storage_uri_class arg.
-
-    Returns:
-      WildcardIterator.IterUris(), over which caller can iterate.
-    """
-    return wildcard_iterator.wildcard_iterator(
-        uri_or_str, cls.proj_id_handler, cls.mock_bucket_storage_uri,
-        debug=debug)
-
-  @staticmethod
-  def _test_storage_uri(uri_str, default_scheme='file', debug=0,
-                        validate=True):
-    """
-    Convenience method for instantiating a testing
-    instance of StorageUri, without having to specify
-    bucket_storage_uri_class=mock_storage_service.MockBucketStorageUri.
-    Also naming the factory method this way makes it clearer in the test
-    code that StorageUri needs to be set up for testing.
-
-    Args, Returns, and Raises are same as for boto.storage_uri(), except there's
-    no bucket_storage_uri_class arg.
-    """
-    return boto.storage_uri(uri_str, default_scheme, debug, validate,
-                            GSMockBucketStorageUri)
-
-  def CreateBucket(self, bucket_name=None, test_objects=0, storage_class=None):
-    """Creates a test bucket.
-
-    The bucket and all of its contents will be deleted after the test.
-
-    Args:
-      bucket_name: Create the bucket with this name. If not provided, a
-                   temporary test bucket name is constructed.
-      test_objects: The number of objects that should be placed in the bucket or
-                    a list of object names to place in the bucket. Defaults to
-                    0.
-      storage_class: storage class to use. If not provided we us standard.
-
-    Returns:
-      StorageUri for the created bucket.
-    """
-    bucket_name = bucket_name or self.MakeTempName('bucket')
-    bucket_uri = boto.storage_uri(
-        'gs://%s' % bucket_name.lower(),
-        suppress_consec_slashes=False,
-        bucket_storage_uri_class=GSMockBucketStorageUri)
-    bucket_uri.create_bucket(storage_class=storage_class)
-    self.bucket_uris.append(bucket_uri)
-    try:
-      iter(test_objects)
-    except TypeError:
-      test_objects = [self.MakeTempName('obj') for _ in range(test_objects)]
-    for i, name in enumerate(test_objects):
-      self.CreateObject(bucket_uri=bucket_uri, object_name=name,
-                        contents='test %d' % i)
-    return bucket_uri
-
-  def CreateObject(self, bucket_uri=None, object_name=None, contents=None):
-    """Creates a test object.
-
-    Args:
-      bucket: The URI of the bucket to place the object in. If not specified, a
-              new temporary bucket is created.
-      object_name: The name to use for the object. If not specified, a temporary
-                   test object name is constructed.
-      contents: The contents to write to the object. If not specified, the key
-                is not written to, which means that it isn't actually created
-                yet on the server.
-
-    Returns:
-      A StorageUri for the created object.
-    """
-    bucket_uri = bucket_uri or self.CreateBucket()
-    object_name = object_name or self.MakeTempName('obj')
-    key_uri = bucket_uri.clone_replace_name(object_name)
-    if contents is not None:
-      key_uri.set_contents_from_string(contents)
-    return key_uri