googlecloud 0.0.4 → 0.0.6

data/packages/gsutil/gslib/tests/testcase/__init__.py

@@ -0,0 +1,18 @@
+# Copyright 2013 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Contains test case base classes for gsutil tests."""
+
+from integration_testcase import GsUtilIntegrationTestCase
+from unit_testcase import GsUtilUnitTestCase

data/packages/gsutil/gslib/tests/testcase/base.py

@@ -0,0 +1,89 @@
+import os.path
+import random
+import shutil
+import tempfile
+
+from gslib.tests.util import unittest
+
+
+MAX_BUCKET_LENGTH = 63
+
+
+class GsUtilTestCase(unittest.TestCase):
+  """Base test case class for unit and integration tests."""
+
+  def setUp(self):
+    self.tempdirs = []
+
+  def tearDown(self):
+    while self.tempdirs:
+      tmpdir = self.tempdirs.pop()
+      shutil.rmtree(tmpdir, ignore_errors=True)
+
+  def assertNumLines(self, text, numlines):
+    self.assertEqual(text.count('\n'), numlines)
+
+  def MakeTempName(self, kind):
+    """Creates a temporary name that is most-likely unique.
+
+    Args:
+      kind: A string indicating what kind of test name this is.
+
+    Returns:
+      The temporary name.
+    """
+    name = 'gsutil-test-%s-%s' % (self._testMethodName, kind)
+    name = name[:MAX_BUCKET_LENGTH-9]
+    name = '%s-%08x' % (name, random.randrange(256**4))
+    return name
+
+  def CreateTempDir(self, test_files=0):
+    """Creates a temporary directory on disk.
+
+    The directory and all of its contents will be deleted after the test.
+
+    Args:
+      test_files: The number of test files to place in the directory or a list
+                  of test file names.
+
+    Returns:
+      The path to the new temporary directory.
+    """
+    tmpdir = tempfile.mkdtemp(prefix=self.MakeTempName('directory'))
+    self.tempdirs.append(tmpdir)
+    try:
+      iter(test_files)
+    except TypeError:
+      test_files = [self.MakeTempName('file') for _ in range(test_files)]
+    for i, name in enumerate(test_files):
+      self.CreateTempFile(tmpdir=tmpdir, file_name=name, contents='test %d' % i)
+    return tmpdir
+
+  def CreateTempFile(self, tmpdir=None, contents=None, file_name=None):
+    """Creates a temporary file on disk.
+
+    Args:
+      tmpdir: The temporary directory to place the file in. If not specified, a
+              new temporary directory is created.
+      file_name: The name to use for the file. If not specified, a temporary
+                 test file name is constructed. This can also be a tuple, where
+                 ('dir', 'foo') means to create a file named 'foo' inside a
+                 subdirectory named 'dir'.
+      contents: The contents to write to the file. If not specified, a test
+                string is constructed and written to the file.
+
+    Returns:
+      The path to the new temporary file.
+    """
+    tmpdir = tmpdir or self.CreateTempDir()
+    file_name = file_name or self.MakeTempName('file')
+    if isinstance(file_name, basestring):
+      fpath = os.path.join(tmpdir, file_name)
+    else:
+      fpath = os.path.join(tmpdir, *file_name)
+    if not os.path.isdir(os.path.dirname(fpath)):
+      os.makedirs(os.path.dirname(fpath))
+    with open(fpath, 'w') as f:
+      contents = contents or self.MakeTempName('contents')
+      f.write(contents)
+    return fpath

data/packages/gsutil/gslib/tests/testcase/integration_testcase.py

@@ -0,0 +1,197 @@
+# Copyright 2013 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Contains gsutil base integration test case class."""
+
+import logging
+import os.path
+import subprocess
+import sys
+
+import boto
+from boto.exception import GSResponseError
+
+from gslib.project_id import ProjectIdHandler
+import gslib.tests.util as util
+from gslib.tests.util import Retry
+from gslib.tests.util import unittest
+from gslib.util import IS_WINDOWS
+import base
+
+
+CURDIR = os.path.abspath(os.path.dirname(__file__))
+TESTS_DIR = os.path.split(CURDIR)[0]
+GSLIB_DIR = os.path.split(TESTS_DIR)[0]
+GSUTIL_DIR = os.path.split(GSLIB_DIR)[0]
+GSUTIL_PATH = os.path.join(GSUTIL_DIR, 'gsutil')
+LOGGER = logging.getLogger('integration-test')
+
+
+@unittest.skipUnless(util.RUN_INTEGRATION_TESTS,
+                     'Not running integration tests.')
+class GsUtilIntegrationTestCase(base.GsUtilTestCase):
+  """Base class for gsutil integration tests."""
+  GROUP_TEST_ADDRESS = 'gs-discussion@googlegroups.com'
+  GROUP_TEST_ID = '00b4903a97d097895ab58ef505d535916a712215b79c3e54932c2eb502ad97f5'
+  USER_TEST_ADDRESS = 'gs-team@google.com'
+  USER_TEST_ID = '00b4903a9703325c6bfc98992d72e75600387a64b3b6bee9ef74613ef8842080'
+  DOMAIN_TEST = 'google.com'
+
+  def setUp(self):
+    super(GsUtilIntegrationTestCase, self).setUp()
+    self.bucket_uris = []
+
+    # Set up API version and project ID handler.
+    self.api_version = boto.config.get_value(
+        'GSUtil', 'default_api_version', '1')
+    self.proj_id_handler = ProjectIdHandler()
+
+  # Retry with an exponential backoff if a server error is received. This
+  # ensures that we try *really* hard to clean up after ourselves.
+  @Retry(GSResponseError, logger=LOGGER)
+  def tearDown(self):
+    super(GsUtilIntegrationTestCase, self).tearDown()
+
+    while self.bucket_uris:
+      bucket_uri = self.bucket_uris[-1]
+      bucket_list = list(bucket_uri.list_bucket(all_versions=True))
+      while bucket_list:
+        for k in bucket_list:
+          k.delete()
+        bucket_list = list(bucket_uri.list_bucket(all_versions=True))
+      bucket_uri.delete_bucket()
+      self.bucket_uris.pop()
+
+  def CreateBucket(self, bucket_name=None, test_objects=0, storage_class=None):
+    """Creates a test bucket.
+
+    The bucket and all of its contents will be deleted after the test.
+
+    Args:
+      bucket_name: Create the bucket with this name. If not provided, a
+                   temporary test bucket name is constructed.
+      test_objects: The number of objects that should be placed in the bucket.
+                    Defaults to 0.
+      storage_class: storage class to use. If not provided we us standard.
+
+    Returns:
+      StorageUri for the created bucket.
+    """
+    bucket_name = bucket_name or self.MakeTempName('bucket')
+
+    bucket_uri = boto.storage_uri('gs://%s' % bucket_name.lower(),
+                                  suppress_consec_slashes=False)
+
+    # Apply API version and project ID headers if necessary.
+    headers = {'x-goog-api-version': self.api_version}
+    self.proj_id_handler.FillInProjectHeaderIfNeeded(
+        'test', bucket_uri, headers)
+
+    bucket_uri.create_bucket(storage_class=storage_class, headers=headers)
+    self.bucket_uris.append(bucket_uri)
+    for i in range(test_objects):
+      self.CreateObject(bucket_uri=bucket_uri,
+                        object_name=self.MakeTempName('obj'),
+                        contents='test %d' % i)
+    return bucket_uri
+
+  def CreateVersionedBucket(self, bucket_name=None, test_objects=0):
+    """Creates a versioned test bucket.
+
+    The bucket and all of its contents will be deleted after the test.
+
+    Args:
+      bucket_name: Create the bucket with this name. If not provided, a
+                   temporary test bucket name is constructed.
+      test_objects: The number of objects that should be placed in the bucket.
+                    Defaults to 0.
+
+    Returns:
+      StorageUri for the created bucket with versioning enabled.
+    """
+    bucket_uri = self.CreateBucket(bucket_name=bucket_name,
+                               test_objects=test_objects)
+    bucket_uri.configure_versioning(True)
+    return bucket_uri
+
+  def CreateObject(self, bucket_uri=None, object_name=None, contents=None):
+    """Creates a test object.
+
+    Args:
+      bucket: The URI of the bucket to place the object in. If not specified, a
+              new temporary bucket is created.
+      object_name: The name to use for the object. If not specified, a temporary
+                   test object name is constructed.
+      contents: The contents to write to the object. If not specified, the key
+                is not written to, which means that it isn't actually created
+                yet on the server.
+
+    Returns:
+      A StorageUri for the created object.
+    """
+    bucket_uri = bucket_uri or self.CreateBucket()
+    object_name = object_name or self.MakeTempName('obj')
+    key_uri = bucket_uri.clone_replace_name(object_name)
+    if contents is not None:
+      key_uri.set_contents_from_string(contents)
+    return key_uri
+
+  def RunGsUtil(self, cmd, return_status=False, return_stdout=False,
+                return_stderr=False, expected_status=0, stdin=None):
+    """Runs the gsutil command.
+
+    Args:
+      cmd: The command to run, as a list, e.g. ['cp', 'foo', 'bar']
+      return_status: If True, the exit status code is returned.
+      return_stdout: If True, the standard output of the command is returned.
+      return_stderr: If True, the standard error of the command is returned.
+      expected_status: The expected return code. If not specified, defaults to
+                       0. If the return code is a different value, an exception
+                       is raised.
+      stdin: A string of data to pipe to the process as standard input.
+
+    Returns:
+      A tuple containing the desired return values specified by the return_*
+      arguments.
+    """
+    cmd = [GSUTIL_PATH] + cmd
+    if IS_WINDOWS:
+      cmd = [sys.executable] + cmd
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                         stdin=subprocess.PIPE)
+    (stdout, stderr) = p.communicate(stdin)
+    status = p.returncode
+
+    if expected_status is not None:
+      self.assertEqual(
+          status, expected_status,
+          msg='Expected status %d, got %d.\nCommand:\n%s\n\nstderr:\n%s' % (
+              expected_status, status, ' '.join(cmd), stderr))
+
+    toreturn = []
+    if return_status:
+      toreturn.append(status)
+    if return_stdout:
+      if IS_WINDOWS:
+        stdout = stdout.replace('\r\n', '\n')
+      toreturn.append(stdout)
+    if return_stderr:
+      if IS_WINDOWS:
+        stderr = stderr.replace('\r\n', '\n')
+      toreturn.append(stderr)
+
+    if len(toreturn) == 1:
+      return toreturn[0]
+    elif toreturn:
+      return tuple(toreturn)

data/packages/gsutil/gslib/tests/testcase/unit_testcase.py

@@ -0,0 +1,230 @@
+# Copyright 2013 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Contains gsutil base unit test case class."""
+
+import os.path
+import sys
+import tempfile
+
+import boto
+
+from gslib import wildcard_iterator
+from gslib.command_runner import CommandRunner
+from gslib.project_id import ProjectIdHandler
+import gslib.tests.util as util
+from gslib.tests.util import unittest
+from tests.integration.s3 import mock_storage_service
+import base
+
+
+CURDIR = os.path.abspath(os.path.dirname(__file__))
+TESTS_DIR = os.path.split(CURDIR)[0]
+GSLIB_DIR = os.path.split(TESTS_DIR)[0]
+GSUTIL_DIR = os.path.split(GSLIB_DIR)[0]
+BOTO_DIR = os.path.join(GSUTIL_DIR, 'boto')
+
+
+class GSMockConnection(mock_storage_service.MockConnection):
+
+  def __init__(self, *args, **kwargs):
+    kwargs['provider'] = 'gs'
+    super(GSMockConnection, self).__init__(*args, **kwargs)
+
+mock_connection = GSMockConnection()
+
+
+class GSMockBucketStorageUri(mock_storage_service.MockBucketStorageUri):
+
+  def connect(self, access_key_id=None, secret_access_key=None):
+    return mock_connection
+
+
+@unittest.skipUnless(util.RUN_UNIT_TESTS,
+                     'Not running integration tests.')
+class GsUtilUnitTestCase(base.GsUtilTestCase):
+  """Base class for gsutil unit tests."""
+
+  @classmethod
+  def setUpClass(cls):
+    base.GsUtilTestCase.setUpClass()
+    cls.mock_bucket_storage_uri = GSMockBucketStorageUri
+    cls.proj_id_handler = ProjectIdHandler()
+    config_file_list = boto.pyami.config.BotoConfigLocations
+    # Use "gsutil_test_commands" as a fake UserAgent. This value will never be
+    # sent via HTTP because we're using MockStorageService here.
+    cls.command_runner = CommandRunner(GSUTIL_DIR, BOTO_DIR,
+                                       config_file_list, 'gsutil_test_commands',
+                                       cls.mock_bucket_storage_uri)
+
+  def setUp(self):
+    super(GsUtilUnitTestCase, self).setUp()
+    self.bucket_uris = []
+
+  def RunCommand(self, command_name, args=None, headers=None, debug=0,
+                 test_method=None, return_stdout=False, cwd=None):
+    """
+    Method for calling gslib.command_runner.CommandRunner, passing
+    parallel_operations=False for all tests, optionally saving/returning stdout
+    output. We run all tests multi-threaded, to exercise those more complicated
+    code paths.
+    TODO: change to run with parallel_operations=True for all tests. At
+    present when you do this it causes many test failures.
+
+    Args:
+      command_name: The name of the command being run.
+      args: Command-line args (arg0 = actual arg, not command name ala bash).
+      headers: Dictionary containing optional HTTP headers to pass to boto.
+      debug: Debug level to pass in to boto connection (range 0..3).
+      parallel_operations: Should command operations be executed in parallel?
+      test_method: Optional general purpose method for testing purposes.
+                   Application and semantics of this method will vary by
+                   command and test type.
+      cwd: The working directory that should be switched to before running the
+           command. The working directory will be reset back to its original
+           value after running the command. If not specified, the working
+           directory is left unchanged.
+      return_stdout: If true will save and return stdout produced by command.
+    """
+    if util.VERBOSE_OUTPUT:
+      sys.stderr.write('\nRunning test of %s %s\n' %
+                       (command_name, ' '.join(args)))
+    if return_stdout:
+      # Redirect stdout temporarily, to save output to a file.
+      fh, outfile = tempfile.mkstemp()
+      os.close(fh)
+    elif not util.VERBOSE_OUTPUT:
+      outfile = os.devnull
+    else:
+      outfile = None
+
+    stdout_sav = sys.stdout
+    output = None
+    cwd_sav = None
+    try:
+      cwd_sav = os.getcwd()
+    except OSError:
+      # This can happen if the current working directory no longer exists.
+      pass
+    try:
+      if outfile:
+        fp = open(outfile, 'w')
+        sys.stdout = fp
+      if cwd:
+        os.chdir(cwd)
+      self.command_runner.RunNamedCommand(
+          command_name, args=args, headers=headers, debug=debug,
+          parallel_operations=False, test_method=test_method)
+    finally:
+      if cwd and cwd_sav:
+        os.chdir(cwd_sav)
+      if outfile:
+        fp.close()
+        sys.stdout = stdout_sav
+        with open(outfile, 'r') as f:
+          output = f.read()
+        if return_stdout:
+          os.unlink(outfile)
+
+    if output is not None and return_stdout:
+      return output
+
+  @classmethod
+  def _test_wildcard_iterator(cls, uri_or_str, debug=0):
+    """
+    Convenience method for instantiating a testing instance of
+    WildCardIterator, without having to specify all the params of that class
+    (like bucket_storage_uri_class=mock_storage_service.MockBucketStorageUri).
+    Also naming the factory method this way makes it clearer in the test code
+    that WildcardIterator needs to be set up for testing.
+
+    Args are same as for wildcard_iterator.wildcard_iterator(), except there's
+    no bucket_storage_uri_class arg.
+
+    Returns:
+      WildcardIterator.IterUris(), over which caller can iterate.
+    """
+    return wildcard_iterator.wildcard_iterator(
+        uri_or_str, cls.proj_id_handler, cls.mock_bucket_storage_uri,
+        debug=debug)
+
+  @staticmethod
+  def _test_storage_uri(uri_str, default_scheme='file', debug=0,
+                        validate=True):
+    """
+    Convenience method for instantiating a testing
+    instance of StorageUri, without having to specify
+    bucket_storage_uri_class=mock_storage_service.MockBucketStorageUri.
+    Also naming the factory method this way makes it clearer in the test
+    code that StorageUri needs to be set up for testing.
+
+    Args, Returns, and Raises are same as for boto.storage_uri(), except there's
+    no bucket_storage_uri_class arg.
+    """
+    return boto.storage_uri(uri_str, default_scheme, debug, validate,
+                            GSMockBucketStorageUri)
+
+  def CreateBucket(self, bucket_name=None, test_objects=0, storage_class=None):
+    """Creates a test bucket.
+
+    The bucket and all of its contents will be deleted after the test.
+
+    Args:
+      bucket_name: Create the bucket with this name. If not provided, a
+                   temporary test bucket name is constructed.
+      test_objects: The number of objects that should be placed in the bucket or
+                    a list of object names to place in the bucket. Defaults to
+                    0.
+      storage_class: storage class to use. If not provided we us standard.
+
+    Returns:
+      StorageUri for the created bucket.
+    """
+    bucket_name = bucket_name or self.MakeTempName('bucket')
+    bucket_uri = boto.storage_uri(
+        'gs://%s' % bucket_name.lower(),
+        suppress_consec_slashes=False,
+        bucket_storage_uri_class=GSMockBucketStorageUri)
+    bucket_uri.create_bucket(storage_class=storage_class)
+    self.bucket_uris.append(bucket_uri)
+    try:
+      iter(test_objects)
+    except TypeError:
+      test_objects = [self.MakeTempName('obj') for _ in range(test_objects)]
+    for i, name in enumerate(test_objects):
+      self.CreateObject(bucket_uri=bucket_uri, object_name=name,
+                        contents='test %d' % i)
+    return bucket_uri
+
+  def CreateObject(self, bucket_uri=None, object_name=None, contents=None):
+    """Creates a test object.
+
+    Args:
+      bucket: The URI of the bucket to place the object in. If not specified, a
+              new temporary bucket is created.
+      object_name: The name to use for the object. If not specified, a temporary
+                   test object name is constructed.
+      contents: The contents to write to the object. If not specified, the key
+                is not written to, which means that it isn't actually created
+                yet on the server.
+
+    Returns:
+      A StorageUri for the created object.
+    """
+    bucket_uri = bucket_uri or self.CreateBucket()
+    object_name = object_name or self.MakeTempName('obj')
+    key_uri = bucket_uri.clone_replace_name(object_name)
+    if contents is not None:
+      key_uri.set_contents_from_string(contents)
+    return key_uri