gcloud 0.0.4 → 0.0.6

data/packages/gsutil/gslib/no_op_auth_plugin.py

@@ -0,0 +1,30 @@
+# Copyright 2011 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This code implements a no-op auth plugin, which allows users to use
+# gsutil for accessing publicly readable buckets and objects without first
+# signing up for an account.
+
+from boto.auth_handler import AuthHandler
+
+
+class NoOpAuth(AuthHandler):
+
+  capability = ['s3']
+
+  def __init__(self, path, config, provider):
+    pass
+
+  def add_auth(self, http_request):
+    pass

data/packages/gsutil/gslib/plurality_checkable_iterator.py

@@ -0,0 +1,56 @@
+# Copyright 2012 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Iterator wrapper that allows you to check whether the wrapped iterator
+is empty and whether it has more than 1 element.
+"""
+
+class PluralityCheckableIterator(object):
+
+  def __init__(self, it):
+    self.it = it.__iter__()
+    self.head = []
+    # Populate first 2 elems into head so we can check whether iterator has
+    # more than 1 item.
+    for i in range(0, 2):
+      self.__populate_head__()
+
+  def __populate_head__(self):
+    try:
+      e = self.it.next()
+      self.underlying_iter_empty = False
+      self.head.append(e)
+    except StopIteration:
+      # Indicates we can no longer call next() on underlying iterator, but
+      # there could still be elements left to iterate in head.
+      self.underlying_iter_empty = True
+
+  def __iter__(self):
+    while len(self.head) > 0:
+      yield self.next()
+    else:
+      raise StopIteration()
+
+  def next(self):
+    # Backfill into head each time we pop an element so we can always check
+    # for emptiness and for has_plurality().
+    self.__populate_head__()
+    return self.head.pop(0)
+
+  def is_empty(self):
+    return len(self.head) == 0
+
+  def has_plurality(self):
+    return len(self.head) > 1

data/packages/gsutil/gslib/project_id.py

@@ -0,0 +1,67 @@
+# Copyright 2011 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import boto
+
+from gslib.exception import ProjectIdException
+from gslib.wildcard_iterator import WILDCARD_BUCKET_ITERATOR
+
+GOOG_PROJ_ID_HDR = 'x-goog-project-id'
+
+
+class ProjectIdHandler(object):
+  """Google Project ID header handling."""
+
+  def __init__(self):
+    """Instantiates Project ID handler. Call after boto config file loaded."""
+    config = boto.config
+    self.project_id = config.get_value('GSUtil', 'default_project_id')
+
+  def SetProjectId(self, project_id):
+    """Overrides project ID value from config file default.
+
+    Args:
+      project_id: Project ID to use.
+    """
+    self.project_id = project_id
+
+  def FillInProjectHeaderIfNeeded(self, command, uri, headers):
+    """Fills project ID header into headers if defined and applicable.
+
+    Args:
+      command: The command being run.
+      uri: The URI against which this command is being run.
+      headers: Dictionary containing optional HTTP headers to pass to boto.
+          Must not be None.
+    """
+
+    # We only include the project ID header if it's a GS URI and a project_id
+    # was specified and
+    # (it's an 'mb', 'disablelogging, or 'enablelogging' command or
+    #  a boto request in integration tests or
+    #  (an 'ls' command that doesn't specify a bucket or wildcarded bucket)).
+    if (uri.scheme.lower() == 'gs' and self.project_id
+        and (command == 'mb' or command == 'disablelogging'
+             or command == 'enablelogging'
+             or command == 'test'
+             or (command == 'ls' and not uri.names_bucket())
+             or (command == WILDCARD_BUCKET_ITERATOR))):
+      # Note: check for None (as opposed to "not headers") here because
+      # it's ok to pass empty headers.
+      if headers is None:
+        raise ProjectIdException(
+            'FillInProjectHeaderIfNeeded called with headers=None')
+      headers[GOOG_PROJ_ID_HDR] = self.project_id
+    elif headers.has_key(GOOG_PROJ_ID_HDR):
+      del headers[GOOG_PROJ_ID_HDR]

data/packages/gsutil/gslib/storage_uri_builder.py

@@ -0,0 +1,56 @@
+# Copyright 2012 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""
+Class that holds state (bucket_storage_uri_class and debug) needed for
+instantiating StorageUri objects. The StorageUri func defined in this class
+uses that state plus gsutil default flag values to instantiate this frequently
+constructed object with just one param for most cases.
+"""
+
+import boto
+from gslib.exception import CommandException
+
+
+class StorageUriBuilder(object):
+
+  def __init__(self, debug, bucket_storage_uri_class):
+    """
+    Args:
+      debug: Debug level to pass in to boto connection (range 0..3).
+      bucket_storage_uri_class: Class to instantiate for cloud StorageUris.
+                                Settable for testing/mocking.
+    """
+    self.bucket_storage_uri_class = bucket_storage_uri_class
+    self.debug = debug
+
+  def StorageUri(self, uri_str, is_latest=False):
+    """
+    Instantiates StorageUri using class state and gsutil default flag values.
+
+    Args:
+      uri_str: StorageUri naming bucket or object.
+      is_latest: boolean indicating whether this versioned object represents the
+          current version.
+
+    Returns:
+      boto.StorageUri for given uri_str.
+
+    Raises:
+      InvalidUriError: if uri_str not valid.
+    """
+    return boto.storage_uri(
+        uri_str, 'file', debug=self.debug, validate=False,
+        bucket_storage_uri_class=self.bucket_storage_uri_class,
+        suppress_consec_slashes=False, is_latest=is_latest)

data/packages/gsutil/gslib/tests/__init__.py

@@ -0,0 +1,15 @@
+# Copyright 2013 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Contains test cases for gsutil."""

data/packages/gsutil/gslib/tests/test_chacl.py

@@ -0,0 +1,236 @@
+# Copyright 2013 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import time
+
+from gslib.command import _ThreadedLogger as ThreadedLogger
+import gslib.commands.chacl as chacl
+import gslib.tests.testcase as case
+from gslib.tests.util import ObjectToURI as suri
+
+
+class ChaclIntegrationTest(case.GsUtilIntegrationTestCase):
+  """Tests gslib.commands.chacl."""
+  logger = ThreadedLogger()
+
+  def setUp(self):
+    super(ChaclIntegrationTest, self).setUp()
+    self.sample_uri = self.CreateBucket()
+    self.logger = ThreadedLogger()
+
+  def testAclChangeWithUserId(self):
+    change = chacl.AclChange(self.USER_TEST_ID + ':r',
+                             scope_type=chacl.ChangeType.USER,
+                             logger=self.logger)
+    acl = self.sample_uri.get_acl()
+    change.Execute(self.sample_uri, acl)
+    self._AssertHas(acl, 'READ', 'UserById', self.USER_TEST_ID)
+
+  def testAclChangeWithGroupId(self):
+    change = chacl.AclChange(self.GROUP_TEST_ID + ':r',
+                             scope_type=chacl.ChangeType.GROUP,
+                             logger=self.logger)
+    acl = self.sample_uri.get_acl()
+    change.Execute(self.sample_uri, acl)
+    self._AssertHas(acl, 'READ', 'GroupById', self.GROUP_TEST_ID)
+
+  def testAclChangeWithUserEmail(self):
+    change = chacl.AclChange(self.USER_TEST_ADDRESS + ':r',
+                             scope_type=chacl.ChangeType.USER,
+                             logger=self.logger)
+    acl = self.sample_uri.get_acl()
+    change.Execute(self.sample_uri, acl)
+    self._AssertHas(acl, 'READ', 'UserByEmail', self.USER_TEST_ADDRESS)
+
+  def testAclChangeWithGroupEmail(self):
+    change = chacl.AclChange(self.GROUP_TEST_ADDRESS + ':fc',
+                             scope_type=chacl.ChangeType.GROUP,
+                             logger=self.logger)
+    acl = self.sample_uri.get_acl()
+    change.Execute(self.sample_uri, acl)
+    self._AssertHas(acl, 'FULL_CONTROL', 'GroupByEmail',
+                    self.GROUP_TEST_ADDRESS)
+
+  def testAclChangeWithDomain(self):
+    change = chacl.AclChange(self.DOMAIN_TEST + ':READ',
+                             scope_type=chacl.ChangeType.GROUP,
+                             logger=self.logger)
+    acl = self.sample_uri.get_acl()
+    change.Execute(self.sample_uri, acl)
+    self._AssertHas(acl, 'READ', 'GroupByDomain', self.DOMAIN_TEST)
+
+  def testAclChangeWithAllUsers(self):
+    change = chacl.AclChange('AllUsers:WRITE',
+                             scope_type=chacl.ChangeType.GROUP,
+                             logger=self.logger)
+    acl = self.sample_uri.get_acl()
+    change.Execute(self.sample_uri, acl)
+    self._AssertHas(acl, 'WRITE', 'AllUsers')
+
+  def testAclChangeWithAllAuthUsers(self):
+    change = chacl.AclChange('AllAuthenticatedUsers:READ',
+                             scope_type=chacl.ChangeType.GROUP,
+                             logger=self.logger)
+    acl = self.sample_uri.get_acl()
+    change.Execute(self.sample_uri, acl)
+    self._AssertHas(acl, 'READ', 'AllAuthenticatedUsers')
+
+  def testAclDelWithUser(self):
+    add = chacl.AclChange(self.USER_TEST_ADDRESS + ':READ',
+                          scope_type=chacl.ChangeType.USER,
+                          logger=self.logger)
+    acl = self.sample_uri.get_acl()
+    add.Execute(self.sample_uri, acl)
+    self._AssertHas(acl, 'READ', 'UserByEmail', self.USER_TEST_ADDRESS)
+
+    remove = chacl.AclDel(self.USER_TEST_ADDRESS,
+                          logger=self.logger)
+    remove.Execute(self.sample_uri, acl)
+    self._AssertHasNo(acl, 'READ', 'UserByEmail', self.USER_TEST_ADDRESS)
+
+  def testAclDelWithGroup(self):
+    add = chacl.AclChange(self.USER_TEST_ADDRESS + ':READ',
+                          scope_type=chacl.ChangeType.GROUP,
+                          logger=self.logger)
+    acl = self.sample_uri.get_acl()
+    add.Execute(self.sample_uri, acl)
+    self._AssertHas(acl, 'READ', 'GroupByEmail', self.USER_TEST_ADDRESS)
+
+    remove = chacl.AclDel(self.USER_TEST_ADDRESS,
+                          logger=self.logger)
+    remove.Execute(self.sample_uri, acl)
+    self._AssertHasNo(acl, 'READ', 'GroupByEmail', self.GROUP_TEST_ADDRESS)
+
+  #
+  # Here are a whole lot of verbose asserts
+  #
+
+  def _AssertHas(self, current_acl, perm, scope, value=None):
+    matches = list(self._YieldMatchingEntries(current_acl, perm, scope, value))
+    self.assertEqual(1, len(matches))
+
+  def _AssertHasNo(self, current_acl, perm, scope, value=None):
+    matches = list(self._YieldMatchingEntries(current_acl, perm, scope, value))
+    self.assertEqual(0, len(matches))
+
+  def _YieldMatchingEntries(self, current_acl, perm, scope, value=None):
+    """Generator that finds entries that match the change descriptor."""
+    for entry in current_acl.entries.entry_list:
+      if entry.scope.type == scope:
+        if scope in ['UserById', 'GroupById']:
+          if value == entry.scope.id:
+            yield entry
+        elif scope in ['UserByEmail', 'GroupByEmail']:
+          if value == entry.scope.email_address:
+            yield entry
+        elif scope == 'GroupByDomain':
+          if value == entry.scope.domain:
+            yield entry
+        elif scope in ['AllUsers', 'AllAuthenticatedUsers']:
+          yield entry
+        else:
+          raise CommandException('Found an unrecognized ACL '
+                                 'entry type, aborting.')
+
+  def _MakeScopeRegex(self, scope_type, email_address, perm):
+    template_regex = (
+        r'<Scope type="{0}">\s*<EmailAddress>\s*{1}\s*</EmailAddress>\s*'
+        r'</Scope>\s*<Permission>\s*{2}\s*</Permission>')
+    return template_regex.format(scope_type, email_address, perm)
+
+  def testBucketAclChange(self):
+    test_regex = self._MakeScopeRegex(
+        'UserByEmail', self.USER_TEST_ADDRESS, 'FULL_CONTROL')
+    xml = self.RunGsUtil(
+        ['getacl', suri(self.sample_uri)], return_stdout=True)
+    self.assertNotRegexpMatches(xml, test_regex)
+
+    self.RunGsUtil(
+        ['chacl', '-u', self.USER_TEST_ADDRESS+':fc', suri(self.sample_uri)])
+    xml = self.RunGsUtil(
+        ['getacl', suri(self.sample_uri)], return_stdout=True)
+    self.assertRegexpMatches(xml, test_regex)
+
+    self.RunGsUtil(
+        ['chacl', '-d', self.USER_TEST_ADDRESS, suri(self.sample_uri)])
+    xml = self.RunGsUtil(
+        ['getacl', suri(self.sample_uri)], return_stdout=True)
+    self.assertNotRegexpMatches(xml, test_regex)
+
+  def testObjectAclChange(self):
+    obj = self.CreateObject(bucket_uri=self.sample_uri, contents='something')
+    test_regex = self._MakeScopeRegex(
+        'GroupByEmail', self.GROUP_TEST_ADDRESS, 'READ')
+    xml = self.RunGsUtil(['getacl', suri(obj)], return_stdout=True)
+    self.assertNotRegexpMatches(xml, test_regex)
+
+    self.RunGsUtil(['chacl', '-g', self.GROUP_TEST_ADDRESS+':READ', suri(obj)])
+    xml = self.RunGsUtil(['getacl', suri(obj)], return_stdout=True)
+    self.assertRegexpMatches(xml, test_regex)
+
+    self.RunGsUtil(['chacl', '-d', self.GROUP_TEST_ADDRESS, suri(obj)])
+    xml = self.RunGsUtil(['getacl', suri(obj)], return_stdout=True)
+    self.assertNotRegexpMatches(xml, test_regex)
+
+  def testMultithreadedAclChange(self, count=10):
+    objects = []
+    for i in range(count):
+      objects.append(self.CreateObject(
+          bucket_uri=self.sample_uri,
+          contents='something {0}'.format(i)))
+
+    test_regex = self._MakeScopeRegex(
+        'GroupByEmail', self.GROUP_TEST_ADDRESS, 'READ')
+    xmls = []
+    for obj in objects:
+      xmls.append(self.RunGsUtil(['getacl', suri(obj)], return_stdout=True))
+    for xml in xmls:
+      self.assertNotRegexpMatches(xml, test_regex)
+
+    uris = [suri(obj) for obj in objects]
+    self.RunGsUtil(['-m', '-DD', 'chacl', '-g',
+                    self.GROUP_TEST_ADDRESS+':READ'] + uris)
+
+    xmls = []
+    for obj in objects:
+      xmls.append(self.RunGsUtil(['getacl', suri(obj)], return_stdout=True))
+    for xml in xmls:
+      self.assertRegexpMatches(xml, test_regex)
+
+  def testMultiVersionSupport(self):
+    bucket = self.CreateVersionedBucket()
+    object_name = self.MakeTempName('obj')
+    obj = self.CreateObject(
+        bucket_uri=bucket, object_name=object_name, contents='One thing')
+    # Create another on the same URI, giving us a second version.
+    self.CreateObject(
+        bucket_uri=bucket, object_name=object_name, contents='Another thing')
+    # Bucket listings are eventually consistent, so wait to increase the likelihood 
+    # we'll see both versions of the object we just created. 
+    time.sleep(0.5)
+    stdout = self.RunGsUtil(['ls', '-a', suri(obj)], return_stdout=True)
+    obj_v1, obj_v2 = stdout.strip().split('\n')
+
+    test_regex = self._MakeScopeRegex(
+        'GroupByEmail', self.GROUP_TEST_ADDRESS, 'READ')
+    xml = self.RunGsUtil(['getacl', obj_v1], return_stdout=True)
+    self.assertNotRegexpMatches(xml, test_regex)
+
+    self.RunGsUtil(['chacl', '-g', self.GROUP_TEST_ADDRESS+':READ', obj_v1])
+    xml = self.RunGsUtil(['getacl', obj_v1], return_stdout=True)
+    self.assertRegexpMatches(xml, test_regex)
+
+    xml = self.RunGsUtil(['getacl', obj_v2], return_stdout=True)
+    self.assertNotRegexpMatches(xml, test_regex)