googlecloud 0.0.2 → 0.0.4

data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/file.py

@@ -0,0 +1,107 @@
+# Copyright (C) 2010 Google Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Utilities for OAuth.
+
+Utilities for making it easier to work with OAuth 2.0
+credentials.
+"""
+
+
+
+import os
+import stat
+import threading
+
+from anyjson import simplejson
+from client import Storage as BaseStorage
+from client import Credentials
+
+
+class Storage(BaseStorage):
+  """Store and retrieve a single credential to and from a file."""
+
+  def __init__(self, filename):
+    self._filename = filename
+    self._lock = threading.Lock()
+
+  def acquire_lock(self):
+    """Acquires any lock necessary to access this Storage.
+
+    This lock is not reentrant."""
+    self._lock.acquire()
+
+  def release_lock(self):
+    """Release the Storage lock.
+
+    Trying to release a lock that isn't held will result in a
+    RuntimeError.
+    """
+    self._lock.release()
+
+  def locked_get(self):
+    """Retrieve Credential from file.
+
+    Returns:
+      oauth2client.client.Credentials
+    """
+    credentials = None
+    try:
+      f = open(self._filename, 'rb')
+      content = f.read()
+      f.close()
+    except IOError:
+      return credentials
+
+    try:
+      credentials = Credentials.new_from_json(content)
+      credentials.set_store(self)
+    except ValueError:
+      pass
+
+    return credentials
+
+  def _create_file_if_needed(self):
+    """Create an empty file if necessary.
+
+    This method will not initialize the file. Instead it implements a
+    simple version of "touch" to ensure the file has been created.
+    """
+    if not os.path.exists(self._filename):
+      old_umask = os.umask(0177)
+      try:
+        open(self._filename, 'a+b').close()
+      finally:
+        os.umask(old_umask)
+
+  def locked_put(self, credentials):
+    """Write Credentials to file.
+
+    Args:
+      credentials: Credentials, the credentials to store.
+    """
+
+    self._create_file_if_needed()
+    f = open(self._filename, 'wb')
+    f.write(credentials.to_json())
+    f.close()
+
+  def locked_delete(self):
+    """Delete Credentials file.
+
+    Args:
+      credentials: Credentials, the credentials to store.
+    """
+
+    os.unlink(self._filename)

data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/locked_file.py

@@ -0,0 +1,343 @@
+# Copyright 2011 Google Inc. All Rights Reserved.
+
+"""Locked file interface that should work on Unix and Windows pythons.
+
+This module first tries to use fcntl locking to ensure serialized access
+to a file, then falls back on a lock file if that is unavialable.
+
+Usage:
+    f = LockedFile('filename', 'r+b', 'rb')
+    f.open_and_lock()
+    if f.is_locked():
+      print 'Acquired filename with r+b mode'
+      f.file_handle().write('locked data')
+    else:
+      print 'Aquired filename with rb mode'
+    f.unlock_and_close()
+"""
+
+
+
+import errno
+import logging
+import os
+import time
+
+logger = logging.getLogger(__name__)
+
+
+class AlreadyLockedException(Exception):
+  """Trying to lock a file that has already been locked by the LockedFile."""
+  pass
+
+
+class _Opener(object):
+  """Base class for different locking primitives."""
+
+  def __init__(self, filename, mode, fallback_mode):
+    """Create an Opener.
+
+    Args:
+      filename: string, The pathname of the file.
+      mode: string, The preferred mode to access the file with.
+      fallback_mode: string, The mode to use if locking fails.
+    """
+    self._locked = False
+    self._filename = filename
+    self._mode = mode
+    self._fallback_mode = fallback_mode
+    self._fh = None
+
+  def is_locked(self):
+    """Was the file locked."""
+    return self._locked
+
+  def file_handle(self):
+    """The file handle to the file.  Valid only after opened."""
+    return self._fh
+
+  def filename(self):
+    """The filename that is being locked."""
+    return self._filename
+
+  def open_and_lock(self, timeout, delay):
+    """Open the file and lock it.
+
+    Args:
+      timeout: float, How long to try to lock for.
+      delay: float, How long to wait between retries.
+    """
+    pass
+
+  def unlock_and_close(self):
+    """Unlock and close the file."""
+    pass
+
+
+class _PosixOpener(_Opener):
+  """Lock files using Posix advisory lock files."""
+
+  def open_and_lock(self, timeout, delay):
+    """Open the file and lock it.
+
+    Tries to create a .lock file next to the file we're trying to open.
+
+    Args:
+      timeout: float, How long to try to lock for.
+      delay: float, How long to wait between retries.
+
+    Raises:
+      AlreadyLockedException: if the lock is already acquired.
+      IOError: if the open fails.
+    """
+    if self._locked:
+      raise AlreadyLockedException('File %s is already locked' %
+                                   self._filename)
+    self._locked = False
+
+    try:
+      self._fh = open(self._filename, self._mode)
+    except IOError, e:
+      # If we can't access with _mode, try _fallback_mode and don't lock.
+      if e.errno == errno.EACCES:
+        self._fh = open(self._filename, self._fallback_mode)
+        return
+
+    lock_filename = self._posix_lockfile(self._filename)
+    start_time = time.time()
+    while True:
+      try:
+        self._lock_fd = os.open(lock_filename,
+                                os.O_CREAT|os.O_EXCL|os.O_RDWR)
+        self._locked = True
+        break
+
+      except OSError, e:
+        if e.errno != errno.EEXIST:
+          raise
+        if (time.time() - start_time) >= timeout:
+          logger.warn('Could not acquire lock %s in %s seconds' % (
+              lock_filename, timeout))
+          # Close the file and open in fallback_mode.
+          if self._fh:
+            self._fh.close()
+          self._fh = open(self._filename, self._fallback_mode)
+          return
+        time.sleep(delay)
+
+  def unlock_and_close(self):
+    """Unlock a file by removing the .lock file, and close the handle."""
+    if self._locked:
+      lock_filename = self._posix_lockfile(self._filename)
+      os.unlink(lock_filename)
+      os.close(self._lock_fd)
+      self._locked = False
+      self._lock_fd = None
+    if self._fh:
+      self._fh.close()
+
+  def _posix_lockfile(self, filename):
+    """The name of the lock file to use for posix locking."""
+    return '%s.lock' % filename
+
+
+try:
+  import fcntl
+
+  class _FcntlOpener(_Opener):
+    """Open, lock, and unlock a file using fcntl.lockf."""
+
+    def open_and_lock(self, timeout, delay):
+      """Open the file and lock it.
+
+      Args:
+        timeout: float, How long to try to lock for.
+        delay: float, How long to wait between retries
+
+      Raises:
+        AlreadyLockedException: if the lock is already acquired.
+        IOError: if the open fails.
+      """
+      if self._locked:
+        raise AlreadyLockedException('File %s is already locked' %
+                                     self._filename)
+      start_time = time.time()
+
+      try:
+        self._fh = open(self._filename, self._mode)
+      except IOError, e:
+        # If we can't access with _mode, try _fallback_mode and don't lock.
+        if e.errno == errno.EACCES:
+          self._fh = open(self._filename, self._fallback_mode)
+          return
+
+      # We opened in _mode, try to lock the file.
+      while True:
+        try:
+          fcntl.lockf(self._fh.fileno(), fcntl.LOCK_EX)
+          self._locked = True
+          return
+        except IOError, e:
+          # If not retrying, then just pass on the error.
+          if timeout == 0:
+            raise e
+          if e.errno != errno.EACCES:
+            raise e
+          # We could not acquire the lock.  Try again.
+          if (time.time() - start_time) >= timeout:
+            logger.warn('Could not lock %s in %s seconds' % (
+                self._filename, timeout))
+            if self._fh:
+              self._fh.close()
+            self._fh = open(self._filename, self._fallback_mode)
+            return
+          time.sleep(delay)
+
+    def unlock_and_close(self):
+      """Close and unlock the file using the fcntl.lockf primitive."""
+      if self._locked:
+        fcntl.lockf(self._fh.fileno(), fcntl.LOCK_UN)
+      self._locked = False
+      if self._fh:
+        self._fh.close()
+except ImportError:
+  _FcntlOpener = None
+
+
+try:
+  import pywintypes
+  import win32con
+  import win32file
+
+  class _Win32Opener(_Opener):
+    """Open, lock, and unlock a file using windows primitives."""
+
+    # Error #33:
+    #  'The process cannot access the file because another process'
+    FILE_IN_USE_ERROR = 33
+
+    # Error #158:
+    #  'The segment is already unlocked.'
+    FILE_ALREADY_UNLOCKED_ERROR = 158
+
+    def open_and_lock(self, timeout, delay):
+      """Open the file and lock it.
+
+      Args:
+        timeout: float, How long to try to lock for.
+        delay: float, How long to wait between retries
+
+      Raises:
+        AlreadyLockedException: if the lock is already acquired.
+        IOError: if the open fails.
+      """
+      if self._locked:
+        raise AlreadyLockedException('File %s is already locked' %
+                                     self._filename)
+      start_time = time.time()
+
+      try:
+        self._fh = open(self._filename, self._mode)
+      except IOError, e:
+        # If we can't access with _mode, try _fallback_mode and don't lock.
+        if e.errno == errno.EACCES:
+          self._fh = open(self._filename, self._fallback_mode)
+          return
+
+      # We opened in _mode, try to lock the file.
+      while True:
+        try:
+          hfile = win32file._get_osfhandle(self._fh.fileno())
+          win32file.LockFileEx(
+              hfile,
+              (win32con.LOCKFILE_FAIL_IMMEDIATELY|
+               win32con.LOCKFILE_EXCLUSIVE_LOCK), 0, -0x10000,
+              pywintypes.OVERLAPPED())
+          self._locked = True
+          return
+        except pywintypes.error, e:
+          if timeout == 0:
+            raise e
+
+          # If the error is not that the file is already in use, raise.
+          if e[0] != _Win32Opener.FILE_IN_USE_ERROR:
+            raise
+
+          # We could not acquire the lock.  Try again.
+          if (time.time() - start_time) >= timeout:
+            logger.warn('Could not lock %s in %s seconds' % (
+                self._filename, timeout))
+            if self._fh:
+              self._fh.close()
+            self._fh = open(self._filename, self._fallback_mode)
+            return
+          time.sleep(delay)
+
+    def unlock_and_close(self):
+      """Close and unlock the file using the win32 primitive."""
+      if self._locked:
+        try:
+          hfile = win32file._get_osfhandle(self._fh.fileno())
+          win32file.UnlockFileEx(hfile, 0, -0x10000, pywintypes.OVERLAPPED())
+        except pywintypes.error, e:
+          if e[0] != _Win32Opener.FILE_ALREADY_UNLOCKED_ERROR:
+            raise
+      self._locked = False
+      if self._fh:
+        self._fh.close()
+except ImportError:
+  _Win32Opener = None
+
+
+class LockedFile(object):
+  """Represent a file that has exclusive access."""
+
+  def __init__(self, filename, mode, fallback_mode, use_native_locking=True):
+    """Construct a LockedFile.
+
+    Args:
+      filename: string, The path of the file to open.
+      mode: string, The mode to try to open the file with.
+      fallback_mode: string, The mode to use if locking fails.
+      use_native_locking: bool, Whether or not fcntl/win32 locking is used.
+    """
+    opener = None
+    if not opener and use_native_locking:
+      if _Win32Opener:
+        opener = _Win32Opener(filename, mode, fallback_mode)
+      if _FcntlOpener:
+        opener = _FcntlOpener(filename, mode, fallback_mode)
+
+    if not opener:
+      opener = _PosixOpener(filename, mode, fallback_mode)
+
+    self._opener = opener
+
+  def filename(self):
+    """Return the filename we were constructed with."""
+    return self._opener._filename
+
+  def file_handle(self):
+    """Return the file_handle to the opened file."""
+    return self._opener.file_handle()
+
+  def is_locked(self):
+    """Return whether we successfully locked the file."""
+    return self._opener.is_locked()
+
+  def open_and_lock(self, timeout=0, delay=0.05):
+    """Open the file, trying to lock it.
+
+    Args:
+      timeout: float, The number of seconds to try to acquire the lock.
+      delay: float, The number of seconds to wait between retry attempts.
+
+    Raises:
+      AlreadyLockedException: if the lock is already acquired.
+      IOError: if the open fails.
+    """
+    self._opener.open_and_lock(timeout, delay)
+
+  def unlock_and_close(self):
+    """Unlock and close a file."""
+    self._opener.unlock_and_close()

data/packages/gcutil-1.7.1/lib/google_api_python_client/oauth2client/multistore_file.py

@@ -0,0 +1,379 @@
+# Copyright 2011 Google Inc. All Rights Reserved.
+
+"""Multi-credential file store with lock support.
+
+This module implements a JSON credential store where multiple
+credentials can be stored in one file.  That file supports locking
+both in a single process and across processes.
+
+The credential themselves are keyed off of:
+* client_id
+* user_agent
+* scope
+
+The format of the stored data is like so:
+{
+  'file_version': 1,
+  'data': [
+    {
+      'key': {
+        'clientId': '<client id>',
+        'userAgent': '<user agent>',
+        'scope': '<scope>'
+      },
+      'credential': {
+        # JSON serialized Credentials.
+      }
+    }
+  ]
+}
+"""
+
+
+
+import base64
+import errno
+import logging
+import os
+import threading
+
+from anyjson import simplejson
+from client import Storage as BaseStorage
+from client import Credentials
+from locked_file import LockedFile
+
+logger = logging.getLogger(__name__)
+
+# A dict from 'filename'->_MultiStore instances
+_multistores = {}
+_multistores_lock = threading.Lock()
+
+
+class Error(Exception):
+  """Base error for this module."""
+  pass
+
+
+class NewerCredentialStoreError(Error):
+  """The credential store is a newer version that supported."""
+  pass
+
+
+def get_credential_storage(filename, client_id, user_agent, scope,
+                           warn_on_readonly=True):
+  """Get a Storage instance for a credential.
+
+  Args:
+    filename: The JSON file storing a set of credentials
+    client_id: The client_id for the credential
+    user_agent: The user agent for the credential
+    scope: string or list of strings, Scope(s) being requested
+    warn_on_readonly: if True, log a warning if the store is readonly
+
+  Returns:
+    An object derived from client.Storage for getting/setting the
+    credential.
+  """
+  filename = os.path.realpath(os.path.expanduser(filename))
+  _multistores_lock.acquire()
+  try:
+    multistore = _multistores.setdefault(
+        filename, _MultiStore(filename, warn_on_readonly))
+  finally:
+    _multistores_lock.release()
+  if type(scope) is list:
+    scope = ' '.join(scope)
+  return multistore._get_storage(client_id, user_agent, scope)
+
+
+class _MultiStore(object):
+  """A file backed store for multiple credentials."""
+
+  def __init__(self, filename, warn_on_readonly=True):
+    """Initialize the class.
+
+    This will create the file if necessary.
+    """
+    self._file = LockedFile(filename, 'r+b', 'rb')
+    self._thread_lock = threading.Lock()
+    self._read_only = False
+    self._warn_on_readonly = warn_on_readonly
+
+    self._create_file_if_needed()
+
+    # Cache of deserialized store.  This is only valid after the
+    # _MultiStore is locked or _refresh_data_cache is called.  This is
+    # of the form of:
+    #
+    # (client_id, user_agent, scope) -> OAuth2Credential
+    #
+    # If this is None, then the store hasn't been read yet.
+    self._data = None
+
+  class _Storage(BaseStorage):
+    """A Storage object that knows how to read/write a single credential."""
+
+    def __init__(self, multistore, client_id, user_agent, scope):
+      self._multistore = multistore
+      self._client_id = client_id
+      self._user_agent = user_agent
+      self._scope = scope
+
+    def acquire_lock(self):
+      """Acquires any lock necessary to access this Storage.
+
+      This lock is not reentrant.
+      """
+      self._multistore._lock()
+
+    def release_lock(self):
+      """Release the Storage lock.
+
+      Trying to release a lock that isn't held will result in a
+      RuntimeError.
+      """
+      self._multistore._unlock()
+
+    def locked_get(self):
+      """Retrieve credential.
+
+      The Storage lock must be held when this is called.
+
+      Returns:
+        oauth2client.client.Credentials
+      """
+      credential = self._multistore._get_credential(
+          self._client_id, self._user_agent, self._scope)
+      if credential:
+        credential.set_store(self)
+      return credential
+
+    def locked_put(self, credentials):
+      """Write a credential.
+
+      The Storage lock must be held when this is called.
+
+      Args:
+        credentials: Credentials, the credentials to store.
+      """
+      self._multistore._update_credential(credentials, self._scope)
+
+    def locked_delete(self):
+      """Delete a credential.
+
+      The Storage lock must be held when this is called.
+
+      Args:
+        credentials: Credentials, the credentials to store.
+      """
+      self._multistore._delete_credential(self._client_id, self._user_agent,
+          self._scope)
+
+  def _create_file_if_needed(self):
+    """Create an empty file if necessary.
+
+    This method will not initialize the file. Instead it implements a
+    simple version of "touch" to ensure the file has been created.
+    """
+    if not os.path.exists(self._file.filename()):
+      old_umask = os.umask(0177)
+      try:
+        open(self._file.filename(), 'a+b').close()
+      finally:
+        os.umask(old_umask)
+
+  def _lock(self):
+    """Lock the entire multistore."""
+    self._thread_lock.acquire()
+    self._file.open_and_lock()
+    if not self._file.is_locked():
+      self._read_only = True
+      if self._warn_on_readonly:
+        logger.warn('The credentials file (%s) is not writable. Opening in '
+                    'read-only mode. Any refreshed credentials will only be '
+                    'valid for this run.' % self._file.filename())
+    if os.path.getsize(self._file.filename()) == 0:
+      logger.debug('Initializing empty multistore file')
+      # The multistore is empty so write out an empty file.
+      self._data = {}
+      self._write()
+    elif not self._read_only or self._data is None:
+      # Only refresh the data if we are read/write or we haven't
+      # cached the data yet.  If we are readonly, we assume is isn't
+      # changing out from under us and that we only have to read it
+      # once.  This prevents us from whacking any new access keys that
+      # we have cached in memory but were unable to write out.
+      self._refresh_data_cache()
+
+  def _unlock(self):
+    """Release the lock on the multistore."""
+    self._file.unlock_and_close()
+    self._thread_lock.release()
+
+  def _locked_json_read(self):
+    """Get the raw content of the multistore file.
+
+    The multistore must be locked when this is called.
+
+    Returns:
+      The contents of the multistore decoded as JSON.
+    """
+    assert self._thread_lock.locked()
+    self._file.file_handle().seek(0)
+    return simplejson.load(self._file.file_handle())
+
+  def _locked_json_write(self, data):
+    """Write a JSON serializable data structure to the multistore.
+
+    The multistore must be locked when this is called.
+
+    Args:
+      data: The data to be serialized and written.
+    """
+    assert self._thread_lock.locked()
+    if self._read_only:
+      return
+    self._file.file_handle().seek(0)
+    simplejson.dump(data, self._file.file_handle(), sort_keys=True, indent=2)
+    self._file.file_handle().truncate()
+
+  def _refresh_data_cache(self):
+    """Refresh the contents of the multistore.
+
+    The multistore must be locked when this is called.
+
+    Raises:
+      NewerCredentialStoreError: Raised when a newer client has written the
+        store.
+    """
+    self._data = {}
+    try:
+      raw_data = self._locked_json_read()
+    except Exception:
+      logger.warn('Credential data store could not be loaded. '
+                  'Will ignore and overwrite.')
+      return
+
+    version = 0
+    try:
+      version = raw_data['file_version']
+    except Exception:
+      logger.warn('Missing version for credential data store. It may be '
+                  'corrupt or an old version. Overwriting.')
+    if version > 1:
+      raise NewerCredentialStoreError(
+          'Credential file has file_version of %d. '
+          'Only file_version of 1 is supported.' % version)
+
+    credentials = []
+    try:
+      credentials = raw_data['data']
+    except (TypeError, KeyError):
+      pass
+
+    for cred_entry in credentials:
+      try:
+        (key, credential) = self._decode_credential_from_json(cred_entry)
+        self._data[key] = credential
+      except:
+        # If something goes wrong loading a credential, just ignore it
+        logger.info('Error decoding credential, skipping', exc_info=True)
+
+  def _decode_credential_from_json(self, cred_entry):
+    """Load a credential from our JSON serialization.
+
+    Args:
+      cred_entry: A dict entry from the data member of our format
+
+    Returns:
+      (key, cred) where the key is the key tuple and the cred is the
+        OAuth2Credential object.
+    """
+    raw_key = cred_entry['key']
+    client_id = raw_key['clientId']
+    user_agent = raw_key['userAgent']
+    scope = raw_key['scope']
+    key = (client_id, user_agent, scope)
+    credential = None
+    credential = Credentials.new_from_json(simplejson.dumps(cred_entry['credential']))
+    return (key, credential)
+
+  def _write(self):
+    """Write the cached data back out.
+
+    The multistore must be locked.
+    """
+    raw_data = {'file_version': 1}
+    raw_creds = []
+    raw_data['data'] = raw_creds
+    for (cred_key, cred) in self._data.items():
+      raw_key = {
+          'clientId': cred_key[0],
+          'userAgent': cred_key[1],
+          'scope': cred_key[2]
+          }
+      raw_cred = simplejson.loads(cred.to_json())
+      raw_creds.append({'key': raw_key, 'credential': raw_cred})
+    self._locked_json_write(raw_data)
+
+  def _get_credential(self, client_id, user_agent, scope):
+    """Get a credential from the multistore.
+
+    The multistore must be locked.
+
+    Args:
+      client_id: The client_id for the credential
+      user_agent: The user agent for the credential
+      scope: A string for the scope(s) being requested
+
+    Returns:
+      The credential specified or None if not present
+    """
+    key = (client_id, user_agent, scope)
+
+    return self._data.get(key, None)
+
+  def _update_credential(self, cred, scope):
+    """Update a credential and write the multistore.
+
+    This must be called when the multistore is locked.
+
+    Args:
+      cred: The OAuth2Credential to update/set
+      scope: The scope(s) that this credential covers
+    """
+    key = (cred.client_id, cred.user_agent, scope)
+    self._data[key] = cred
+    self._write()
+
+  def _delete_credential(self, client_id, user_agent, scope):
+    """Delete a credential and write the multistore.
+
+    This must be called when the multistore is locked.
+
+    Args:
+      client_id: The client_id for the credential
+      user_agent: The user agent for the credential
+      scope: The scope(s) that this credential covers
+    """
+    key = (client_id, user_agent, scope)
+    try:
+      del self._data[key]
+    except KeyError:
+      pass
+    self._write()
+
+  def _get_storage(self, client_id, user_agent, scope):
+    """Get a Storage object to get/set a credential.
+
+    This Storage is a 'view' into the multistore.
+
+    Args:
+      client_id: The client_id for the credential
+      user_agent: The user agent for the credential
+      scope: A string for the scope(s) being requested
+
+    Returns:
+      A Storage object that can be used to get/set this cred
+    """
+    return self._Storage(self, client_id, user_agent, scope)