googlecloud 0.0.2 → 0.0.4

data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/thread_pool.py

@@ -0,0 +1,188 @@
+# Copyright 2012 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A simple thread pool class for doing multiple concurrent API operations."""
+
+
+
+import logging
+import Queue
+import sys
+import threading
+import time
+import traceback
+
+LOGGER = logging.getLogger('gcutil-logs')
+
+
+class ThreadPoolError(Exception):
+  """An error occurred in this module."""
+  pass
+
+
+class Operation(object):
+  """An operation to be executed by the threadpool.
+
+  Override this and implement the Run() method.
+  """
+
+  def __init__(self):
+    """Initializer."""
+    self._result = None
+    self._raised_exception = False
+
+  def Run(self):
+    """Override this method to execute this operation."""
+    raise NotImplementedError('pure virtual method called')
+
+  def _DoOperation(self):
+    """Internal runner that captures result."""
+    try:
+      self._result = self.Run()
+    except:  # pylint: disable-msg=W0702
+      self._raised_exception = True
+      a = sys.exc_info()
+      # If a string was thrown, a[1] is None.  In Python 2.5, if an exception
+      # was thrown without a message, a is a 1-tuple.  Otherwise, the exception
+      # object is in a[1].
+      if len(a) < 2 or a[1] is None:
+        self._result = a[0]
+      else:
+        self._result = a[1]
+      LOGGER.debug(traceback.format_exc())
+
+  def Result(self):
+    """Get the operation's result.
+
+    If the operation is incomplete the return value will be None.  If the
+    operation raised an exception, the return value will be the exception
+    object.
+
+    Returns:
+      The operation's result.
+    """
+    return self._result
+
+  def RaisedException(self):
+    """Did the operation raise an exception?
+
+    Will be False if the operation has not yet completed.
+
+    Returns:
+      True if an exception was raised by the operation.
+    """
+    return self._raised_exception
+
+
+class Worker(threading.Thread):
+  """Thread executing tasks from a given tasks queue."""
+
+  def __init__(self, queue):
+    threading.Thread.__init__(self)
+    self._queue = queue
+    self.daemon = True
+    self.start()
+
+  def run(self):
+    # pylint: disable-msg=W0212
+    while True:
+      op = self._queue.get()
+      if op is None:
+        self._queue.task_done()
+        break
+      op._DoOperation()  # pylint: disable-msg=W0212
+      self._queue.task_done()
+
+
+class ThreadPool(object):
+  """Pool of threads consuming tasks from a queue.
+
+  Note that operations on the thread pool itself (submitting, waiting,
+  shutdown) are not, themselves, multithread safe.
+  """
+
+  # States
+  _NOT_RUNNING = 0
+  _RUNNING = 1
+  _TERMINATING = 3
+  _TERMINATED = 4
+
+  def __init__(self, num_threads):
+    self._queue = Queue.Queue()
+    self._num_threads = num_threads
+    self._state = self._NOT_RUNNING
+
+    self._workers = []
+    for _ in range(num_threads):
+      self._workers.append(Worker(self._queue))
+    self._state = self._RUNNING
+
+  def __del__(self):
+    # Shut down everything so that we don't leak memory.
+    if self._state == self._RUNNING:
+      self.WaitShutdown()
+
+  def Add(self, op):
+    """Add an operation to the queue.
+
+    Note that this is not thread safe.
+
+    Args:
+      op: An Operation object to add to the thread pool queue
+
+    Raises:
+      ThreadPoolError: if not in running state.
+      ValueError: if op isn't an Operation object
+    """
+    if self._state != self._RUNNING:
+      raise ThreadPoolError('ThreadPool not running')
+    if not isinstance(op, Operation):
+      raise ValueError('Nonoperation argument to AddOperation')
+    self._queue.put(op)
+
+  def _InternalWait(self):
+    """Wait for all items to clear.
+
+    This will come up for air once in a while so that we can capture
+    keyboard interrupt.  Unfortunately Queue.join() isn't
+    interruptable.
+    """
+    while not self._queue.empty():
+      time.sleep(0.2)
+
+  def WaitAll(self):
+    """Wait for completion of all the tasks in the queue.
+
+    Note that this is not thread safe.
+
+    Raises:
+      ThreadPoolError: if not in running state.
+    """
+    if self._state != self._RUNNING:
+      raise ThreadPoolError('ThreadPool not running')
+    self._InternalWait()
+
+  def WaitShutdown(self):
+    """Wait for completion of all tasks and shut down the ThreadPool.
+
+    Note that this is not thread safe.
+    """
+    if self._state != self._RUNNING:
+      raise ThreadPoolError('ThreadPool not running')
+    self._state = self._TERMINATING
+    # Inject a set of sentinal values to have the workers exit.
+    for _ in range(self._num_threads):
+      self._queue.put(None)
+    self._InternalWait()
+    self._state = self._TERMINATED

data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/thread_pool_test.py

@@ -0,0 +1,88 @@
+# Copyright 2012 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Tests for thread_pool."""
+
+
+
+import path_initializer
+path_initializer.InitializeSysPath()
+
+import time
+
+import unittest
+from gcutil import thread_pool
+
+
+class TestOperation(thread_pool.Operation):
+  def __init__(self, raise_exception=False, sleep_time=0):
+    thread_pool.Operation.__init__(self)
+    self.raise_exception = raise_exception
+    self.sleep_time = sleep_time
+
+  def Run(self):
+    if self.sleep_time:
+      time.sleep(self.sleep_time)
+    if self.raise_exception:
+      raise Exception('Exception!')
+    return 42
+
+
+class ThreadPoolTest(unittest.TestCase):
+  def testBasic(self):
+    """Test basic start up and shutdown."""
+    tp = thread_pool.ThreadPool(3)
+    tp.WaitShutdown()
+
+  def testSubmit(self):
+    tp = thread_pool.ThreadPool(3)
+
+    ops = []
+    for _ in xrange(20):
+      op = TestOperation()
+      ops.append(op)
+      tp.Add(op)
+    tp.WaitShutdown()
+    for op in ops:
+      self.assertEqual(op.Result(), 42)
+      self.assertFalse(op.RaisedException())
+
+  def testLongOps(self):
+    tp = thread_pool.ThreadPool(3)
+
+    ops = []
+    for _ in xrange(10):
+      op = TestOperation(sleep_time=0.1)
+      ops.append(op)
+      tp.Add(op)
+    tp.WaitShutdown()
+    for op in ops:
+      self.assertEqual(op.Result(), 42)
+      self.assertFalse(op.RaisedException())
+
+  def testExceptionOps(self):
+    tp = thread_pool.ThreadPool(3)
+
+    ops = []
+    for _ in xrange(20):
+      op = TestOperation(raise_exception=0.1)
+      ops.append(op)
+      tp.Add(op)
+    tp.WaitShutdown()
+    for op in ops:
+      self.assertEqual(str(op.Result()), 'Exception!')
+      self.assertTrue(op.RaisedException())
+
+if __name__ == '__main__':
+  unittest.main()

data/packages/gcutil-1.7.1/lib/google_compute_engine/gcutil/utils.py

@@ -0,0 +1,208 @@
+# Copyright 2012 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""A set of utility functions."""
+
+import cStringIO
+import numbers
+import socket
+import sys
+
+
+
+
+def SimpleName(entity):
+  if entity is None:
+    return ''
+
+  elif isinstance(entity, basestring):
+    if 'projects/google/' in entity:
+      return 'google/' + entity.split('/')[-1]
+    else:
+      return entity.split('/')[-1]
+
+  elif isinstance(entity, numbers.Number):
+    return str(entity)
+
+  raise ValueError('Expected number or string: ' + str(entity))
+
+
+def FlattenList(list):
+  """Flattens a list of lists."""
+  return [item for sublist in list for item in sublist]
+
+
+def RegexesToFilterExpression(regexes, op='eq'):
+  """Converts a list of regular expressions to a filter expression on name.
+
+  Args:
+    regexes: A list of regular expressions to use for matching
+      resource names. Since resource names cannot contain whitespace
+      characters, regular expressions are split on whitespace (e.g.,
+      '[a-z]+ [0-9]+' will be treated as two separate regular expressions).
+
+  Returns:
+    The Google Compute Engine filter expression or None
+      if regexes evaluates to False.
+  """
+  if not regexes:
+    return None
+  regexes = FlattenList(regex.split() for regex in regexes)
+  return 'name %s %s' % (op, '|'.join(regexes))
+
+
+def SimplePrint(text, *args, **kwargs):
+  """Prints the given text without a new-line character at the end."""
+  print text.format(*args, **kwargs),
+  sys.stdout.flush()
+
+
+def ListStrings(strings, prefix='  '):
+  """Returns a string containing each item in strings on its own line.
+
+  Args:
+    strings: The list of strings to place in the result.
+    prefix: A string to place before each name.
+
+  Returns:
+    A string containing the names.
+  """
+  strings = sorted(strings)
+  buf = cStringIO.StringIO()
+  for string in strings:
+    buf.write(prefix + str(string) + '\n')
+  return buf.getvalue().rstrip()
+
+
+def Proceed(message=None):
+  """Prompts the user to proceed.
+
+  Args:
+    message: An optional message to include before
+      'Proceed? [y/N] ' is printed.
+
+  Returns:
+    True if the user answers yes.
+  """
+  message = ((message or '') + ' Proceed? [y/N] ').lstrip()
+  return raw_input(message).strip().lower() == 'y'
+
+
+def ParseProtocol(protocol_string):
+  """Attempt to parse a protocol number from a string.
+
+  Args:
+    protocol_string: The string to parse.
+
+  Returns:
+    The corresponding protocol number.
+
+  Raises:
+    ValueError: If the protocol_string is not a valid protocol string.
+  """
+  try:
+    protocol = socket.getprotobyname(protocol_string)
+  except (socket.error, TypeError):
+    try:
+      protocol = int(protocol_string)
+    except (ValueError, TypeError):
+      raise ValueError('Invalid protocol: %s' % protocol_string)
+
+  return protocol
+
+
+def ReplacePortNames(port_range_string):
+  """Replace port names with port numbers in a port-range string.
+
+  Args:
+    port_range_string: The string to parse.
+
+  Returns:
+    A port range string specifying ports only by number.
+
+  Raises:
+    ValueError: If the port_range_string is the wrong type or malformed.
+  """
+  if not isinstance(port_range_string, basestring):
+    raise ValueError('Invalid port range: %s' % port_range_string)
+
+  ports = port_range_string.split('-')
+  if len(ports) not in [1, 2]:
+    raise ValueError('Invalid port range: %s' % port_range_string)
+
+  try:
+    low_port = socket.getservbyname(ports[0])
+  except socket.error:
+    low_port = int(ports[0])
+
+  try:
+    high_port = socket.getservbyname(ports[-1])
+  except socket.error:
+    high_port = int(ports[-1])
+
+  if low_port == high_port:
+    return '%d' % low_port
+  else:
+    return '%d-%d' % (low_port, high_port)
+
+
+def Singularize(string):
+  """A naive function for singularizing Compute Engine collection names."""
+  return string[:len(string) - 1] if string.endswith('s') else string
+
+
+def All(func, project, max_results=None, filter=None, zone=None):
+  """Calls the given list function while taking care of paging logic.
+
+  Args:
+    func: A Google Compute Engine list function.
+    project: The project to query.
+    max_results: The maximum number of items to return.
+    filter: The filter expression to plumb through.
+    zone: The zone for list functions that require a zone.
+
+  Returns:
+    A list of the resources.
+  """
+  params = {
+      'project': project,
+      'maxResults': max_results,
+      'filter': filter}
+
+  if zone:
+    params['zone'] = zone
+
+  items = []
+  while True:
+    res = func(**params).execute()
+    kind = res.get('kind')
+    items.extend(res.get('items', []))
+
+    next_page_token = res.get('nextPageToken')
+    if not next_page_token:
+      break
+
+    params['pageToken'] = next_page_token
+
+  if max_results is not None:
+    items = items[:max_results]
+  return {'kind': kind,
+          'items': items}
+
+
+def AllNames(func, project, max_results=None, filter=None, zone=None):
+  """Like All, except returns a list of the names of the resources."""
+  list_res = All(
+      func, project, max_results=max_results, filter=filter, zone=zone)
+  return [resource.get('name') for resource in list_res.get('items', [])]