gcloud 0.0.7 → 0.1.0

data/packages/gsutil/gslib/commands/mb.py

@@ -1,172 +0,0 @@
-# Copyright 2011 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from gslib.command import Command
-from gslib.command import COMMAND_NAME
-from gslib.command import COMMAND_NAME_ALIASES
-from gslib.command import CONFIG_REQUIRED
-from gslib.command import FILE_URIS_OK
-from gslib.command import MAX_ARGS
-from gslib.command import MIN_ARGS
-from gslib.command import PROVIDER_URIS_OK
-from gslib.command import SUPPORTED_SUB_ARGS
-from gslib.command import URIS_START_ARG
-from gslib.exception import CommandException
-from gslib.help_provider import HELP_NAME
-from gslib.help_provider import HELP_NAME_ALIASES
-from gslib.help_provider import HELP_ONE_LINE_SUMMARY
-from gslib.help_provider import HELP_TEXT
-from gslib.help_provider import HelpType
-from gslib.help_provider import HELP_TYPE
-from gslib.util import NO_MAX
-
-_detailed_help_text = ("""
-<B>SYNOPSIS</B>
-  gsutil mb [-c storage_class] [-l location] [-p proj_id] uri...
-
-
-<B>DESCRIPTION</B>
-  The mb command creates a new bucket. Google Cloud Storage has a single
-  namespace, so you will not be allowed to create a bucket with a name already
-  in use by another user. You can, however, carve out parts of the bucket name
-  space corresponding to your company's domain name (see "gsutil help naming").
-
-  If you don't specify a project ID using the -p option, the bucket
-  will be created using the default project ID specified in your gsutil
-  configuration file (see "gsutil help config"). For more details about
-  projects see "gsutil help projects".
-
-  The -c and -l options specify the storage class and location, respectively,
-  for the bucket. Once a bucket is created in a given location and with a
-  given storage class, it cannot be moved to a different location, and the
-  storage class cannot be changed. Instead, you would need to create a new
-  bucket and move the data over and then delete the original bucket.
-
-
-<B>BUCKET STORAGE CLASSES</B>
-  If you don't specify a -c option, the bucket will be created with the default
-  (standard) storage class.
-
-  If you specify -c DURABLE_REDUCED_AVAILABILITY (or -c DRA), it causes the data
-  stored in the bucket to use durable reduced availability storage. Buckets
-  created with this storage class have lower availability than standard storage
-  class buckets, but durability equal to that of buckets created with standard
-  storage class. This option allows users to reduce costs for data for which
-  lower availability is acceptable. Durable Reduced Availability storage would
-  not be appropriate for "hot" objects (i.e., objects being accessed frequently)
-  or for interactive workloads; however, it might be appropriate for other types
-  of applications. See the online documentation for pricing and SLA details.
-
-
-<B>BUCKET LOCATIONS</B>
-  If you don't specify a -l option, the bucket will be created in the default
-  location (US). Otherwise, you can specify one of the available locations:
-  US (United States) or EU (Europe).
-
-
-<B>OPTIONS</B>
-  -c storage_class  Can be DRA (or DURABLE_REDUCED_AVAILABILITY) or S (or
-                    STANDARD). Default is STANDARD.
-
-  -l location       Can be US or EU. Default is US. Locations are case
-                    insensitive.
-
-  -p proj_id        Specifies the project ID under which to create the bucket.
-""")
-
-
-class MbCommand(Command):
-  """Implementation of gsutil mb command."""
-
-  # Command specification (processed by parent class).
-  command_spec = {
-    # Name of command.
-    COMMAND_NAME : 'mb',
-    # List of command name aliases.
-    COMMAND_NAME_ALIASES : ['makebucket', 'createbucket', 'md', 'mkdir'],
-    # Min number of args required by this command.
-    MIN_ARGS : 1,
-    # Max number of args required by this command, or NO_MAX.
-    MAX_ARGS : NO_MAX,
-    # Getopt-style string specifying acceptable sub args.
-    SUPPORTED_SUB_ARGS : 'c:l:p:',
-    # True if file URIs acceptable for this command.
-    FILE_URIS_OK : False,
-    # True if provider-only URIs acceptable for this command.
-    PROVIDER_URIS_OK : False,
-    # Index in args of first URI arg.
-    URIS_START_ARG : 0,
-    # True if must configure gsutil before running command.
-    CONFIG_REQUIRED : True,
-  }
-  help_spec = {
-    # Name of command or auxiliary help info for which this help applies.
-    HELP_NAME : 'mb',
-    # List of help name aliases.
-    HELP_NAME_ALIASES : ['createbucket', 'makebucket', 'md', 'mkdir',
-                         'location', 'dra', 'dras', 'reduced_availability',
-                         'durable_reduced_availability',
-                         'rr', 'reduced_redundancy',
-                         'standard', 'storage class' ],
-    # Type of help:
-    HELP_TYPE : HelpType.COMMAND_HELP,
-    # One line summary of this help.
-    HELP_ONE_LINE_SUMMARY : 'Make buckets',
-    # The full help text.
-    HELP_TEXT : _detailed_help_text,
-  }
-
-  # Command entry point.
-  def RunCommand(self):
-    location = ''
-    storage_class = ''
-    if self.sub_opts:
-      for o, a in self.sub_opts:
-        if o == '-l':
-          location = a
-        elif o == '-p':
-          self.proj_id_handler.SetProjectId(a)
-        elif o == '-c':
-          storage_class = self._Normalize_Storage_Class(a)
-
-    if not self.headers:
-      headers = {}
-    else:
-      headers = self.headers.copy()
-
-    for bucket_uri_str in self.args:
-      bucket_uri = self.suri_builder.StorageUri(bucket_uri_str)
-      if not bucket_uri.names_bucket():
-        raise CommandException('The mb command requires a URI that specifies a '
-                               'bucket.\n"%s" is not valid.' % bucket_uri)
-      self.proj_id_handler.FillInProjectHeaderIfNeeded('mb', bucket_uri,
-                                                       headers)
-      print 'Creating %s...' % bucket_uri
-      # Pass storage_class param only if this is a GCS bucket. (In S3 the
-      # storage class is specified on the key object.)
-      if bucket_uri.scheme == 'gs':
-        bucket_uri.create_bucket(headers=headers, location=location,
-                                 storage_class=storage_class)
-      else:
-        bucket_uri.create_bucket(headers=headers, location=location)
-
-    return 0
-
-  def _Normalize_Storage_Class(self, sc):
-    sc = sc.upper()
-    if sc in ('DRA', 'DURABLE_REDUCED_AVAILABILITY'):
-      return 'DURABLE_REDUCED_AVAILABILITY'
-    if sc in ('S', 'STD', 'STANDARD'):
-      return 'STANDARD'
-    return sc

data/packages/gsutil/gslib/commands/mv.py

@@ -1,159 +0,0 @@
-# Copyright 2011 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-from gslib.command import Command
-from gslib.command import COMMAND_NAME
-from gslib.command import COMMAND_NAME_ALIASES
-from gslib.command import CONFIG_REQUIRED
-from gslib.command import FILE_URIS_OK
-from gslib.command import MAX_ARGS
-from gslib.command import MIN_ARGS
-from gslib.command import PROVIDER_URIS_OK
-from gslib.command import SUPPORTED_SUB_ARGS
-from gslib.command import URIS_START_ARG
-from gslib.exception import CommandException
-from gslib.help_provider import HELP_NAME
-from gslib.help_provider import HELP_NAME_ALIASES
-from gslib.help_provider import HELP_ONE_LINE_SUMMARY
-from gslib.help_provider import HELP_TEXT
-from gslib.help_provider import HelpType
-from gslib.help_provider import HELP_TYPE
-from gslib.util import NO_MAX
-
-_detailed_help_text = ("""
-<B>SYNOPSIS</B>
-  gsutil mv [-p] src_uri dst_uri
-    - or -
-  gsutil mv [-p] uri... dst_uri
-
-
-<B>DESCRIPTION</B>
-  The gsutil mv command allows you to move data between your local file
-  system and the cloud, move data within the cloud, and move data between
-  cloud storage providers. For example, to move all objects from a
-  bucket to a local directory you could use:
-
-    gsutil mv gs://my_bucket dir
-
-  Similarly, to move all objects from a local directory to a bucket you could
-  use:
-
-    gsutil mv ./dir gs://my_bucket
-
-
-<B>RENAMING BUCKET SUBDIRECTORIES</B>
-  You can use the gsutil mv command to rename subdirectories. For example,
-  the command:
-
-    gsutil mv gs://my_bucket/olddir gs://my_bucket/newdir
-
-  would rename all objects and subdirectories under gs://my_bucket/olddir to be
-  under gs://my_bucket/newdir, otherwise preserving the subdirectory structure.
-
-  If you do a rename as specified above and you want to preserve ACLs, you
-  should use the -p option (see OPTIONS).
-
-  Note that when using mv to rename bucket subdirectories you cannot specify
-  the source URI using wildcards. You need to spell out the complete name:
-
-    gsutil mv gs://my_bucket/olddir gs://my_bucket/newdir
-
-  If you have a large number of files to move you might want to use the
-  gsutil -m option, to perform a multi-threaded/multi-processing move:
-
-    gsutil -m mv gs://my_bucket/olddir gs://my_bucket/newdir
-
-
-<B>NON-ATOMIC OPERATION</B>
-  Unlike the case with many file systems, the gsutil mv command does not
-  perform a single atomic operation. Rather, it performs a copy from source
-  to destination followed by removing the source for each object.
-
-
-<B>OPTIONS</B>
-  -p          Causes ACL to be preserved when moving in the cloud. Note that
-              this option has performance and cost implications, because it
-              is essentially performing three requests (getacl, cp, setacl).
-              (The performance issue can be mitigated to some degree by
-              using gsutil -m cp to cause multi-threaded/multi-processing
-              copying.)
-""")
-
-
-class MvCommand(Command):
-  """Implementation of gsutil mv command.
-     Note that there is no atomic rename operation - this command is simply
-     a shorthand for 'cp' followed by 'rm'.
-  """
-
-  # Command specification (processed by parent class).
-  command_spec = {
-    # Name of command.
-    COMMAND_NAME : 'mv',
-    # List of command name aliases.
-    COMMAND_NAME_ALIASES : ['move', 'ren', 'rename'],
-    # Min number of args required by this command.
-    MIN_ARGS : 2,
-    # Max number of args required by this command, or NO_MAX.
-    MAX_ARGS : NO_MAX,
-    # Getopt-style string specifying acceptable sub args.
-    SUPPORTED_SUB_ARGS : 'pv',
-    # True if file URIs acceptable for this command.
-    FILE_URIS_OK : True,
-    # True if provider-only URIs acceptable for this command.
-    PROVIDER_URIS_OK : False,
-    # Index in args of first URI arg.
-    URIS_START_ARG : 0,
-    # True if must configure gsutil before running command.
-    CONFIG_REQUIRED : True,
-  }
-  help_spec = {
-    # Name of command or auxiliary help info for which this help applies.
-    HELP_NAME : 'mv',
-    # List of help name aliases.
-    HELP_NAME_ALIASES : ['move', 'rename'],
-    # Type of help:
-    HELP_TYPE : HelpType.COMMAND_HELP,
-    # One line summary of this help.
-    HELP_ONE_LINE_SUMMARY : 'Move/rename objects and/or subdirectories',
-    # The full help text.
-    HELP_TEXT : _detailed_help_text,
-  }
-
-  # Command entry point.
-  def RunCommand(self):
-    # Check each source arg up, refusing to delete a bucket src URI (force users
-    # to explicitly do that as a separate operation).
-    for arg_to_check in self.args[0:-1]:
-      if self.suri_builder.StorageUri(arg_to_check).names_bucket():
-        raise CommandException('You cannot move a source bucket using the mv '
-                               'command. If you meant to move\nall objects in '
-                               'the bucket, you can use a command like:\n'
-                               '\tgsutil mv %s/* %s' %
-                               (arg_to_check, self.args[-1]))
-
-    # Insert command-line opts in front of args so they'll be picked up by cp
-    # and rm commands (e.g., for -p option). Use undocumented (internal
-    # use-only) cp -M option, which causes each original object to be deleted
-    # after successfully copying to its destination, and also causes naming
-    # behavior consistent with Unix mv naming behavior (see comments in
-    # _ConstructDstUri in cp.py).
-    unparsed_args = ['-M']
-    if self.recursion_requested:
-      unparsed_args.append('-R')
-    unparsed_args.extend(self.unparsed_args)
-    self.command_runner.RunNamedCommand('cp', unparsed_args, self.headers,
-                                        self.debug, self.parallel_operations)
-
-    return 0

data/packages/gsutil/gslib/commands/perfdiag.py

@@ -1,903 +0,0 @@
-# Copyright 2012 Google Inc. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-"""Contains the perfdiag gsutil command."""
-
-import calendar
-from collections import defaultdict
-import contextlib
-import datetime
-import json
-import math
-import multiprocessing
-import os
-import re
-import socket
-import string
-import subprocess
-import tempfile
-import time
-
-import boto.gs.connection
-
-from gslib.command import Command
-from gslib.command import COMMAND_NAME
-from gslib.command import COMMAND_NAME_ALIASES
-from gslib.command import CONFIG_REQUIRED
-from gslib.command import FILE_URIS_OK
-from gslib.command import MAX_ARGS
-from gslib.command import MIN_ARGS
-from gslib.command import PROVIDER_URIS_OK
-from gslib.command import SUPPORTED_SUB_ARGS
-from gslib.command import URIS_START_ARG
-from gslib.commands import config
-from gslib.exception import CommandException
-from gslib.help_provider import HELP_NAME
-from gslib.help_provider import HELP_NAME_ALIASES
-from gslib.help_provider import HELP_ONE_LINE_SUMMARY
-from gslib.help_provider import HELP_TEXT
-from gslib.help_provider import HELP_TYPE
-from gslib.help_provider import HelpType
-from gslib.util import IS_LINUX
-from gslib.util import MakeBitsHumanReadable
-from gslib.util import MakeHumanReadable
-from gslib.util import Percentile
-
-_detailed_help_text = ("""
-<B>SYNOPSIS</B>
-  gsutil perfdiag [-i in.json] [-o out.json]
-                  [-n iterations] [-c concurrency] [-s size] [-t tests] uri...
-
-
-<B>DESCRIPTION</B>
-  The perfdiag command runs a suite of diagnostic tests for a given Google
-  Storage bucket.
-
-  The 'uri' parameter must name an existing bucket (e.g. gs://foo) to which
-  the user has write permission. Several test files will be uploaded to and
-  downloaded from this bucket. All test files will be deleted at the completion
-  of the diagnostic if it finishes successfully.
-
-  gsutil performance can be impacted by many factors at the client, server,
-  and in-between, such as: CPU speed; available memory; the access path to the
-  local disk; network bandwidth; contention and error rates along the path
-  between gsutil and Google; operating system buffering configuration; and
-  firewalls and other network elements. The perfdiag command is provided so
-  that customers can run a known measurement suite when troubleshooting
-  performance problems.
-
-
-<B>PROVIDING DIAGNOSTIC OUTPUT TO GOOGLE CLOUD STORAGE TEAM</B>
-  If the Google Cloud Storage Team asks you to run a performance diagnostic
-  please use the following command, and email the output file (output.json)
-  to gs-team@google.com:
-
-    gsutil perfdiag -o output.json gs://your-bucket
-
-
-<B>OPTIONS</B>
-  -n          Sets the number of iterations performed when downloading and
-              uploading files during latency and throughput tests. Defaults to
-              5.
-
-  -c          Sets the level of concurrency to use while running throughput
-              experiments. The default value of 1 will only run a single read
-              or write operation concurrently.
-
-  -s          Sets the size (in bytes) of the test file used to perform read
-              and write throughput tests. The default is 1 MiB.
-
-  -t          Sets the list of diagnostic tests to perform. The default is to
-              run all diagnostic tests. Must be a comma-separated list
-              containing one or more of the following:
-
-                  lat: Runs N iterations (set with -n) of writing the file,
-                       retrieving its metadata, reading the file, and deleting
-                       the file. Records the latency of each operation.
-
-                rthru: Runs N (set with -n) read operations, with at most C
-                       (set with -c) reads outstanding at any given time.
-
-                wthru: Runs N (set with -n) write operations, with at most C
-                       (set with -c) writes outstanding at any given time.
-
-  -o          Writes the results of the diagnostic to an output file. The output
-              is a JSON file containing system information and performance
-              diagnostic results. The file can be read and reported later using
-              the -i option.
-
-  -i          Reads the JSON output file created using the -o command and prints
-              a formatted description of the results.
-
-
-<B>NOTE</B>
-  The perfdiag command collects system information. It collects your IP address,
-  executes DNS queries to Google servers and collects the results, and collects
-  network statistics information from the output of netstat -s. None of this
-  information will be sent to Google unless you choose to send it.
-""")
-
-
-class PerfDiagCommand(Command):
-  """Implementation of gsutil perfdiag command."""
-
-  # Command specification (processed by parent class).
-  command_spec = {
-      # Name of command.
-      COMMAND_NAME: 'perfdiag',
-      # List of command name aliases.
-      COMMAND_NAME_ALIASES: ['diag', 'diagnostic', 'perf', 'performance'],
-      # Min number of args required by this command.
-      MIN_ARGS: 0,
-      # Max number of args required by this command, or NO_MAX.
-      MAX_ARGS: 1,
-      # Getopt-style string specifying acceptable sub args.
-      SUPPORTED_SUB_ARGS: 'n:c:s:t:i:o:',
-      # True if file URIs acceptable for this command.
-      FILE_URIS_OK: False,
-      # True if provider-only URIs acceptable for this command.
-      PROVIDER_URIS_OK: False,
-      # Index in args of first URI arg.
-      URIS_START_ARG: 0,
-      # True if must configure gsutil before running command.
-      CONFIG_REQUIRED: True,
-  }
-  help_spec = {
-      # Name of command or auxiliary help info for which this help applies.
-      HELP_NAME: 'perfdiag',
-      # List of help name aliases.
-      HELP_NAME_ALIASES: [],
-      # Type of help:
-      HELP_TYPE: HelpType.COMMAND_HELP,
-      # One line summary of this help.
-      HELP_ONE_LINE_SUMMARY: 'Run performance diagnostic',
-      # The full help text.
-      HELP_TEXT: _detailed_help_text,
-  }
-
-  # Byte sizes to use for testing files.
-  # TODO: Consider letting the user specify these sizes with a configuration
-  # parameter.
-  test_file_sizes = (
-      0,  # 0 bytes
-      1024,  # 1 KB
-      102400,  # 100 KB
-      1048576,  # 1MB
-  )
-
-  # List of all diagnostic tests.
-  ALL_DIAG_TESTS = ('rthru', 'wthru', 'lat')
-
-  # Google Cloud Storage API endpoint host.
-  GOOGLE_API_HOST = boto.gs.connection.GSConnection.DefaultHost
-
-  def _WindowedExec(self, cmd, n, w, raise_on_error=True):
-    """Executes a command n times with a window size of w.
-
-    Up to w instances of the command will be executed and left outstanding at a
-    time until n instances of the command have completed.
-
-    Args:
-      cmd: List containing the command to execute.
-      n: Number of times the command will be executed.
-      w: Window size of outstanding commands being executed.
-      raise_on_error: See _Exec.
-
-    Raises:
-      Exception: If raise_on_error is set to True and any process exits with a
-      non-zero return code.
-    """
-    if self.debug:
-      print 'Running command:', cmd
-    devnull_f = open(os.devnull, 'w')
-    num_finished = 0
-    running = []
-    while len(running) or num_finished < n:
-      # Fires off new commands that can be executed.
-      while len(running) < w and num_finished + len(running) < n:
-        print 'Starting concurrent command: %s' % (' '.join(cmd))
-        p = subprocess.Popen(cmd, stdout=devnull_f, stderr=devnull_f)
-        running.append(p)
-
-      # Checks for finished commands.
-      prev_running = running
-      running = []
-      for p in prev_running:
-        retcode = p.poll()
-        if retcode is None:
-          running.append(p)
-        elif raise_on_error and retcode:
-          raise CommandException("Received non-zero return code (%d) from "
-                                 "subprocess '%s'." % (retcode, ' '.join(cmd)))
-        else:
-          num_finished += 1
-
-  def _Exec(self, cmd, raise_on_error=True, return_output=False,
-            mute_stderr=False):
-    """Executes a command in a subprocess.
-
-    Args:
-      cmd: List containing the command to execute.
-      raise_on_error: Whether or not to raise an exception when a process exits
-          with a non-zero return code.
-      return_output: If set to True, the return value of the function is the
-          stdout of the process.
-      mute_stderr: If set to True, the stderr of the process is not printed to
-          the console.
-
-    Returns:
-      The return code of the process or the stdout if return_output is set.
-
-    Raises:
-      Exception: If raise_on_error is set to True and any process exits with a
-      non-zero return code.
-    """
-    if self.debug:
-      print 'Running command:', cmd
-    stderr = subprocess.PIPE if mute_stderr else None
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=stderr)
-    (stdoutdata, stderrdata) = p.communicate()
-    if raise_on_error and p.returncode:
-      raise CommandException("Received non-zero return code (%d) from "
-                             "subprocess '%s'." % (p.returncode, ' '.join(cmd)))
-    return stdoutdata if return_output else p.returncode
-
-  def _GsUtil(self, cmd, raise_on_error=True, return_output=False,
-              mute_stderr=False):
-    """Executes a gsutil command in a subprocess.
-
-    Args:
-      cmd: A list containing the arguments to the gsutil program, e.g. ['ls',
-          'gs://foo'].
-      raise_on_error: see _Exec.
-      return_output: see _Exec.
-      mute_stderr: see _Exec.
-
-    Returns:
-      The return code of the process or the stdout if return_output is set.
-    """
-    cmd = self.gsutil_exec_list + cmd
-    return self._Exec(cmd, raise_on_error=raise_on_error,
-                      return_output=return_output, mute_stderr=mute_stderr)
-
-  def _SetUp(self):
-    """Performs setup operations needed before diagnostics can be run."""
-
-    # Stores test result data.
-    self.results = {}
-    # List of test files in a temporary location on disk for latency ops.
-    self.latency_files = []
-    # Maps each test file path to its size in bytes.
-    self.file_sizes = {}
-    # Maps each test file to its contents as a string.
-    self.file_contents = {}
-
-    def _MakeFile(file_size):
-      """Creates a temporary file of the given size and returns its path."""
-      fd, fpath = tempfile.mkstemp(suffix='.bin', prefix='gsutil_test_file',
-                                   text=False)
-      self.file_sizes[fpath] = file_size
-      f = os.fdopen(fd, 'wb')
-      f.write(os.urandom(file_size))
-      f.close()
-      f = open(fpath, 'rb')
-      self.file_contents[fpath] = f.read()
-      f.close()
-      return fpath
-
-    # Create files for latency tests.
-    for file_size in self.test_file_sizes:
-      fpath = _MakeFile(file_size)
-      self.latency_files.append(fpath)
-
-    # Local file on disk for write throughput tests.
-    self.thru_local_file = _MakeFile(self.thru_filesize)
-    # Remote file to write/read from during throughput tests.
-    self.thru_remote_file = (str(self.bucket_uri) +
-                             os.path.basename(self.thru_local_file))
-
-  def _TearDown(self):
-    """Performs operations to clean things up after performing diagnostics."""
-    for fpath in self.latency_files + [self.thru_local_file]:
-      try:
-        os.remove(fpath)
-      except OSError:
-        pass
-
-    self._GsUtil(['rm', self.thru_remote_file], raise_on_error=False,
-                 mute_stderr=True)
-
-  @contextlib.contextmanager
-  def _Time(self, key, bucket):
-    """A context manager that measures time.
-
-    A context manager that prints a status message before and after executing
-    the inner command and times how long the inner command takes. Keeps track of
-    the timing, aggregated by the given key.
-
-    Args:
-      key: The key to insert the timing value into a dictionary bucket.
-      bucket: A dictionary to place the timing value in.
-
-    Yields:
-      For the context manager.
-    """
-    print key, 'starting...'
-    t0 = time.time()
-    yield
-    t1 = time.time()
-    bucket[key].append(t1 - t0)
-    print key, 'done.'
-
-  def _RunLatencyTests(self):
-    """Runs latency tests."""
-    # Stores timing information for each category of operation.
-    self.results['latency'] = defaultdict(list)
-
-    for i in range(self.num_iterations):
-      print
-      print 'Running latency iteration %d...' % (i+1)
-      for fpath in self.latency_files:
-        basename = os.path.basename(fpath)
-        gsbucket = str(self.bucket_uri)
-        gsuri = gsbucket + basename
-        file_size = self.file_sizes[fpath]
-        readable_file_size = MakeHumanReadable(file_size)
-
-        print
-        print ("File of size %(size)s located on disk at '%(fpath)s' being "
-               "diagnosed in the cloud at '%(gsuri)s'."
-               % {'size': readable_file_size,
-                  'fpath': fpath,
-                  'gsuri': gsuri})
-
-        k = self.bucket.key_class(self.bucket)
-        k.key = basename
-
-        with self._Time('UPLOAD_%d' % file_size, self.results['latency']):
-          k.set_contents_from_string(self.file_contents[fpath])
-        with self._Time('METADATA_%d' % file_size, self.results['latency']):
-          k.exists()
-        with self._Time('DOWNLOAD_%d' % file_size, self.results['latency']):
-          k.get_contents_as_string()
-        with self._Time('DELETE_%d' % file_size, self.results['latency']):
-          k.delete()
-
-  def _RunReadThruTests(self):
-    """Runs read throughput tests."""
-    self.results['read_throughput'] = {'file_size': self.thru_filesize,
-                                       'num_times': self.num_iterations,
-                                       'concurrency': self.concurrency}
-
-    # Copy the file to remote location before reading.
-    self._GsUtil(['cp', self.thru_local_file, self.thru_remote_file])
-
-    if self.concurrency == 1:
-      k = self.bucket.key_class(self.bucket)
-      k.key = os.path.basename(self.thru_local_file)
-      # Warm up the TCP connection by transferring a couple times first.
-      for i in range(2):
-        k.get_contents_as_string()
-      t0 = time.time()
-      for i in range(self.num_iterations):
-        k.get_contents_as_string()
-      t1 = time.time()
-    else:
-      cmd = self.gsutil_exec_list + ['cp', self.thru_remote_file, os.devnull]
-      t0 = time.time()
-      self._WindowedExec(cmd, self.num_iterations, self.concurrency)
-      t1 = time.time()
-
-    time_took = t1 - t0
-    total_bytes_copied = self.thru_filesize * self.num_iterations
-    bytes_per_second = total_bytes_copied / time_took
-
-    self.results['read_throughput']['time_took'] = time_took
-    self.results['read_throughput']['total_bytes_copied'] = total_bytes_copied
-    self.results['read_throughput']['bytes_per_second'] = bytes_per_second
-
-  def _RunWriteThruTests(self):
-    """Runs write throughput tests."""
-    self.results['write_throughput'] = {'file_size': self.thru_filesize,
-                                        'num_copies': self.num_iterations,
-                                        'concurrency': self.concurrency}
-
-    if self.concurrency == 1:
-      k = self.bucket.key_class(self.bucket)
-      k.key = os.path.basename(self.thru_local_file)
-      # Warm up the TCP connection by transferring a couple times first.
-      for i in range(2):
-        k.set_contents_from_string(self.file_contents[self.thru_local_file])
-      t0 = time.time()
-      for i in range(self.num_iterations):
-        k.set_contents_from_string(self.file_contents[self.thru_local_file])
-      t1 = time.time()
-    else:
-      cmd = self.gsutil_exec_list + ['cp', self.thru_local_file,
-                                     self.thru_remote_file]
-      t0 = time.time()
-      self._WindowedExec(cmd, self.num_iterations, self.concurrency)
-      t1 = time.time()
-
-    time_took = t1 - t0
-    total_bytes_copied = self.thru_filesize * self.num_iterations
-    bytes_per_second = total_bytes_copied / time_took
-
-    self.results['write_throughput']['time_took'] = time_took
-    self.results['write_throughput']['total_bytes_copied'] = total_bytes_copied
-    self.results['write_throughput']['bytes_per_second'] = bytes_per_second
-
-  def _GetDiskCounters(self):
-    """Retrieves disk I/O statistics for all disks.
-
-    Adapted from the psutil module's psutil._pslinux.disk_io_counters:
-      http://code.google.com/p/psutil/source/browse/trunk/psutil/_pslinux.py
-
-    Originally distributed under under a BSD license.
-    Original Copyright (c) 2009, Jay Loden, Dave Daeschler, Giampaolo Rodola.
-
-    Returns:
-      A dictionary containing disk names mapped to the disk counters from
-      /disk/diskstats.
-    """
-    # iostat documentation states that sectors are equivalent with blocks and
-    # have a size of 512 bytes since 2.4 kernels. This value is needed to
-    # calculate the amount of disk I/O in bytes.
-    sector_size = 512
-
-    partitions = []
-    with open('/proc/partitions', 'r') as f:
-      lines = f.readlines()[2:]
-      for line in lines:
-        _, _, _, name = line.split()
-        if name[-1].isdigit():
-          partitions.append(name)
-
-    retdict = {}
-    with open('/proc/diskstats', 'r') as f:
-      for line in f:
-        values = line.split()[:11]
-        _, _, name, reads, _, rbytes, rtime, writes, _, wbytes, wtime = values
-        if name in partitions:
-          rbytes = int(rbytes) * sector_size
-          wbytes = int(wbytes) * sector_size
-          reads = int(reads)
-          writes = int(writes)
-          rtime = int(rtime)
-          wtime = int(wtime)
-          retdict[name] = (reads, writes, rbytes, wbytes, rtime, wtime)
-    return retdict
-
-  def _GetTcpStats(self):
-    """Tries to parse out TCP packet information from netstat output.
-
-    Returns:
-       A dictionary containing TCP information
-    """
-    # netstat return code is non-zero for -s on Linux, so don't raise on error.
-    netstat_output = self._Exec(['netstat', '-s'], return_output=True,
-                                raise_on_error=False)
-    netstat_output = netstat_output.strip().lower()
-    found_tcp = False
-    tcp_retransmit = None
-    tcp_received = None
-    tcp_sent = None
-    for line in netstat_output.split('\n'):
-      # Header for TCP section is "Tcp:" in Linux/Mac and
-      # "TCP Statistics for" in Windows.
-      if 'tcp:' in line or 'tcp statistics' in line:
-        found_tcp = True
-
-      # Linux == "segments retransmited" (sic), Mac == "retransmit timeouts"
-      # Windows == "segments retransmitted".
-      if (found_tcp and tcp_retransmit is None and
-          ('segments retransmited' in line or 'retransmit timeouts' in line or
-           'segments retransmitted' in line)):
-        tcp_retransmit = ''.join(c for c in line if c in string.digits)
-
-      # Linux+Windows == "segments received", Mac == "packets received".
-      if (found_tcp and tcp_received is None and
-          ('segments received' in line or 'packets received' in line)):
-        tcp_received = ''.join(c for c in line if c in string.digits)
-
-      # Linux == "segments send out" (sic), Mac+Windows == "packets sent".
-      if (found_tcp and tcp_sent is None and
-          ('segments send out' in line or 'packets sent' in line or
-           'segments sent' in line)):
-        tcp_sent = ''.join(c for c in line if c in string.digits)
-
-    result = {}
-    try:
-      result['tcp_retransmit'] = int(tcp_retransmit)
-      result['tcp_received'] = int(tcp_received)
-      result['tcp_sent'] = int(tcp_sent)
-    except (ValueError, TypeError):
-      result['tcp_retransmit'] = None
-      result['tcp_received'] = None
-      result['tcp_sent'] = None
-
-    return result
-
-  def _CollectSysInfo(self):
-    """Collects system information."""
-    sysinfo = {}
-
-    # Get the local IP address from socket lib.
-    sysinfo['ip_address'] = socket.gethostbyname(socket.gethostname())
-    # Record the temporary directory used since it can affect performance, e.g.
-    # when on a networked filesystem.
-    sysinfo['tempdir'] = tempfile.gettempdir()
-
-    # Produces an RFC 2822 compliant GMT timestamp.
-    sysinfo['gmt_timestamp'] = time.strftime('%a, %d %b %Y %H:%M:%S +0000',
-                                             time.gmtime())
-
-    # Execute a CNAME lookup on Google DNS to find what Google server
-    # it's routing to.
-    cmd = ['nslookup', '-type=CNAME', self.GOOGLE_API_HOST]
-    nslookup_cname_output = self._Exec(cmd, return_output=True)
-    m = re.search(r' = (?P<googserv>[^.]+)\.', nslookup_cname_output)
-    sysinfo['googserv_route'] = m.group('googserv') if m else None
-
-    # Look up IP addresses for Google Server.
-    (hostname, aliaslist, ipaddrlist) = socket.gethostbyname_ex(
-        self.GOOGLE_API_HOST)
-    sysinfo['googserv_ips'] = ipaddrlist
-
-    # Reverse lookup the hostnames for the Google Server IPs.
-    sysinfo['googserv_hostnames'] = []
-    for googserv_ip in ipaddrlist:
-      (hostname, aliaslist, ipaddrlist) = socket.gethostbyaddr(googserv_ip)
-      sysinfo['googserv_hostnames'].append(hostname)
-
-    # Query o-o to find out what the Google DNS thinks is the user's IP.
-    cmd = ['nslookup', '-type=TXT', 'o-o.myaddr.google.com.']
-    nslookup_txt_output = self._Exec(cmd, return_output=True)
-    m = re.search(r'text\s+=\s+"(?P<dnsip>[\.\d]+)"', nslookup_txt_output)
-    sysinfo['dns_o-o_ip'] = m.group('dnsip') if m else None
-
-    # Try and find the number of CPUs in the system if available.
-    try:
-      sysinfo['cpu_count'] = multiprocessing.cpu_count()
-    except NotImplementedError:
-      sysinfo['cpu_count'] = None
-
-    # For *nix platforms, obtain the CPU load.
-    try:
-      sysinfo['load_avg'] = list(os.getloadavg())
-    except (AttributeError, OSError):
-      sysinfo['load_avg'] = None
-
-    # Try and collect memory information from /proc/meminfo if possible.
-    mem_total = None
-    mem_free = None
-    mem_buffers = None
-    mem_cached = None
-
-    try:
-      with open('/proc/meminfo', 'r') as f:
-        for line in f:
-          if line.startswith('MemTotal'):
-            mem_total = (int(''.join(c for c in line if c in string.digits))
-                         * 1000)
-          elif line.startswith('MemFree'):
-            mem_free = (int(''.join(c for c in line if c in string.digits))
-                        * 1000)
-          elif line.startswith('Buffers'):
-            mem_buffers = (int(''.join(c for c in line if c in string.digits))
-                           * 1000)
-          elif line.startswith('Cached'):
-            mem_cached = (int(''.join(c for c in line if c in string.digits))
-                          * 1000)
-    except (IOError, ValueError):
-      pass
-
-    sysinfo['meminfo'] = {'mem_total': mem_total,
-                          'mem_free': mem_free,
-                          'mem_buffers': mem_buffers,
-                          'mem_cached': mem_cached}
-
-    # Get configuration attributes from config module.
-    sysinfo['gsutil_config'] = {}
-    for attr in dir(config):
-      attr_value = getattr(config, attr)
-      # Filter out multiline strings that are not useful.
-      if attr.isupper() and not (isinstance(attr_value, basestring) and
-                                 '\n' in attr_value):
-        sysinfo['gsutil_config'][attr] = attr_value
-
-    self.results['sysinfo'] = sysinfo
-
-  def _DisplayStats(self, trials):
-    """Prints out mean, standard deviation, median, and 90th percentile."""
-    n = len(trials)
-    mean = float(sum(trials)) / n
-    stdev = math.sqrt(sum((x - mean)**2 for x in trials) / n)
-
-    print str(n).rjust(6), '',
-    print ('%.1f' % (mean * 1000)).rjust(9), '',
-    print ('%.1f' % (stdev * 1000)).rjust(12), '',
-    print ('%.1f' % (Percentile(trials, 0.5) * 1000)).rjust(11), '',
-    print ('%.1f' % (Percentile(trials, 0.9) * 1000)).rjust(11), ''
-
-  def _DisplayResults(self):
-    """Displays results collected from diagnostic run."""
-    print
-    print '=' * 78
-    print 'DIAGNOSTIC RESULTS'.center(78)
-    print '=' * 78
-
-    if 'latency' in self.results:
-      print
-      print '-' * 78
-      print 'Latency'.center(78)
-      print '-' * 78
-      print ('Operation       Size  Trials  Mean (ms)  Std Dev (ms)  '
-             'Median (ms)  90th % (ms)')
-      print ('=========  =========  ======  =========  ============  '
-             '===========  ===========')
-      for key in sorted(self.results['latency']):
-        trials = sorted(self.results['latency'][key])
-        op, numbytes = key.split('_')
-        numbytes = int(numbytes)
-        if op == 'METADATA':
-          print 'Metadata'.rjust(9), '',
-          print MakeHumanReadable(numbytes).rjust(9), '',
-          self._DisplayStats(trials)
-        if op == 'DOWNLOAD':
-          print 'Download'.rjust(9), '',
-          print MakeHumanReadable(numbytes).rjust(9), '',
-          self._DisplayStats(trials)
-        if op == 'UPLOAD':
-          print 'Upload'.rjust(9), '',
-          print MakeHumanReadable(numbytes).rjust(9), '',
-          self._DisplayStats(trials)
-        if op == 'DELETE':
-          print 'Delete'.rjust(9), '',
-          print MakeHumanReadable(numbytes).rjust(9), '',
-          self._DisplayStats(trials)
-
-    if 'write_throughput' in self.results:
-      print
-      print '-' * 78
-      print 'Write Throughput'.center(78)
-      print '-' * 78
-      write_thru = self.results['write_throughput']
-      print 'Copied a %s file %d times for a total transfer size of %s.' % (
-          MakeHumanReadable(write_thru['file_size']),
-          write_thru['num_copies'],
-          MakeHumanReadable(write_thru['total_bytes_copied']))
-      print 'Write throughput: %s/s.' % (
-          MakeBitsHumanReadable(write_thru['bytes_per_second'] * 8))
-
-    if 'read_throughput' in self.results:
-      print
-      print '-' * 78
-      print 'Read Throughput'.center(78)
-      print '-' * 78
-      read_thru = self.results['read_throughput']
-      print 'Copied a %s file %d times for a total transfer size of %s.' % (
-          MakeHumanReadable(read_thru['file_size']),
-          read_thru['num_times'],
-          MakeHumanReadable(read_thru['total_bytes_copied']))
-      print 'Read throughput: %s/s.' % (
-          MakeBitsHumanReadable(read_thru['bytes_per_second'] * 8))
-
-    if 'sysinfo' in self.results:
-      print
-      print '-' * 78
-      print 'System Information'.center(78)
-      print '-' * 78
-      info = self.results['sysinfo']
-      print 'IP Address: \n  %s' % info['ip_address']
-      print 'Temporary Directory: \n  %s' % info['tempdir']
-      print 'Bucket URI: \n  %s' % self.results['bucket_uri']
-
-      if 'gmt_timestamp' in info:
-        ts_string = info['gmt_timestamp']
-        timetuple = None
-        try:
-          # Convert RFC 2822 string to Linux timestamp.
-          timetuple = time.strptime(ts_string, '%a, %d %b %Y %H:%M:%S +0000')
-        except ValueError:
-          pass
-
-        if timetuple:
-          # Converts the GMT time tuple to local Linux timestamp.
-          localtime = calendar.timegm(timetuple)
-          localdt = datetime.datetime.fromtimestamp(localtime)
-          print 'Measurement time: \n %s' % localdt.strftime(
-              '%Y-%m-%d %I-%M-%S %p %Z')
-
-      print 'Google Server: \n  %s' % info['googserv_route']
-      print ('Google Server IP Addresses: \n  %s' %
-             ('\n  '.join(info['googserv_ips'])))
-      print ('Google Server Hostnames: \n  %s' %
-             ('\n  '.join(info['googserv_hostnames'])))
-      print 'Google DNS thinks your IP is: \n  %s' % info['dns_o-o_ip']
-      print 'CPU Count: \n  %s' % info['cpu_count']
-      print 'CPU Load Average: \n  %s' % info['load_avg']
-      try:
-        print ('Total Memory: \n  %s' %
-               MakeHumanReadable(info['meminfo']['mem_total']))
-        # Free memory is really MemFree + Buffers + Cached.
-        print 'Free Memory: \n  %s' % MakeHumanReadable(
-            info['meminfo']['mem_free'] +
-            info['meminfo']['mem_buffers'] +
-            info['meminfo']['mem_cached'])
-      except TypeError:
-        pass
-
-      netstat_after = info['netstat_end']
-      netstat_before = info['netstat_start']
-      for tcp_type in ('sent', 'received', 'retransmit'):
-        try:
-          delta = (netstat_after['tcp_%s' % tcp_type] -
-                   netstat_before['tcp_%s' % tcp_type])
-          print 'TCP segments %s during test:\n  %d' % (tcp_type, delta)
-        except TypeError:
-          pass
-
-      if 'disk_counters_end' in info and 'disk_counters_start' in info:
-        print 'Disk Counter Deltas:\n',
-        disk_after = info['disk_counters_end']
-        disk_before = info['disk_counters_start']
-        print '', 'disk'.rjust(6),
-        for colname in ['reads', 'writes', 'rbytes', 'wbytes', 'rtime',
-                        'wtime']:
-          print colname.rjust(8),
-        print
-        for diskname in sorted(disk_after):
-          before = disk_before[diskname]
-          after = disk_after[diskname]
-          (reads1, writes1, rbytes1, wbytes1, rtime1, wtime1) = before
-          (reads2, writes2, rbytes2, wbytes2, rtime2, wtime2) = after
-          print '', diskname.rjust(6),
-          deltas = [reads2-reads1, writes2-writes1, rbytes2-rbytes1,
-                    wbytes2-wbytes1, rtime2-rtime1, wtime2-wtime1]
-          for delta in deltas:
-            print str(delta).rjust(8),
-          print
-
-    if self.output_file:
-      with open(self.output_file, 'w') as f:
-        json.dump(self.results, f, indent=2)
-      print
-      print "Output file written to '%s'." % self.output_file
-
-    print
-
-  def _ParsePositiveInteger(self, val, msg):
-    """Tries to convert val argument to a positive integer.
-
-    Args:
-      val: The value (as a string) to convert to a positive integer.
-      msg: The error message to place in the CommandException on an error.
-
-    Returns:
-      A valid positive integer.
-
-    Raises:
-      CommandException: If the supplied value is not a valid positive integer.
-    """
-    try:
-      val = int(val)
-      if val < 1:
-        raise CommandException(msg)
-      return val
-    except ValueError:
-      raise CommandException(msg)
-
-  def _ParseArgs(self):
-    """Parses arguments for perfdiag command."""
-    # From -n.
-    self.num_iterations = 5
-    # From -c.
-    self.concurrency = 1
-    # From -s.
-    self.thru_filesize = 1048576
-    # From -t.
-    self.diag_tests = self.ALL_DIAG_TESTS
-    # From -o.
-    self.output_file = None
-    # From -i.
-    self.input_file = None
-
-    if self.sub_opts:
-      for o, a in self.sub_opts:
-        if o == '-n':
-          self.num_iterations = self._ParsePositiveInteger(
-              a, 'The -n parameter must be a positive integer.')
-        if o == '-c':
-          self.concurrency = self._ParsePositiveInteger(
-              a, 'The -c parameter must be a positive integer.')
-        if o == '-s':
-          self.thru_filesize = self._ParsePositiveInteger(
-              a, 'The -s parameter must be a positive integer.')
-        if o == '-t':
-          self.diag_tests = []
-          for test_name in a.strip().split(','):
-            if test_name.lower() not in self.ALL_DIAG_TESTS:
-              raise CommandException("List of test names (-t) contains invalid "
-                                     "test name '%s'." % test_name)
-            self.diag_tests.append(test_name)
-        if o == '-o':
-          self.output_file = os.path.abspath(a)
-        if o == '-i':
-          self.input_file = os.path.abspath(a)
-          if not os.path.isfile(self.input_file):
-            raise CommandException("Invalid input file (-i): '%s'." % a)
-          try:
-            with open(self.input_file, 'r') as f:
-              self.results = json.load(f)
-              print "Read input file: '%s'." % self.input_file
-          except ValueError:
-            raise CommandException("Could not decode input file (-i): '%s'." %
-                                   a)
-          return
-
-    if not self.args:
-      raise CommandException('Wrong number of arguments for "perfdiag" '
-                             'command.')
-    self.bucket_uri = self.suri_builder.StorageUri(self.args[0])
-    if not self.bucket_uri.names_bucket():
-      raise CommandException('The perfdiag command requires a URI that '
-                             'specifies a bucket.\n"%s" is not '
-                             'valid.' % self.bucket_uri)
-    self.bucket = self.bucket_uri.get_bucket()
-
-  # Command entry point.
-  def RunCommand(self):
-    """Called by gsutil when the command is being invoked."""
-    self._ParseArgs()
-
-    if self.input_file:
-      self._DisplayResults()
-      return 0
-
-    print 'Number of iterations to run: %d' % self.num_iterations
-    print 'Base bucket URI: %s' % self.bucket_uri
-    print 'Concurrency level: %d' % self.concurrency
-    print 'Throughput file size: %s' % MakeHumanReadable(self.thru_filesize)
-    print 'Diagnostics to run: %s' % (', '.join(self.diag_tests))
-
-    try:
-      self._SetUp()
-
-      # Collect generic system info.
-      self._CollectSysInfo()
-      # Collect netstat info and disk counters before tests (and again later).
-      self.results['sysinfo']['netstat_start'] = self._GetTcpStats()
-      if IS_LINUX:
-        self.results['sysinfo']['disk_counters_start'] = self._GetDiskCounters()
-      # Record bucket URI.
-      self.results['bucket_uri'] = str(self.bucket_uri)
-
-      if 'lat' in self.diag_tests:
-        self._RunLatencyTests()
-      if 'rthru' in self.diag_tests:
-        self._RunReadThruTests()
-      if 'wthru' in self.diag_tests:
-        self._RunWriteThruTests()
-
-      # Collect netstat info and disk counters after tests.
-      self.results['sysinfo']['netstat_end'] = self._GetTcpStats()
-      if IS_LINUX:
-        self.results['sysinfo']['disk_counters_end'] = self._GetDiskCounters()
-
-      self._DisplayResults()
-    finally:
-      self._TearDown()
-
-    return 0