gcloud 0.0.7 → 0.1.0

data/packages/gsutil/boto/boto/pyami/bootstrap.py

@@ -1,134 +0,0 @@
-# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish, dis-
-# tribute, sublicense, and/or sell copies of the Software, and to permit
-# persons to whom the Software is furnished to do so, subject to the fol-
-# lowing conditions:
-#
-# The above copyright notice and this permission notice shall be included
-# in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
-# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
-# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
-# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-#
-import os
-import boto
-from boto.utils import get_instance_metadata, get_instance_userdata
-from boto.pyami.config import Config, BotoConfigPath
-from boto.pyami.scriptbase import ScriptBase
-import time
-
-class Bootstrap(ScriptBase):
-    """
-    The Bootstrap class is instantiated and run as part of the PyAMI
-    instance initialization process.  The methods in this class will
-    be run from the rc.local script of the instance and will be run
-    as the root user.
-
-    The main purpose of this class is to make sure the boto distribution
-    on the instance is the one required.
-    """
-
-    def __init__(self):
-        self.working_dir = '/mnt/pyami'
-        self.write_metadata()
-        ScriptBase.__init__(self)
-
-    def write_metadata(self):
-        fp = open(os.path.expanduser(BotoConfigPath), 'w')
-        fp.write('[Instance]\n')
-        inst_data = get_instance_metadata()
-        for key in inst_data:
-            fp.write('%s = %s\n' % (key, inst_data[key]))
-        user_data = get_instance_userdata()
-        fp.write('\n%s\n' % user_data)
-        fp.write('[Pyami]\n')
-        fp.write('working_dir = %s\n' % self.working_dir)
-        fp.close()
-        # This file has the AWS credentials, should we lock it down?
-        # os.chmod(BotoConfigPath, stat.S_IREAD | stat.S_IWRITE)
-        # now that we have written the file, read it into a pyami Config object
-        boto.config = Config()
-        boto.init_logging()
-
-    def create_working_dir(self):
-        boto.log.info('Working directory: %s' % self.working_dir)
-        if not os.path.exists(self.working_dir):
-            os.mkdir(self.working_dir)
-
-    def load_boto(self):
-        update = boto.config.get('Boto', 'boto_update', 'svn:HEAD')
-        if update.startswith('svn'):
-            if update.find(':') >= 0:
-                method, version = update.split(':')
-                version = '-r%s' % version
-            else:
-                version = '-rHEAD'
-            location = boto.config.get('Boto', 'boto_location', '/usr/local/boto')
-            self.run('svn update %s %s' % (version, location))
-        elif update.startswith('git'):
-            location = boto.config.get('Boto', 'boto_location', '/usr/share/python-support/python-boto/boto')
-            num_remaining_attempts = 10
-            while num_remaining_attempts > 0:
-                num_remaining_attempts -= 1
-                try:
-                    self.run('git pull', cwd=location)
-                    num_remaining_attempts = 0
-                except Exception, e:
-                    boto.log.info('git pull attempt failed with the following exception. Trying again in a bit. %s', e)
-                    time.sleep(2)
-            if update.find(':') >= 0:
-                method, version = update.split(':')
-            else:
-                version = 'master'
-            self.run('git checkout %s' % version, cwd=location)
-        else:
-            # first remove the symlink needed when running from subversion
-            self.run('rm /usr/local/lib/python2.5/site-packages/boto')
-            self.run('easy_install %s' % update)
-
-    def fetch_s3_file(self, s3_file):
-        try:
-            from boto.utils import fetch_file
-            f = fetch_file(s3_file)
-            path = os.path.join(self.working_dir, s3_file.split("/")[-1])
-            open(path, "w").write(f.read())
-        except:
-            boto.log.exception('Problem Retrieving file: %s' % s3_file)
-            path = None
-        return path
-
-    def load_packages(self):
-        package_str = boto.config.get('Pyami', 'packages')
-        if package_str:
-            packages = package_str.split(',')
-            for package in packages:
-                package = package.strip()
-                if package.startswith('s3:'):
-                    package = self.fetch_s3_file(package)
-                if package:
-                    # if the "package" is really a .py file, it doesn't have to
-                    # be installed, just being in the working dir is enough
-                    if not package.endswith('.py'):
-                        self.run('easy_install -Z %s' % package, exit_on_error=False)
-
-    def main(self):
-        self.create_working_dir()
-        self.load_boto()
-        self.load_packages()
-        self.notify('Bootstrap Completed for %s' % boto.config.get_instance('instance-id'))
-
-if __name__ == "__main__":
-    # because bootstrap starts before any logging configuration can be loaded from
-    # the boto config files, we will manually enable logging to /var/log/boto.log
-    boto.set_file_logger('bootstrap', '/var/log/boto.log')
-    bs = Bootstrap()
-    bs.main()

data/packages/gsutil/boto/boto/pyami/config.py

@@ -1,229 +0,0 @@
-# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/
-# Copyright (c) 2011 Chris Moyer http://coredumped.org/
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish, dis-
-# tribute, sublicense, and/or sell copies of the Software, and to permit
-# persons to whom the Software is furnished to do so, subject to the fol-
-# lowing conditions:
-#
-# The above copyright notice and this permission notice shall be included
-# in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
-# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
-# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-#
-import StringIO, os, re
-import warnings
-import ConfigParser
-import boto
-
-# If running in Google App Engine there is no "user" and
-# os.path.expanduser() will fail. Attempt to detect this case and use a
-# no-op expanduser function in this case.
-try:
-  os.path.expanduser('~')
-  expanduser = os.path.expanduser
-except (AttributeError, ImportError):
-  # This is probably running on App Engine.
-  expanduser = (lambda x: x)
-
-# By default we use two locations for the boto configurations,
-# /etc/boto.cfg and ~/.boto (which works on Windows and Unix).
-BotoConfigPath = '/etc/boto.cfg'
-BotoConfigLocations = [BotoConfigPath]
-UserConfigPath = os.path.join(expanduser('~'), '.boto')
-BotoConfigLocations.append(UserConfigPath)
-
-# If there's a BOTO_CONFIG variable set, we load ONLY 
-# that variable
-if 'BOTO_CONFIG' in os.environ:
-    BotoConfigLocations = [expanduser(os.environ['BOTO_CONFIG'])]
-
-# If there's a BOTO_PATH variable set, we use anything there
-# as the current configuration locations, split with colons
-elif 'BOTO_PATH' in os.environ:
-    BotoConfigLocations = []
-    for path in os.environ['BOTO_PATH'].split(":"):
-        BotoConfigLocations.append(expanduser(path))
-
-
-class Config(ConfigParser.SafeConfigParser):
-
-    def __init__(self, path=None, fp=None, do_load=True):
-        ConfigParser.SafeConfigParser.__init__(self, {'working_dir' : '/mnt/pyami',
-                                                      'debug' : '0'})
-        if do_load:
-            if path:
-                self.load_from_path(path)
-            elif fp:
-                self.readfp(fp)
-            else:
-                self.read(BotoConfigLocations)
-            if "AWS_CREDENTIAL_FILE" in os.environ:
-                full_path = expanduser(os.environ['AWS_CREDENTIAL_FILE'])
-                try:
-                    self.load_credential_file(full_path)
-                except IOError:
-                    warnings.warn('Unable to load AWS_CREDENTIAL_FILE (%s)' % full_path)
-
-    def load_credential_file(self, path):
-        """Load a credential file as is setup like the Java utilities"""
-        c_data = StringIO.StringIO()
-        c_data.write("[Credentials]\n")
-        for line in open(path, "r").readlines():
-            c_data.write(line.replace("AWSAccessKeyId", "aws_access_key_id").replace("AWSSecretKey", "aws_secret_access_key"))
-        c_data.seek(0)
-        self.readfp(c_data)
-
-    def load_from_path(self, path):
-        file = open(path)
-        for line in file.readlines():
-            match = re.match("^#import[\s\t]*([^\s^\t]*)[\s\t]*$", line)
-            if match:
-                extended_file = match.group(1)
-                (dir, file) = os.path.split(path)
-                self.load_from_path(os.path.join(dir, extended_file))
-        self.read(path)
-
-    def save_option(self, path, section, option, value):
-        """
-        Write the specified Section.Option to the config file specified by path.
-        Replace any previous value.  If the path doesn't exist, create it.
-        Also add the option the the in-memory config.
-        """
-        config = ConfigParser.SafeConfigParser()
-        config.read(path)
-        if not config.has_section(section):
-            config.add_section(section)
-        config.set(section, option, value)
-        fp = open(path, 'w')
-        config.write(fp)
-        fp.close()
-        if not self.has_section(section):
-            self.add_section(section)
-        self.set(section, option, value)
-
-    def save_user_option(self, section, option, value):
-        self.save_option(UserConfigPath, section, option, value)
-
-    def save_system_option(self, section, option, value):
-        self.save_option(BotoConfigPath, section, option, value)
-
-    def get_instance(self, name, default=None):
-        try:
-            val = self.get('Instance', name)
-        except:
-            val = default
-        return val
-
-    def get_user(self, name, default=None):
-        try:
-            val = self.get('User', name)
-        except:
-            val = default
-        return val
-
-    def getint_user(self, name, default=0):
-        try:
-            val = self.getint('User', name)
-        except:
-            val = default
-        return val
-
-    def get_value(self, section, name, default=None):
-        return self.get(section, name, default)
-
-    def get(self, section, name, default=None):
-        try:
-            val = ConfigParser.SafeConfigParser.get(self, section, name)
-        except:
-            val = default
-        return val
-    
-    def getint(self, section, name, default=0):
-        try:
-            val = ConfigParser.SafeConfigParser.getint(self, section, name)
-        except:
-            val = int(default)
-        return val
-    
-    def getfloat(self, section, name, default=0.0):
-        try:
-            val = ConfigParser.SafeConfigParser.getfloat(self, section, name)
-        except:
-            val = float(default)
-        return val
-
-    def getbool(self, section, name, default=False):
-        if self.has_option(section, name):
-            val = self.get(section, name)
-            if val.lower() == 'true':
-                val = True
-            else:
-                val = False
-        else:
-            val = default
-        return val
-    
-    def setbool(self, section, name, value):
-        if value:
-            self.set(section, name, 'true')
-        else:
-            self.set(section, name, 'false')
-    
-    def dump(self):
-        s = StringIO.StringIO()
-        self.write(s)
-        print s.getvalue()
-
-    def dump_safe(self, fp=None):
-        if not fp:
-            fp = StringIO.StringIO()
-        for section in self.sections():
-            fp.write('[%s]\n' % section)
-            for option in self.options(section):
-                if option == 'aws_secret_access_key':
-                    fp.write('%s = xxxxxxxxxxxxxxxxxx\n' % option)
-                else:
-                    fp.write('%s = %s\n' % (option, self.get(section, option)))
-    
-    def dump_to_sdb(self, domain_name, item_name):
-        from boto.compat import json
-        sdb = boto.connect_sdb()
-        domain = sdb.lookup(domain_name)
-        if not domain:
-            domain = sdb.create_domain(domain_name)
-        item = domain.new_item(item_name)
-        item.active = False
-        for section in self.sections():
-            d = {}
-            for option in self.options(section):
-                d[option] = self.get(section, option)
-            item[section] = json.dumps(d)
-        item.save()
-
-    def load_from_sdb(self, domain_name, item_name):
-        from boto.compat import json
-        sdb = boto.connect_sdb()
-        domain = sdb.lookup(domain_name)
-        item = domain.get_item(item_name)
-        for section in item.keys():
-            if not self.has_section(section):
-                self.add_section(section)
-            d = json.loads(item[section])
-            for attr_name in d.keys():
-                attr_value = d[attr_name]
-                if attr_value == None:
-                    attr_value = 'None'
-                if isinstance(attr_value, bool):
-                    self.setbool(section, attr_name, attr_value)
-                else:
-                    self.set(section, attr_name, attr_value)

data/packages/gsutil/boto/boto/pyami/copybot.cfg

@@ -1,60 +0,0 @@
-#
-# Your AWS Credentials
-#
-[Credentials]
-aws_access_key_id = <AWS Access Key Here>
-aws_secret_access_key = <AWS Secret Key Here>
-
-#
-# If you want to use a separate set of credentials when writing
-# to the destination bucket, put them here
-#dest_aws_access_key_id = <AWS Access Key Here>
-#dest_aws_secret_access_key = <AWS Secret Key Here>
-
-#
-# Fill out this section if you want emails from CopyBot
-# when it starts and stops
-#
-[Notification]
-#smtp_host = <your smtp host>
-#smtp_user = <your smtp username, if necessary>
-#smtp_pass = <your smtp password, if necessary>
-#smtp_from = <email address for From: field>
-#smtp_to = <email address for To: field>
-
-#
-# If you leave this section as is, it will automatically
-# update boto from subversion upon start up.
-# If you don't want that to happen, comment this out
-#
-[Boto]
-boto_location = /usr/local/boto
-boto_update = svn:HEAD
-
-#
-# This tells the Pyami code in boto what scripts
-# to run during startup
-#
-[Pyami]
-scripts = boto.pyami.copybot.CopyBot
-
-#
-# Source bucket and Destination Bucket, obviously.
-# If the Destination bucket does not exist, it will
-# attempt to create it.
-# If exit_on_completion is false, the instance
-# will keep running after the copy operation is
-# complete which might be handy for debugging.
-# If copy_acls is false, the ACL's will not be
-# copied with the objects to the new bucket.
-# If replace_dst is false, copybot will not
-# will only store the source file in the dest if
-# that file does not already exist.  If it's true
-# it will replace it even if it does exist.
-#
-[CopyBot]
-src_bucket = <your source bucket name>
-dst_bucket = <your destination bucket name>
-exit_on_completion = true
-copy_acls = true
-replace_dst = true

data/packages/gsutil/boto/boto/pyami/copybot.py

@@ -1,97 +0,0 @@
-# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/
-#
-# Permission is hereby granted, free of charge, to any person obtaining a
-# copy of this software and associated documentation files (the
-# "Software"), to deal in the Software without restriction, including
-# without limitation the rights to use, copy, modify, merge, publish, dis-
-# tribute, sublicense, and/or sell copies of the Software, and to permit
-# persons to whom the Software is furnished to do so, subject to the fol-
-# lowing conditions:
-#
-# The above copyright notice and this permission notice shall be included
-# in all copies or substantial portions of the Software.
-#
-# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
-# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
-# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
-# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
-# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-# IN THE SOFTWARE.
-#
-import boto
-from boto.pyami.scriptbase import ScriptBase
-import os, StringIO
-
-class CopyBot(ScriptBase):
-
-    def __init__(self):
-        ScriptBase.__init__(self)
-        self.wdir = boto.config.get('Pyami', 'working_dir')
-        self.log_file = '%s.log' % self.instance_id
-        self.log_path = os.path.join(self.wdir, self.log_file)
-        boto.set_file_logger(self.name, self.log_path)
-        self.src_name = boto.config.get(self.name, 'src_bucket')
-        self.dst_name = boto.config.get(self.name, 'dst_bucket')
-        self.replace = boto.config.getbool(self.name, 'replace_dst', True)
-        s3 = boto.connect_s3()
-        self.src = s3.lookup(self.src_name)
-        if not self.src:
-            boto.log.error('Source bucket does not exist: %s' % self.src_name)
-        dest_access_key = boto.config.get(self.name, 'dest_aws_access_key_id', None)
-        if dest_access_key:
-            dest_secret_key = boto.config.get(self.name, 'dest_aws_secret_access_key', None)
-            s3 = boto.connect(dest_access_key, dest_secret_key)
-        self.dst = s3.lookup(self.dst_name)
-        if not self.dst:
-            self.dst = s3.create_bucket(self.dst_name)
-
-    def copy_bucket_acl(self):
-        if boto.config.get(self.name, 'copy_acls', True):
-            acl = self.src.get_xml_acl()
-            self.dst.set_xml_acl(acl)
-
-    def copy_key_acl(self, src, dst):
-        if boto.config.get(self.name, 'copy_acls', True):
-            acl = src.get_xml_acl()
-            dst.set_xml_acl(acl)
-
-    def copy_keys(self):
-        boto.log.info('src=%s' % self.src.name)
-        boto.log.info('dst=%s' % self.dst.name)
-        try:
-            for key in self.src:
-                if not self.replace:
-                    exists = self.dst.lookup(key.name)
-                    if exists:
-                        boto.log.info('key=%s already exists in %s, skipping' % (key.name, self.dst.name))
-                        continue
-                boto.log.info('copying %d bytes from key=%s' % (key.size, key.name))
-                prefix, base = os.path.split(key.name)
-                path = os.path.join(self.wdir, base)
-                key.get_contents_to_filename(path)
-                new_key = self.dst.new_key(key.name)
-                new_key.set_contents_from_filename(path)
-                self.copy_key_acl(key, new_key)
-                os.unlink(path)
-        except:
-            boto.log.exception('Error copying key: %s' % key.name)
-
-    def copy_log(self):
-        key = self.dst.new_key(self.log_file)
-        key.set_contents_from_filename(self.log_path)
-        
-    def main(self):
-        fp = StringIO.StringIO()
-        boto.config.dump_safe(fp)
-        self.notify('%s (%s) Starting' % (self.name, self.instance_id), fp.getvalue())
-        if self.src and self.dst:
-            self.copy_keys()
-        if self.dst:
-            self.copy_log()
-        self.notify('%s (%s) Stopping' % (self.name, self.instance_id),
-                    'Copy Operation Complete')
-        if boto.config.getbool(self.name, 'exit_on_completion', True):
-            ec2 = boto.connect_ec2()
-            ec2.terminate_instances([self.instance_id])
-