gcloud 0.0.4 → 0.0.6

data/packages/gsutil/boto/boto/pyami/bootstrap.py

@@ -0,0 +1,134 @@
+# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+import os
+import boto
+from boto.utils import get_instance_metadata, get_instance_userdata
+from boto.pyami.config import Config, BotoConfigPath
+from boto.pyami.scriptbase import ScriptBase
+import time
+
+class Bootstrap(ScriptBase):
+    """
+    The Bootstrap class is instantiated and run as part of the PyAMI
+    instance initialization process.  The methods in this class will
+    be run from the rc.local script of the instance and will be run
+    as the root user.
+
+    The main purpose of this class is to make sure the boto distribution
+    on the instance is the one required.
+    """
+
+    def __init__(self):
+        self.working_dir = '/mnt/pyami'
+        self.write_metadata()
+        ScriptBase.__init__(self)
+
+    def write_metadata(self):
+        fp = open(os.path.expanduser(BotoConfigPath), 'w')
+        fp.write('[Instance]\n')
+        inst_data = get_instance_metadata()
+        for key in inst_data:
+            fp.write('%s = %s\n' % (key, inst_data[key]))
+        user_data = get_instance_userdata()
+        fp.write('\n%s\n' % user_data)
+        fp.write('[Pyami]\n')
+        fp.write('working_dir = %s\n' % self.working_dir)
+        fp.close()
+        # This file has the AWS credentials, should we lock it down?
+        # os.chmod(BotoConfigPath, stat.S_IREAD | stat.S_IWRITE)
+        # now that we have written the file, read it into a pyami Config object
+        boto.config = Config()
+        boto.init_logging()
+
+    def create_working_dir(self):
+        boto.log.info('Working directory: %s' % self.working_dir)
+        if not os.path.exists(self.working_dir):
+            os.mkdir(self.working_dir)
+
+    def load_boto(self):
+        update = boto.config.get('Boto', 'boto_update', 'svn:HEAD')
+        if update.startswith('svn'):
+            if update.find(':') >= 0:
+                method, version = update.split(':')
+                version = '-r%s' % version
+            else:
+                version = '-rHEAD'
+            location = boto.config.get('Boto', 'boto_location', '/usr/local/boto')
+            self.run('svn update %s %s' % (version, location))
+        elif update.startswith('git'):
+            location = boto.config.get('Boto', 'boto_location', '/usr/share/python-support/python-boto/boto')
+            num_remaining_attempts = 10
+            while num_remaining_attempts > 0:
+                num_remaining_attempts -= 1
+                try:
+                    self.run('git pull', cwd=location)
+                    num_remaining_attempts = 0
+                except Exception, e:
+                    boto.log.info('git pull attempt failed with the following exception. Trying again in a bit. %s', e)
+                    time.sleep(2)
+            if update.find(':') >= 0:
+                method, version = update.split(':')
+            else:
+                version = 'master'
+            self.run('git checkout %s' % version, cwd=location)
+        else:
+            # first remove the symlink needed when running from subversion
+            self.run('rm /usr/local/lib/python2.5/site-packages/boto')
+            self.run('easy_install %s' % update)
+
+    def fetch_s3_file(self, s3_file):
+        try:
+            from boto.utils import fetch_file
+            f = fetch_file(s3_file)
+            path = os.path.join(self.working_dir, s3_file.split("/")[-1])
+            open(path, "w").write(f.read())
+        except:
+            boto.log.exception('Problem Retrieving file: %s' % s3_file)
+            path = None
+        return path
+
+    def load_packages(self):
+        package_str = boto.config.get('Pyami', 'packages')
+        if package_str:
+            packages = package_str.split(',')
+            for package in packages:
+                package = package.strip()
+                if package.startswith('s3:'):
+                    package = self.fetch_s3_file(package)
+                if package:
+                    # if the "package" is really a .py file, it doesn't have to
+                    # be installed, just being in the working dir is enough
+                    if not package.endswith('.py'):
+                        self.run('easy_install -Z %s' % package, exit_on_error=False)
+
+    def main(self):
+        self.create_working_dir()
+        self.load_boto()
+        self.load_packages()
+        self.notify('Bootstrap Completed for %s' % boto.config.get_instance('instance-id'))
+
+if __name__ == "__main__":
+    # because bootstrap starts before any logging configuration can be loaded from
+    # the boto config files, we will manually enable logging to /var/log/boto.log
+    boto.set_file_logger('bootstrap', '/var/log/boto.log')
+    bs = Bootstrap()
+    bs.main()

data/packages/gsutil/boto/boto/pyami/config.py

@@ -0,0 +1,229 @@
+# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/
+# Copyright (c) 2011 Chris Moyer http://coredumped.org/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+import StringIO, os, re
+import warnings
+import ConfigParser
+import boto
+
+# If running in Google App Engine there is no "user" and
+# os.path.expanduser() will fail. Attempt to detect this case and use a
+# no-op expanduser function in this case.
+try:
+  os.path.expanduser('~')
+  expanduser = os.path.expanduser
+except (AttributeError, ImportError):
+  # This is probably running on App Engine.
+  expanduser = (lambda x: x)
+
+# By default we use two locations for the boto configurations,
+# /etc/boto.cfg and ~/.boto (which works on Windows and Unix).
+BotoConfigPath = '/etc/boto.cfg'
+BotoConfigLocations = [BotoConfigPath]
+UserConfigPath = os.path.join(expanduser('~'), '.boto')
+BotoConfigLocations.append(UserConfigPath)
+
+# If there's a BOTO_CONFIG variable set, we load ONLY 
+# that variable
+if 'BOTO_CONFIG' in os.environ:
+    BotoConfigLocations = [expanduser(os.environ['BOTO_CONFIG'])]
+
+# If there's a BOTO_PATH variable set, we use anything there
+# as the current configuration locations, split with colons
+elif 'BOTO_PATH' in os.environ:
+    BotoConfigLocations = []
+    for path in os.environ['BOTO_PATH'].split(":"):
+        BotoConfigLocations.append(expanduser(path))
+
+
+class Config(ConfigParser.SafeConfigParser):
+
+    def __init__(self, path=None, fp=None, do_load=True):
+        ConfigParser.SafeConfigParser.__init__(self, {'working_dir' : '/mnt/pyami',
+                                                      'debug' : '0'})
+        if do_load:
+            if path:
+                self.load_from_path(path)
+            elif fp:
+                self.readfp(fp)
+            else:
+                self.read(BotoConfigLocations)
+            if "AWS_CREDENTIAL_FILE" in os.environ:
+                full_path = expanduser(os.environ['AWS_CREDENTIAL_FILE'])
+                try:
+                    self.load_credential_file(full_path)
+                except IOError:
+                    warnings.warn('Unable to load AWS_CREDENTIAL_FILE (%s)' % full_path)
+
+    def load_credential_file(self, path):
+        """Load a credential file as is setup like the Java utilities"""
+        c_data = StringIO.StringIO()
+        c_data.write("[Credentials]\n")
+        for line in open(path, "r").readlines():
+            c_data.write(line.replace("AWSAccessKeyId", "aws_access_key_id").replace("AWSSecretKey", "aws_secret_access_key"))
+        c_data.seek(0)
+        self.readfp(c_data)
+
+    def load_from_path(self, path):
+        file = open(path)
+        for line in file.readlines():
+            match = re.match("^#import[\s\t]*([^\s^\t]*)[\s\t]*$", line)
+            if match:
+                extended_file = match.group(1)
+                (dir, file) = os.path.split(path)
+                self.load_from_path(os.path.join(dir, extended_file))
+        self.read(path)
+
+    def save_option(self, path, section, option, value):
+        """
+        Write the specified Section.Option to the config file specified by path.
+        Replace any previous value.  If the path doesn't exist, create it.
+        Also add the option the the in-memory config.
+        """
+        config = ConfigParser.SafeConfigParser()
+        config.read(path)
+        if not config.has_section(section):
+            config.add_section(section)
+        config.set(section, option, value)
+        fp = open(path, 'w')
+        config.write(fp)
+        fp.close()
+        if not self.has_section(section):
+            self.add_section(section)
+        self.set(section, option, value)
+
+    def save_user_option(self, section, option, value):
+        self.save_option(UserConfigPath, section, option, value)
+
+    def save_system_option(self, section, option, value):
+        self.save_option(BotoConfigPath, section, option, value)
+
+    def get_instance(self, name, default=None):
+        try:
+            val = self.get('Instance', name)
+        except:
+            val = default
+        return val
+
+    def get_user(self, name, default=None):
+        try:
+            val = self.get('User', name)
+        except:
+            val = default
+        return val
+
+    def getint_user(self, name, default=0):
+        try:
+            val = self.getint('User', name)
+        except:
+            val = default
+        return val
+
+    def get_value(self, section, name, default=None):
+        return self.get(section, name, default)
+
+    def get(self, section, name, default=None):
+        try:
+            val = ConfigParser.SafeConfigParser.get(self, section, name)
+        except:
+            val = default
+        return val
+    
+    def getint(self, section, name, default=0):
+        try:
+            val = ConfigParser.SafeConfigParser.getint(self, section, name)
+        except:
+            val = int(default)
+        return val
+    
+    def getfloat(self, section, name, default=0.0):
+        try:
+            val = ConfigParser.SafeConfigParser.getfloat(self, section, name)
+        except:
+            val = float(default)
+        return val
+
+    def getbool(self, section, name, default=False):
+        if self.has_option(section, name):
+            val = self.get(section, name)
+            if val.lower() == 'true':
+                val = True
+            else:
+                val = False
+        else:
+            val = default
+        return val
+    
+    def setbool(self, section, name, value):
+        if value:
+            self.set(section, name, 'true')
+        else:
+            self.set(section, name, 'false')
+    
+    def dump(self):
+        s = StringIO.StringIO()
+        self.write(s)
+        print s.getvalue()
+
+    def dump_safe(self, fp=None):
+        if not fp:
+            fp = StringIO.StringIO()
+        for section in self.sections():
+            fp.write('[%s]\n' % section)
+            for option in self.options(section):
+                if option == 'aws_secret_access_key':
+                    fp.write('%s = xxxxxxxxxxxxxxxxxx\n' % option)
+                else:
+                    fp.write('%s = %s\n' % (option, self.get(section, option)))
+    
+    def dump_to_sdb(self, domain_name, item_name):
+        from boto.compat import json
+        sdb = boto.connect_sdb()
+        domain = sdb.lookup(domain_name)
+        if not domain:
+            domain = sdb.create_domain(domain_name)
+        item = domain.new_item(item_name)
+        item.active = False
+        for section in self.sections():
+            d = {}
+            for option in self.options(section):
+                d[option] = self.get(section, option)
+            item[section] = json.dumps(d)
+        item.save()
+
+    def load_from_sdb(self, domain_name, item_name):
+        from boto.compat import json
+        sdb = boto.connect_sdb()
+        domain = sdb.lookup(domain_name)
+        item = domain.get_item(item_name)
+        for section in item.keys():
+            if not self.has_section(section):
+                self.add_section(section)
+            d = json.loads(item[section])
+            for attr_name in d.keys():
+                attr_value = d[attr_name]
+                if attr_value == None:
+                    attr_value = 'None'
+                if isinstance(attr_value, bool):
+                    self.setbool(section, attr_name, attr_value)
+                else:
+                    self.set(section, attr_name, attr_value)

data/packages/gsutil/boto/boto/pyami/copybot.cfg

@@ -0,0 +1,60 @@
+#
+# Your AWS Credentials
+#
+[Credentials]
+aws_access_key_id = <AWS Access Key Here>
+aws_secret_access_key = <AWS Secret Key Here>
+
+#
+# If you want to use a separate set of credentials when writing
+# to the destination bucket, put them here
+#dest_aws_access_key_id = <AWS Access Key Here>
+#dest_aws_secret_access_key = <AWS Secret Key Here>
+
+#
+# Fill out this section if you want emails from CopyBot
+# when it starts and stops
+#
+[Notification]
+#smtp_host = <your smtp host>
+#smtp_user = <your smtp username, if necessary>
+#smtp_pass = <your smtp password, if necessary>
+#smtp_from = <email address for From: field>
+#smtp_to = <email address for To: field>
+
+#
+# If you leave this section as is, it will automatically
+# update boto from subversion upon start up.
+# If you don't want that to happen, comment this out
+#
+[Boto]
+boto_location = /usr/local/boto
+boto_update = svn:HEAD
+
+#
+# This tells the Pyami code in boto what scripts
+# to run during startup
+#
+[Pyami]
+scripts = boto.pyami.copybot.CopyBot
+
+#
+# Source bucket and Destination Bucket, obviously.
+# If the Destination bucket does not exist, it will
+# attempt to create it.
+# If exit_on_completion is false, the instance
+# will keep running after the copy operation is
+# complete which might be handy for debugging.
+# If copy_acls is false, the ACL's will not be
+# copied with the objects to the new bucket.
+# If replace_dst is false, copybot will not
+# will only store the source file in the dest if
+# that file does not already exist.  If it's true
+# it will replace it even if it does exist.
+#
+[CopyBot]
+src_bucket = <your source bucket name>
+dst_bucket = <your destination bucket name>
+exit_on_completion = true
+copy_acls = true
+replace_dst = true

data/packages/gsutil/boto/boto/pyami/copybot.py

@@ -0,0 +1,97 @@
+# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+import boto
+from boto.pyami.scriptbase import ScriptBase
+import os, StringIO
+
+class CopyBot(ScriptBase):
+
+    def __init__(self):
+        ScriptBase.__init__(self)
+        self.wdir = boto.config.get('Pyami', 'working_dir')
+        self.log_file = '%s.log' % self.instance_id
+        self.log_path = os.path.join(self.wdir, self.log_file)
+        boto.set_file_logger(self.name, self.log_path)
+        self.src_name = boto.config.get(self.name, 'src_bucket')
+        self.dst_name = boto.config.get(self.name, 'dst_bucket')
+        self.replace = boto.config.getbool(self.name, 'replace_dst', True)
+        s3 = boto.connect_s3()
+        self.src = s3.lookup(self.src_name)
+        if not self.src:
+            boto.log.error('Source bucket does not exist: %s' % self.src_name)
+        dest_access_key = boto.config.get(self.name, 'dest_aws_access_key_id', None)
+        if dest_access_key:
+            dest_secret_key = boto.config.get(self.name, 'dest_aws_secret_access_key', None)
+            s3 = boto.connect(dest_access_key, dest_secret_key)
+        self.dst = s3.lookup(self.dst_name)
+        if not self.dst:
+            self.dst = s3.create_bucket(self.dst_name)
+
+    def copy_bucket_acl(self):
+        if boto.config.get(self.name, 'copy_acls', True):
+            acl = self.src.get_xml_acl()
+            self.dst.set_xml_acl(acl)
+
+    def copy_key_acl(self, src, dst):
+        if boto.config.get(self.name, 'copy_acls', True):
+            acl = src.get_xml_acl()
+            dst.set_xml_acl(acl)
+
+    def copy_keys(self):
+        boto.log.info('src=%s' % self.src.name)
+        boto.log.info('dst=%s' % self.dst.name)
+        try:
+            for key in self.src:
+                if not self.replace:
+                    exists = self.dst.lookup(key.name)
+                    if exists:
+                        boto.log.info('key=%s already exists in %s, skipping' % (key.name, self.dst.name))
+                        continue
+                boto.log.info('copying %d bytes from key=%s' % (key.size, key.name))
+                prefix, base = os.path.split(key.name)
+                path = os.path.join(self.wdir, base)
+                key.get_contents_to_filename(path)
+                new_key = self.dst.new_key(key.name)
+                new_key.set_contents_from_filename(path)
+                self.copy_key_acl(key, new_key)
+                os.unlink(path)
+        except:
+            boto.log.exception('Error copying key: %s' % key.name)
+
+    def copy_log(self):
+        key = self.dst.new_key(self.log_file)
+        key.set_contents_from_filename(self.log_path)
+        
+    def main(self):
+        fp = StringIO.StringIO()
+        boto.config.dump_safe(fp)
+        self.notify('%s (%s) Starting' % (self.name, self.instance_id), fp.getvalue())
+        if self.src and self.dst:
+            self.copy_keys()
+        if self.dst:
+            self.copy_log()
+        self.notify('%s (%s) Stopping' % (self.name, self.instance_id),
+                    'Copy Operation Complete')
+        if boto.config.getbool(self.name, 'exit_on_completion', True):
+            ec2 = boto.connect_ec2()
+            ec2.terminate_instances([self.instance_id])
+