gcloud 0.0.4 → 0.0.6

data/packages/gsutil/boto/boto/pyami/installers/ubuntu/mysql.py

@@ -0,0 +1,109 @@
+# Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+"""
+This installer will install mysql-server on an Ubuntu machine.
+In addition to the normal installation done by apt-get, it will
+also configure the new MySQL server to store it's data files in
+a different location.  By default, this is /mnt but that can be
+configured in the [MySQL] section of the boto config file passed
+to the instance.
+"""
+from boto.pyami.installers.ubuntu.installer import Installer
+import os
+import boto
+from boto.utils import ShellCommand
+from ConfigParser import SafeConfigParser
+import time
+
+ConfigSection = """
+[MySQL]
+root_password = <will be used as MySQL root password, default none>
+data_dir = <new data dir for MySQL, default is /mnt>
+"""
+
+class MySQL(Installer):
+
+    def install(self):
+        self.run('apt-get update')
+        self.run('apt-get -y install mysql-server', notify=True, exit_on_error=True)
+
+#    def set_root_password(self, password=None):
+#        if not password:
+#            password = boto.config.get('MySQL', 'root_password')
+#        if password:
+#            self.run('mysqladmin -u root password %s' % password)
+#        return password
+
+    def change_data_dir(self, password=None):
+        data_dir = boto.config.get('MySQL', 'data_dir', '/mnt')
+        fresh_install = False;
+        is_mysql_running_command = ShellCommand('mysqladmin ping') # exit status 0 if mysql is running
+        is_mysql_running_command.run()
+        if is_mysql_running_command.getStatus() == 0:
+            # mysql is running. This is the state apt-get will leave it in. If it isn't running, 
+            # that means mysql was already installed on the AMI and there's no need to stop it,
+            # saving 40 seconds on instance startup.
+            time.sleep(10) #trying to stop mysql immediately after installing it fails
+            # We need to wait until mysql creates the root account before we kill it
+            # or bad things will happen
+            i = 0
+            while self.run("echo 'quit' | mysql -u root") != 0 and i<5:
+                time.sleep(5)
+                i = i + 1
+            self.run('/etc/init.d/mysql stop')
+            self.run("pkill -9 mysql")
+
+        mysql_path = os.path.join(data_dir, 'mysql')
+        if not os.path.exists(mysql_path):
+            self.run('mkdir %s' % mysql_path)
+            fresh_install = True;
+        self.run('chown -R mysql:mysql %s' % mysql_path)
+        fp = open('/etc/mysql/conf.d/use_mnt.cnf', 'w')
+        fp.write('# created by pyami\n')
+        fp.write('# use the %s volume for data\n' % data_dir)
+        fp.write('[mysqld]\n')
+        fp.write('datadir = %s\n' % mysql_path)
+        fp.write('log_bin = %s\n' % os.path.join(mysql_path, 'mysql-bin.log'))
+        fp.close()
+        if fresh_install:
+            self.run('cp -pr /var/lib/mysql/* %s/' % mysql_path)
+            self.start('mysql')
+        else:
+            #get the password ubuntu expects to use:
+            config_parser = SafeConfigParser()
+            config_parser.read('/etc/mysql/debian.cnf')
+            password = config_parser.get('client', 'password')
+            # start the mysql deamon, then mysql with the required grant statement piped into it:
+            self.start('mysql')
+            time.sleep(10) #time for mysql to start
+            grant_command = "echo \"GRANT ALL PRIVILEGES ON *.* TO 'debian-sys-maint'@'localhost' IDENTIFIED BY '%s' WITH GRANT OPTION;\" | mysql" % password
+            while self.run(grant_command) != 0:
+                time.sleep(5)
+            # leave mysqld running
+
+    def main(self):
+        self.install()
+        # change_data_dir runs 'mysql -u root' which assumes there is no mysql password, i
+        # and changing that is too ugly to be worth it:
+        #self.set_root_password()
+        self.change_data_dir()
+        

data/packages/gsutil/boto/boto/pyami/installers/ubuntu/trac.py

@@ -0,0 +1,139 @@
+# Copyright (c) 2008 Chris Moyer http://coredumped.org
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+from boto.pyami.installers.ubuntu.installer import Installer
+import boto
+import os
+
+class Trac(Installer):
+    """
+    Install Trac and DAV-SVN
+    Sets up a Vhost pointing to [Trac]->home
+    Using the config parameter [Trac]->hostname
+    Sets up a trac environment for every directory found under [Trac]->data_dir
+
+    [Trac]
+    name = My Foo Server
+    hostname = trac.foo.com
+    home = /mnt/sites/trac
+    data_dir = /mnt/trac
+    svn_dir = /mnt/subversion
+    server_admin = root@foo.com
+    sdb_auth_domain = users
+    # Optional
+    SSLCertificateFile = /mnt/ssl/foo.crt
+    SSLCertificateKeyFile = /mnt/ssl/foo.key
+    SSLCertificateChainFile = /mnt/ssl/FooCA.crt
+
+    """
+
+    def install(self):
+        self.run('apt-get -y install trac', notify=True, exit_on_error=True)
+        self.run('apt-get -y install libapache2-svn', notify=True, exit_on_error=True)
+        self.run("a2enmod ssl")
+        self.run("a2enmod mod_python")
+        self.run("a2enmod dav_svn")
+        self.run("a2enmod rewrite")
+        # Make sure that boto.log is writable by everyone so that subversion post-commit hooks can 
+        # write to it.
+        self.run("touch /var/log/boto.log")
+        self.run("chmod a+w /var/log/boto.log")
+
+    def setup_vhost(self):
+        domain = boto.config.get("Trac", "hostname").strip()
+        if domain:
+            domain_info = domain.split('.')
+            cnf = open("/etc/apache2/sites-available/%s" % domain_info[0], "w")
+            cnf.write("NameVirtualHost *:80\n")
+            if boto.config.get("Trac", "SSLCertificateFile"):
+                cnf.write("NameVirtualHost *:443\n\n")
+                cnf.write("<VirtualHost *:80>\n")
+                cnf.write("\tServerAdmin %s\n" % boto.config.get("Trac", "server_admin").strip())
+                cnf.write("\tServerName %s\n" % domain)
+                cnf.write("\tRewriteEngine On\n")
+                cnf.write("\tRewriteRule ^(.*)$ https://%s$1\n" % domain)
+                cnf.write("</VirtualHost>\n\n")
+
+                cnf.write("<VirtualHost *:443>\n")
+            else:
+                cnf.write("<VirtualHost *:80>\n")
+
+            cnf.write("\tServerAdmin %s\n" % boto.config.get("Trac", "server_admin").strip())
+            cnf.write("\tServerName %s\n" % domain)
+            cnf.write("\tDocumentRoot %s\n" % boto.config.get("Trac", "home").strip())
+
+            cnf.write("\t<Directory %s>\n" % boto.config.get("Trac", "home").strip())
+            cnf.write("\t\tOptions FollowSymLinks Indexes MultiViews\n")
+            cnf.write("\t\tAllowOverride All\n")
+            cnf.write("\t\tOrder allow,deny\n")
+            cnf.write("\t\tallow from all\n")
+            cnf.write("\t</Directory>\n")
+
+            cnf.write("\t<Location />\n")
+            cnf.write("\t\tAuthType Basic\n")
+            cnf.write("\t\tAuthName \"%s\"\n" % boto.config.get("Trac", "name"))
+            cnf.write("\t\tRequire valid-user\n")
+            cnf.write("\t\tAuthUserFile /mnt/apache/passwd/passwords\n")
+            cnf.write("\t</Location>\n")
+
+            data_dir = boto.config.get("Trac", "data_dir")
+            for env in os.listdir(data_dir):
+                if(env[0] != "."):
+                    cnf.write("\t<Location /trac/%s>\n" % env)
+                    cnf.write("\t\tSetHandler mod_python\n")
+                    cnf.write("\t\tPythonInterpreter main_interpreter\n")
+                    cnf.write("\t\tPythonHandler trac.web.modpython_frontend\n")
+                    cnf.write("\t\tPythonOption TracEnv %s/%s\n" % (data_dir, env))
+                    cnf.write("\t\tPythonOption TracUriRoot /trac/%s\n" % env)
+                    cnf.write("\t</Location>\n")
+
+            svn_dir = boto.config.get("Trac", "svn_dir")
+            for env in os.listdir(svn_dir):
+                if(env[0] != "."):
+                    cnf.write("\t<Location /svn/%s>\n" % env)
+                    cnf.write("\t\tDAV svn\n")
+                    cnf.write("\t\tSVNPath %s/%s\n" % (svn_dir, env))
+                    cnf.write("\t</Location>\n")
+
+            cnf.write("\tErrorLog /var/log/apache2/error.log\n")
+            cnf.write("\tLogLevel warn\n")
+            cnf.write("\tCustomLog /var/log/apache2/access.log combined\n")
+            cnf.write("\tServerSignature On\n")
+            SSLCertificateFile = boto.config.get("Trac", "SSLCertificateFile")
+            if SSLCertificateFile:
+                cnf.write("\tSSLEngine On\n")
+                cnf.write("\tSSLCertificateFile %s\n" % SSLCertificateFile)
+
+            SSLCertificateKeyFile = boto.config.get("Trac", "SSLCertificateKeyFile")
+            if SSLCertificateKeyFile:
+                cnf.write("\tSSLCertificateKeyFile %s\n" % SSLCertificateKeyFile)
+
+            SSLCertificateChainFile = boto.config.get("Trac", "SSLCertificateChainFile")
+            if SSLCertificateChainFile:
+                cnf.write("\tSSLCertificateChainFile %s\n" % SSLCertificateChainFile)
+            cnf.write("</VirtualHost>\n")
+            cnf.close()
+            self.run("a2ensite %s" % domain_info[0])
+            self.run("/etc/init.d/apache2 force-reload")
+
+    def main(self):
+        self.install()
+        self.setup_vhost()

data/packages/gsutil/boto/boto/pyami/launch_ami.py

@@ -0,0 +1,178 @@
+#!/usr/bin/env python
+# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+#
+import getopt
+import sys
+import imp
+import time
+import boto
+
+usage_string = """
+SYNOPSIS
+    launch_ami.py -a ami_id [-b script_bucket] [-s script_name]
+                  [-m module] [-c class_name] [-r] 
+                  [-g group] [-k key_name] [-n num_instances]
+                  [-w] [extra_data]
+    Where:
+        ami_id - the id of the AMI you wish to launch
+        module - The name of the Python module containing the class you
+                 want to run when the instance is started.  If you use this
+                 option the Python module must already be stored on the
+                 instance in a location that is on the Python path.
+        script_file - The name of a local Python module that you would like
+                      to have copied to S3 and then run on the instance
+                      when it is started.  The specified module must be
+                      import'able (i.e. in your local Python path).  It
+                      will then be copied to the specified bucket in S3
+                      (see the -b option).  Once the new instance(s)
+                      start up the script will be copied from S3 and then
+                      run locally on the instance.
+        class_name - The name of the class to be instantiated within the
+                     module or script file specified.
+        script_bucket - the name of the bucket in which the script will be
+                        stored
+        group - the name of the security group the instance will run in
+        key_name - the name of the keypair to use when launching the AMI
+        num_instances - how many instances of the AMI to launch (default 1)
+        input_queue_name - Name of SQS to read input messages from
+        output_queue_name - Name of SQS to write output messages to
+        extra_data - additional name-value pairs that will be passed as
+                     userdata to the newly launched instance.  These should
+                     be of the form "name=value"
+        The -r option reloads the Python module to S3 without launching
+        another instance.  This can be useful during debugging to allow
+        you to test a new version of your script without shutting down
+        your instance and starting up another one.
+        The -w option tells the script to run synchronously, meaning to
+        wait until the instance is actually up and running.  It then prints
+        the IP address and internal and external DNS names before exiting.
+"""
+
+def usage():
+    print usage_string
+    sys.exit()
+
+def main():
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], 'a:b:c:g:hi:k:m:n:o:rs:w',
+                                   ['ami', 'bucket', 'class', 'group', 'help',
+                                    'inputqueue', 'keypair', 'module',
+                                    'numinstances', 'outputqueue',
+                                    'reload', 'script_name', 'wait'])
+    except:
+        usage()
+    params = {'module_name' : None,
+              'script_name' : None,
+              'class_name' : None,
+              'script_bucket' : None,
+              'group' : 'default',
+              'keypair' : None,
+              'ami' : None,
+              'num_instances' : 1,
+              'input_queue_name' : None,
+              'output_queue_name' : None}
+    reload = None
+    wait = None
+    for o, a in opts:
+        if o in ('-a', '--ami'):
+            params['ami'] = a
+        if o in ('-b', '--bucket'):
+            params['script_bucket'] = a
+        if o in ('-c', '--class'):
+            params['class_name'] = a
+        if o in ('-g', '--group'):
+            params['group'] = a
+        if o in ('-h', '--help'):
+            usage()
+        if o in ('-i', '--inputqueue'):
+            params['input_queue_name'] = a
+        if o in ('-k', '--keypair'):
+            params['keypair'] = a
+        if o in ('-m', '--module'):
+            params['module_name'] = a
+        if o in ('-n', '--num_instances'):
+            params['num_instances'] = int(a)
+        if o in ('-o', '--outputqueue'):
+            params['output_queue_name'] = a
+        if o in ('-r', '--reload'):
+            reload = True
+        if o in ('-s', '--script'):
+            params['script_name'] = a
+        if o in ('-w', '--wait'):
+            wait = True
+
+    # check required fields
+    required = ['ami']
+    for pname in required:
+        if not params.get(pname, None):
+            print '%s is required' % pname
+            usage()
+    if params['script_name']:
+        # first copy the desired module file to S3 bucket
+        if reload:
+            print 'Reloading module %s to S3' % params['script_name']
+        else:
+            print 'Copying module %s to S3' % params['script_name']
+        l = imp.find_module(params['script_name'])
+        c = boto.connect_s3()
+        bucket = c.get_bucket(params['script_bucket'])
+        key = bucket.new_key(params['script_name']+'.py')
+        key.set_contents_from_file(l[0])
+        params['script_md5'] = key.md5
+    # we have everything we need, now build userdata string
+    l = []
+    for k, v in params.items():
+        if v:
+            l.append('%s=%s' % (k, v))
+    c = boto.connect_ec2()
+    l.append('aws_access_key_id=%s' % c.aws_access_key_id)
+    l.append('aws_secret_access_key=%s' % c.aws_secret_access_key)
+    for kv in args:
+        l.append(kv)
+    s = '|'.join(l)
+    if not reload:
+        rs = c.get_all_images([params['ami']])
+        img = rs[0]
+        r = img.run(user_data=s, key_name=params['keypair'],
+                    security_groups=[params['group']],
+                    max_count=params.get('num_instances', 1))
+        print 'AMI: %s - %s (Started)' % (params['ami'], img.location)
+        print 'Reservation %s contains the following instances:' % r.id
+        for i in r.instances:
+            print '\t%s' % i.id
+        if wait:
+            running = False
+            while not running:
+                time.sleep(30)
+                [i.update() for i in r.instances]
+                status = [i.state for i in r.instances]
+                print status
+                if status.count('running') == len(r.instances):
+                    running = True
+            for i in r.instances:
+                print 'Instance: %s' % i.ami_launch_index
+                print 'Public DNS Name: %s' % i.public_dns_name
+                print 'Private DNS Name: %s' % i.private_dns_name
+
+if __name__ == "__main__":
+    main()
+

data/packages/gsutil/boto/boto/pyami/scriptbase.py

@@ -0,0 +1,44 @@
+import os
+import sys
+from boto.utils import ShellCommand, get_ts
+import boto
+import boto.utils
+
+class ScriptBase:
+
+    def __init__(self, config_file=None):
+        self.instance_id = boto.config.get('Instance', 'instance-id', 'default')
+        self.name = self.__class__.__name__
+        self.ts = get_ts()
+        if config_file:
+            boto.config.read(config_file)
+
+    def notify(self, subject, body=''):
+        boto.utils.notify(subject, body)
+
+    def mkdir(self, path):
+        if not os.path.isdir(path):
+            try:
+                os.mkdir(path)
+            except:
+                boto.log.error('Error creating directory: %s' % path)
+
+    def umount(self, path):
+        if os.path.ismount(path):
+            self.run('umount %s' % path)
+
+    def run(self, command, notify=True, exit_on_error=False, cwd=None):
+        self.last_command = ShellCommand(command, cwd=cwd)
+        if self.last_command.status != 0:
+            boto.log.error('Error running command: "%s". Output: "%s"' % (command, self.last_command.output))
+            if notify:
+                self.notify('Error encountered', \
+                        'Error running the following command:\n\t%s\n\nCommand output:\n\t%s' % \
+                        (command, self.last_command.output))
+            if exit_on_error:
+                sys.exit(-1)
+        return self.last_command.status
+
+    def main(self):
+        pass
+