googlecloud 0.0.4 → 0.0.6

data/packages/gsutil/boto/boto/services/service.py

@@ -0,0 +1,161 @@
+# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+import boto
+from boto.services.message import ServiceMessage
+from boto.services.servicedef import ServiceDef
+from boto.pyami.scriptbase import ScriptBase
+from boto.utils import get_ts
+import time
+import os
+import mimetypes
+
+
+class Service(ScriptBase):
+
+    # Time required to process a transaction
+    ProcessingTime = 60
+
+    def __init__(self, config_file=None, mimetype_files=None):
+        ScriptBase.__init__(self, config_file)
+        self.name = self.__class__.__name__
+        self.working_dir = boto.config.get('Pyami', 'working_dir')
+        self.sd = ServiceDef(config_file)
+        self.retry_count = self.sd.getint('retry_count', 5)
+        self.loop_delay = self.sd.getint('loop_delay', 30)
+        self.processing_time = self.sd.getint('processing_time', 60)
+        self.input_queue = self.sd.get_obj('input_queue')
+        self.output_queue = self.sd.get_obj('output_queue')
+        self.output_domain = self.sd.get_obj('output_domain')
+        if mimetype_files:
+            mimetypes.init(mimetype_files)
+
+    def split_key(key):
+        if key.find(';') < 0:
+            t = (key, '')
+        else:
+            key, type = key.split(';')
+            label, mtype = type.split('=')
+            t = (key, mtype)
+        return t
+
+    def read_message(self):
+        boto.log.info('read_message')
+        message = self.input_queue.read(self.processing_time)
+        if message:
+            boto.log.info(message.get_body())
+            key = 'Service-Read'
+            message[key] = get_ts()
+        return message
+
+    # retrieve the source file from S3
+    def get_file(self, message):
+        bucket_name = message['Bucket']
+        key_name = message['InputKey']
+        file_name = os.path.join(self.working_dir, message.get('OriginalFileName', 'in_file'))
+        boto.log.info('get_file: %s/%s to %s' % (bucket_name, key_name, file_name))
+        bucket = boto.lookup('s3', bucket_name)
+        key = bucket.new_key(key_name)
+        key.get_contents_to_filename(os.path.join(self.working_dir, file_name))
+        return file_name
+
+    # process source file, return list of output files
+    def process_file(self, in_file_name, msg):
+        return []
+
+    # store result file in S3
+    def put_file(self, bucket_name, file_path, key_name=None):
+        boto.log.info('putting file %s as %s.%s' % (file_path, bucket_name, key_name))
+        bucket = boto.lookup('s3', bucket_name)
+        key = bucket.new_key(key_name)
+        key.set_contents_from_filename(file_path)
+        return key
+
+    def save_results(self, results, input_message, output_message):
+        output_keys = []
+        for file, type in results:
+            if 'OutputBucket' in input_message:
+                output_bucket = input_message['OutputBucket']
+            else:
+                output_bucket = input_message['Bucket']
+            key_name = os.path.split(file)[1]
+            key = self.put_file(output_bucket, file, key_name)
+            output_keys.append('%s;type=%s' % (key.name, type))
+        output_message['OutputKey'] = ','.join(output_keys)
+            
+    # write message to each output queue
+    def write_message(self, message):
+        message['Service-Write'] = get_ts()
+        message['Server'] = self.name
+        if 'HOSTNAME' in os.environ:
+            message['Host'] = os.environ['HOSTNAME']
+        else:
+            message['Host'] = 'unknown'
+        message['Instance-ID'] = self.instance_id
+        if self.output_queue:
+            boto.log.info('Writing message to SQS queue: %s' % self.output_queue.id)
+            self.output_queue.write(message)
+        if self.output_domain:
+            boto.log.info('Writing message to SDB domain: %s' % self.output_domain.name)
+            item_name = '/'.join([message['Service-Write'], message['Bucket'], message['InputKey']])
+            self.output_domain.put_attributes(item_name, message)
+
+    # delete message from input queue
+    def delete_message(self, message):
+        boto.log.info('deleting message from %s' % self.input_queue.id)
+        self.input_queue.delete_message(message)
+
+    # to clean up any files, etc. after each iteration
+    def cleanup(self):
+        pass
+
+    def shutdown(self):
+        on_completion = self.sd.get('on_completion', 'shutdown')
+        if on_completion == 'shutdown':
+            if self.instance_id:
+                time.sleep(60)
+                c = boto.connect_ec2()
+                c.terminate_instances([self.instance_id])
+
+    def main(self, notify=False):
+        self.notify('Service: %s Starting' % self.name)
+        empty_reads = 0
+        while self.retry_count < 0 or empty_reads < self.retry_count:
+            try:
+                input_message = self.read_message()
+                if input_message:
+                    empty_reads = 0
+                    output_message = ServiceMessage(None, input_message.get_body())
+                    input_file = self.get_file(input_message)
+                    results = self.process_file(input_file, output_message)
+                    self.save_results(results, input_message, output_message)
+                    self.write_message(output_message)
+                    self.delete_message(input_message)
+                    self.cleanup()
+                else:
+                    empty_reads += 1
+                    time.sleep(self.loop_delay)
+            except Exception:
+                boto.log.exception('Service Failed')
+                empty_reads += 1
+        self.notify('Service: %s Shutting Down' % self.name)
+        self.shutdown()
+

data/packages/gsutil/boto/boto/services/servicedef.py

@@ -0,0 +1,91 @@
+# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+from boto.pyami.config import Config
+from boto.services.message import ServiceMessage
+import boto
+
+class ServiceDef(Config):
+
+    def __init__(self, config_file, aws_access_key_id=None, aws_secret_access_key=None):
+        Config.__init__(self, config_file)
+        self.aws_access_key_id = aws_access_key_id
+        self.aws_secret_access_key = aws_secret_access_key
+        script = Config.get(self, 'Pyami', 'scripts')
+        if script:
+            self.name = script.split('.')[-1]
+        else:
+            self.name = None
+
+
+    def get(self, name, default=None):
+        return Config.get(self, self.name, name, default)
+
+    def has_option(self, option):
+        return Config.has_option(self, self.name, option)
+
+    def getint(self, option, default=0):
+        try:
+            val = Config.get(self, self.name, option)
+            val = int(val)
+        except:
+            val = int(default)
+        return val
+        
+    def getbool(self, option, default=False):
+        try:
+            val = Config.get(self, self.name, option)
+            if val.lower() == 'true':
+                val = True
+            else:
+                val = False
+        except:
+            val = default
+        return val
+        
+    def get_obj(self, name):
+        """
+        Returns the AWS object associated with a given option.
+
+        The heuristics used are a bit lame.  If the option name contains
+        the word 'bucket' it is assumed to be an S3 bucket, if the name
+        contains the word 'queue' it is assumed to be an SQS queue and
+        if it contains the word 'domain' it is assumed to be a SimpleDB
+        domain.  If the option name specified does not exist in the
+        config file or if the AWS object cannot be retrieved this
+        returns None.
+        """
+        val = self.get(name)
+        if not val:
+            return None
+        if name.find('queue') >= 0:
+            obj = boto.lookup('sqs', val)
+            if obj:
+                obj.set_message_class(ServiceMessage)
+        elif name.find('bucket') >= 0:
+            obj = boto.lookup('s3', val)
+        elif name.find('domain') >= 0:
+            obj = boto.lookup('sdb', val)
+        else:
+            obj = None
+        return obj
+
+    

data/packages/gsutil/boto/boto/services/sonofmmm.cfg

@@ -0,0 +1,43 @@
+#
+# Your AWS Credentials
+# You only need to supply these in this file if you are not using
+# the boto tools to start your service
+#
+#[Credentials]
+#aws_access_key_id = <AWS Access Key Here>
+#aws_secret_access_key = <AWS Secret Key Here>
+
+#
+# Fill out this section if you want emails from the service
+# when it starts and stops
+#
+#[Notification]
+#smtp_host = <your smtp host>
+#smtp_user = <your smtp username, if necessary>
+#smtp_pass = <your smtp password, if necessary>
+#smtp_from = <email address for From: field>
+#smtp_to = <email address for To: field>
+
+[Pyami]
+scripts = boto.services.sonofmmm.SonOfMMM
+
+[SonOfMMM]
+# id of the AMI to be launched
+ami_id = ami-dc799cb5
+# number of times service will read an empty queue before exiting
+# a negative value will cause the service to run forever
+retry_count = 5
+# seconds to wait after empty queue read before reading again
+loop_delay = 10
+# average time it takes to process a transaction
+# controls invisibility timeout of messages
+processing_time = 60
+ffmpeg_args = -y -i %%s -f mov -r 29.97 -b 1200kb -mbd 2 -flags +4mv+trell -aic 2 -cmp 2 -subcmp 2 -ar 48000 -ab 19200 -s 320x240 -vcodec mpeg4 -acodec libfaac %%s
+output_mimetype = video/quicktime
+output_ext = .mov
+input_bucket = <S3 bucket where source videos live>
+output_bucket = <S3 bucket where converted videos should be stored>
+output_domain = <SimpleDB domain to store results - optional>
+output_queue = <SQS queue to store results - optional>
+input_queue = <SQS queue where work to be done will be queued up>
+

data/packages/gsutil/boto/boto/services/sonofmmm.py

@@ -0,0 +1,81 @@
+# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+import boto
+from boto.services.service import Service
+from boto.services.message import ServiceMessage
+import os
+import mimetypes
+
+class SonOfMMM(Service):
+
+    def __init__(self, config_file=None):
+        Service.__init__(self, config_file)
+        self.log_file = '%s.log' % self.instance_id
+        self.log_path = os.path.join(self.working_dir, self.log_file)
+        boto.set_file_logger(self.name, self.log_path)
+        if self.sd.has_option('ffmpeg_args'):
+            self.command = '/usr/local/bin/ffmpeg ' + self.sd.get('ffmpeg_args')
+        else:
+            self.command = '/usr/local/bin/ffmpeg -y -i %s %s'
+        self.output_mimetype = self.sd.get('output_mimetype')
+        if self.sd.has_option('output_ext'):
+            self.output_ext = self.sd.get('output_ext')
+        else:
+            self.output_ext = mimetypes.guess_extension(self.output_mimetype)
+        self.output_bucket = self.sd.get_obj('output_bucket')
+        self.input_bucket = self.sd.get_obj('input_bucket')
+        # check to see if there are any messages queue
+        # if not, create messages for all files in input_bucket
+        m = self.input_queue.read(1)
+        if not m:
+            self.queue_files()
+
+    def queue_files(self):
+        boto.log.info('Queueing files from %s' % self.input_bucket.name)
+        for key in self.input_bucket:
+            boto.log.info('Queueing %s' % key.name)
+            m = ServiceMessage()
+            if self.output_bucket:
+                d = {'OutputBucket' : self.output_bucket.name}
+            else:
+                d = None
+            m.for_key(key, d)
+            self.input_queue.write(m)
+
+    def process_file(self, in_file_name, msg):
+        base, ext = os.path.splitext(in_file_name)
+        out_file_name = os.path.join(self.working_dir,
+                                     base+self.output_ext)
+        command = self.command % (in_file_name, out_file_name)
+        boto.log.info('running:\n%s' % command)
+        status = self.run(command)
+        if status == 0:
+            return [(out_file_name, self.output_mimetype)]
+        else:
+            return []
+
+    def shutdown(self):
+        if os.path.isfile(self.log_path):
+            if self.output_bucket:
+                key = self.output_bucket.new_key(self.log_file)
+                key.set_contents_from_filename(self.log_path)
+        Service.shutdown(self)

data/packages/gsutil/boto/boto/services/submit.py

@@ -0,0 +1,88 @@
+# Copyright (c) 2006,2007 Mitch Garnaat http://garnaat.org/
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish, dis-
+# tribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the fol-
+# lowing conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
+# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
+# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
+# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
+# IN THE SOFTWARE.
+
+import time
+import os
+
+
+class Submitter:
+
+    def __init__(self, sd):
+        self.sd = sd
+        self.input_bucket = self.sd.get_obj('input_bucket')
+        self.output_bucket = self.sd.get_obj('output_bucket')
+        self.output_domain = self.sd.get_obj('output_domain')
+        self.queue = self.sd.get_obj('input_queue')
+
+    def get_key_name(self, fullpath, prefix):
+        key_name = fullpath[len(prefix):]
+        l = key_name.split(os.sep)
+        return '/'.join(l)
+
+    def write_message(self, key, metadata):
+        if self.queue:
+            m = self.queue.new_message()
+            m.for_key(key, metadata)
+            if self.output_bucket:
+                m['OutputBucket'] = self.output_bucket.name
+            self.queue.write(m)
+
+    def submit_file(self, path, metadata=None, cb=None, num_cb=0, prefix='/'):
+        if not metadata:
+            metadata = {}
+        key_name = self.get_key_name(path, prefix)
+        k = self.input_bucket.new_key(key_name)
+        k.update_metadata(metadata)
+        k.set_contents_from_filename(path, replace=False, cb=cb, num_cb=num_cb)
+        self.write_message(k, metadata)
+
+    def submit_path(self, path, tags=None, ignore_dirs=None, cb=None, num_cb=0, status=False, prefix='/'):
+        path = os.path.expanduser(path)
+        path = os.path.expandvars(path)
+        path = os.path.abspath(path)
+        total = 0
+        metadata = {}
+        if tags:
+            metadata['Tags'] = tags
+        l = []
+        for t in time.gmtime():
+            l.append(str(t))
+        metadata['Batch'] = '_'.join(l)
+        if self.output_domain:
+            self.output_domain.put_attributes(metadata['Batch'], {'type' : 'Batch'})
+        if os.path.isdir(path):
+            for root, dirs, files in os.walk(path):
+                if ignore_dirs:
+                    for ignore in ignore_dirs:
+                        if ignore in dirs:
+                            dirs.remove(ignore)
+                for file in files:
+                    fullpath = os.path.join(root, file)
+                    if status:
+                        print 'Submitting %s' % fullpath
+                    self.submit_file(fullpath, metadata, cb, num_cb, prefix)
+                    total += 1
+        elif os.path.isfile(path):
+            self.submit_file(path, metadata, cb, num_cb)
+            total += 1
+        else:
+            print 'problem with %s' % path
+        return (metadata['Batch'], total)