elastic-mapreduce 0.0.1

data/lib/amazon/coral/querystringmap.rb

@@ -0,0 +1,93 @@
+#
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+require 'amazon/coral/urlencoding'
+
+module Amazon
+  module Coral
+
+    # A hash containing query string parameters that produces a query
+    # string via to_s.  Also consumes hashes representing hierarchies of
+    # data to encode as query parameters.
+    class QueryStringMap < Hash
+
+      # Instantiate a QueryStringMap with the contents of the specified
+      # hash.  If no hash is provided, an empty map is created.
+      def initialize(hash = {})
+        add_flattened(hash)
+      end
+
+      # Returns the query string representation of this map by collapsing
+      # its key-value pairs into URL parameters.
+      def to_s
+        qstr = ''
+        isFirst = true
+        each_pair { |k,v|
+          if isFirst then
+            isFirst = false
+          else
+            qstr << '&'
+          end
+          qstr << UrlEncoding.encode(k.to_s)
+          unless(v.nil?) then
+            qstr << '='
+            qstr << UrlEncoding.encode(v.to_s)
+          end
+        }
+        return qstr
+      end
+
+      private
+      def add_flattened(hash)
+        stack = []
+        add_flattened_helper(stack, hash)
+      end
+
+      def add_flattened_helper(stack, obj)
+        return if obj.nil?
+        
+        case obj
+        when Hash
+
+            obj.each_pair { |k,v|
+            stack.push(k)
+            add_flattened_helper(stack, v)
+            stack.pop
+          }
+
+        when Array
+
+            # Do artificial list member wrapping (Coral requires this
+            # level of indirection, but doesn't validate the member name)
+            stack.push("member")
+
+          obj.each_index { |i|
+            v = obj[i]
+            stack.push(i + 1) # query string arrays are 1-based
+            add_flattened_helper(stack, v)
+            stack.pop
+          }
+
+          stack.pop
+
+        else
+
+          # this works for symbols also, because sym.id2name == sym.to_s
+          self[get_key(stack)] = obj.to_s
+
+        end
+      end
+
+      def get_key(stack)
+        key = ''
+        stack.each_index { |i|
+          key << '.' if(i > 0)
+          key << stack[i].to_s
+        }
+        return key
+      end
+
+    end
+
+  end
+end

data/lib/amazon/coral/service.rb

@@ -0,0 +1,130 @@
+#
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+require 'amazon/coral/option'
+require 'amazon/coral/orchestrator'
+require 'amazon/coral/dispatcher'
+require 'amazon/coral/call'
+require 'amazon/coral/awsquery'
+require 'amazon/coral/simplelog'
+
+module Amazon
+  module Coral
+
+    # Provides a simple command-line interface to call remote services.
+    class Service
+
+      @@command_arguments = [
+                             Option.new({:long => 'help', :short => 'h'}),
+                             Option.new({:long => 'url', :short => 'u', :parameters => 1}),
+                             Option.new({:long => 'awsAccessKey', :short => 'a', :parameters => 1}),
+                             Option.new({:long => 'awsSecretKey', :short => 's', :parameters => 1}),
+                             Option.new({:long => 'v0'}),
+                             Option.new({:long => 'v1'}),
+                             Option.new({:long => 'timeout', :parameters => 1}),
+                             Option.new({:long => 'connect_timeout', :parameters => 1}),
+                             Option.new({:long => 'input', :short => 'i', :parameters => 1}),
+                             Option.new({:long => 'operation', :short => 'o', :parameters => 1}),
+                             Option.new({:long => 'verbose', :short => 'v'})];
+
+      # Initializes a Service object with the specified arguments.
+      # Possible arguments include:
+      # [:orchestrator_helper]
+      #   A class that responds to self.new_orchestrator create the necessary orchestrator.
+      #   By default the AwsQueryChainHelper is used.
+      # [:service]
+      #   The name of the service to be called.
+      # [:operations]
+      #   A list naming the operations available on the remote service.
+      def initialize(args)
+        @orchestrator_helper_class = args[:orchestrator_helper]
+        @orchestrator_helper_class = AwsQuery if @orchestrator_helper_class.nil?
+
+        @service_name = args[:service]
+        @operation_names = args[:operations]
+      end
+
+      # Runs the command line client application.
+      def main
+
+        if ARGV.length == 0 then
+          print_usage
+          exit
+        end
+
+        args = Option.parse(@@command_arguments, ARGV)
+        if(args.length == 0 || !args['help'].nil?) then
+          print_usage
+          exit
+        end
+
+
+
+        raise "the 'url' parameter is required" if(args['url'].nil?)
+        url = args['url'][0]
+        
+        input = nil
+        input = eval(args['input'][0]) unless args['input'].nil?
+
+        raise "the 'operation' parameter is required" if(args['operation'].nil?)
+        operation = args['operation'][0]
+        raise "operation '#{operation}' is not valid for this service" unless @operation_names.include?(operation)
+
+        verbose = !args['verbose'].nil?
+
+        timeout = Float(args['timeout'][0]) unless args['timeout'].nil?
+        connect_timeout = Float(args['connect_timeout'][0]) unless args['connect_timeout'].nil?
+
+        aws_access_key = nil
+        aws_secret_key = nil
+        signature_algorithm = nil
+
+        if(!args['awsAccessKey'].nil? && !args['awsSecretKey'].nil?) then
+          aws_access_key = args['awsAccessKey'][0]
+          aws_secret_key = args['awsSecretKey'][0]
+          signature_algorithm = :V2
+          signature_algorithm = :V0 if !args['v0'].nil?
+          signature_algorithm = :V1 if !args['v1'].nil?
+        end
+
+        helper_args = {:endpoint => url, :signature_algorithm => signature_algorithm, :verbose => verbose, :timeout => timeout, :connect_timeout => connect_timeout}
+
+        orchestrator = @orchestrator_helper_class.new_orchestrator(helper_args)
+        dispatcher = Dispatcher.new(orchestrator, @service_name, operation)
+        call = Call.new(dispatcher)
+
+        call.identity[:aws_access_key] = aws_access_key
+        call.identity[:aws_secret_key] = aws_secret_key
+
+        output = call.call(input)
+
+        puts output.inspect
+      end
+
+
+
+      # Prints to STDOUT a help message describing how to use the application.
+      def print_usage
+        puts "#{@service_name} ruby client"
+        puts "Usage:"
+        puts "  -h --help"
+        puts "  -u --url"
+        puts "  -o --operation OPERATION"
+        puts "  -i --input INPUT"
+        puts "  -a --awsAccessKey KEY"
+        puts "  -s --awsSecretKey SECRET_KEY"
+        puts "  --v0"
+        puts "  --v1"
+        puts "  -v --verbose"
+        puts ""
+        puts "Available operations:"
+        @operation_names.each { |name|
+          puts("  #{name}")
+        } unless @operation_names.nil?
+      end
+
+    end
+
+  end
+end
+

data/lib/amazon/coral/simplelog.rb

@@ -0,0 +1,98 @@
+#
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+require 'amazon/coral/logfactory'
+require 'logger'
+
+module Amazon
+  module Coral
+
+    # Wraps Ruby's built in Logger to prepend a context string to log messages.
+    # This is useful to prefix log messages with the name of the originating class, etc.
+    class WrappedLogger
+      def initialize(key, logger)
+        @key = key
+        @logger = logger
+      end
+
+      def debug(s)
+        @logger.debug(format(s))
+      end
+      def info(s)
+        @logger.info(format(s))
+      end
+      def warn(s)
+        @logger.warn(format(s))
+      end
+      def error(s)
+        @logger.error(format(s))
+      end
+      def fatal(s)
+        @logger.fatal(format(s))
+      end
+
+      def debug?
+        @logger.debug?
+      end
+      def info?
+        @logger.info?
+      end
+      def warn?
+        @logger.warn?
+      end
+      def error?
+        @logger.error?
+      end
+      def fatal?
+        @logger.fatal?
+      end
+
+      def format(s)
+        return "#{@key}: #{s}"
+      end
+    end
+
+    #
+    # Provides a LogFactory implementation that supplies WrappedLogger objects to requestors.
+    # The key provided to getLog is prepended to log messages from each Logger.
+    #
+    # Copyright:: Copyright (c) 2008 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+    #
+    class SimpleLogFactory < LogFactory
+      def initialize(output, level)
+        @output = output
+        @level = level
+      end
+
+      def getLog(key)
+        logger = Logger.new(@output)
+        logger.level = @level
+
+        return WrappedLogger.new(key, logger)
+      end
+    end
+
+    #
+    # Provides a straightforward facility to configure SimpleLog as the active logging mechanism.
+    #
+    # Copyright:: Copyright (c) 2008 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+    #
+    class SimpleLog
+      # Registers a SimpleLogFactory with the specified output IO object and logging level.
+      #
+      # To set logging to its highest level and send output to the console, use:
+      #   SimpleLog.install(STDOUT, Logger:DEBUG)
+      #
+      # To send logging output to a file:
+      #   SimpleLog.install(File.new('/tmp/simplelog.log', 'r'), Logger::INFO)
+      #
+      # Installing a new LogFactory will not affect objects that have already retrieved their log instances,
+      # it's best to initialize logging as early as possible in your code to ensure that all your code gets
+      # the proper configuration.
+      def SimpleLog.install(output, level)
+        LogFactory.setInstance(SimpleLogFactory.new(output, level))
+      end
+    end
+
+  end
+end

data/lib/amazon/coral/urlencoding.rb

@@ -0,0 +1,19 @@
+#
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+require 'cgi'
+
+module Amazon
+  module Coral
+
+    # Performs AWS's preferred method of URLEncoding.
+    class UrlEncoding
+      
+      # Convert a string into URL encoded form.
+      def UrlEncoding.encode(plaintext)
+        CGI.escape(plaintext.to_s).gsub("+", "%20").gsub("%7E", "~")
+      end
+    end
+    
+  end
+end
+

data/lib/amazon/coral/v0signaturehandler.rb

@@ -0,0 +1,33 @@
+#
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+require 'openssl'
+require 'base64'
+require 'time'
+require 'amazon/coral/handler'
+require 'amazon/coral/v0signaturehelper'
+
+module Amazon
+  module Coral
+    
+    # Applies an AWS version 0 signature to the outgoing request.
+    class V0SignatureHandler < Handler
+      
+      def before(job)
+        request = job.request
+        identity = request[:identity]
+        aws_access_key = identity[:aws_access_key]
+        aws_secret_key = identity[:aws_secret_key]
+        
+        query_string_map = request[:query_string_map]
+        
+        return if aws_access_key.nil? || aws_secret_key.nil? || query_string_map.nil?;
+        
+        V0SignatureHelper.new(aws_access_key, aws_secret_key).sign({:query_string_map => query_string_map})
+      end
+
+    end
+    
+  end
+end
+

data/lib/amazon/coral/v0signaturehelper.rb

@@ -0,0 +1,83 @@
+#
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+require 'openssl'
+require 'base64'
+require 'time'
+
+module Amazon
+  module Coral
+
+    # Performs AWS V0 Signatures on QueryStringMap objects.
+    class V0SignatureHelper 
+      def initialize(aws_access_key_id, aws_secret_key)
+        @aws_access_key_id = aws_access_key_id.to_s
+        @aws_secret_key = aws_secret_key.to_s
+      end
+
+      def sign(args)
+        signT(Time.now.iso8601, args)
+      end
+
+      def signT(time, args)
+        query_string_map = args[:query_string_map]
+        add_fields(query_string_map, time)
+        query_string_map['Signature'] = compute_signature(canonicalize(args))
+      end
+
+      def canonicalize(args)
+        query_string_map = args[:query_string_map]
+
+        service_name = query_string_map['ServiceName']
+        action = query_string_map['Action']
+        operation = query_string_map['Operation']
+        timestamp = query_string_map['Timestamp']
+        expires = query_string_map['Expires']
+        credential = query_string_map['Credential']
+
+        canonical = ''
+
+        canonical << service_name unless service_name.nil?
+
+        if !action.nil? then
+          canonical << action
+        elsif !operation.nil? then
+          canonical << operation
+        else
+          raise 'Query string must contain Action or Operation'
+        end
+        
+        if timestamp.nil? and expires.nil? then
+          raise 'Query string must contain Timestamp or Expires'
+        elsif !timestamp.nil? and !expires.nil? then
+          raise 'Query string may contain only one of Timestamp or Expires'
+        elsif !timestamp.nil? then
+          canonical << timestamp
+        else
+          canonical << expires
+        end
+
+        canonical << credential unless credential.nil?
+
+        return canonical
+      end
+
+      def compute_signature(canonical)
+        digest = OpenSSL::Digest::Digest.new('sha1')
+        return Base64.encode64(OpenSSL::HMAC.digest(digest, @aws_secret_key, canonical)).strip
+      end
+
+      def add_fields(query_string_map, time)
+        query_string_map['AWSAccessKeyId'] = @aws_access_key_id
+        query_string_map['SignatureVersion'] = '0'
+        query_string_map['SignatureMethod'] = 'HmacSHA1'
+        query_string_map['Timestamp'] = time.to_s
+      end
+
+      def sort(hash)
+        hash.sort { |a,b| a[0].downcase <=> b[0].downcase }
+      end
+    end
+
+  end
+end

data/lib/amazon/coral/v1signaturehandler.rb

@@ -0,0 +1,32 @@
+#
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+require 'openssl'
+require 'base64'
+require 'time'
+require 'amazon/coral/handler'
+require 'amazon/coral/v1signaturehelper'
+
+module Amazon
+  module Coral
+
+    # Applies an AWS version 1 signature to the outgoing request.
+    class V1SignatureHandler < Handler
+
+      def before(job)
+        request = job.request
+        identity = request[:identity]
+        aws_access_key = identity[:aws_access_key]
+        aws_secret_key = identity[:aws_secret_key]
+
+        query_string_map = request[:query_string_map]
+
+        return if aws_access_key.nil? || aws_secret_key.nil? || query_string_map.nil?;
+
+        V1SignatureHelper.new(aws_access_key, aws_secret_key).sign({:query_string_map => query_string_map})
+      end
+
+    end
+
+  end
+end

data/lib/amazon/coral/v1signaturehelper.rb

@@ -0,0 +1,58 @@
+#
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+require 'openssl'
+require 'base64'
+require 'time'
+
+module Amazon
+  module Coral
+    class V1SignatureHelper 
+      def initialize(aws_access_key_id, aws_secret_key)
+        @aws_access_key_id = aws_access_key_id.to_s
+        @aws_secret_key = aws_secret_key.to_s
+      end
+      
+      def sign(args)
+        signT(Time.now.iso8601, args)
+      end
+      
+      def signT(time, args)
+        query_string_map = args[:query_string_map]
+        add_fields(query_string_map, time)
+        query_string_map['Signature'] = compute_signature(canonicalize(args))
+      end
+      
+      def canonicalize(args)
+        query_string_map = args[:query_string_map]
+        
+        # exclude any existing Signature parameter from the canonical string
+        sorted = sort(query_string_map.reject { |k, v| k == 'Signature' })
+        
+        canonical = ''
+        sorted.each do |v|
+          canonical << v[0]
+          canonical << v[1] unless(v[1].nil?)
+        end
+        
+        return canonical
+      end
+      
+      def compute_signature(canonical)
+        digest = OpenSSL::Digest::Digest.new('sha1')
+        return Base64.encode64(OpenSSL::HMAC.digest(digest, @aws_secret_key, canonical)).strip
+      end
+      
+      def add_fields(query_string_map, time)
+        query_string_map['AWSAccessKeyId'] = @aws_access_key_id
+        query_string_map['SignatureVersion'] = '1'
+        query_string_map['SignatureMethod'] = 'HmacSHA1'
+        query_string_map['Timestamp'] = time.to_s
+      end
+      
+      def sort(hash)
+        hash.sort { |a,b| a[0].downcase <=> b[0].downcase }
+      end
+    end
+  end
+end

data/lib/amazon/coral/v2signaturehandler.rb

@@ -0,0 +1,46 @@
+#
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+require 'openssl'
+require 'base64'
+require 'time'
+require 'amazon/coral/handler'
+require 'amazon/coral/v2signaturehelper'
+require 'amazon/coral/logfactory'
+
+module Amazon
+  module Coral
+    class V2SignatureHandler < Handler
+
+      def initialize
+        @log = LogFactory.getLog('Amazon::Coral::V2SignatureHandler')
+      end
+
+      def before(job)
+        request = job.request
+        identity = request[:identity]
+        aws_access_key = identity[:aws_access_key]
+        aws_secret_key = identity[:aws_secret_key]
+
+        query_string_map = request[:query_string_map]
+        http_uri = request[:http_uri]
+        uri = http_uri.path
+        verb = request[:http_verb]
+
+        host = "#{http_uri.host}"
+        host << ":#{http_uri.port}" unless http_uri.port.nil?
+
+        @log.debug("uri: #{uri} verb: #{verb} host: #{host}")
+
+        return if aws_access_key.nil? || aws_secret_key.nil? || query_string_map.nil? || 
+          uri.nil? || verb.nil? || host.nil?;
+
+        V2SignatureHelper.new(aws_access_key, aws_secret_key).sign({
+          :query_string_map => query_string_map, :uri => uri, :verb => verb, :host => host
+        })
+        
+        request[:http_host] = host
+      end
+    end
+  end
+end

data/lib/amazon/coral/v2signaturehelper.rb

@@ -0,0 +1,76 @@
+#
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+require 'openssl'
+require 'base64'
+require 'time'
+
+module Amazon
+  module Coral
+
+    # Performs AWS V2 signatures on QueryStringMap objects.
+    class V2SignatureHelper 
+      def initialize(aws_access_key_id, aws_secret_key)
+        @aws_access_key_id = aws_access_key_id.to_s
+        @aws_secret_key = aws_secret_key.to_s
+      end
+
+      def sign(args)
+        signT(Time.now.iso8601, args)
+      end
+
+      def signT(time, args)
+        query_string_map = args[:query_string_map]
+        add_fields(query_string_map, time)
+        query_string_map['Signature'] = compute_signature(canonicalize(args))
+      end
+
+      def canonicalize(args)
+        query_string_map = args[:query_string_map]
+        uri = args[:uri]
+        verb = args[:verb]
+        host = args[:host].downcase
+
+        # exclude any existing Signature parameter from the canonical string
+        sorted = sort(query_string_map.reject { |k, v| k == 'Signature' })
+        
+        canonical = "#{verb}\n#{host}\n#{uri}\n"
+        isFirst = true
+
+        sorted.each { |v|
+          if(isFirst) then
+            isFirst = false
+          else
+            canonical << '&'
+          end
+
+          canonical << UrlEncoding.encode(v[0])
+          unless(v[1].nil?) then
+            canonical << '='
+            canonical << UrlEncoding.encode(v[1])
+          end
+        }
+
+        return canonical
+      end
+
+      def compute_signature(canonical)
+        digest = OpenSSL::Digest::Digest.new('sha256')
+        return Base64.encode64(OpenSSL::HMAC.digest(digest, @aws_secret_key, canonical)).strip
+      end
+
+      def add_fields(query_string_map, time)
+        query_string_map['AWSAccessKeyId'] = @aws_access_key_id
+        query_string_map['SignatureVersion'] = '2'
+        query_string_map['SignatureMethod'] = 'HmacSHA256'
+        query_string_map['Timestamp'] = time.to_s
+      end
+
+      def sort(hash)
+        hash.sort
+      end
+
+    end
+
+  end
+end