elastic-mapreduce 0.0.1

data/lib/amazon/coral/elasticmapreduceclient.rb

@@ -0,0 +1,198 @@
+#
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+require 'amazon/coral/dispatcher'
+require 'amazon/coral/call'
+require 'amazon/coral/service'
+
+module Amazon
+  module Coral
+
+    # Client interface for calling ElasticMapReduce.
+    #
+    # The client supports two mechanisms to invoke each remote service call: a simple approach
+    # which directly calls the remote service, or a Call based mechanism that allows you to
+    # control aspects of the outgoing request such as request-id and identity attributes.
+    #
+    # Each instance of a client interface is backed by an Orchestrator object which manages
+    # the processing of each request to the remote service.
+    # Clients can be instantiated with a custom orchestrator or with presets corresponding to
+    # particular protocols.  Inputs and return values to the direct service-call methods and
+    # the Call.call methods are hashes.
+    class ElasticMapReduceClient
+      # Construct a new client.  Takes an orchestrator through which to process requests.
+      # See additional constructors below to use pre-configured orchestrators for specific protocols.
+      #
+      # [orchestrator]
+      #   The Orchestrator is responsible for actually making the remote service call.  Clients
+      #   construct requests, hand them off to the orchestrator, and receive responses in return.
+      def initialize(orchestrator)
+        @addJobFlowStepsDispatcher = Dispatcher.new(orchestrator, 'ElasticMapReduce', 'AddJobFlowSteps')
+        @addInstanceGroupsDispatcher = Dispatcher.new(orchestrator, 'ElasticMapReduce', 'AddInstanceGroups')
+        @terminateJobFlowsDispatcher = Dispatcher.new(orchestrator, 'ElasticMapReduce', 'TerminateJobFlows')
+        @setTerminationProtectionDispatcher = Dispatcher.new(orchestrator, 'ElasticMapReduce', 'SetTerminationProtection')
+        @describeJobFlowsDispatcher = Dispatcher.new(orchestrator, 'ElasticMapReduce', 'DescribeJobFlows')
+        @runJobFlowDispatcher = Dispatcher.new(orchestrator, 'ElasticMapReduce', 'RunJobFlow')
+        @modifyInstanceGroupsDispatcher = Dispatcher.new(orchestrator, 'ElasticMapReduce', 'ModifyInstanceGroups')
+      end
+
+
+      # Instantiates a call object to invoke the AddJobFlowSteps operation:
+      #
+      # Example usage:
+      #   my_call = my_client.newAddJobFlowStepsCall
+      #   # set identity information if needed
+      #   my_call.identity[:aws_access_key] = my_access_key
+      #   my_call.identity[:aws_secret_key] = my_secret_key
+      #   # make the remote call
+      #   my_call.call(my_input)
+      #   # retrieve the request-id returned by the server
+      #   my_request_id = my_call.request_id
+      def newAddJobFlowStepsCall
+        Call.new(@addJobFlowStepsDispatcher)
+      end
+
+      # Instantiates a call object to invoke the AddInstanceGroups operation:
+      #
+      # Example usage:
+      #   my_call = my_client.newAddInstanceGroupsCall
+      #   # set identity information if needed
+      #   my_call.identity[:aws_access_key] = my_access_key
+      #   my_call.identity[:aws_secret_key] = my_secret_key
+      #   # make the remote call
+      #   my_call.call(my_input)
+      #   # retrieve the request-id returned by the server
+      #   my_request_id = my_call.request_id
+      def newAddInstanceGroupsCall
+        Call.new(@addInstanceGroupsDispatcher)
+      end
+
+      # Instantiates a call object to invoke the TerminateJobFlows operation:
+      #
+      # Example usage:
+      #   my_call = my_client.newTerminateJobFlowsCall
+      #   # set identity information if needed
+      #   my_call.identity[:aws_access_key] = my_access_key
+      #   my_call.identity[:aws_secret_key] = my_secret_key
+      #   # make the remote call
+      #   my_call.call(my_input)
+      #   # retrieve the request-id returned by the server
+      #   my_request_id = my_call.request_id
+      def newTerminateJobFlowsCall
+        Call.new(@terminateJobFlowsDispatcher)
+      end
+
+      # Instantiates a call object to invoke the SetTerminationProtection operation:
+      def newSetTerminationProtectionCall
+        Call.new(@setTerminationProtectionDispatcher)
+      end
+
+      # Instantiates a call object to invoke the DescribeJobFlows operation:
+      #
+      # Example usage:
+      #   my_call = my_client.newDescribeJobFlowsCall
+      #   # set identity information if needed
+      #   my_call.identity[:aws_access_key] = my_access_key
+      #   my_call.identity[:aws_secret_key] = my_secret_key
+      #   # make the remote call
+      #   my_output = my_call.call(my_input)
+      #   # retrieve the request-id returned by the server
+      #   my_request_id = my_call.request_id
+      def newDescribeJobFlowsCall
+        Call.new(@describeJobFlowsDispatcher)
+      end
+
+      # Instantiates a call object to invoke the RunJobFlow operation:
+      #
+      # Example usage:
+      #   my_call = my_client.newRunJobFlowCall
+      #   # set identity information if needed
+      #   my_call.identity[:aws_access_key] = my_access_key
+      #   my_call.identity[:aws_secret_key] = my_secret_key
+      #   # make the remote call
+      #   my_output = my_call.call(my_input)
+      #   # retrieve the request-id returned by the server
+      #   my_request_id = my_call.request_id
+      def newRunJobFlowCall
+        Call.new(@runJobFlowDispatcher)
+      end
+
+      def newModifyInstanceGroupsCall
+        Call.new(@modifyInstanceGroupsDispatcher)
+      end
+
+
+      # Shorthand method to invoke the AddJobFlowSteps operation:
+      #
+      # Example usage:
+      #   my_client.AddJobFlowSteps(my_input)
+      def AddJobFlowSteps(input = {})
+        newAddJobFlowStepsCall.call(input)
+      end
+      
+      # Shorthand method to invoke the AddInstanceGroups operation:
+      #
+      # Example usage:
+      #   my_client.AddInstanceGroups(my_input)
+      def AddInstanceGroups(input = {})
+        newAddInstanceGroupsCall.call(input)
+      end
+
+      # Shorthand method to invoke the TerminateJobFlows operation:
+      #
+      # Example usage:
+      #   my_client.TerminateJobFlows(my_input)
+      def TerminateJobFlows(input = {})
+        newTerminateJobFlowsCall.call(input)
+      end
+
+      # Shorthand method to invoke the SetTerminationProtection operation:
+      def SetTerminationProtection(input = {})
+        newSetTerminationProtectionCall.call(input)
+      end
+
+      # Shorthand method to invoke the DescribeJobFlows operation:
+      #
+      # Example usage:
+      #   my_output = my_client.DescribeJobFlows(my_input)
+      def DescribeJobFlows(input = {})
+        newDescribeJobFlowsCall.call(input)
+      end
+
+      # Shorthand method to invoke the RunJobFlow operation:
+      #
+      # Example usage:
+      #   my_output = my_client.RunJobFlow(my_input)
+      def RunJobFlow(input = {})
+        newRunJobFlowCall.call(input)
+      end
+
+
+      def ModifyInstanceGroups(input = {})
+        newModifyInstanceGroupsCall.call(input)
+      end
+
+
+      # Instantiates the client with an orchestrator configured for use with AWS/QUERY.
+      # Use of this constructor is deprecated in favor of using the AwsQuery class:
+      #   client = ElasticMapReduceClient.new(AwsQuery.new_orchestrator(args))
+      def ElasticMapReduceClient.new_aws_query(args)
+        require 'amazon/coral/awsquery'
+        ElasticMapReduceClient.new(AwsQuery.new_orchestrator(args))
+      end
+
+    end
+
+    # allow running from the command line
+    Service.new(:service => 'ElasticMapReduce', :operations => [
+                                                                'AddJobFlowSteps',
+                                                                'AddInstanceGroups',
+                                                                'TerminateJobFlows',
+                                                                'SetTerminationProtection',
+                                                                'DescribeJobFlows',
+                                                                'RunJobFlow',
+                                                                'ModifyInstanceGroups'
+                                                               ]).main if caller.empty?
+  end
+end
+

data/lib/amazon/coral/handler.rb

@@ -0,0 +1,20 @@
+#
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+module Amazon
+  module Coral
+    
+    class Handler
+      
+      # Operate on the specified Job on the "outbound" side of the execution
+      def before(job)
+      end
+      
+      # Operation on the specified Job on the "inbound" side of the execution
+      def after(job)
+      end
+      
+    end
+    
+  end
+end

data/lib/amazon/coral/httpdelegationhelper.rb

@@ -0,0 +1,27 @@
+#
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+module Amazon
+  module Coral
+    
+    class HttpDelegationHelper
+      def self.add_delegation_token(delegate_identity, request_identity)
+        token = ""
+        first = true
+        
+        delegate_identity.each do |k,v|
+          if(first)
+            first = false
+          else
+            token << ';'
+          end
+          
+          token << "#{k}=#{v}"
+        end
+        
+        request_identity[:http_delegation] = token if(token.length > 0)
+      end
+    end
+    
+  end
+end

data/lib/amazon/coral/httpdestinationhandler.rb

@@ -0,0 +1,36 @@
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+require 'uri'
+require 'amazon/coral/handler'
+require 'amazon/coral/logfactory'
+
+module Amazon
+  module Coral
+
+    # Attaches the specified endpoint URI to the outgoing request.
+    class HttpDestinationHandler < Handler
+
+      # Initialize an HttpDestinationHandler with the specified endpoint
+      # URI.
+      def initialize(endpoint)
+        @log = LogFactory.getLog('Amazon::Coral::HttpDestinationHandler')
+
+        @uri = case endpoint
+               when URI
+                   endpoint
+               else
+                 URI.parse(endpoint)
+               end
+        @uri.path = '/' if @uri.path.nil? || @uri.path.empty?
+      end
+
+      def before(job)
+        job.request[:http_verb] = 'GET'
+        job.request[:http_uri] = @uri.clone
+
+        @log.debug "Initial request URI #{@uri}"
+      end
+    end
+
+  end
+end

data/lib/amazon/coral/httphandler.rb

@@ -0,0 +1,124 @@
+#
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+require 'uri'
+require 'net/http'
+require 'net/https'
+require 'amazon/coral/handler'
+require 'amazon/coral/logfactory'
+
+module Amazon
+  module Coral
+
+    # Executes HTTP requests via the Net::HTTP library.  Supports HTTP,
+    # HTTPS and client X509 certificates.
+    class HttpHandler < Handler
+      
+      # Instantiate a new HttpHandler with the specified arguments.  Possible arguments include:
+      # [:verbose]
+      #   If true, the handler will output the URI it is requesting to STDOUT.
+      #   This may be useful for debugging purposes.
+      # [:ca_file]
+      #   This parameter's value points to a valid .pem certificate file to enable the 
+      #   client to validate server certificates when using SSL.
+      #   If this parameter is not specified, the client operates in insecure mode and does not
+      #   validate that server certificates come from a trusted source.
+      # [:timeout]
+      #   This value (in seconds) will be used for every socket operation during the request.
+      #   Note that since a request can involve many socket operations, calls that timeout may 
+      #   actually take more time than this value.  If unspecified, defaults to 5.0 seconds.  
+      #   A value of zero will result in an infinite timeout.
+      # [:connect_timeout]
+      #   This value (in seconds) will be used as the timeout for opening a connection to the 
+      #   service.  If unspecified, defaults to 5.0 seconds.  A value of zero will result in
+      #   an infinite timeout.
+      def initialize(args = {})
+        @log = LogFactory.getLog('Amazon::Coral::HttpHandler')
+
+        @verbose = args[:verbose]
+        @ca_file = args[:ca_file]
+        @connect_timeout = args[:connect_timeout]
+        @timeout = args[:timeout]
+
+        @connect_timeout = 5.0 if @connect_timeout.nil?
+        @timeout = 5.0 if @timeout.nil?
+
+        raise ArgumentError, "connect_timeout must be non-negative" if @connect_timeout < 0
+        raise ArgumentError, "timeout must be non-negative" if @timeout < 0
+      end
+
+      def before(job)
+        identity = job.request[:identity]
+        request_id = job.request[:id]
+        uri = job.request[:http_uri]
+        verb = job.request[:http_verb]
+        query_map = job.request[:http_query_map]
+
+        verb = 'GET' if verb.nil?
+
+        headers = {}
+        headers['x-amzn-RequestId'] = "#{request_id}"
+        headers['x-amzn-Delegation'] = identity[:http_delegation] unless identity[:http_delegation].nil?
+        headers['Authorization'] = identity[:http_authorization] unless identity[:http_authorization].nil?
+        headers['Host'] = job.request[:http_host] unless job.request[:http_host].nil?
+        headers['User-Agent'] = 'ruby-client'
+        
+        result = http_request(uri, headers, verb, query_map, identity[:http_client_x509], identity[:http_client_x509_key])
+
+        @log.info "Response code: #{result.code}"
+
+        job.reply[:response] = result
+        job.reply[:value] = result.body
+        job.reply[:http_status_code] = result.code
+        job.reply[:http_status_message] = result.message
+        job.reply[:http_content] = nil # TODO: get content-type header
+      end
+
+      private
+      def http_request(uri, headers, verb, query_map = nil, cert = nil, key = nil)
+        if @verbose then
+          puts "Requesting URL:\n#{uri}\nQuery string:\n#{query_map}\nHeaders:\n#{headers}\n" 
+        end
+
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.read_timeout = @timeout;
+        http.open_timeout = @connect_timeout;
+
+        if(uri.scheme == 'https')
+          # enable SSL
+          http.use_ssl = true
+
+          # if we haven't been given CA certificates to check, disable certificate verification (otherwise we'll get repeated warnings to STDOUT)
+          if @ca_file.nil?
+            http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          else
+            http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+            http.ca_file = @ca_file
+          end
+
+          # negotiate with the client certificate, if one is present
+          unless(cert.nil? || key.nil?)
+            http.cert = OpenSSL::X509::Certificate.new(cert)
+            http.key = OpenSSL::PKey::RSA.new(key)
+          end
+        end
+
+        if verb == 'GET'
+          request = Net::HTTP::Get.new("#{uri.path}?#{uri.query}", headers)
+        elsif verb == 'POST'
+          request = Net::HTTP::Post.new("#{uri.path}?#{uri.query}", headers)
+          request.set_form_data(query_map)
+        else
+          raise "Unrecognized http_verb: #{http_verb}"
+        end
+
+        http.start { |http|
+          http.request(request)
+        }
+
+      end
+
+    end
+
+  end
+end

data/lib/amazon/coral/identityhandler.rb

@@ -0,0 +1,32 @@
+#
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+require 'amazon/coral/handler'
+
+module Amazon
+  module Coral
+
+    # Preloads identity information into request objects.  Useful if all
+    # requests through a client will utilize the same identity.
+    class IdentityHandler < Handler
+
+      # Instantiates an IdentityHandler with a hash of identity attributes
+      # to be contributed to the request's identity.  Attributes specified
+      # here will not override those explicitly associated with a request.
+      def initialize(attributes)
+        @attributes = attributes.to_hash
+      end
+
+      def before(job)
+        identity = job.request[:identity]
+
+        # for each shared identity attribute, set it on the request's
+        # identity IFF no attribute with that key is already present
+        @attributes.each { |k,v|
+          identity[k] = v unless identity.has_key?(k)
+        }
+      end
+    end
+  end
+end
+

data/lib/amazon/coral/job.rb

@@ -0,0 +1,25 @@
+#
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+module Amazon
+  module Coral
+
+    class Job
+      def initialize(request)
+        @request = request
+        @reply = {}
+      end
+
+      # Returns the hash of request attributes
+      def request
+        @request
+      end
+
+      # Returns the hash of reply attributes
+      def reply
+        @reply
+      end
+    end
+
+  end
+end

data/lib/amazon/coral/logfactory.rb

@@ -0,0 +1,35 @@
+#
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+require 'logger'
+
+module Amazon
+module Coral
+
+# A simple log retrieval interface to allow injection of common logging frameworks.
+class LogFactory
+
+  @@instance = LogFactory.new
+
+  # Invokes the singleton LogFactory instance to retrieve a logger for a given key.
+  def LogFactory.getLog(key)
+    return @@instance.getLog(key)
+  end
+
+  # Specifies a LogFactory instance which will handle log requests.
+  # Call this method early in execution prior to instantiating handlers to replace the default no-op log.
+  def LogFactory.setInstance(instance)
+    @@instance = instance
+  end
+
+  # Default logging implementation which returns a null logger.
+  def getLog(key)
+    log = Logger.new(nil)
+    log.level = Logger::FATAL
+    return log
+  end
+
+end
+
+end
+end

data/lib/amazon/coral/option.rb

@@ -0,0 +1,70 @@
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+module Amazon
+  module Coral
+
+    # A simple library for processing command line arguments.
+    class Option
+      def initialize(args)
+        @long = args[:long]
+        @short = args[:short]
+        @num_parameters = args[:parameters]
+        @description = args[:description]
+      end
+
+      # Returns the long form of this option's name
+      def long
+        @long
+      end
+      # Returns the short form of this option's name
+      def short
+        @short
+      end
+      # Returns a text description of this option, if present
+      def description
+        @description
+      end
+
+      # Consume the arguments of this option from the argument vector and store them in the provided hash
+      # Returns the incremented counter of current position within the argument vector.
+      def consume(argv, i, hash)
+        i = i + 1
+        hash[@long] = []
+        unless @num_parameters.nil?
+          @num_parameters.times do
+            raise "Option #{@long} requires #{@num_parameters} parameter(s)" if argv.length <= i
+            hash[@long] << argv[i]
+            i = i + 1
+          end
+        end
+
+        return i
+      end
+
+      # Using the provided list of arguments (defined as Option objects), parse the given argument vector.
+      def Option.parse(arguments, argv)
+        long_map = {}
+        short_map = {}
+        arguments.each { |p|
+          long_map["--#{p.long}"] = p unless p.long.nil?
+          short_map["-#{p.short}"] = p unless p.short.nil?
+        }
+
+
+        h = {}
+        i = 0
+        while i < argv.length
+          arg = argv[i]
+          a = long_map[arg]
+          a = short_map[arg] if a.nil?
+          raise "Unrecognized argument '#{arg}'" if a.nil?
+
+          i = a.consume(argv, i, h)
+        end
+
+        return h
+      end
+    end
+
+  end
+end

data/lib/amazon/coral/orchestrator.rb

@@ -0,0 +1,49 @@
+#
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+require 'amazon/coral/job'
+require 'amazon/coral/logfactory'
+
+module Amazon
+  module Coral
+    
+    # Directs a Job through a Handler chain for processing.
+    class Orchestrator
+      
+      # Instantiate an orchestrator with the given list of Handlers.
+      def initialize(handlers)
+        @log = LogFactory.getLog('Amazon::Coral::Orchestrator')
+        @handlers = handlers
+        
+        @log.info "Initialized with handlers: #{handlers}"
+      end
+      
+      # Direct the specified request down the Handler chain, invoking first each before method,
+      # then in reverse order each after method.  If any exceptions are thrown along the way, orchestration
+      # will stop immediately.
+      def orchestrate(request)
+        @log.debug "Processing request #{request}"
+        
+        job = Job.new(request)
+        
+        stack = []
+        
+        @handlers.each { |handler|
+          stack << handler
+          
+          @log.debug "Invoking #{handler}.before()"
+          handler.before(job)
+        }
+        
+        stack.reverse.each { |handler|
+          @log.debug "Invoking #{handler}.after()"
+          handler.after(job)
+        }
+        
+        return job.reply
+      end
+      
+    end
+    
+  end
+end