elastic-mapreduce 0.0.1

data/lib/amazon/retry_delegator.rb

@@ -0,0 +1,66 @@
+#
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+require 'amazon/stderr_logger.rb'
+
+module Amazon
+
+  # RetryDelegator
+  #   this is a wrapper around a client that will retry if exceptions are raised.
+  class RetryDelegator
+    def initialize(client, options={})
+      @client            = client
+      @log               = options[:log] || StdErrLogger.new
+      @backoff_seconds   = options[:backoff_seconds] || 2
+      @backoff_mult      = options[:backoff_mult] || 1.5
+      @retries           = options[:retries] || 8
+      @retry_if          = options[:retry_if]
+      @pass_exceptions   = options[:pass_exceptions] || [ScriptError, SignalException, ArgumentError, StandardError]
+      @retry_exceptions  = options[:retry_exceptions] || [IOError, EOFError, RuntimeError]
+    end
+
+    def is_retry_exception(e)
+      if @retry_exceptions then
+        for retry_exception in @retry_exceptions do
+          return true if e.is_a?(retry_exception)
+        end
+      end
+      if @pass_exceptions then
+        for pass_exception in @pass_exceptions do
+          return false if e.is_a?(pass_exception)
+        end
+        return true
+      else
+        return false
+      end
+    end
+
+    def method_missing(method, *args)
+      backoff_seconds = @backoff_seconds
+      backoff_mult = @backoff_mult
+      retries_remaining = @retries
+      begin
+        response = @client.send(method, *args)
+        if @retry_if && @retry_if.call(response) then
+          raise "Retriable invalid response returned from #{method}: #{response.inspect}"
+        end
+        return response
+      rescue Exception => e
+        if retries_remaining > 0 && is_retry_exception(e) then
+          if @log != nil then
+            @log.info "Exception #{e} while calling #{method} on #{@client.class}, retrying in #{@backoff_seconds * backoff_mult} seconds."
+          end
+          sleep(@backoff_seconds * backoff_mult)
+          backoff_mult *= 2
+          retries_remaining -= 1
+          retry
+        else
+          if @log != nil then
+            @log.info "Exception #{e} while calling #{method} on #{@client.class}, failing"
+          end
+          raise e
+        end
+      end
+    end
+  end
+end

data/lib/amazon/stderr_logger.rb

@@ -0,0 +1,23 @@
+#
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+module Amazon
+  class StdErrLogger
+    INFO = { :level => 4, :string => "INFO" }
+
+    def initialize(level=nil)
+      @level = level || INFO[:level]
+      @file  = STDERR
+    end
+
+    def message(level, msg)
+      if level[:level] <= @level then
+        @file.puts(level[:string] + " " + msg)
+      end
+    end
+
+    def info(*args)
+      message(INFO, *args)
+    end
+  end
+end

data/lib/client.rb

@@ -0,0 +1,117 @@
+#
+# Copyright 2008-2010 Amazon.com, Inc. or its affiliates.  All Rights Reserved.
+
+require 'credentials'
+require 'amazon/retry_delegator'
+require 'amazon/coral/elasticmapreduceclient'
+
+class EmrClient
+  attr_accessor :commands, :logger, :options
+
+  def initialize(commands, logger, client_class)
+    @commands = commands
+    @logger = logger
+    @options = commands.global_options
+
+    @config = {
+      :endpoint            => @options[:endpoint] || "https://elasticmapreduce.amazonaws.com",
+      :ca_file             => File.join(File.dirname(__FILE__), "cacert.pem"),
+      :aws_access_key      => @options[:aws_access_id],
+      :aws_secret_key      => @options[:aws_secret_key],
+      :signature_algorithm => :V2,
+      :content_type        => 'JSON',
+      :verbose             => (@options[:verbose] != nil)
+    }
+
+    @client = Amazon::RetryDelegator.new(
+      client_class.new_aws_query(@config),
+      :retry_if => Proc.new { |*opts| self.is_retryable_error_response(*opts) }
+    )
+  end
+
+  def is_retryable_error_response(response)
+    if response == nil then
+      false
+    else
+      ret = false
+      if response['Error'] then 
+        # note: 'Timeout' is not retryable because the operation might have completed just the connection timed out
+        ret ||= ['InternalFailure', 'Throttling', 'ServiceUnavailable'].include?(response['Error']['Code'])
+      end
+      ret 
+    end
+  end
+
+  def is_error_response(response)
+    response != nil && response.key?('Error')
+  end
+
+  def raise_on_error(response)
+    if is_error_response(response) then
+      raise RuntimeError, response["Error"]["Message"]
+    end
+    return response
+  end
+
+  def describe_jobflow_with_id(jobflow_id)
+    logger.trace "DescribeJobFlows('JobFlowIds' => [ #{jobflow_id} ])"
+    result = @client.DescribeJobFlows('JobFlowIds' => [ jobflow_id ], 'DescriptionType' => 'EXTENDED')
+    logger.trace result.inspect
+    raise_on_error(result)
+    if result == nil || result['JobFlows'].size() == 0 then
+      raise RuntimeError, "Jobflow with id #{jobflow_id} not found"
+    end
+    return result['JobFlows'].first
+  end
+
+  def add_steps(jobflow_id, steps)
+    logger.trace "AddJobFlowSteps('JobFlowId' => #{jobflow_id.inspect}, 'Steps' => #{steps.inspect})"
+    result = @client.AddJobFlowSteps('JobFlowId' => jobflow_id, 'Steps' => steps)
+    logger.trace result.inspect
+    return raise_on_error(result)
+  end
+
+  def run_jobflow(jobflow)
+    logger.trace "RunJobFlow(#{jobflow.inspect})"
+    result = @client.RunJobFlow(jobflow)
+    logger.trace result.inspect
+    return raise_on_error(result)
+  end
+
+  def describe_jobflow(options)
+    logger.trace "DescribeJobFlows(#{options.inspect})"
+    result = @client.DescribeJobFlows(options.merge('DescriptionType' => 'EXTENDED'))
+    logger.trace result.inspect
+    return raise_on_error(result)
+  end
+
+  def set_termination_protection(jobflow_ids, protected)
+    logger.trace "SetTerminationProtection('JobFlowIds' => #{jobflow_ids.inspect}, 'TerminationProtected' => #{protected})"
+    result = @client.SetTerminationProtection('JobFlowIds' => jobflow_ids, 'TerminationProtected' => protected)
+    logger.trace result.inspect
+    return raise_on_error(result)
+  end
+
+  def terminate_jobflows(jobflow_ids)
+    logger.trace "TerminateJobFlows('JobFlowIds' => #{jobflow_ids.inspect})"
+    result = @client.TerminateJobFlows('JobFlowIds' => jobflow_ids)
+    logger.trace result.inspect
+    return raise_on_error(result)
+  end
+
+  def modify_instance_groups(options)
+    logger.trace "ModifyInstanceGroups(#{options.inspect})"
+    result = @client.ModifyInstanceGroups(options)
+    logger.trace result.inspect
+    return raise_on_error(result)
+  end    
+
+  def add_instance_groups(options)
+    logger.trace "AddInstanceGroups(#{options.inspect})"
+    result = @client.AddInstanceGroups(options)
+    logger.trace result.inspect
+    return raise_on_error(result)
+  end    
+
+end
+