elasticity 1.5 → 2.0

data/Rakefile

@@ -4,8 +4,9 @@ Bundler::GemHelper.install_tasks
 require 'rake/testtask'
 require 'rspec/core/rake_task'
 
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.verbose = false
+end
+
 desc 'Run specs'
 task :default => :spec
-
-desc "Run specs"
-RSpec::Core::RakeTask.new

data/elasticity.gemspec

@@ -8,16 +8,14 @@ Gem::Specification.new do |s|
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Robert Slifka"]
   s.homepage    = "http://www.github.com/rslifka/elasticity"
-  s.summary     = %q{Programmatic access to Amazon's Elastic Map Reduce service.}
-  s.description = %q{Programmatic access to Amazon's Elastic Map Reduce service, driven by the Sharethrough team's requirements for belting out EMR jobs.}
+  s.summary     = %q{Streamlined, programmatic access to Amazon's Elastic Map Reduce service.}
+  s.description = %q{Streamlined, Programmatic access to Amazon's Elastic Map Reduce service, driven by the Sharethrough team's requirements for belting out EMR jobs.}
 
   s.add_dependency("rest-client")
   s.add_dependency("nokogiri")
 
   s.add_development_dependency("rake")
-  s.add_development_dependency("rspec",   ">= 2.8.0")
-  s.add_development_dependency("vcr",     "~> 2.0")
-  s.add_development_dependency("webmock", "~> 1.8.0")
+  s.add_development_dependency("rspec",   "~> 2.10.0")
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/elasticity.rb

@@ -4,16 +4,21 @@ require 'time'
 require 'rest_client'
 require 'nokogiri'
 
+require 'elasticity/support/conditional_raise'
+
 require 'elasticity/aws_request'
 require 'elasticity/emr'
-require 'elasticity/job_flow'
+
+require 'elasticity/hadoop_bootstrap_action'
 require 'elasticity/job_flow_step'
 
-require 'elasticity/simple_job'
+require 'elasticity/job_flow'
+require 'elasticity/job_flow_status'
+require 'elasticity/job_flow_status_step'
 
-require 'elasticity/custom_jar_job'
-require 'elasticity/hive_job'
-require 'elasticity/pig_job'
+require 'elasticity/custom_jar_step'
+require 'elasticity/hive_step'
+require 'elasticity/pig_step'
 
 module Elasticity
 end

data/lib/elasticity/aws_request.rb

@@ -2,52 +2,113 @@ module Elasticity
 
   class AwsRequest
 
+    attr_reader :access_key
+    attr_reader :secret_key
+    attr_reader :options
+    attr_reader :host
+    attr_reader :protocol
+
     # Supported values for options:
     #  :region - AWS region (e.g. us-west-1)
     #  :secure - true or false, default true.
-    def initialize(aws_access_key_id, aws_secret_access_key, options = {})
-      @access_key = aws_access_key_id
-      @secret_key = aws_secret_access_key
-      @options = {:secure => true}.merge(options)
+    def initialize(access, secret, options = {})
+      @access_key = access
+      @secret_key = secret
+      @host = options[:region] ? "elasticmapreduce.#{options[:region]}.amazonaws.com" : 'elasticmapreduce.amazonaws.com'
+      @protocol = {:secure => true}.merge(options)[:secure] ? 'https' : 'http'
     end
 
-    def aws_emr_request(params)
-      host = @options[:region] ? "elasticmapreduce.#{@options[:region]}.amazonaws.com" : "elasticmapreduce.amazonaws.com"
-      protocol = @options[:secure] ? "https" : "http"
+    def submit(ruby_params)
+      aws_params = AwsRequest.convert_ruby_to_aws(ruby_params)
+      signed_params = sign_params(aws_params)
+      begin
+        RestClient.post("#@protocol://#@host", signed_params, :content_type => 'application/x-www-form-urlencoded; charset=utf-8')
+      rescue RestClient::BadRequest => e
+        raise ArgumentError, AwsRequest.parse_error_response(e.http_body)
+      end
+    end
 
-      signed_params = sign_params(params, "GET", host, "/")
-      signed_request = "#{protocol}://#{host}?#{signed_params}"
-      RestClient.get signed_request
+    def ==(other)
+      return false unless other.is_a? AwsRequest
+      return false unless @access_key == other.access_key
+      return false unless @secret_key == other.secret_key
+      return false unless @options == other.options
+      true
     end
 
+    private
+
     # (Used from RightScale's right_aws gem.)
     # EC2, SQS, SDB and EMR requests must be signed by this guy.
     # See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
     #      http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1928
-    def sign_params(service_hash, http_verb, host, uri)
+    def sign_params(service_hash)
+      uri = '/' # TODO: Why are we hard-coding this?
       service_hash["AWSAccessKeyId"] = @access_key
       service_hash["Timestamp"] = Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z")
       service_hash["SignatureVersion"] = "2"
-      service_hash['SignatureMethod'] = 'HmacSHA256'
+      service_hash["SignatureMethod"] = "HmacSHA256"
       canonical_string = service_hash.keys.sort.map do |key|
         "#{AwsRequest.aws_escape(key)}=#{AwsRequest.aws_escape(service_hash[key])}"
       end.join('&')
-      string_to_sign = "#{http_verb.to_s.upcase}\n#{host.downcase}\n#{uri}\n#{canonical_string}"
+      string_to_sign = "POST\n#{@host.downcase}\n#{uri}\n#{canonical_string}"
       signature = AwsRequest.aws_escape(Base64.encode64(OpenSSL::HMAC.digest("sha256", @secret_key, string_to_sign)).strip)
       "#{canonical_string}&Signature=#{signature}"
     end
 
-    class << self
+    # (Used from RightScale's right_aws gem)
+    # Escape a string according to Amazon's rules.
+    # See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
+    def self.aws_escape(param)
+      param.to_s.gsub(/([^a-zA-Z0-9._~-]+)/n) do
+        '%' + $1.unpack('H2' * $1.size).join('%').upcase
+      end
+    end
 
-      # (Used from RightScale's right_aws gem)
-      # Escape a string according to Amazon's rules.
-      # See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
-      def aws_escape(param)
-        param.to_s.gsub(/([^a-zA-Z0-9._~-]+)/n) do
-          '%' + $1.unpack('H2' * $1.size).join('%').upcase
+    # Since we use the same structure as AWS, we can generate AWS param names
+    # from the Ruby versions of those names (and the param nesting).
+    def self.convert_ruby_to_aws(params)
+      result = {}
+      params.each do |key, value|
+        case value
+          when Array
+            prefix = "#{camelize(key.to_s)}.member"
+            value.each_with_index do |item, index|
+              if item.is_a?(String)
+                result["#{prefix}.#{index+1}"] = item
+              else
+                convert_ruby_to_aws(item).each do |nested_key, nested_value|
+                  result["#{prefix}.#{index+1}.#{nested_key}"] = nested_value
+                end
+              end
+            end
+          when Hash
+            prefix = "#{camelize(key.to_s)}"
+            convert_ruby_to_aws(value).each do |nested_key, nested_value|
+              result["#{prefix}.#{nested_key}"] = nested_value
+            end
+          else
+            result[camelize(key.to_s)] = value
         end
       end
+      result
+    end
+
+    # (Used from Rails' ActiveSupport)
+    def self.camelize(lower_case_and_underscored_word, first_letter_in_uppercase = true)
+      if first_letter_in_uppercase
+        lower_case_and_underscored_word.to_s.gsub(/\/(.?)/) { "::" + $1.upcase }.gsub(/(^|_)(.)/) { $2.upcase }
+      else
+        lower_case_and_underscored_word.first + camelize(lower_case_and_underscored_word)[1..-1]
+      end
+    end
 
+    # AWS error responses all follow the same form.  Extract the message from
+    # the error document.
+    def self.parse_error_response(error_xml)
+      xml_doc = Nokogiri::XML(error_xml)
+      xml_doc.remove_namespaces!
+      xml_doc.xpath("/ErrorResponse/Error/Message").text
     end
 
   end

data/lib/elasticity/custom_jar_step.rb

@@ -0,0 +1,33 @@
+module Elasticity
+
+  class CustomJarStep
+
+    include JobFlowStep
+
+    attr_accessor :name
+    attr_accessor :jar
+    attr_accessor :arguments
+    attr_accessor :action_on_failure
+
+    def initialize(jar)
+      @name = "Elasticity Custom Jar Step (#{jar})"
+      @jar = jar
+      @arguments = []
+      @action_on_failure = 'TERMINATE_JOB_FLOW'
+    end
+
+    def to_aws_step(job_flow)
+      step = {
+        :action_on_failure => @action_on_failure,
+        :hadoop_jar_step => {
+          :jar => @jar
+        },
+        :name => @name
+      }
+      step[:hadoop_jar_step][:args] = @arguments unless @arguments.empty?
+      step
+    end
+
+  end
+
+end

data/lib/elasticity/emr.rb

@@ -2,6 +2,8 @@ module Elasticity
 
   class EMR
 
+    attr_reader :aws_request
+
     def initialize(aws_access_key_id, aws_secret_access_key, options = {})
       @aws_request = Elasticity::AwsRequest.new(aws_access_key_id, aws_secret_access_key, options)
     end
@@ -12,18 +14,14 @@ module Elasticity
     #
     # Raises ArgumentError if the specified jobflow does not exist.
     def describe_jobflow(jobflow_id)
-      begin
-        aws_result = @aws_request.aws_emr_request(EMR.convert_ruby_to_aws({
-          :operation => "DescribeJobFlows",
-          :job_flow_ids => [jobflow_id]
-        }))
-        xml_doc = Nokogiri::XML(aws_result)
-        xml_doc.remove_namespaces!
-        yield aws_result if block_given?
-        JobFlow.from_members_nodeset(xml_doc.xpath("/DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member")).first
-      rescue RestClient::BadRequest => e
-        raise ArgumentError, EMR.parse_error_response(e.http_body)
-      end
+      aws_result = @aws_request.submit({
+        :operation => 'DescribeJobFlows',
+        :job_flow_ids => [jobflow_id]
+      })
+      xml_doc = Nokogiri::XML(aws_result)
+      xml_doc.remove_namespaces!
+      yield aws_result if block_given?
+      JobFlowStatus.from_members_nodeset(xml_doc.xpath('/DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member')).first
     end
 
     # Lists all jobflows in all states.
@@ -33,13 +31,13 @@ module Elasticity
     #
     #   describe_jobflows(:CreatedBefore => "2011-10-04")
     def describe_jobflows(params = {})
-      aws_result = @aws_request.aws_emr_request(EMR.convert_ruby_to_aws(
-        params.merge({:operation => "DescribeJobFlows"}))
+      aws_result = @aws_request.submit(
+        params.merge({:operation => 'DescribeJobFlows'})
       )
       xml_doc = Nokogiri::XML(aws_result)
       xml_doc.remove_namespaces!
       yield aws_result if block_given?
-      JobFlow.from_members_nodeset(xml_doc.xpath("/DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member"))
+      JobFlowStatus.from_members_nodeset(xml_doc.xpath('/DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member'))
     end
 
     # Adds a new group of instances to the specified jobflow.  Elasticity maps a
@@ -62,23 +60,19 @@ module Elasticity
     #   ["ig-2GOVEN6HVJZID", "ig-1DU9M2UQMM051", "ig-3DZRW4Y2X4S", ...]
     def add_instance_groups(jobflow_id, instance_group_configs)
       params = {
-        :operation => "AddInstanceGroups",
+        :operation => 'AddInstanceGroups',
         :job_flow_id => jobflow_id,
         :instance_groups => instance_group_configs
       }
-      begin
-        aws_result = @aws_request.aws_emr_request(EMR.convert_ruby_to_aws(params))
-        xml_doc = Nokogiri::XML(aws_result)
-        xml_doc.remove_namespaces!
-        instance_group_ids = []
-        xml_doc.xpath("/AddInstanceGroupsResponse/AddInstanceGroupsResult/InstanceGroupIds/member").each do |member|
-          instance_group_ids << member.text
-        end
-        yield aws_result if block_given?
-        instance_group_ids
-      rescue RestClient::BadRequest => e
-        raise ArgumentError, EMR.parse_error_response(e.http_body)
+      aws_result = @aws_request.submit(params)
+      xml_doc = Nokogiri::XML(aws_result)
+      xml_doc.remove_namespaces!
+      instance_group_ids = []
+      xml_doc.xpath('/AddInstanceGroupsResponse/AddInstanceGroupsResult/InstanceGroupIds/member').each do |member|
+        instance_group_ids << member.text
       end
+      yield aws_result if block_given?
+      instance_group_ids
     end
 
     # Add a step (or steps) to the specified job flow.
@@ -102,15 +96,11 @@ module Elasticity
     #   })
     def add_jobflow_steps(jobflow_id, steps_config)
       params = {
-        :operation => "AddJobFlowSteps",
+        :operation => 'AddJobFlowSteps',
         :job_flow_id => jobflow_id
       }.merge!(steps_config)
-      begin
-        aws_result = @aws_request.aws_emr_request(EMR.convert_ruby_to_aws(params))
-        yield aws_result if block_given?
-      rescue RestClient::BadRequest => e
-        raise ArgumentError, EMR.parse_error_response(e.http_body)
-      end
+      aws_result = @aws_request.submit(params)
+      yield aws_result if block_given?
     end
 
     # Set the number of instances in the specified instance groups to the
@@ -123,15 +113,11 @@ module Elasticity
     #   {"ig-1" => 40, "ig-2" => 5, ...}
     def modify_instance_groups(instance_group_config)
       params = {
-        :operation => "ModifyInstanceGroups",
+        :operation => 'ModifyInstanceGroups',
         :instance_groups => instance_group_config.map { |k, v| {:instance_group_id => k, :instance_count => v} }
       }
-      begin
-        aws_result = @aws_request.aws_emr_request(EMR.convert_ruby_to_aws(params))
-        yield aws_result if block_given?
-      rescue RestClient::BadRequest => e
-        raise ArgumentError, EMR.parse_error_response(e.http_body)
-      end
+      aws_result = @aws_request.submit(params)
+      yield aws_result if block_given?
     end
 
     # Start a job flow with the specified configuration.  This is a very thin
@@ -191,17 +177,13 @@ module Elasticity
     #   })
     def run_job_flow(job_flow_config)
       params = {
-        :operation => "RunJobFlow",
+        :operation => 'RunJobFlow',
       }.merge!(job_flow_config)
-      begin
-        aws_result = @aws_request.aws_emr_request(EMR.convert_ruby_to_aws(params))
-        yield aws_result if block_given?
-        xml_doc = Nokogiri::XML(aws_result)
-        xml_doc.remove_namespaces!
-        xml_doc.xpath("/RunJobFlowResponse/RunJobFlowResult/JobFlowId").text
-      rescue RestClient::BadRequest => e
-        raise ArgumentError, EMR.parse_error_response(e.http_body)
-      end
+      aws_result = @aws_request.submit(params)
+      yield aws_result if block_given?
+      xml_doc = Nokogiri::XML(aws_result)
+      xml_doc.remove_namespaces!
+      xml_doc.xpath('/RunJobFlowResponse/RunJobFlowResult/JobFlowId').text
     end
 
     # Enabled or disable "termination protection" on the specified job flows.
@@ -214,16 +196,12 @@ module Elasticity
     #   ["j-1B4D1XP0C0A35", "j-1YG2MYL0HVYS5", ...]
     def set_termination_protection(jobflow_ids, protection_enabled=true)
       params = {
-        :operation => "SetTerminationProtection",
+        :operation => 'SetTerminationProtection',
         :termination_protected => protection_enabled,
         :job_flow_ids => jobflow_ids
       }
-      begin
-        aws_result = @aws_request.aws_emr_request(EMR.convert_ruby_to_aws(params))
-        yield aws_result if block_given?
-      rescue RestClient::BadRequest => e
-        raise ArgumentError, EMR.parse_error_response(e.http_body)
-      end
+      aws_result = @aws_request.submit(params)
+      yield aws_result if block_given?
     end
 
     # Terminate the specified jobflow.  Amazon does not define a return value
@@ -232,74 +210,24 @@ module Elasticity
     # flow does not exist.
     def terminate_jobflows(jobflow_id)
       params = {
-        :operation => "TerminateJobFlows",
+        :operation => 'TerminateJobFlows',
         :job_flow_ids => [jobflow_id]
       }
-      begin
-        aws_result = @aws_request.aws_emr_request(EMR.convert_ruby_to_aws(params))
-        yield aws_result if block_given?
-      rescue RestClient::BadRequest
-        raise ArgumentError, "Job flow '#{jobflow_id}' does not exist."
-      end
+      aws_result = @aws_request.submit(params)
+      yield aws_result if block_given?
     end
 
     # Pass the specified params hash directly through to the AWS request URL.
     # Use this if you want to perform an operation that hasn't yet been wrapped
     # by Elasticity or you just want to see the response XML for yourself :)
     def direct(params)
-      @aws_request.aws_emr_request(params)
+      @aws_request.submit(params)
     end
 
-    private
-
-    class << self
-
-      # AWS error responses all follow the same form.  Extract the message from
-      # the error document.
-      def parse_error_response(error_xml)
-        xml_doc = Nokogiri::XML(error_xml)
-        xml_doc.remove_namespaces!
-        xml_doc.xpath("/ErrorResponse/Error/Message").text
-      end
-
-      # Since we use the same structure as AWS, we can generate AWS param names
-      # from the Ruby versions of those names (and the param nesting).
-      def convert_ruby_to_aws(params)
-        result = {}
-        params.each do |key, value|
-          case value
-            when Array
-              prefix = "#{camelize(key.to_s)}.member"
-              value.each_with_index do |item, index|
-                if item.is_a?(String)
-                  result["#{prefix}.#{index+1}"] = item
-                else
-                  convert_ruby_to_aws(item).each do |nested_key, nested_value|
-                    result["#{prefix}.#{index+1}.#{nested_key}"] = nested_value
-                  end
-                end
-              end
-            when Hash
-              prefix = "#{camelize(key.to_s)}"
-              convert_ruby_to_aws(value).each do |nested_key, nested_value|
-                result["#{prefix}.#{nested_key}"] = nested_value
-              end
-            else
-              result[camelize(key.to_s)] = value
-          end
-        end
-        result
-      end
-
-      # (Used from Rails' ActiveSupport)
-      def camelize(lower_case_and_underscored_word, first_letter_in_uppercase = true)
-        if first_letter_in_uppercase
-          lower_case_and_underscored_word.to_s.gsub(/\/(.?)/) { "::" + $1.upcase }.gsub(/(^|_)(.)/) { $2.upcase }
-        else
-          lower_case_and_underscored_word.first + camelize(lower_case_and_underscored_word)[1..-1]
-        end
-      end
-
+    def ==(other)
+      return false unless other.is_a? EMR
+      return false unless @aws_request == other.aws_request
+      true
     end
 
   end