elasticity 5.0.1 → 5.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 214e0a6224f1c25d0cab456008171d242f5b2cb6
-  data.tar.gz: 417acbc3a8b312eabea4a86d4db2b1b9f726e8fa
+  metadata.gz: fd9e39257ac81bf6cb0a18f38d680bc0e6bdc5f1
+  data.tar.gz: d6f3dd0970a81a164e548e52707b29419f3ce107
 SHA512:
-  metadata.gz: 856569d07a95865b11b3d54ebbbdad531b646200fd1df2eb609b75f408eb1be2d1b2eae0bc5625da1a222ebfed5c0e8b4eefc54dd36a4df0e66f0f43f290e5b3
-  data.tar.gz: c46610d0917cd5089dccc6aa8712007d536b3a3ed5f94acb8cc7328a4c07e218efe3000c26f32508a03dae53ca28b98eae9b8de2df8d65e2d363e8bb4ac0ea06
+  metadata.gz: b47d9a75474f3afb40a34c9f0f7bd846fa250e0e15551b3106cd92cd804a710e8c818ac2770b65f9c57e5760d985c5a641db668094dcb9c57f0c2a694866634e
+  data.tar.gz: c6d00853cf474fb771bde33fdb50828ec9923d9918d059b06409fdd6ab7709184bf71db78943df23df1b30a7710504b574ccb3687b0cdf7bbf3d6c961752a8e1

data/HISTORY.md

@@ -1,3 +1,8 @@
+## 5.0.2 - April 27, 2015
+
+- Fix for issue [#83](https://github.com/rslifka/elasticity/issues/83), `elasticity` has now transitioned to the AWS [Signature Version 4 Signing Process](http://docs.aws.amazon.com/general/latest/gr/signature-version-4.html).
+- Removed the ability to create insecure (HTTP) connections to the EMR endpoints.
+
 ## 5.0.1 - April 12, 2015
 
 - Bear with me here :)  Backmerged into 4.0.4 to add [IAM Service Role support](http://docs.aws.amazon.com/ElasticMapReduce/latest/DeveloperGuide/emr-iam-roles-creatingroles.html) per @alexanderdean.  As part of the forward merge, bumping the version to trigger an update.

data/lib/elasticity.rb

@@ -5,7 +5,12 @@ require 'rest_client'
 require 'nokogiri'
 require 'fog'
 
-require 'elasticity/aws_request'
+require 'elasticity/version'
+
+require 'elasticity/aws_utils'
+require 'elasticity/aws_session'
+require 'elasticity/aws_request_v2'
+require 'elasticity/aws_request_v4'
 require 'elasticity/emr'
 
 require 'elasticity/sync_to_s3'

data/lib/elasticity/aws_request_v2.rb

@@ -0,0 +1,42 @@
+module Elasticity
+
+  class AwsRequestV2
+
+    def initialize(aws_session, ruby_service_hash)
+      @aws_session = aws_session
+      @ruby_service_hash = ruby_service_hash
+    end
+
+    def url
+      "https://elasticmapreduce.#{@aws_session.region}.amazonaws.com"
+    end
+
+    def headers
+      {
+        :content_type => 'application/x-www-form-urlencoded; charset=utf-8'
+      }
+    end
+
+    # (Used from RightScale's right_aws gem.)
+    # EC2, SQS, SDB and EMR requests must be signed by this guy.
+    # See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
+    #      http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1928
+    def payload
+      service_hash = AwsUtils.convert_ruby_to_aws(@ruby_service_hash)
+      service_hash.merge!({
+          'AWSAccessKeyId' => @aws_session.access_key,
+          'Timestamp' => Time.now.utc.strftime('%Y-%m-%dT%H:%M:%S.000Z'),
+          'SignatureVersion' => '2',
+          'SignatureMethod' => 'HmacSHA256'
+        })
+      canonical_string = service_hash.keys.sort.map do |key|
+        "#{AwsUtils.aws_escape(key)}=#{AwsUtils.aws_escape(service_hash[key])}"
+      end.join('&')
+      string_to_sign = "POST\n#{@aws_session.host.downcase}\n/\n#{canonical_string}"
+      signature = AwsUtils.aws_escape(Base64.encode64(OpenSSL::HMAC.digest('sha256', @aws_session.secret_key, string_to_sign)).strip)
+      "#{canonical_string}&Signature=#{signature}"
+    end
+
+  end
+
+end

data/lib/elasticity/aws_request_v4.rb

@@ -0,0 +1,98 @@
+module Elasticity
+
+  # To help ensure correctness, Amazon has provided a step-by-step guide of
+  # query-and-response conversations for various types of API calls.
+  #
+  #   http://docs.aws.amazon.com/general/latest/gr/signature-v4-test-suite.html
+  #
+  # We are working with POSTs only, where the body of the POST contains the
+  # service details, so the 'post-x-www-form-urlencoded-parameters' suite is
+  # the most applicable.
+  class AwsRequestV4
+
+    SERVICE_NAME = 'elasticmapreduce'
+
+    def initialize(aws_session, ruby_service_hash)
+      @aws_session = aws_session
+
+      @ruby_service_hash = ruby_service_hash
+      @operation = @ruby_service_hash[:operation]
+      @ruby_service_hash.delete(:operation)
+
+      @timestamp = Time.now.utc
+    end
+
+    def headers
+      {
+        'Authorization' => "AWS4-HMAC-SHA256 Credential=#{@aws_session.access_key}/#{credential_scope}, SignedHeaders=content-type;host;user-agent;x-amz-content-sha256;x-amz-date;x-amz-target, Signature=#{aws_v4_signature}",
+        'Content-Type' => 'application/x-amz-json-1.1',
+        'Host' => host,
+        'User-Agent' => "elasticity/#{Elasticity::VERSION}",
+        'X-Amz-Content-SHA256' => Digest::SHA256.hexdigest(payload),
+        'X-Amz-Date' => @timestamp.strftime('%Y%m%dT%H%M%SZ'),
+        'X-Amz-Target' => "ElasticMapReduce.#{@operation}",
+      }
+    end
+
+    def url
+      "https://#{host}"
+    end
+
+    def payload
+      AwsUtils.convert_ruby_to_aws_v4(@ruby_service_hash).to_json
+    end
+
+    private
+
+    def host
+      "elasticmapreduce.#{@aws_session.region}.amazonaws.com"
+    end
+
+    def credential_scope
+      "#{@timestamp.strftime('%Y%m%d')}/#{@aws_session.region}/#{SERVICE_NAME}/aws4_request"
+    end
+
+    # Task 1: Create a Canonical Request For Signature Version 4
+    #   http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+    def canonical_request
+      [
+        'POST',
+        '/',
+        '',
+        'content-type:application/x-amz-json-1.1',
+        "host:#{host}",
+        "user-agent:elasticity/#{Elasticity::VERSION}",
+        "x-amz-content-sha256:#{Digest::SHA256.hexdigest(payload)}",
+        "x-amz-date:#{@timestamp.strftime('%Y%m%dT%H%M%SZ')}",
+        "x-amz-target:ElasticMapReduce.#{@operation}",
+        '',
+        'content-type;host;user-agent;x-amz-content-sha256;x-amz-date;x-amz-target',
+        Digest::SHA256.hexdigest(payload)
+      ].join("\n")
+    end
+
+    # Task 2: Create a String to Sign for Signature Version 4
+    #   http://docs.aws.amazon.com/general/latest/gr/sigv4-create-string-to-sign.html
+    def string_to_sign
+      [
+        'AWS4-HMAC-SHA256',
+        @timestamp.strftime('%Y%m%dT%H%M%SZ'),
+        credential_scope,
+        Digest::SHA256.hexdigest(canonical_request)
+      ].join("\n")
+    end
+
+    # Task 3: Calculate the AWS Signature Version 4
+    #   http://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
+    def aws_v4_signature
+      date = OpenSSL::HMAC.digest('sha256', 'AWS4' + @aws_session.secret_key, @timestamp.strftime('%Y%m%d'))
+      region = OpenSSL::HMAC.digest('sha256', date, @aws_session.region)
+      service = OpenSSL::HMAC.digest('sha256', region, SERVICE_NAME)
+      signing_key = OpenSSL::HMAC.digest('sha256', service, 'aws4_request')
+
+      OpenSSL::HMAC.hexdigest('sha256', signing_key, string_to_sign)
+    end
+
+  end
+
+end

data/lib/elasticity/aws_session.rb

@@ -0,0 +1,72 @@
+module Elasticity
+
+  class MissingKeyError < StandardError;
+  end
+  class MissingRegionError < StandardError;
+  end
+
+  class AwsSession
+
+    attr_reader :access_key
+    attr_reader :secret_key
+    attr_reader :host
+    attr_reader :region
+
+    # Supported values for options:
+    #  :region - AWS region (e.g. us-west-1)
+    #  :secure - true or false, default true.
+    def initialize(access=nil, secret=nil, options={})
+      # There is a cryptic error if this isn't set
+      if options.has_key?(:region) && options[:region] == nil
+        raise MissingRegionError, 'A valid :region is required to connect to EMR'
+      end
+      options[:region] = 'us-east-1' unless options[:region]
+      @region = options[:region]
+
+      @access_key = get_access_key(access)
+      @secret_key = get_secret_key(secret)
+      @host = "elasticmapreduce.#@region.amazonaws.com"
+    end
+
+    def submit(ruby_service_hash)
+      aws_request = AwsRequestV4.new(self, ruby_service_hash)
+      begin
+        RestClient.post(aws_request.url, aws_request.payload, aws_request.headers)
+      rescue RestClient::BadRequest => e
+        raise ArgumentError, AwsSession.parse_error_response(e.http_body)
+      end
+    end
+
+    def ==(other)
+      return false unless other.is_a? AwsSession
+      return false unless @access_key == other.access_key
+      return false unless @secret_key == other.secret_key
+      return false unless @host == other.host
+      true
+    end
+
+    private
+
+    def get_access_key(access)
+      return access if access
+      return ENV['AWS_ACCESS_KEY_ID'] if ENV['AWS_ACCESS_KEY_ID']
+      raise MissingKeyError, 'Please provide an access key or set AWS_ACCESS_KEY_ID.'
+    end
+
+    def get_secret_key(secret)
+      return secret if secret
+      return ENV['AWS_SECRET_ACCESS_KEY'] if ENV['AWS_SECRET_ACCESS_KEY']
+      raise MissingKeyError, 'Please provide a secret key or set AWS_SECRET_ACCESS_KEY.'
+    end
+
+    # AWS error responses all follow the same form.  Extract the message from
+    # the error document.
+    def self.parse_error_response(error_xml)
+      xml_doc = Nokogiri::XML(error_xml)
+      xml_doc.remove_namespaces!
+      xml_doc.xpath('/ErrorResponse/Error/Message').text
+    end
+
+  end
+
+end

data/lib/elasticity/aws_utils.rb

@@ -0,0 +1,65 @@
+module Elasticity
+
+  class AwsUtils
+
+    # Escape a string according to Amazon's rules.
+    # See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
+    def self.aws_escape(param)
+      param.to_s.gsub(/([^a-zA-Z0-9._~-]+)/n) do
+        '%' + $1.unpack('H2' * $1.size).join('%').upcase
+      end
+    end
+
+    # With the advent of v4 signing, we can skip the complex translation from v2
+    # and ship the JSON over with nearly the same structure.
+    def self.convert_ruby_to_aws_v4(value)
+      case value
+        when Array
+          return value.map{|v| convert_ruby_to_aws_v4(v)}
+        when Hash
+          result = {}
+          value.each do |k,v|
+            result[camelize(k.to_s)] = convert_ruby_to_aws_v4(v)
+          end
+          return result
+        else
+          return value
+      end
+    end
+
+    # Since we use the same structure as AWS, we can generate AWS param names
+    # from the Ruby versions of those names (and the param nesting).
+    def self.convert_ruby_to_aws(params)
+      result = {}
+      params.each do |key, value|
+        case value
+          when Array
+            prefix = "#{camelize(key.to_s)}.member"
+            value.each_with_index do |item, index|
+              if item.is_a?(String)
+                result["#{prefix}.#{index+1}"] = item
+              else
+                convert_ruby_to_aws(item).each do |nested_key, nested_value|
+                  result["#{prefix}.#{index+1}.#{nested_key}"] = nested_value
+                end
+              end
+            end
+          when Hash
+            prefix = "#{camelize(key.to_s)}"
+            convert_ruby_to_aws(value).each do |nested_key, nested_value|
+              result["#{prefix}.#{nested_key}"] = nested_value
+            end
+          else
+            result[camelize(key.to_s)] = value
+        end
+      end
+      result
+    end
+
+    def self.camelize(word)
+      word.to_s.gsub(/\/(.?)/) { '::' + $1.upcase }.gsub(/(^|_)(.)/) { $2.upcase }
+    end
+
+  end
+
+end

data/lib/elasticity/emr.rb

@@ -5,7 +5,7 @@ module Elasticity
     attr_reader :aws_request
 
     def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, options = {})
-      @aws_request = Elasticity::AwsRequest.new(aws_access_key_id, aws_secret_access_key, options)
+      @aws_request = Elasticity::AwsSession.new(aws_access_key_id, aws_secret_access_key, options)
     end
 
     # Describe a specific jobflow.

data/lib/elasticity/version.rb

@@ -1,3 +1,3 @@
 module Elasticity
-  VERSION = '5.0.1'
+  VERSION = '5.0.2'
 end

data/spec/lib/elasticity/aws_request_v2_spec.rb

@@ -0,0 +1,38 @@
+describe Elasticity::AwsRequestV2 do
+
+  before do
+    Timecop.freeze(Time.at(1302461096))
+  end
+
+  after do
+    Timecop.return
+  end
+
+  subject do
+    Elasticity::AwsRequestV2.new(
+      Elasticity::AwsSession.new('access', 'secret'),
+      {:operation => 'RunJobFlow', :name => 'Elasticity Job Flow'}
+    )
+  end
+
+  describe '#url' do
+    it 'should construct a proper endpoint' do
+      subject.url.should == 'https://elasticmapreduce.us-east-1.amazonaws.com'
+    end
+  end
+
+  describe '#headers' do
+    it 'should create the proper headers' do
+      subject.headers.should == {
+        :content_type => 'application/x-www-form-urlencoded; charset=utf-8'
+      }
+    end
+  end
+
+  describe '#payload' do
+    it 'should payload up the place' do
+      subject.payload.should == 'AWSAccessKeyId=access&Name=Elasticity%20Job%20Flow&Operation=RunJobFlow&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2011-04-10T18%3A44%3A56.000Z&Signature=5x6YilYHOjgM%2F6nalIOf62txOKoLFGBYyIivoHb%2F27k%3D'
+    end
+  end
+
+end

data/spec/lib/elasticity/aws_request_v4_spec.rb

@@ -0,0 +1,80 @@
+describe Elasticity::AwsRequestV4 do
+
+  before do
+    Timecop.freeze(Time.at(1315611360))
+  end
+
+  after do
+    Timecop.return
+  end
+
+  subject do
+    Elasticity::AwsRequestV4.new(
+      Elasticity::AwsSession.new('access', 'secret'),
+      {:operation => 'DescribeJobFlows', :job_flow_ids => ['TEST_JOBFLOW_ID']}
+    )
+  end
+
+  describe '#url' do
+    it 'should construct a proper endpoint' do
+      subject.url.should == 'https://elasticmapreduce.us-east-1.amazonaws.com'
+    end
+  end
+
+  describe '#headers' do
+    it 'should create the proper headers' do
+      subject.headers.should == {
+        'Authorization' => "AWS4-HMAC-SHA256 Credential=access/20110909/us-east-1/elasticmapreduce/aws4_request, SignedHeaders=content-type;host;user-agent;x-amz-content-sha256;x-amz-date;x-amz-target, Signature=#{subject.send(:aws_v4_signature)}",
+        'Content-Type' => 'application/x-amz-json-1.1',
+        'Host' => 'elasticmapreduce.us-east-1.amazonaws.com',
+        'User-Agent' => "elasticity/#{Elasticity::VERSION}",
+        'X-Amz-Content-SHA256' => Digest::SHA256.hexdigest(subject.payload),
+        'X-Amz-Date' => '20110909T233600Z',
+        'X-Amz-Target' => 'ElasticMapReduce.DescribeJobFlows',
+      }
+    end
+  end
+
+  describe '#payload' do
+    it 'should create the proper payload' do
+      subject.payload.should == '{"JobFlowIds":["TEST_JOBFLOW_ID"]}'
+    end
+  end
+
+  describe '.canonical_request' do
+    it 'should create the proper canonical request' do
+      subject.send(:canonical_request).should == [
+        'POST',
+        '/',
+        '',
+        'content-type:application/x-amz-json-1.1',
+        'host:elasticmapreduce.us-east-1.amazonaws.com',
+        "user-agent:elasticity/#{Elasticity::VERSION}",
+        "x-amz-content-sha256:#{Digest::SHA256.hexdigest(subject.payload)}",
+        'x-amz-date:20110909T233600Z',
+        'x-amz-target:ElasticMapReduce.DescribeJobFlows',
+        '',
+        'content-type;host;user-agent;x-amz-content-sha256;x-amz-date;x-amz-target',
+        "#{Digest::SHA256.hexdigest(subject.payload)}"
+      ].join("\n")
+    end
+  end
+
+  describe '.string_to_sign' do
+    it 'should create the proper string to sign' do
+      subject.send(:string_to_sign).should == [
+        'AWS4-HMAC-SHA256',
+        '20110909T233600Z',
+        '20110909/us-east-1/elasticmapreduce/aws4_request',
+        "#{Digest::SHA256.hexdigest(subject.send(:canonical_request))}"
+      ].join("\n")
+    end
+  end
+
+  describe '.aws_v4_signature' do
+    it 'should create the proper signature' do
+      subject.send(:aws_v4_signature).should == '3e88b95410e6828f80b4ec476bcf7e23ab8dd380b22ffcb1d5f7e86390346f68'
+    end
+  end
+
+end