elasticity 5.0.3 → 6.0

data/lib/elasticity/emr.rb

@@ -4,49 +4,8 @@ module Elasticity
 
     attr_reader :aws_request
 
-    def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, options = {})
-      @aws_request = Elasticity::AwsSession.new(aws_access_key_id, aws_secret_access_key, options)
-    end
-
-    # Describe a specific jobflow.
-    #
-    #   describe_jobflow("j-3UN6WX5RRO2AG")
-    #
-    # Raises ArgumentError if the specified jobflow does not exist.
-    def describe_jobflow(jobflow_id)
-      aws_result = @aws_request.submit({
-        :operation => 'DescribeJobFlows',
-        :job_flow_ids => [jobflow_id]
-      })
-      xml_doc = Nokogiri::XML(aws_result)
-      xml_doc.remove_namespaces!
-      yield aws_result if block_given?
-      JobFlowStatus.from_members_nodeset(xml_doc.xpath('/DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member')).first
-    end
-
-    # This is primarily for debugging purposes, providing insight into how
-    # Amazon internally represents jobs.  It's used to reverse-engineer
-    # how API calls construct jobflows.
-    def describe_jobflow_xml(jobflow_id)
-      describe_jobflow(jobflow_id) do |xml|
-        return xml
-      end
-    end
-
-    # Lists all jobflows in all states.
-    #
-    # To override this behaviour, pass additional filters as specified in the AWS
-    # documentation - http://docs.amazonwebservices.com/ElasticMapReduce/latest/API/index.html?API_DescribeJobFlows.html.
-    #
-    #   describe_jobflows(:CreatedBefore => "2011-10-04")
-    def describe_jobflows(params = {})
-      aws_result = @aws_request.submit(
-        params.merge({:operation => 'DescribeJobFlows'})
-      )
-      xml_doc = Nokogiri::XML(aws_result)
-      xml_doc.remove_namespaces!
-      yield aws_result if block_given?
-      JobFlowStatus.from_members_nodeset(xml_doc.xpath('/DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member'))
+    def initialize(options = {})
+      @aws_request = Elasticity::AwsSession.new(options)
     end
 
     # Adds a new group of instances to the specified jobflow.  Elasticity maps a
@@ -56,11 +15,11 @@ module Elasticity
     #
     #   instance_group_config = {
     #     :bid_price => 5,
+    #     :market => "SPOT",
+    #     :name => "Go Canucks Go!",
     #     :instance_count => 1,
     #     :instance_role => "TASK",
-    #     :market => "SPOT",
-    #     :name => "Go Canucks Go!"
-    #     :type => "m1.small",
+    #     :instance_type => "m1.small"
     #   }
     #
     # add_instance_groups takes an array of {}.  Returns an array of the instance IDs
@@ -74,42 +33,179 @@ module Elasticity
         :instance_groups => instance_group_configs
       }
       aws_result = @aws_request.submit(params)
-      xml_doc = Nokogiri::XML(aws_result)
-      xml_doc.remove_namespaces!
-      instance_group_ids = []
-      xml_doc.xpath('/AddInstanceGroupsResponse/AddInstanceGroupsResult/InstanceGroupIds/member').each do |member|
-        instance_group_ids << member.text
-      end
       yield aws_result if block_given?
-      instance_group_ids
+      JSON.parse(aws_result)['InstanceGroupIds']
     end
 
     # Add a step (or steps) to the specified job flow.
     #
-    #   emr.add_jobflow_step("j-123", {
-    #     :steps => [
-    #       {
-    #         :action_on_failure => "TERMINATE_JOB_FLOW",
-    #         :hadoop_jar_step => {
-    #           :args => [
-    #             "s3://elasticmapreduce/libs/pig/pig-script",
-    #               "--base-path",
-    #               "s3://elasticmapreduce/libs/pig/",
-    #               "--install-pig"
-    #           ],
-    #           :jar => "s3://elasticmapreduce/libs/script-runner/script-runner.jar"
-    #         },
-    #         :name => "Setup Pig"
-    #       }
-    #     ]
-    #   })
+    #   emr.add_jobflow_step("j-123", [
+    #     {
+    #       :action_on_failure => "TERMINATE_JOB_FLOW",
+    #       :hadoop_jar_step => {
+    #         :args => [
+    #           "s3://elasticmapreduce/libs/pig/pig-script",
+    #             "--base-path",
+    #             "s3://elasticmapreduce/libs/pig/",
+    #             "--install-pig"
+    #         ],
+    #         :jar => "s3://elasticmapreduce/libs/script-runner/script-runner.jar"
+    #       },
+    #       :name => "Setup Pig"
+    #     }
+    #   ])
     def add_jobflow_steps(jobflow_id, steps_config)
       params = {
         :operation => 'AddJobFlowSteps',
-        :job_flow_id => jobflow_id
-      }.merge!(steps_config)
+        :job_flow_id => jobflow_id,
+        :steps => steps_config
+      }
+      aws_result = @aws_request.submit(params)
+      yield aws_result if block_given?
+      JSON.parse(aws_result)
+    end
+
+    # Sets the specified tags on all instances in the specified jobflow
+    #
+    #   emr.add_tags('j-123', [{:key => 'key1', :value => 'value1'}, {:key => 'key_only2'}])
+    #
+    # See http://docs.aws.amazon.com/ElasticMapReduce/latest/API/API_AddTags.html
+    def add_tags(jobflow_id, tags)
+      params = {
+        :operation => 'AddTags',
+        :resource_id => jobflow_id,
+        :tags => tags
+      }
+      aws_result = @aws_request.submit(params)
+      yield aws_result if block_given?
+    end
+
+    # Provides details about the specified jobflow
+    #
+    #   emr.describe_cluster('j-123')
+    #
+    # http://docs.aws.amazon.com/ElasticMapReduce/latest/API/API_DescribeCluster.html
+    def describe_cluster(jobflow_id)
+      params = {
+        :operation => 'DescribeCluster',
+        :cluster_id => jobflow_id,
+      }
+      aws_result = @aws_request.submit(params)
+      yield aws_result if block_given?
+      JSON.parse(aws_result)
+    end
+
+    # Provides details about the specified step within an existing jobflow
+    #
+    #   emr.describe_step('j-123', 'step-456')
+    #
+    # http://docs.aws.amazon.com/ElasticMapReduce/latest/API/API_DescribeStep.html
+    def describe_step(jobflow_id, step_id)
+      params = {
+        :operation => 'DescribeStep',
+        :cluster_id => jobflow_id,
+        :step_id => step_id
+      }
+      aws_result = @aws_request.submit(params)
+      yield aws_result if block_given?
+      JSON.parse(aws_result)
+    end
+
+    # List the clusters given specified filtering
+    #
+    #   emr.list_clusters({
+    #     :states => ['status1', 'status2', ...],
+    #     :created_before => Time,                # Amazon times are in UTC
+    #     :created_after => Time,                 # Amazon times are in UTC
+    #     :marker => 'marker'                     # Retrieve from a prior call to list_clusters
+    #   })
+    #
+    # http://docs.aws.amazon.com/ElasticMapReduce/latest/API/API_ListClusters.html
+    def list_clusters(options={})
+      params = {
+        :operation => 'ListClusters'
+      }
+      params.merge!(:cluster_states => options[:states]) if options[:states]
+      params.merge!(:created_before => options[:created_before].to_i) if options[:created_before]
+      params.merge!(:created_after => options[:created_after].to_i) if options[:created_after]
+      params.merge!(:marker => options[:marker]) if options[:marker]
+      aws_result = @aws_request.submit(params)
+      yield aws_result if block_given?
+      JSON.parse(aws_result)
+    end
+
+    # List the instances in a cluster given specified filtering
+    #
+    #   emr.list_instances('j-123', {
+    #     :types => ['MASTER', 'CORE', 'TASK'],
+    #     :marker => 'marker'                     # Retrieve from a prior call to list_clusters
+    #   })
+    #
+    # http://docs.aws.amazon.com/ElasticMapReduce/latest/API/API_ListInstances.html
+    def list_instances(jobflow_id, options={})
+      params = {
+        :operation => 'ListInstances',
+        :cluster_id => jobflow_id
+      }
+      params.merge!(:instance_group_id => options[:instance_group_id]) if options[:instance_group_id]
+      params.merge!(:instance_group_types => options[:instance_group_types]) if options[:instance_group_types]
+      params.merge!(:marker => options[:marker]) if options[:marker]
+      aws_result = @aws_request.submit(params)
+      yield aws_result if block_given?
+      JSON.parse(aws_result)
+    end
+
+    # List the instance groups in the specified jobflow
+    #
+    #   emr.list_instance_groups('j-123')
+    #
+    # http://docs.aws.amazon.com/ElasticMapReduce/latest/API/API_ListInstanceGroups.html
+    def list_instance_groups(jobflow_id)
+      params = {
+        :operation => 'ListInstanceGroups',
+        :cluster_id => jobflow_id,
+      }
+      aws_result = @aws_request.submit(params)
+      yield aws_result if block_given?
+      JSON.parse(aws_result)
+    end
+
+    # List the bootstrap actions in the specified jobflow
+    #
+    #   emr.list_bootstrap_actions('j-123')
+    #
+    # http://docs.aws.amazon.com/ElasticMapReduce/latest/API/API_ListBootstrapActions.html
+    def list_bootstrap_actions(jobflow_id)
+      params = {
+        :operation => 'ListBootstrapActions',
+        :cluster_id => jobflow_id,
+      }
+      aws_result = @aws_request.submit(params)
+      yield aws_result if block_given?
+      JSON.parse(aws_result)
+    end
+
+    # List the steps in a job flow given specified filtering
+    #
+    #   emr.list_steps('j-123', {
+    #     :types => ['MASTER', 'CORE', 'TASK'],
+    #     :step_ids => ['ID-1', 'ID-2']
+    #     :step_states => ['PENDING', 'RUNNING', ...]
+    #     :marker => 'marker'                         # Retrieve from a prior call to list_steps
+    #   })
+    #
+    # http://docs.aws.amazon.com/ElasticMapReduce/latest/API/API_ListSteps.html
+    def list_steps(jobflow_id, options={})
+      params = {
+        :operation => 'ListSteps',
+        :cluster_id => jobflow_id,
+      }
+      params.merge!(:step_ids => options[:step_ids]) if options[:step_ids]
+      params.merge!(:step_states => options[:step_states]) if options[:step_states]
+      params.merge!(:marker => options[:marker]) if options[:marker]
       aws_result = @aws_request.submit(params)
       yield aws_result if block_given?
+      JSON.parse(aws_result)
     end
 
     # Set the number of instances in the specified instance groups to the
@@ -129,6 +225,21 @@ module Elasticity
       yield aws_result if block_given?
     end
 
+    # Remove the specified tags on all instances in the specified jobflow
+    #
+    #   emr.remove_tags('j-123', ['key1','key_only2'])
+    #
+    # See http://docs.aws.amazon.com/ElasticMapReduce/latest/API/API_RemoveTags.html
+    def remove_tags(jobflow_id, keys)
+      params = {
+        :operation => 'RemoveTags',
+        :resource_id => jobflow_id,
+        :tag_keys => keys
+      }
+      aws_result = @aws_request.submit(params)
+      yield aws_result if block_given?
+    end
+
     # Start a job flow with the specified configuration.  This is a very thin
     # wrapper around the AWS API, so in order to use it directly you'll need
     # to have the PDF API reference handy, which can be found here:
@@ -211,14 +322,33 @@ module Elasticity
       yield aws_result if block_given?
     end
 
-    # Terminate the specified jobflow.  Amazon does not define a return value
-    # for this operation, so you'll need to poll #describe_jobflows to see
-    # the state of the jobflow.  Raises ArgumentError if the specified job
-    # flow does not exist.
-    def terminate_jobflows(jobflow_id)
+    # Whether or not all IAM users in this account can access the job flows.
+    #
+    # Takes an [] of job flow IDs.
+    #
+    #   ["j-1B4D1XP0C0A35", "j-1YG2MYL0HVYS5", ...]
+    #
+    # http://docs.aws.amazon.com/ElasticMapReduce/latest/API/API_SetVisibleToAllUsers.html
+    def set_visible_to_all_users(jobflow_ids, visible=true)
+      params = {
+        :operation => 'SetVisibleToAllUsers',
+        :visible_to_all_users => visible,
+        :job_flow_ids => jobflow_ids
+      }
+      aws_result = @aws_request.submit(params)
+      yield aws_result if block_given?
+    end
+
+    # Terminate the specified jobflows.  Amazon does not define a return value
+    # for this operation, so you'll need to poll to see the state of the jobflow.
+    #
+    # Takes an [] of job flow IDs.
+    #
+    #   ["j-1B4D1XP0C0A35", "j-1YG2MYL0HVYS5", ...]
+    def terminate_jobflows(jobflow_ids)
       params = {
         :operation => 'TerminateJobFlows',
-        :job_flow_ids => [jobflow_id]
+        :job_flow_ids => jobflow_ids
       }
       aws_result = @aws_request.submit(params)
       yield aws_result if block_given?

data/lib/elasticity/job_flow.rb

@@ -25,10 +25,7 @@ module Elasticity
     attr_accessor :job_flow_role
     attr_accessor :service_role
 
-    attr_reader :access_key
-    attr_reader :secret_key
-
-    def initialize(access=nil, secret=nil)
+    def initialize
       @action_on_failure = 'TERMINATE_JOB_FLOW'
       @name = 'Elasticity Job Flow'
       @ami_version = 'latest'
@@ -36,8 +33,6 @@ module Elasticity
       self.placement = 'us-east-1a'
       @enable_debugging = false
 
-      @access_key = access
-      @secret_key = secret
       @visible_to_all_users = false
 
       @bootstrap_actions = []
@@ -50,16 +45,13 @@ module Elasticity
       @instance_count = 2
       @master_instance_type = 'm1.small'
       @slave_instance_type = 'm1.small'
-
-      @access_key = access
-      @secret_key = secret
     end
 
-    def self.from_jobflow_id(access, secret, jobflow_id, region = 'us-east-1')
-      JobFlow.new(access, secret).tap do |j|
+    def self.from_jobflow_id(jobflow_id, region = 'us-east-1')
+      JobFlow.new.tap do |j|
         j.instance_variable_set(:@region, region)
         j.instance_variable_set(:@jobflow_id, jobflow_id)
-        j.instance_variable_set(:@installed_steps, j.status.installed_steps)
+        j.instance_variable_set(:@installed_steps, ClusterStepStatus.installed_steps(j.cluster_step_status))
       end
     end
 
@@ -154,11 +146,18 @@ module Elasticity
       emr.terminate_jobflows(@jobflow_id)
     end
 
-    def status
+    def cluster_status
+      if !is_jobflow_running?
+        raise JobFlowNotStartedError, 'Please #run this job flow before attempting to retrieve status.'
+      end
+      ClusterStatus.from_aws_data(emr.describe_cluster(@jobflow_id))
+    end
+
+    def cluster_step_status
       if !is_jobflow_running?
         raise JobFlowNotStartedError, 'Please #run this job flow before attempting to retrieve status.'
       end
-      emr.describe_jobflow(@jobflow_id)
+      ClusterStepStatus.from_aws_list_data(emr.list_steps(@jobflow_id))
     end
 
     def wait_for_completion(&on_wait)
@@ -169,12 +168,12 @@ module Elasticity
     private
 
     def retry_check
-      jf_status = status
-      return status.active?, jf_status
+      jf_status = cluster_status
+      return cluster_status.active?, jf_status
     end
 
     def emr
-      @emr ||= Elasticity::EMR.new(@access_key, @secret_key, :region => @region)
+      @emr ||= Elasticity::EMR.new(:region => @region)
     end
 
     def is_jobflow_running?

data/lib/elasticity/version.rb

@@ -1,3 +1,3 @@
 module Elasticity
-  VERSION = '5.0.3'
+  VERSION = '6.0'
 end

data/spec/factories/cluster_status_factory.rb

@@ -0,0 +1,12 @@
+FactoryGirl.define do
+  factory :cluster_status, class: Elasticity::ClusterStatus do
+    cluster_id 'CLUSTER_ID'
+    state 'TERMINATED'
+    created_at Time.at(1436788464.415)
+    ready_at Time.at(1436788842.195)
+    ended_at Time.at(1436791032.097)
+    last_state_change_reason 'ALL_STEPS_COMPLETED'
+    master_public_dns_name 'ec2-54-81-173-103.compute-1.amazonaws.com'
+    normalized_instance_hours 999
+  end
+end

data/spec/factories/cluster_step_status_factory.rb

@@ -0,0 +1,17 @@
+FactoryGirl.define do
+  factory :cluster_step_status, class: Elasticity::ClusterStepStatus do 
+    action_on_failure 'TERMINATE_CLUSTER'
+    args ['36', '3', '0',]
+    jar 's3n://elasticmapreduce/samples/cloudburst/cloudburst.jar'
+    main_class 'MAIN_CLASS'
+    step_id 's-OYPPAC4XPPUC'
+    properties 'Key1' => 'Value1', 'Key2' => 'Value2'
+    name 'Elasticity Custom Jar Step'
+    state 'COMPLETED'
+    state_change_reason 'ALL_STEPS_COMPLETED'
+    state_change_reason_message 'Steps completed'
+    created_at Time.at(1436788464.416)
+    started_at Time.at(1436788841.237)
+    ended_at Time.at(1436790944.162)
+  end
+end

data/spec/lib/elasticity/aws_request_v4_spec.rb

@@ -2,6 +2,11 @@ describe Elasticity::AwsRequestV4 do
 
   before do
     Timecop.freeze(Time.at(1315611360))
+
+    Elasticity.configure do |c|
+      c.access_key = 'access'
+      c.secret_key = 'secret'
+    end
   end
 
   after do
@@ -10,11 +15,41 @@ describe Elasticity::AwsRequestV4 do
 
   subject do
     Elasticity::AwsRequestV4.new(
-      Elasticity::AwsSession.new('access', 'secret'),
+      Elasticity::AwsSession.new,
       {:operation => 'DescribeJobFlows', :job_flow_ids => ['TEST_JOBFLOW_ID']}
     )
   end
 
+  describe '.initialize' do
+
+    describe 'access key' do
+      context 'when not provided' do
+        it 'should be an error' do
+          Elasticity.configure do |c|
+            c.access_key = nil
+          end
+          expect {
+            Elasticity::AwsRequestV4.new(nil, {})
+          }.to raise_error(ArgumentError, '.access_key must be set in the configuration block')
+        end
+      end
+    end
+
+    describe 'secret key' do
+      context 'when not provided' do
+        it 'should be an error' do
+          Elasticity.configure do |c|
+            c.secret_key = nil
+          end
+          expect {
+            Elasticity::AwsRequestV4.new(nil, {})
+          }.to raise_error(ArgumentError, '.secret_key must be set in the configuration block')
+        end
+      end
+    end
+
+  end
+
   describe '#url' do
     it 'should construct a proper endpoint' do
       subject.url.should == 'https://elasticmapreduce.us-east-1.amazonaws.com'
@@ -22,16 +57,31 @@ describe Elasticity::AwsRequestV4 do
   end
 
   describe '#headers' do
-    it 'should create the proper headers' do
-      subject.headers.should == {
+
+    let(:base_headers) {
+      {
         'Authorization' => "AWS4-HMAC-SHA256 Credential=access/20110909/us-east-1/elasticmapreduce/aws4_request, SignedHeaders=content-type;host;user-agent;x-amz-content-sha256;x-amz-date;x-amz-target, Signature=#{subject.send(:aws_v4_signature)}",
         'Content-Type' => 'application/x-amz-json-1.1',
         'Host' => 'elasticmapreduce.us-east-1.amazonaws.com',
         'User-Agent' => "elasticity/#{Elasticity::VERSION}",
         'X-Amz-Content-SHA256' => Digest::SHA256.hexdigest(subject.payload),
         'X-Amz-Date' => '20110909T233600Z',
-        'X-Amz-Target' => 'ElasticMapReduce.DescribeJobFlows',
+        'X-Amz-Target' => 'ElasticMapReduce.DescribeJobFlows'
       }
+    }
+
+    context 'when a security token is specified' do
+      it 'should create the proper headers' do
+        Elasticity.configure {|c| c.security_token = 'SECURITY_TOKEN' }
+        subject.headers.should == base_headers.merge('X-Amz-Security-Token' => 'SECURITY_TOKEN')
+      end
+    end
+
+    context 'when a security token is not specified' do
+      it 'should create the proper headers' do
+        Elasticity.configure {|c| c.security_token = nil }
+        subject.headers.should == base_headers
+      end
     end
   end