elasticity 1.5 → 2.0

data/lib/elasticity/hadoop_bootstrap_action.rb

@@ -0,0 +1,27 @@
+module Elasticity
+
+  class HadoopBootstrapAction
+
+    attr_accessor :name
+    attr_accessor :option
+    attr_accessor :value
+
+    def initialize(option, value)
+      @name = 'Elasticity Bootstrap Action (Configure Hadoop)'
+      @option = option
+      @value = value
+    end
+
+    def to_aws_bootstrap_action
+      {
+        :name => @name,
+        :script_bootstrap_action => {
+          :path => 's3n://elasticmapreduce/bootstrap-actions/configure-hadoop',
+          :args => [@option, @value]
+        }
+      }
+    end
+
+  end
+
+end

data/lib/elasticity/hive_step.rb

@@ -0,0 +1,57 @@
+module Elasticity
+
+  class HiveStep
+
+    include JobFlowStep
+
+    attr_accessor :name
+    attr_accessor :script
+    attr_accessor :variables
+    attr_accessor :action_on_failure
+
+    def initialize(script)
+      @name = "Elasticity Hive Step (#{script})"
+      @script = script
+      @variables = { }
+      @action_on_failure = 'TERMINATE_JOB_FLOW'
+    end
+
+    def to_aws_step(job_flow)
+      args = %w(s3://elasticmapreduce/libs/hive/hive-script --run-hive-script --args)
+      args.concat(['-f', @script])
+      @variables.keys.sort.each do |name|
+        args.concat(['-d', "#{name}=#{@variables[name]}"])
+      end
+      {
+        :name => @name,
+        :action_on_failure => @action_on_failure,
+        :hadoop_jar_step => {
+          :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
+          :args => args
+        }
+      }
+    end
+
+    def self.requires_installation?
+      true
+    end
+
+    def self.aws_installation_step
+      {
+        :action_on_failure => 'TERMINATE_JOB_FLOW',
+        :hadoop_jar_step => {
+          :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
+          :args => [
+            's3://elasticmapreduce/libs/hive/hive-script',
+              '--base-path',
+              's3://elasticmapreduce/libs/hive/',
+              '--install-hive'
+          ],
+        },
+        :name => 'Elasticity - Install Hive'
+      }
+    end
+
+  end
+
+end

data/lib/elasticity/job_flow.rb

@@ -1,51 +1,121 @@
 module Elasticity
 
+  class JobFlowRunningError < StandardError; end
+  class JobFlowNotStartedError < StandardError; end
+  class JobFlowMissingStepsError < StandardError; end
+
   class JobFlow
 
+    attr_accessor :action_on_failure
+    attr_accessor :ec2_key_name
     attr_accessor :name
-    attr_accessor :jobflow_id
-    attr_accessor :state
-    attr_accessor :steps
-    attr_accessor :created_at
-    attr_accessor :started_at
-    attr_accessor :ready_at
+    attr_accessor :hadoop_version
     attr_accessor :instance_count
+    attr_accessor :log_uri
     attr_accessor :master_instance_type
     attr_accessor :slave_instance_type
-    attr_accessor :last_state_change_reason
-
-    def initialize
-      @steps = []
-    end
-
-    # Create a jobflow from an AWS <member> (Nokogiri::XML::Element):
-    #   /DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member
-    def self.from_member_element(xml_element)
-      jobflow = JobFlow.new
-      jobflow.name = xml_element.xpath("./Name").text.strip
-      jobflow.jobflow_id = xml_element.xpath("./JobFlowId").text.strip
-      jobflow.state = xml_element.xpath("./ExecutionStatusDetail/State").text.strip
-      jobflow.last_state_change_reason = xml_element.xpath("./ExecutionStatusDetail/LastStateChangeReason").text.strip
-      jobflow.steps = JobFlowStep.from_members_nodeset(xml_element.xpath("./Steps/member"))
-      jobflow.created_at = Time.parse(xml_element.xpath("./ExecutionStatusDetail/CreationDateTime").text.strip)
-      started_at = xml_element.xpath("./ExecutionStatusDetail/StartDateTime").text.strip
-      jobflow.started_at = (started_at == "") ? (nil) : (Time.parse(started_at))
-      ready_at = xml_element.xpath("./ExecutionStatusDetail/ReadyDateTime").text.strip
-      jobflow.ready_at = (ready_at == "") ? (nil) : (Time.parse(ready_at))
-      jobflow.instance_count = xml_element.xpath("./Instances/InstanceCount").text.strip
-      jobflow.master_instance_type = xml_element.xpath("./Instances/MasterInstanceType").text.strip
-      jobflow.slave_instance_type = xml_element.xpath("./Instances/SlaveInstanceType").text.strip
-      jobflow
-    end
-
-    # Create JobFlows from a collection of AWS <member> nodes (Nokogiri::XML::NodeSet):
-    #   /DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows
-    def self.from_members_nodeset(members_nodeset)
-      jobflows = []
-      members_nodeset.each do |member|
-        jobflows << from_member_element(member)
+    attr_accessor :ami_version
+    attr_accessor :keep_job_flow_alive_when_no_steps
+    attr_accessor :ec2_subnet_id
+
+    def initialize(access, secret)
+      @action_on_failure = 'TERMINATE_JOB_FLOW'
+      @ec2_key_name = 'default'
+      @hadoop_version = '0.20.205'
+      @instance_count = 2
+      @master_instance_type = 'm1.small'
+      @name = 'Elasticity Job Flow'
+      @slave_instance_type = 'm1.small'
+      @ami_version = 'latest'
+      @keep_job_flow_alive_when_no_steps = false
+
+      @emr = Elasticity::EMR.new(access, secret)
+
+      @bootstrap_actions = []
+      @jobflow_steps = []
+      @installed_steps = []
+    end
+
+    def instance_count=(count)
+      raise ArgumentError, 'Instance count cannot be set to less than 2 (requested 1)' unless count > 1
+      @instance_count = count
+    end
+
+    def add_bootstrap_action(bootstrap_action)
+      raise_if is_jobflow_running?, JobFlowRunningError, 'To modify bootstrap actions, please create a new job flow.'
+      @bootstrap_actions << bootstrap_action
+    end
+
+    def add_step(jobflow_step)
+      if is_jobflow_running?
+        jobflow_steps = []
+        if jobflow_step.class.send(:requires_installation?) && !@installed_steps.include?(jobflow_step.class)
+          jobflow_steps << jobflow_step.class.send(:aws_installation_step)
+        end
+        jobflow_steps << jobflow_step.to_aws_step(self)
+        @emr.add_jobflow_steps(@jobflow_id, {:steps => jobflow_steps})
+      else
+        @jobflow_steps << jobflow_step
+      end
+    end
+
+    def run
+      raise_if @jobflow_steps.empty?, JobFlowMissingStepsError, 'Cannot run a job flow without adding steps.  Please use #add_step.'
+      raise_if is_jobflow_running?, JobFlowRunningError, 'Cannot run a job flow multiple times.  To do more with this job flow, please use #add_step.'
+      @jobflow_id ||= @emr.run_job_flow(jobflow_config)
+    end
+
+    def shutdown
+      raise_unless is_jobflow_running?, JobFlowNotStartedError, 'Cannot #shutdown a job flow that has not yet been #run.'
+      @emr.terminate_jobflows(@jobflow_id)
+    end
+
+    def status
+      raise_unless is_jobflow_running?, JobFlowNotStartedError, 'Please #run this job flow before attempting to retrieve status.'
+      @emr.describe_jobflow(@jobflow_id)
+    end
+
+    private
+
+    def is_jobflow_running?
+      @jobflow_id
+    end
+
+    def jobflow_config
+      config = jobflow_preamble
+      config[:steps] = jobflow_steps
+      config[:log_uri] = @log_uri if @log_uri
+      config[:bootstrap_actions] = @bootstrap_actions.map{|a| a.to_aws_bootstrap_action} unless @bootstrap_actions.empty?
+      config
+    end
+
+    def jobflow_preamble
+      {
+        :name => @name,
+        :ami_version => @ami_version,
+        :instances => {
+          :keep_job_flow_alive_when_no_steps => @keep_job_flow_alive_when_no_steps,
+          :ec2_key_name => @ec2_key_name,
+          :hadoop_version => @hadoop_version,
+          :instance_count => @instance_count,
+          :master_instance_type => @master_instance_type,
+          :slave_instance_type => @slave_instance_type,
+        }
+      }.tap do |preamble|
+        preamble.merge!(:ec2_subnet_id => @ec2_subnet_id) if @ec2_subnet_id
+      end
+    end
+
+    def jobflow_steps
+      steps = []
+      @jobflow_steps.each do |step|
+        if step.class.send(:requires_installation?) && !@installed_steps.include?(step.class)
+          steps << step.class.send(:aws_installation_step)
+          @installed_steps << step.class
+        end
+        steps << step.to_aws_step(self)
       end
-      jobflows
+      steps
     end
 
   end

data/lib/elasticity/job_flow_status.rb

@@ -0,0 +1,53 @@
+module Elasticity
+
+  class JobFlowStatus
+
+    attr_accessor :name
+    attr_accessor :jobflow_id
+    attr_accessor :state
+    attr_accessor :steps
+    attr_accessor :created_at
+    attr_accessor :started_at
+    attr_accessor :ready_at
+    attr_accessor :instance_count
+    attr_accessor :master_instance_type
+    attr_accessor :slave_instance_type
+    attr_accessor :last_state_change_reason
+
+    def initialize
+      @steps = []
+    end
+
+    # Create a jobflow from an AWS <member> (Nokogiri::XML::Element):
+    #   /DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member
+    def self.from_member_element(xml_element)
+      jobflow = JobFlowStatus.new
+      jobflow.name = xml_element.xpath('./Name').text.strip
+      jobflow.jobflow_id = xml_element.xpath('./JobFlowId').text.strip
+      jobflow.state = xml_element.xpath('./ExecutionStatusDetail/State').text.strip
+      jobflow.last_state_change_reason = xml_element.xpath('./ExecutionStatusDetail/LastStateChangeReason').text.strip
+      jobflow.steps = JobFlowStatusStep.from_members_nodeset(xml_element.xpath('./Steps/member'))
+      jobflow.created_at = Time.parse(xml_element.xpath('./ExecutionStatusDetail/CreationDateTime').text.strip)
+      started_at = xml_element.xpath('./ExecutionStatusDetail/StartDateTime').text.strip
+      jobflow.started_at = (started_at == '') ? (nil) : (Time.parse(started_at))
+      ready_at = xml_element.xpath('./ExecutionStatusDetail/ReadyDateTime').text.strip
+      jobflow.ready_at = (ready_at == '') ? (nil) : (Time.parse(ready_at))
+      jobflow.instance_count = xml_element.xpath('./Instances/InstanceCount').text.strip
+      jobflow.master_instance_type = xml_element.xpath('./Instances/MasterInstanceType').text.strip
+      jobflow.slave_instance_type = xml_element.xpath('./Instances/SlaveInstanceType').text.strip
+      jobflow
+    end
+
+    # Create JobFlows from a collection of AWS <member> nodes (Nokogiri::XML::NodeSet):
+    #   /DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows
+    def self.from_members_nodeset(members_nodeset)
+      jobflows = []
+      members_nodeset.each do |member|
+        jobflows << from_member_element(member)
+      end
+      jobflows
+    end
+
+  end
+
+end

data/lib/elasticity/job_flow_status_step.rb

@@ -0,0 +1,35 @@
+module Elasticity
+
+  class JobFlowStatusStep
+
+    attr_accessor :name
+    attr_accessor :state
+    attr_accessor :started_at
+    attr_accessor :ended_at
+
+    # Create a job flow from an AWS <member> (Nokogiri::XML::Element):
+    #   /DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member/Steps/member
+    def self.from_member_element(xml_element)
+      job_flow_step = JobFlowStatusStep.new
+      job_flow_step.name = xml_element.xpath("./StepConfig/Name").text.strip
+      job_flow_step.state = xml_element.xpath("./ExecutionStatusDetail/State").text.strip
+      started_at = xml_element.xpath("./ExecutionStatusDetail/StartDateTime").text.strip
+      job_flow_step.started_at = (started_at == "") ? (nil) : (Time.parse(started_at))
+      ended_at = xml_element.xpath("./ExecutionStatusDetail/EndDateTime").text.strip
+      job_flow_step.ended_at = (ended_at == "") ? (nil) : (Time.parse(ended_at))
+      job_flow_step
+    end
+
+    # Create JobFlowSteps from a collection of AWS <member> nodes (Nokogiri::XML::NodeSet):
+    #   /DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member/Steps/member
+    def self.from_members_nodeset(members_nodeset)
+      jobflow_steps = []
+      members_nodeset.each do |member|
+        jobflow_steps << from_member_element(member)
+      end
+      jobflow_steps
+    end
+
+  end
+
+end

data/lib/elasticity/job_flow_step.rb

@@ -1,33 +1,25 @@
 module Elasticity
 
-  class JobFlowStep
-
-    attr_accessor :name
-    attr_accessor :state
-    attr_accessor :started_at
-    attr_accessor :ended_at
-
-    # Create a job flow from an AWS <member> (Nokogiri::XML::Element):
-    #   /DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member/Steps/member
-    def self.from_member_element(xml_element)
-      job_flow_step = JobFlowStep.new
-      job_flow_step.name = xml_element.xpath("./StepConfig/Name").text.strip
-      job_flow_step.state = xml_element.xpath("./ExecutionStatusDetail/State").text.strip
-      started_at = xml_element.xpath("./ExecutionStatusDetail/StartDateTime").text.strip
-      job_flow_step.started_at = (started_at == "") ? (nil) : (Time.parse(started_at))
-      ended_at = xml_element.xpath("./ExecutionStatusDetail/EndDateTime").text.strip
-      job_flow_step.ended_at = (ended_at == "") ? (nil) : (Time.parse(ended_at))
-      job_flow_step
+  module JobFlowStep
+
+    def to_aws_step(jobflow_step)
+      raise RuntimeError, '#to_aws_step is required to be defined on all job flow steps.'
     end
 
-    # Create JobFlowSteps from a collection of AWS <member> nodes (Nokogiri::XML::NodeSet):
-    #   /DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member/Steps/member
-    def self.from_members_nodeset(members_nodeset)
-      jobflow_steps = []
-      members_nodeset.each do |member|
-        jobflow_steps << from_member_element(member)
+    module ClassMethods
+
+      def requires_installation?
+        false
+      end
+
+      def aws_installation_step
+        raise RuntimeError, '.aws_installation_step is required to be defined when a step requires installation (e.g. Pig, Hive).'
       end
-      jobflow_steps
+
+    end
+
+    def self.included(base)
+      base.extend(ClassMethods)
     end
 
   end

data/lib/elasticity/pig_step.rb

@@ -0,0 +1,82 @@
+module Elasticity
+
+  class PigStep
+
+    include JobFlowStep
+
+    attr_accessor :name
+    attr_accessor :script
+    attr_accessor :variables
+    attr_accessor :action_on_failure
+
+    def initialize(script)
+      @name = "Elasticity Pig Step (#{script})"
+      @script = script
+      @variables = { }
+      @action_on_failure = 'TERMINATE_JOB_FLOW'
+    end
+
+    def to_aws_step(job_flow)
+      args = %w(s3://elasticmapreduce/libs/pig/pig-script --run-pig-script --args)
+      @variables.keys.sort.each do |name|
+        args.concat(['-p', "#{name}=#{@variables[name]}"])
+      end
+      args.concat(['-p', "E_PARALLELS=#{parallels(job_flow.slave_instance_type, job_flow.instance_count)}"])
+      args << @script
+      {
+        :action_on_failure => @action_on_failure,
+        :hadoop_jar_step => {
+          :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
+          :args => args,
+        },
+        :name => @name
+      }
+    end
+
+    def self.requires_installation?
+      true
+    end
+
+    def self.aws_installation_step
+      {
+        :action_on_failure => 'TERMINATE_JOB_FLOW',
+        :hadoop_jar_step => {
+          :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
+          :args => [
+            's3://elasticmapreduce/libs/pig/pig-script',
+              '--base-path',
+              's3://elasticmapreduce/libs/pig/',
+              '--install-pig'
+          ],
+        },
+        :name => 'Elasticity - Install Pig'
+      }
+    end
+
+    private
+
+    # Calculate a common-sense default value of PARALLELS using the following
+    # formula from the Pig Cookbook:
+    #
+    #   <num machines> * <num reduce slots per machine> * 0.9
+    #
+    # With the following reducer configuration (from an AWS forum post):
+    #
+    #   m1.small   1
+    #   m1.large   2
+    #   m1.xlarge  4
+    #   c1.medium  2
+    #   c1.xlarge  4
+    def parallels(slave_instance_type, instance_count)
+      reduce_slots = Hash.new(1)
+      reduce_slots['m1.small'] = 1
+      reduce_slots['m1.large'] = 2
+      reduce_slots['m1.xlarge'] = 4
+      reduce_slots['c1.medium'] = 2
+      reduce_slots['c1.xlarge'] = 4
+      ((instance_count - 1).to_f * reduce_slots[slave_instance_type].to_f * 0.9).ceil
+    end
+
+  end
+
+end