elasticity 2.1 → 2.1.1

data/Gemfile

@@ -1,4 +1,4 @@
-source "http://rubygems.org"
+source 'http://rubygems.org'
 
 # Specify your gem's dependencies in elasticity.gemspec
 gemspec

data/HISTORY.md

@@ -1,3 +1,8 @@
+## 2.1.1 - July 22, 2012
+
++ ```JobFlow::from_jobflow_id``` factory method added so that you can operate on running job flows (add steps, shutdown, status, etc.) that you didn't start in the same Ruby instance.
++ Updated to rspec-2.11.
+
 ## 2.1 - July 7, 2012
 
 + TASK instance group support added.

data/README.md

@@ -59,7 +59,7 @@ Job flows are the center of the EMR universe.  The general order of operations i
   1. (optional) Add additional steps.
   1. (optional) Shutdown the job flow.
 
-## 1 - Creating Job Flows
+## 1 - Create a Job Flow
 
 Only your AWS credentials are needed.
 
@@ -67,6 +67,14 @@ Only your AWS credentials are needed.
 jobflow = Elasticity::JobFlow.new('AWS access key', 'AWS secret key')
 ```
 
+If you want to access a job flow that's already running:
+
+```
+jobflow = Elasticity::JobFlow.from_jobflow_id('AWS access key', 'AWS secret key', 'jobflow ID')
+```
+
+This is useful if you'd like to attach to a running job flow and add more steps, etc.
+
 ## 2 - Specifying Job Flow Options
 
 Configuration job flow options, shown below with default values.  Note that these defaults are subject to change - they are reasonable defaults at the time(s) I work on them (e.g. the latest version of Hadoop).

data/elasticity.gemspec

@@ -1,24 +1,24 @@
 # -*- encoding: utf-8 -*-
-$:.push File.expand_path("../lib", __FILE__)
-require "elasticity/version"
+$:.push File.expand_path('../lib', __FILE__)
+require 'elasticity/version'
 
 Gem::Specification.new do |s|
-  s.name        = "elasticity"
+  s.name        = 'elasticity'
   s.version     = Elasticity::VERSION
   s.platform    = Gem::Platform::RUBY
-  s.authors     = ["Robert Slifka"]
-  s.homepage    = "http://www.github.com/rslifka/elasticity"
+  s.authors     = ['Robert Slifka']
+  s.homepage    = 'http://www.github.com/rslifka/elasticity'
   s.summary     = %q{Streamlined, programmatic access to Amazon's Elastic Map Reduce service.}
   s.description = %q{Streamlined, Programmatic access to Amazon's Elastic Map Reduce service, driven by the Sharethrough team's requirements for belting out EMR jobs.}
 
-  s.add_dependency("rest-client")
-  s.add_dependency("nokogiri")
+  s.add_dependency('rest-client')
+  s.add_dependency('nokogiri')
 
-  s.add_development_dependency("rake")
-  s.add_development_dependency("rspec",   "~> 2.10.0")
+  s.add_development_dependency('rake')
+  s.add_development_dependency('rspec', '~> 2.11.0')
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
-  s.require_paths = ["lib"]
+  s.require_paths = %w(lib)
 end

data/lib/elasticity/job_flow.rb

@@ -41,6 +41,13 @@ module Elasticity
       @emr = Elasticity::EMR.new(access, secret)
     end
 
+    def self.from_jobflow_id(access, secret, jobflow_id)
+      JobFlow.new(access, secret).tap do |j|
+        j.instance_variable_set(:@jobflow_id, jobflow_id)
+        j.instance_variable_set(:@installed_steps, j.status.installed_steps)
+      end
+    end
+
     def instance_count=(count)
       raise ArgumentError, 'Instance count cannot be set to less than 2 (requested 1)' unless count > 1
       @instance_groups[:core].count = count - 1
@@ -80,8 +87,8 @@ module Elasticity
     def add_step(jobflow_step)
       if is_jobflow_running?
         jobflow_steps = []
-        if jobflow_step.class.send(:requires_installation?) && !@installed_steps.include?(jobflow_step.class)
-          jobflow_steps << jobflow_step.class.send(:aws_installation_step)
+        if jobflow_step.requires_installation? && !@installed_steps.include?(jobflow_step.class)
+          jobflow_steps << jobflow_step.aws_installation_step
         end
         jobflow_steps << jobflow_step.to_aws_step(self)
         @emr.add_jobflow_steps(@jobflow_id, {:steps => jobflow_steps})
@@ -93,7 +100,7 @@ module Elasticity
     def run
       raise_if @jobflow_steps.empty?, JobFlowMissingStepsError, 'Cannot run a job flow without adding steps.  Please use #add_step.'
       raise_if is_jobflow_running?, JobFlowRunningError, 'Cannot run a job flow multiple times.  To do more with this job flow, please use #add_step.'
-      @jobflow_id ||= @emr.run_job_flow(jobflow_config)
+      @jobflow_id = @emr.run_job_flow(jobflow_config)
     end
 
     def shutdown
@@ -109,7 +116,7 @@ module Elasticity
     private
 
     def is_jobflow_running?
-      @jobflow_id
+      !@jobflow_id.nil?
     end
 
     def jobflow_config

data/lib/elasticity/job_flow_status.rb

@@ -13,28 +13,42 @@ module Elasticity
     attr_accessor :master_instance_type
     attr_accessor :slave_instance_type
     attr_accessor :last_state_change_reason
+    attr_accessor :installed_steps
 
     def initialize
       @steps = []
+      @installed_steps = []
     end
 
     # Create a jobflow from an AWS <member> (Nokogiri::XML::Element):
     #   /DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member
     def self.from_member_element(xml_element)
       jobflow = JobFlowStatus.new
+
       jobflow.name = xml_element.xpath('./Name').text.strip
       jobflow.jobflow_id = xml_element.xpath('./JobFlowId').text.strip
       jobflow.state = xml_element.xpath('./ExecutionStatusDetail/State').text.strip
       jobflow.last_state_change_reason = xml_element.xpath('./ExecutionStatusDetail/LastStateChangeReason').text.strip
+
       jobflow.steps = JobFlowStatusStep.from_members_nodeset(xml_element.xpath('./Steps/member'))
+
+      step_names = jobflow.steps.map(&:name)
+      Elasticity::JobFlowStep.steps_requiring_installation.each do |step|
+        jobflow.installed_steps << step if step_names.include?(step.aws_installation_step[:name])
+      end
+
       jobflow.created_at = Time.parse(xml_element.xpath('./ExecutionStatusDetail/CreationDateTime').text.strip)
+
       started_at = xml_element.xpath('./ExecutionStatusDetail/StartDateTime').text.strip
       jobflow.started_at = (started_at == '') ? (nil) : (Time.parse(started_at))
+
       ready_at = xml_element.xpath('./ExecutionStatusDetail/ReadyDateTime').text.strip
       jobflow.ready_at = (ready_at == '') ? (nil) : (Time.parse(ready_at))
+
       jobflow.instance_count = xml_element.xpath('./Instances/InstanceCount').text.strip
       jobflow.master_instance_type = xml_element.xpath('./Instances/MasterInstanceType').text.strip
       jobflow.slave_instance_type = xml_element.xpath('./Instances/SlaveInstanceType').text.strip
+
       jobflow
     end
 

data/lib/elasticity/job_flow_status_step.rb

@@ -11,11 +11,11 @@ module Elasticity
     #   /DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member/Steps/member
     def self.from_member_element(xml_element)
       job_flow_step = JobFlowStatusStep.new
-      job_flow_step.name = xml_element.xpath("./StepConfig/Name").text.strip
-      job_flow_step.state = xml_element.xpath("./ExecutionStatusDetail/State").text.strip
-      started_at = xml_element.xpath("./ExecutionStatusDetail/StartDateTime").text.strip
-      job_flow_step.started_at = (started_at == "") ? (nil) : (Time.parse(started_at))
-      ended_at = xml_element.xpath("./ExecutionStatusDetail/EndDateTime").text.strip
+      job_flow_step.name = xml_element.xpath('./StepConfig/Name').text.strip
+      job_flow_step.state = xml_element.xpath('./ExecutionStatusDetail/State').text.strip
+      started_at = xml_element.xpath('./ExecutionStatusDetail/StartDateTime').text.strip
+      job_flow_step.started_at = (started_at == '') ? (nil) : (Time.parse(started_at))
+      ended_at = xml_element.xpath('./ExecutionStatusDetail/EndDateTime').text.strip
       job_flow_step.ended_at = (ended_at == "") ? (nil) : (Time.parse(ended_at))
       job_flow_step
     end

data/lib/elasticity/job_flow_step.rb

@@ -2,10 +2,20 @@ module Elasticity
 
   module JobFlowStep
 
+    @@step_klasses = []
+
     def to_aws_step(jobflow_step)
       raise RuntimeError, '#to_aws_step is required to be defined on all job flow steps.'
     end
 
+    def requires_installation?
+      self.class.requires_installation?
+    end
+
+    def aws_installation_step
+      self.class.aws_installation_step
+    end
+
     module ClassMethods
 
       def requires_installation?
@@ -20,6 +30,11 @@ module Elasticity
 
     def self.included(base)
       base.extend(ClassMethods)
+      @@step_klasses << base
+    end
+
+    def self.steps_requiring_installation
+      @@step_klasses.select{|klass| klass.requires_installation?}
     end
 
   end

data/lib/elasticity/version.rb

@@ -1,3 +1,3 @@
 module Elasticity
-  VERSION = '2.1'
+  VERSION = '2.1.1'
 end

data/spec/lib/elasticity/job_flow_spec.rb

@@ -483,4 +483,48 @@ describe Elasticity::JobFlow do
 
   end
 
+  describe '.from_jobflow_id' do
+    before do
+      Elasticity::JobFlow.any_instance.stub_chain(:status, :installed_steps => [])
+    end
+
+    let(:jobflow) { Elasticity::JobFlow.from_jobflow_id('ACCESS', 'SECRET', 'JOBFLOW_ID') }
+
+    it 'should create a jobflow with the specified credentials' do
+      Elasticity::EMR.should_receive(:new).with('ACCESS', 'SECRET')
+      Elasticity::JobFlow.from_jobflow_id('ACCESS', 'SECRET', '_')
+    end
+
+    it 'should create a jobflow' do
+      jobflow.should be_a Elasticity::JobFlow
+    end
+
+    it 'should create a running jobflow' do
+      jobflow.send(:is_jobflow_running?).should == true
+    end
+
+    it 'should remember the jobflow ID' do
+      jobflow.instance_variable_get(:@jobflow_id).should == 'JOBFLOW_ID'
+    end
+
+    context 'when no steps have been installed' do
+      before do
+        Elasticity::JobFlow.any_instance.should_receive(:status).and_return(double('Elasticity::JobFlowStatus', :installed_steps => []))
+      end
+      it 'should show that no steps are installed' do
+        jobflow.instance_variable_get(:@installed_steps).should == []
+      end
+    end
+
+    context 'when steps have been installed do' do
+      before do
+        Elasticity::JobFlow.any_instance.should_receive(:status).and_return(double('Elasticity::JobFlowStatus', :installed_steps => [Elasticity::HiveStep, Elasticity::PigStep]))
+      end
+      it 'should show that no steps are installed' do
+        jobflow.instance_variable_get(:@installed_steps).should =~ [Elasticity::PigStep, Elasticity::HiveStep]
+      end
+    end
+
+  end
+
 end

data/spec/lib/elasticity/job_flow_status_spec.rb

@@ -1,13 +1,43 @@
 describe Elasticity::JobFlowStatus do
 
-  before do
-    describe_jobflows_xml = <<-JOBFLOWS
+  let(:hive_setup_config) do
+    <<-XML
+      <member>
+        <StepConfig>
+          <Name>Elasticity - Install Hive</Name>
+        </StepConfig>
+        <ExecutionStatusDetail>
+          <State>FAILED</State>
+        </ExecutionStatusDetail>
+      </member>
+    XML
+  end
+
+  let(:pig_setup_config) do
+    <<-XML
+      <member>
+        <StepConfig>
+          <Name>Elasticity - Install Pig</Name>
+        </StepConfig>
+        <ExecutionStatusDetail>
+          <State>FAILED</State>
+        </ExecutionStatusDetail>
+      </member>
+    XML
+  end
+
+  let(:setup_config) do
+    hive_setup_config
+  end
+
+  let(:describe_jobflows_xml) do
+    <<-XML
       <DescribeJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
         <DescribeJobFlowsResult>
           <JobFlows>
             <member>
               <JobFlowId>j-p</JobFlowId>
-              <Name>Pig Job</Name>
+              <Name>Hive Job 1</Name>
               <ExecutionStatusDetail>
                 <CreationDateTime>
                    2011-10-04T21:49:16Z
@@ -24,14 +54,7 @@ describe Elasticity::JobFlowStatus do
                 <State>TERMINATED</State>
               </ExecutionStatusDetail>
               <Steps>
-                <member>
-                  <StepConfig>
-                    <Name>Setup Hive</Name>
-                  </StepConfig>
-                  <ExecutionStatusDetail>
-                    <State>FAILED</State>
-                  </ExecutionStatusDetail>
-                </member>
+                #{setup_config}
                 <member>
                   <StepConfig>
                     <Name>Run Hive Script</Name>
@@ -63,7 +86,7 @@ describe Elasticity::JobFlowStatus do
             </member>
             <member>
               <JobFlowId>j-h</JobFlowId>
-              <Name>Hive Job</Name>
+              <Name>Hive Job 2</Name>
               <ExecutionStatusDetail>
                 <CreationDateTime>
                    2011-10-04T22:49:16Z
@@ -104,43 +127,79 @@ describe Elasticity::JobFlowStatus do
           </JobFlows>
         </DescribeJobFlowsResult>
       </DescribeJobFlowsResponse>
-    JOBFLOWS
+    XML
+  end
+
+  let(:members_nodeset) do
     describe_jobflows_document = Nokogiri::XML(describe_jobflows_xml)
     describe_jobflows_document.remove_namespaces!
-    @members_nodeset = describe_jobflows_document.xpath('/DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member')
+    describe_jobflows_document.xpath('/DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member')
   end
 
-  describe ".from_xml" do
-    it "should return a JobFlow with the appropriate fields initialized" do
-      jobflow = Elasticity::JobFlowStatus.from_member_element(@members_nodeset[0])
-      jobflow.name.should == "Pig Job"
-      jobflow.jobflow_id.should == "j-p"
-      jobflow.state.should == "TERMINATED"
-      jobflow.steps.map(&:name).should == ["Setup Hive", "Run Hive Script"]
-      jobflow.steps.map(&:state).should == ["FAILED", "PENDING"]
-      jobflow.created_at.should == Time.parse("2011-10-04T21:49:16Z")
-      jobflow.started_at.should == Time.parse("2011-10-04T21:49:17Z")
-      jobflow.ready_at.should == Time.parse("2011-10-04T21:49:18Z")
-      jobflow.master_instance_type.should == "m1.small"
-      jobflow.slave_instance_type.should == "m1.small"
-      jobflow.instance_count.should == "4"
-      jobflow.last_state_change_reason.should == "Steps completed with errors"
+  let(:single_jobflow) { Elasticity::JobFlowStatus.from_member_element(members_nodeset[0]) }
+
+  let(:multiple_jobflows) { Elasticity::JobFlowStatus.from_members_nodeset(members_nodeset) }
+
+  describe '.from_xml' do
+    it 'should return a JobFlow with the appropriate fields initialized' do
+      single_jobflow.name.should == 'Hive Job 1'
+      single_jobflow.jobflow_id.should == 'j-p'
+      single_jobflow.state.should == 'TERMINATED'
+      single_jobflow.steps.map(&:name).should == ['Elasticity - Install Hive', 'Run Hive Script']
+      single_jobflow.steps.map(&:state).should == %w(FAILED PENDING)
+      single_jobflow.created_at.should == Time.parse('2011-10-04T21:49:16Z')
+      single_jobflow.started_at.should == Time.parse('2011-10-04T21:49:17Z')
+      single_jobflow.ready_at.should == Time.parse('2011-10-04T21:49:18Z')
+      single_jobflow.master_instance_type.should == 'm1.small'
+      single_jobflow.slave_instance_type.should == 'm1.small'
+      single_jobflow.instance_count.should == '4'
+      single_jobflow.last_state_change_reason.should == 'Steps completed with errors'
     end
   end
 
-  describe ".from_jobflows_nodeset" do
-    it "should return JobFlows with the appropriate fields initialized" do
-      jobflow = Elasticity::JobFlowStatus.from_members_nodeset(@members_nodeset)
-      jobflow.map(&:name).should == ["Pig Job", "Hive Job"]
-      jobflow.map(&:jobflow_id).should == ["j-p", "j-h"]
-      jobflow.map(&:state).should == ["TERMINATED", "TERMINATED"]
-      jobflow.map(&:created_at).should == [Time.parse("2011-10-04T21:49:16Z"), Time.parse("2011-10-04T22:49:16Z")]
-      jobflow.map(&:started_at).should == [Time.parse("2011-10-04T21:49:17Z"), nil]
-      jobflow.map(&:ready_at).should == [Time.parse("2011-10-04T21:49:18Z"), nil]
-      jobflow.map(&:master_instance_type).should == ["m1.small","c1.medium"]
-      jobflow.map(&:slave_instance_type).should == ["m1.small", "c1.medium"]
-      jobflow.map(&:instance_count).should == ["4","2"]
-      jobflow.map(&:last_state_change_reason).should == ["Steps completed with errors", "Steps completed"]
+  describe '.from_jobflows_nodeset' do
+    it 'should return JobFlows with the appropriate fields initialized' do
+      multiple_jobflows.map(&:name).should == ['Hive Job 1', 'Hive Job 2']
+      multiple_jobflows.map(&:jobflow_id).should == %w(j-p j-h)
+      multiple_jobflows.map(&:state).should == %w(TERMINATED TERMINATED)
+      multiple_jobflows.map(&:created_at).should == [Time.parse('2011-10-04T21:49:16Z'), Time.parse('2011-10-04T22:49:16Z')]
+      multiple_jobflows.map(&:started_at).should == [Time.parse('2011-10-04T21:49:17Z'), nil]
+      multiple_jobflows.map(&:ready_at).should == [Time.parse('2011-10-04T21:49:18Z'), nil]
+      multiple_jobflows.map(&:master_instance_type).should == %w(m1.small c1.medium)
+      multiple_jobflows.map(&:slave_instance_type).should == %w(m1.small c1.medium)
+      multiple_jobflows.map(&:instance_count).should == %w(4 2)
+      multiple_jobflows.map(&:last_state_change_reason).should == ['Steps completed with errors', 'Steps completed']
+    end
+  end
+
+  describe '#installed_steps' do
+
+    context 'when nothing has been installed' do
+      let(:setup_config) { }
+      it 'should be empty' do
+        single_jobflow.installed_steps.should == []
+      end
+    end
+
+    context 'when Hive has been installed by Elasticity' do
+      let(:setup_config) { hive_setup_config }
+      it 'should include HiveStep' do
+        single_jobflow.installed_steps.should == [Elasticity::HiveStep]
+      end
+    end
+
+    context 'when Pig has been installed by Elasticity' do
+      let(:setup_config) { pig_setup_config }
+      it 'should include PigStep' do
+        single_jobflow.installed_steps.should == [Elasticity::PigStep]
+      end
+    end
+
+    context 'when more than one step has been installed by Elasticity' do
+      let(:setup_config) { hive_setup_config + pig_setup_config }
+      it 'should include all of them' do
+        single_jobflow.installed_steps.should =~ [Elasticity::HiveStep, Elasticity::PigStep]
+      end
     end
   end
 

data/spec/lib/elasticity/job_flow_step_spec.rb

@@ -20,6 +20,13 @@ describe Elasticity::JobFlowStep do
 
   end
 
+  describe '#requires_installation?' do
+    it 'should delegate to the class method' do
+      FakeStep.should_receive(:requires_installation?).and_return(true)
+      subject.requires_installation?.should == true
+    end
+  end
+
   describe '.requires_installation?' do
     it 'should be false by default' do
       FakeStep.requires_installation?.should be_false
@@ -34,4 +41,17 @@ describe Elasticity::JobFlowStep do
     end
   end
 
+  describe '#aws_installation_step' do
+    it 'should delegate to the class method' do
+      FakeStep.should_receive(:aws_installation_step).and_return('AWS_INSTALLATION_STEP')
+      subject.aws_installation_step.should == 'AWS_INSTALLATION_STEP'
+    end
+  end
+
+  describe '.steps_requiring_installation' do
+    it 'should list all of the steps requiring installation' do
+      Elasticity::JobFlowStep.steps_requiring_installation.should =~ [Elasticity::PigStep, Elasticity::HiveStep]
+    end
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elasticity
 version: !ruby/object:Gem::Version
-  version: '2.1'
+  version: 2.1.1
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-07 00:00:00.000000000 Z
+date: 2012-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -66,7 +66,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.10.0
+        version: 2.11.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -74,7 +74,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.10.0
+        version: 2.11.0
 description: Streamlined, Programmatic access to Amazon's Elastic Map Reduce service,
   driven by the Sharethrough team's requirements for belting out EMR jobs.
 email: 
@@ -133,18 +133,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -3013863256678255822
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
-      segments:
-      - 0
-      hash: -3013863256678255822
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24