elasticity 2.2 → 2.3

data/HISTORY.md

@@ -1,3 +1,8 @@
+## 2.3 - July 28, 2012
+
++ ```JobFlow``` now supports specifying availbility zone instance specification via ```JobFlow#placement=```.
++ ```JobFlow::from_jobflow_id``` now supports region specification so that jobs created in regions other than us-east-1 can be recreated.
+
 ## 2.2 - July 23, 2012
 
 + Hadoop streaming jobs are now supported via ```Elasticity::StreamingStep```.

data/README.md

@@ -70,10 +70,10 @@ jobflow = Elasticity::JobFlow.new('AWS access key', 'AWS secret key')
 If you want to access a job flow that's already running:
 
 ```ruby
-jobflow = Elasticity::JobFlow.from_jobflow_id('AWS access key', 'AWS secret key', 'jobflow ID')
+jobflow = Elasticity::JobFlow.from_jobflow_id('AWS access key', 'AWS secret key', 'jobflow ID', 'region')
 ```
 
-This is useful if you'd like to attach to a running job flow and add more steps, etc.
+This is useful if you'd like to attach to a running job flow and add more steps, etc.  The ```region``` parameter is necessary because job flows are only accessible from the the API when you connect to the same endpoint that created them (e.g. us-west-1).  If you don't specify the ```region``` parameter, us-east-1 is assumed.
 
 ## 2 - Specifying Job Flow Options
 
@@ -90,6 +90,7 @@ jobflow.hadoop_version                    = '0.20.205'
 jobflow.keep_job_flow_alive_when_no_steps = true
 jobflow.log_uri                           = nil
 jobflow.name                              = 'Elasticity Job Flow'
+jobflow.placement                         = 'us-east-1a'
 jobflow.instance_count                    = 2
 jobflow.master_instance_type              = 'm1.small'
 jobflow.slave_instance_type               = 'm1.small'

data/lib/elasticity/aws_request.rb

@@ -4,7 +4,6 @@ module Elasticity
 
     attr_reader :access_key
     attr_reader :secret_key
-    attr_reader :options
     attr_reader :host
     attr_reader :protocol
 
@@ -14,7 +13,7 @@ module Elasticity
     def initialize(access, secret, options = {})
       @access_key = access
       @secret_key = secret
-      @host = options[:region] ? "elasticmapreduce.#{options[:region]}.amazonaws.com" : 'elasticmapreduce.amazonaws.com'
+      @host = "elasticmapreduce.#{{:region => 'us-east-1'}.merge(options)[:region]}.amazonaws.com"
       @protocol = {:secure => true}.merge(options)[:secure] ? 'https' : 'http'
     end
 
@@ -32,7 +31,8 @@ module Elasticity
       return false unless other.is_a? AwsRequest
       return false unless @access_key == other.access_key
       return false unless @secret_key == other.secret_key
-      return false unless @options == other.options
+      return false unless @host == other.host
+      return false unless @protocol == other.protocol
       true
     end
 

data/lib/elasticity/job_flow.rb

@@ -17,6 +17,7 @@ module Elasticity
     attr_accessor :ami_version
     attr_accessor :keep_job_flow_alive_when_no_steps
     attr_accessor :ec2_subnet_id
+    attr_accessor :placement
 
     def initialize(access, secret)
       @action_on_failure = 'TERMINATE_JOB_FLOW'
@@ -25,6 +26,10 @@ module Elasticity
       @name = 'Elasticity Job Flow'
       @ami_version = 'latest'
       @keep_job_flow_alive_when_no_steps = false
+      @placement = 'us-east-1a'
+
+      @access = access
+      @secret = secret
 
       @bootstrap_actions = []
       @jobflow_steps = []
@@ -33,23 +38,24 @@ module Elasticity
       @instance_groups = {}
       set_master_instance_group(Elasticity::InstanceGroup.new)
       set_core_instance_group(Elasticity::InstanceGroup.new)
-
       @instance_count = 2
       @master_instance_type = 'm1.small'
       @slave_instance_type = 'm1.small'
 
-      @emr = Elasticity::EMR.new(access, secret)
+      @access = access
+      @secret = secret
     end
 
-    def self.from_jobflow_id(access, secret, jobflow_id)
+    def self.from_jobflow_id(access, secret, jobflow_id, region = 'us-east-1')
       JobFlow.new(access, secret).tap do |j|
+        j.instance_variable_set(:@region, region)
         j.instance_variable_set(:@jobflow_id, jobflow_id)
         j.instance_variable_set(:@installed_steps, j.status.installed_steps)
       end
     end
 
     def instance_count=(count)
-      raise ArgumentError, 'Instance count cannot be set to less than 2 (requested 1)' unless count > 1
+      raise ArgumentError, "Instance count cannot be set to less than 2 (requested #{count})" unless count > 1
       @instance_groups[:core].count = count - 1
       @instance_count = count
     end
@@ -91,7 +97,7 @@ module Elasticity
           jobflow_steps << jobflow_step.aws_installation_step
         end
         jobflow_steps << jobflow_step.to_aws_step(self)
-        @emr.add_jobflow_steps(@jobflow_id, {:steps => jobflow_steps})
+        emr.add_jobflow_steps(@jobflow_id, {:steps => jobflow_steps})
       else
         @jobflow_steps << jobflow_step
       end
@@ -100,21 +106,26 @@ module Elasticity
     def run
       raise_if @jobflow_steps.empty?, JobFlowMissingStepsError, 'Cannot run a job flow without adding steps.  Please use #add_step.'
       raise_if is_jobflow_running?, JobFlowRunningError, 'Cannot run a job flow multiple times.  To do more with this job flow, please use #add_step.'
-      @jobflow_id = @emr.run_job_flow(jobflow_config)
+      @jobflow_id = emr.run_job_flow(jobflow_config)
     end
 
     def shutdown
       raise_unless is_jobflow_running?, JobFlowNotStartedError, 'Cannot #shutdown a job flow that has not yet been #run.'
-      @emr.terminate_jobflows(@jobflow_id)
+      emr.terminate_jobflows(@jobflow_id)
     end
 
     def status
       raise_unless is_jobflow_running?, JobFlowNotStartedError, 'Please #run this job flow before attempting to retrieve status.'
-      @emr.describe_jobflow(@jobflow_id)
+      emr.describe_jobflow(@jobflow_id)
     end
 
     private
 
+    def emr
+      @region ||= @placement.match(/(\w+-\w+-\d+)/)[0]
+      @emr ||= Elasticity::EMR.new(@access, @secret, :region => @region)
+    end
+
     def is_jobflow_running?
       !@jobflow_id.nil?
     end

data/lib/elasticity/version.rb

@@ -1,3 +1,3 @@
 module Elasticity
-  VERSION = '2.2'
+  VERSION = '2.3'
 end

data/spec/lib/elasticity/aws_request_spec.rb

@@ -14,7 +14,7 @@ describe Elasticity::AwsRequest do
   describe '#host' do
 
     context 'when the region is not specified' do
-      its(:host) { should == 'elasticmapreduce.amazonaws.com' }
+      its(:host) { should == 'elasticmapreduce.us-east-1.amazonaws.com' }
     end
 
     context 'when the region is specified' do
@@ -60,7 +60,7 @@ describe Elasticity::AwsRequest do
   describe '#sign_params' do
     it 'should sign according to AWS rules' do
       signed_params = subject.send(:sign_params, {})
-      signed_params.should == 'AWSAccessKeyId=access&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2011-04-10T18%3A44%3A56.000Z&Signature=QwQIiizWrfvWuLNnzmCMfaeXFfh9IQTvOix5MNVTh2s%3D'
+      signed_params.should == 'AWSAccessKeyId=access&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2011-04-10T18%3A44%3A56.000Z&Signature=t%2BccC38VxCKyk2ROTKo9vnECsntKoU0RBAFklHWP5bE%3D'
     end
   end
 
@@ -85,13 +85,13 @@ describe Elasticity::AwsRequest do
     context 'when there is an EMR error with the request' do
       let(:error_message) { 'ERROR_MESSAGE' }
       let(:error_xml) do
-        <<-ERROR
+        <<-XML
           <ErrorResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
             <Error>
               <Message>#{error_message}</Message>
             </Error>
           </ErrorResponse>
-        ERROR
+        XML
       end
       let(:error) do
         RestClient::BadRequest.new.tap do |error|
@@ -110,24 +110,39 @@ describe Elasticity::AwsRequest do
   end
 
   describe '#==' do
-    let(:same_object) { subject }
-    let(:same_values) { Elasticity::AwsRequest.new('access', 'secret', {}) }
-    let(:diff_type) { Object.new }
-
-    it { should == same_object }
-    it { should == same_values }
-    it { should_not == diff_type }
-
-    it 'should be false on deep comparison' do
-      {
-        :@access_key => '_',
-        :@secret_key => '_',
-        :@options => {:foo => :bar}
-      }.each do |variable, value|
-        other = Elasticity::AwsRequest.new('aws_access_key_id', 'aws_secret_access_key', {})
-        other.instance_variable_set(variable, value)
-        subject.should_not == other
+
+    describe 'basic equality checks with subject' do
+      let(:same_object) { subject }
+      let(:same_values) { Elasticity::AwsRequest.new('access', 'secret', {}) }
+      let(:diff_type) { Object.new }
+
+      it { should == same_object }
+      it { should == same_values }
+      it { should_not == diff_type }
+    end
+
+    describe 'deep comparisons' do
+
+      it 'should fail on access key check' do
+        Elasticity::AwsRequest.new('access', '_').should_not == Elasticity::AwsRequest.new('_', '_')
+      end
+
+      it 'should fail on secret key check' do
+        Elasticity::AwsRequest.new('_', 'secret').should_not == Elasticity::AwsRequest.new('_', '_')
+      end
+
+      it 'should fail on host check' do
+        aws1 = Elasticity::AwsRequest.new('_', '_', :region => 'us-east-1')
+        aws2 = Elasticity::AwsRequest.new('_', '_', :region => 'us-west-1')
+        aws1.should_not == aws2
       end
+
+      it 'should fail on protocol check' do
+        aws1 = Elasticity::AwsRequest.new('_', '_', :secure => true)
+        aws2 = Elasticity::AwsRequest.new('_', '_', :secure => false)
+        aws1.should_not == aws2
+      end
+
     end
 
   end

data/spec/lib/elasticity/job_flow_integration_spec.rb

@@ -3,7 +3,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
   let(:emr) { double('Elasticity::EMR') }
 
   before do
-    Elasticity::EMR.should_receive(:new).with('access', 'secret').and_return(emr)
+    Elasticity::EMR.should_receive(:new).with('access', 'secret', :region => 'us-west-1').and_return(emr)
   end
 
   describe 'Hive' do
@@ -17,6 +17,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
 
     let(:hive_jobflow) do
       Elasticity::JobFlow.new('access', 'secret').tap do |jf|
+        jf.placement = 'us-west-1b'
         jf.log_uri = 's3n://slif-test/output/logs'
         jf.add_step(hive_step)
       end
@@ -93,6 +94,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
 
     let(:pig_jobflow) do
       Elasticity::JobFlow.new('access', 'secret').tap do |jf|
+        jf.placement = 'us-west-1b'
         jf.instance_count = 8
         jf.slave_instance_type = 'm1.xlarge'
         jf.log_uri = 's3n://slif-test/output/logs'
@@ -177,6 +179,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
 
     let(:custom_jar_jobflow) do
       Elasticity::JobFlow.new('access', 'secret').tap do |jf|
+        jf.placement = 'us-west-1b'
         jf.log_uri = 's3n://slif-test/output/logs'
         jf.add_step(custom_jar_step)
       end

data/spec/lib/elasticity/job_flow_spec.rb

@@ -1,10 +1,5 @@
 describe Elasticity::JobFlow do
 
-  before do
-    # Ensure we don't accidentally submit to EMR for all of these examples
-    Elasticity::EMR.stub(:new).and_return(double('Elasticity::EMR', :run_job_flow => '_'))
-  end
-
   subject do
     Elasticity::JobFlow.new('access', 'secret')
   end
@@ -19,6 +14,7 @@ describe Elasticity::JobFlow do
   its(:slave_instance_type) { should == 'm1.small' }
   its(:ami_version) { should == 'latest' }
   its(:keep_job_flow_alive_when_no_steps) { should == false }
+  its(:placement) { should == 'us-east-1a' }
 
   describe '#instance_count=' do
 
@@ -102,6 +98,7 @@ describe Elasticity::JobFlow do
 
     context 'when the jobflow is already started' do
       before do
+        Elasticity::EMR.any_instance.stub(:run_job_flow => '_')
         subject.add_step(Elasticity::CustomJarStep.new('_'))
         subject.run
       end
@@ -127,7 +124,7 @@ describe Elasticity::JobFlow do
       end
 
       before do
-        Elasticity::EMR.should_receive(:new).with('access', 'secret').and_return(emr)
+        Elasticity::EMR.stub(:new).and_return(emr)
         running_jobflow.run
       end
 
@@ -380,20 +377,20 @@ describe Elasticity::JobFlow do
         let(:emr) { double('Elasticity::EMR', :run_job_flow => 'JOBFLOW_ID') }
 
         it 'should run the job with the supplied EMR credentials' do
-          Elasticity::EMR.should_receive(:new).with('STEP_TEST_ACCESS', 'STEP_TEST_SECRET').and_return(emr)
+          Elasticity::EMR.stub(:new).with('STEP_TEST_ACCESS', 'STEP_TEST_SECRET', :region => 'us-east-1').and_return(emr)
           emr.should_receive(:run_job_flow)
           jobflow_with_steps.run
         end
 
         it 'should run the job with the jobflow config' do
-          Elasticity::EMR.stub(:new).with('STEP_TEST_ACCESS', 'STEP_TEST_SECRET').and_return(emr)
+          Elasticity::EMR.stub(:new).and_return(emr)
           jobflow_with_steps.stub(:jobflow_config).and_return('JOBFLOW_CONFIG')
           emr.should_receive(:run_job_flow).with('JOBFLOW_CONFIG')
           jobflow_with_steps.run
         end
 
         it 'should return the jobflow ID' do
-          Elasticity::EMR.stub(:new).with('STEP_TEST_ACCESS', 'STEP_TEST_SECRET').and_return(emr)
+          Elasticity::EMR.stub(:new).and_return(emr)
           jobflow_with_steps.run.should == 'JOBFLOW_ID'
         end
 
@@ -401,6 +398,7 @@ describe Elasticity::JobFlow do
 
       context 'when the jobflow has already been run' do
         before do
+          Elasticity::EMR.any_instance.stub(:run_job_flow => '_')
           jobflow_with_steps.run
         end
         it 'should raise an error' do
@@ -484,15 +482,29 @@ describe Elasticity::JobFlow do
   end
 
   describe '.from_jobflow_id' do
+
     before do
       Elasticity::JobFlow.any_instance.stub_chain(:status, :installed_steps => [])
     end
 
     let(:jobflow) { Elasticity::JobFlow.from_jobflow_id('ACCESS', 'SECRET', 'JOBFLOW_ID') }
 
-    it 'should create a jobflow with the specified credentials' do
-      Elasticity::EMR.should_receive(:new).with('ACCESS', 'SECRET')
-      Elasticity::JobFlow.from_jobflow_id('ACCESS', 'SECRET', '_')
+    describe 'creating a jobflow with the specified credentials' do
+
+      context 'when the region is not specified' do
+        it 'should use the default of us-east-1a' do
+          j = Elasticity::JobFlow.from_jobflow_id('ACCESS', 'SECRET', '_')
+          j.send(:emr).should == Elasticity::EMR.new('ACCESS', 'SECRET', :region => 'us-east-1')
+        end
+      end
+
+      context 'when the region is specified' do
+        it 'should use the specified region' do
+          j = Elasticity::JobFlow.from_jobflow_id('ACCESS', 'SECRET', '_', 'us-west-1')
+          j.send(:emr).should == Elasticity::EMR.new('ACCESS', 'SECRET', :region => 'us-west-1')
+        end
+      end
+
     end
 
     it 'should create a jobflow' do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elasticity
 version: !ruby/object:Gem::Version
-  version: '2.2'
+  version: '2.3'
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-23 00:00:00.000000000 Z
+date: 2012-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client