elasticity 5.0.3 → 6.0

data/spec/lib/elasticity/aws_session_spec.rb

@@ -2,22 +2,21 @@ describe Elasticity::AwsSession do
 
   before do
     Timecop.freeze(Time.at(1302461096))
+
+    Elasticity.configure do |c|
+      c.access_key = 'access'
+      c.secret_key = 'secret'
+    end
   end
 
   after do
     Timecop.return
   end
 
-  subject do
-    Elasticity::AwsSession.new('access', 'secret')
-  end
-
   describe '#initialize' do
 
     context 'when access and/or secret keys are provided' do
       it 'should set them to the provided values' do
-        subject.access_key.should == 'access'
-        subject.secret_key.should == 'secret'
         subject.region.should == 'us-east-1'
       end
     end
@@ -25,71 +24,15 @@ describe Elasticity::AwsSession do
     context 'when :region is nil' do
       it 'should be an error' do
         expect {
-          Elasticity::AwsSession.new('_', '_', :region => nil)
+          Elasticity::AwsSession.new(:region => nil)
         }.to raise_error Elasticity::MissingRegionError, 'A valid :region is required to connect to EMR'
       end
     end
 
     context 'when :region is specified' do
-      Elasticity::AwsSession.new('_', '_', :region => 'TEST_REGION').region.should == 'TEST_REGION'
-    end
-
-    context 'when either access or secret key is not provided or nil' do
-
-      context 'when the proper environment variables are set' do
-
-        context 'when access and secret key are not provided' do
-          let(:default_values) { Elasticity::AwsSession.new }
-          before do
-            ENV.stub(:[]).with('AWS_ACCESS_KEY_ID').and_return('ENV_ACCESS')
-            ENV.stub(:[]).with('AWS_SECRET_ACCESS_KEY').and_return('ENV_SECRET')
-          end
-          it 'should set access and secret keys' do
-            default_values.access_key.should == 'ENV_ACCESS'
-            default_values.secret_key.should == 'ENV_SECRET'
-          end
-        end
-
-        context 'when access and secret key are nil' do
-          let(:nil_values) { Elasticity::AwsSession.new(nil, nil) }
-          before do
-            ENV.stub(:[]).with('AWS_ACCESS_KEY_ID').and_return('ENV_ACCESS')
-            ENV.stub(:[]).with('AWS_SECRET_ACCESS_KEY').and_return('ENV_SECRET')
-          end
-          it 'should set access and secret keys' do
-            nil_values.access_key.should == 'ENV_ACCESS'
-            nil_values.secret_key.should == 'ENV_SECRET'
-          end
-        end
-
-      end
-
-      context 'when the environment variables are not set' do
-        let(:missing_something) { Elasticity::AwsSession.new }
-        context 'when the access key is not set' do
-          before do
-            ENV.stub(:[]).with('AWS_ACCESS_KEY_ID').and_return(nil)
-            ENV.stub(:[]).with('AWS_SECRET_ACCESS_KEY').and_return('_')
-          end
-          it 'should raise an error' do
-            expect {
-              missing_something.access_key
-            }.to raise_error(Elasticity::MissingKeyError, 'Please provide an access key or set AWS_ACCESS_KEY_ID.')
-          end
-        end
-        context 'when the secret key is not set' do
-          before do
-            ENV.stub(:[]).with('AWS_ACCESS_KEY_ID').and_return('_')
-            ENV.stub(:[]).with('AWS_SECRET_ACCESS_KEY').and_return(nil)
-          end
-          it 'should raise an error' do
-            expect {
-              missing_something.access_key
-            }.to raise_error(Elasticity::MissingKeyError, 'Please provide a secret key or set AWS_SECRET_ACCESS_KEY.')
-          end
-        end
+      it 'should be assigned' do
+        Elasticity::AwsSession.new(:region => 'TEST_REGION').region.should == 'TEST_REGION'
       end
-
     end
 
   end
@@ -97,12 +40,14 @@ describe Elasticity::AwsSession do
   describe '#host' do
 
     context 'when the region is not specified' do
-      its(:host) { should == 'elasticmapreduce.us-east-1.amazonaws.com' }
+      it 'should use the default value' do
+        expect(subject.host).to eq('elasticmapreduce.us-east-1.amazonaws.com')
+      end
     end
 
     context 'when the region is specified' do
       let(:request_with_region) do
-        Elasticity::AwsSession.new('_', '_', {:region => 'us-west-1'})
+        Elasticity::AwsSession.new(:region => 'us-west-1')
       end
       it 'should incorporate the region into the hostname' do
         request_with_region.host.should == 'elasticmapreduce.us-west-1.amazonaws.com'
@@ -131,18 +76,15 @@ describe Elasticity::AwsSession do
 
     context 'when there is an EMR error with the request' do
       let(:error_message) { 'ERROR_MESSAGE' }
-      let(:error_xml) do
-        <<-XML
-          <ErrorResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
-            <Error>
-              <Message>#{error_message}</Message>
-            </Error>
-          </ErrorResponse>
-        XML
+      let(:error_type) { 'ERROR_TYPE' }
+      let(:error_json) do
+        <<-JSON
+          { "__type" : "#{error_type}", "message" : "#{error_message}" }
+        JSON
       end
       let(:error) do
         RestClient::BadRequest.new.tap do |error|
-          error.stub(:http_body => error_xml)
+          error.stub(:http_body => error_json)
         end
       end
 
@@ -150,7 +92,7 @@ describe Elasticity::AwsSession do
         RestClient.should_receive(:post).and_raise(error)
         expect {
           subject.submit({})
-        }.to raise_error(ArgumentError, error_message)
+        }.to raise_error(ArgumentError, "AWS EMR API Error (#{error_type}): #{error_message}")
       end
     end
 
@@ -160,7 +102,7 @@ describe Elasticity::AwsSession do
 
     describe 'basic equality checks with subject' do
       let(:same_object) { subject }
-      let(:same_values) { Elasticity::AwsSession.new('access', 'secret', {}) }
+      let(:same_values) { Elasticity::AwsSession.new }
       let(:diff_type) { Object.new }
 
       it { should == same_object }
@@ -170,17 +112,9 @@ describe Elasticity::AwsSession do
 
     describe 'deep comparisons' do
 
-      it 'should fail on access key check' do
-        Elasticity::AwsSession.new('access', '_').should_not == Elasticity::AwsSession.new('_', '_')
-      end
-
-      it 'should fail on secret key check' do
-        Elasticity::AwsSession.new('_', 'secret').should_not == Elasticity::AwsSession.new('_', '_')
-      end
-
       it 'should fail on host check' do
-        aws1 = Elasticity::AwsSession.new('_', '_', :region => 'us-east-1')
-        aws2 = Elasticity::AwsSession.new('_', '_', :region => 'us-west-1')
+        aws1 = Elasticity::AwsSession.new(:region => 'us-east-1')
+        aws2 = Elasticity::AwsSession.new(:region => 'us-west-1')
         aws1.should_not == aws2
       end
 

data/spec/lib/elasticity/aws_utils_spec.rb

@@ -56,50 +56,4 @@ describe Elasticity::AwsUtils do
     end
   end
 
-  describe '.convert_ruby_to_aws' do
-    it 'should convert the params' do
-      add_jobflow_steps_params = {
-        :job_flow_id => 'j-1',
-        :steps => [
-          {
-            :action_on_failure => 'CONTINUE',
-            :name => 'First New Job Step',
-            :hadoop_jar_step => {
-              :args => %w(arg1 arg2 arg3),
-              :jar => 'first_step.jar',
-              :main_class => 'first_class.jar'
-            }
-          },
-          {
-            :action_on_failure => 'CANCEL_AND_WAIT',
-            :name => 'Second New Job Step',
-            :hadoop_jar_step => {
-              :args => %w(arg4 arg5 arg6),
-              :jar => 'second_step.jar',
-              :main_class => 'second_class.jar'
-            }
-          }
-        ]
-      }
-      expected_result = {
-        'JobFlowId' => 'j-1',
-        'Steps.member.1.Name' => 'First New Job Step',
-        'Steps.member.1.ActionOnFailure' => 'CONTINUE',
-        'Steps.member.1.HadoopJarStep.Jar' => 'first_step.jar',
-        'Steps.member.1.HadoopJarStep.MainClass' => 'first_class.jar',
-        'Steps.member.1.HadoopJarStep.Args.member.1' => 'arg1',
-        'Steps.member.1.HadoopJarStep.Args.member.2' => 'arg2',
-        'Steps.member.1.HadoopJarStep.Args.member.3' => 'arg3',
-        'Steps.member.2.Name' => 'Second New Job Step',
-        'Steps.member.2.ActionOnFailure' => 'CANCEL_AND_WAIT',
-        'Steps.member.2.HadoopJarStep.Jar' => 'second_step.jar',
-        'Steps.member.2.HadoopJarStep.MainClass' => 'second_class.jar',
-        'Steps.member.2.HadoopJarStep.Args.member.1' => 'arg4',
-        'Steps.member.2.HadoopJarStep.Args.member.2' => 'arg5',
-        'Steps.member.2.HadoopJarStep.Args.member.3' => 'arg6'
-      }
-      Elasticity::AwsUtils.send(:convert_ruby_to_aws, add_jobflow_steps_params).should == expected_result
-    end
-  end
-
 end

data/spec/lib/elasticity/bootstrap_action_spec.rb

@@ -4,9 +4,13 @@ describe Elasticity::BootstrapAction do
     Elasticity::BootstrapAction.new('script', 'arg1', 'arg2')
   end
 
-  its(:name) { should == 'Elasticity Bootstrap Action' }
-  its(:script) { should == 'script' }
-  its(:arguments) { should == %w(arg1 arg2) }
+  describe '.initialize' do
+    it 'should set the fields appropriately' do
+      expect(subject.name).to eql('Elasticity Bootstrap Action')
+      expect(subject.script).to eql('script')
+      expect(subject.arguments).to eql(%w(arg1 arg2))
+    end
+  end
 
   describe '#to_aws_bootstrap_action' do
 

data/spec/lib/elasticity/cluster_status_spec.rb

@@ -0,0 +1,98 @@
+describe Elasticity::ClusterStatus do
+
+  let(:cluster_state) { 'TERMINATED' }
+  let(:aws_cluster_status) do
+    <<-JSON
+      {
+        "Cluster": {
+          "Applications": [
+            {
+              "Name": "hadoop",
+              "Version": "1.0.3"
+            }
+          ],
+          "AutoTerminate": true,
+          "Configurations": [
+
+          ],
+          "Ec2InstanceAttributes": {
+            "Ec2AvailabilityZone": "us-east-1a",
+            "EmrManagedMasterSecurityGroup": "sg-b7de0adf",
+            "EmrManagedSlaveSecurityGroup": "sg-89de0ae1"
+          },
+          "Id": "j-3T0PHNUXCY7SX",
+          "MasterPublicDnsName": "ec2-54-81-173-103.compute-1.amazonaws.com",
+          "Name": "Elasticity Job Flow",
+          "NormalizedInstanceHours": 2,
+          "RequestedAmiVersion": "latest",
+          "RunningAmiVersion": "2.4.2",
+          "Status": {
+            "State": "#{cluster_state}",
+            "StateChangeReason": {
+              "Code": "ALL_STEPS_COMPLETED",
+              "Message": "Steps completed"
+            },
+            "Timeline": {
+              "CreationDateTime": 1436788464.415,
+              "EndDateTime": 1436791032.097,
+              "ReadyDateTime": 1436788842.195
+            }
+          },
+          "Tags": [
+            {
+              "Key": "key",
+              "Value": "value"
+            }
+          ],
+          "TerminationProtected": false,
+          "VisibleToAllUsers": false
+        }
+      }
+    JSON
+  end
+
+  subject do
+    Elasticity::ClusterStatus.from_aws_data(JSON.parse(aws_cluster_status))
+  end
+
+  describe '.from_aws_data' do
+    it 'should hydate properly' do
+      expect(subject.name).to eql('Elasticity Job Flow')
+      expect(subject.cluster_id).to eql('j-3T0PHNUXCY7SX')
+      expect(subject.state).to eql('TERMINATED')
+      expect(subject.created_at).to eql(Time.at(1436788464.415))
+      expect(subject.ready_at).to eql(Time.at(1436788842.195))
+      expect(subject.ended_at).to eql(Time.at(1436791032.097))
+      expect(subject.last_state_change_reason).to eql('ALL_STEPS_COMPLETED')
+      expect(subject.master_public_dns_name).to eql('ec2-54-81-173-103.compute-1.amazonaws.com')
+      expect(subject.normalized_instance_hours).to eql(2)
+    end
+  end
+
+  describe '#active?' do
+
+    context 'when the jobflow status is terminal' do
+      %w{COMPLETED TERMINATED FAILED _}.each do |status|
+        context "when the jobflow is #{status}" do
+          let(:cluster_state) {status}
+          it 'is not active' do
+            expect(subject.active?).to be false
+          end
+        end
+      end
+    end
+
+    context 'when the jobflow status is not terminal' do
+      %w{RUNNING STARTING BOOTSTRAPPING WAITING SHUTTING_DOWN}.each do |status|
+        context "when the jobflow is #{status}" do
+          let(:cluster_state) {status}
+          it 'is active' do
+            expect(subject.active?).to be true
+          end
+        end
+      end
+    end
+
+  end
+
+end

data/spec/lib/elasticity/cluster_step_status_spec.rb

@@ -0,0 +1,80 @@
+describe Elasticity::ClusterStepStatus do
+
+  let(:aws_cluster_steps) do
+    <<-JSON
+      {
+          "Steps": [
+              {
+                  "ActionOnFailure": "TERMINATE_CLUSTER",
+                  "Config": {
+                      "Args": [
+                          "36",
+                          "3",
+                          "0"
+                      ],
+                      "Jar": "s3n://elasticmapreduce/samples/cloudburst/cloudburst.jar",
+                      "MainClass" : "MAIN_CLASS",
+                      "Properties": {
+                          "Key1" : "Value1",
+                          "Key2" : "Value2"
+                      }
+                  },
+                  "Id": "s-OYPPAC4XPPUC",
+                  "Name": "Elasticity Custom Jar Step",
+                  "Status": {
+                      "State": "COMPLETED",
+                      "StateChangeReason": {
+                        "Code": "ALL_STEPS_COMPLETED",
+                        "Message": "Steps completed"
+                      },
+                      "Timeline": {
+                          "CreationDateTime": 1436788464.416,
+                          "EndDateTime": 1436790944.162,
+                          "StartDateTime": 1436788841.237
+                      }
+                  }
+              }
+          ]
+      }
+    JSON
+  end
+
+  describe '.from_aws_list_data' do
+    let(:cluster_step_statuses) { Elasticity::ClusterStepStatus.from_aws_list_data(JSON.parse(aws_cluster_steps)) }
+
+    it 'should extract the proper number of steps' do
+      expect(cluster_step_statuses.length).to eql(1)
+    end
+
+    it 'should hydate properly' do
+      status = cluster_step_statuses[0]
+      expect(status.action_on_failure).to eql('TERMINATE_CLUSTER')
+      expect(status.args).to eql(['36', '3', '0',])
+      expect(status.jar).to eql('s3n://elasticmapreduce/samples/cloudburst/cloudburst.jar')
+      expect(status.main_class).to eql('MAIN_CLASS')
+      expect(status.step_id).to eql('s-OYPPAC4XPPUC')
+      expect(status.properties).to eql({'Key1' => 'Value1', 'Key2' => 'Value2'})
+      expect(status.name).to eql('Elasticity Custom Jar Step')
+      expect(status.state).to eql('COMPLETED')
+      expect(status.state_change_reason).to eql('ALL_STEPS_COMPLETED')
+      expect(status.state_change_reason_message).to eql('Steps completed')
+      expect(status.created_at).to eql(Time.at(1436788464.416))
+      expect(status.started_at).to eql(Time.at(1436788841.237))
+      expect(status.ended_at).to eql(Time.at(1436790944.162))
+    end
+  end
+
+  describe '.installed_steps' do
+    let(:installed_cluster_step_statuses) do
+      step_names = Elasticity::JobFlowStep.steps_requiring_installation.map { |s| s.aws_installation_step_name }
+      step_names.map { |name| build(:cluster_step_status, :name => name) }
+    end
+
+    it 'should return a list of steps that are installed' do
+      expect(Elasticity::ClusterStepStatus.installed_steps(installed_cluster_step_statuses)).to match_array([
+            Elasticity::PigStep, Elasticity::HiveStep
+          ])
+    end
+  end
+
+end

data/spec/lib/elasticity/custom_jar_step_spec.rb

@@ -6,10 +6,13 @@ describe Elasticity::CustomJarStep do
 
   it { should be_a Elasticity::JobFlowStep }
 
-  its(:name) { should == 'Elasticity Custom Jar Step' }
-  its(:jar) { should == 'jar' }
-  its(:arguments) { should == [] }
-  its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
+  describe '.initialize' do
+    it 'should set the fields appropriately' do
+      expect(subject.name).to eql('Elasticity Custom Jar Step')
+      expect(subject.jar).to eql('jar')
+      expect(subject.arguments).to eql([])
+    end
+  end
 
   describe '#to_aws_step' do
 
@@ -19,7 +22,7 @@ describe Elasticity::CustomJarStep do
       let(:cjs_with_no_args) { Elasticity::CustomJarStep.new('jar') }
 
       it 'should convert to aws step format' do
-        cjs_with_no_args.to_aws_step(Elasticity::JobFlow.new('access', 'secret')).should == {
+        cjs_with_no_args.to_aws_step(Elasticity::JobFlow.new).should == {
           :action_on_failure => 'TERMINATE_JOB_FLOW',
           :hadoop_jar_step => {
             :jar => 'jar'
@@ -37,7 +40,7 @@ describe Elasticity::CustomJarStep do
       end
 
       it 'should convert to aws step format' do
-        cjs_with_args.to_aws_step(Elasticity::JobFlow.new('access', 'secret')).should == {
+        cjs_with_args.to_aws_step(Elasticity::JobFlow.new).should == {
           :action_on_failure => 'TERMINATE_JOB_FLOW',
           :hadoop_jar_step => {
             :jar => 'jar',
@@ -52,7 +55,7 @@ describe Elasticity::CustomJarStep do
 
   describe '.requires_installation?' do
     it 'should not require installation' do
-      Elasticity::CustomJarStep.requires_installation?.should be_false
+      expect(Elasticity::CustomJarStep.requires_installation?).to be false
     end
   end