elasticity 1.5 → 2.0

data/spec/lib/elasticity/job_flow_status_spec.rb

@@ -0,0 +1,147 @@
+describe Elasticity::JobFlowStatus do
+
+  before do
+    describe_jobflows_xml = <<-JOBFLOWS
+      <DescribeJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
+        <DescribeJobFlowsResult>
+          <JobFlows>
+            <member>
+              <JobFlowId>j-p</JobFlowId>
+              <Name>Pig Job</Name>
+              <ExecutionStatusDetail>
+                <CreationDateTime>
+                   2011-10-04T21:49:16Z
+                </CreationDateTime>
+                <LastStateChangeReason>
+                   Steps completed with errors
+                </LastStateChangeReason>
+                <StartDateTime>
+                   2011-10-04T21:49:17Z
+                </StartDateTime>
+                <ReadyDateTime>
+                   2011-10-04T21:49:18Z
+                </ReadyDateTime>
+                <State>TERMINATED</State>
+              </ExecutionStatusDetail>
+              <Steps>
+                <member>
+                  <StepConfig>
+                    <Name>Setup Hive</Name>
+                  </StepConfig>
+                  <ExecutionStatusDetail>
+                    <State>FAILED</State>
+                  </ExecutionStatusDetail>
+                </member>
+                <member>
+                  <StepConfig>
+                    <Name>Run Hive Script</Name>
+                  </StepConfig>
+                  <ExecutionStatusDetail>
+                    <State>PENDING</State>
+                  </ExecutionStatusDetail>
+                </member>
+              </Steps>
+              <Instances>
+                 <Placement>
+                    <AvailabilityZone>
+                      eu-west-1a
+                    </AvailabilityZone>
+                 </Placement>
+                 <SlaveInstanceType>
+                    m1.small
+                 </SlaveInstanceType>
+                 <MasterInstanceType>
+                    m1.small
+                 </MasterInstanceType>
+                 <Ec2KeyName>
+                    myec2keyname
+                 </Ec2KeyName>
+                 <InstanceCount>
+                    4
+                 </InstanceCount>
+              </Instances>
+            </member>
+            <member>
+              <JobFlowId>j-h</JobFlowId>
+              <Name>Hive Job</Name>
+              <ExecutionStatusDetail>
+                <CreationDateTime>
+                   2011-10-04T22:49:16Z
+                </CreationDateTime>
+                <StartDateTime>
+
+                </StartDateTime>
+                <ReadyDateTime>
+
+                </ReadyDateTime>
+                <State>
+                  TERMINATED
+                </State>
+                <LastStateChangeReason>
+                  Steps completed
+                </LastStateChangeReason>
+              </ExecutionStatusDetail>
+              <Instances>
+                 <Placement>
+                    <AvailabilityZone>
+                      eu-west-1b
+                    </AvailabilityZone>
+                 </Placement>
+                 <SlaveInstanceType>
+                    c1.medium
+                 </SlaveInstanceType>
+                 <MasterInstanceType>
+                    c1.medium
+                 </MasterInstanceType>
+                 <Ec2KeyName>
+                    myec2keyname
+                 </Ec2KeyName>
+                 <InstanceCount>
+                    2
+                 </InstanceCount>
+              </Instances>
+            </member>
+          </JobFlows>
+        </DescribeJobFlowsResult>
+      </DescribeJobFlowsResponse>
+    JOBFLOWS
+    describe_jobflows_document = Nokogiri::XML(describe_jobflows_xml)
+    describe_jobflows_document.remove_namespaces!
+    @members_nodeset = describe_jobflows_document.xpath('/DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member')
+  end
+
+  describe ".from_xml" do
+    it "should return a JobFlow with the appropriate fields initialized" do
+      jobflow = Elasticity::JobFlowStatus.from_member_element(@members_nodeset[0])
+      jobflow.name.should == "Pig Job"
+      jobflow.jobflow_id.should == "j-p"
+      jobflow.state.should == "TERMINATED"
+      jobflow.steps.map(&:name).should == ["Setup Hive", "Run Hive Script"]
+      jobflow.steps.map(&:state).should == ["FAILED", "PENDING"]
+      jobflow.created_at.should == Time.parse("2011-10-04T21:49:16Z")
+      jobflow.started_at.should == Time.parse("2011-10-04T21:49:17Z")
+      jobflow.ready_at.should == Time.parse("2011-10-04T21:49:18Z")
+      jobflow.master_instance_type.should == "m1.small"
+      jobflow.slave_instance_type.should == "m1.small"
+      jobflow.instance_count.should == "4"
+      jobflow.last_state_change_reason.should == "Steps completed with errors"
+    end
+  end
+
+  describe ".from_jobflows_nodeset" do
+    it "should return JobFlows with the appropriate fields initialized" do
+      jobflow = Elasticity::JobFlowStatus.from_members_nodeset(@members_nodeset)
+      jobflow.map(&:name).should == ["Pig Job", "Hive Job"]
+      jobflow.map(&:jobflow_id).should == ["j-p", "j-h"]
+      jobflow.map(&:state).should == ["TERMINATED", "TERMINATED"]
+      jobflow.map(&:created_at).should == [Time.parse("2011-10-04T21:49:16Z"), Time.parse("2011-10-04T22:49:16Z")]
+      jobflow.map(&:started_at).should == [Time.parse("2011-10-04T21:49:17Z"), nil]
+      jobflow.map(&:ready_at).should == [Time.parse("2011-10-04T21:49:18Z"), nil]
+      jobflow.map(&:master_instance_type).should == ["m1.small","c1.medium"]
+      jobflow.map(&:slave_instance_type).should == ["m1.small", "c1.medium"]
+      jobflow.map(&:instance_count).should == ["4","2"]
+      jobflow.map(&:last_state_change_reason).should == ["Steps completed with errors", "Steps completed"]
+    end
+  end
+
+end

data/spec/lib/elasticity/job_flow_status_step_spec.rb

@@ -0,0 +1,73 @@
+describe Elasticity::JobFlowStatusStep do
+
+  before do
+    describe_jobflows_xml = <<-JOBFLOWS
+      <DescribeJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
+        <DescribeJobFlowsResult>
+          <JobFlows>
+            <member>
+              <JobFlowId>j-p</JobFlowId>
+              <Name>Pig Job</Name>
+              <ExecutionStatusDetail>
+                <State>TERMINATED</State>
+              </ExecutionStatusDetail>
+              <Steps>
+                <member>
+                  <StepConfig>
+                    <Name>Setup Hive</Name>
+                  </StepConfig>
+                  <ExecutionStatusDetail>
+                    <State>FAILED</State>
+                    <StartDateTime>
+                       2011-10-04T21:49:16Z
+                    </StartDateTime>
+                    <EndDateTime>
+                       2011-10-04T21:51:16Z
+                    </EndDateTime>
+                  </ExecutionStatusDetail>
+                </member>
+                <member>
+                  <StepConfig>
+                    <Name>Run Hive Script</Name>
+                  </StepConfig>
+                  <ExecutionStatusDetail>
+                    <State>PENDING</State>
+                    <StartDateTime>
+                    </StartDateTime>
+                    <EndDateTime>
+                    </EndDateTime>
+                  </ExecutionStatusDetail>
+                </member>
+              </Steps>
+            </member>
+          </JobFlows>
+        </DescribeJobFlowsResult>
+      </DescribeJobFlowsResponse>
+    JOBFLOWS
+    describe_jobflows_document = Nokogiri::XML(describe_jobflows_xml)
+    describe_jobflows_document.remove_namespaces!
+    @members_nodeset = describe_jobflows_document.xpath('/DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member/Steps/member')
+  end
+
+  describe ".from_xml" do
+    it "should return a JobFlowStep with the appropriate fields initialized" do
+      jobflow_step = Elasticity::JobFlowStatusStep.from_member_element(@members_nodeset[0])
+      jobflow_step.name.should == "Setup Hive"
+      jobflow_step.state.should == "FAILED"
+      jobflow_step.started_at.should == Time.parse("2011-10-04T21:49:16Z")
+      jobflow_step.ended_at.should == Time.parse("2011-10-04T21:51:16Z")
+    end
+  end
+
+  describe ".from_steps_nodeset" do
+    it "should return JobFlowSteps with the appropriate fields initialized" do
+      jobflow_steps = Elasticity::JobFlowStatusStep.from_members_nodeset(@members_nodeset)
+      jobflow_steps.map(&:name).should == ["Setup Hive", "Run Hive Script"]
+      jobflow_steps.map(&:state).should == ["FAILED", "PENDING"]
+      jobflow_steps.map(&:started_at).should == [Time.parse("2011-10-04T21:49:16Z"), nil]
+      jobflow_steps.map(&:ended_at).should == [Time.parse("2011-10-04T21:51:16Z"), nil]
+    end
+  end
+
+
+end

data/spec/lib/elasticity/job_flow_step_spec.rb

@@ -1,75 +1,37 @@
-require 'spec_helper'
-
 describe Elasticity::JobFlowStep do
 
-  before do
-    describe_jobflows_xml = <<-JOBFLOWS
-      <DescribeJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
-        <DescribeJobFlowsResult>
-          <JobFlows>
-            <member>
-              <JobFlowId>j-p</JobFlowId>
-              <Name>Pig Job</Name>
-              <ExecutionStatusDetail>
-                <State>TERMINATED</State>
-              </ExecutionStatusDetail>
-              <Steps>
-                <member>
-                  <StepConfig>
-                    <Name>Setup Hive</Name>
-                  </StepConfig>
-                  <ExecutionStatusDetail>
-                    <State>FAILED</State>
-                    <StartDateTime>
-                       2011-10-04T21:49:16Z
-                    </StartDateTime>
-                    <EndDateTime>
-                       2011-10-04T21:51:16Z
-                    </EndDateTime>
-                  </ExecutionStatusDetail>
-                </member>
-                <member>
-                  <StepConfig>
-                    <Name>Run Hive Script</Name>
-                  </StepConfig>
-                  <ExecutionStatusDetail>
-                    <State>PENDING</State>
-                    <StartDateTime>
-                    </StartDateTime>
-                    <EndDateTime>
-                    </EndDateTime>
-                  </ExecutionStatusDetail>
-                </member>
-              </Steps>
-            </member>
-          </JobFlows>
-        </DescribeJobFlowsResult>
-      </DescribeJobFlowsResponse>
-    JOBFLOWS
-    describe_jobflows_document = Nokogiri::XML(describe_jobflows_xml)
-    describe_jobflows_document.remove_namespaces!
-    @members_nodeset = describe_jobflows_document.xpath('/DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member/Steps/member')
+  class FakeStep
+    include Elasticity::JobFlowStep
   end
 
-  describe ".from_xml" do
-    it "should return a JobFlowStep with the appropriate fields initialized" do
-      jobflow_step = Elasticity::JobFlowStep.from_member_element(@members_nodeset[0])
-      jobflow_step.name.should == "Setup Hive"
-      jobflow_step.state.should == "FAILED"
-      jobflow_step.started_at.should == Time.parse("2011-10-04T21:49:16Z")
-      jobflow_step.ended_at.should == Time.parse("2011-10-04T21:51:16Z")
+  subject { FakeStep.new }
+
+  describe '#to_aws_step' do
+
+    it 'should take a job flow as context' do
+      subject.should respond_to(:to_aws_step).with(1).argument
+    end
+
+    it 'should raise an error by default' do
+      expect {
+        subject.to_aws_step(nil)
+      }.to raise_error(RuntimeError, '#to_aws_step is required to be defined on all job flow steps.')
     end
+
   end
 
-  describe ".from_steps_nodeset" do
-    it "should return JobFlowSteps with the appropriate fields initialized" do
-      jobflow_steps = Elasticity::JobFlowStep.from_members_nodeset(@members_nodeset)
-      jobflow_steps.map(&:name).should == ["Setup Hive", "Run Hive Script"]
-      jobflow_steps.map(&:state).should == ["FAILED", "PENDING"]
-      jobflow_steps.map(&:started_at).should == [Time.parse("2011-10-04T21:49:16Z"), nil]
-      jobflow_steps.map(&:ended_at).should == [Time.parse("2011-10-04T21:51:16Z"), nil]
+  describe '.requires_installation?' do
+    it 'should be false by default' do
+      FakeStep.requires_installation?.should be_false
     end
   end
 
+  describe '.aws_installation_step' do
+    it 'should raise an error by default' do
+      expect {
+        FakeStep.aws_installation_step
+      }.to raise_error(RuntimeError, '.aws_installation_step is required to be defined when a step requires installation (e.g. Pig, Hive).')
+    end
+  end
 
-end
+end

data/spec/lib/elasticity/pig_step_spec.rb

@@ -0,0 +1,104 @@
+describe Elasticity::PigStep do
+
+  subject do
+    Elasticity::PigStep.new('script.pig')
+  end
+
+  it { should be_a Elasticity::JobFlowStep }
+
+  its(:name) { should == 'Elasticity Pig Step (script.pig)' }
+  its(:script) { should == 'script.pig' }
+  its(:variables) { should == { } }
+  its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
+
+  describe '#to_aws_step' do
+
+    it 'should convert to aws step format' do
+      step = subject.to_aws_step(Elasticity::JobFlow.new('access', 'secret'))
+      step[:name].should == 'Elasticity Pig Step (script.pig)'
+      step[:action_on_failure].should == 'TERMINATE_JOB_FLOW'
+      step[:hadoop_jar_step][:jar].should == 's3://elasticmapreduce/libs/script-runner/script-runner.jar'
+      step[:hadoop_jar_step][:args].should start_with([
+        's3://elasticmapreduce/libs/pig/pig-script',
+          '--run-pig-script',
+          '--args',
+          '-p'
+      ])
+      step[:hadoop_jar_step][:args][4] =~ /^E_PARALLELS=\d+$/
+    end
+
+    describe 'E_PARALLELS' do
+      it 'should include the correct value of E_PARALLELS' do
+        job_flow = Elasticity::JobFlow.new('access', 'secret')
+        job_flow.instance_count = 8
+        {
+          '_' => 7,
+          'm1.small' => 7,
+          'm1.large' => 13,
+          'c1.medium' => 13,
+          'c1.xlarge' => 26
+        }.each do |instance_type, value|
+          job_flow.slave_instance_type = instance_type
+          step = subject.to_aws_step(job_flow)
+          step[:hadoop_jar_step][:args][4].should == "E_PARALLELS=#{value}"
+        end
+      end
+    end
+
+    context 'when variables are not provided' do
+      let(:ps_with_no_variables) { Elasticity::PigStep.new('script.pig') }
+
+      it 'should convert to aws step format' do
+        step = ps_with_no_variables.to_aws_step(Elasticity::JobFlow.new('access', 'secret'))
+        step[:hadoop_jar_step][:args][5].should == 'script.pig'
+      end
+    end
+
+    context 'when variables are provided' do
+      let(:ps_with_variables) do
+        Elasticity::PigStep.new('script.pig').tap do |ps|
+          ps.variables = {
+            'VAR1' => 'VALUE1',
+            'VAR2' => 'VALUE2'
+          }
+        end
+      end
+
+      it 'should convert to aws step format' do
+        step = ps_with_variables.to_aws_step(Elasticity::JobFlow.new('access', 'secret'))
+        step[:hadoop_jar_step][:args][3..9].should == [
+          '-p', 'VAR1=VALUE1',
+            '-p', 'VAR2=VALUE2',
+            '-p', 'E_PARALLELS=1',
+            'script.pig'
+        ]
+      end
+    end
+
+  end
+
+  describe '.requires_installation?' do
+    it 'should require installation' do
+      Elasticity::PigStep.requires_installation?.should be_true
+    end
+  end
+
+  describe '.aws_installation_step' do
+    it 'should provide a means to install Pig' do
+      Elasticity::PigStep.aws_installation_step.should == {
+        :action_on_failure => 'TERMINATE_JOB_FLOW',
+        :hadoop_jar_step => {
+          :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
+          :args => [
+            's3://elasticmapreduce/libs/pig/pig-script',
+              '--base-path',
+              's3://elasticmapreduce/libs/pig/',
+              '--install-pig'
+          ],
+        },
+        :name => 'Elasticity - Install Pig'
+      }
+    end
+  end
+
+end

data/spec/lib/elasticity/support/conditional_raise_spec.rb

@@ -0,0 +1,35 @@
+describe 'Conditional Raising' do
+
+  describe '#raise_if' do
+
+    it 'should raise the specified error if the condition is true' do
+      expect {
+        raise_if true, RuntimeError, 'MESSAGE'
+      }.to raise_error(RuntimeError, 'MESSAGE')
+    end
+
+    it 'should not raise the specified error if the condition is false' do
+      expect {
+        raise_if false, RuntimeError, 'MESSAGE'
+      }.to_not raise_error(RuntimeError, 'MESSAGE')
+    end
+
+  end
+
+  describe '#raise_unless' do
+
+    it 'should not raise the specified error unless the condition is true' do
+      expect {
+        raise_unless true, RuntimeError, 'MESSAGE'
+      }.to_not raise_error(RuntimeError, 'MESSAGE')
+    end
+
+    it 'should raise the specified error unless the condition is false' do
+      expect {
+        raise_unless false, RuntimeError, 'MESSAGE'
+      }.to raise_error(RuntimeError, 'MESSAGE')
+    end
+
+  end
+
+end