elasticity 1.5 → 2.0

data/spec/lib/elasticity/hadoop_bootstrap_action_spec.rb

@@ -0,0 +1,26 @@
+describe Elasticity::HadoopBootstrapAction do
+
+  subject do
+    Elasticity::HadoopBootstrapAction.new('option', 'value')
+  end
+
+  its(:name) { should == 'Elasticity Bootstrap Action (Configure Hadoop)' }
+  its(:option) { should == 'option' }
+  its(:value) { should == 'value' }
+
+  describe '#to_aws_bootstrap_action' do
+
+    it 'should create a bootstrap action' do
+      subject.to_aws_bootstrap_action.should ==
+        {
+          :name => 'Elasticity Bootstrap Action (Configure Hadoop)',
+          :script_bootstrap_action => {
+            :path => 's3n://elasticmapreduce/bootstrap-actions/configure-hadoop',
+            :args => ['option', 'value']
+          }
+        }
+    end
+
+  end
+
+end

data/spec/lib/elasticity/hive_step_spec.rb

@@ -0,0 +1,74 @@
+describe Elasticity::HiveStep do
+
+  subject do
+    Elasticity::HiveStep.new('script.hql')
+  end
+
+  it { should be_a Elasticity::JobFlowStep }
+
+  its(:name) { should == 'Elasticity Hive Step (script.hql)' }
+  its(:script) { should == 'script.hql' }
+  its(:variables) { should == { } }
+  its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
+
+  describe '#to_aws_step' do
+
+    it 'should convert to aws step format' do
+      step = subject.to_aws_step(Elasticity::JobFlow.new('access', 'secret'))
+      step[:name].should == 'Elasticity Hive Step (script.hql)'
+      step[:action_on_failure].should == 'TERMINATE_JOB_FLOW'
+      step[:hadoop_jar_step][:jar].should == 's3://elasticmapreduce/libs/script-runner/script-runner.jar'
+      step[:hadoop_jar_step][:args].should start_with([
+        's3://elasticmapreduce/libs/hive/hive-script',
+          '--run-hive-script',
+          '--args',
+          '-f',
+          'script.hql'
+      ])
+    end
+
+    context 'when variables are provided' do
+      let(:hs_with_variables) do
+        Elasticity::HiveStep.new('script.pig').tap do |hs|
+          hs.variables = {
+            'VAR1' => 'VALUE1',
+            'VAR2' => 'VALUE2'
+          }
+        end
+      end
+
+      it 'should convert to aws step format' do
+        step = hs_with_variables.to_aws_step(Elasticity::JobFlow.new('access', 'secret'))
+        step[:hadoop_jar_step][:args][5..9].should == %w(-d VAR1=VALUE1 -d VAR2=VALUE2)
+      end
+    end
+
+  end
+
+  describe '.requires_installation?' do
+    it 'should require installation' do
+      Elasticity::HiveStep.requires_installation?.should be_true
+    end
+  end
+
+  describe '.aws_installation_step' do
+
+    it 'should provide a means to install Hive' do
+      Elasticity::HiveStep.aws_installation_step.should == {
+        :action_on_failure => 'TERMINATE_JOB_FLOW',
+        :hadoop_jar_step => {
+          :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
+          :args => [
+            's3://elasticmapreduce/libs/hive/hive-script',
+              '--base-path',
+              's3://elasticmapreduce/libs/hive/',
+              '--install-hive'
+          ],
+        },
+        :name => 'Elasticity - Install Hive'
+      }
+    end
+
+  end
+
+end

data/spec/lib/elasticity/job_flow_integration_spec.rb

@@ -0,0 +1,197 @@
+describe 'Elasticity::JobFlow Integration Examples' do
+
+  let(:emr) { double('Elasticity::EMR') }
+
+  before do
+    Elasticity::EMR.should_receive(:new).with('access', 'secret').and_return(emr)
+  end
+
+  describe 'Hive' do
+
+    let(:hive_step) do
+      Elasticity::HiveStep.new('s3n://slif-hive/test.q').tap do |hs|
+        hs.variables = {'OUTPUT' => 's3n://slif-test/output'}
+        hs.action_on_failure = 'CONTINUE'
+      end
+    end
+
+    let(:hive_jobflow) do
+      Elasticity::JobFlow.new('access', 'secret').tap do |jf|
+        jf.log_uri = 's3n://slif-test/output/logs'
+        jf.add_step(hive_step)
+      end
+    end
+
+    it 'should launch the Hive job with the specified EMR credentials' do
+      emr.should_receive(:run_job_flow).with({
+        :name => 'Elasticity Job Flow',
+        :log_uri => 's3n://slif-test/output/logs',
+        :ami_version => 'latest',
+        :instances => {
+          :keep_job_flow_alive_when_no_steps => false,
+          :ec2_key_name => 'default',
+          :hadoop_version => '0.20.205',
+          :instance_count => 2,
+          :master_instance_type => 'm1.small',
+          :slave_instance_type => 'm1.small',
+        },
+        :steps => [
+          {
+            :action_on_failure => 'TERMINATE_JOB_FLOW',
+            :hadoop_jar_step => {
+              :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
+              :args => [
+                's3://elasticmapreduce/libs/hive/hive-script',
+                  '--base-path',
+                  's3://elasticmapreduce/libs/hive/',
+                  '--install-hive'
+              ],
+            },
+            :name => 'Elasticity - Install Hive'
+          },
+            {
+              :action_on_failure => 'CONTINUE',
+              :hadoop_jar_step => {
+                :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
+                :args => [
+                  's3://elasticmapreduce/libs/hive/hive-script',
+                    '--run-hive-script',
+                    '--args',
+                    '-f', 's3n://slif-hive/test.q',
+                    '-d', 'OUTPUT=s3n://slif-test/output'
+                ],
+              },
+              :name => 'Elasticity Hive Step (s3n://slif-hive/test.q)'
+            }
+        ]
+      }).and_return('HIVE_JOBFLOW_ID')
+
+      hive_jobflow.run.should == 'HIVE_JOBFLOW_ID'
+    end
+
+  end
+
+  describe 'Pig' do
+
+    let(:pig_step) do
+      Elasticity::PigStep.new('s3n://slif-pig-test/test.pig').tap do |ps|
+        ps.variables = {'OUTPUT' => 's3n://slif-pig-test/output', 'XREFS' => 's3n://slif-pig-test/xrefs'}
+        ps.action_on_failure = 'CONTINUE'
+      end
+    end
+
+    let(:pig_jobflow) do
+      Elasticity::JobFlow.new('access', 'secret').tap do |jf|
+        jf.instance_count = 8
+        jf.slave_instance_type = 'm1.xlarge'
+        jf.log_uri = 's3n://slif-test/output/logs'
+        jf.add_step(pig_step)
+      end
+    end
+
+    it 'should launch the Pig job with the specified EMR credentials' do
+      emr.should_receive(:run_job_flow).with({
+        :name => 'Elasticity Job Flow',
+        :log_uri => 's3n://slif-test/output/logs',
+        :ami_version => 'latest',
+        :instances => {
+          :keep_job_flow_alive_when_no_steps => false,
+          :ec2_key_name => 'default',
+          :hadoop_version => '0.20.205',
+          :instance_count => 8,
+          :master_instance_type => 'm1.small',
+          :slave_instance_type => 'm1.xlarge',
+        },
+        :steps => [
+          {
+            :action_on_failure => 'TERMINATE_JOB_FLOW',
+            :hadoop_jar_step => {
+              :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
+              :args => [
+                's3://elasticmapreduce/libs/pig/pig-script',
+                  '--base-path',
+                  's3://elasticmapreduce/libs/pig/',
+                  '--install-pig'
+              ],
+            },
+            :name => 'Elasticity - Install Pig'
+          },
+            {
+              :action_on_failure => 'CONTINUE',
+              :hadoop_jar_step => {
+                :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
+                :args => [
+                  's3://elasticmapreduce/libs/pig/pig-script',
+                    '--run-pig-script',
+                    '--args',
+                    '-p', 'OUTPUT=s3n://slif-pig-test/output',
+                    '-p', 'XREFS=s3n://slif-pig-test/xrefs',
+                    '-p', 'E_PARALLELS=26',
+                    's3n://slif-pig-test/test.pig'
+                ],
+              },
+              :name => 'Elasticity Pig Step (s3n://slif-pig-test/test.pig)'
+            }
+        ]
+      }).and_return('PIG_JOBFLOW_ID')
+
+      pig_jobflow.run.should == 'PIG_JOBFLOW_ID'
+    end
+
+  end
+
+  describe 'Custom Jar' do
+
+    let(:custom_jar_step) do
+      Elasticity::CustomJarStep.new('s3n://elasticmapreduce/samples/cloudburst/cloudburst.jar').tap do |cj|
+        cj.arguments = [
+          's3n://elasticmapreduce/samples/cloudburst/input/s_suis.br',
+            's3n://elasticmapreduce/samples/cloudburst/input/100k.br',
+            's3n://slif_hadoop_test/cloudburst/output/2011-12-09',
+        ]
+        cj.action_on_failure = 'TERMINATE_JOB_FLOW'
+      end
+    end
+
+    let(:custom_jar_jobflow) do
+      Elasticity::JobFlow.new('access', 'secret').tap do |jf|
+        jf.log_uri = 's3n://slif-test/output/logs'
+        jf.add_step(custom_jar_step)
+      end
+    end
+
+    it 'should launch the Custom Jar job with the specified EMR credentials' do
+      emr.should_receive(:run_job_flow).with({
+        :name => 'Elasticity Job Flow',
+        :log_uri => 's3n://slif-test/output/logs',
+        :ami_version => 'latest',
+        :instances => {
+          :keep_job_flow_alive_when_no_steps => false,
+          :ec2_key_name => 'default',
+          :hadoop_version => '0.20.205',
+          :instance_count => 2,
+          :master_instance_type => 'm1.small',
+          :slave_instance_type => 'm1.small',
+        },
+        :steps => [
+          {
+            :action_on_failure => 'TERMINATE_JOB_FLOW',
+            :hadoop_jar_step => {
+              :jar => 's3n://elasticmapreduce/samples/cloudburst/cloudburst.jar',
+              :args => [
+                's3n://elasticmapreduce/samples/cloudburst/input/s_suis.br',
+                  's3n://elasticmapreduce/samples/cloudburst/input/100k.br',
+                  's3n://slif_hadoop_test/cloudburst/output/2011-12-09',
+              ],
+            },
+            :name => 'Elasticity Custom Jar Step (s3n://elasticmapreduce/samples/cloudburst/cloudburst.jar)'
+          }
+        ]
+      }).and_return('CUSTOM_JAR_JOBFLOW_ID')
+
+      custom_jar_jobflow.run.should == 'CUSTOM_JAR_JOBFLOW_ID'
+    end
+
+  end
+
+end

data/spec/lib/elasticity/job_flow_spec.rb

@@ -1,149 +1,380 @@
-require 'spec_helper'
-
 describe Elasticity::JobFlow do
 
   before do
-    describe_jobflows_xml = <<-JOBFLOWS
-      <DescribeJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
-        <DescribeJobFlowsResult>
-          <JobFlows>
-            <member>
-              <JobFlowId>j-p</JobFlowId>
-              <Name>Pig Job</Name>
-              <ExecutionStatusDetail>
-                <CreationDateTime>
-                   2011-10-04T21:49:16Z
-                </CreationDateTime>
-                <LastStateChangeReason>
-                   Steps completed with errors
-                </LastStateChangeReason>
-                <StartDateTime>
-                   2011-10-04T21:49:17Z
-                </StartDateTime>
-                <ReadyDateTime>
-                   2011-10-04T21:49:18Z
-                </ReadyDateTime>
-                <State>TERMINATED</State>
-              </ExecutionStatusDetail>
-              <Steps>
-                <member>
-                  <StepConfig>
-                    <Name>Setup Hive</Name>
-                  </StepConfig>
-                  <ExecutionStatusDetail>
-                    <State>FAILED</State>
-                  </ExecutionStatusDetail>
-                </member>
-                <member>
-                  <StepConfig>
-                    <Name>Run Hive Script</Name>
-                  </StepConfig>
-                  <ExecutionStatusDetail>
-                    <State>PENDING</State>
-                  </ExecutionStatusDetail>
-                </member>
-              </Steps>
-              <Instances>
-                 <Placement>
-                    <AvailabilityZone>
-                      eu-west-1a
-                    </AvailabilityZone>
-                 </Placement>
-                 <SlaveInstanceType>
-                    m1.small
-                 </SlaveInstanceType>
-                 <MasterInstanceType>
-                    m1.small
-                 </MasterInstanceType>
-                 <Ec2KeyName>
-                    myec2keyname
-                 </Ec2KeyName>
-                 <InstanceCount>
-                    4
-                 </InstanceCount>
-              </Instances>
-            </member>
-            <member>
-              <JobFlowId>j-h</JobFlowId>
-              <Name>Hive Job</Name>
-              <ExecutionStatusDetail>
-                <CreationDateTime>
-                   2011-10-04T22:49:16Z
-                </CreationDateTime>
-                <StartDateTime>
-
-                </StartDateTime>
-                <ReadyDateTime>
-                   
-                </ReadyDateTime>
-                <State>
-                  TERMINATED
-                </State>
-                <LastStateChangeReason>
-                  Steps completed
-                </LastStateChangeReason>
-              </ExecutionStatusDetail>
-              <Instances>
-                 <Placement>
-                    <AvailabilityZone>
-                      eu-west-1b
-                    </AvailabilityZone>
-                 </Placement>
-                 <SlaveInstanceType>
-                    c1.medium
-                 </SlaveInstanceType>
-                 <MasterInstanceType>
-                    c1.medium
-                 </MasterInstanceType>
-                 <Ec2KeyName>
-                    myec2keyname
-                 </Ec2KeyName>
-                 <InstanceCount>
-                    2
-                 </InstanceCount>
-              </Instances>
-            </member>
-          </JobFlows>
-        </DescribeJobFlowsResult>
-      </DescribeJobFlowsResponse>
-    JOBFLOWS
-    describe_jobflows_document = Nokogiri::XML(describe_jobflows_xml)
-    describe_jobflows_document.remove_namespaces!
-    @members_nodeset = describe_jobflows_document.xpath('/DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member')
+    # Ensure we don't accidentally submit to EMR for all of these examples
+    Elasticity::EMR.stub(:new).and_return(double('Elasticity::EMR', :run_job_flow => '_'))
+  end
+
+  subject do
+    Elasticity::JobFlow.new('access', 'secret')
+  end
+
+  its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
+  its(:ec2_key_name) { should == 'default' }
+  its(:hadoop_version) { should == '0.20.205' }
+  its(:instance_count) { should == 2 }
+  its(:log_uri) { should == nil }
+  its(:master_instance_type) { should == 'm1.small' }
+  its(:name) { should == 'Elasticity Job Flow' }
+  its(:slave_instance_type) { should == 'm1.small' }
+  its(:ami_version) { should == 'latest' }
+  its(:keep_job_flow_alive_when_no_steps) { should == false }
+
+  describe '#instance_count=' do
+
+    context 'when set to more than 1' do
+      it 'should set the number of instances' do
+        subject.instance_count = 10
+        subject.instance_count.should == 10
+      end
+    end
+
+    context 'when set to less than 2' do
+      it 'should be an error' do
+        expect {
+          subject.instance_count = 1
+        }.to raise_error(ArgumentError, 'Instance count cannot be set to less than 2 (requested 1)')
+      end
+    end
+
+  end
+
+  describe '#add_bootstrap_action' do
+
+    context 'when the jobflow is not yet started' do
+      it 'should not raise an error' do
+        expect {
+          subject.add_bootstrap_action(nil)
+        }.to_not raise_error
+      end
+    end
+
+    context 'when the jobflow is already started' do
+      before do
+        subject.add_step(Elasticity::CustomJarStep.new('_'))
+        subject.run
+      end
+      it 'should raise an error' do
+        expect {
+          subject.add_bootstrap_action(nil)
+        }.to raise_error(Elasticity::JobFlowRunningError, 'To modify bootstrap actions, please create a new job flow.')
+      end
+    end
+
+  end
+
+  describe '#add_step' do
+
+    context 'when the jobflow is already running' do
+
+      let(:emr) { double('Elasticity::EMR', :run_job_flow => 'RUNNING_JOBFLOW_ID') }
+
+      let(:running_jobflow) do
+        Elasticity::JobFlow.new('access', 'secret').tap do |jf|
+          jf.add_step(Elasticity::PigStep.new('_'))
+        end
+      end
+
+      before do
+        Elasticity::EMR.should_receive(:new).with('access', 'secret').and_return(emr)
+        running_jobflow.run
+      end
+
+      context 'when the step requires installation' do
+
+        context 'when the installation has already happened' do
+          let(:additional_step) { Elasticity::PigStep.new('_') }
+
+          it 'should submit the step' do
+            emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', {
+              :steps => [additional_step.to_aws_step(running_jobflow)]
+            })
+            running_jobflow.add_step(additional_step)
+          end
+        end
+
+        context 'when the installation has not yet happened' do
+          let(:additional_step) { Elasticity::HiveStep.new('_') }
+
+          it 'should submit the installation step and the step' do
+            emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', {
+              :steps => [
+                Elasticity::HiveStep.aws_installation_step,
+                  additional_step.to_aws_step(running_jobflow)
+              ]
+            })
+            running_jobflow.add_step(additional_step)
+          end
+        end
+
+      end
+
+      context 'when the step does not require installation' do
+
+        let(:additional_step) { Elasticity::CustomJarStep.new('jar') }
+
+        it 'should submit the step' do
+          emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', {
+            :steps => [additional_step.to_aws_step(running_jobflow)]
+          })
+          running_jobflow.add_step(additional_step)
+        end
+
+      end
+
+    end
+
+    context 'when the jobflow is not yet running' do
+      # This behaviour is tested in #jobflow_config
+    end
+  end
+
+  describe '#jobflow_config' do
+
+    it 'should incorporate the job flow preamble' do
+      subject.stub(:jobflow_preamble).and_return({:preamble => 'PREAMBLE'})
+      subject.send(:jobflow_config).should be_a_hash_including({:preamble => 'PREAMBLE'})
+    end
+
+    describe 'steps' do
+
+      let(:jobflow_steps) { [Elasticity::HiveStep.new('script.hql'), Elasticity::PigStep.new('script.pig'), Elasticity::CustomJarStep.new('script.jar')] }
+      let(:jobflow_with_steps) do
+        Elasticity::JobFlow.new('_', '_').tap do |jobflow|
+          jobflow_steps.each { |s| jobflow.add_step(s) }
+        end
+      end
+      let(:aws_steps) do
+        [
+          Elasticity::HiveStep.aws_installation_step,
+            jobflow_steps[0].to_aws_step(jobflow_with_steps),
+            Elasticity::PigStep.aws_installation_step,
+            jobflow_steps[1].to_aws_step(jobflow_with_steps),
+            jobflow_steps[2].to_aws_step(jobflow_with_steps),
+        ]
+      end
+
+      it 'should incorporate the installation and run steps into the jobflow config' do
+        jobflow_with_steps.send(:jobflow_config).should be_a_hash_including({:steps => aws_steps})
+      end
+
+      context 'when there are more than one installable step of the same type' do
+        before do
+          jobflow_steps << Elasticity::HiveStep.new('script.hql')
+          aws_steps << jobflow_steps.last.to_aws_step(jobflow_with_steps)
+        end
+        it 'should not include the installation step more than once' do
+          jobflow_with_steps.send(:jobflow_config).should be_a_hash_including({:steps => aws_steps})
+        end
+      end
+
+    end
+
+    describe 'log URI' do
+
+      context 'when a log URI is specified' do
+        let(:jobflow_with_log_uri) do
+          Elasticity::JobFlow.new('_', '_').tap do |jf|
+            jf.log_uri = 'LOG_URI'
+          end
+        end
+        it 'should incorporate it into the jobflow config' do
+          jobflow_with_log_uri.send(:jobflow_config).should be_a_hash_including({:log_uri => 'LOG_URI'})
+        end
+      end
+
+      context 'when a log URI is not specified' do
+        let(:jobflow_with_no_log_uri) do
+          Elasticity::JobFlow.new('_', '_').tap do |jf|
+            jf.log_uri = nil
+          end
+        end
+        it 'should not make space for it in the jobflow config' do
+          jobflow_with_no_log_uri.send(:jobflow_config).should_not have_key(:log_uri)
+        end
+      end
+
+    end
+
+    describe 'bootstrap actions' do
+
+      context 'when bootstrap actions are specified' do
+        let(:hadoop_bootstrap_actions) do
+          [
+            Elasticity::HadoopBootstrapAction.new('OPTION1', 'VALUE1'),
+              Elasticity::HadoopBootstrapAction.new('OPTION1', 'VALUE2'),
+              Elasticity::HadoopBootstrapAction.new('OPTION2', 'VALUE3')
+          ]
+        end
+        let(:jobflow_with_bootstrap_actions) do
+          Elasticity::JobFlow.new('_', '_').tap do |jf|
+            hadoop_bootstrap_actions.each do |action|
+              jf.add_bootstrap_action(action)
+            end
+          end
+        end
+        it 'should include them in the jobflow config' do
+          bootstrap_actions = hadoop_bootstrap_actions.map { |a| a.to_aws_bootstrap_action }
+          jobflow_with_bootstrap_actions.send(:jobflow_config).should be_a_hash_including({
+            :bootstrap_actions => bootstrap_actions
+          })
+        end
+      end
+
+      context 'when bootstrap actions are not specified' do
+        it 'should not make space for them in the jobflow config' do
+          subject.send(:jobflow_config).should_not have_key(:bootstrap_actions)
+        end
+      end
+
+    end
+
   end
 
-  describe ".from_xml" do
-    it "should return a JobFlow with the appropriate fields initialized" do
-      jobflow = Elasticity::JobFlow.from_member_element(@members_nodeset[0])
-      jobflow.name.should == "Pig Job"
-      jobflow.jobflow_id.should == "j-p"
-      jobflow.state.should == "TERMINATED"
-      jobflow.steps.map(&:name).should == ["Setup Hive", "Run Hive Script"]
-      jobflow.steps.map(&:state).should == ["FAILED", "PENDING"]
-      jobflow.created_at.should == Time.parse("2011-10-04T21:49:16Z")
-      jobflow.started_at.should == Time.parse("2011-10-04T21:49:17Z")
-      jobflow.ready_at.should == Time.parse("2011-10-04T21:49:18Z")
-      jobflow.master_instance_type.should == "m1.small"
-      jobflow.slave_instance_type.should == "m1.small"
-      jobflow.instance_count.should == "4"
-      jobflow.last_state_change_reason.should == "Steps completed with errors"
+  describe '#jobflow_preamble' do
+
+    let(:basic_preamble) do
+      {
+        :name => 'Elasticity Job Flow',
+        :ami_version => 'latest',
+        :instances => {
+          :keep_job_flow_alive_when_no_steps => false,
+          :ec2_key_name => 'default',
+          :hadoop_version => '0.20.205',
+          :instance_count => 2,
+          :master_instance_type => 'm1.small',
+          :slave_instance_type => 'm1.small',
+        }
+      }
     end
+
+    it 'should create a jobflow configuration section' do
+      subject.send(:jobflow_preamble).should == basic_preamble
+    end
+
+    context 'when a VPC subnet ID is specified' do
+      it 'should include it in the preamble' do
+        subject.ec2_subnet_id = 'subnet-118b9d79'
+        subject.send(:jobflow_preamble).should be_a_hash_including({:ec2_subnet_id => 'subnet-118b9d79'})
+      end
+    end
+
   end
 
-  describe ".from_jobflows_nodeset" do
-    it "should return JobFlows with the appropriate fields initialized" do
-      jobflow = Elasticity::JobFlow.from_members_nodeset(@members_nodeset)
-      jobflow.map(&:name).should == ["Pig Job", "Hive Job"]
-      jobflow.map(&:jobflow_id).should == ["j-p", "j-h"]
-      jobflow.map(&:state).should == ["TERMINATED", "TERMINATED"]
-      jobflow.map(&:created_at).should == [Time.parse("2011-10-04T21:49:16Z"), Time.parse("2011-10-04T22:49:16Z")]
-      jobflow.map(&:started_at).should == [Time.parse("2011-10-04T21:49:17Z"), nil]
-      jobflow.map(&:ready_at).should == [Time.parse("2011-10-04T21:49:18Z"), nil]
-      jobflow.map(&:master_instance_type).should == ["m1.small","c1.medium"]
-      jobflow.map(&:slave_instance_type).should == ["m1.small", "c1.medium"]
-      jobflow.map(&:instance_count).should == ["4","2"]
-      jobflow.map(&:last_state_change_reason).should == ["Steps completed with errors", "Steps completed"]
+  describe '#run' do
+
+    context 'when there are steps added' do
+      let(:jobflow_with_steps) do
+        Elasticity::JobFlow.new('STEP_TEST_ACCESS', 'STEP_TEST_SECRET').tap do |jf|
+          jf.add_step(Elasticity::CustomJarStep.new('_'))
+        end
+      end
+
+      context 'when the jobflow has not yet been run' do
+        let(:emr) { double('Elasticity::EMR', :run_job_flow => 'JOBFLOW_ID') }
+
+        it 'should run the job with the supplied EMR credentials' do
+          Elasticity::EMR.should_receive(:new).with('STEP_TEST_ACCESS', 'STEP_TEST_SECRET').and_return(emr)
+          emr.should_receive(:run_job_flow)
+          jobflow_with_steps.run
+        end
+
+        it 'should run the job with the jobflow config' do
+          Elasticity::EMR.stub(:new).with('STEP_TEST_ACCESS', 'STEP_TEST_SECRET').and_return(emr)
+          jobflow_with_steps.stub(:jobflow_config).and_return('JOBFLOW_CONFIG')
+          emr.should_receive(:run_job_flow).with('JOBFLOW_CONFIG')
+          jobflow_with_steps.run
+        end
+
+        it 'should return the jobflow ID' do
+          Elasticity::EMR.stub(:new).with('STEP_TEST_ACCESS', 'STEP_TEST_SECRET').and_return(emr)
+          jobflow_with_steps.run.should == 'JOBFLOW_ID'
+        end
+
+      end
+
+      context 'when the jobflow has already been run' do
+        before do
+          jobflow_with_steps.run
+        end
+        it 'should raise an error' do
+          expect {
+            jobflow_with_steps.run
+          }.to raise_error(Elasticity::JobFlowRunningError, 'Cannot run a job flow multiple times.  To do more with this job flow, please use #add_step.')
+        end
+      end
+
+    end
+
+    context 'when there are no steps added' do
+      let(:jobflow_with_no_steps) { Elasticity::JobFlow.new('_', '_') }
+      it 'should raise an error' do
+        expect {
+          jobflow_with_no_steps.run
+        }.to raise_error(Elasticity::JobFlowMissingStepsError, 'Cannot run a job flow without adding steps.  Please use #add_step.')
+      end
+    end
+
+  end
+
+  describe '#status' do
+
+    context 'before the jobflow has been run' do
+      it 'should raise an error' do
+        expect {
+          subject.status
+        }.to raise_error(Elasticity::JobFlowNotStartedError, 'Please #run this job flow before attempting to retrieve status.')
+      end
+    end
+
+    context 'after the jobflow has been run' do
+      let(:emr) { double('Elasticity::EMR', :run_job_flow => 'JOBFLOW_ID') }
+      let(:running_jobflow) { Elasticity::JobFlow.new('_', '_') }
+      let(:jobflow_status) do
+        Elasticity::JobFlowStatus.new.tap do |js|
+          js.stub(:state => 'TERMINATED')
+        end
+      end
+      before do
+        Elasticity::EMR.stub(:new).and_return(emr)
+        running_jobflow.add_step(Elasticity::CustomJarStep.new('_'))
+        running_jobflow.run
+      end
+      it 'should return the AWS status' do
+        emr.should_receive(:describe_jobflow).with('JOBFLOW_ID').and_return(jobflow_status)
+        status = running_jobflow.status
+        status.should be_a(Elasticity::JobFlowStatus)
+        status.state.should == 'TERMINATED'
+      end
     end
+
+  end
+
+  describe '#shutdown' do
+
+    context 'when the jobflow has not yet been started' do
+      let(:unstarted_job_flow) { Elasticity::JobFlow.new('_', '_')}
+      it 'should be an error' do
+        expect {
+          unstarted_job_flow.shutdown
+        }.to raise_error(Elasticity::JobFlowNotStartedError, 'Cannot #shutdown a job flow that has not yet been #run.')
+      end
+    end
+
+    context 'when the jobflow has been started' do
+      let(:emr) { double('Elasticity::EMR', :run_job_flow => 'JOBFLOW_ID') }
+      let(:running_jobflow) { Elasticity::JobFlow.new('_', '_') }
+      before do
+        Elasticity::EMR.stub(:new).and_return(emr)
+        running_jobflow.add_step(Elasticity::CustomJarStep.new('_'))
+        running_jobflow.run
+      end
+      it 'should shutdown the running jobflow' do
+        emr.should_receive(:terminate_jobflows).with('JOBFLOW_ID')
+        running_jobflow.shutdown
+      end
+    end
+
   end
 
-end
+end