elasticity 5.0.3 → 6.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/HISTORY.md +26 -0
  3. data/README.md +35 -28
  4. data/elasticity.gemspec +2 -2
  5. data/lib/elasticity.rb +5 -3
  6. data/lib/elasticity/aws_request_v4.rb +15 -3
  7. data/lib/elasticity/aws_session.rb +4 -23
  8. data/lib/elasticity/aws_utils.rb +0 -29
  9. data/lib/elasticity/cluster_status.rb +38 -0
  10. data/lib/elasticity/cluster_step_status.rb +51 -0
  11. data/lib/elasticity/emr.rb +208 -78
  12. data/lib/elasticity/job_flow.rb +16 -17
  13. data/lib/elasticity/version.rb +1 -1
  14. data/spec/factories/cluster_status_factory.rb +12 -0
  15. data/spec/factories/cluster_step_status_factory.rb +17 -0
  16. data/spec/lib/elasticity/aws_request_v4_spec.rb +54 -4
  17. data/spec/lib/elasticity/aws_session_spec.rb +22 -88
  18. data/spec/lib/elasticity/aws_utils_spec.rb +0 -46
  19. data/spec/lib/elasticity/bootstrap_action_spec.rb +7 -3
  20. data/spec/lib/elasticity/cluster_status_spec.rb +98 -0
  21. data/spec/lib/elasticity/cluster_step_status_spec.rb +80 -0
  22. data/spec/lib/elasticity/custom_jar_step_spec.rb +10 -7
  23. data/spec/lib/elasticity/emr_spec.rb +422 -132
  24. data/spec/lib/elasticity/ganglia_bootstrap_action_spec.rb +8 -3
  25. data/spec/lib/elasticity/hadoop_bootstrap_action_spec.rb +8 -3
  26. data/spec/lib/elasticity/hadoop_file_bootstrap_action_spec.rb +7 -3
  27. data/spec/lib/elasticity/hive_step_spec.rb +21 -17
  28. data/spec/lib/elasticity/instance_group_spec.rb +9 -5
  29. data/spec/lib/elasticity/job_flow_integration_spec.rb +4 -4
  30. data/spec/lib/elasticity/job_flow_spec.rb +102 -76
  31. data/spec/lib/elasticity/job_flow_step_spec.rb +1 -1
  32. data/spec/lib/elasticity/looper_spec.rb +1 -1
  33. data/spec/lib/elasticity/pig_step_spec.rb +13 -9
  34. data/spec/lib/elasticity/s3distcp_step_spec.rb +7 -5
  35. data/spec/lib/elasticity/script_step_spec.rb +11 -6
  36. data/spec/lib/elasticity/setup_hadoop_debugging_step_spec.rb +9 -5
  37. data/spec/lib/elasticity/streaming_step_spec.rb +13 -9
  38. data/spec/spec_helper.rb +8 -0
  39. data/spec/support/factory_girl.rb +8 -0
  40. metadata +24 -21
  41. data/lib/elasticity/aws_request_v2.rb +0 -42
  42. data/lib/elasticity/job_flow_status.rb +0 -91
  43. data/lib/elasticity/job_flow_status_step.rb +0 -38
  44. data/spec/lib/elasticity/aws_request_v2_spec.rb +0 -38
  45. data/spec/lib/elasticity/job_flow_status_spec.rb +0 -265
  46. data/spec/lib/elasticity/job_flow_status_step_spec.rb +0 -80
@@ -6,8 +6,13 @@ describe Elasticity::GangliaBootstrapAction do
6
6
 
7
7
  it { should be_a Elasticity::BootstrapAction }
8
8
 
9
- its(:name) { should == 'Elasticity Bootstrap Action (Install Ganglia)' }
10
- its(:arguments) { should == [] }
11
- its(:script) { should == 's3://elasticmapreduce/bootstrap-actions/install-ganglia' }
9
+ describe '.intialize' do
10
+ it 'should set the fields appropriately' do
11
+ expect(subject.name).to eql('Elasticity Bootstrap Action (Install Ganglia)')
12
+ expect(subject.arguments).to eql([])
13
+ expect(subject.script).to eql('s3://elasticmapreduce/bootstrap-actions/install-ganglia')
14
+ end
15
+ end
16
+
12
17
 
13
18
  end
@@ -6,8 +6,13 @@ describe Elasticity::HadoopBootstrapAction do
6
6
 
7
7
  it { should be_a Elasticity::BootstrapAction }
8
8
 
9
- its(:name) { should == 'Elasticity Bootstrap Action (Configure Hadoop)' }
10
- its(:arguments) { should == %w(option value) }
11
- its(:script) { should == 's3n://elasticmapreduce/bootstrap-actions/configure-hadoop' }
9
+ describe '.initialize' do
10
+ it 'should set the fields appropriately' do
11
+ expect(subject.name).to eql('Elasticity Bootstrap Action (Configure Hadoop)')
12
+ expect(subject.arguments).to eql(%w(option value))
13
+ expect(subject.script).to eql('s3n://elasticmapreduce/bootstrap-actions/configure-hadoop')
14
+ end
15
+ end
16
+
12
17
 
13
18
  end
@@ -6,8 +6,12 @@ describe Elasticity::HadoopFileBootstrapAction do
6
6
 
7
7
  it { should be_a Elasticity::BootstrapAction }
8
8
 
9
- its(:name) { should == 'Elasticity Bootstrap Action (Configure Hadoop via File)' }
10
- its(:arguments) { should == %w(--mapred-config-file config_file) }
11
- its(:script) { should == 's3n://elasticmapreduce/bootstrap-actions/configure-hadoop' }
9
+ describe '.initialize' do
10
+ it 'should set the fields appropriately' do
11
+ expect(subject.name).to eql('Elasticity Bootstrap Action (Configure Hadoop via File)')
12
+ expect(subject.arguments).to eql(%w(--mapred-config-file config_file))
13
+ expect(subject.script).to eql('s3n://elasticmapreduce/bootstrap-actions/configure-hadoop')
14
+ end
15
+ end
12
16
 
13
17
  end
@@ -6,29 +6,33 @@ describe Elasticity::HiveStep do
6
6
 
7
7
  it { should be_a Elasticity::JobFlowStep }
8
8
 
9
- its(:name) { should == 'Elasticity Hive Step (script.hql)' }
10
- its(:script) { should == 'script.hql' }
11
- its(:variables) { should == {} }
12
- its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
9
+ describe '.initialize' do
10
+ it 'should set the fields appropriately' do
11
+ expect(subject.name).to eql('Elasticity Hive Step (script.hql)')
12
+ expect(subject.script).to eql('script.hql')
13
+ expect(subject.variables).to eql({})
14
+ expect(subject.action_on_failure).to eql('TERMINATE_JOB_FLOW')
15
+ end
16
+ end
13
17
 
14
18
  describe '#to_aws_step' do
15
19
 
16
20
  it 'should convert to aws step format' do
17
- step = subject.to_aws_step(Elasticity::JobFlow.new('access', 'secret'))
21
+ step = subject.to_aws_step(Elasticity::JobFlow.new)
18
22
  step[:name].should == 'Elasticity Hive Step (script.hql)'
19
23
  step[:action_on_failure].should == 'TERMINATE_JOB_FLOW'
20
24
  step[:hadoop_jar_step][:jar].should == 's3://elasticmapreduce/libs/script-runner/script-runner.jar'
21
25
  step[:hadoop_jar_step][:args].should start_with([
22
- 's3://elasticmapreduce/libs/hive/hive-script',
23
- '--base-path',
24
- 's3://elasticmapreduce/libs/hive/',
25
- '--hive-versions',
26
- 'latest',
27
- '--run-hive-script',
28
- '--args',
29
- '-f',
30
- 'script.hql'
31
- ])
26
+ 's3://elasticmapreduce/libs/hive/hive-script',
27
+ '--base-path',
28
+ 's3://elasticmapreduce/libs/hive/',
29
+ '--hive-versions',
30
+ 'latest',
31
+ '--run-hive-script',
32
+ '--args',
33
+ '-f',
34
+ 'script.hql'
35
+ ])
32
36
  end
33
37
 
34
38
  context 'when variables are provided' do
@@ -42,7 +46,7 @@ describe Elasticity::HiveStep do
42
46
  end
43
47
 
44
48
  it 'should convert to aws step format' do
45
- step = hs_with_variables.to_aws_step(Elasticity::JobFlow.new('access', 'secret'))
49
+ step = hs_with_variables.to_aws_step(Elasticity::JobFlow.new)
46
50
  step[:hadoop_jar_step][:args][9..13].should == %w(-d VAR1=VALUE1 -d VAR2=VALUE2)
47
51
  end
48
52
  end
@@ -51,7 +55,7 @@ describe Elasticity::HiveStep do
51
55
 
52
56
  describe '.requires_installation?' do
53
57
  it 'should require installation' do
54
- Elasticity::HiveStep.requires_installation?.should be_true
58
+ expect(Elasticity::HiveStep.requires_installation?).to be true
55
59
  end
56
60
  end
57
61
 
@@ -1,10 +1,14 @@
1
1
  describe Elasticity::InstanceGroup do
2
2
 
3
- its(:bid_price) { should == nil }
4
- its(:count) { should == 1 }
5
- its(:type) { should == 'm1.small' }
6
- its(:market) { should == 'ON_DEMAND' }
7
- its(:role) { should == 'CORE' }
3
+ describe '.initialize' do
4
+ it 'should set the fields appropriately' do
5
+ expect(subject.bid_price).to eql(nil)
6
+ expect(subject.count).to eql(1)
7
+ expect(subject.type).to eql('m1.small')
8
+ expect(subject.market).to eql('ON_DEMAND')
9
+ expect(subject.role).to eql('CORE')
10
+ end
11
+ end
8
12
 
9
13
  describe '#count=' do
10
14
 
@@ -3,7 +3,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
3
3
  let(:emr) { double('Elasticity::EMR') }
4
4
 
5
5
  before do
6
- Elasticity::EMR.should_receive(:new).with('access', 'secret', :region => 'us-west-1').and_return(emr)
6
+ Elasticity::EMR.should_receive(:new).with(:region => 'us-west-1').and_return(emr)
7
7
  end
8
8
 
9
9
  describe 'Hive' do
@@ -16,7 +16,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
16
16
  end
17
17
 
18
18
  let(:hive_jobflow) do
19
- Elasticity::JobFlow.new('access', 'secret').tap do |jf|
19
+ Elasticity::JobFlow.new.tap do |jf|
20
20
  jf.placement = 'us-west-1a'
21
21
  jf.log_uri = 's3n://slif-test/output/logs'
22
22
  jf.add_step(hive_step)
@@ -101,7 +101,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
101
101
  end
102
102
 
103
103
  let(:pig_jobflow) do
104
- Elasticity::JobFlow.new('access', 'secret').tap do |jf|
104
+ Elasticity::JobFlow.new.tap do |jf|
105
105
  jf.placement = 'us-west-1c'
106
106
  jf.instance_count = 8
107
107
  jf.slave_instance_type = 'm1.xlarge'
@@ -189,7 +189,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
189
189
  end
190
190
 
191
191
  let(:custom_jar_jobflow) do
192
- Elasticity::JobFlow.new('access', 'secret').tap do |jf|
192
+ Elasticity::JobFlow.new.tap do |jf|
193
193
  jf.placement = 'us-west-1b'
194
194
  jf.log_uri = 's3n://slif-test/output/logs'
195
195
  jf.add_step(custom_jar_step)
@@ -1,41 +1,30 @@
1
1
  describe Elasticity::JobFlow do
2
2
 
3
3
  subject do
4
- Elasticity::JobFlow.new('access', 'secret')
4
+ Elasticity::JobFlow.new
5
5
  end
6
6
 
7
- its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
8
- its(:ec2_key_name) { should == nil }
9
- its(:name) { should == 'Elasticity Job Flow' }
10
- its(:instance_count) { should == 2 }
11
- its(:log_uri) { should == nil }
12
- its(:master_instance_type) { should == 'm1.small' }
13
- its(:slave_instance_type) { should == 'm1.small' }
14
- its(:ami_version) { should == 'latest' }
15
- its(:keep_job_flow_alive_when_no_steps) { should == false }
16
- its(:ec2_subnet_id) { should == nil }
17
- its(:placement) { should == 'us-east-1a' }
18
- its(:region) { should == 'us-east-1' }
19
- its(:visible_to_all_users) { should == false }
20
- its(:enable_debugging) { should == false }
21
- its(:job_flow_role) { should == nil }
22
- its(:service_role) { should == nil }
23
-
24
- its(:access_key) { should == 'access' }
25
- its(:secret_key) { should == 'secret' }
26
-
27
7
  describe '.initialize' do
28
- it 'should set the access and secret keys to nil by default' do
29
- Elasticity::JobFlow.new.tap do |j|
30
- j.access_key.should == nil
31
- j.secret_key.should == nil
32
- end
33
- Elasticity::JobFlow.new('_') do |j|
34
- j.secret_key.should == nil
35
- end
8
+ it 'should set the fields appropriately' do
9
+ expect(subject.action_on_failure).to eql('TERMINATE_JOB_FLOW')
10
+ expect(subject.ec2_key_name).to eql(nil)
11
+ expect(subject.name).to eql('Elasticity Job Flow')
12
+ expect(subject.instance_count).to eql(2)
13
+ expect(subject.log_uri).to eql(nil)
14
+ expect(subject.master_instance_type).to eql('m1.small')
15
+ expect(subject.slave_instance_type).to eql('m1.small')
16
+ expect(subject.ami_version).to eql('latest')
17
+ expect(subject.keep_job_flow_alive_when_no_steps).to eql(false)
18
+ expect(subject.ec2_subnet_id).to eql(nil)
19
+ expect(subject.placement).to eql('us-east-1a')
20
+ expect(subject.region).to eql('us-east-1')
21
+ expect(subject.visible_to_all_users).to eql(false)
22
+ expect(subject.enable_debugging).to eql(false)
23
+ expect(subject.job_flow_role).to eql(nil)
24
+ expect(subject.service_role).to eql(nil)
36
25
  end
37
26
  end
38
-
27
+
39
28
  describe '#placement=' do
40
29
 
41
30
  context 'when the placement is set' do
@@ -213,7 +202,7 @@ describe Elasticity::JobFlow do
213
202
  let(:emr) { double('Elasticity::EMR', :run_job_flow => 'RUNNING_JOBFLOW_ID') }
214
203
 
215
204
  let(:running_jobflow) do
216
- Elasticity::JobFlow.new('access', 'secret').tap do |jf|
205
+ Elasticity::JobFlow.new.tap do |jf|
217
206
  jf.add_step(Elasticity::PigStep.new('_'))
218
207
  end
219
208
  end
@@ -230,8 +219,8 @@ describe Elasticity::JobFlow do
230
219
 
231
220
  it 'should submit the step' do
232
221
  emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', {
233
- :steps => [additional_step.to_aws_step(running_jobflow)]
234
- })
222
+ :steps => [additional_step.to_aws_step(running_jobflow)]
223
+ })
235
224
  running_jobflow.add_step(additional_step)
236
225
  end
237
226
  end
@@ -241,8 +230,8 @@ describe Elasticity::JobFlow do
241
230
 
242
231
  it 'should submit the installation step and the step' do
243
232
  emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', {
244
- :steps => Elasticity::HiveStep.aws_installation_steps << additional_step.to_aws_step(running_jobflow)
245
- })
233
+ :steps => Elasticity::HiveStep.aws_installation_steps << additional_step.to_aws_step(running_jobflow)
234
+ })
246
235
  running_jobflow.add_step(additional_step)
247
236
  end
248
237
  end
@@ -255,8 +244,8 @@ describe Elasticity::JobFlow do
255
244
 
256
245
  it 'should submit the step' do
257
246
  emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', {
258
- :steps => [additional_step.to_aws_step(running_jobflow)]
259
- })
247
+ :steps => [additional_step.to_aws_step(running_jobflow)]
248
+ })
260
249
  running_jobflow.add_step(additional_step)
261
250
  end
262
251
 
@@ -280,17 +269,17 @@ describe Elasticity::JobFlow do
280
269
 
281
270
  let(:jobflow_steps) { [Elasticity::HiveStep.new('script.hql'), Elasticity::PigStep.new('script.pig'), Elasticity::CustomJarStep.new('script.jar')] }
282
271
  let(:jobflow_with_steps) do
283
- Elasticity::JobFlow.new('_', '_').tap do |jobflow|
272
+ Elasticity::JobFlow.new.tap do |jobflow|
284
273
  jobflow_steps.each { |s| jobflow.add_step(s) }
285
274
  end
286
275
  end
287
276
  let(:aws_steps) do
288
277
  [
289
278
  Elasticity::HiveStep.aws_installation_steps,
290
- jobflow_steps[0].to_aws_step(jobflow_with_steps),
291
- Elasticity::PigStep.aws_installation_steps,
292
- jobflow_steps[1].to_aws_step(jobflow_with_steps),
293
- jobflow_steps[2].to_aws_step(jobflow_with_steps),
279
+ jobflow_steps[0].to_aws_step(jobflow_with_steps),
280
+ Elasticity::PigStep.aws_installation_steps,
281
+ jobflow_steps[1].to_aws_step(jobflow_with_steps),
282
+ jobflow_steps[2].to_aws_step(jobflow_with_steps),
294
283
  ].flatten
295
284
  end
296
285
 
@@ -325,7 +314,7 @@ describe Elasticity::JobFlow do
325
314
 
326
315
  context 'when a log URI is specified' do
327
316
  let(:jobflow_with_log_uri) do
328
- Elasticity::JobFlow.new('_', '_').tap do |jf|
317
+ Elasticity::JobFlow.new.tap do |jf|
329
318
  jf.log_uri = 'LOG_URI'
330
319
  end
331
320
  end
@@ -336,7 +325,7 @@ describe Elasticity::JobFlow do
336
325
 
337
326
  context 'when a log URI is not specified' do
338
327
  let(:jobflow_with_no_log_uri) do
339
- Elasticity::JobFlow.new('_', '_').tap do |jf|
328
+ Elasticity::JobFlow.new.tap do |jf|
340
329
  jf.log_uri = nil
341
330
  end
342
331
  end
@@ -351,7 +340,7 @@ describe Elasticity::JobFlow do
351
340
 
352
341
  context 'when a job flow role is specified' do
353
342
  let(:jobflow_with_job_flow_role) do
354
- Elasticity::JobFlow.new('_', '_').tap do |jf|
343
+ Elasticity::JobFlow.new.tap do |jf|
355
344
  jf.job_flow_role = 'JOB_FLOW_ROLE'
356
345
  end
357
346
  end
@@ -362,7 +351,7 @@ describe Elasticity::JobFlow do
362
351
 
363
352
  context 'when a job flow role is not specified' do
364
353
  let(:jobflow_with_no_job_flow_role) do
365
- Elasticity::JobFlow.new('_', '_').tap do |jf|
354
+ Elasticity::JobFlow.new.tap do |jf|
366
355
  jf.job_flow_role = nil
367
356
  end
368
357
  end
@@ -377,7 +366,7 @@ describe Elasticity::JobFlow do
377
366
 
378
367
  context 'when a service role is specified' do
379
368
  let(:jobflow_with_service_role) do
380
- Elasticity::JobFlow.new('_', '_').tap do |jf|
369
+ Elasticity::JobFlow.new.tap do |jf|
381
370
  jf.service_role = 'SERVICE_ROLE'
382
371
  end
383
372
  end
@@ -388,7 +377,7 @@ describe Elasticity::JobFlow do
388
377
 
389
378
  context 'when a service role is not specified' do
390
379
  let(:jobflow_with_no_service_role) do
391
- Elasticity::JobFlow.new('_', '_').tap do |jf|
380
+ Elasticity::JobFlow.new.tap do |jf|
392
381
  jf.service_role = nil
393
382
  end
394
383
  end
@@ -410,7 +399,7 @@ describe Elasticity::JobFlow do
410
399
  ]
411
400
  end
412
401
  let(:jobflow_with_bootstrap_actions) do
413
- Elasticity::JobFlow.new('_', '_').tap do |jf|
402
+ Elasticity::JobFlow.new.tap do |jf|
414
403
  hadoop_bootstrap_actions.each do |action|
415
404
  jf.add_bootstrap_action(action)
416
405
  end
@@ -419,8 +408,8 @@ describe Elasticity::JobFlow do
419
408
  it 'should include them in the jobflow config' do
420
409
  bootstrap_actions = hadoop_bootstrap_actions.map { |a| a.to_aws_bootstrap_action }
421
410
  jobflow_with_bootstrap_actions.send(:jobflow_config).should be_a_hash_including({
422
- :bootstrap_actions => bootstrap_actions
423
- })
411
+ :bootstrap_actions => bootstrap_actions
412
+ })
424
413
  end
425
414
  end
426
415
 
@@ -544,7 +533,7 @@ describe Elasticity::JobFlow do
544
533
 
545
534
  context 'when there are steps added' do
546
535
  let(:jobflow_with_steps) do
547
- Elasticity::JobFlow.new('STEP_TEST_ACCESS', 'STEP_TEST_SECRET').tap do |jf|
536
+ Elasticity::JobFlow.new.tap do |jf|
548
537
  jf.add_step(Elasticity::CustomJarStep.new('_'))
549
538
  end
550
539
  end
@@ -553,7 +542,7 @@ describe Elasticity::JobFlow do
553
542
  let(:emr) { double('Elasticity::EMR', :run_job_flow => 'JOBFLOW_ID') }
554
543
 
555
544
  it 'should run the job with the supplied EMR credentials' do
556
- Elasticity::EMR.stub(:new).with('STEP_TEST_ACCESS', 'STEP_TEST_SECRET', :region => 'us-east-1').and_return(emr)
545
+ Elasticity::EMR.stub(:new).with(:region => 'us-east-1').and_return(emr)
557
546
  emr.should_receive(:run_job_flow)
558
547
  jobflow_with_steps.run
559
548
  end
@@ -588,34 +577,68 @@ describe Elasticity::JobFlow do
588
577
 
589
578
  end
590
579
 
591
- describe '#status' do
580
+ describe '#cluster_status' do
592
581
 
593
582
  context 'before the jobflow has been run' do
594
583
  it 'should raise an error' do
595
584
  expect {
596
- subject.status
585
+ subject.cluster_status
597
586
  }.to raise_error(Elasticity::JobFlowNotStartedError, 'Please #run this job flow before attempting to retrieve status.')
598
587
  end
599
588
  end
600
589
 
601
590
  context 'after the jobflow has been run' do
602
591
  let(:emr) { double('Elasticity::EMR', :run_job_flow => 'JOBFLOW_ID') }
603
- let(:running_jobflow) { Elasticity::JobFlow.new('_', '_') }
604
- let(:jobflow_status) do
605
- Elasticity::JobFlowStatus.new.tap do |js|
606
- js.stub(:state => 'TERMINATED')
607
- end
592
+ let(:running_jobflow) { Elasticity::JobFlow.new }
593
+ let(:aws_cluster_status) { JSON.parse('{ "Cluster": { "Status": { "State": "TERMINATED" } } }') }
594
+
595
+ before do
596
+ Elasticity::EMR.stub(:new).and_return(emr)
597
+ running_jobflow.add_step(Elasticity::CustomJarStep.new('_'))
598
+ running_jobflow.run
599
+ end
600
+
601
+ it 'should return the AWS status' do
602
+ emr.should_receive(:describe_cluster).with('JOBFLOW_ID').and_return(aws_cluster_status)
603
+ Elasticity::ClusterStatus.should_receive(:from_aws_data).with(aws_cluster_status).and_return(build(:cluster_status))
604
+
605
+ status = running_jobflow.cluster_status
606
+ expect(status).to be_a(Elasticity::ClusterStatus)
607
+ expect(status.state).to eql('TERMINATED')
608
608
  end
609
+ end
610
+
611
+ end
612
+
613
+ describe '#cluster_step_status' do
614
+
615
+ context 'before the jobflow has been run' do
616
+ it 'should raise an error' do
617
+ expect {
618
+ subject.cluster_step_status
619
+ }.to raise_error(Elasticity::JobFlowNotStartedError, 'Please #run this job flow before attempting to retrieve status.')
620
+ end
621
+ end
622
+
623
+ context 'after the jobflow has been run' do
624
+ let(:emr) { double('Elasticity::EMR', :run_job_flow => 'JOBFLOW_ID') }
625
+ let(:running_jobflow) { Elasticity::JobFlow.new }
626
+
609
627
  before do
610
628
  Elasticity::EMR.stub(:new).and_return(emr)
611
629
  running_jobflow.add_step(Elasticity::CustomJarStep.new('_'))
612
630
  running_jobflow.run
613
631
  end
632
+
614
633
  it 'should return the AWS status' do
615
- emr.should_receive(:describe_jobflow).with('JOBFLOW_ID').and_return(jobflow_status)
616
- status = running_jobflow.status
617
- status.should be_a(Elasticity::JobFlowStatus)
618
- status.state.should == 'TERMINATED'
634
+ emr.should_receive(:list_steps).with('JOBFLOW_ID').and_return('AWS_CLUSTER_STATUS')
635
+ Elasticity::ClusterStepStatus.should_receive(:from_aws_list_data).with('AWS_CLUSTER_STATUS').and_return([build(:cluster_step_status)])
636
+
637
+ status = running_jobflow.cluster_step_status
638
+ status.each do |s|
639
+ expect(s).to be_a(Elasticity::ClusterStepStatus)
640
+ end
641
+ expect(status[0].state).to eql('COMPLETED')
619
642
  end
620
643
  end
621
644
 
@@ -624,7 +647,7 @@ describe Elasticity::JobFlow do
624
647
  describe '#shutdown' do
625
648
 
626
649
  context 'when the jobflow has not yet been started' do
627
- let(:unstarted_job_flow) { Elasticity::JobFlow.new('_', '_') }
650
+ let(:unstarted_job_flow) { Elasticity::JobFlow.new }
628
651
  it 'should be an error' do
629
652
  expect {
630
653
  unstarted_job_flow.shutdown
@@ -634,7 +657,7 @@ describe Elasticity::JobFlow do
634
657
 
635
658
  context 'when the jobflow has been started' do
636
659
  let(:emr) { double('Elasticity::EMR', :run_job_flow => 'JOBFLOW_ID') }
637
- let(:running_jobflow) { Elasticity::JobFlow.new('_', '_') }
660
+ let(:running_jobflow) { Elasticity::JobFlow.new }
638
661
  before do
639
662
  Elasticity::EMR.stub(:new).and_return(emr)
640
663
  running_jobflow.add_step(Elasticity::CustomJarStep.new('_'))
@@ -664,7 +687,7 @@ describe Elasticity::JobFlow do
664
687
  context 'when the jobflow is active' do
665
688
  let(:jobflow_status) { double(:active? => true) }
666
689
  before do
667
- subject.stub(:status).and_return(jobflow_status)
690
+ subject.stub(:cluster_status).and_return(jobflow_status)
668
691
  end
669
692
  it 'returns true and the result of #status' do
670
693
  subject.send(:retry_check).should == [true, jobflow_status]
@@ -674,7 +697,7 @@ describe Elasticity::JobFlow do
674
697
  context 'when the jobflow is not active' do
675
698
  let(:jobflow_status) { double(:active? => false) }
676
699
  before do
677
- subject.stub(:status).and_return(jobflow_status)
700
+ subject.stub(:cluster_status).and_return(jobflow_status)
678
701
  end
679
702
  it 'returns true and the result of #status' do
680
703
  subject.send(:retry_check).should == [false, jobflow_status]
@@ -686,24 +709,24 @@ describe Elasticity::JobFlow do
686
709
  describe '.from_jobflow_id' do
687
710
 
688
711
  before do
689
- Elasticity::JobFlow.any_instance.stub_chain(:status, :installed_steps => [])
712
+ Elasticity::JobFlow.any_instance.stub(:cluster_step_status).and_return([])
690
713
  end
691
714
 
692
- let(:jobflow) { Elasticity::JobFlow.from_jobflow_id('ACCESS', 'SECRET', 'JOBFLOW_ID') }
715
+ let(:jobflow) { Elasticity::JobFlow.from_jobflow_id('JOBFLOW_ID') }
693
716
 
694
717
  describe 'creating a jobflow with the specified credentials' do
695
718
 
696
719
  context 'when the region is not specified' do
697
720
  it 'should use the default of us-east-1a' do
698
- j = Elasticity::JobFlow.from_jobflow_id('ACCESS', 'SECRET', '_')
699
- j.send(:emr).should == Elasticity::EMR.new('ACCESS', 'SECRET', :region => 'us-east-1')
721
+ j = Elasticity::JobFlow.from_jobflow_id('_')
722
+ j.send(:emr).should == Elasticity::EMR.new(:region => 'us-east-1')
700
723
  end
701
724
  end
702
725
 
703
726
  context 'when the region is specified' do
704
727
  it 'should use the specified region' do
705
- j = Elasticity::JobFlow.from_jobflow_id('ACCESS', 'SECRET', '_', 'us-west-1')
706
- j.send(:emr).should == Elasticity::EMR.new('ACCESS', 'SECRET', :region => 'us-west-1')
728
+ j = Elasticity::JobFlow.from_jobflow_id('_', 'us-west-1')
729
+ j.send(:emr).should == Elasticity::EMR.new(:region => 'us-west-1')
707
730
  end
708
731
  end
709
732
 
@@ -723,7 +746,7 @@ describe Elasticity::JobFlow do
723
746
 
724
747
  context 'when no steps have been installed' do
725
748
  before do
726
- Elasticity::JobFlow.any_instance.should_receive(:status).and_return(double('Elasticity::JobFlowStatus', :installed_steps => []))
749
+ Elasticity::JobFlow.any_instance.should_receive(:cluster_step_status).and_return([])
727
750
  end
728
751
  it 'should show that no steps are installed' do
729
752
  jobflow.instance_variable_get(:@installed_steps).should == []
@@ -732,7 +755,10 @@ describe Elasticity::JobFlow do
732
755
 
733
756
  context 'when steps have been installed do' do
734
757
  before do
735
- Elasticity::JobFlow.any_instance.should_receive(:status).and_return(double('Elasticity::JobFlowStatus', :installed_steps => [Elasticity::HiveStep, Elasticity::PigStep]))
758
+ Elasticity::JobFlow.any_instance.should_receive(:cluster_step_status).and_return([
759
+ build(:cluster_step_status, :name => Elasticity::PigStep.aws_installation_step_name),
760
+ build(:cluster_step_status, :name => Elasticity::HiveStep.aws_installation_step_name)
761
+ ])
736
762
  end
737
763
  it 'should show that no steps are installed' do
738
764
  jobflow.instance_variable_get(:@installed_steps).should =~ [Elasticity::PigStep, Elasticity::HiveStep]