elasticity 5.0.3 → 6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/HISTORY.md +26 -0
  3. data/README.md +35 -28
  4. data/elasticity.gemspec +2 -2
  5. data/lib/elasticity.rb +5 -3
  6. data/lib/elasticity/aws_request_v4.rb +15 -3
  7. data/lib/elasticity/aws_session.rb +4 -23
  8. data/lib/elasticity/aws_utils.rb +0 -29
  9. data/lib/elasticity/cluster_status.rb +38 -0
  10. data/lib/elasticity/cluster_step_status.rb +51 -0
  11. data/lib/elasticity/emr.rb +208 -78
  12. data/lib/elasticity/job_flow.rb +16 -17
  13. data/lib/elasticity/version.rb +1 -1
  14. data/spec/factories/cluster_status_factory.rb +12 -0
  15. data/spec/factories/cluster_step_status_factory.rb +17 -0
  16. data/spec/lib/elasticity/aws_request_v4_spec.rb +54 -4
  17. data/spec/lib/elasticity/aws_session_spec.rb +22 -88
  18. data/spec/lib/elasticity/aws_utils_spec.rb +0 -46
  19. data/spec/lib/elasticity/bootstrap_action_spec.rb +7 -3
  20. data/spec/lib/elasticity/cluster_status_spec.rb +98 -0
  21. data/spec/lib/elasticity/cluster_step_status_spec.rb +80 -0
  22. data/spec/lib/elasticity/custom_jar_step_spec.rb +10 -7
  23. data/spec/lib/elasticity/emr_spec.rb +422 -132
  24. data/spec/lib/elasticity/ganglia_bootstrap_action_spec.rb +8 -3
  25. data/spec/lib/elasticity/hadoop_bootstrap_action_spec.rb +8 -3
  26. data/spec/lib/elasticity/hadoop_file_bootstrap_action_spec.rb +7 -3
  27. data/spec/lib/elasticity/hive_step_spec.rb +21 -17
  28. data/spec/lib/elasticity/instance_group_spec.rb +9 -5
  29. data/spec/lib/elasticity/job_flow_integration_spec.rb +4 -4
  30. data/spec/lib/elasticity/job_flow_spec.rb +102 -76
  31. data/spec/lib/elasticity/job_flow_step_spec.rb +1 -1
  32. data/spec/lib/elasticity/looper_spec.rb +1 -1
  33. data/spec/lib/elasticity/pig_step_spec.rb +13 -9
  34. data/spec/lib/elasticity/s3distcp_step_spec.rb +7 -5
  35. data/spec/lib/elasticity/script_step_spec.rb +11 -6
  36. data/spec/lib/elasticity/setup_hadoop_debugging_step_spec.rb +9 -5
  37. data/spec/lib/elasticity/streaming_step_spec.rb +13 -9
  38. data/spec/spec_helper.rb +8 -0
  39. data/spec/support/factory_girl.rb +8 -0
  40. metadata +24 -21
  41. data/lib/elasticity/aws_request_v2.rb +0 -42
  42. data/lib/elasticity/job_flow_status.rb +0 -91
  43. data/lib/elasticity/job_flow_status_step.rb +0 -38
  44. data/spec/lib/elasticity/aws_request_v2_spec.rb +0 -38
  45. data/spec/lib/elasticity/job_flow_status_spec.rb +0 -265
  46. data/spec/lib/elasticity/job_flow_status_step_spec.rb +0 -80
@@ -6,8 +6,13 @@ describe Elasticity::GangliaBootstrapAction do
6
6
 
7
7
  it { should be_a Elasticity::BootstrapAction }
8
8
 
9
- its(:name) { should == 'Elasticity Bootstrap Action (Install Ganglia)' }
10
- its(:arguments) { should == [] }
11
- its(:script) { should == 's3://elasticmapreduce/bootstrap-actions/install-ganglia' }
9
+ describe '.intialize' do
10
+ it 'should set the fields appropriately' do
11
+ expect(subject.name).to eql('Elasticity Bootstrap Action (Install Ganglia)')
12
+ expect(subject.arguments).to eql([])
13
+ expect(subject.script).to eql('s3://elasticmapreduce/bootstrap-actions/install-ganglia')
14
+ end
15
+ end
16
+
12
17
 
13
18
  end
@@ -6,8 +6,13 @@ describe Elasticity::HadoopBootstrapAction do
6
6
 
7
7
  it { should be_a Elasticity::BootstrapAction }
8
8
 
9
- its(:name) { should == 'Elasticity Bootstrap Action (Configure Hadoop)' }
10
- its(:arguments) { should == %w(option value) }
11
- its(:script) { should == 's3n://elasticmapreduce/bootstrap-actions/configure-hadoop' }
9
+ describe '.initialize' do
10
+ it 'should set the fields appropriately' do
11
+ expect(subject.name).to eql('Elasticity Bootstrap Action (Configure Hadoop)')
12
+ expect(subject.arguments).to eql(%w(option value))
13
+ expect(subject.script).to eql('s3n://elasticmapreduce/bootstrap-actions/configure-hadoop')
14
+ end
15
+ end
16
+
12
17
 
13
18
  end
@@ -6,8 +6,12 @@ describe Elasticity::HadoopFileBootstrapAction do
6
6
 
7
7
  it { should be_a Elasticity::BootstrapAction }
8
8
 
9
- its(:name) { should == 'Elasticity Bootstrap Action (Configure Hadoop via File)' }
10
- its(:arguments) { should == %w(--mapred-config-file config_file) }
11
- its(:script) { should == 's3n://elasticmapreduce/bootstrap-actions/configure-hadoop' }
9
+ describe '.initialize' do
10
+ it 'should set the fields appropriately' do
11
+ expect(subject.name).to eql('Elasticity Bootstrap Action (Configure Hadoop via File)')
12
+ expect(subject.arguments).to eql(%w(--mapred-config-file config_file))
13
+ expect(subject.script).to eql('s3n://elasticmapreduce/bootstrap-actions/configure-hadoop')
14
+ end
15
+ end
12
16
 
13
17
  end
@@ -6,29 +6,33 @@ describe Elasticity::HiveStep do
6
6
 
7
7
  it { should be_a Elasticity::JobFlowStep }
8
8
 
9
- its(:name) { should == 'Elasticity Hive Step (script.hql)' }
10
- its(:script) { should == 'script.hql' }
11
- its(:variables) { should == {} }
12
- its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
9
+ describe '.initialize' do
10
+ it 'should set the fields appropriately' do
11
+ expect(subject.name).to eql('Elasticity Hive Step (script.hql)')
12
+ expect(subject.script).to eql('script.hql')
13
+ expect(subject.variables).to eql({})
14
+ expect(subject.action_on_failure).to eql('TERMINATE_JOB_FLOW')
15
+ end
16
+ end
13
17
 
14
18
  describe '#to_aws_step' do
15
19
 
16
20
  it 'should convert to aws step format' do
17
- step = subject.to_aws_step(Elasticity::JobFlow.new('access', 'secret'))
21
+ step = subject.to_aws_step(Elasticity::JobFlow.new)
18
22
  step[:name].should == 'Elasticity Hive Step (script.hql)'
19
23
  step[:action_on_failure].should == 'TERMINATE_JOB_FLOW'
20
24
  step[:hadoop_jar_step][:jar].should == 's3://elasticmapreduce/libs/script-runner/script-runner.jar'
21
25
  step[:hadoop_jar_step][:args].should start_with([
22
- 's3://elasticmapreduce/libs/hive/hive-script',
23
- '--base-path',
24
- 's3://elasticmapreduce/libs/hive/',
25
- '--hive-versions',
26
- 'latest',
27
- '--run-hive-script',
28
- '--args',
29
- '-f',
30
- 'script.hql'
31
- ])
26
+ 's3://elasticmapreduce/libs/hive/hive-script',
27
+ '--base-path',
28
+ 's3://elasticmapreduce/libs/hive/',
29
+ '--hive-versions',
30
+ 'latest',
31
+ '--run-hive-script',
32
+ '--args',
33
+ '-f',
34
+ 'script.hql'
35
+ ])
32
36
  end
33
37
 
34
38
  context 'when variables are provided' do
@@ -42,7 +46,7 @@ describe Elasticity::HiveStep do
42
46
  end
43
47
 
44
48
  it 'should convert to aws step format' do
45
- step = hs_with_variables.to_aws_step(Elasticity::JobFlow.new('access', 'secret'))
49
+ step = hs_with_variables.to_aws_step(Elasticity::JobFlow.new)
46
50
  step[:hadoop_jar_step][:args][9..13].should == %w(-d VAR1=VALUE1 -d VAR2=VALUE2)
47
51
  end
48
52
  end
@@ -51,7 +55,7 @@ describe Elasticity::HiveStep do
51
55
 
52
56
  describe '.requires_installation?' do
53
57
  it 'should require installation' do
54
- Elasticity::HiveStep.requires_installation?.should be_true
58
+ expect(Elasticity::HiveStep.requires_installation?).to be true
55
59
  end
56
60
  end
57
61
 
@@ -1,10 +1,14 @@
1
1
  describe Elasticity::InstanceGroup do
2
2
 
3
- its(:bid_price) { should == nil }
4
- its(:count) { should == 1 }
5
- its(:type) { should == 'm1.small' }
6
- its(:market) { should == 'ON_DEMAND' }
7
- its(:role) { should == 'CORE' }
3
+ describe '.initialize' do
4
+ it 'should set the fields appropriately' do
5
+ expect(subject.bid_price).to eql(nil)
6
+ expect(subject.count).to eql(1)
7
+ expect(subject.type).to eql('m1.small')
8
+ expect(subject.market).to eql('ON_DEMAND')
9
+ expect(subject.role).to eql('CORE')
10
+ end
11
+ end
8
12
 
9
13
  describe '#count=' do
10
14
 
@@ -3,7 +3,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
3
3
  let(:emr) { double('Elasticity::EMR') }
4
4
 
5
5
  before do
6
- Elasticity::EMR.should_receive(:new).with('access', 'secret', :region => 'us-west-1').and_return(emr)
6
+ Elasticity::EMR.should_receive(:new).with(:region => 'us-west-1').and_return(emr)
7
7
  end
8
8
 
9
9
  describe 'Hive' do
@@ -16,7 +16,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
16
16
  end
17
17
 
18
18
  let(:hive_jobflow) do
19
- Elasticity::JobFlow.new('access', 'secret').tap do |jf|
19
+ Elasticity::JobFlow.new.tap do |jf|
20
20
  jf.placement = 'us-west-1a'
21
21
  jf.log_uri = 's3n://slif-test/output/logs'
22
22
  jf.add_step(hive_step)
@@ -101,7 +101,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
101
101
  end
102
102
 
103
103
  let(:pig_jobflow) do
104
- Elasticity::JobFlow.new('access', 'secret').tap do |jf|
104
+ Elasticity::JobFlow.new.tap do |jf|
105
105
  jf.placement = 'us-west-1c'
106
106
  jf.instance_count = 8
107
107
  jf.slave_instance_type = 'm1.xlarge'
@@ -189,7 +189,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
189
189
  end
190
190
 
191
191
  let(:custom_jar_jobflow) do
192
- Elasticity::JobFlow.new('access', 'secret').tap do |jf|
192
+ Elasticity::JobFlow.new.tap do |jf|
193
193
  jf.placement = 'us-west-1b'
194
194
  jf.log_uri = 's3n://slif-test/output/logs'
195
195
  jf.add_step(custom_jar_step)
@@ -1,41 +1,30 @@
1
1
  describe Elasticity::JobFlow do
2
2
 
3
3
  subject do
4
- Elasticity::JobFlow.new('access', 'secret')
4
+ Elasticity::JobFlow.new
5
5
  end
6
6
 
7
- its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
8
- its(:ec2_key_name) { should == nil }
9
- its(:name) { should == 'Elasticity Job Flow' }
10
- its(:instance_count) { should == 2 }
11
- its(:log_uri) { should == nil }
12
- its(:master_instance_type) { should == 'm1.small' }
13
- its(:slave_instance_type) { should == 'm1.small' }
14
- its(:ami_version) { should == 'latest' }
15
- its(:keep_job_flow_alive_when_no_steps) { should == false }
16
- its(:ec2_subnet_id) { should == nil }
17
- its(:placement) { should == 'us-east-1a' }
18
- its(:region) { should == 'us-east-1' }
19
- its(:visible_to_all_users) { should == false }
20
- its(:enable_debugging) { should == false }
21
- its(:job_flow_role) { should == nil }
22
- its(:service_role) { should == nil }
23
-
24
- its(:access_key) { should == 'access' }
25
- its(:secret_key) { should == 'secret' }
26
-
27
7
  describe '.initialize' do
28
- it 'should set the access and secret keys to nil by default' do
29
- Elasticity::JobFlow.new.tap do |j|
30
- j.access_key.should == nil
31
- j.secret_key.should == nil
32
- end
33
- Elasticity::JobFlow.new('_') do |j|
34
- j.secret_key.should == nil
35
- end
8
+ it 'should set the fields appropriately' do
9
+ expect(subject.action_on_failure).to eql('TERMINATE_JOB_FLOW')
10
+ expect(subject.ec2_key_name).to eql(nil)
11
+ expect(subject.name).to eql('Elasticity Job Flow')
12
+ expect(subject.instance_count).to eql(2)
13
+ expect(subject.log_uri).to eql(nil)
14
+ expect(subject.master_instance_type).to eql('m1.small')
15
+ expect(subject.slave_instance_type).to eql('m1.small')
16
+ expect(subject.ami_version).to eql('latest')
17
+ expect(subject.keep_job_flow_alive_when_no_steps).to eql(false)
18
+ expect(subject.ec2_subnet_id).to eql(nil)
19
+ expect(subject.placement).to eql('us-east-1a')
20
+ expect(subject.region).to eql('us-east-1')
21
+ expect(subject.visible_to_all_users).to eql(false)
22
+ expect(subject.enable_debugging).to eql(false)
23
+ expect(subject.job_flow_role).to eql(nil)
24
+ expect(subject.service_role).to eql(nil)
36
25
  end
37
26
  end
38
-
27
+
39
28
  describe '#placement=' do
40
29
 
41
30
  context 'when the placement is set' do
@@ -213,7 +202,7 @@ describe Elasticity::JobFlow do
213
202
  let(:emr) { double('Elasticity::EMR', :run_job_flow => 'RUNNING_JOBFLOW_ID') }
214
203
 
215
204
  let(:running_jobflow) do
216
- Elasticity::JobFlow.new('access', 'secret').tap do |jf|
205
+ Elasticity::JobFlow.new.tap do |jf|
217
206
  jf.add_step(Elasticity::PigStep.new('_'))
218
207
  end
219
208
  end
@@ -230,8 +219,8 @@ describe Elasticity::JobFlow do
230
219
 
231
220
  it 'should submit the step' do
232
221
  emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', {
233
- :steps => [additional_step.to_aws_step(running_jobflow)]
234
- })
222
+ :steps => [additional_step.to_aws_step(running_jobflow)]
223
+ })
235
224
  running_jobflow.add_step(additional_step)
236
225
  end
237
226
  end
@@ -241,8 +230,8 @@ describe Elasticity::JobFlow do
241
230
 
242
231
  it 'should submit the installation step and the step' do
243
232
  emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', {
244
- :steps => Elasticity::HiveStep.aws_installation_steps << additional_step.to_aws_step(running_jobflow)
245
- })
233
+ :steps => Elasticity::HiveStep.aws_installation_steps << additional_step.to_aws_step(running_jobflow)
234
+ })
246
235
  running_jobflow.add_step(additional_step)
247
236
  end
248
237
  end
@@ -255,8 +244,8 @@ describe Elasticity::JobFlow do
255
244
 
256
245
  it 'should submit the step' do
257
246
  emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', {
258
- :steps => [additional_step.to_aws_step(running_jobflow)]
259
- })
247
+ :steps => [additional_step.to_aws_step(running_jobflow)]
248
+ })
260
249
  running_jobflow.add_step(additional_step)
261
250
  end
262
251
 
@@ -280,17 +269,17 @@ describe Elasticity::JobFlow do
280
269
 
281
270
  let(:jobflow_steps) { [Elasticity::HiveStep.new('script.hql'), Elasticity::PigStep.new('script.pig'), Elasticity::CustomJarStep.new('script.jar')] }
282
271
  let(:jobflow_with_steps) do
283
- Elasticity::JobFlow.new('_', '_').tap do |jobflow|
272
+ Elasticity::JobFlow.new.tap do |jobflow|
284
273
  jobflow_steps.each { |s| jobflow.add_step(s) }
285
274
  end
286
275
  end
287
276
  let(:aws_steps) do
288
277
  [
289
278
  Elasticity::HiveStep.aws_installation_steps,
290
- jobflow_steps[0].to_aws_step(jobflow_with_steps),
291
- Elasticity::PigStep.aws_installation_steps,
292
- jobflow_steps[1].to_aws_step(jobflow_with_steps),
293
- jobflow_steps[2].to_aws_step(jobflow_with_steps),
279
+ jobflow_steps[0].to_aws_step(jobflow_with_steps),
280
+ Elasticity::PigStep.aws_installation_steps,
281
+ jobflow_steps[1].to_aws_step(jobflow_with_steps),
282
+ jobflow_steps[2].to_aws_step(jobflow_with_steps),
294
283
  ].flatten
295
284
  end
296
285
 
@@ -325,7 +314,7 @@ describe Elasticity::JobFlow do
325
314
 
326
315
  context 'when a log URI is specified' do
327
316
  let(:jobflow_with_log_uri) do
328
- Elasticity::JobFlow.new('_', '_').tap do |jf|
317
+ Elasticity::JobFlow.new.tap do |jf|
329
318
  jf.log_uri = 'LOG_URI'
330
319
  end
331
320
  end
@@ -336,7 +325,7 @@ describe Elasticity::JobFlow do
336
325
 
337
326
  context 'when a log URI is not specified' do
338
327
  let(:jobflow_with_no_log_uri) do
339
- Elasticity::JobFlow.new('_', '_').tap do |jf|
328
+ Elasticity::JobFlow.new.tap do |jf|
340
329
  jf.log_uri = nil
341
330
  end
342
331
  end
@@ -351,7 +340,7 @@ describe Elasticity::JobFlow do
351
340
 
352
341
  context 'when a job flow role is specified' do
353
342
  let(:jobflow_with_job_flow_role) do
354
- Elasticity::JobFlow.new('_', '_').tap do |jf|
343
+ Elasticity::JobFlow.new.tap do |jf|
355
344
  jf.job_flow_role = 'JOB_FLOW_ROLE'
356
345
  end
357
346
  end
@@ -362,7 +351,7 @@ describe Elasticity::JobFlow do
362
351
 
363
352
  context 'when a job flow role is not specified' do
364
353
  let(:jobflow_with_no_job_flow_role) do
365
- Elasticity::JobFlow.new('_', '_').tap do |jf|
354
+ Elasticity::JobFlow.new.tap do |jf|
366
355
  jf.job_flow_role = nil
367
356
  end
368
357
  end
@@ -377,7 +366,7 @@ describe Elasticity::JobFlow do
377
366
 
378
367
  context 'when a service role is specified' do
379
368
  let(:jobflow_with_service_role) do
380
- Elasticity::JobFlow.new('_', '_').tap do |jf|
369
+ Elasticity::JobFlow.new.tap do |jf|
381
370
  jf.service_role = 'SERVICE_ROLE'
382
371
  end
383
372
  end
@@ -388,7 +377,7 @@ describe Elasticity::JobFlow do
388
377
 
389
378
  context 'when a service role is not specified' do
390
379
  let(:jobflow_with_no_service_role) do
391
- Elasticity::JobFlow.new('_', '_').tap do |jf|
380
+ Elasticity::JobFlow.new.tap do |jf|
392
381
  jf.service_role = nil
393
382
  end
394
383
  end
@@ -410,7 +399,7 @@ describe Elasticity::JobFlow do
410
399
  ]
411
400
  end
412
401
  let(:jobflow_with_bootstrap_actions) do
413
- Elasticity::JobFlow.new('_', '_').tap do |jf|
402
+ Elasticity::JobFlow.new.tap do |jf|
414
403
  hadoop_bootstrap_actions.each do |action|
415
404
  jf.add_bootstrap_action(action)
416
405
  end
@@ -419,8 +408,8 @@ describe Elasticity::JobFlow do
419
408
  it 'should include them in the jobflow config' do
420
409
  bootstrap_actions = hadoop_bootstrap_actions.map { |a| a.to_aws_bootstrap_action }
421
410
  jobflow_with_bootstrap_actions.send(:jobflow_config).should be_a_hash_including({
422
- :bootstrap_actions => bootstrap_actions
423
- })
411
+ :bootstrap_actions => bootstrap_actions
412
+ })
424
413
  end
425
414
  end
426
415
 
@@ -544,7 +533,7 @@ describe Elasticity::JobFlow do
544
533
 
545
534
  context 'when there are steps added' do
546
535
  let(:jobflow_with_steps) do
547
- Elasticity::JobFlow.new('STEP_TEST_ACCESS', 'STEP_TEST_SECRET').tap do |jf|
536
+ Elasticity::JobFlow.new.tap do |jf|
548
537
  jf.add_step(Elasticity::CustomJarStep.new('_'))
549
538
  end
550
539
  end
@@ -553,7 +542,7 @@ describe Elasticity::JobFlow do
553
542
  let(:emr) { double('Elasticity::EMR', :run_job_flow => 'JOBFLOW_ID') }
554
543
 
555
544
  it 'should run the job with the supplied EMR credentials' do
556
- Elasticity::EMR.stub(:new).with('STEP_TEST_ACCESS', 'STEP_TEST_SECRET', :region => 'us-east-1').and_return(emr)
545
+ Elasticity::EMR.stub(:new).with(:region => 'us-east-1').and_return(emr)
557
546
  emr.should_receive(:run_job_flow)
558
547
  jobflow_with_steps.run
559
548
  end
@@ -588,34 +577,68 @@ describe Elasticity::JobFlow do
588
577
 
589
578
  end
590
579
 
591
- describe '#status' do
580
+ describe '#cluster_status' do
592
581
 
593
582
  context 'before the jobflow has been run' do
594
583
  it 'should raise an error' do
595
584
  expect {
596
- subject.status
585
+ subject.cluster_status
597
586
  }.to raise_error(Elasticity::JobFlowNotStartedError, 'Please #run this job flow before attempting to retrieve status.')
598
587
  end
599
588
  end
600
589
 
601
590
  context 'after the jobflow has been run' do
602
591
  let(:emr) { double('Elasticity::EMR', :run_job_flow => 'JOBFLOW_ID') }
603
- let(:running_jobflow) { Elasticity::JobFlow.new('_', '_') }
604
- let(:jobflow_status) do
605
- Elasticity::JobFlowStatus.new.tap do |js|
606
- js.stub(:state => 'TERMINATED')
607
- end
592
+ let(:running_jobflow) { Elasticity::JobFlow.new }
593
+ let(:aws_cluster_status) { JSON.parse('{ "Cluster": { "Status": { "State": "TERMINATED" } } }') }
594
+
595
+ before do
596
+ Elasticity::EMR.stub(:new).and_return(emr)
597
+ running_jobflow.add_step(Elasticity::CustomJarStep.new('_'))
598
+ running_jobflow.run
599
+ end
600
+
601
+ it 'should return the AWS status' do
602
+ emr.should_receive(:describe_cluster).with('JOBFLOW_ID').and_return(aws_cluster_status)
603
+ Elasticity::ClusterStatus.should_receive(:from_aws_data).with(aws_cluster_status).and_return(build(:cluster_status))
604
+
605
+ status = running_jobflow.cluster_status
606
+ expect(status).to be_a(Elasticity::ClusterStatus)
607
+ expect(status.state).to eql('TERMINATED')
608
608
  end
609
+ end
610
+
611
+ end
612
+
613
+ describe '#cluster_step_status' do
614
+
615
+ context 'before the jobflow has been run' do
616
+ it 'should raise an error' do
617
+ expect {
618
+ subject.cluster_step_status
619
+ }.to raise_error(Elasticity::JobFlowNotStartedError, 'Please #run this job flow before attempting to retrieve status.')
620
+ end
621
+ end
622
+
623
+ context 'after the jobflow has been run' do
624
+ let(:emr) { double('Elasticity::EMR', :run_job_flow => 'JOBFLOW_ID') }
625
+ let(:running_jobflow) { Elasticity::JobFlow.new }
626
+
609
627
  before do
610
628
  Elasticity::EMR.stub(:new).and_return(emr)
611
629
  running_jobflow.add_step(Elasticity::CustomJarStep.new('_'))
612
630
  running_jobflow.run
613
631
  end
632
+
614
633
  it 'should return the AWS status' do
615
- emr.should_receive(:describe_jobflow).with('JOBFLOW_ID').and_return(jobflow_status)
616
- status = running_jobflow.status
617
- status.should be_a(Elasticity::JobFlowStatus)
618
- status.state.should == 'TERMINATED'
634
+ emr.should_receive(:list_steps).with('JOBFLOW_ID').and_return('AWS_CLUSTER_STATUS')
635
+ Elasticity::ClusterStepStatus.should_receive(:from_aws_list_data).with('AWS_CLUSTER_STATUS').and_return([build(:cluster_step_status)])
636
+
637
+ status = running_jobflow.cluster_step_status
638
+ status.each do |s|
639
+ expect(s).to be_a(Elasticity::ClusterStepStatus)
640
+ end
641
+ expect(status[0].state).to eql('COMPLETED')
619
642
  end
620
643
  end
621
644
 
@@ -624,7 +647,7 @@ describe Elasticity::JobFlow do
624
647
  describe '#shutdown' do
625
648
 
626
649
  context 'when the jobflow has not yet been started' do
627
- let(:unstarted_job_flow) { Elasticity::JobFlow.new('_', '_') }
650
+ let(:unstarted_job_flow) { Elasticity::JobFlow.new }
628
651
  it 'should be an error' do
629
652
  expect {
630
653
  unstarted_job_flow.shutdown
@@ -634,7 +657,7 @@ describe Elasticity::JobFlow do
634
657
 
635
658
  context 'when the jobflow has been started' do
636
659
  let(:emr) { double('Elasticity::EMR', :run_job_flow => 'JOBFLOW_ID') }
637
- let(:running_jobflow) { Elasticity::JobFlow.new('_', '_') }
660
+ let(:running_jobflow) { Elasticity::JobFlow.new }
638
661
  before do
639
662
  Elasticity::EMR.stub(:new).and_return(emr)
640
663
  running_jobflow.add_step(Elasticity::CustomJarStep.new('_'))
@@ -664,7 +687,7 @@ describe Elasticity::JobFlow do
664
687
  context 'when the jobflow is active' do
665
688
  let(:jobflow_status) { double(:active? => true) }
666
689
  before do
667
- subject.stub(:status).and_return(jobflow_status)
690
+ subject.stub(:cluster_status).and_return(jobflow_status)
668
691
  end
669
692
  it 'returns true and the result of #status' do
670
693
  subject.send(:retry_check).should == [true, jobflow_status]
@@ -674,7 +697,7 @@ describe Elasticity::JobFlow do
674
697
  context 'when the jobflow is not active' do
675
698
  let(:jobflow_status) { double(:active? => false) }
676
699
  before do
677
- subject.stub(:status).and_return(jobflow_status)
700
+ subject.stub(:cluster_status).and_return(jobflow_status)
678
701
  end
679
702
  it 'returns true and the result of #status' do
680
703
  subject.send(:retry_check).should == [false, jobflow_status]
@@ -686,24 +709,24 @@ describe Elasticity::JobFlow do
686
709
  describe '.from_jobflow_id' do
687
710
 
688
711
  before do
689
- Elasticity::JobFlow.any_instance.stub_chain(:status, :installed_steps => [])
712
+ Elasticity::JobFlow.any_instance.stub(:cluster_step_status).and_return([])
690
713
  end
691
714
 
692
- let(:jobflow) { Elasticity::JobFlow.from_jobflow_id('ACCESS', 'SECRET', 'JOBFLOW_ID') }
715
+ let(:jobflow) { Elasticity::JobFlow.from_jobflow_id('JOBFLOW_ID') }
693
716
 
694
717
  describe 'creating a jobflow with the specified credentials' do
695
718
 
696
719
  context 'when the region is not specified' do
697
720
  it 'should use the default of us-east-1a' do
698
- j = Elasticity::JobFlow.from_jobflow_id('ACCESS', 'SECRET', '_')
699
- j.send(:emr).should == Elasticity::EMR.new('ACCESS', 'SECRET', :region => 'us-east-1')
721
+ j = Elasticity::JobFlow.from_jobflow_id('_')
722
+ j.send(:emr).should == Elasticity::EMR.new(:region => 'us-east-1')
700
723
  end
701
724
  end
702
725
 
703
726
  context 'when the region is specified' do
704
727
  it 'should use the specified region' do
705
- j = Elasticity::JobFlow.from_jobflow_id('ACCESS', 'SECRET', '_', 'us-west-1')
706
- j.send(:emr).should == Elasticity::EMR.new('ACCESS', 'SECRET', :region => 'us-west-1')
728
+ j = Elasticity::JobFlow.from_jobflow_id('_', 'us-west-1')
729
+ j.send(:emr).should == Elasticity::EMR.new(:region => 'us-west-1')
707
730
  end
708
731
  end
709
732
 
@@ -723,7 +746,7 @@ describe Elasticity::JobFlow do
723
746
 
724
747
  context 'when no steps have been installed' do
725
748
  before do
726
- Elasticity::JobFlow.any_instance.should_receive(:status).and_return(double('Elasticity::JobFlowStatus', :installed_steps => []))
749
+ Elasticity::JobFlow.any_instance.should_receive(:cluster_step_status).and_return([])
727
750
  end
728
751
  it 'should show that no steps are installed' do
729
752
  jobflow.instance_variable_get(:@installed_steps).should == []
@@ -732,7 +755,10 @@ describe Elasticity::JobFlow do
732
755
 
733
756
  context 'when steps have been installed do' do
734
757
  before do
735
- Elasticity::JobFlow.any_instance.should_receive(:status).and_return(double('Elasticity::JobFlowStatus', :installed_steps => [Elasticity::HiveStep, Elasticity::PigStep]))
758
+ Elasticity::JobFlow.any_instance.should_receive(:cluster_step_status).and_return([
759
+ build(:cluster_step_status, :name => Elasticity::PigStep.aws_installation_step_name),
760
+ build(:cluster_step_status, :name => Elasticity::HiveStep.aws_installation_step_name)
761
+ ])
736
762
  end
737
763
  it 'should show that no steps are installed' do
738
764
  jobflow.instance_variable_get(:@installed_steps).should =~ [Elasticity::PigStep, Elasticity::HiveStep]