elasticity 5.0.3 → 6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/HISTORY.md +26 -0
- data/README.md +35 -28
- data/elasticity.gemspec +2 -2
- data/lib/elasticity.rb +5 -3
- data/lib/elasticity/aws_request_v4.rb +15 -3
- data/lib/elasticity/aws_session.rb +4 -23
- data/lib/elasticity/aws_utils.rb +0 -29
- data/lib/elasticity/cluster_status.rb +38 -0
- data/lib/elasticity/cluster_step_status.rb +51 -0
- data/lib/elasticity/emr.rb +208 -78
- data/lib/elasticity/job_flow.rb +16 -17
- data/lib/elasticity/version.rb +1 -1
- data/spec/factories/cluster_status_factory.rb +12 -0
- data/spec/factories/cluster_step_status_factory.rb +17 -0
- data/spec/lib/elasticity/aws_request_v4_spec.rb +54 -4
- data/spec/lib/elasticity/aws_session_spec.rb +22 -88
- data/spec/lib/elasticity/aws_utils_spec.rb +0 -46
- data/spec/lib/elasticity/bootstrap_action_spec.rb +7 -3
- data/spec/lib/elasticity/cluster_status_spec.rb +98 -0
- data/spec/lib/elasticity/cluster_step_status_spec.rb +80 -0
- data/spec/lib/elasticity/custom_jar_step_spec.rb +10 -7
- data/spec/lib/elasticity/emr_spec.rb +422 -132
- data/spec/lib/elasticity/ganglia_bootstrap_action_spec.rb +8 -3
- data/spec/lib/elasticity/hadoop_bootstrap_action_spec.rb +8 -3
- data/spec/lib/elasticity/hadoop_file_bootstrap_action_spec.rb +7 -3
- data/spec/lib/elasticity/hive_step_spec.rb +21 -17
- data/spec/lib/elasticity/instance_group_spec.rb +9 -5
- data/spec/lib/elasticity/job_flow_integration_spec.rb +4 -4
- data/spec/lib/elasticity/job_flow_spec.rb +102 -76
- data/spec/lib/elasticity/job_flow_step_spec.rb +1 -1
- data/spec/lib/elasticity/looper_spec.rb +1 -1
- data/spec/lib/elasticity/pig_step_spec.rb +13 -9
- data/spec/lib/elasticity/s3distcp_step_spec.rb +7 -5
- data/spec/lib/elasticity/script_step_spec.rb +11 -6
- data/spec/lib/elasticity/setup_hadoop_debugging_step_spec.rb +9 -5
- data/spec/lib/elasticity/streaming_step_spec.rb +13 -9
- data/spec/spec_helper.rb +8 -0
- data/spec/support/factory_girl.rb +8 -0
- metadata +24 -21
- data/lib/elasticity/aws_request_v2.rb +0 -42
- data/lib/elasticity/job_flow_status.rb +0 -91
- data/lib/elasticity/job_flow_status_step.rb +0 -38
- data/spec/lib/elasticity/aws_request_v2_spec.rb +0 -38
- data/spec/lib/elasticity/job_flow_status_spec.rb +0 -265
- data/spec/lib/elasticity/job_flow_status_step_spec.rb +0 -80
@@ -6,8 +6,13 @@ describe Elasticity::GangliaBootstrapAction do
|
|
6
6
|
|
7
7
|
it { should be_a Elasticity::BootstrapAction }
|
8
8
|
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
describe '.intialize' do
|
10
|
+
it 'should set the fields appropriately' do
|
11
|
+
expect(subject.name).to eql('Elasticity Bootstrap Action (Install Ganglia)')
|
12
|
+
expect(subject.arguments).to eql([])
|
13
|
+
expect(subject.script).to eql('s3://elasticmapreduce/bootstrap-actions/install-ganglia')
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
12
17
|
|
13
18
|
end
|
@@ -6,8 +6,13 @@ describe Elasticity::HadoopBootstrapAction do
|
|
6
6
|
|
7
7
|
it { should be_a Elasticity::BootstrapAction }
|
8
8
|
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
describe '.initialize' do
|
10
|
+
it 'should set the fields appropriately' do
|
11
|
+
expect(subject.name).to eql('Elasticity Bootstrap Action (Configure Hadoop)')
|
12
|
+
expect(subject.arguments).to eql(%w(option value))
|
13
|
+
expect(subject.script).to eql('s3n://elasticmapreduce/bootstrap-actions/configure-hadoop')
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
12
17
|
|
13
18
|
end
|
@@ -6,8 +6,12 @@ describe Elasticity::HadoopFileBootstrapAction do
|
|
6
6
|
|
7
7
|
it { should be_a Elasticity::BootstrapAction }
|
8
8
|
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
describe '.initialize' do
|
10
|
+
it 'should set the fields appropriately' do
|
11
|
+
expect(subject.name).to eql('Elasticity Bootstrap Action (Configure Hadoop via File)')
|
12
|
+
expect(subject.arguments).to eql(%w(--mapred-config-file config_file))
|
13
|
+
expect(subject.script).to eql('s3n://elasticmapreduce/bootstrap-actions/configure-hadoop')
|
14
|
+
end
|
15
|
+
end
|
12
16
|
|
13
17
|
end
|
@@ -6,29 +6,33 @@ describe Elasticity::HiveStep do
|
|
6
6
|
|
7
7
|
it { should be_a Elasticity::JobFlowStep }
|
8
8
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
9
|
+
describe '.initialize' do
|
10
|
+
it 'should set the fields appropriately' do
|
11
|
+
expect(subject.name).to eql('Elasticity Hive Step (script.hql)')
|
12
|
+
expect(subject.script).to eql('script.hql')
|
13
|
+
expect(subject.variables).to eql({})
|
14
|
+
expect(subject.action_on_failure).to eql('TERMINATE_JOB_FLOW')
|
15
|
+
end
|
16
|
+
end
|
13
17
|
|
14
18
|
describe '#to_aws_step' do
|
15
19
|
|
16
20
|
it 'should convert to aws step format' do
|
17
|
-
step = subject.to_aws_step(Elasticity::JobFlow.new
|
21
|
+
step = subject.to_aws_step(Elasticity::JobFlow.new)
|
18
22
|
step[:name].should == 'Elasticity Hive Step (script.hql)'
|
19
23
|
step[:action_on_failure].should == 'TERMINATE_JOB_FLOW'
|
20
24
|
step[:hadoop_jar_step][:jar].should == 's3://elasticmapreduce/libs/script-runner/script-runner.jar'
|
21
25
|
step[:hadoop_jar_step][:args].should start_with([
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
26
|
+
's3://elasticmapreduce/libs/hive/hive-script',
|
27
|
+
'--base-path',
|
28
|
+
's3://elasticmapreduce/libs/hive/',
|
29
|
+
'--hive-versions',
|
30
|
+
'latest',
|
31
|
+
'--run-hive-script',
|
32
|
+
'--args',
|
33
|
+
'-f',
|
34
|
+
'script.hql'
|
35
|
+
])
|
32
36
|
end
|
33
37
|
|
34
38
|
context 'when variables are provided' do
|
@@ -42,7 +46,7 @@ describe Elasticity::HiveStep do
|
|
42
46
|
end
|
43
47
|
|
44
48
|
it 'should convert to aws step format' do
|
45
|
-
step = hs_with_variables.to_aws_step(Elasticity::JobFlow.new
|
49
|
+
step = hs_with_variables.to_aws_step(Elasticity::JobFlow.new)
|
46
50
|
step[:hadoop_jar_step][:args][9..13].should == %w(-d VAR1=VALUE1 -d VAR2=VALUE2)
|
47
51
|
end
|
48
52
|
end
|
@@ -51,7 +55,7 @@ describe Elasticity::HiveStep do
|
|
51
55
|
|
52
56
|
describe '.requires_installation?' do
|
53
57
|
it 'should require installation' do
|
54
|
-
Elasticity::HiveStep.requires_installation
|
58
|
+
expect(Elasticity::HiveStep.requires_installation?).to be true
|
55
59
|
end
|
56
60
|
end
|
57
61
|
|
@@ -1,10 +1,14 @@
|
|
1
1
|
describe Elasticity::InstanceGroup do
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
3
|
+
describe '.initialize' do
|
4
|
+
it 'should set the fields appropriately' do
|
5
|
+
expect(subject.bid_price).to eql(nil)
|
6
|
+
expect(subject.count).to eql(1)
|
7
|
+
expect(subject.type).to eql('m1.small')
|
8
|
+
expect(subject.market).to eql('ON_DEMAND')
|
9
|
+
expect(subject.role).to eql('CORE')
|
10
|
+
end
|
11
|
+
end
|
8
12
|
|
9
13
|
describe '#count=' do
|
10
14
|
|
@@ -3,7 +3,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
|
|
3
3
|
let(:emr) { double('Elasticity::EMR') }
|
4
4
|
|
5
5
|
before do
|
6
|
-
Elasticity::EMR.should_receive(:new).with(
|
6
|
+
Elasticity::EMR.should_receive(:new).with(:region => 'us-west-1').and_return(emr)
|
7
7
|
end
|
8
8
|
|
9
9
|
describe 'Hive' do
|
@@ -16,7 +16,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
|
|
16
16
|
end
|
17
17
|
|
18
18
|
let(:hive_jobflow) do
|
19
|
-
Elasticity::JobFlow.new
|
19
|
+
Elasticity::JobFlow.new.tap do |jf|
|
20
20
|
jf.placement = 'us-west-1a'
|
21
21
|
jf.log_uri = 's3n://slif-test/output/logs'
|
22
22
|
jf.add_step(hive_step)
|
@@ -101,7 +101,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
|
|
101
101
|
end
|
102
102
|
|
103
103
|
let(:pig_jobflow) do
|
104
|
-
Elasticity::JobFlow.new
|
104
|
+
Elasticity::JobFlow.new.tap do |jf|
|
105
105
|
jf.placement = 'us-west-1c'
|
106
106
|
jf.instance_count = 8
|
107
107
|
jf.slave_instance_type = 'm1.xlarge'
|
@@ -189,7 +189,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
|
|
189
189
|
end
|
190
190
|
|
191
191
|
let(:custom_jar_jobflow) do
|
192
|
-
Elasticity::JobFlow.new
|
192
|
+
Elasticity::JobFlow.new.tap do |jf|
|
193
193
|
jf.placement = 'us-west-1b'
|
194
194
|
jf.log_uri = 's3n://slif-test/output/logs'
|
195
195
|
jf.add_step(custom_jar_step)
|
@@ -1,41 +1,30 @@
|
|
1
1
|
describe Elasticity::JobFlow do
|
2
2
|
|
3
3
|
subject do
|
4
|
-
Elasticity::JobFlow.new
|
4
|
+
Elasticity::JobFlow.new
|
5
5
|
end
|
6
6
|
|
7
|
-
its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
|
8
|
-
its(:ec2_key_name) { should == nil }
|
9
|
-
its(:name) { should == 'Elasticity Job Flow' }
|
10
|
-
its(:instance_count) { should == 2 }
|
11
|
-
its(:log_uri) { should == nil }
|
12
|
-
its(:master_instance_type) { should == 'm1.small' }
|
13
|
-
its(:slave_instance_type) { should == 'm1.small' }
|
14
|
-
its(:ami_version) { should == 'latest' }
|
15
|
-
its(:keep_job_flow_alive_when_no_steps) { should == false }
|
16
|
-
its(:ec2_subnet_id) { should == nil }
|
17
|
-
its(:placement) { should == 'us-east-1a' }
|
18
|
-
its(:region) { should == 'us-east-1' }
|
19
|
-
its(:visible_to_all_users) { should == false }
|
20
|
-
its(:enable_debugging) { should == false }
|
21
|
-
its(:job_flow_role) { should == nil }
|
22
|
-
its(:service_role) { should == nil }
|
23
|
-
|
24
|
-
its(:access_key) { should == 'access' }
|
25
|
-
its(:secret_key) { should == 'secret' }
|
26
|
-
|
27
7
|
describe '.initialize' do
|
28
|
-
it 'should set the
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
8
|
+
it 'should set the fields appropriately' do
|
9
|
+
expect(subject.action_on_failure).to eql('TERMINATE_JOB_FLOW')
|
10
|
+
expect(subject.ec2_key_name).to eql(nil)
|
11
|
+
expect(subject.name).to eql('Elasticity Job Flow')
|
12
|
+
expect(subject.instance_count).to eql(2)
|
13
|
+
expect(subject.log_uri).to eql(nil)
|
14
|
+
expect(subject.master_instance_type).to eql('m1.small')
|
15
|
+
expect(subject.slave_instance_type).to eql('m1.small')
|
16
|
+
expect(subject.ami_version).to eql('latest')
|
17
|
+
expect(subject.keep_job_flow_alive_when_no_steps).to eql(false)
|
18
|
+
expect(subject.ec2_subnet_id).to eql(nil)
|
19
|
+
expect(subject.placement).to eql('us-east-1a')
|
20
|
+
expect(subject.region).to eql('us-east-1')
|
21
|
+
expect(subject.visible_to_all_users).to eql(false)
|
22
|
+
expect(subject.enable_debugging).to eql(false)
|
23
|
+
expect(subject.job_flow_role).to eql(nil)
|
24
|
+
expect(subject.service_role).to eql(nil)
|
36
25
|
end
|
37
26
|
end
|
38
|
-
|
27
|
+
|
39
28
|
describe '#placement=' do
|
40
29
|
|
41
30
|
context 'when the placement is set' do
|
@@ -213,7 +202,7 @@ describe Elasticity::JobFlow do
|
|
213
202
|
let(:emr) { double('Elasticity::EMR', :run_job_flow => 'RUNNING_JOBFLOW_ID') }
|
214
203
|
|
215
204
|
let(:running_jobflow) do
|
216
|
-
Elasticity::JobFlow.new
|
205
|
+
Elasticity::JobFlow.new.tap do |jf|
|
217
206
|
jf.add_step(Elasticity::PigStep.new('_'))
|
218
207
|
end
|
219
208
|
end
|
@@ -230,8 +219,8 @@ describe Elasticity::JobFlow do
|
|
230
219
|
|
231
220
|
it 'should submit the step' do
|
232
221
|
emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', {
|
233
|
-
|
234
|
-
|
222
|
+
:steps => [additional_step.to_aws_step(running_jobflow)]
|
223
|
+
})
|
235
224
|
running_jobflow.add_step(additional_step)
|
236
225
|
end
|
237
226
|
end
|
@@ -241,8 +230,8 @@ describe Elasticity::JobFlow do
|
|
241
230
|
|
242
231
|
it 'should submit the installation step and the step' do
|
243
232
|
emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', {
|
244
|
-
|
245
|
-
|
233
|
+
:steps => Elasticity::HiveStep.aws_installation_steps << additional_step.to_aws_step(running_jobflow)
|
234
|
+
})
|
246
235
|
running_jobflow.add_step(additional_step)
|
247
236
|
end
|
248
237
|
end
|
@@ -255,8 +244,8 @@ describe Elasticity::JobFlow do
|
|
255
244
|
|
256
245
|
it 'should submit the step' do
|
257
246
|
emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', {
|
258
|
-
|
259
|
-
|
247
|
+
:steps => [additional_step.to_aws_step(running_jobflow)]
|
248
|
+
})
|
260
249
|
running_jobflow.add_step(additional_step)
|
261
250
|
end
|
262
251
|
|
@@ -280,17 +269,17 @@ describe Elasticity::JobFlow do
|
|
280
269
|
|
281
270
|
let(:jobflow_steps) { [Elasticity::HiveStep.new('script.hql'), Elasticity::PigStep.new('script.pig'), Elasticity::CustomJarStep.new('script.jar')] }
|
282
271
|
let(:jobflow_with_steps) do
|
283
|
-
Elasticity::JobFlow.new
|
272
|
+
Elasticity::JobFlow.new.tap do |jobflow|
|
284
273
|
jobflow_steps.each { |s| jobflow.add_step(s) }
|
285
274
|
end
|
286
275
|
end
|
287
276
|
let(:aws_steps) do
|
288
277
|
[
|
289
278
|
Elasticity::HiveStep.aws_installation_steps,
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
279
|
+
jobflow_steps[0].to_aws_step(jobflow_with_steps),
|
280
|
+
Elasticity::PigStep.aws_installation_steps,
|
281
|
+
jobflow_steps[1].to_aws_step(jobflow_with_steps),
|
282
|
+
jobflow_steps[2].to_aws_step(jobflow_with_steps),
|
294
283
|
].flatten
|
295
284
|
end
|
296
285
|
|
@@ -325,7 +314,7 @@ describe Elasticity::JobFlow do
|
|
325
314
|
|
326
315
|
context 'when a log URI is specified' do
|
327
316
|
let(:jobflow_with_log_uri) do
|
328
|
-
Elasticity::JobFlow.new
|
317
|
+
Elasticity::JobFlow.new.tap do |jf|
|
329
318
|
jf.log_uri = 'LOG_URI'
|
330
319
|
end
|
331
320
|
end
|
@@ -336,7 +325,7 @@ describe Elasticity::JobFlow do
|
|
336
325
|
|
337
326
|
context 'when a log URI is not specified' do
|
338
327
|
let(:jobflow_with_no_log_uri) do
|
339
|
-
Elasticity::JobFlow.new
|
328
|
+
Elasticity::JobFlow.new.tap do |jf|
|
340
329
|
jf.log_uri = nil
|
341
330
|
end
|
342
331
|
end
|
@@ -351,7 +340,7 @@ describe Elasticity::JobFlow do
|
|
351
340
|
|
352
341
|
context 'when a job flow role is specified' do
|
353
342
|
let(:jobflow_with_job_flow_role) do
|
354
|
-
Elasticity::JobFlow.new
|
343
|
+
Elasticity::JobFlow.new.tap do |jf|
|
355
344
|
jf.job_flow_role = 'JOB_FLOW_ROLE'
|
356
345
|
end
|
357
346
|
end
|
@@ -362,7 +351,7 @@ describe Elasticity::JobFlow do
|
|
362
351
|
|
363
352
|
context 'when a job flow role is not specified' do
|
364
353
|
let(:jobflow_with_no_job_flow_role) do
|
365
|
-
Elasticity::JobFlow.new
|
354
|
+
Elasticity::JobFlow.new.tap do |jf|
|
366
355
|
jf.job_flow_role = nil
|
367
356
|
end
|
368
357
|
end
|
@@ -377,7 +366,7 @@ describe Elasticity::JobFlow do
|
|
377
366
|
|
378
367
|
context 'when a service role is specified' do
|
379
368
|
let(:jobflow_with_service_role) do
|
380
|
-
Elasticity::JobFlow.new
|
369
|
+
Elasticity::JobFlow.new.tap do |jf|
|
381
370
|
jf.service_role = 'SERVICE_ROLE'
|
382
371
|
end
|
383
372
|
end
|
@@ -388,7 +377,7 @@ describe Elasticity::JobFlow do
|
|
388
377
|
|
389
378
|
context 'when a service role is not specified' do
|
390
379
|
let(:jobflow_with_no_service_role) do
|
391
|
-
Elasticity::JobFlow.new
|
380
|
+
Elasticity::JobFlow.new.tap do |jf|
|
392
381
|
jf.service_role = nil
|
393
382
|
end
|
394
383
|
end
|
@@ -410,7 +399,7 @@ describe Elasticity::JobFlow do
|
|
410
399
|
]
|
411
400
|
end
|
412
401
|
let(:jobflow_with_bootstrap_actions) do
|
413
|
-
Elasticity::JobFlow.new
|
402
|
+
Elasticity::JobFlow.new.tap do |jf|
|
414
403
|
hadoop_bootstrap_actions.each do |action|
|
415
404
|
jf.add_bootstrap_action(action)
|
416
405
|
end
|
@@ -419,8 +408,8 @@ describe Elasticity::JobFlow do
|
|
419
408
|
it 'should include them in the jobflow config' do
|
420
409
|
bootstrap_actions = hadoop_bootstrap_actions.map { |a| a.to_aws_bootstrap_action }
|
421
410
|
jobflow_with_bootstrap_actions.send(:jobflow_config).should be_a_hash_including({
|
422
|
-
|
423
|
-
|
411
|
+
:bootstrap_actions => bootstrap_actions
|
412
|
+
})
|
424
413
|
end
|
425
414
|
end
|
426
415
|
|
@@ -544,7 +533,7 @@ describe Elasticity::JobFlow do
|
|
544
533
|
|
545
534
|
context 'when there are steps added' do
|
546
535
|
let(:jobflow_with_steps) do
|
547
|
-
Elasticity::JobFlow.new
|
536
|
+
Elasticity::JobFlow.new.tap do |jf|
|
548
537
|
jf.add_step(Elasticity::CustomJarStep.new('_'))
|
549
538
|
end
|
550
539
|
end
|
@@ -553,7 +542,7 @@ describe Elasticity::JobFlow do
|
|
553
542
|
let(:emr) { double('Elasticity::EMR', :run_job_flow => 'JOBFLOW_ID') }
|
554
543
|
|
555
544
|
it 'should run the job with the supplied EMR credentials' do
|
556
|
-
Elasticity::EMR.stub(:new).with(
|
545
|
+
Elasticity::EMR.stub(:new).with(:region => 'us-east-1').and_return(emr)
|
557
546
|
emr.should_receive(:run_job_flow)
|
558
547
|
jobflow_with_steps.run
|
559
548
|
end
|
@@ -588,34 +577,68 @@ describe Elasticity::JobFlow do
|
|
588
577
|
|
589
578
|
end
|
590
579
|
|
591
|
-
describe '#
|
580
|
+
describe '#cluster_status' do
|
592
581
|
|
593
582
|
context 'before the jobflow has been run' do
|
594
583
|
it 'should raise an error' do
|
595
584
|
expect {
|
596
|
-
subject.
|
585
|
+
subject.cluster_status
|
597
586
|
}.to raise_error(Elasticity::JobFlowNotStartedError, 'Please #run this job flow before attempting to retrieve status.')
|
598
587
|
end
|
599
588
|
end
|
600
589
|
|
601
590
|
context 'after the jobflow has been run' do
|
602
591
|
let(:emr) { double('Elasticity::EMR', :run_job_flow => 'JOBFLOW_ID') }
|
603
|
-
let(:running_jobflow) { Elasticity::JobFlow.new
|
604
|
-
let(:
|
605
|
-
|
606
|
-
|
607
|
-
|
592
|
+
let(:running_jobflow) { Elasticity::JobFlow.new }
|
593
|
+
let(:aws_cluster_status) { JSON.parse('{ "Cluster": { "Status": { "State": "TERMINATED" } } }') }
|
594
|
+
|
595
|
+
before do
|
596
|
+
Elasticity::EMR.stub(:new).and_return(emr)
|
597
|
+
running_jobflow.add_step(Elasticity::CustomJarStep.new('_'))
|
598
|
+
running_jobflow.run
|
599
|
+
end
|
600
|
+
|
601
|
+
it 'should return the AWS status' do
|
602
|
+
emr.should_receive(:describe_cluster).with('JOBFLOW_ID').and_return(aws_cluster_status)
|
603
|
+
Elasticity::ClusterStatus.should_receive(:from_aws_data).with(aws_cluster_status).and_return(build(:cluster_status))
|
604
|
+
|
605
|
+
status = running_jobflow.cluster_status
|
606
|
+
expect(status).to be_a(Elasticity::ClusterStatus)
|
607
|
+
expect(status.state).to eql('TERMINATED')
|
608
608
|
end
|
609
|
+
end
|
610
|
+
|
611
|
+
end
|
612
|
+
|
613
|
+
describe '#cluster_step_status' do
|
614
|
+
|
615
|
+
context 'before the jobflow has been run' do
|
616
|
+
it 'should raise an error' do
|
617
|
+
expect {
|
618
|
+
subject.cluster_step_status
|
619
|
+
}.to raise_error(Elasticity::JobFlowNotStartedError, 'Please #run this job flow before attempting to retrieve status.')
|
620
|
+
end
|
621
|
+
end
|
622
|
+
|
623
|
+
context 'after the jobflow has been run' do
|
624
|
+
let(:emr) { double('Elasticity::EMR', :run_job_flow => 'JOBFLOW_ID') }
|
625
|
+
let(:running_jobflow) { Elasticity::JobFlow.new }
|
626
|
+
|
609
627
|
before do
|
610
628
|
Elasticity::EMR.stub(:new).and_return(emr)
|
611
629
|
running_jobflow.add_step(Elasticity::CustomJarStep.new('_'))
|
612
630
|
running_jobflow.run
|
613
631
|
end
|
632
|
+
|
614
633
|
it 'should return the AWS status' do
|
615
|
-
emr.should_receive(:
|
616
|
-
|
617
|
-
|
618
|
-
status
|
634
|
+
emr.should_receive(:list_steps).with('JOBFLOW_ID').and_return('AWS_CLUSTER_STATUS')
|
635
|
+
Elasticity::ClusterStepStatus.should_receive(:from_aws_list_data).with('AWS_CLUSTER_STATUS').and_return([build(:cluster_step_status)])
|
636
|
+
|
637
|
+
status = running_jobflow.cluster_step_status
|
638
|
+
status.each do |s|
|
639
|
+
expect(s).to be_a(Elasticity::ClusterStepStatus)
|
640
|
+
end
|
641
|
+
expect(status[0].state).to eql('COMPLETED')
|
619
642
|
end
|
620
643
|
end
|
621
644
|
|
@@ -624,7 +647,7 @@ describe Elasticity::JobFlow do
|
|
624
647
|
describe '#shutdown' do
|
625
648
|
|
626
649
|
context 'when the jobflow has not yet been started' do
|
627
|
-
let(:unstarted_job_flow) { Elasticity::JobFlow.new
|
650
|
+
let(:unstarted_job_flow) { Elasticity::JobFlow.new }
|
628
651
|
it 'should be an error' do
|
629
652
|
expect {
|
630
653
|
unstarted_job_flow.shutdown
|
@@ -634,7 +657,7 @@ describe Elasticity::JobFlow do
|
|
634
657
|
|
635
658
|
context 'when the jobflow has been started' do
|
636
659
|
let(:emr) { double('Elasticity::EMR', :run_job_flow => 'JOBFLOW_ID') }
|
637
|
-
let(:running_jobflow) { Elasticity::JobFlow.new
|
660
|
+
let(:running_jobflow) { Elasticity::JobFlow.new }
|
638
661
|
before do
|
639
662
|
Elasticity::EMR.stub(:new).and_return(emr)
|
640
663
|
running_jobflow.add_step(Elasticity::CustomJarStep.new('_'))
|
@@ -664,7 +687,7 @@ describe Elasticity::JobFlow do
|
|
664
687
|
context 'when the jobflow is active' do
|
665
688
|
let(:jobflow_status) { double(:active? => true) }
|
666
689
|
before do
|
667
|
-
subject.stub(:
|
690
|
+
subject.stub(:cluster_status).and_return(jobflow_status)
|
668
691
|
end
|
669
692
|
it 'returns true and the result of #status' do
|
670
693
|
subject.send(:retry_check).should == [true, jobflow_status]
|
@@ -674,7 +697,7 @@ describe Elasticity::JobFlow do
|
|
674
697
|
context 'when the jobflow is not active' do
|
675
698
|
let(:jobflow_status) { double(:active? => false) }
|
676
699
|
before do
|
677
|
-
subject.stub(:
|
700
|
+
subject.stub(:cluster_status).and_return(jobflow_status)
|
678
701
|
end
|
679
702
|
it 'returns true and the result of #status' do
|
680
703
|
subject.send(:retry_check).should == [false, jobflow_status]
|
@@ -686,24 +709,24 @@ describe Elasticity::JobFlow do
|
|
686
709
|
describe '.from_jobflow_id' do
|
687
710
|
|
688
711
|
before do
|
689
|
-
Elasticity::JobFlow.any_instance.
|
712
|
+
Elasticity::JobFlow.any_instance.stub(:cluster_step_status).and_return([])
|
690
713
|
end
|
691
714
|
|
692
|
-
let(:jobflow) { Elasticity::JobFlow.from_jobflow_id('
|
715
|
+
let(:jobflow) { Elasticity::JobFlow.from_jobflow_id('JOBFLOW_ID') }
|
693
716
|
|
694
717
|
describe 'creating a jobflow with the specified credentials' do
|
695
718
|
|
696
719
|
context 'when the region is not specified' do
|
697
720
|
it 'should use the default of us-east-1a' do
|
698
|
-
j = Elasticity::JobFlow.from_jobflow_id('
|
699
|
-
j.send(:emr).should == Elasticity::EMR.new(
|
721
|
+
j = Elasticity::JobFlow.from_jobflow_id('_')
|
722
|
+
j.send(:emr).should == Elasticity::EMR.new(:region => 'us-east-1')
|
700
723
|
end
|
701
724
|
end
|
702
725
|
|
703
726
|
context 'when the region is specified' do
|
704
727
|
it 'should use the specified region' do
|
705
|
-
j = Elasticity::JobFlow.from_jobflow_id('
|
706
|
-
j.send(:emr).should == Elasticity::EMR.new(
|
728
|
+
j = Elasticity::JobFlow.from_jobflow_id('_', 'us-west-1')
|
729
|
+
j.send(:emr).should == Elasticity::EMR.new(:region => 'us-west-1')
|
707
730
|
end
|
708
731
|
end
|
709
732
|
|
@@ -723,7 +746,7 @@ describe Elasticity::JobFlow do
|
|
723
746
|
|
724
747
|
context 'when no steps have been installed' do
|
725
748
|
before do
|
726
|
-
Elasticity::JobFlow.any_instance.should_receive(:
|
749
|
+
Elasticity::JobFlow.any_instance.should_receive(:cluster_step_status).and_return([])
|
727
750
|
end
|
728
751
|
it 'should show that no steps are installed' do
|
729
752
|
jobflow.instance_variable_get(:@installed_steps).should == []
|
@@ -732,7 +755,10 @@ describe Elasticity::JobFlow do
|
|
732
755
|
|
733
756
|
context 'when steps have been installed do' do
|
734
757
|
before do
|
735
|
-
Elasticity::JobFlow.any_instance.should_receive(:
|
758
|
+
Elasticity::JobFlow.any_instance.should_receive(:cluster_step_status).and_return([
|
759
|
+
build(:cluster_step_status, :name => Elasticity::PigStep.aws_installation_step_name),
|
760
|
+
build(:cluster_step_status, :name => Elasticity::HiveStep.aws_installation_step_name)
|
761
|
+
])
|
736
762
|
end
|
737
763
|
it 'should show that no steps are installed' do
|
738
764
|
jobflow.instance_variable_get(:@installed_steps).should =~ [Elasticity::PigStep, Elasticity::HiveStep]
|