elasticity 5.0.3 → 6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/HISTORY.md +26 -0
- data/README.md +35 -28
- data/elasticity.gemspec +2 -2
- data/lib/elasticity.rb +5 -3
- data/lib/elasticity/aws_request_v4.rb +15 -3
- data/lib/elasticity/aws_session.rb +4 -23
- data/lib/elasticity/aws_utils.rb +0 -29
- data/lib/elasticity/cluster_status.rb +38 -0
- data/lib/elasticity/cluster_step_status.rb +51 -0
- data/lib/elasticity/emr.rb +208 -78
- data/lib/elasticity/job_flow.rb +16 -17
- data/lib/elasticity/version.rb +1 -1
- data/spec/factories/cluster_status_factory.rb +12 -0
- data/spec/factories/cluster_step_status_factory.rb +17 -0
- data/spec/lib/elasticity/aws_request_v4_spec.rb +54 -4
- data/spec/lib/elasticity/aws_session_spec.rb +22 -88
- data/spec/lib/elasticity/aws_utils_spec.rb +0 -46
- data/spec/lib/elasticity/bootstrap_action_spec.rb +7 -3
- data/spec/lib/elasticity/cluster_status_spec.rb +98 -0
- data/spec/lib/elasticity/cluster_step_status_spec.rb +80 -0
- data/spec/lib/elasticity/custom_jar_step_spec.rb +10 -7
- data/spec/lib/elasticity/emr_spec.rb +422 -132
- data/spec/lib/elasticity/ganglia_bootstrap_action_spec.rb +8 -3
- data/spec/lib/elasticity/hadoop_bootstrap_action_spec.rb +8 -3
- data/spec/lib/elasticity/hadoop_file_bootstrap_action_spec.rb +7 -3
- data/spec/lib/elasticity/hive_step_spec.rb +21 -17
- data/spec/lib/elasticity/instance_group_spec.rb +9 -5
- data/spec/lib/elasticity/job_flow_integration_spec.rb +4 -4
- data/spec/lib/elasticity/job_flow_spec.rb +102 -76
- data/spec/lib/elasticity/job_flow_step_spec.rb +1 -1
- data/spec/lib/elasticity/looper_spec.rb +1 -1
- data/spec/lib/elasticity/pig_step_spec.rb +13 -9
- data/spec/lib/elasticity/s3distcp_step_spec.rb +7 -5
- data/spec/lib/elasticity/script_step_spec.rb +11 -6
- data/spec/lib/elasticity/setup_hadoop_debugging_step_spec.rb +9 -5
- data/spec/lib/elasticity/streaming_step_spec.rb +13 -9
- data/spec/spec_helper.rb +8 -0
- data/spec/support/factory_girl.rb +8 -0
- metadata +24 -21
- data/lib/elasticity/aws_request_v2.rb +0 -42
- data/lib/elasticity/job_flow_status.rb +0 -91
- data/lib/elasticity/job_flow_status_step.rb +0 -38
- data/spec/lib/elasticity/aws_request_v2_spec.rb +0 -38
- data/spec/lib/elasticity/job_flow_status_spec.rb +0 -265
- data/spec/lib/elasticity/job_flow_status_step_spec.rb +0 -80
@@ -6,8 +6,13 @@ describe Elasticity::GangliaBootstrapAction do
|
|
6
6
|
|
7
7
|
it { should be_a Elasticity::BootstrapAction }
|
8
8
|
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
describe '.intialize' do
|
10
|
+
it 'should set the fields appropriately' do
|
11
|
+
expect(subject.name).to eql('Elasticity Bootstrap Action (Install Ganglia)')
|
12
|
+
expect(subject.arguments).to eql([])
|
13
|
+
expect(subject.script).to eql('s3://elasticmapreduce/bootstrap-actions/install-ganglia')
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
12
17
|
|
13
18
|
end
|
@@ -6,8 +6,13 @@ describe Elasticity::HadoopBootstrapAction do
|
|
6
6
|
|
7
7
|
it { should be_a Elasticity::BootstrapAction }
|
8
8
|
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
describe '.initialize' do
|
10
|
+
it 'should set the fields appropriately' do
|
11
|
+
expect(subject.name).to eql('Elasticity Bootstrap Action (Configure Hadoop)')
|
12
|
+
expect(subject.arguments).to eql(%w(option value))
|
13
|
+
expect(subject.script).to eql('s3n://elasticmapreduce/bootstrap-actions/configure-hadoop')
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
12
17
|
|
13
18
|
end
|
@@ -6,8 +6,12 @@ describe Elasticity::HadoopFileBootstrapAction do
|
|
6
6
|
|
7
7
|
it { should be_a Elasticity::BootstrapAction }
|
8
8
|
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
describe '.initialize' do
|
10
|
+
it 'should set the fields appropriately' do
|
11
|
+
expect(subject.name).to eql('Elasticity Bootstrap Action (Configure Hadoop via File)')
|
12
|
+
expect(subject.arguments).to eql(%w(--mapred-config-file config_file))
|
13
|
+
expect(subject.script).to eql('s3n://elasticmapreduce/bootstrap-actions/configure-hadoop')
|
14
|
+
end
|
15
|
+
end
|
12
16
|
|
13
17
|
end
|
@@ -6,29 +6,33 @@ describe Elasticity::HiveStep do
|
|
6
6
|
|
7
7
|
it { should be_a Elasticity::JobFlowStep }
|
8
8
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
9
|
+
describe '.initialize' do
|
10
|
+
it 'should set the fields appropriately' do
|
11
|
+
expect(subject.name).to eql('Elasticity Hive Step (script.hql)')
|
12
|
+
expect(subject.script).to eql('script.hql')
|
13
|
+
expect(subject.variables).to eql({})
|
14
|
+
expect(subject.action_on_failure).to eql('TERMINATE_JOB_FLOW')
|
15
|
+
end
|
16
|
+
end
|
13
17
|
|
14
18
|
describe '#to_aws_step' do
|
15
19
|
|
16
20
|
it 'should convert to aws step format' do
|
17
|
-
step = subject.to_aws_step(Elasticity::JobFlow.new
|
21
|
+
step = subject.to_aws_step(Elasticity::JobFlow.new)
|
18
22
|
step[:name].should == 'Elasticity Hive Step (script.hql)'
|
19
23
|
step[:action_on_failure].should == 'TERMINATE_JOB_FLOW'
|
20
24
|
step[:hadoop_jar_step][:jar].should == 's3://elasticmapreduce/libs/script-runner/script-runner.jar'
|
21
25
|
step[:hadoop_jar_step][:args].should start_with([
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
26
|
+
's3://elasticmapreduce/libs/hive/hive-script',
|
27
|
+
'--base-path',
|
28
|
+
's3://elasticmapreduce/libs/hive/',
|
29
|
+
'--hive-versions',
|
30
|
+
'latest',
|
31
|
+
'--run-hive-script',
|
32
|
+
'--args',
|
33
|
+
'-f',
|
34
|
+
'script.hql'
|
35
|
+
])
|
32
36
|
end
|
33
37
|
|
34
38
|
context 'when variables are provided' do
|
@@ -42,7 +46,7 @@ describe Elasticity::HiveStep do
|
|
42
46
|
end
|
43
47
|
|
44
48
|
it 'should convert to aws step format' do
|
45
|
-
step = hs_with_variables.to_aws_step(Elasticity::JobFlow.new
|
49
|
+
step = hs_with_variables.to_aws_step(Elasticity::JobFlow.new)
|
46
50
|
step[:hadoop_jar_step][:args][9..13].should == %w(-d VAR1=VALUE1 -d VAR2=VALUE2)
|
47
51
|
end
|
48
52
|
end
|
@@ -51,7 +55,7 @@ describe Elasticity::HiveStep do
|
|
51
55
|
|
52
56
|
describe '.requires_installation?' do
|
53
57
|
it 'should require installation' do
|
54
|
-
Elasticity::HiveStep.requires_installation
|
58
|
+
expect(Elasticity::HiveStep.requires_installation?).to be true
|
55
59
|
end
|
56
60
|
end
|
57
61
|
|
@@ -1,10 +1,14 @@
|
|
1
1
|
describe Elasticity::InstanceGroup do
|
2
2
|
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
3
|
+
describe '.initialize' do
|
4
|
+
it 'should set the fields appropriately' do
|
5
|
+
expect(subject.bid_price).to eql(nil)
|
6
|
+
expect(subject.count).to eql(1)
|
7
|
+
expect(subject.type).to eql('m1.small')
|
8
|
+
expect(subject.market).to eql('ON_DEMAND')
|
9
|
+
expect(subject.role).to eql('CORE')
|
10
|
+
end
|
11
|
+
end
|
8
12
|
|
9
13
|
describe '#count=' do
|
10
14
|
|
@@ -3,7 +3,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
|
|
3
3
|
let(:emr) { double('Elasticity::EMR') }
|
4
4
|
|
5
5
|
before do
|
6
|
-
Elasticity::EMR.should_receive(:new).with(
|
6
|
+
Elasticity::EMR.should_receive(:new).with(:region => 'us-west-1').and_return(emr)
|
7
7
|
end
|
8
8
|
|
9
9
|
describe 'Hive' do
|
@@ -16,7 +16,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
|
|
16
16
|
end
|
17
17
|
|
18
18
|
let(:hive_jobflow) do
|
19
|
-
Elasticity::JobFlow.new
|
19
|
+
Elasticity::JobFlow.new.tap do |jf|
|
20
20
|
jf.placement = 'us-west-1a'
|
21
21
|
jf.log_uri = 's3n://slif-test/output/logs'
|
22
22
|
jf.add_step(hive_step)
|
@@ -101,7 +101,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
|
|
101
101
|
end
|
102
102
|
|
103
103
|
let(:pig_jobflow) do
|
104
|
-
Elasticity::JobFlow.new
|
104
|
+
Elasticity::JobFlow.new.tap do |jf|
|
105
105
|
jf.placement = 'us-west-1c'
|
106
106
|
jf.instance_count = 8
|
107
107
|
jf.slave_instance_type = 'm1.xlarge'
|
@@ -189,7 +189,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
|
|
189
189
|
end
|
190
190
|
|
191
191
|
let(:custom_jar_jobflow) do
|
192
|
-
Elasticity::JobFlow.new
|
192
|
+
Elasticity::JobFlow.new.tap do |jf|
|
193
193
|
jf.placement = 'us-west-1b'
|
194
194
|
jf.log_uri = 's3n://slif-test/output/logs'
|
195
195
|
jf.add_step(custom_jar_step)
|
@@ -1,41 +1,30 @@
|
|
1
1
|
describe Elasticity::JobFlow do
|
2
2
|
|
3
3
|
subject do
|
4
|
-
Elasticity::JobFlow.new
|
4
|
+
Elasticity::JobFlow.new
|
5
5
|
end
|
6
6
|
|
7
|
-
its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
|
8
|
-
its(:ec2_key_name) { should == nil }
|
9
|
-
its(:name) { should == 'Elasticity Job Flow' }
|
10
|
-
its(:instance_count) { should == 2 }
|
11
|
-
its(:log_uri) { should == nil }
|
12
|
-
its(:master_instance_type) { should == 'm1.small' }
|
13
|
-
its(:slave_instance_type) { should == 'm1.small' }
|
14
|
-
its(:ami_version) { should == 'latest' }
|
15
|
-
its(:keep_job_flow_alive_when_no_steps) { should == false }
|
16
|
-
its(:ec2_subnet_id) { should == nil }
|
17
|
-
its(:placement) { should == 'us-east-1a' }
|
18
|
-
its(:region) { should == 'us-east-1' }
|
19
|
-
its(:visible_to_all_users) { should == false }
|
20
|
-
its(:enable_debugging) { should == false }
|
21
|
-
its(:job_flow_role) { should == nil }
|
22
|
-
its(:service_role) { should == nil }
|
23
|
-
|
24
|
-
its(:access_key) { should == 'access' }
|
25
|
-
its(:secret_key) { should == 'secret' }
|
26
|
-
|
27
7
|
describe '.initialize' do
|
28
|
-
it 'should set the
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
8
|
+
it 'should set the fields appropriately' do
|
9
|
+
expect(subject.action_on_failure).to eql('TERMINATE_JOB_FLOW')
|
10
|
+
expect(subject.ec2_key_name).to eql(nil)
|
11
|
+
expect(subject.name).to eql('Elasticity Job Flow')
|
12
|
+
expect(subject.instance_count).to eql(2)
|
13
|
+
expect(subject.log_uri).to eql(nil)
|
14
|
+
expect(subject.master_instance_type).to eql('m1.small')
|
15
|
+
expect(subject.slave_instance_type).to eql('m1.small')
|
16
|
+
expect(subject.ami_version).to eql('latest')
|
17
|
+
expect(subject.keep_job_flow_alive_when_no_steps).to eql(false)
|
18
|
+
expect(subject.ec2_subnet_id).to eql(nil)
|
19
|
+
expect(subject.placement).to eql('us-east-1a')
|
20
|
+
expect(subject.region).to eql('us-east-1')
|
21
|
+
expect(subject.visible_to_all_users).to eql(false)
|
22
|
+
expect(subject.enable_debugging).to eql(false)
|
23
|
+
expect(subject.job_flow_role).to eql(nil)
|
24
|
+
expect(subject.service_role).to eql(nil)
|
36
25
|
end
|
37
26
|
end
|
38
|
-
|
27
|
+
|
39
28
|
describe '#placement=' do
|
40
29
|
|
41
30
|
context 'when the placement is set' do
|
@@ -213,7 +202,7 @@ describe Elasticity::JobFlow do
|
|
213
202
|
let(:emr) { double('Elasticity::EMR', :run_job_flow => 'RUNNING_JOBFLOW_ID') }
|
214
203
|
|
215
204
|
let(:running_jobflow) do
|
216
|
-
Elasticity::JobFlow.new
|
205
|
+
Elasticity::JobFlow.new.tap do |jf|
|
217
206
|
jf.add_step(Elasticity::PigStep.new('_'))
|
218
207
|
end
|
219
208
|
end
|
@@ -230,8 +219,8 @@ describe Elasticity::JobFlow do
|
|
230
219
|
|
231
220
|
it 'should submit the step' do
|
232
221
|
emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', {
|
233
|
-
|
234
|
-
|
222
|
+
:steps => [additional_step.to_aws_step(running_jobflow)]
|
223
|
+
})
|
235
224
|
running_jobflow.add_step(additional_step)
|
236
225
|
end
|
237
226
|
end
|
@@ -241,8 +230,8 @@ describe Elasticity::JobFlow do
|
|
241
230
|
|
242
231
|
it 'should submit the installation step and the step' do
|
243
232
|
emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', {
|
244
|
-
|
245
|
-
|
233
|
+
:steps => Elasticity::HiveStep.aws_installation_steps << additional_step.to_aws_step(running_jobflow)
|
234
|
+
})
|
246
235
|
running_jobflow.add_step(additional_step)
|
247
236
|
end
|
248
237
|
end
|
@@ -255,8 +244,8 @@ describe Elasticity::JobFlow do
|
|
255
244
|
|
256
245
|
it 'should submit the step' do
|
257
246
|
emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', {
|
258
|
-
|
259
|
-
|
247
|
+
:steps => [additional_step.to_aws_step(running_jobflow)]
|
248
|
+
})
|
260
249
|
running_jobflow.add_step(additional_step)
|
261
250
|
end
|
262
251
|
|
@@ -280,17 +269,17 @@ describe Elasticity::JobFlow do
|
|
280
269
|
|
281
270
|
let(:jobflow_steps) { [Elasticity::HiveStep.new('script.hql'), Elasticity::PigStep.new('script.pig'), Elasticity::CustomJarStep.new('script.jar')] }
|
282
271
|
let(:jobflow_with_steps) do
|
283
|
-
Elasticity::JobFlow.new
|
272
|
+
Elasticity::JobFlow.new.tap do |jobflow|
|
284
273
|
jobflow_steps.each { |s| jobflow.add_step(s) }
|
285
274
|
end
|
286
275
|
end
|
287
276
|
let(:aws_steps) do
|
288
277
|
[
|
289
278
|
Elasticity::HiveStep.aws_installation_steps,
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
279
|
+
jobflow_steps[0].to_aws_step(jobflow_with_steps),
|
280
|
+
Elasticity::PigStep.aws_installation_steps,
|
281
|
+
jobflow_steps[1].to_aws_step(jobflow_with_steps),
|
282
|
+
jobflow_steps[2].to_aws_step(jobflow_with_steps),
|
294
283
|
].flatten
|
295
284
|
end
|
296
285
|
|
@@ -325,7 +314,7 @@ describe Elasticity::JobFlow do
|
|
325
314
|
|
326
315
|
context 'when a log URI is specified' do
|
327
316
|
let(:jobflow_with_log_uri) do
|
328
|
-
Elasticity::JobFlow.new
|
317
|
+
Elasticity::JobFlow.new.tap do |jf|
|
329
318
|
jf.log_uri = 'LOG_URI'
|
330
319
|
end
|
331
320
|
end
|
@@ -336,7 +325,7 @@ describe Elasticity::JobFlow do
|
|
336
325
|
|
337
326
|
context 'when a log URI is not specified' do
|
338
327
|
let(:jobflow_with_no_log_uri) do
|
339
|
-
Elasticity::JobFlow.new
|
328
|
+
Elasticity::JobFlow.new.tap do |jf|
|
340
329
|
jf.log_uri = nil
|
341
330
|
end
|
342
331
|
end
|
@@ -351,7 +340,7 @@ describe Elasticity::JobFlow do
|
|
351
340
|
|
352
341
|
context 'when a job flow role is specified' do
|
353
342
|
let(:jobflow_with_job_flow_role) do
|
354
|
-
Elasticity::JobFlow.new
|
343
|
+
Elasticity::JobFlow.new.tap do |jf|
|
355
344
|
jf.job_flow_role = 'JOB_FLOW_ROLE'
|
356
345
|
end
|
357
346
|
end
|
@@ -362,7 +351,7 @@ describe Elasticity::JobFlow do
|
|
362
351
|
|
363
352
|
context 'when a job flow role is not specified' do
|
364
353
|
let(:jobflow_with_no_job_flow_role) do
|
365
|
-
Elasticity::JobFlow.new
|
354
|
+
Elasticity::JobFlow.new.tap do |jf|
|
366
355
|
jf.job_flow_role = nil
|
367
356
|
end
|
368
357
|
end
|
@@ -377,7 +366,7 @@ describe Elasticity::JobFlow do
|
|
377
366
|
|
378
367
|
context 'when a service role is specified' do
|
379
368
|
let(:jobflow_with_service_role) do
|
380
|
-
Elasticity::JobFlow.new
|
369
|
+
Elasticity::JobFlow.new.tap do |jf|
|
381
370
|
jf.service_role = 'SERVICE_ROLE'
|
382
371
|
end
|
383
372
|
end
|
@@ -388,7 +377,7 @@ describe Elasticity::JobFlow do
|
|
388
377
|
|
389
378
|
context 'when a service role is not specified' do
|
390
379
|
let(:jobflow_with_no_service_role) do
|
391
|
-
Elasticity::JobFlow.new
|
380
|
+
Elasticity::JobFlow.new.tap do |jf|
|
392
381
|
jf.service_role = nil
|
393
382
|
end
|
394
383
|
end
|
@@ -410,7 +399,7 @@ describe Elasticity::JobFlow do
|
|
410
399
|
]
|
411
400
|
end
|
412
401
|
let(:jobflow_with_bootstrap_actions) do
|
413
|
-
Elasticity::JobFlow.new
|
402
|
+
Elasticity::JobFlow.new.tap do |jf|
|
414
403
|
hadoop_bootstrap_actions.each do |action|
|
415
404
|
jf.add_bootstrap_action(action)
|
416
405
|
end
|
@@ -419,8 +408,8 @@ describe Elasticity::JobFlow do
|
|
419
408
|
it 'should include them in the jobflow config' do
|
420
409
|
bootstrap_actions = hadoop_bootstrap_actions.map { |a| a.to_aws_bootstrap_action }
|
421
410
|
jobflow_with_bootstrap_actions.send(:jobflow_config).should be_a_hash_including({
|
422
|
-
|
423
|
-
|
411
|
+
:bootstrap_actions => bootstrap_actions
|
412
|
+
})
|
424
413
|
end
|
425
414
|
end
|
426
415
|
|
@@ -544,7 +533,7 @@ describe Elasticity::JobFlow do
|
|
544
533
|
|
545
534
|
context 'when there are steps added' do
|
546
535
|
let(:jobflow_with_steps) do
|
547
|
-
Elasticity::JobFlow.new
|
536
|
+
Elasticity::JobFlow.new.tap do |jf|
|
548
537
|
jf.add_step(Elasticity::CustomJarStep.new('_'))
|
549
538
|
end
|
550
539
|
end
|
@@ -553,7 +542,7 @@ describe Elasticity::JobFlow do
|
|
553
542
|
let(:emr) { double('Elasticity::EMR', :run_job_flow => 'JOBFLOW_ID') }
|
554
543
|
|
555
544
|
it 'should run the job with the supplied EMR credentials' do
|
556
|
-
Elasticity::EMR.stub(:new).with(
|
545
|
+
Elasticity::EMR.stub(:new).with(:region => 'us-east-1').and_return(emr)
|
557
546
|
emr.should_receive(:run_job_flow)
|
558
547
|
jobflow_with_steps.run
|
559
548
|
end
|
@@ -588,34 +577,68 @@ describe Elasticity::JobFlow do
|
|
588
577
|
|
589
578
|
end
|
590
579
|
|
591
|
-
describe '#
|
580
|
+
describe '#cluster_status' do
|
592
581
|
|
593
582
|
context 'before the jobflow has been run' do
|
594
583
|
it 'should raise an error' do
|
595
584
|
expect {
|
596
|
-
subject.
|
585
|
+
subject.cluster_status
|
597
586
|
}.to raise_error(Elasticity::JobFlowNotStartedError, 'Please #run this job flow before attempting to retrieve status.')
|
598
587
|
end
|
599
588
|
end
|
600
589
|
|
601
590
|
context 'after the jobflow has been run' do
|
602
591
|
let(:emr) { double('Elasticity::EMR', :run_job_flow => 'JOBFLOW_ID') }
|
603
|
-
let(:running_jobflow) { Elasticity::JobFlow.new
|
604
|
-
let(:
|
605
|
-
|
606
|
-
|
607
|
-
|
592
|
+
let(:running_jobflow) { Elasticity::JobFlow.new }
|
593
|
+
let(:aws_cluster_status) { JSON.parse('{ "Cluster": { "Status": { "State": "TERMINATED" } } }') }
|
594
|
+
|
595
|
+
before do
|
596
|
+
Elasticity::EMR.stub(:new).and_return(emr)
|
597
|
+
running_jobflow.add_step(Elasticity::CustomJarStep.new('_'))
|
598
|
+
running_jobflow.run
|
599
|
+
end
|
600
|
+
|
601
|
+
it 'should return the AWS status' do
|
602
|
+
emr.should_receive(:describe_cluster).with('JOBFLOW_ID').and_return(aws_cluster_status)
|
603
|
+
Elasticity::ClusterStatus.should_receive(:from_aws_data).with(aws_cluster_status).and_return(build(:cluster_status))
|
604
|
+
|
605
|
+
status = running_jobflow.cluster_status
|
606
|
+
expect(status).to be_a(Elasticity::ClusterStatus)
|
607
|
+
expect(status.state).to eql('TERMINATED')
|
608
608
|
end
|
609
|
+
end
|
610
|
+
|
611
|
+
end
|
612
|
+
|
613
|
+
describe '#cluster_step_status' do
|
614
|
+
|
615
|
+
context 'before the jobflow has been run' do
|
616
|
+
it 'should raise an error' do
|
617
|
+
expect {
|
618
|
+
subject.cluster_step_status
|
619
|
+
}.to raise_error(Elasticity::JobFlowNotStartedError, 'Please #run this job flow before attempting to retrieve status.')
|
620
|
+
end
|
621
|
+
end
|
622
|
+
|
623
|
+
context 'after the jobflow has been run' do
|
624
|
+
let(:emr) { double('Elasticity::EMR', :run_job_flow => 'JOBFLOW_ID') }
|
625
|
+
let(:running_jobflow) { Elasticity::JobFlow.new }
|
626
|
+
|
609
627
|
before do
|
610
628
|
Elasticity::EMR.stub(:new).and_return(emr)
|
611
629
|
running_jobflow.add_step(Elasticity::CustomJarStep.new('_'))
|
612
630
|
running_jobflow.run
|
613
631
|
end
|
632
|
+
|
614
633
|
it 'should return the AWS status' do
|
615
|
-
emr.should_receive(:
|
616
|
-
|
617
|
-
|
618
|
-
status
|
634
|
+
emr.should_receive(:list_steps).with('JOBFLOW_ID').and_return('AWS_CLUSTER_STATUS')
|
635
|
+
Elasticity::ClusterStepStatus.should_receive(:from_aws_list_data).with('AWS_CLUSTER_STATUS').and_return([build(:cluster_step_status)])
|
636
|
+
|
637
|
+
status = running_jobflow.cluster_step_status
|
638
|
+
status.each do |s|
|
639
|
+
expect(s).to be_a(Elasticity::ClusterStepStatus)
|
640
|
+
end
|
641
|
+
expect(status[0].state).to eql('COMPLETED')
|
619
642
|
end
|
620
643
|
end
|
621
644
|
|
@@ -624,7 +647,7 @@ describe Elasticity::JobFlow do
|
|
624
647
|
describe '#shutdown' do
|
625
648
|
|
626
649
|
context 'when the jobflow has not yet been started' do
|
627
|
-
let(:unstarted_job_flow) { Elasticity::JobFlow.new
|
650
|
+
let(:unstarted_job_flow) { Elasticity::JobFlow.new }
|
628
651
|
it 'should be an error' do
|
629
652
|
expect {
|
630
653
|
unstarted_job_flow.shutdown
|
@@ -634,7 +657,7 @@ describe Elasticity::JobFlow do
|
|
634
657
|
|
635
658
|
context 'when the jobflow has been started' do
|
636
659
|
let(:emr) { double('Elasticity::EMR', :run_job_flow => 'JOBFLOW_ID') }
|
637
|
-
let(:running_jobflow) { Elasticity::JobFlow.new
|
660
|
+
let(:running_jobflow) { Elasticity::JobFlow.new }
|
638
661
|
before do
|
639
662
|
Elasticity::EMR.stub(:new).and_return(emr)
|
640
663
|
running_jobflow.add_step(Elasticity::CustomJarStep.new('_'))
|
@@ -664,7 +687,7 @@ describe Elasticity::JobFlow do
|
|
664
687
|
context 'when the jobflow is active' do
|
665
688
|
let(:jobflow_status) { double(:active? => true) }
|
666
689
|
before do
|
667
|
-
subject.stub(:
|
690
|
+
subject.stub(:cluster_status).and_return(jobflow_status)
|
668
691
|
end
|
669
692
|
it 'returns true and the result of #status' do
|
670
693
|
subject.send(:retry_check).should == [true, jobflow_status]
|
@@ -674,7 +697,7 @@ describe Elasticity::JobFlow do
|
|
674
697
|
context 'when the jobflow is not active' do
|
675
698
|
let(:jobflow_status) { double(:active? => false) }
|
676
699
|
before do
|
677
|
-
subject.stub(:
|
700
|
+
subject.stub(:cluster_status).and_return(jobflow_status)
|
678
701
|
end
|
679
702
|
it 'returns true and the result of #status' do
|
680
703
|
subject.send(:retry_check).should == [false, jobflow_status]
|
@@ -686,24 +709,24 @@ describe Elasticity::JobFlow do
|
|
686
709
|
describe '.from_jobflow_id' do
|
687
710
|
|
688
711
|
before do
|
689
|
-
Elasticity::JobFlow.any_instance.
|
712
|
+
Elasticity::JobFlow.any_instance.stub(:cluster_step_status).and_return([])
|
690
713
|
end
|
691
714
|
|
692
|
-
let(:jobflow) { Elasticity::JobFlow.from_jobflow_id('
|
715
|
+
let(:jobflow) { Elasticity::JobFlow.from_jobflow_id('JOBFLOW_ID') }
|
693
716
|
|
694
717
|
describe 'creating a jobflow with the specified credentials' do
|
695
718
|
|
696
719
|
context 'when the region is not specified' do
|
697
720
|
it 'should use the default of us-east-1a' do
|
698
|
-
j = Elasticity::JobFlow.from_jobflow_id('
|
699
|
-
j.send(:emr).should == Elasticity::EMR.new(
|
721
|
+
j = Elasticity::JobFlow.from_jobflow_id('_')
|
722
|
+
j.send(:emr).should == Elasticity::EMR.new(:region => 'us-east-1')
|
700
723
|
end
|
701
724
|
end
|
702
725
|
|
703
726
|
context 'when the region is specified' do
|
704
727
|
it 'should use the specified region' do
|
705
|
-
j = Elasticity::JobFlow.from_jobflow_id('
|
706
|
-
j.send(:emr).should == Elasticity::EMR.new(
|
728
|
+
j = Elasticity::JobFlow.from_jobflow_id('_', 'us-west-1')
|
729
|
+
j.send(:emr).should == Elasticity::EMR.new(:region => 'us-west-1')
|
707
730
|
end
|
708
731
|
end
|
709
732
|
|
@@ -723,7 +746,7 @@ describe Elasticity::JobFlow do
|
|
723
746
|
|
724
747
|
context 'when no steps have been installed' do
|
725
748
|
before do
|
726
|
-
Elasticity::JobFlow.any_instance.should_receive(:
|
749
|
+
Elasticity::JobFlow.any_instance.should_receive(:cluster_step_status).and_return([])
|
727
750
|
end
|
728
751
|
it 'should show that no steps are installed' do
|
729
752
|
jobflow.instance_variable_get(:@installed_steps).should == []
|
@@ -732,7 +755,10 @@ describe Elasticity::JobFlow do
|
|
732
755
|
|
733
756
|
context 'when steps have been installed do' do
|
734
757
|
before do
|
735
|
-
Elasticity::JobFlow.any_instance.should_receive(:
|
758
|
+
Elasticity::JobFlow.any_instance.should_receive(:cluster_step_status).and_return([
|
759
|
+
build(:cluster_step_status, :name => Elasticity::PigStep.aws_installation_step_name),
|
760
|
+
build(:cluster_step_status, :name => Elasticity::HiveStep.aws_installation_step_name)
|
761
|
+
])
|
736
762
|
end
|
737
763
|
it 'should show that no steps are installed' do
|
738
764
|
jobflow.instance_variable_get(:@installed_steps).should =~ [Elasticity::PigStep, Elasticity::HiveStep]
|