elasticity 2.5.6 → 2.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +1 -0
- data/.ruby-gemset +2 -0
- data/.ruby-version +1 -0
- data/.travis.yml +1 -2
- data/Guardfile +5 -0
- data/HISTORY.md +14 -0
- data/README.md +27 -9
- data/elasticity.gemspec +7 -6
- data/lib/elasticity.rb +2 -0
- data/lib/elasticity/job_flow.rb +23 -1
- data/lib/elasticity/looper.rb +23 -0
- data/lib/elasticity/setup_hadoop_debugging_step.rb +14 -0
- data/lib/elasticity/version.rb +1 -1
- data/spec/lib/elasticity/job_flow_spec.rb +82 -0
- data/spec/lib/elasticity/looper_spec.rb +87 -0
- data/spec/lib/elasticity/setup_hadoop_debugging_step_spec.rb +10 -0
- metadata +52 -52
- data/.rvmrc +0 -1
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 3109d268557e25b4de8f9219f267bb9fcc2ee0c5
|
4
|
+
data.tar.gz: f42cd6f42d22afcc7fe14353afc0a15d6ca60a68
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 8ff4ccbb6af6cffe65c60884685204db81c5d4be67dfafffc422d8d0e21e9ba710f75c05c4e86f37599c6b492424137a34b9cbf7907798900ea71779b60de611
|
7
|
+
data.tar.gz: b71875d6ca3db0f7ca14c431121c52bffbdf094deaed4fcb81929ef3e78f949d3796c98c9718a810230b96e653b518e113c7c6a0838b48b140985e54d80ae356
|
data/.gitignore
CHANGED
data/.ruby-gemset
ADDED
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
1.9.2-p320
|
data/.travis.yml
CHANGED
data/Guardfile
ADDED
data/HISTORY.md
CHANGED
@@ -1,3 +1,17 @@
|
|
1
|
+
## 2.6 - August 17, 2013
|
2
|
+
|
3
|
+
+ Added debugging support via `JobFlow#enable_debugging`. Note that this requires `JobFlow#log_uri` to be set.
|
4
|
+
+ Added job flow completion polling via `JobFlow#wait_for_completion`.
|
5
|
+
+ Added testing to support Ruby 2.0.
|
6
|
+
+ Removed support for REE and 1.8.7 as these are now unsupported versions of Ruby. Common strategy in the Ruby community has been to only perform a minor version bump in this case; Elasticity is following suit.
|
7
|
+
+ Now specifing minor versions in the gemspec. With the release of Ruby 2.0, I'm anticipating breaking changes coming to many gems, and hoping that this mitigates those effects.
|
8
|
+
|
9
|
+
### Development
|
10
|
+
+ Dev update: guard-rspec added.
|
11
|
+
+ Dev update: All development dependencies now require Ruby >= 1.9.2.
|
12
|
+
+ Dev update: Latest version of Ruby 1.9.2 (p320).
|
13
|
+
+ Dev update: Migrated from .rvmrc => .ruby-version and .ruby-gemset.
|
14
|
+
|
1
15
|
## 2.5.6 - February 9, 2013
|
2
16
|
|
3
17
|
+ Pull request from [Aaron Olson](https://github.com/airolson), removing requirement that a ```JobFlow``` has steps before running.
|
data/README.md
CHANGED
@@ -1,12 +1,10 @@
|
|
1
1
|
[![Gem Version](https://badge.fury.io/rb/elasticity.png)](http://badge.fury.io/rb/elasticity)
|
2
2
|
|
3
|
-
**(
|
3
|
+
**(August 17, 2013)** Taking requests! I have a few ideas for what might be cool features though I'd rather work on what the community wants. Go ahead and file an issue!
|
4
4
|
|
5
5
|
Elasticity provides programmatic access to Amazon's Elastic Map Reduce service. The aim is to conveniently abstract away the complex EMR REST API and make working with job flows more productive and more enjoyable.
|
6
6
|
|
7
|
-
|
8
|
-
|
9
|
-
[![Build Status](https://secure.travis-ci.org/rslifka/elasticity.png)](http://travis-ci.org/rslifka/elasticity) REE, 1.8.7, 1.9.2, 1.9.3
|
7
|
+
[![Build Status](https://secure.travis-ci.org/rslifka/elasticity.png)](http://travis-ci.org/rslifka/elasticity) 1.9.2, 1.9.3, 2.0.0
|
10
8
|
|
11
9
|
Elasticity provides two ways to access EMR:
|
12
10
|
|
@@ -22,7 +20,7 @@ gem install elasticity
|
|
22
20
|
or in your Gemfile
|
23
21
|
|
24
22
|
```
|
25
|
-
gem 'elasticity', '~> 2.
|
23
|
+
gem 'elasticity', '~> 2.6'
|
26
24
|
```
|
27
25
|
|
28
26
|
This will ensure that you protect yourself from API changes, which will only be made in major revisions.
|
@@ -63,10 +61,11 @@ Job flows are the center of the EMR universe. The general order of operations i
|
|
63
61
|
1. Specify options.
|
64
62
|
1. (optional) Configure instance groups.
|
65
63
|
1. (optional) Add bootstrap actions.
|
66
|
-
1. Add steps.
|
64
|
+
1. (optional) Add steps.
|
67
65
|
1. (optional) Upload assets.
|
68
66
|
1. Run the job flow.
|
69
67
|
1. (optional) Add additional steps.
|
68
|
+
1. (optional) Wait for the job flow to complete.
|
70
69
|
1. (optional) Shutdown the job flow.
|
71
70
|
|
72
71
|
## 1 - Create a Job Flow
|
@@ -107,6 +106,7 @@ jobflow.keep_job_flow_alive_when_no_steps = false
|
|
107
106
|
jobflow.ami_version = 'latest'
|
108
107
|
jobflow.hadoop_version = '1.0.3'
|
109
108
|
jobflow.log_uri = nil
|
109
|
+
jobflow.enable_debugging = false # Requires a log_uri to enable
|
110
110
|
|
111
111
|
jobflow.ec2_key_name = nil
|
112
112
|
jobflow.ec2_subnet_id = nil
|
@@ -199,9 +199,9 @@ action = Elasticity::HadoopFileBootstrapAction.new('s3n://my-bucket/job-config.x
|
|
199
199
|
jobflow.add_bootstrap_action(action)
|
200
200
|
```
|
201
201
|
|
202
|
-
## 5 - Add Steps
|
202
|
+
## 5 - Add Steps (optional)
|
203
203
|
|
204
|
-
Each type of step has ```#name``` and ```#action_on_failure``` fields that can be
|
204
|
+
Each type of step has ```#name``` and ```#action_on_failure``` fields that can be specified. Apart from that, steps are configured differently - exhaustively described below.
|
205
205
|
|
206
206
|
### Adding a Pig Step
|
207
207
|
|
@@ -317,7 +317,25 @@ jobflow_id = jobflow.run
|
|
317
317
|
|
318
318
|
Steps can be added to a running jobflow just by calling ```#add_step``` on the job flow exactly how you add them prior to submitting the job.
|
319
319
|
|
320
|
-
## 9 -
|
320
|
+
## 9 - Wait For the Job Flow to Complete (optional)
|
321
|
+
|
322
|
+
Elasticity has the ability to block until the status of a job flow is not STARTING or RUNNING. There are two flavours. Without a status callback:
|
323
|
+
|
324
|
+
```ruby
|
325
|
+
# Blocks until status changes
|
326
|
+
jobflow.wait_for_completion
|
327
|
+
```
|
328
|
+
|
329
|
+
And with a status callback, providing the elapsed time and an instance of ```Elasticity::JobFlowStatus``` so you can inspect the progress of the job.
|
330
|
+
|
331
|
+
```ruby
|
332
|
+
# Blocks until status changes, calling back every 60 seconds
|
333
|
+
jobflow.wait_for_completion do |elapsed_time, job_flow_status|
|
334
|
+
puts "Waiting for #{seconds}, jobflow status: #{status.state}"
|
335
|
+
end
|
336
|
+
```
|
337
|
+
|
338
|
+
## 10 - Shut Down the Job Flow (optional)
|
321
339
|
|
322
340
|
By default, job flows are set to terminate when there are no more running steps. You can tell the job flow to stay alive when it has nothing left to do:
|
323
341
|
|
data/elasticity.gemspec
CHANGED
@@ -11,14 +11,15 @@ Gem::Specification.new do |s|
|
|
11
11
|
s.summary = %q{Streamlined, programmatic access to Amazon's Elastic Map Reduce service.}
|
12
12
|
s.description = %q{Streamlined, programmatic access to Amazon's Elastic Map Reduce service, driven by the Sharethrough team's requirements for belting out EMR jobs.}
|
13
13
|
|
14
|
-
s.add_dependency('rest-client')
|
15
|
-
s.add_dependency('nokogiri')
|
16
|
-
s.add_dependency('fog')
|
14
|
+
s.add_dependency('rest-client', '~> 1.0')
|
15
|
+
s.add_dependency('nokogiri', '~> 1.0')
|
16
|
+
s.add_dependency('fog', '~> 1.0')
|
17
17
|
|
18
|
+
s.add_development_dependency('fakefs', '~> 0.4.0')
|
19
|
+
s.add_development_dependency('guard-rspec', '~> 3.0')
|
18
20
|
s.add_development_dependency('rake', '~> 0.9')
|
19
|
-
s.add_development_dependency('rspec', '~> 2.
|
20
|
-
s.add_development_dependency('timecop', '~> 0.
|
21
|
-
s.add_development_dependency('fakefs', '~> 0.4')
|
21
|
+
s.add_development_dependency('rspec', '~> 2.14.0')
|
22
|
+
s.add_development_dependency('timecop', '~> 0.6')
|
22
23
|
|
23
24
|
s.files = `git ls-files`.split("\n")
|
24
25
|
s.test_files = `git ls-files -- {spec,features}/*`.split("\n")
|
data/lib/elasticity.rb
CHANGED
@@ -15,6 +15,7 @@ require 'elasticity/hadoop_bootstrap_action'
|
|
15
15
|
require 'elasticity/hadoop_file_bootstrap_action'
|
16
16
|
require 'elasticity/job_flow_step'
|
17
17
|
|
18
|
+
require 'elasticity/looper'
|
18
19
|
require 'elasticity/job_flow'
|
19
20
|
require 'elasticity/instance_group'
|
20
21
|
|
@@ -22,6 +23,7 @@ require 'elasticity/job_flow_status'
|
|
22
23
|
require 'elasticity/job_flow_status_step'
|
23
24
|
|
24
25
|
require 'elasticity/custom_jar_step'
|
26
|
+
require 'elasticity/setup_hadoop_debugging_step'
|
25
27
|
require 'elasticity/hive_step'
|
26
28
|
require 'elasticity/pig_step'
|
27
29
|
require 'elasticity/streaming_step'
|
data/lib/elasticity/job_flow.rb
CHANGED
@@ -3,6 +3,7 @@ module Elasticity
|
|
3
3
|
class JobFlowRunningError < StandardError; end
|
4
4
|
class JobFlowNotStartedError < StandardError; end
|
5
5
|
class JobFlowMissingStepsError < StandardError; end
|
6
|
+
class LogUriMissingError < StandardError; end
|
6
7
|
|
7
8
|
class JobFlow
|
8
9
|
|
@@ -19,6 +20,7 @@ module Elasticity
|
|
19
20
|
attr_accessor :ec2_subnet_id
|
20
21
|
attr_accessor :placement
|
21
22
|
attr_accessor :visible_to_all_users
|
23
|
+
attr_accessor :enable_debugging
|
22
24
|
|
23
25
|
attr_reader :access_key
|
24
26
|
attr_reader :secret_key
|
@@ -30,6 +32,7 @@ module Elasticity
|
|
30
32
|
@ami_version = 'latest'
|
31
33
|
@keep_job_flow_alive_when_no_steps = false
|
32
34
|
@placement = 'us-east-1a'
|
35
|
+
@enable_debugging = false
|
33
36
|
|
34
37
|
@access_key = access
|
35
38
|
@secret_key = secret
|
@@ -58,6 +61,13 @@ module Elasticity
|
|
58
61
|
end
|
59
62
|
end
|
60
63
|
|
64
|
+
def enable_debugging=(enabled)
|
65
|
+
if enabled
|
66
|
+
raise LogUriMissingError, 'To enable debugging, please set a #log_uri' unless @log_uri
|
67
|
+
end
|
68
|
+
@enable_debugging = enabled
|
69
|
+
end
|
70
|
+
|
61
71
|
def instance_count=(count)
|
62
72
|
raise ArgumentError, "Instance count cannot be set to less than 2 (requested #{count})" unless count > 1
|
63
73
|
@instance_groups[:core].count = count - 1
|
@@ -130,8 +140,18 @@ module Elasticity
|
|
130
140
|
emr.describe_jobflow(@jobflow_id)
|
131
141
|
end
|
132
142
|
|
143
|
+
def wait_for_completion(&on_wait)
|
144
|
+
l = Elasticity::Looper.new(method(:retry_check), on_wait)
|
145
|
+
l.go
|
146
|
+
end
|
147
|
+
|
133
148
|
private
|
134
149
|
|
150
|
+
def retry_check
|
151
|
+
jf_status = status
|
152
|
+
return status.state == 'RUNNING' || status.state == 'STARTING', jf_status
|
153
|
+
end
|
154
|
+
|
135
155
|
def emr
|
136
156
|
@region ||= @placement.match(/(\w+-\w+-\d+)/)[0]
|
137
157
|
@emr ||= Elasticity::EMR.new(@access_key, @secret_key, :region => @region)
|
@@ -143,7 +163,9 @@ module Elasticity
|
|
143
163
|
|
144
164
|
def jobflow_config
|
145
165
|
config = jobflow_preamble
|
146
|
-
|
166
|
+
steps = jobflow_steps
|
167
|
+
steps.insert(0, Elasticity::SetupHadoopDebuggingStep.new.to_aws_step(self)) if @enable_debugging
|
168
|
+
config[:steps] = steps
|
147
169
|
config[:log_uri] = @log_uri if @log_uri
|
148
170
|
config[:bootstrap_actions] = @bootstrap_actions.map{|a| a.to_aws_bootstrap_action} unless @bootstrap_actions.empty?
|
149
171
|
config
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module Elasticity
|
2
|
+
|
3
|
+
class Looper
|
4
|
+
|
5
|
+
def initialize(on_retry_check, on_wait = nil, poll_interval = 60)
|
6
|
+
@on_retry_check = on_retry_check
|
7
|
+
@on_wait = on_wait
|
8
|
+
@poll_interval = poll_interval
|
9
|
+
end
|
10
|
+
|
11
|
+
def go
|
12
|
+
start_time = Time.now
|
13
|
+
loop do
|
14
|
+
should_continue, *results = @on_retry_check.call
|
15
|
+
return unless should_continue
|
16
|
+
@on_wait.call(Time.now - start_time, *results) if @on_wait
|
17
|
+
sleep(@poll_interval)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
@@ -0,0 +1,14 @@
|
|
1
|
+
module Elasticity
|
2
|
+
|
3
|
+
class SetupHadoopDebuggingStep < CustomJarStep
|
4
|
+
|
5
|
+
def initialize
|
6
|
+
@name = 'Elasticity Setup Hadoop Debugging'
|
7
|
+
@jar = 's3://elasticmapreduce/libs/script-runner/script-runner.jar'
|
8
|
+
@arguments = ['s3://elasticmapreduce/libs/state-pusher/0.1/fetch']
|
9
|
+
@action_on_failure = 'TERMINATE_JOB_FLOW'
|
10
|
+
end
|
11
|
+
|
12
|
+
end
|
13
|
+
|
14
|
+
end
|
data/lib/elasticity/version.rb
CHANGED
@@ -19,6 +19,7 @@ describe Elasticity::JobFlow do
|
|
19
19
|
its(:keep_job_flow_alive_when_no_steps) { should == false }
|
20
20
|
its(:placement) { should == 'us-east-1a' }
|
21
21
|
its(:visible_to_all_users) { should == false }
|
22
|
+
its(:enable_debugging) { should == false }
|
22
23
|
|
23
24
|
describe '.initialize' do
|
24
25
|
it 'should set the access and secret keys to nil by default' do
|
@@ -32,6 +33,32 @@ describe Elasticity::JobFlow do
|
|
32
33
|
end
|
33
34
|
end
|
34
35
|
|
36
|
+
describe '#enable_debugging=' do
|
37
|
+
|
38
|
+
context 'when a log_uri is present' do
|
39
|
+
before do
|
40
|
+
subject.log_uri = '_'
|
41
|
+
end
|
42
|
+
it 'should set enable_debugging' do
|
43
|
+
subject.enable_debugging = true
|
44
|
+
subject.enable_debugging.should == true
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
context 'when a log_uri is not present' do
|
49
|
+
before do
|
50
|
+
subject.log_uri = nil
|
51
|
+
end
|
52
|
+
it 'should raise an error' do
|
53
|
+
expect {
|
54
|
+
subject.enable_debugging = true
|
55
|
+
}.to raise_error(Elasticity::LogUriMissingError, 'To enable debugging, please set a #log_uri')
|
56
|
+
subject.enable_debugging.should == false
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
61
|
+
|
35
62
|
describe '#instance_count=' do
|
36
63
|
|
37
64
|
context 'when set to more than 1' do
|
@@ -229,6 +256,17 @@ describe Elasticity::JobFlow do
|
|
229
256
|
end
|
230
257
|
end
|
231
258
|
|
259
|
+
context 'when debugging is enabled' do
|
260
|
+
before do
|
261
|
+
jobflow_with_steps.log_uri = '_'
|
262
|
+
jobflow_with_steps.enable_debugging = true
|
263
|
+
aws_steps.insert(0, Elasticity::SetupHadoopDebuggingStep.new.to_aws_step(jobflow_with_steps))
|
264
|
+
end
|
265
|
+
it 'should incorporate the step to setup Hadoop debugging' do
|
266
|
+
jobflow_with_steps.send(:jobflow_config).should be_a_hash_including({:steps => aws_steps})
|
267
|
+
end
|
268
|
+
end
|
269
|
+
|
232
270
|
end
|
233
271
|
|
234
272
|
describe 'log URI' do
|
@@ -502,6 +540,50 @@ describe Elasticity::JobFlow do
|
|
502
540
|
|
503
541
|
end
|
504
542
|
|
543
|
+
describe '#wait_for_completion' do
|
544
|
+
let(:client_block) { Proc.new {} }
|
545
|
+
let(:fake_looper) { double(:looper, :go => nil) }
|
546
|
+
|
547
|
+
it 'should kick off a looper' do
|
548
|
+
Elasticity::Looper.should_receive(:new).with(subject.method(:retry_check), client_block).and_return(fake_looper)
|
549
|
+
fake_looper.should_receive(:go)
|
550
|
+
subject.wait_for_completion(&client_block)
|
551
|
+
end
|
552
|
+
end
|
553
|
+
|
554
|
+
describe '#retry_check' do
|
555
|
+
|
556
|
+
context 'when the jobflow is RUNNING' do
|
557
|
+
let(:jobflow_status) { double(:state => 'RUNNING') }
|
558
|
+
before do
|
559
|
+
subject.stub(:status).and_return(jobflow_status)
|
560
|
+
end
|
561
|
+
it 'returns true and the result of #status' do
|
562
|
+
subject.send(:retry_check).should == [true, jobflow_status]
|
563
|
+
end
|
564
|
+
end
|
565
|
+
|
566
|
+
context 'when the jobflow is STARTING' do
|
567
|
+
let(:jobflow_status) { double(:state => 'STARTING') }
|
568
|
+
before do
|
569
|
+
subject.stub(:status).and_return(jobflow_status)
|
570
|
+
end
|
571
|
+
it 'returns true and the result of #status' do
|
572
|
+
subject.send(:retry_check).should == [true, jobflow_status]
|
573
|
+
end
|
574
|
+
end
|
575
|
+
|
576
|
+
context 'when the jobflow is != RUNNING or STARTING' do
|
577
|
+
let(:jobflow_status) { double(:state => '_') }
|
578
|
+
before do
|
579
|
+
subject.stub(:status).and_return(jobflow_status)
|
580
|
+
end
|
581
|
+
it 'returns false and the result of #status' do
|
582
|
+
subject.send(:retry_check).should == [false, jobflow_status]
|
583
|
+
end
|
584
|
+
end
|
585
|
+
end
|
586
|
+
|
505
587
|
describe '.from_jobflow_id' do
|
506
588
|
|
507
589
|
before do
|
@@ -0,0 +1,87 @@
|
|
1
|
+
describe Elasticity::Looper do
|
2
|
+
|
3
|
+
before do
|
4
|
+
Elasticity::Looper.any_instance.stub(:sleep)
|
5
|
+
end
|
6
|
+
|
7
|
+
context 'when you should wait' do
|
8
|
+
|
9
|
+
let(:client) do
|
10
|
+
double(:client).tap do |c|
|
11
|
+
# Retry on the first pass through and don't retry on the second
|
12
|
+
c.stub(:on_retry_check).and_return(true, false)
|
13
|
+
c.stub(:on_wait)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
context 'and then you should not wait' do
|
18
|
+
|
19
|
+
context 'when a wait callback is provided' do
|
20
|
+
|
21
|
+
it 'communicates that waiting occurs only once' do
|
22
|
+
client.should_receive(:on_wait).once
|
23
|
+
l = Elasticity::Looper.new(client.method(:on_retry_check), client.method(:on_wait))
|
24
|
+
l.go
|
25
|
+
end
|
26
|
+
|
27
|
+
it 'communicates waiting occurs with elapsed wait time and arguments that on_retry_check returns' do
|
28
|
+
# Freeze time at the start and then 60 seconds ahead when sleep is called
|
29
|
+
Timecop.freeze(Time.at(1300000000))
|
30
|
+
Elasticity::Looper.any_instance.stub(:sleep) do
|
31
|
+
Timecop.freeze(Time.at(1300000060))
|
32
|
+
end
|
33
|
+
|
34
|
+
client.stub(:on_retry_check).and_return([true, 'TEST1', 'TEST2'], [true, 'TEST3'], false)
|
35
|
+
client.should_receive(:on_wait).with(0, 'TEST1', 'TEST2')
|
36
|
+
client.should_receive(:on_wait).with(60, 'TEST3')
|
37
|
+
|
38
|
+
l = Elasticity::Looper.new(client.method(:on_retry_check), client.method(:on_wait))
|
39
|
+
l.go
|
40
|
+
|
41
|
+
Timecop.return
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
45
|
+
|
46
|
+
context 'when a wait callback is not provided' do
|
47
|
+
it 'still works' do
|
48
|
+
l = Elasticity::Looper.new(client.method(:on_retry_check), nil)
|
49
|
+
l.go
|
50
|
+
|
51
|
+
l = Elasticity::Looper.new(client.method(:on_retry_check))
|
52
|
+
l.go
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
|
58
|
+
context 'when no poll interval is specified' do
|
59
|
+
it 'should poll every 60 seconds' do
|
60
|
+
Elasticity::Looper.any_instance.should_receive(:sleep).with(60)
|
61
|
+
l = Elasticity::Looper.new(client.method(:on_retry_check), client.method(:on_wait))
|
62
|
+
l.go
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
context 'when a custom poll interview is specified' do
|
67
|
+
it 'should poll at that interval' do
|
68
|
+
Elasticity::Looper.any_instance.should_receive(:sleep).with(999)
|
69
|
+
l = Elasticity::Looper.new(client.method(:on_retry_check), client.method(:on_wait), 999)
|
70
|
+
l.go
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
end
|
75
|
+
|
76
|
+
context 'when you should not wait' do
|
77
|
+
|
78
|
+
it 'does not communicate that waiting is about to occur' do
|
79
|
+
client = double(:client, on_retry_check: false, on_wait: nil)
|
80
|
+
client.should_not receive(:on_wait)
|
81
|
+
l = Elasticity::Looper.new(client.method(:on_retry_check), client.method(:on_wait))
|
82
|
+
l.go
|
83
|
+
end
|
84
|
+
|
85
|
+
end
|
86
|
+
|
87
|
+
end
|
@@ -0,0 +1,10 @@
|
|
1
|
+
describe Elasticity::SetupHadoopDebuggingStep do
|
2
|
+
|
3
|
+
it { should be_a Elasticity::CustomJarStep }
|
4
|
+
|
5
|
+
its(:name) { should == 'Elasticity Setup Hadoop Debugging' }
|
6
|
+
its(:jar) { should == 's3://elasticmapreduce/libs/script-runner/script-runner.jar' }
|
7
|
+
its(:arguments) { should == ['s3://elasticmapreduce/libs/state-pusher/0.1/fetch'] }
|
8
|
+
its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
|
9
|
+
|
10
|
+
end
|
metadata
CHANGED
@@ -1,128 +1,127 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elasticity
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 2.
|
5
|
-
prerelease:
|
4
|
+
version: '2.6'
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Robert Slifka
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date: 2013-
|
11
|
+
date: 2013-08-17 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: rest-client
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
|
-
- -
|
17
|
+
- - ~>
|
20
18
|
- !ruby/object:Gem::Version
|
21
|
-
version: '0'
|
19
|
+
version: '1.0'
|
22
20
|
type: :runtime
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
|
-
- -
|
24
|
+
- - ~>
|
28
25
|
- !ruby/object:Gem::Version
|
29
|
-
version: '0'
|
26
|
+
version: '1.0'
|
30
27
|
- !ruby/object:Gem::Dependency
|
31
28
|
name: nokogiri
|
32
29
|
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
30
|
requirements:
|
35
|
-
- -
|
31
|
+
- - ~>
|
36
32
|
- !ruby/object:Gem::Version
|
37
|
-
version: '0'
|
33
|
+
version: '1.0'
|
38
34
|
type: :runtime
|
39
35
|
prerelease: false
|
40
36
|
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
37
|
requirements:
|
43
|
-
- -
|
38
|
+
- - ~>
|
44
39
|
- !ruby/object:Gem::Version
|
45
|
-
version: '0'
|
40
|
+
version: '1.0'
|
46
41
|
- !ruby/object:Gem::Dependency
|
47
42
|
name: fog
|
48
43
|
requirement: !ruby/object:Gem::Requirement
|
49
|
-
none: false
|
50
44
|
requirements:
|
51
|
-
- -
|
45
|
+
- - ~>
|
52
46
|
- !ruby/object:Gem::Version
|
53
|
-
version: '0'
|
47
|
+
version: '1.0'
|
54
48
|
type: :runtime
|
55
49
|
prerelease: false
|
56
50
|
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
none: false
|
58
51
|
requirements:
|
59
|
-
- -
|
52
|
+
- - ~>
|
60
53
|
- !ruby/object:Gem::Version
|
61
|
-
version: '0'
|
54
|
+
version: '1.0'
|
62
55
|
- !ruby/object:Gem::Dependency
|
63
|
-
name:
|
56
|
+
name: fakefs
|
64
57
|
requirement: !ruby/object:Gem::Requirement
|
65
|
-
none: false
|
66
58
|
requirements:
|
67
59
|
- - ~>
|
68
60
|
- !ruby/object:Gem::Version
|
69
|
-
version:
|
61
|
+
version: 0.4.0
|
70
62
|
type: :development
|
71
63
|
prerelease: false
|
72
64
|
version_requirements: !ruby/object:Gem::Requirement
|
73
|
-
none: false
|
74
65
|
requirements:
|
75
66
|
- - ~>
|
76
67
|
- !ruby/object:Gem::Version
|
77
|
-
version:
|
68
|
+
version: 0.4.0
|
78
69
|
- !ruby/object:Gem::Dependency
|
79
|
-
name: rspec
|
70
|
+
name: guard-rspec
|
80
71
|
requirement: !ruby/object:Gem::Requirement
|
81
|
-
none: false
|
82
72
|
requirements:
|
83
73
|
- - ~>
|
84
74
|
- !ruby/object:Gem::Version
|
85
|
-
version:
|
75
|
+
version: '3.0'
|
86
76
|
type: :development
|
87
77
|
prerelease: false
|
88
78
|
version_requirements: !ruby/object:Gem::Requirement
|
89
|
-
none: false
|
90
79
|
requirements:
|
91
80
|
- - ~>
|
92
81
|
- !ruby/object:Gem::Version
|
93
|
-
version:
|
82
|
+
version: '3.0'
|
94
83
|
- !ruby/object:Gem::Dependency
|
95
|
-
name:
|
84
|
+
name: rake
|
96
85
|
requirement: !ruby/object:Gem::Requirement
|
97
|
-
none: false
|
98
86
|
requirements:
|
99
87
|
- - ~>
|
100
88
|
- !ruby/object:Gem::Version
|
101
|
-
version: '0.
|
89
|
+
version: '0.9'
|
102
90
|
type: :development
|
103
91
|
prerelease: false
|
104
92
|
version_requirements: !ruby/object:Gem::Requirement
|
105
|
-
none: false
|
106
93
|
requirements:
|
107
94
|
- - ~>
|
108
95
|
- !ruby/object:Gem::Version
|
109
|
-
version: '0.
|
96
|
+
version: '0.9'
|
110
97
|
- !ruby/object:Gem::Dependency
|
111
|
-
name:
|
98
|
+
name: rspec
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ~>
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: 2.14.0
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ~>
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: 2.14.0
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: timecop
|
112
113
|
requirement: !ruby/object:Gem::Requirement
|
113
|
-
none: false
|
114
114
|
requirements:
|
115
115
|
- - ~>
|
116
116
|
- !ruby/object:Gem::Version
|
117
|
-
version: '0.
|
117
|
+
version: '0.6'
|
118
118
|
type: :development
|
119
119
|
prerelease: false
|
120
120
|
version_requirements: !ruby/object:Gem::Requirement
|
121
|
-
none: false
|
122
121
|
requirements:
|
123
122
|
- - ~>
|
124
123
|
- !ruby/object:Gem::Version
|
125
|
-
version: '0.
|
124
|
+
version: '0.6'
|
126
125
|
description: Streamlined, programmatic access to Amazon's Elastic Map Reduce service,
|
127
126
|
driven by the Sharethrough team's requirements for belting out EMR jobs.
|
128
127
|
email:
|
@@ -132,9 +131,11 @@ extra_rdoc_files: []
|
|
132
131
|
files:
|
133
132
|
- .gitignore
|
134
133
|
- .rspec
|
135
|
-
- .
|
134
|
+
- .ruby-gemset
|
135
|
+
- .ruby-version
|
136
136
|
- .travis.yml
|
137
137
|
- Gemfile
|
138
|
+
- Guardfile
|
138
139
|
- HISTORY.md
|
139
140
|
- LICENSE
|
140
141
|
- README.md
|
@@ -153,7 +154,9 @@ files:
|
|
153
154
|
- lib/elasticity/job_flow_status.rb
|
154
155
|
- lib/elasticity/job_flow_status_step.rb
|
155
156
|
- lib/elasticity/job_flow_step.rb
|
157
|
+
- lib/elasticity/looper.rb
|
156
158
|
- lib/elasticity/pig_step.rb
|
159
|
+
- lib/elasticity/setup_hadoop_debugging_step.rb
|
157
160
|
- lib/elasticity/streaming_step.rb
|
158
161
|
- lib/elasticity/sync_to_s3.rb
|
159
162
|
- lib/elasticity/version.rb
|
@@ -170,40 +173,35 @@ files:
|
|
170
173
|
- spec/lib/elasticity/job_flow_status_spec.rb
|
171
174
|
- spec/lib/elasticity/job_flow_status_step_spec.rb
|
172
175
|
- spec/lib/elasticity/job_flow_step_spec.rb
|
176
|
+
- spec/lib/elasticity/looper_spec.rb
|
173
177
|
- spec/lib/elasticity/pig_step_spec.rb
|
178
|
+
- spec/lib/elasticity/setup_hadoop_debugging_step_spec.rb
|
174
179
|
- spec/lib/elasticity/streaming_step_spec.rb
|
175
180
|
- spec/lib/elasticity/sync_to_s3_spec.rb
|
176
181
|
- spec/spec_helper.rb
|
177
182
|
- spec/support/be_a_hash_including_matcher.rb
|
178
183
|
homepage: http://www.github.com/rslifka/elasticity
|
179
184
|
licenses: []
|
185
|
+
metadata: {}
|
180
186
|
post_install_message:
|
181
187
|
rdoc_options: []
|
182
188
|
require_paths:
|
183
189
|
- lib
|
184
190
|
required_ruby_version: !ruby/object:Gem::Requirement
|
185
|
-
none: false
|
186
191
|
requirements:
|
187
192
|
- - ! '>='
|
188
193
|
- !ruby/object:Gem::Version
|
189
194
|
version: '0'
|
190
|
-
segments:
|
191
|
-
- 0
|
192
|
-
hash: -3179594525299845016
|
193
195
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
194
|
-
none: false
|
195
196
|
requirements:
|
196
197
|
- - ! '>='
|
197
198
|
- !ruby/object:Gem::Version
|
198
199
|
version: '0'
|
199
|
-
segments:
|
200
|
-
- 0
|
201
|
-
hash: -3179594525299845016
|
202
200
|
requirements: []
|
203
201
|
rubyforge_project:
|
204
|
-
rubygems_version:
|
202
|
+
rubygems_version: 2.0.6
|
205
203
|
signing_key:
|
206
|
-
specification_version:
|
204
|
+
specification_version: 4
|
207
205
|
summary: Streamlined, programmatic access to Amazon's Elastic Map Reduce service.
|
208
206
|
test_files:
|
209
207
|
- spec/lib/elasticity/aws_request_spec.rb
|
@@ -219,7 +217,9 @@ test_files:
|
|
219
217
|
- spec/lib/elasticity/job_flow_status_spec.rb
|
220
218
|
- spec/lib/elasticity/job_flow_status_step_spec.rb
|
221
219
|
- spec/lib/elasticity/job_flow_step_spec.rb
|
220
|
+
- spec/lib/elasticity/looper_spec.rb
|
222
221
|
- spec/lib/elasticity/pig_step_spec.rb
|
222
|
+
- spec/lib/elasticity/setup_hadoop_debugging_step_spec.rb
|
223
223
|
- spec/lib/elasticity/streaming_step_spec.rb
|
224
224
|
- spec/lib/elasticity/sync_to_s3_spec.rb
|
225
225
|
- spec/spec_helper.rb
|
data/.rvmrc
DELETED
@@ -1 +0,0 @@
|
|
1
|
-
rvm use ruby-1.9.3-p385@elasticity --create
|