elasticity 3.0.3 → 3.0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/HISTORY.md +4 -0
- data/README.md +5 -3
- data/lib/elasticity/job_flow.rb +5 -0
- data/lib/elasticity/s3distcp_step.rb +2 -6
- data/lib/elasticity/version.rb +1 -1
- data/spec/lib/elasticity/job_flow_spec.rb +14 -0
- data/spec/lib/elasticity/s3distcp_step_spec.rb +2 -3
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
MmY0NTJjZmJkMmVhNTYwNTJlZmY3MjAwOTE0NDhmYjczY2MxNzAwOA==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
MmIwNDRmMjM3MTA3NzliZDM4ZWY4MGZmODRlNTYwODlhYTMwYmYwZQ==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
YzI5NzgyMjc5YTlmZjljMjY5MzQ2MzU4ZmZiOWFiYmViMDgwMmYyNDJiZjE2
|
10
|
+
NzY4MTllZGFlN2Q1ZjcyNTVjZGIxYWQ2NTNmMmE4NDE2MTNmOTc1ODcyYjZl
|
11
|
+
MTljMDM3OWZhNWM2OThkYjBkMjI1MGE4YzgyMWZkZjQ3M2ZlMDU=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
NmU2N2MxM2EyODdhZGMyYzQ2MTA1YmQ4ZGFiMTI2MTM3ZGQ0Mzg0MGM3ODg3
|
14
|
+
MjQ2NTljMTg5NzYxODZhM2UyNzQ2NTZjMTU1MmIxOTRmNWJiNmVmYmNlMTBi
|
15
|
+
MDg3ZjlmZTE1ZTUyODBmMDQ0MjI0NzBhN2ZmOGFhMGRmZTI2MmU=
|
data/HISTORY.md
CHANGED
@@ -1,3 +1,7 @@
|
|
1
|
+
## 3.0.4 - April 30, 2014
|
2
|
+
|
3
|
+
- Fix for issues [#65](https://github.com/rslifka/elasticity/issues/65) and [#66](https://github.com/rslifka/elasticity/issues/66).
|
4
|
+
|
1
5
|
## 3.0.3 - April 10, 2014
|
2
6
|
|
3
7
|
- Fix for issues [#63](https://github.com/rslifka/elasticity/issues/63) and [#64](https://github.com/rslifka/elasticity/issues/64).
|
data/README.md
CHANGED
@@ -106,6 +106,9 @@ These options are sent up as part of job flow submission (i.e. ```JobFlow#run```
|
|
106
106
|
```ruby
|
107
107
|
jobflow.name = 'Elasticity Job Flow'
|
108
108
|
|
109
|
+
# For new AWS accounts, this is required to be set
|
110
|
+
jobflow.ec2_subnet_id = nil
|
111
|
+
|
109
112
|
jobflow.action_on_failure = 'TERMINATE_JOB_FLOW'
|
110
113
|
jobflow.keep_job_flow_alive_when_no_steps = false
|
111
114
|
jobflow.ami_version = 'latest'
|
@@ -114,7 +117,6 @@ jobflow.log_uri = nil
|
|
114
117
|
jobflow.enable_debugging = false # Requires a log_uri to enable
|
115
118
|
|
116
119
|
jobflow.ec2_key_name = nil
|
117
|
-
jobflow.ec2_subnet_id = nil
|
118
120
|
jobflow.visible_to_all_users = false
|
119
121
|
jobflow.placement = 'us-east-1a'
|
120
122
|
jobflow.instance_count = 2
|
@@ -300,8 +302,8 @@ jobflow.add_step(script_step)
|
|
300
302
|
For a complete list of supported arguments, please see the [Amazon EMR guide](http://docs.aws.amazon.com/ElasticMapReduce/latest/DeveloperGuide/UsingEMR_s3distcp.html).
|
301
303
|
|
302
304
|
```ruby
|
303
|
-
|
304
|
-
copy_step =
|
305
|
+
copy_step = Elasticity::S3DistCpStep.new
|
306
|
+
copy_step.arguments = [...]
|
305
307
|
|
306
308
|
jobflow.add_step(copy_step)
|
307
309
|
```
|
data/lib/elasticity/job_flow.rb
CHANGED
@@ -84,6 +84,11 @@ module Elasticity
|
|
84
84
|
@slave_instance_type = type
|
85
85
|
end
|
86
86
|
|
87
|
+
def ec2_subnet_id=(subnet_id)
|
88
|
+
@ec2_subnet_id = subnet_id
|
89
|
+
@placement = nil
|
90
|
+
end
|
91
|
+
|
87
92
|
def add_bootstrap_action(bootstrap_action)
|
88
93
|
if is_jobflow_running?
|
89
94
|
raise JobFlowRunningError, 'To modify bootstrap actions, please create a new job flow.'
|
@@ -2,13 +2,9 @@ module Elasticity
|
|
2
2
|
|
3
3
|
class S3DistCpStep < CustomJarStep
|
4
4
|
|
5
|
-
def initialize
|
5
|
+
def initialize
|
6
|
+
super('/home/hadoop/lib/emr-s3distcp-1.0.jar')
|
6
7
|
@name = 'Elasticity S3DistCp Step'
|
7
|
-
@jar = '/home/hadoop/lib/emr-s3distcp-1.0.jar'
|
8
|
-
@arguments = []
|
9
|
-
options.each do |argument, value|
|
10
|
-
@arguments << '--arg' << argument.to_s << '--arg' << value
|
11
|
-
end
|
12
8
|
end
|
13
9
|
|
14
10
|
end
|
data/lib/elasticity/version.rb
CHANGED
@@ -129,6 +129,20 @@ describe Elasticity::JobFlow do
|
|
129
129
|
|
130
130
|
end
|
131
131
|
|
132
|
+
describe '#ec2_subnet_id=' do
|
133
|
+
it 'should set ec2_subnet_id' do
|
134
|
+
subject.ec2_subnet_id = 'TEST_ID'
|
135
|
+
subject.ec2_subnet_id.should == 'TEST_ID'
|
136
|
+
end
|
137
|
+
|
138
|
+
it 'should unset placement (which has a default value) because having both set is EMR-invalid' do
|
139
|
+
subject.placement = 'TEST_PLACEMENT'
|
140
|
+
|
141
|
+
subject.ec2_subnet_id = '_'
|
142
|
+
subject.placement.should == nil
|
143
|
+
end
|
144
|
+
end
|
145
|
+
|
132
146
|
describe '#add_bootstrap_action' do
|
133
147
|
|
134
148
|
context 'when the jobflow is not yet started' do
|
@@ -2,10 +2,9 @@ describe Elasticity::S3DistCpStep do
|
|
2
2
|
|
3
3
|
it { should be_a Elasticity::CustomJarStep }
|
4
4
|
|
5
|
-
subject { Elasticity::S3DistCpStep.new({'arg1' => 'value1', :arg2 => 'value2'}) }
|
6
|
-
|
7
5
|
its(:name) { should == 'Elasticity S3DistCp Step' }
|
8
6
|
its(:jar) { should == '/home/hadoop/lib/emr-s3distcp-1.0.jar' }
|
9
|
-
its(:arguments) { should ==
|
7
|
+
its(:arguments) { should == [] }
|
8
|
+
its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
|
10
9
|
|
11
10
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elasticity
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 3.0.
|
4
|
+
version: 3.0.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Robert Slifka
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-04-
|
11
|
+
date: 2014-04-30 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rest-client
|