elasticity 2.3 → 2.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/HISTORY.md +7 -0
- data/README.md +1 -3
- data/lib/elasticity/custom_jar_step.rb +1 -1
- data/lib/elasticity/hive_step.rb +4 -2
- data/lib/elasticity/job_flow.rb +2 -3
- data/lib/elasticity/job_flow_status.rb +4 -0
- data/lib/elasticity/pig_step.rb +1 -1
- data/lib/elasticity/streaming_step.rb +1 -1
- data/lib/elasticity/version.rb +1 -1
- data/spec/lib/elasticity/hive_step_spec.rb +4 -2
- data/spec/lib/elasticity/job_flow_integration_spec.rb +6 -7
- data/spec/lib/elasticity/job_flow_spec.rb +11 -4
- data/spec/lib/elasticity/job_flow_status_spec.rb +5 -0
- metadata +8 -2
data/HISTORY.md
CHANGED
@@ -1,3 +1,10 @@
|
|
1
|
+
## 2.3.1 - August 23, 2012
|
2
|
+
|
3
|
+
+ Birthday release! ;)
|
4
|
+
+ Bumped the default version of Hadoop to 1.0.3.
|
5
|
+
+ Amazon now requires the --hive-versions argument when installing Hive (thanks to Johannes Wuerbach).
|
6
|
+
+ ```JobFlowStatus#master_public_dns_name``` is now available (thanks to Johannes Wuerbach).
|
7
|
+
|
1
8
|
## 2.3 - July 28, 2012
|
2
9
|
|
3
10
|
+ ```JobFlow``` now supports specifying availbility zone instance specification via ```JobFlow#placement=```.
|
data/README.md
CHANGED
@@ -35,7 +35,7 @@ jobflow = Elasticity::JobFlow.new('AWS access key', 'AWS secret key')
|
|
35
35
|
step = Elasticity::CustomJarStep.new('s3n://elasticmapreduce/samples/cloudburst/cloudburst.jar')
|
36
36
|
|
37
37
|
# Here are the arguments to pass to the jar
|
38
|
-
|
38
|
+
step.arguments = %w(s3n://elasticmapreduce/samples/cloudburst/input/s_suis.br s3n://elasticmapreduce/samples/cloudburst/input/100k.br s3n://slif-output/cloudburst/output/2012-06-22 36 3 0 1 240 48 24 24 128 16)
|
39
39
|
|
40
40
|
# Add the step to the jobflow
|
41
41
|
jobflow.add_step(step)
|
@@ -126,7 +126,6 @@ ig.count = 10 # Provision 10 instances
|
|
126
126
|
ig.type = 'c1.medium' # See the EMR docs for a list of supported types
|
127
127
|
ig.set_on_demand_instances # This is the default setting
|
128
128
|
|
129
|
-
|
130
129
|
jobflow.set_core_instance_group(ig)
|
131
130
|
```
|
132
131
|
|
@@ -140,7 +139,6 @@ ig.count = 10 # Provision 10 instances
|
|
140
139
|
ig.type = 'c1.medium' # See the EMR docs for a list of supported types
|
141
140
|
ig.set_spot_instances(0.25) # Makes this a SPOT group with a $0.25 bid price
|
142
141
|
|
143
|
-
|
144
142
|
jobflow.set_core_instance_group(ig)
|
145
143
|
```
|
146
144
|
|
data/lib/elasticity/hive_step.rb
CHANGED
@@ -2,7 +2,7 @@ module Elasticity
|
|
2
2
|
|
3
3
|
class HiveStep
|
4
4
|
|
5
|
-
include JobFlowStep
|
5
|
+
include Elasticity::JobFlowStep
|
6
6
|
|
7
7
|
attr_accessor :name
|
8
8
|
attr_accessor :script
|
@@ -45,7 +45,9 @@ module Elasticity
|
|
45
45
|
's3://elasticmapreduce/libs/hive/hive-script',
|
46
46
|
'--base-path',
|
47
47
|
's3://elasticmapreduce/libs/hive/',
|
48
|
-
'--install-hive'
|
48
|
+
'--install-hive',
|
49
|
+
'--hive-versions',
|
50
|
+
'latest'
|
49
51
|
],
|
50
52
|
},
|
51
53
|
:name => 'Elasticity - Install Hive'
|
data/lib/elasticity/job_flow.rb
CHANGED
@@ -21,8 +21,7 @@ module Elasticity
|
|
21
21
|
|
22
22
|
def initialize(access, secret)
|
23
23
|
@action_on_failure = 'TERMINATE_JOB_FLOW'
|
24
|
-
@
|
25
|
-
@hadoop_version = '0.20.205'
|
24
|
+
@hadoop_version = '1.0.3'
|
26
25
|
@name = 'Elasticity Job Flow'
|
27
26
|
@ami_version = 'latest'
|
28
27
|
@keep_job_flow_alive_when_no_steps = false
|
@@ -144,12 +143,12 @@ module Elasticity
|
|
144
143
|
:ami_version => @ami_version,
|
145
144
|
:instances => {
|
146
145
|
:keep_job_flow_alive_when_no_steps => @keep_job_flow_alive_when_no_steps,
|
147
|
-
:ec2_key_name => @ec2_key_name,
|
148
146
|
:hadoop_version => @hadoop_version,
|
149
147
|
:instance_groups => jobflow_instance_groups
|
150
148
|
}
|
151
149
|
}
|
152
150
|
preamble.merge!(:ec2_subnet_id => @ec2_subnet_id) if @ec2_subnet_id
|
151
|
+
preamble[:instances].merge!(:ec2_key_name => @ec2_key_name) if @ec2_key_name
|
153
152
|
preamble
|
154
153
|
end
|
155
154
|
|
@@ -14,6 +14,7 @@ module Elasticity
|
|
14
14
|
attr_accessor :slave_instance_type
|
15
15
|
attr_accessor :last_state_change_reason
|
16
16
|
attr_accessor :installed_steps
|
17
|
+
attr_accessor :master_public_dns_name
|
17
18
|
|
18
19
|
def initialize
|
19
20
|
@steps = []
|
@@ -49,6 +50,9 @@ module Elasticity
|
|
49
50
|
jobflow.master_instance_type = xml_element.xpath('./Instances/MasterInstanceType').text.strip
|
50
51
|
jobflow.slave_instance_type = xml_element.xpath('./Instances/SlaveInstanceType').text.strip
|
51
52
|
|
53
|
+
master_public_dns_name = xml_element.xpath('./Instances/MasterPublicDnsName').text.strip
|
54
|
+
jobflow.master_public_dns_name = (master_public_dns_name == '') ? (nil) : (master_public_dns_name)
|
55
|
+
|
52
56
|
jobflow
|
53
57
|
end
|
54
58
|
|
data/lib/elasticity/pig_step.rb
CHANGED
data/lib/elasticity/version.rb
CHANGED
@@ -8,7 +8,7 @@ describe Elasticity::HiveStep do
|
|
8
8
|
|
9
9
|
its(:name) { should == 'Elasticity Hive Step (script.hql)' }
|
10
10
|
its(:script) { should == 'script.hql' }
|
11
|
-
its(:variables) { should == {
|
11
|
+
its(:variables) { should == {} }
|
12
12
|
its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
|
13
13
|
|
14
14
|
describe '#to_aws_step' do
|
@@ -62,7 +62,9 @@ describe Elasticity::HiveStep do
|
|
62
62
|
's3://elasticmapreduce/libs/hive/hive-script',
|
63
63
|
'--base-path',
|
64
64
|
's3://elasticmapreduce/libs/hive/',
|
65
|
-
'--install-hive'
|
65
|
+
'--install-hive',
|
66
|
+
'--hive-versions',
|
67
|
+
'latest'
|
66
68
|
],
|
67
69
|
},
|
68
70
|
:name => 'Elasticity - Install Hive'
|
@@ -30,8 +30,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
|
|
30
30
|
:ami_version => 'latest',
|
31
31
|
:instances => {
|
32
32
|
:keep_job_flow_alive_when_no_steps => false,
|
33
|
-
:
|
34
|
-
:hadoop_version => '0.20.205',
|
33
|
+
:hadoop_version => '1.0.3',
|
35
34
|
:instance_groups => [
|
36
35
|
{
|
37
36
|
:instance_count => 1,
|
@@ -56,7 +55,9 @@ describe 'Elasticity::JobFlow Integration Examples' do
|
|
56
55
|
's3://elasticmapreduce/libs/hive/hive-script',
|
57
56
|
'--base-path',
|
58
57
|
's3://elasticmapreduce/libs/hive/',
|
59
|
-
'--install-hive'
|
58
|
+
'--install-hive',
|
59
|
+
'--hive-versions',
|
60
|
+
'latest'
|
60
61
|
],
|
61
62
|
},
|
62
63
|
:name => 'Elasticity - Install Hive'
|
@@ -109,8 +110,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
|
|
109
110
|
:ami_version => 'latest',
|
110
111
|
:instances => {
|
111
112
|
:keep_job_flow_alive_when_no_steps => false,
|
112
|
-
:
|
113
|
-
:hadoop_version => '0.20.205',
|
113
|
+
:hadoop_version => '1.0.3',
|
114
114
|
:instance_groups => [
|
115
115
|
{
|
116
116
|
:instance_count => 1,
|
@@ -192,8 +192,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
|
|
192
192
|
:ami_version => 'latest',
|
193
193
|
:instances => {
|
194
194
|
:keep_job_flow_alive_when_no_steps => false,
|
195
|
-
:
|
196
|
-
:hadoop_version => '0.20.205',
|
195
|
+
:hadoop_version => '1.0.3',
|
197
196
|
:instance_groups => [
|
198
197
|
{
|
199
198
|
:instance_count => 1,
|
@@ -5,8 +5,9 @@ describe Elasticity::JobFlow do
|
|
5
5
|
end
|
6
6
|
|
7
7
|
its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
|
8
|
-
its(:ec2_key_name) { should ==
|
9
|
-
its(:
|
8
|
+
its(:ec2_key_name) { should == nil }
|
9
|
+
its(:ec2_subnet_id) { should == nil }
|
10
|
+
its(:hadoop_version) { should == '1.0.3' }
|
10
11
|
its(:instance_count) { should == 2 }
|
11
12
|
its(:log_uri) { should == nil }
|
12
13
|
its(:master_instance_type) { should == 'm1.small' }
|
@@ -340,8 +341,7 @@ describe Elasticity::JobFlow do
|
|
340
341
|
:ami_version => 'latest',
|
341
342
|
:instances => {
|
342
343
|
:keep_job_flow_alive_when_no_steps => false,
|
343
|
-
:
|
344
|
-
:hadoop_version => '0.20.205',
|
344
|
+
:hadoop_version => '1.0.3',
|
345
345
|
:instance_groups => ['INSTANCE_GROUP_CONFIGURATION']
|
346
346
|
}
|
347
347
|
}
|
@@ -355,6 +355,13 @@ describe Elasticity::JobFlow do
|
|
355
355
|
subject.send(:jobflow_preamble).should == basic_preamble
|
356
356
|
end
|
357
357
|
|
358
|
+
context 'when a key name is provided' do
|
359
|
+
it 'should include it in the preamble' do
|
360
|
+
subject.ec2_key_name = 'default'
|
361
|
+
subject.send(:jobflow_preamble)[:instances].should be_a_hash_including({:ec2_key_name => 'default'})
|
362
|
+
end
|
363
|
+
end
|
364
|
+
|
358
365
|
context 'when a VPC subnet ID is specified' do
|
359
366
|
it 'should include it in the preamble' do
|
360
367
|
subject.ec2_subnet_id = 'subnet-118b9d79'
|
@@ -65,6 +65,9 @@ describe Elasticity::JobFlowStatus do
|
|
65
65
|
</member>
|
66
66
|
</Steps>
|
67
67
|
<Instances>
|
68
|
+
<MasterPublicDnsName>
|
69
|
+
ec2-107-22-77-99.compute-1.amazonaws.com
|
70
|
+
</MasterPublicDnsName>
|
68
71
|
<Placement>
|
69
72
|
<AvailabilityZone>
|
70
73
|
eu-west-1a
|
@@ -154,6 +157,7 @@ describe Elasticity::JobFlowStatus do
|
|
154
157
|
single_jobflow.slave_instance_type.should == 'm1.small'
|
155
158
|
single_jobflow.instance_count.should == '4'
|
156
159
|
single_jobflow.last_state_change_reason.should == 'Steps completed with errors'
|
160
|
+
single_jobflow.master_public_dns_name.should == 'ec2-107-22-77-99.compute-1.amazonaws.com'
|
157
161
|
end
|
158
162
|
end
|
159
163
|
|
@@ -169,6 +173,7 @@ describe Elasticity::JobFlowStatus do
|
|
169
173
|
multiple_jobflows.map(&:slave_instance_type).should == %w(m1.small c1.medium)
|
170
174
|
multiple_jobflows.map(&:instance_count).should == %w(4 2)
|
171
175
|
multiple_jobflows.map(&:last_state_change_reason).should == ['Steps completed with errors', 'Steps completed']
|
176
|
+
multiple_jobflows.map(&:master_public_dns_name).should == ['ec2-107-22-77-99.compute-1.amazonaws.com', nil]
|
172
177
|
end
|
173
178
|
end
|
174
179
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elasticity
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.3.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-08-23 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rest-client
|
@@ -135,12 +135,18 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
135
135
|
- - ! '>='
|
136
136
|
- !ruby/object:Gem::Version
|
137
137
|
version: '0'
|
138
|
+
segments:
|
139
|
+
- 0
|
140
|
+
hash: 812718050856923137
|
138
141
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
139
142
|
none: false
|
140
143
|
requirements:
|
141
144
|
- - ! '>='
|
142
145
|
- !ruby/object:Gem::Version
|
143
146
|
version: '0'
|
147
|
+
segments:
|
148
|
+
- 0
|
149
|
+
hash: 812718050856923137
|
144
150
|
requirements: []
|
145
151
|
rubyforge_project:
|
146
152
|
rubygems_version: 1.8.24
|