elasticity 2.5 → 2.5.1
Sign up to get free protection for your applications and to get access to all the features.
- data/.rvmrc +1 -1
- data/HISTORY.md +5 -0
- data/README.md +4 -2
- data/elasticity.gemspec +2 -2
- data/lib/elasticity/hive_step.rb +1 -8
- data/lib/elasticity/job_flow.rb +4 -1
- data/lib/elasticity/version.rb +1 -1
- data/spec/lib/elasticity/job_flow_integration_spec.rb +14 -4
- data/spec/lib/elasticity/job_flow_spec.rb +4 -1
- metadata +8 -14
data/.rvmrc
CHANGED
@@ -1 +1 @@
|
|
1
|
-
rvm use ruby-1.9.3-
|
1
|
+
rvm use ruby-1.9.3-p327@elasticity --create
|
data/HISTORY.md
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
## 2.5.1 - November 28, 2012
|
2
|
+
|
3
|
+
+ When ```JobFlow#placement``` is specified, instances are created in that availability zone. Previously, this setting was only used to derive the EMR API endpoint to connect to (i.e. the region).
|
4
|
+
+ Updated development depdencies.
|
5
|
+
|
1
6
|
## 2.5 - September 29, 2012
|
2
7
|
|
3
8
|
+ ```SyncToS3``` supports S3 region specification.
|
data/README.md
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
(2012-11-23) Hey all, I'm still out here. Elasticity remains part of our daily workflow at [Sharethrough](http://www.sharethrough.com/) and after a year of work, our usage patterns have stabilized. Submit an issue if you have suggestions!
|
2
|
+
|
1
3
|
Elasticity provides programmatic access to Amazon's Elastic Map Reduce service. The aim is to conveniently abstract away the complex EMR REST API and make working with job flows more productive and more enjoyable.
|
2
4
|
|
3
5
|
[![Build Status](https://secure.travis-ci.org/rslifka/elasticity.png)](http://travis-ci.org/rslifka/elasticity) REE, 1.8.7, 1.9.2, 1.9.3
|
@@ -339,13 +341,13 @@ Unfortunately, the documentation is sometimes incorrect and sometimes missing.
|
|
339
341
|
|
340
342
|
* AWS signing was used from [RightScale's](http://www.rightscale.com/) amazing [right_aws gem](https://github.com/rightscale/right_aws) which works extraordinarily well! If you need access to any AWS service (EC2, S3, etc.), have a look.
|
341
343
|
* <code>camelize</code> was used from ActiveSupport to assist in converting parmeters to AWS request format.
|
342
|
-
* Thanks to the following people who have contributed patches or helpful suggestions: [Ryan Weald](https://github.com/rweald), [Aram Price](https://github.com/aramprice/), [Wouter Broekhof](https://github.com/wouter/)
|
344
|
+
* Thanks to the following people who have contributed patches or helpful suggestions: [Ryan Weald](https://github.com/rweald), [Aram Price](https://github.com/aramprice/), [Wouter Broekhof](https://github.com/wouter/), [Menno van der Sman](https://github.com/menno), [Michael Tibben](https://github.com/mtibben) and [Alexander Dean](https://github.com/alexanderdean).
|
343
345
|
|
344
346
|
|
345
347
|
# License
|
346
348
|
|
347
349
|
```
|
348
|
-
Copyright 2011-
|
350
|
+
Copyright 2011-2013 Robert Slifka
|
349
351
|
|
350
352
|
Licensed under the Apache License, Version 2.0 (the "License");
|
351
353
|
you may not use this file except in compliance with the License.
|
data/elasticity.gemspec
CHANGED
@@ -16,8 +16,8 @@ Gem::Specification.new do |s|
|
|
16
16
|
s.add_dependency('fog')
|
17
17
|
|
18
18
|
s.add_development_dependency('rake', '~> 0.9')
|
19
|
-
s.add_development_dependency('rspec', '~> 2.
|
20
|
-
s.add_development_dependency('timecop')
|
19
|
+
s.add_development_dependency('rspec', '~> 2.12.0')
|
20
|
+
s.add_development_dependency('timecop', '~> 0.5')
|
21
21
|
s.add_development_dependency('fakefs', '~> 0.4')
|
22
22
|
|
23
23
|
s.files = `git ls-files`.split("\n")
|
data/lib/elasticity/hive_step.rb
CHANGED
@@ -41,14 +41,7 @@ module Elasticity
|
|
41
41
|
:action_on_failure => 'TERMINATE_JOB_FLOW',
|
42
42
|
:hadoop_jar_step => {
|
43
43
|
:jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
|
44
|
-
:args =>
|
45
|
-
's3://elasticmapreduce/libs/hive/hive-script',
|
46
|
-
'--base-path',
|
47
|
-
's3://elasticmapreduce/libs/hive/',
|
48
|
-
'--install-hive',
|
49
|
-
'--hive-versions',
|
50
|
-
'latest'
|
51
|
-
],
|
44
|
+
:args => %w(s3://elasticmapreduce/libs/hive/hive-script --base-path s3://elasticmapreduce/libs/hive/ --install-hive --hive-versions latest)
|
52
45
|
},
|
53
46
|
:name => 'Elasticity - Install Hive'
|
54
47
|
}
|
data/lib/elasticity/job_flow.rb
CHANGED
@@ -157,7 +157,10 @@ module Elasticity
|
|
157
157
|
:instances => {
|
158
158
|
:keep_job_flow_alive_when_no_steps => @keep_job_flow_alive_when_no_steps,
|
159
159
|
:hadoop_version => @hadoop_version,
|
160
|
-
:instance_groups => jobflow_instance_groups
|
160
|
+
:instance_groups => jobflow_instance_groups,
|
161
|
+
:placement => {
|
162
|
+
:availability_zone => @placement
|
163
|
+
}
|
161
164
|
}
|
162
165
|
}
|
163
166
|
preamble.merge!(:ec2_subnet_id => @ec2_subnet_id) if @ec2_subnet_id
|
data/lib/elasticity/version.rb
CHANGED
@@ -17,7 +17,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
|
|
17
17
|
|
18
18
|
let(:hive_jobflow) do
|
19
19
|
Elasticity::JobFlow.new('access', 'secret').tap do |jf|
|
20
|
-
jf.placement = 'us-west-
|
20
|
+
jf.placement = 'us-west-1a'
|
21
21
|
jf.log_uri = 's3n://slif-test/output/logs'
|
22
22
|
jf.add_step(hive_step)
|
23
23
|
end
|
@@ -45,6 +45,9 @@ describe 'Elasticity::JobFlow Integration Examples' do
|
|
45
45
|
:market => 'ON_DEMAND'
|
46
46
|
},
|
47
47
|
],
|
48
|
+
:placement => {
|
49
|
+
:availability_zone => 'us-west-1a'
|
50
|
+
},
|
48
51
|
},
|
49
52
|
:steps => [
|
50
53
|
{
|
@@ -99,7 +102,7 @@ describe 'Elasticity::JobFlow Integration Examples' do
|
|
99
102
|
|
100
103
|
let(:pig_jobflow) do
|
101
104
|
Elasticity::JobFlow.new('access', 'secret').tap do |jf|
|
102
|
-
jf.placement = 'us-west-
|
105
|
+
jf.placement = 'us-west-1c'
|
103
106
|
jf.instance_count = 8
|
104
107
|
jf.slave_instance_type = 'm1.xlarge'
|
105
108
|
jf.log_uri = 's3n://slif-test/output/logs'
|
@@ -128,8 +131,12 @@ describe 'Elasticity::JobFlow Integration Examples' do
|
|
128
131
|
:instance_type => 'm1.xlarge',
|
129
132
|
:market => 'ON_DEMAND'
|
130
133
|
},
|
131
|
-
]
|
134
|
+
],
|
135
|
+
:placement => {
|
136
|
+
:availability_zone => 'us-west-1c'
|
137
|
+
},
|
132
138
|
},
|
139
|
+
|
133
140
|
:steps => [
|
134
141
|
{
|
135
142
|
:action_on_failure => 'TERMINATE_JOB_FLOW',
|
@@ -210,7 +217,10 @@ describe 'Elasticity::JobFlow Integration Examples' do
|
|
210
217
|
:instance_type => 'm1.small',
|
211
218
|
:market => 'ON_DEMAND'
|
212
219
|
},
|
213
|
-
]
|
220
|
+
],
|
221
|
+
:placement => {
|
222
|
+
:availability_zone => 'us-west-1b'
|
223
|
+
},
|
214
224
|
},
|
215
225
|
:steps => [
|
216
226
|
{
|
@@ -356,7 +356,10 @@ describe Elasticity::JobFlow do
|
|
356
356
|
:instances => {
|
357
357
|
:keep_job_flow_alive_when_no_steps => false,
|
358
358
|
:hadoop_version => '1.0.3',
|
359
|
-
:instance_groups => ['INSTANCE_GROUP_CONFIGURATION']
|
359
|
+
:instance_groups => ['INSTANCE_GROUP_CONFIGURATION'],
|
360
|
+
:placement => {
|
361
|
+
:availability_zone => 'us-east-1a'
|
362
|
+
}
|
360
363
|
}
|
361
364
|
}
|
362
365
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elasticity
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version:
|
4
|
+
version: 2.5.1
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-
|
12
|
+
date: 2012-11-29 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: rest-client
|
@@ -82,7 +82,7 @@ dependencies:
|
|
82
82
|
requirements:
|
83
83
|
- - ~>
|
84
84
|
- !ruby/object:Gem::Version
|
85
|
-
version: 2.
|
85
|
+
version: 2.12.0
|
86
86
|
type: :development
|
87
87
|
prerelease: false
|
88
88
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -90,23 +90,23 @@ dependencies:
|
|
90
90
|
requirements:
|
91
91
|
- - ~>
|
92
92
|
- !ruby/object:Gem::Version
|
93
|
-
version: 2.
|
93
|
+
version: 2.12.0
|
94
94
|
- !ruby/object:Gem::Dependency
|
95
95
|
name: timecop
|
96
96
|
requirement: !ruby/object:Gem::Requirement
|
97
97
|
none: false
|
98
98
|
requirements:
|
99
|
-
- -
|
99
|
+
- - ~>
|
100
100
|
- !ruby/object:Gem::Version
|
101
|
-
version: '0'
|
101
|
+
version: '0.5'
|
102
102
|
type: :development
|
103
103
|
prerelease: false
|
104
104
|
version_requirements: !ruby/object:Gem::Requirement
|
105
105
|
none: false
|
106
106
|
requirements:
|
107
|
-
- -
|
107
|
+
- - ~>
|
108
108
|
- !ruby/object:Gem::Version
|
109
|
-
version: '0'
|
109
|
+
version: '0.5'
|
110
110
|
- !ruby/object:Gem::Dependency
|
111
111
|
name: fakefs
|
112
112
|
requirement: !ruby/object:Gem::Requirement
|
@@ -187,18 +187,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
187
187
|
- - ! '>='
|
188
188
|
- !ruby/object:Gem::Version
|
189
189
|
version: '0'
|
190
|
-
segments:
|
191
|
-
- 0
|
192
|
-
hash: -4541363325950914992
|
193
190
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
194
191
|
none: false
|
195
192
|
requirements:
|
196
193
|
- - ! '>='
|
197
194
|
- !ruby/object:Gem::Version
|
198
195
|
version: '0'
|
199
|
-
segments:
|
200
|
-
- 0
|
201
|
-
hash: -4541363325950914992
|
202
196
|
requirements: []
|
203
197
|
rubyforge_project:
|
204
198
|
rubygems_version: 1.8.24
|