elasticity 0.3.5 → 0.4
Sign up to get free protection for your applications and to get access to all the features.
data/README.mediawiki
CHANGED
@@ -6,6 +6,8 @@ Each Elasticity call provides some level of wrapping to save you the trouble of
|
|
6
6
|
|
7
7
|
Alternatively, you may opt for "direct" access to the API where you specify the params and Elasticity takes care of the signing for you, responding with the XML from Amazon. Direct access is described below the API catalog.
|
8
8
|
|
9
|
+
'''BACKLOG''': Have a look at the backlog [https://www.pivotaltracker.com/projects/272429 here] (Pivotal Tracker).
|
10
|
+
|
9
11
|
'''CREDITS''': AWS signing was used from [http://www.rightscale.com/ RightScale's] amazing [https://github.com/rightscale/right_aws right_aws gem] which works extraordinarily well! If you need access to any AWS service (EC2, S3, etc.), have a look.
|
10
12
|
|
11
13
|
== AddInstanceGroups ==
|
@@ -29,7 +31,29 @@ DescribeJobFlows returns detailed information as to the state of all jobs. Curr
|
|
29
31
|
|
30
32
|
== ModifyInstanceGroups ==
|
31
33
|
|
32
|
-
|
34
|
+
A job flow contains several "instance groups" of various types. These instances are where the work for your EMR task occurs. After a job flow has been created, you can find these instance groups in the AWS web UI by clicking on a job flow and then clicking on the "Instance Groups" tab.
|
35
|
+
|
36
|
+
<pre>
|
37
|
+
> emr = Elasticity::EMR.new(ENV["aws_access_key_id"], ENV["aws_secret_key"])
|
38
|
+
> emr.modify_instance_groups({"ig-2T1HNUO61BG3O" => 3})
|
39
|
+
>
|
40
|
+
</pre>
|
41
|
+
|
42
|
+
If there's an error, you'll receive an ArgumentError containing the message from Amazon. For example if you attempt to modify an instance group that's part of a terminated job flow:
|
43
|
+
|
44
|
+
<pre>
|
45
|
+
> emr = Elasticity::EMR.new(ENV["aws_access_key_id"], ENV["aws_secret_key"])
|
46
|
+
> emr.modify_instance_groups({"ig-some_terminated_group" => 3})
|
47
|
+
ArgumentError: An instance group may only be modified when the job flow is running or waiting
|
48
|
+
</pre>
|
49
|
+
|
50
|
+
Or if you attempt to increase the instance count of the MASTER instance group:
|
51
|
+
|
52
|
+
<pre>
|
53
|
+
> emr = Elasticity::EMR.new(ENV["aws_access_key_id"], ENV["aws_secret_key"])
|
54
|
+
> emr.modify_instance_groups({"ig-some_terminated_group" => 3})
|
55
|
+
ArgumentError: A master instance group may not be modified
|
56
|
+
</pre>
|
33
57
|
|
34
58
|
== RunJobFlow ==
|
35
59
|
|
data/lib/elasticity/emr.rb
CHANGED
@@ -15,6 +15,30 @@ module Elasticity
|
|
15
15
|
JobFlow.from_members_nodeset(xml_doc.xpath("/DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member"))
|
16
16
|
end
|
17
17
|
|
18
|
+
# Set the number of instances in the specified instance groups to the
|
19
|
+
# specified counts. Note that this modifies the *request* count, which
|
20
|
+
# is not the same as the *running* count. I.e. you request instances
|
21
|
+
# and then wait for them to be created.
|
22
|
+
#
|
23
|
+
# Takes a {} of instance group IDs => desired instance count.
|
24
|
+
#
|
25
|
+
# {"ig-1" => 40, "ig-2" => 5", ...}
|
26
|
+
def modify_instance_groups(instance_group_config)
|
27
|
+
params = {"Operation" => "ModifyInstanceGroups"}
|
28
|
+
instance_group_config.keys.each_with_index do |instance_group, index|
|
29
|
+
params.merge!(
|
30
|
+
"InstanceGroups.member.#{index+1}.InstanceGroupId" => instance_group,
|
31
|
+
"InstanceGroups.member.#{index+1}.InstanceCount" => instance_group_config[instance_group]
|
32
|
+
)
|
33
|
+
end
|
34
|
+
begin
|
35
|
+
aws_result = @aws_request.aws_emr_request(params)
|
36
|
+
yield aws_result if block_given?
|
37
|
+
rescue RestClient::BadRequest => e
|
38
|
+
raise ArgumentError, parse_error_response(e.http_body)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
18
42
|
# Terminate the specified jobflow. Amazon does not define a return value
|
19
43
|
# for this operation, so you'll need to poll #describe_jobflows to see
|
20
44
|
# the state of the jobflow. Raises ArgumentError if the specified job
|
@@ -39,5 +63,13 @@ module Elasticity
|
|
39
63
|
@aws_request.aws_emr_request(params)
|
40
64
|
end
|
41
65
|
|
66
|
+
private
|
67
|
+
|
68
|
+
def parse_error_response(error_xml)
|
69
|
+
xml_doc = Nokogiri::XML(error_xml)
|
70
|
+
xml_doc.remove_namespaces!
|
71
|
+
xml_doc.xpath("/ErrorResponse/Error/Message").text
|
72
|
+
end
|
73
|
+
|
42
74
|
end
|
43
75
|
end
|
data/lib/elasticity/version.rb
CHANGED
@@ -0,0 +1,32 @@
|
|
1
|
+
---
|
2
|
+
- !ruby/struct:VCR::HTTPInteraction
|
3
|
+
request: !ruby/struct:VCR::Request
|
4
|
+
method: :get
|
5
|
+
uri: !ruby/regexp /^http:\/\/elasticmapreduce.amazonaws.com:80\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&InstanceGroups.member.1.InstanceCount=2&InstanceGroups.member.1.InstanceGroupId=ig-2T1HNUO61BG3O&Operation=ModifyInstanceGroups/
|
6
|
+
body:
|
7
|
+
headers:
|
8
|
+
accept:
|
9
|
+
- "*/*; q=0.5, application/xml"
|
10
|
+
accept-encoding:
|
11
|
+
- gzip, deflate
|
12
|
+
response: !ruby/struct:VCR::Response
|
13
|
+
status: !ruby/struct:VCR::ResponseStatus
|
14
|
+
code: 200
|
15
|
+
message: OK
|
16
|
+
headers:
|
17
|
+
x-amzn-requestid:
|
18
|
+
- 4ef75373-659c-11e0-bdf6-e3d62a364c28
|
19
|
+
content-type:
|
20
|
+
- text/xml
|
21
|
+
date:
|
22
|
+
- Wed, 13 Apr 2011 07:04:43 GMT
|
23
|
+
content-length:
|
24
|
+
- "231"
|
25
|
+
body: |
|
26
|
+
<ModifyInstanceGroupsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
27
|
+
<ResponseMetadata>
|
28
|
+
<RequestId>4ef75373-659c-11e0-bdf6-e3d62a364c28</RequestId>
|
29
|
+
</ResponseMetadata>
|
30
|
+
</ModifyInstanceGroupsResponse>
|
31
|
+
|
32
|
+
http_version: "1.1"
|
@@ -82,6 +82,7 @@ describe Elasticity::EMR do
|
|
82
82
|
describe "unit tests" do
|
83
83
|
|
84
84
|
context "when the jobflow exists" do
|
85
|
+
|
85
86
|
before do
|
86
87
|
@terminate_jobflows_xml = <<-RESPONSE
|
87
88
|
<TerminateJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
@@ -91,6 +92,7 @@ describe Elasticity::EMR do
|
|
91
92
|
</TerminateJobFlowsResponse>
|
92
93
|
RESPONSE
|
93
94
|
end
|
95
|
+
|
94
96
|
it "should terminate the specific jobflow" do
|
95
97
|
aws_request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_key")
|
96
98
|
aws_request.should_receive(:aws_emr_request).with({
|
@@ -101,6 +103,7 @@ describe Elasticity::EMR do
|
|
101
103
|
emr = Elasticity::EMR.new("aws_access_key_id", "aws_secret_key")
|
102
104
|
emr.terminate_jobflows("j-1")
|
103
105
|
end
|
106
|
+
|
104
107
|
context "when a block is given" do
|
105
108
|
it "should yield the XML result" do
|
106
109
|
aws_request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_key")
|
@@ -117,10 +120,11 @@ describe Elasticity::EMR do
|
|
117
120
|
xml_result.should == @terminate_jobflows_xml
|
118
121
|
end
|
119
122
|
end
|
123
|
+
|
120
124
|
end
|
121
125
|
|
122
126
|
context "when the jobflow does not exist" do
|
123
|
-
it "should
|
127
|
+
it "should raise an ArgumentError" do
|
124
128
|
aws_request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_key")
|
125
129
|
aws_request.should_receive(:aws_emr_request).and_raise(RestClient::BadRequest)
|
126
130
|
Elasticity::AwsRequest.should_receive(:new).and_return(aws_request)
|
@@ -134,6 +138,90 @@ describe Elasticity::EMR do
|
|
134
138
|
end
|
135
139
|
end
|
136
140
|
|
141
|
+
describe "#modify_instance_groups" do
|
142
|
+
|
143
|
+
describe "integration happy path" do
|
144
|
+
context "when the instance group exists" do
|
145
|
+
use_vcr_cassette "modify_instance_groups/set_instances_to_3", :record => :none
|
146
|
+
it "should terminate the specified jobflow" do
|
147
|
+
emr = Elasticity::EMR.new(ENV["aws_access_key_id"], ENV["aws_secret_key"])
|
148
|
+
instance_group_config = {"ig-2T1HNUO61BG3O" => 2}
|
149
|
+
emr.modify_instance_groups(instance_group_config)
|
150
|
+
end
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
describe "unit tests" do
|
155
|
+
|
156
|
+
context "when the instance group exists" do
|
157
|
+
it "should modify the specified instance group" do
|
158
|
+
aws_request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_key")
|
159
|
+
aws_request.should_receive(:aws_emr_request).with({
|
160
|
+
"Operation" => "ModifyInstanceGroups",
|
161
|
+
"InstanceGroups.member.1.InstanceGroupId" => "ig-1",
|
162
|
+
"InstanceGroups.member.1.InstanceCount" => 2
|
163
|
+
})
|
164
|
+
Elasticity::AwsRequest.should_receive(:new).and_return(aws_request)
|
165
|
+
emr = Elasticity::EMR.new("aws_access_key_id", "aws_secret_key")
|
166
|
+
emr.modify_instance_groups({"ig-1" => 2})
|
167
|
+
end
|
168
|
+
end
|
169
|
+
|
170
|
+
context "when a block is given" do
|
171
|
+
before do
|
172
|
+
@modify_instance_groups_xml = <<-RESPONSE
|
173
|
+
<ModifyInstanceGroupsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
174
|
+
<ResponseMetadata>
|
175
|
+
<RequestId>4ef75373-659c-11e0-bdf6-e3d62a364c28</RequestId>
|
176
|
+
</ResponseMetadata>
|
177
|
+
</ModifyInstanceGroupsResponse>
|
178
|
+
RESPONSE
|
179
|
+
end
|
180
|
+
it "should yield the XML result" do
|
181
|
+
aws_request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_key")
|
182
|
+
aws_request.should_receive(:aws_emr_request).and_return(@modify_instance_groups_xml)
|
183
|
+
Elasticity::AwsRequest.should_receive(:new).and_return(aws_request)
|
184
|
+
emr = Elasticity::EMR.new("aws_access_key_id", "aws_secret_key")
|
185
|
+
xml_result = nil
|
186
|
+
emr.modify_instance_groups({"ig-1" => 2}) do |xml|
|
187
|
+
xml_result = xml
|
188
|
+
end
|
189
|
+
xml_result.should == @modify_instance_groups_xml
|
190
|
+
end
|
191
|
+
end
|
192
|
+
|
193
|
+
|
194
|
+
context "when there is an error" do
|
195
|
+
|
196
|
+
before do
|
197
|
+
@error_message = "1 validation error detected: Value null at 'instanceGroups.1.member.instanceCount' failed to satisfy constraint: Member must not be null"
|
198
|
+
@error_xml = <<-ERROR
|
199
|
+
<ErrorResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
200
|
+
<Error>
|
201
|
+
<Message>#{@error_message}</Message>
|
202
|
+
</Error>
|
203
|
+
</ErrorResponse>
|
204
|
+
ERROR
|
205
|
+
end
|
206
|
+
|
207
|
+
it "should raise an ArgumentError with the error message" do
|
208
|
+
aws_request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_key")
|
209
|
+
@exception = RestClient::BadRequest.new
|
210
|
+
@exception.should_receive(:http_body).and_return(@error_xml)
|
211
|
+
aws_request.should_receive(:aws_emr_request).and_raise(@exception)
|
212
|
+
Elasticity::AwsRequest.should_receive(:new).and_return(aws_request)
|
213
|
+
emr = Elasticity::EMR.new("aws_access_key_id", "aws_secret_key")
|
214
|
+
lambda {
|
215
|
+
emr.modify_instance_groups({"ig-1" => 2})
|
216
|
+
}.should raise_error(ArgumentError, @error_message)
|
217
|
+
end
|
218
|
+
|
219
|
+
end
|
220
|
+
|
221
|
+
end
|
222
|
+
|
223
|
+
end
|
224
|
+
|
137
225
|
describe "#direct" do
|
138
226
|
|
139
227
|
describe "integration happy path" do
|
metadata
CHANGED
@@ -1,13 +1,12 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: elasticity
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 3
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
|
-
-
|
9
|
-
|
10
|
-
version: 0.3.5
|
8
|
+
- 4
|
9
|
+
version: "0.4"
|
11
10
|
platform: ruby
|
12
11
|
authors:
|
13
12
|
- Robert Slifka
|
@@ -15,7 +14,7 @@ autorequire:
|
|
15
14
|
bindir: bin
|
16
15
|
cert_chain: []
|
17
16
|
|
18
|
-
date: 2011-04-
|
17
|
+
date: 2011-04-13 00:00:00 -07:00
|
19
18
|
default_executable:
|
20
19
|
dependencies:
|
21
20
|
- !ruby/object:Gem::Dependency
|
@@ -131,6 +130,7 @@ files:
|
|
131
130
|
- lib/elasticity/version.rb
|
132
131
|
- spec/fixtures/vcr_cassettes/describe_jobflows/all_jobflows.yml
|
133
132
|
- spec/fixtures/vcr_cassettes/direct/terminate_jobflow.yml
|
133
|
+
- spec/fixtures/vcr_cassettes/modify_instance_groups/set_instances_to_3.yml
|
134
134
|
- spec/fixtures/vcr_cassettes/terminate_jobflows/one_jobflow.yml
|
135
135
|
- spec/lib/elasticity/aws_request_spec.rb
|
136
136
|
- spec/lib/elasticity/emr_spec.rb
|
@@ -173,6 +173,7 @@ summary: Programmatic access to Amazon's Elastic Map Reduce service.
|
|
173
173
|
test_files:
|
174
174
|
- spec/fixtures/vcr_cassettes/describe_jobflows/all_jobflows.yml
|
175
175
|
- spec/fixtures/vcr_cassettes/direct/terminate_jobflow.yml
|
176
|
+
- spec/fixtures/vcr_cassettes/modify_instance_groups/set_instances_to_3.yml
|
176
177
|
- spec/fixtures/vcr_cassettes/terminate_jobflows/one_jobflow.yml
|
177
178
|
- spec/lib/elasticity/aws_request_spec.rb
|
178
179
|
- spec/lib/elasticity/emr_spec.rb
|