elasticity 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +5 -0
- data/.rspec +2 -0
- data/.rvmrc +1 -0
- data/Gemfile +4 -0
- data/README.mediawiki +53 -0
- data/Rakefile +11 -0
- data/elasticity.gemspec +28 -0
- data/lib/elasticity.rb +9 -0
- data/lib/elasticity/aws_request.rb +48 -0
- data/lib/elasticity/emr.rb +35 -0
- data/lib/elasticity/job_flow.rb +35 -0
- data/lib/elasticity/version.rb +3 -0
- data/spec/fixtures/vcr_cassettes/describe_jobflows/all_jobflows.yml +69 -0
- data/spec/fixtures/vcr_cassettes/terminate_jobflows/one_jobflow.yml +32 -0
- data/spec/lib/elasticity/aws_request_spec.rb +27 -0
- data/spec/lib/elasticity/emr_spec.rb +106 -0
- data/spec/lib/elasticity/job_flow_spec.rb +51 -0
- data/spec/spec_helper.rb +19 -0
- metadata +178 -0
data/.rspec
ADDED
data/.rvmrc
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
rvm use ree-1.8.7-2010.02@elasticity --create
|
data/Gemfile
ADDED
data/README.mediawiki
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
= Elasticity =
|
2
|
+
|
3
|
+
Elasticity provides programmatic access to Amazon's Elastic Map Reduce service.
|
4
|
+
|
5
|
+
There are five API actions in the [http://docs.amazonwebservices.com/ElasticMapReduce/latest/DeveloperGuide/index.html EMR API], described below. Please see the Amazon guide for details on these operations because the default values aren't obvious (e.g. the meaning of DescribeJobFlows without parameters).
|
6
|
+
|
7
|
+
'''NOTE''': The AWS plumbing (especially signing/authentication) was used from [http://www.rightscale.com/ RightScale's] amazing [https://github.com/rightscale/right_aws right_aws gem] which works extraordinarily well!
|
8
|
+
|
9
|
+
== AddInstanceGroups ==
|
10
|
+
|
11
|
+
''(not yet supported)''
|
12
|
+
|
13
|
+
== AddJobFlowSteps ==
|
14
|
+
|
15
|
+
''(not yet supported)''
|
16
|
+
|
17
|
+
== DescribeJobFlows ==
|
18
|
+
|
19
|
+
<pre>
|
20
|
+
> emr = Elasticity::EMR.new(ENV["aws_access_key_id"], ENV["aws_secret_key"])
|
21
|
+
> jobflows = emr.describe_jobflows
|
22
|
+
> p jobflows.map(&:name)
|
23
|
+
["Hive Test", "Pig Test", "Interactive Hadoop", "Interactive Hive"]
|
24
|
+
</pre>
|
25
|
+
|
26
|
+
== ModifyInstanceGroups ==
|
27
|
+
|
28
|
+
''(not yet supported)''
|
29
|
+
|
30
|
+
== RunJobFlow ==
|
31
|
+
|
32
|
+
''(not yet supported)''
|
33
|
+
|
34
|
+
== TerminateJobFlows ==
|
35
|
+
|
36
|
+
When the job flow '''exists''', you will receive no output. This is because Amazon does not return anything other than a 200 when you terminate a job flow :) You'll want to continually poll with DescribeJobFlows to see when the job was actually terminated.
|
37
|
+
|
38
|
+
<pre>
|
39
|
+
> emr = Elasticity::EMR.new(ENV["aws_access_key_id"], ENV["aws_secret_key"])
|
40
|
+
> emr.terminate_jobflows("no-flow")
|
41
|
+
>
|
42
|
+
</pre>
|
43
|
+
|
44
|
+
When the job flow '''doesn't exist''':
|
45
|
+
|
46
|
+
<pre>
|
47
|
+
> emr = Elasticity::EMR.new(ENV["aws_access_key_id"], ENV["aws_secret_key"])
|
48
|
+
> emr.terminate_jobflows("no-flow")
|
49
|
+
ArgumentError: Job flow 'no-flow' does not exist.
|
50
|
+
</pre>
|
51
|
+
|
52
|
+
|
53
|
+
<!-- __NOTOC__ -->
|
data/Rakefile
ADDED
data/elasticity.gemspec
ADDED
@@ -0,0 +1,28 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "elasticity/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "elasticity"
|
7
|
+
s.version = Elasticity::VERSION
|
8
|
+
s.platform = Gem::Platform::RUBY
|
9
|
+
s.authors = ["Robert Slifka"]
|
10
|
+
s.homepage = "http://www.github.com/rslifka/elasticity"
|
11
|
+
s.summary = %q{Programmatic access to Amazon's Elastic Map Reduce service.}
|
12
|
+
s.description = %q{Programmatic access to Amazon's Elastic Map Reduce service.}
|
13
|
+
|
14
|
+
s.rubyforge_project = "elasticity"
|
15
|
+
|
16
|
+
s.add_dependency("rest-client")
|
17
|
+
s.add_dependency("nokogiri")
|
18
|
+
|
19
|
+
s.add_development_dependency("rake")
|
20
|
+
s.add_development_dependency("rspec", ">= 2.5.0")
|
21
|
+
s.add_development_dependency("vcr", ">= 1.5.1")
|
22
|
+
s.add_development_dependency("webmock", ">= 1.6.2")
|
23
|
+
|
24
|
+
s.files = `git ls-files`.split("\n")
|
25
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
26
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
27
|
+
s.require_paths = ["lib"]
|
28
|
+
end
|
data/lib/elasticity.rb
ADDED
@@ -0,0 +1,48 @@
|
|
1
|
+
module Elasticity
|
2
|
+
|
3
|
+
class AwsRequest
|
4
|
+
|
5
|
+
def initialize(aws_access_key_id, aws_secret_access_key)
|
6
|
+
@access_key = aws_access_key_id
|
7
|
+
@secret_key = aws_secret_access_key
|
8
|
+
end
|
9
|
+
|
10
|
+
def aws_emr_request(params)
|
11
|
+
signed_params = sign_params(params, "GET", "elasticmapreduce.amazonaws.com", "/")
|
12
|
+
signed_request = "http://elasticmapreduce.amazonaws.com?#{signed_params}"
|
13
|
+
RestClient.get signed_request
|
14
|
+
end
|
15
|
+
|
16
|
+
# (Used from RightScale's right_aws gem.)
|
17
|
+
# EC2, SQS, SDB and EMR requests must be signed by this guy.
|
18
|
+
# See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
|
19
|
+
# http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1928
|
20
|
+
def sign_params(service_hash, http_verb, host, uri)
|
21
|
+
service_hash["AWSAccessKeyId"] = @access_key
|
22
|
+
service_hash["Timestamp"] = Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z")
|
23
|
+
service_hash["SignatureVersion"] = "2"
|
24
|
+
service_hash['SignatureMethod'] = 'HmacSHA256'
|
25
|
+
canonical_string = service_hash.keys.sort.map do |key|
|
26
|
+
"#{AwsRequest.aws_escape(key)}=#{AwsRequest.aws_escape(service_hash[key])}"
|
27
|
+
end.join('&')
|
28
|
+
string_to_sign = "#{http_verb.to_s.upcase}\n#{host.downcase}\n#{uri}\n#{canonical_string}"
|
29
|
+
signature = AwsRequest.aws_escape(Base64.encode64(OpenSSL::HMAC.digest("sha256", @secret_key, string_to_sign)).strip)
|
30
|
+
"#{canonical_string}&Signature=#{signature}"
|
31
|
+
end
|
32
|
+
|
33
|
+
class << self
|
34
|
+
|
35
|
+
# (Used from RightScale's right_aws gem)
|
36
|
+
# Escape a string according to Amazon's rules.
|
37
|
+
# See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
|
38
|
+
def aws_escape(param)
|
39
|
+
param.to_s.gsub(/([^a-zA-Z0-9._~-]+)/n) do
|
40
|
+
'%' + $1.unpack('H2' * $1.size).join('%').upcase
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
45
|
+
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
require "base64"
|
2
|
+
|
3
|
+
module Elasticity
|
4
|
+
|
5
|
+
class EMR
|
6
|
+
|
7
|
+
def initialize(aws_access_key_id, aws_secret_access_key)
|
8
|
+
@aws_request = Elasticity::AwsRequest.new(aws_access_key_id, aws_secret_access_key)
|
9
|
+
end
|
10
|
+
|
11
|
+
# Lists all jobflows in all states.
|
12
|
+
def describe_jobflows
|
13
|
+
aws_result = @aws_request.aws_emr_request({"Operation" => "DescribeJobFlows"})
|
14
|
+
xml_doc = Nokogiri::XML(aws_result)
|
15
|
+
xml_doc.remove_namespaces!
|
16
|
+
JobFlow.from_members_nodeset(xml_doc.xpath("/DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member"))
|
17
|
+
end
|
18
|
+
|
19
|
+
# Terminate the specified jobflow. Amazon does not define a return value
|
20
|
+
# for this operation, so you'll need to poll #describe_jobflows to see
|
21
|
+
# the state of the jobflow. Raises ArgumentError if the specified job
|
22
|
+
# flow does not exist.
|
23
|
+
def terminate_jobflows(jobflow_id)
|
24
|
+
begin
|
25
|
+
@aws_request.aws_emr_request({
|
26
|
+
"Operation" => "TerminateJobFlows",
|
27
|
+
"JobFlowIds.member.1" => jobflow_id
|
28
|
+
})
|
29
|
+
rescue RestClient::BadRequest
|
30
|
+
raise ArgumentError, "Job flow '#{jobflow_id}' does not exist."
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,35 @@
|
|
1
|
+
module Elasticity
|
2
|
+
|
3
|
+
class JobFlow
|
4
|
+
|
5
|
+
attr_accessor :name
|
6
|
+
attr_accessor :jobflow_id
|
7
|
+
attr_accessor :state
|
8
|
+
|
9
|
+
class << self
|
10
|
+
|
11
|
+
# Create a jobflow from an AWS <member> (Nokogiri::XML::Element):
|
12
|
+
# /DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member
|
13
|
+
def from_member_element(xml_element)
|
14
|
+
jobflow = JobFlow.new
|
15
|
+
jobflow.name = xml_element.xpath("./Name").text
|
16
|
+
jobflow.jobflow_id = xml_element.xpath("./JobFlowId").text
|
17
|
+
jobflow.state = xml_element.xpath("./ExecutionStatusDetail/State").text
|
18
|
+
jobflow
|
19
|
+
end
|
20
|
+
|
21
|
+
# Create JobFlows from a collection of AWS <member> nodes (Nokogiri::XML::NodeSet):
|
22
|
+
# /DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows
|
23
|
+
def from_members_nodeset(members_nodeset)
|
24
|
+
jobflows = []
|
25
|
+
members_nodeset.each do |member|
|
26
|
+
jobflows << from_member_element(member)
|
27
|
+
end
|
28
|
+
jobflows
|
29
|
+
end
|
30
|
+
|
31
|
+
end
|
32
|
+
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
---
|
2
|
+
- !ruby/struct:VCR::HTTPInteraction
|
3
|
+
request: !ruby/struct:VCR::Request
|
4
|
+
method: :get
|
5
|
+
uri: !ruby/regexp /^http:\/\/elasticmapreduce.amazonaws.com:80\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&Operation=DescribeJobFlows/
|
6
|
+
body:
|
7
|
+
headers:
|
8
|
+
accept:
|
9
|
+
- "*/*; q=0.5, application/xml"
|
10
|
+
accept-encoding:
|
11
|
+
- gzip, deflate
|
12
|
+
response: !ruby/struct:VCR::Response
|
13
|
+
status: !ruby/struct:VCR::ResponseStatus
|
14
|
+
code: 200
|
15
|
+
message: OK
|
16
|
+
headers:
|
17
|
+
x-amzn-requestid:
|
18
|
+
- 994deb7a-63ac-11e0-bc41-ababd98a870b
|
19
|
+
content-type:
|
20
|
+
- text/xml
|
21
|
+
date:
|
22
|
+
- Sun, 10 Apr 2011 19:56:17 GMT
|
23
|
+
content-encoding:
|
24
|
+
- gzip
|
25
|
+
vary:
|
26
|
+
- Accept-Encoding
|
27
|
+
transfer-encoding:
|
28
|
+
- chunked
|
29
|
+
body: !binary |
|
30
|
+
H4sIAAAAAAAAAO1bW3PaOhB+769gztM5c0bYEld7XGZIgIQQIAXnMnnJyLYC
|
31
|
+
7vGttpw2/fVHtmOwjQkQIKUpM0wSS7urlfTp0+6aSC3iqa6ukAtb6Rj2d29E
|
32
|
+
PMe2PFL4YRqW9/mvKaWOyHHEwB7VVRM7LtF8lRSxiX/aFv7uFVXb5DRb5RDP
|
33
|
+
C4AvgRL8q/GpUJByLPsGDbpYZ9wYPbIGk5gKceNH1tD+QVSf6rY1ppj6XotQ
|
34
|
+
rBvzfiZx6hIcCLQwJbJukgbiIQR8mX1kWBPLUKzAe4lbEEvauGTTCgYgp1Ns
|
35
|
+
TciIYM+2GjJxTd1irVpBeS74HnELLvnmE49KXL5G0ibrdekypypiSWBOpWWS
|
36
|
+
2syg9rxCOy2TGZuShtwe9buDptxuhSPRtEzb0hbtCzKsiuzDB0uWlJjvCLdi
|
37
|
+
S6QT26YedbHTVAMpj0v0DTCzddv/dzSWuPDvTwmfieOlPMyCYSZ2aluP+iTd
|
38
|
+
wbrOsWbbzgV2A5lsb4A27Da8Ug6MOUNXPC7AqUOB61sWcdNPxa/YlbhAf9Fq
|
39
|
+
0514i81z918Zcao/kfAHiEaTuLwpp80BoGCPAAfT6Triq0Zfb0jd8ig2DBCo
|
40
|
+
LNeQuPy1kK5c2yEu1UkSC7HOq7sWAWZMqO8UzsPB06iJ9yBE2tDqMCT6bgL5
|
41
|
+
DxfDk4fO5fCW+ZaRScOKW4ar1QwUSu2AhUI7W7FGbIGd9NNh/+qynX/0o2nl
|
42
|
+
Hf9oGMYAlWXHP1qs1ay8AJJohRPoiMm/qzW+Ati/r8g3t52L0bg3YCdt1jUX
|
43
|
+
7wYYtFSS5og+QzZx4z6moANcrdf5R1WQuIXOpOaVgVViEotmJtd8YnPBim7o
|
44
|
+
9PnetkjD9wBjdwqgykCU7UzNOdfkzPEz1/adzPHI5bgdwikee8RoTLcmp7bP
|
45
|
+
nOMlLrd9IyyWRVRdicVVt1jttVss9oGBtz1obYjk1RdZdoWimz1aCZhYoWT7
|
46
|
+
MmX52SENExY9k7HkXDdszur0sfsfoY3h4KHV7jcHrQCmYUtWMD/EYEej8MjO
|
47
|
+
RoHOopM1opH0VG2DNPrNMWPJxDyDxmUaIXSDwzUBSDjv8L0xX+1enXyZq8cS
|
48
|
+
ufxthucwj7vzLpOM0fR5H9guW2P9J9FioXPbd70GhOy8L+tMGoiumxvieuzc
|
49
|
+
NPgi4uMrKG5bTi+Zbc7pXNS98hVDV1uWF06eqAgghk9UqgIo1ACCMIicHZ8S
|
50
|
+
ANPRdGw/bSAVwamoR57DZm+KXUKnbL0m0weNPDHMzzvzyGgB5wsAl3qEOC9E
|
51
|
+
3DTY3Xs7JdbAjlicuj67i1+TSBD/AnNnd32HYX+QeMiwLvIVEQqHEvbPnKqL
|
52
|
+
EG4c9me19xD2Q5mHIiq/cu/vIOzvWgzNmHU+kULT13TCIBEGdQU5XNtjPnDM
|
53
|
+
Bz56PrAJPYV2tqKT2MLb8oHEMEh453ygVG9fo7v7L1dC76K8ZT7A16tlDVZr
|
54
|
+
O88HlF+eD2wMp53nAzMPaiKsvSkfSKKZ31s+sPqGy67QLvIBA7sT8rHzAbl6
|
55
|
+
1ul1eHR+17s5nHSghN4vG3jZ5DdmA7UKgDwEqFIGAjrmArvPBWpipRKVAQ4o
|
56
|
+
F2BOCWKZf2MuMNfedS4wI+OyiEp7zAVm8f/fQRT3zzH4Pwb/hT8k+F+Pj0I7
|
57
|
+
W/FHbGHL4J8XYXnnwf/xPOebA4FLYGMlFoB4a0k+rscSVt6kPGw6BonmBbDm
|
58
|
+
RbM0bY0YQPF1Qyt+W8sHbR2p7uDqWv68ricUK+x5d6MPr+X58Owei3aEslt/
|
59
|
+
PqbtUxapcfPTsrvhL7sn47XnHuzCL2BhlihGtbvxC04/OBUvT742z7tezaNf
|
60
|
+
+HYtWm8OTtuXl0tofQ9VGSS3Tm/O2u3zYacrbFmVEaq1qlp63P1b2kOpymwA
|
61
|
+
rj1VZZgH7FPaoioThhml+n6qMmvlGtkVOr6lXe8t7eBu3C+1Bmd3/HXncMoy
|
62
|
+
8Hd5R1vhAawBWEEA8r9lUeYRG95hV2VQRUT1Q6vKlKCI0JurMrH2fqoyjIyR
|
63
|
+
iGrvUZUZEcdmwaE1ORZmjoWZwh9TmFmHkkI7W1FIbGGbwkw4TFj/ec+3soNe
|
64
|
+
70sZCa16pd7dMvwvE0FDdQQ/4Jc0N0bT3sJ/JLAcZKvwPwDzq985in14a/i/
|
65
|
+
4lLLrtDW4b8KiybRdN/82PE/7JVu7+XhqDk6GQqHE/+/11vZxC6/MQGAtTKA
|
66
|
+
SAAQBr+rv2UOsMsXszO6D/slbvm/dUnx/4712X2jYYpfDLycUYY+QShrRKlh
|
67
|
+
UC1hla0w4YGiliHAClY0oY7rNV4JKCeWDwdctJrrRCjT+PQ/yLkKP9E2AAA=
|
68
|
+
|
69
|
+
http_version: "1.1"
|
@@ -0,0 +1,32 @@
|
|
1
|
+
---
|
2
|
+
- !ruby/struct:VCR::HTTPInteraction
|
3
|
+
request: !ruby/struct:VCR::Request
|
4
|
+
method: :get
|
5
|
+
uri: !ruby/regexp /^http:\/\/elasticmapreduce.amazonaws.com:80\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&JobFlowIds.member.1=j-1MZ5TVWFJRSKN&Operation=TerminateJobFlows/
|
6
|
+
body:
|
7
|
+
headers:
|
8
|
+
accept:
|
9
|
+
- "*/*; q=0.5, application/xml"
|
10
|
+
accept-encoding:
|
11
|
+
- gzip, deflate
|
12
|
+
response: !ruby/struct:VCR::Response
|
13
|
+
status: !ruby/struct:VCR::ResponseStatus
|
14
|
+
code: 200
|
15
|
+
message: OK
|
16
|
+
headers:
|
17
|
+
x-amzn-requestid:
|
18
|
+
- 83e0154d-63c6-11e0-bc41-ababd98a870b
|
19
|
+
content-type:
|
20
|
+
- text/xml
|
21
|
+
date:
|
22
|
+
- Sun, 10 Apr 2011 23:01:48 GMT
|
23
|
+
content-length:
|
24
|
+
- "225"
|
25
|
+
body: |
|
26
|
+
<TerminateJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
27
|
+
<ResponseMetadata>
|
28
|
+
<RequestId>83e0154d-63c6-11e0-bc41-ababd98a870b</RequestId>
|
29
|
+
</ResponseMetadata>
|
30
|
+
</TerminateJobFlowsResponse>
|
31
|
+
|
32
|
+
http_version: "1.1"
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Elasticity::AwsRequest do
|
4
|
+
|
5
|
+
describe ".aws_escape" do
|
6
|
+
it "should escape according to Amazon's rules" do
|
7
|
+
# Don't encode reserved characters
|
8
|
+
Elasticity::AwsRequest.aws_escape("foo-_.~bar").should == "foo-_.~bar"
|
9
|
+
# Encode as %20, not as +
|
10
|
+
Elasticity::AwsRequest.aws_escape("foo bar").should == "foo%20bar"
|
11
|
+
# Percent encode all other characters with %XY, where X and Y are hex characters 0-9 and uppercase A-F.
|
12
|
+
Elasticity::AwsRequest.aws_escape("foo$&+,/:;=?@bar").should == "foo%24%26%2B%2C%2F%3A%3B%3D%3F%40bar"
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
describe "#sign_params" do
|
17
|
+
before do
|
18
|
+
Time.should_receive(:now).and_return(Time.at(1302461096))
|
19
|
+
end
|
20
|
+
it "should sign according to Amazon's rules" do
|
21
|
+
request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_access_key")
|
22
|
+
signed_params = request.send(:sign_params, {}, "GET", "example.com", "/")
|
23
|
+
signed_params.should == "AWSAccessKeyId=aws_access_key_id&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2011-04-10T18%3A44%3A56.000Z&Signature=jVLfPS056dNmjpCcikBnPmRHJNZ8YGaI7zdmHWUk658%3D"
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
@@ -0,0 +1,106 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Elasticity::EMR do
|
4
|
+
|
5
|
+
describe "#describe_jobflows" do
|
6
|
+
|
7
|
+
describe "integration happy path" do
|
8
|
+
use_vcr_cassette "describe_jobflows/all_jobflows", :record => :none
|
9
|
+
it "should return the names of all running job flows" do
|
10
|
+
emr = Elasticity::EMR.new(ENV["aws_access_key_id"], ENV["aws_secret_key"])
|
11
|
+
jobflows = emr.describe_jobflows
|
12
|
+
jobflows.map(&:name).should == ["WM+RS", "Interactive Audience Hive Test", "Audience (Hive)", "Audience Reporting"]
|
13
|
+
jobflows.map(&:jobflow_id).should == ["j-1MZ5TVWFJRSKN", "j-38EU2XZQP9KJ4", "j-2TDCVGEEHOFI9", "j-NKKQ429D858I"]
|
14
|
+
jobflows.map(&:state).should == ["TERMINATED", "TERMINATED", "TERMINATED", "TERMINATED"]
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
describe "unit tests" do
|
19
|
+
before do
|
20
|
+
@describe_jobflows_xml = <<-JOBFLOWS
|
21
|
+
<DescribeJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
22
|
+
<DescribeJobFlowsResult>
|
23
|
+
<JobFlows>
|
24
|
+
<member>
|
25
|
+
<ExecutionStatusDetail>
|
26
|
+
<State>TERMINATED</State>
|
27
|
+
</ExecutionStatusDetail>
|
28
|
+
<JobFlowId>j-p</JobFlowId>
|
29
|
+
<Name>Pig Job</Name>
|
30
|
+
</member>
|
31
|
+
<member>
|
32
|
+
<ExecutionStatusDetail>
|
33
|
+
<State>TERMINATED</State>
|
34
|
+
</ExecutionStatusDetail>
|
35
|
+
<JobFlowId>j-h</JobFlowId>
|
36
|
+
<Name>Hive Job</Name>
|
37
|
+
</member>
|
38
|
+
</JobFlows>
|
39
|
+
</DescribeJobFlowsResult>
|
40
|
+
</DescribeJobFlowsResponse>
|
41
|
+
JOBFLOWS
|
42
|
+
end
|
43
|
+
it "should return the names of all running job flows" do
|
44
|
+
aws_request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_key")
|
45
|
+
aws_request.should_receive(:aws_emr_request).with({"Operation" => "DescribeJobFlows"}).and_return(@describe_jobflows_xml)
|
46
|
+
Elasticity::AwsRequest.should_receive(:new).and_return(aws_request)
|
47
|
+
emr = Elasticity::EMR.new("aws_access_key_id", "aws_secret_key")
|
48
|
+
jobflows = emr.describe_jobflows
|
49
|
+
jobflows.map(&:name).should == ["Pig Job", "Hive Job"]
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
54
|
+
|
55
|
+
describe "#terminate_jobflows" do
|
56
|
+
|
57
|
+
describe "integration happy path" do
|
58
|
+
context "when the job flow exists" do
|
59
|
+
use_vcr_cassette "terminate_jobflows/one_jobflow", :record => :none
|
60
|
+
it "should terminate the specified jobflow" do
|
61
|
+
emr = Elasticity::EMR.new(ENV["aws_access_key_id"], ENV["aws_secret_key"])
|
62
|
+
emr.terminate_jobflows("j-1MZ5TVWFJRSKN")
|
63
|
+
end
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
describe "unit tests" do
|
68
|
+
|
69
|
+
context "when the jobflow exists" do
|
70
|
+
before do
|
71
|
+
@terminate_jobflows_xml = <<-RESPONSE
|
72
|
+
<TerminateJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
73
|
+
<ResponseMetadata>
|
74
|
+
<RequestId>2690d7eb-ed86-11dd-9877-6fad448a8419</RequestId>
|
75
|
+
</ResponseMetadata>
|
76
|
+
</TerminateJobFlowsResponse>
|
77
|
+
RESPONSE
|
78
|
+
end
|
79
|
+
it "should terminate the specific jobflow" do
|
80
|
+
aws_request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_key")
|
81
|
+
aws_request.should_receive(:aws_emr_request).with({
|
82
|
+
"Operation" => "TerminateJobFlows",
|
83
|
+
"JobFlowIds.member.1" => "j-1"
|
84
|
+
}).and_return(@terminate_jobflows_xml)
|
85
|
+
Elasticity::AwsRequest.should_receive(:new).and_return(aws_request)
|
86
|
+
emr = Elasticity::EMR.new("aws_access_key_id", "aws_secret_key")
|
87
|
+
emr.terminate_jobflows("j-1")
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
context "when the jobflow does not exist" do
|
92
|
+
it "should terminate the specific jobflow" do
|
93
|
+
aws_request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_key")
|
94
|
+
aws_request.should_receive(:aws_emr_request).and_raise(RestClient::BadRequest)
|
95
|
+
Elasticity::AwsRequest.should_receive(:new).and_return(aws_request)
|
96
|
+
emr = Elasticity::EMR.new("aws_access_key_id", "aws_secret_key")
|
97
|
+
lambda {
|
98
|
+
emr.terminate_jobflows("invalid_jobflow_id")
|
99
|
+
}.should raise_error(ArgumentError)
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
describe Elasticity::JobFlow do
|
4
|
+
|
5
|
+
before do
|
6
|
+
describe_jobflows_xml = <<-JOBFLOWS
|
7
|
+
<DescribeJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
|
8
|
+
<DescribeJobFlowsResult>
|
9
|
+
<JobFlows>
|
10
|
+
<member>
|
11
|
+
<ExecutionStatusDetail>
|
12
|
+
<State>TERMINATED</State>
|
13
|
+
</ExecutionStatusDetail>
|
14
|
+
<JobFlowId>j-p</JobFlowId>
|
15
|
+
<Name>Pig Job</Name>
|
16
|
+
</member>
|
17
|
+
<member>
|
18
|
+
<ExecutionStatusDetail>
|
19
|
+
<State>TERMINATED</State>
|
20
|
+
</ExecutionStatusDetail>
|
21
|
+
<JobFlowId>j-h</JobFlowId>
|
22
|
+
<Name>Hive Job</Name>
|
23
|
+
</member>
|
24
|
+
</JobFlows>
|
25
|
+
</DescribeJobFlowsResult>
|
26
|
+
</DescribeJobFlowsResponse>
|
27
|
+
JOBFLOWS
|
28
|
+
@describe_jobflows_document = Nokogiri::XML(describe_jobflows_xml)
|
29
|
+
@describe_jobflows_document.remove_namespaces!
|
30
|
+
@members_nodeset = @describe_jobflows_document.xpath('/DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member')
|
31
|
+
end
|
32
|
+
|
33
|
+
describe ".from_xml" do
|
34
|
+
it "should return a JobFlow with the appropriate fields initialized" do
|
35
|
+
jobflow = Elasticity::JobFlow.from_member_element(@members_nodeset[0])
|
36
|
+
jobflow.name.should == "Pig Job"
|
37
|
+
jobflow.jobflow_id.should == "j-p"
|
38
|
+
jobflow.state.should == "TERMINATED"
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
describe ".from_jobflows_nodeset" do
|
43
|
+
it "should return JobFlows with the appropriate fields initialized" do
|
44
|
+
jobflow = Elasticity::JobFlow.from_members_nodeset(@members_nodeset)
|
45
|
+
jobflow.map(&:name).should == ["Pig Job", "Hive Job"]
|
46
|
+
jobflow.map(&:jobflow_id).should == ["j-p", "j-h"]
|
47
|
+
jobflow.map(&:state).should == ["TERMINATED", "TERMINATED"]
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require 'bundler/setup'
|
3
|
+
|
4
|
+
require 'vcr'
|
5
|
+
|
6
|
+
require 'elasticity'
|
7
|
+
|
8
|
+
ENV["RAILS_ENV"] ||= 'test'
|
9
|
+
|
10
|
+
$:.unshift File.dirname(__FILE__)
|
11
|
+
|
12
|
+
VCR.config do |c|
|
13
|
+
c.cassette_library_dir = 'spec/fixtures/vcr_cassettes'
|
14
|
+
c.stub_with :webmock
|
15
|
+
end
|
16
|
+
|
17
|
+
RSpec.configure do |c|
|
18
|
+
c.extend VCR::RSpec::Macros
|
19
|
+
end
|
metadata
ADDED
@@ -0,0 +1,178 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: elasticity
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 29
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 0
|
9
|
+
- 1
|
10
|
+
version: 0.0.1
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Robert Slifka
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2011-04-10 00:00:00 -07:00
|
19
|
+
default_executable:
|
20
|
+
dependencies:
|
21
|
+
- !ruby/object:Gem::Dependency
|
22
|
+
name: rest-client
|
23
|
+
prerelease: false
|
24
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
hash: 3
|
30
|
+
segments:
|
31
|
+
- 0
|
32
|
+
version: "0"
|
33
|
+
type: :runtime
|
34
|
+
version_requirements: *id001
|
35
|
+
- !ruby/object:Gem::Dependency
|
36
|
+
name: nokogiri
|
37
|
+
prerelease: false
|
38
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
hash: 3
|
44
|
+
segments:
|
45
|
+
- 0
|
46
|
+
version: "0"
|
47
|
+
type: :runtime
|
48
|
+
version_requirements: *id002
|
49
|
+
- !ruby/object:Gem::Dependency
|
50
|
+
name: rake
|
51
|
+
prerelease: false
|
52
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
53
|
+
none: false
|
54
|
+
requirements:
|
55
|
+
- - ">="
|
56
|
+
- !ruby/object:Gem::Version
|
57
|
+
hash: 3
|
58
|
+
segments:
|
59
|
+
- 0
|
60
|
+
version: "0"
|
61
|
+
type: :development
|
62
|
+
version_requirements: *id003
|
63
|
+
- !ruby/object:Gem::Dependency
|
64
|
+
name: rspec
|
65
|
+
prerelease: false
|
66
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
67
|
+
none: false
|
68
|
+
requirements:
|
69
|
+
- - ">="
|
70
|
+
- !ruby/object:Gem::Version
|
71
|
+
hash: 27
|
72
|
+
segments:
|
73
|
+
- 2
|
74
|
+
- 5
|
75
|
+
- 0
|
76
|
+
version: 2.5.0
|
77
|
+
type: :development
|
78
|
+
version_requirements: *id004
|
79
|
+
- !ruby/object:Gem::Dependency
|
80
|
+
name: vcr
|
81
|
+
prerelease: false
|
82
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
83
|
+
none: false
|
84
|
+
requirements:
|
85
|
+
- - ">="
|
86
|
+
- !ruby/object:Gem::Version
|
87
|
+
hash: 1
|
88
|
+
segments:
|
89
|
+
- 1
|
90
|
+
- 5
|
91
|
+
- 1
|
92
|
+
version: 1.5.1
|
93
|
+
type: :development
|
94
|
+
version_requirements: *id005
|
95
|
+
- !ruby/object:Gem::Dependency
|
96
|
+
name: webmock
|
97
|
+
prerelease: false
|
98
|
+
requirement: &id006 !ruby/object:Gem::Requirement
|
99
|
+
none: false
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
hash: 11
|
104
|
+
segments:
|
105
|
+
- 1
|
106
|
+
- 6
|
107
|
+
- 2
|
108
|
+
version: 1.6.2
|
109
|
+
type: :development
|
110
|
+
version_requirements: *id006
|
111
|
+
description: Programmatic access to Amazon's Elastic Map Reduce service.
|
112
|
+
email:
|
113
|
+
executables: []
|
114
|
+
|
115
|
+
extensions: []
|
116
|
+
|
117
|
+
extra_rdoc_files: []
|
118
|
+
|
119
|
+
files:
|
120
|
+
- .gitignore
|
121
|
+
- .rspec
|
122
|
+
- .rvmrc
|
123
|
+
- Gemfile
|
124
|
+
- README.mediawiki
|
125
|
+
- Rakefile
|
126
|
+
- elasticity.gemspec
|
127
|
+
- lib/elasticity.rb
|
128
|
+
- lib/elasticity/aws_request.rb
|
129
|
+
- lib/elasticity/emr.rb
|
130
|
+
- lib/elasticity/job_flow.rb
|
131
|
+
- lib/elasticity/version.rb
|
132
|
+
- spec/fixtures/vcr_cassettes/describe_jobflows/all_jobflows.yml
|
133
|
+
- spec/fixtures/vcr_cassettes/terminate_jobflows/one_jobflow.yml
|
134
|
+
- spec/lib/elasticity/aws_request_spec.rb
|
135
|
+
- spec/lib/elasticity/emr_spec.rb
|
136
|
+
- spec/lib/elasticity/job_flow_spec.rb
|
137
|
+
- spec/spec_helper.rb
|
138
|
+
has_rdoc: true
|
139
|
+
homepage: http://www.github.com/rslifka/elasticity
|
140
|
+
licenses: []
|
141
|
+
|
142
|
+
post_install_message:
|
143
|
+
rdoc_options: []
|
144
|
+
|
145
|
+
require_paths:
|
146
|
+
- lib
|
147
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
148
|
+
none: false
|
149
|
+
requirements:
|
150
|
+
- - ">="
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
hash: 3
|
153
|
+
segments:
|
154
|
+
- 0
|
155
|
+
version: "0"
|
156
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
157
|
+
none: false
|
158
|
+
requirements:
|
159
|
+
- - ">="
|
160
|
+
- !ruby/object:Gem::Version
|
161
|
+
hash: 3
|
162
|
+
segments:
|
163
|
+
- 0
|
164
|
+
version: "0"
|
165
|
+
requirements: []
|
166
|
+
|
167
|
+
rubyforge_project: elasticity
|
168
|
+
rubygems_version: 1.4.1
|
169
|
+
signing_key:
|
170
|
+
specification_version: 3
|
171
|
+
summary: Programmatic access to Amazon's Elastic Map Reduce service.
|
172
|
+
test_files:
|
173
|
+
- spec/fixtures/vcr_cassettes/describe_jobflows/all_jobflows.yml
|
174
|
+
- spec/fixtures/vcr_cassettes/terminate_jobflows/one_jobflow.yml
|
175
|
+
- spec/lib/elasticity/aws_request_spec.rb
|
176
|
+
- spec/lib/elasticity/emr_spec.rb
|
177
|
+
- spec/lib/elasticity/job_flow_spec.rb
|
178
|
+
- spec/spec_helper.rb
|