elasticity 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,5 @@
1
+ *.gem
2
+ .bundle
3
+ Gemfile.lock
4
+ pkg/*
5
+ .idea
data/.rspec ADDED
@@ -0,0 +1,2 @@
1
+ --color
2
+ --format progress
data/.rvmrc ADDED
@@ -0,0 +1 @@
1
+ rvm use ree-1.8.7-2010.02@elasticity --create
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source "http://rubygems.org"
2
+
3
+ # Specify your gem's dependencies in elasticity.gemspec
4
+ gemspec
@@ -0,0 +1,53 @@
1
+ = Elasticity =
2
+
3
+ Elasticity provides programmatic access to Amazon's Elastic Map Reduce service.
4
+
5
+ There are five API actions in the [http://docs.amazonwebservices.com/ElasticMapReduce/latest/DeveloperGuide/index.html EMR API], described below. Please see the Amazon guide for details on these operations because the default values aren't obvious (e.g. the meaning of DescribeJobFlows without parameters).
6
+
7
+ '''NOTE''': The AWS plumbing (especially signing/authentication) was used from [http://www.rightscale.com/ RightScale's] amazing [https://github.com/rightscale/right_aws right_aws gem] which works extraordinarily well!
8
+
9
+ == AddInstanceGroups ==
10
+
11
+ ''(not yet supported)''
12
+
13
+ == AddJobFlowSteps ==
14
+
15
+ ''(not yet supported)''
16
+
17
+ == DescribeJobFlows ==
18
+
19
+ <pre>
20
+ > emr = Elasticity::EMR.new(ENV["aws_access_key_id"], ENV["aws_secret_key"])
21
+ > jobflows = emr.describe_jobflows
22
+ > p jobflows.map(&:name)
23
+ ["Hive Test", "Pig Test", "Interactive Hadoop", "Interactive Hive"]
24
+ </pre>
25
+
26
+ == ModifyInstanceGroups ==
27
+
28
+ ''(not yet supported)''
29
+
30
+ == RunJobFlow ==
31
+
32
+ ''(not yet supported)''
33
+
34
+ == TerminateJobFlows ==
35
+
36
+ When the job flow '''exists''', you will receive no output. This is because Amazon does not return anything other than a 200 when you terminate a job flow :) You'll want to continually poll with DescribeJobFlows to see when the job was actually terminated.
37
+
38
+ <pre>
39
+ > emr = Elasticity::EMR.new(ENV["aws_access_key_id"], ENV["aws_secret_key"])
40
+ > emr.terminate_jobflows("no-flow")
41
+ >
42
+ </pre>
43
+
44
+ When the job flow '''doesn't exist''':
45
+
46
+ <pre>
47
+ > emr = Elasticity::EMR.new(ENV["aws_access_key_id"], ENV["aws_secret_key"])
48
+ > emr.terminate_jobflows("no-flow")
49
+ ArgumentError: Job flow 'no-flow' does not exist.
50
+ </pre>
51
+
52
+
53
+ <!-- __NOTOC__ -->
@@ -0,0 +1,11 @@
1
+ require 'bundler'
2
+ Bundler::GemHelper.install_tasks
3
+
4
+ require 'rake/testtask'
5
+ require 'rspec/core/rake_task'
6
+
7
+ desc 'Run specs'
8
+ task :default => :spec
9
+
10
+ desc "Run specs"
11
+ RSpec::Core::RakeTask.new
@@ -0,0 +1,28 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+ require "elasticity/version"
4
+
5
+ Gem::Specification.new do |s|
6
+ s.name = "elasticity"
7
+ s.version = Elasticity::VERSION
8
+ s.platform = Gem::Platform::RUBY
9
+ s.authors = ["Robert Slifka"]
10
+ s.homepage = "http://www.github.com/rslifka/elasticity"
11
+ s.summary = %q{Programmatic access to Amazon's Elastic Map Reduce service.}
12
+ s.description = %q{Programmatic access to Amazon's Elastic Map Reduce service.}
13
+
14
+ s.rubyforge_project = "elasticity"
15
+
16
+ s.add_dependency("rest-client")
17
+ s.add_dependency("nokogiri")
18
+
19
+ s.add_development_dependency("rake")
20
+ s.add_development_dependency("rspec", ">= 2.5.0")
21
+ s.add_development_dependency("vcr", ">= 1.5.1")
22
+ s.add_development_dependency("webmock", ">= 1.6.2")
23
+
24
+ s.files = `git ls-files`.split("\n")
25
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
26
+ s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
27
+ s.require_paths = ["lib"]
28
+ end
@@ -0,0 +1,9 @@
1
+ require 'rest_client'
2
+ require 'nokogiri'
3
+
4
+ require 'elasticity/aws_request'
5
+ require 'elasticity/emr'
6
+ require 'elasticity/job_flow'
7
+
8
+ module Elasticity
9
+ end
@@ -0,0 +1,48 @@
1
+ module Elasticity
2
+
3
+ class AwsRequest
4
+
5
+ def initialize(aws_access_key_id, aws_secret_access_key)
6
+ @access_key = aws_access_key_id
7
+ @secret_key = aws_secret_access_key
8
+ end
9
+
10
+ def aws_emr_request(params)
11
+ signed_params = sign_params(params, "GET", "elasticmapreduce.amazonaws.com", "/")
12
+ signed_request = "http://elasticmapreduce.amazonaws.com?#{signed_params}"
13
+ RestClient.get signed_request
14
+ end
15
+
16
+ # (Used from RightScale's right_aws gem.)
17
+ # EC2, SQS, SDB and EMR requests must be signed by this guy.
18
+ # See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
19
+ # http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1928
20
+ def sign_params(service_hash, http_verb, host, uri)
21
+ service_hash["AWSAccessKeyId"] = @access_key
22
+ service_hash["Timestamp"] = Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z")
23
+ service_hash["SignatureVersion"] = "2"
24
+ service_hash['SignatureMethod'] = 'HmacSHA256'
25
+ canonical_string = service_hash.keys.sort.map do |key|
26
+ "#{AwsRequest.aws_escape(key)}=#{AwsRequest.aws_escape(service_hash[key])}"
27
+ end.join('&')
28
+ string_to_sign = "#{http_verb.to_s.upcase}\n#{host.downcase}\n#{uri}\n#{canonical_string}"
29
+ signature = AwsRequest.aws_escape(Base64.encode64(OpenSSL::HMAC.digest("sha256", @secret_key, string_to_sign)).strip)
30
+ "#{canonical_string}&Signature=#{signature}"
31
+ end
32
+
33
+ class << self
34
+
35
+ # (Used from RightScale's right_aws gem)
36
+ # Escape a string according to Amazon's rules.
37
+ # See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
38
+ def aws_escape(param)
39
+ param.to_s.gsub(/([^a-zA-Z0-9._~-]+)/n) do
40
+ '%' + $1.unpack('H2' * $1.size).join('%').upcase
41
+ end
42
+ end
43
+
44
+ end
45
+
46
+ end
47
+
48
+ end
@@ -0,0 +1,35 @@
1
+ require "base64"
2
+
3
+ module Elasticity
4
+
5
+ class EMR
6
+
7
+ def initialize(aws_access_key_id, aws_secret_access_key)
8
+ @aws_request = Elasticity::AwsRequest.new(aws_access_key_id, aws_secret_access_key)
9
+ end
10
+
11
+ # Lists all jobflows in all states.
12
+ def describe_jobflows
13
+ aws_result = @aws_request.aws_emr_request({"Operation" => "DescribeJobFlows"})
14
+ xml_doc = Nokogiri::XML(aws_result)
15
+ xml_doc.remove_namespaces!
16
+ JobFlow.from_members_nodeset(xml_doc.xpath("/DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member"))
17
+ end
18
+
19
+ # Terminate the specified jobflow. Amazon does not define a return value
20
+ # for this operation, so you'll need to poll #describe_jobflows to see
21
+ # the state of the jobflow. Raises ArgumentError if the specified job
22
+ # flow does not exist.
23
+ def terminate_jobflows(jobflow_id)
24
+ begin
25
+ @aws_request.aws_emr_request({
26
+ "Operation" => "TerminateJobFlows",
27
+ "JobFlowIds.member.1" => jobflow_id
28
+ })
29
+ rescue RestClient::BadRequest
30
+ raise ArgumentError, "Job flow '#{jobflow_id}' does not exist."
31
+ end
32
+ end
33
+
34
+ end
35
+ end
@@ -0,0 +1,35 @@
1
+ module Elasticity
2
+
3
+ class JobFlow
4
+
5
+ attr_accessor :name
6
+ attr_accessor :jobflow_id
7
+ attr_accessor :state
8
+
9
+ class << self
10
+
11
+ # Create a jobflow from an AWS <member> (Nokogiri::XML::Element):
12
+ # /DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member
13
+ def from_member_element(xml_element)
14
+ jobflow = JobFlow.new
15
+ jobflow.name = xml_element.xpath("./Name").text
16
+ jobflow.jobflow_id = xml_element.xpath("./JobFlowId").text
17
+ jobflow.state = xml_element.xpath("./ExecutionStatusDetail/State").text
18
+ jobflow
19
+ end
20
+
21
+ # Create JobFlows from a collection of AWS <member> nodes (Nokogiri::XML::NodeSet):
22
+ # /DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows
23
+ def from_members_nodeset(members_nodeset)
24
+ jobflows = []
25
+ members_nodeset.each do |member|
26
+ jobflows << from_member_element(member)
27
+ end
28
+ jobflows
29
+ end
30
+
31
+ end
32
+
33
+ end
34
+
35
+ end
@@ -0,0 +1,3 @@
1
+ module Elasticity
2
+ VERSION = "0.0.1"
3
+ end
@@ -0,0 +1,69 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^http:\/\/elasticmapreduce.amazonaws.com:80\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&Operation=DescribeJobFlows/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 200
15
+ message: OK
16
+ headers:
17
+ x-amzn-requestid:
18
+ - 994deb7a-63ac-11e0-bc41-ababd98a870b
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Sun, 10 Apr 2011 19:56:17 GMT
23
+ content-encoding:
24
+ - gzip
25
+ vary:
26
+ - Accept-Encoding
27
+ transfer-encoding:
28
+ - chunked
29
+ body: !binary |
30
+ H4sIAAAAAAAAAO1bW3PaOhB+769gztM5c0bYEld7XGZIgIQQIAXnMnnJyLYC
31
+ 7vGttpw2/fVHtmOwjQkQIKUpM0wSS7urlfTp0+6aSC3iqa6ukAtb6Rj2d29E
32
+ PMe2PFL4YRqW9/mvKaWOyHHEwB7VVRM7LtF8lRSxiX/aFv7uFVXb5DRb5RDP
33
+ C4AvgRL8q/GpUJByLPsGDbpYZ9wYPbIGk5gKceNH1tD+QVSf6rY1ppj6XotQ
34
+ rBvzfiZx6hIcCLQwJbJukgbiIQR8mX1kWBPLUKzAe4lbEEvauGTTCgYgp1Ns
35
+ TciIYM+2GjJxTd1irVpBeS74HnELLvnmE49KXL5G0ibrdekypypiSWBOpWWS
36
+ 2syg9rxCOy2TGZuShtwe9buDptxuhSPRtEzb0hbtCzKsiuzDB0uWlJjvCLdi
37
+ S6QT26YedbHTVAMpj0v0DTCzddv/dzSWuPDvTwmfieOlPMyCYSZ2aluP+iTd
38
+ wbrOsWbbzgV2A5lsb4A27Da8Ug6MOUNXPC7AqUOB61sWcdNPxa/YlbhAf9Fq
39
+ 0514i81z918Zcao/kfAHiEaTuLwpp80BoGCPAAfT6Triq0Zfb0jd8ig2DBCo
40
+ LNeQuPy1kK5c2yEu1UkSC7HOq7sWAWZMqO8UzsPB06iJ9yBE2tDqMCT6bgL5
41
+ DxfDk4fO5fCW+ZaRScOKW4ar1QwUSu2AhUI7W7FGbIGd9NNh/+qynX/0o2nl
42
+ Hf9oGMYAlWXHP1qs1ay8AJJohRPoiMm/qzW+Ati/r8g3t52L0bg3YCdt1jUX
43
+ 7wYYtFSS5og+QzZx4z6moANcrdf5R1WQuIXOpOaVgVViEotmJtd8YnPBim7o
44
+ 9PnetkjD9wBjdwqgykCU7UzNOdfkzPEz1/adzPHI5bgdwikee8RoTLcmp7bP
45
+ nOMlLrd9IyyWRVRdicVVt1jttVss9oGBtz1obYjk1RdZdoWimz1aCZhYoWT7
46
+ MmX52SENExY9k7HkXDdszur0sfsfoY3h4KHV7jcHrQCmYUtWMD/EYEej8MjO
47
+ RoHOopM1opH0VG2DNPrNMWPJxDyDxmUaIXSDwzUBSDjv8L0xX+1enXyZq8cS
48
+ ufxthucwj7vzLpOM0fR5H9guW2P9J9FioXPbd70GhOy8L+tMGoiumxvieuzc
49
+ NPgi4uMrKG5bTi+Zbc7pXNS98hVDV1uWF06eqAgghk9UqgIo1ACCMIicHZ8S
50
+ ANPRdGw/bSAVwamoR57DZm+KXUKnbL0m0weNPDHMzzvzyGgB5wsAl3qEOC9E
51
+ 3DTY3Xs7JdbAjlicuj67i1+TSBD/AnNnd32HYX+QeMiwLvIVEQqHEvbPnKqL
52
+ EG4c9me19xD2Q5mHIiq/cu/vIOzvWgzNmHU+kULT13TCIBEGdQU5XNtjPnDM
53
+ Bz56PrAJPYV2tqKT2MLb8oHEMEh453ygVG9fo7v7L1dC76K8ZT7A16tlDVZr
54
+ O88HlF+eD2wMp53nAzMPaiKsvSkfSKKZ31s+sPqGy67QLvIBA7sT8rHzAbl6
55
+ 1ul1eHR+17s5nHSghN4vG3jZ5DdmA7UKgDwEqFIGAjrmArvPBWpipRKVAQ4o
56
+ F2BOCWKZf2MuMNfedS4wI+OyiEp7zAVm8f/fQRT3zzH4Pwb/hT8k+F+Pj0I7
57
+ W/FHbGHL4J8XYXnnwf/xPOebA4FLYGMlFoB4a0k+rscSVt6kPGw6BonmBbDm
58
+ RbM0bY0YQPF1Qyt+W8sHbR2p7uDqWv68ricUK+x5d6MPr+X58Owei3aEslt/
59
+ PqbtUxapcfPTsrvhL7sn47XnHuzCL2BhlihGtbvxC04/OBUvT742z7tezaNf
60
+ +HYtWm8OTtuXl0tofQ9VGSS3Tm/O2u3zYacrbFmVEaq1qlp63P1b2kOpymwA
61
+ rj1VZZgH7FPaoioThhml+n6qMmvlGtkVOr6lXe8t7eBu3C+1Bmd3/HXncMoy
62
+ 8Hd5R1vhAawBWEEA8r9lUeYRG95hV2VQRUT1Q6vKlKCI0JurMrH2fqoyjIyR
63
+ iGrvUZUZEcdmwaE1ORZmjoWZwh9TmFmHkkI7W1FIbGGbwkw4TFj/ec+3soNe
64
+ 70sZCa16pd7dMvwvE0FDdQQ/4Jc0N0bT3sJ/JLAcZKvwPwDzq985in14a/i/
65
+ 4lLLrtDW4b8KiybRdN/82PE/7JVu7+XhqDk6GQqHE/+/11vZxC6/MQGAtTKA
66
+ SAAQBr+rv2UOsMsXszO6D/slbvm/dUnx/4712X2jYYpfDLycUYY+QShrRKlh
67
+ UC1hla0w4YGiliHAClY0oY7rNV4JKCeWDwdctJrrRCjT+PQ/yLkKP9E2AAA=
68
+
69
+ http_version: "1.1"
@@ -0,0 +1,32 @@
1
+ ---
2
+ - !ruby/struct:VCR::HTTPInteraction
3
+ request: !ruby/struct:VCR::Request
4
+ method: :get
5
+ uri: !ruby/regexp /^http:\/\/elasticmapreduce.amazonaws.com:80\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&JobFlowIds.member.1=j-1MZ5TVWFJRSKN&Operation=TerminateJobFlows/
6
+ body:
7
+ headers:
8
+ accept:
9
+ - "*/*; q=0.5, application/xml"
10
+ accept-encoding:
11
+ - gzip, deflate
12
+ response: !ruby/struct:VCR::Response
13
+ status: !ruby/struct:VCR::ResponseStatus
14
+ code: 200
15
+ message: OK
16
+ headers:
17
+ x-amzn-requestid:
18
+ - 83e0154d-63c6-11e0-bc41-ababd98a870b
19
+ content-type:
20
+ - text/xml
21
+ date:
22
+ - Sun, 10 Apr 2011 23:01:48 GMT
23
+ content-length:
24
+ - "225"
25
+ body: |
26
+ <TerminateJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
27
+ <ResponseMetadata>
28
+ <RequestId>83e0154d-63c6-11e0-bc41-ababd98a870b</RequestId>
29
+ </ResponseMetadata>
30
+ </TerminateJobFlowsResponse>
31
+
32
+ http_version: "1.1"
@@ -0,0 +1,27 @@
1
+ require 'spec_helper'
2
+
3
+ describe Elasticity::AwsRequest do
4
+
5
+ describe ".aws_escape" do
6
+ it "should escape according to Amazon's rules" do
7
+ # Don't encode reserved characters
8
+ Elasticity::AwsRequest.aws_escape("foo-_.~bar").should == "foo-_.~bar"
9
+ # Encode as %20, not as +
10
+ Elasticity::AwsRequest.aws_escape("foo bar").should == "foo%20bar"
11
+ # Percent encode all other characters with %XY, where X and Y are hex characters 0-9 and uppercase A-F.
12
+ Elasticity::AwsRequest.aws_escape("foo$&+,/:;=?@bar").should == "foo%24%26%2B%2C%2F%3A%3B%3D%3F%40bar"
13
+ end
14
+ end
15
+
16
+ describe "#sign_params" do
17
+ before do
18
+ Time.should_receive(:now).and_return(Time.at(1302461096))
19
+ end
20
+ it "should sign according to Amazon's rules" do
21
+ request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_access_key")
22
+ signed_params = request.send(:sign_params, {}, "GET", "example.com", "/")
23
+ signed_params.should == "AWSAccessKeyId=aws_access_key_id&SignatureMethod=HmacSHA256&SignatureVersion=2&Timestamp=2011-04-10T18%3A44%3A56.000Z&Signature=jVLfPS056dNmjpCcikBnPmRHJNZ8YGaI7zdmHWUk658%3D"
24
+ end
25
+ end
26
+
27
+ end
@@ -0,0 +1,106 @@
1
+ require 'spec_helper'
2
+
3
+ describe Elasticity::EMR do
4
+
5
+ describe "#describe_jobflows" do
6
+
7
+ describe "integration happy path" do
8
+ use_vcr_cassette "describe_jobflows/all_jobflows", :record => :none
9
+ it "should return the names of all running job flows" do
10
+ emr = Elasticity::EMR.new(ENV["aws_access_key_id"], ENV["aws_secret_key"])
11
+ jobflows = emr.describe_jobflows
12
+ jobflows.map(&:name).should == ["WM+RS", "Interactive Audience Hive Test", "Audience (Hive)", "Audience Reporting"]
13
+ jobflows.map(&:jobflow_id).should == ["j-1MZ5TVWFJRSKN", "j-38EU2XZQP9KJ4", "j-2TDCVGEEHOFI9", "j-NKKQ429D858I"]
14
+ jobflows.map(&:state).should == ["TERMINATED", "TERMINATED", "TERMINATED", "TERMINATED"]
15
+ end
16
+ end
17
+
18
+ describe "unit tests" do
19
+ before do
20
+ @describe_jobflows_xml = <<-JOBFLOWS
21
+ <DescribeJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
22
+ <DescribeJobFlowsResult>
23
+ <JobFlows>
24
+ <member>
25
+ <ExecutionStatusDetail>
26
+ <State>TERMINATED</State>
27
+ </ExecutionStatusDetail>
28
+ <JobFlowId>j-p</JobFlowId>
29
+ <Name>Pig Job</Name>
30
+ </member>
31
+ <member>
32
+ <ExecutionStatusDetail>
33
+ <State>TERMINATED</State>
34
+ </ExecutionStatusDetail>
35
+ <JobFlowId>j-h</JobFlowId>
36
+ <Name>Hive Job</Name>
37
+ </member>
38
+ </JobFlows>
39
+ </DescribeJobFlowsResult>
40
+ </DescribeJobFlowsResponse>
41
+ JOBFLOWS
42
+ end
43
+ it "should return the names of all running job flows" do
44
+ aws_request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_key")
45
+ aws_request.should_receive(:aws_emr_request).with({"Operation" => "DescribeJobFlows"}).and_return(@describe_jobflows_xml)
46
+ Elasticity::AwsRequest.should_receive(:new).and_return(aws_request)
47
+ emr = Elasticity::EMR.new("aws_access_key_id", "aws_secret_key")
48
+ jobflows = emr.describe_jobflows
49
+ jobflows.map(&:name).should == ["Pig Job", "Hive Job"]
50
+ end
51
+ end
52
+
53
+ end
54
+
55
+ describe "#terminate_jobflows" do
56
+
57
+ describe "integration happy path" do
58
+ context "when the job flow exists" do
59
+ use_vcr_cassette "terminate_jobflows/one_jobflow", :record => :none
60
+ it "should terminate the specified jobflow" do
61
+ emr = Elasticity::EMR.new(ENV["aws_access_key_id"], ENV["aws_secret_key"])
62
+ emr.terminate_jobflows("j-1MZ5TVWFJRSKN")
63
+ end
64
+ end
65
+ end
66
+
67
+ describe "unit tests" do
68
+
69
+ context "when the jobflow exists" do
70
+ before do
71
+ @terminate_jobflows_xml = <<-RESPONSE
72
+ <TerminateJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
73
+ <ResponseMetadata>
74
+ <RequestId>2690d7eb-ed86-11dd-9877-6fad448a8419</RequestId>
75
+ </ResponseMetadata>
76
+ </TerminateJobFlowsResponse>
77
+ RESPONSE
78
+ end
79
+ it "should terminate the specific jobflow" do
80
+ aws_request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_key")
81
+ aws_request.should_receive(:aws_emr_request).with({
82
+ "Operation" => "TerminateJobFlows",
83
+ "JobFlowIds.member.1" => "j-1"
84
+ }).and_return(@terminate_jobflows_xml)
85
+ Elasticity::AwsRequest.should_receive(:new).and_return(aws_request)
86
+ emr = Elasticity::EMR.new("aws_access_key_id", "aws_secret_key")
87
+ emr.terminate_jobflows("j-1")
88
+ end
89
+ end
90
+
91
+ context "when the jobflow does not exist" do
92
+ it "should terminate the specific jobflow" do
93
+ aws_request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_key")
94
+ aws_request.should_receive(:aws_emr_request).and_raise(RestClient::BadRequest)
95
+ Elasticity::AwsRequest.should_receive(:new).and_return(aws_request)
96
+ emr = Elasticity::EMR.new("aws_access_key_id", "aws_secret_key")
97
+ lambda {
98
+ emr.terminate_jobflows("invalid_jobflow_id")
99
+ }.should raise_error(ArgumentError)
100
+ end
101
+ end
102
+
103
+ end
104
+ end
105
+
106
+ end
@@ -0,0 +1,51 @@
1
+ require 'spec_helper'
2
+
3
+ describe Elasticity::JobFlow do
4
+
5
+ before do
6
+ describe_jobflows_xml = <<-JOBFLOWS
7
+ <DescribeJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
8
+ <DescribeJobFlowsResult>
9
+ <JobFlows>
10
+ <member>
11
+ <ExecutionStatusDetail>
12
+ <State>TERMINATED</State>
13
+ </ExecutionStatusDetail>
14
+ <JobFlowId>j-p</JobFlowId>
15
+ <Name>Pig Job</Name>
16
+ </member>
17
+ <member>
18
+ <ExecutionStatusDetail>
19
+ <State>TERMINATED</State>
20
+ </ExecutionStatusDetail>
21
+ <JobFlowId>j-h</JobFlowId>
22
+ <Name>Hive Job</Name>
23
+ </member>
24
+ </JobFlows>
25
+ </DescribeJobFlowsResult>
26
+ </DescribeJobFlowsResponse>
27
+ JOBFLOWS
28
+ @describe_jobflows_document = Nokogiri::XML(describe_jobflows_xml)
29
+ @describe_jobflows_document.remove_namespaces!
30
+ @members_nodeset = @describe_jobflows_document.xpath('/DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member')
31
+ end
32
+
33
+ describe ".from_xml" do
34
+ it "should return a JobFlow with the appropriate fields initialized" do
35
+ jobflow = Elasticity::JobFlow.from_member_element(@members_nodeset[0])
36
+ jobflow.name.should == "Pig Job"
37
+ jobflow.jobflow_id.should == "j-p"
38
+ jobflow.state.should == "TERMINATED"
39
+ end
40
+ end
41
+
42
+ describe ".from_jobflows_nodeset" do
43
+ it "should return JobFlows with the appropriate fields initialized" do
44
+ jobflow = Elasticity::JobFlow.from_members_nodeset(@members_nodeset)
45
+ jobflow.map(&:name).should == ["Pig Job", "Hive Job"]
46
+ jobflow.map(&:jobflow_id).should == ["j-p", "j-h"]
47
+ jobflow.map(&:state).should == ["TERMINATED", "TERMINATED"]
48
+ end
49
+ end
50
+
51
+ end
@@ -0,0 +1,19 @@
1
+ require 'rubygems'
2
+ require 'bundler/setup'
3
+
4
+ require 'vcr'
5
+
6
+ require 'elasticity'
7
+
8
+ ENV["RAILS_ENV"] ||= 'test'
9
+
10
+ $:.unshift File.dirname(__FILE__)
11
+
12
+ VCR.config do |c|
13
+ c.cassette_library_dir = 'spec/fixtures/vcr_cassettes'
14
+ c.stub_with :webmock
15
+ end
16
+
17
+ RSpec.configure do |c|
18
+ c.extend VCR::RSpec::Macros
19
+ end
metadata ADDED
@@ -0,0 +1,178 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: elasticity
3
+ version: !ruby/object:Gem::Version
4
+ hash: 29
5
+ prerelease:
6
+ segments:
7
+ - 0
8
+ - 0
9
+ - 1
10
+ version: 0.0.1
11
+ platform: ruby
12
+ authors:
13
+ - Robert Slifka
14
+ autorequire:
15
+ bindir: bin
16
+ cert_chain: []
17
+
18
+ date: 2011-04-10 00:00:00 -07:00
19
+ default_executable:
20
+ dependencies:
21
+ - !ruby/object:Gem::Dependency
22
+ name: rest-client
23
+ prerelease: false
24
+ requirement: &id001 !ruby/object:Gem::Requirement
25
+ none: false
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ hash: 3
30
+ segments:
31
+ - 0
32
+ version: "0"
33
+ type: :runtime
34
+ version_requirements: *id001
35
+ - !ruby/object:Gem::Dependency
36
+ name: nokogiri
37
+ prerelease: false
38
+ requirement: &id002 !ruby/object:Gem::Requirement
39
+ none: false
40
+ requirements:
41
+ - - ">="
42
+ - !ruby/object:Gem::Version
43
+ hash: 3
44
+ segments:
45
+ - 0
46
+ version: "0"
47
+ type: :runtime
48
+ version_requirements: *id002
49
+ - !ruby/object:Gem::Dependency
50
+ name: rake
51
+ prerelease: false
52
+ requirement: &id003 !ruby/object:Gem::Requirement
53
+ none: false
54
+ requirements:
55
+ - - ">="
56
+ - !ruby/object:Gem::Version
57
+ hash: 3
58
+ segments:
59
+ - 0
60
+ version: "0"
61
+ type: :development
62
+ version_requirements: *id003
63
+ - !ruby/object:Gem::Dependency
64
+ name: rspec
65
+ prerelease: false
66
+ requirement: &id004 !ruby/object:Gem::Requirement
67
+ none: false
68
+ requirements:
69
+ - - ">="
70
+ - !ruby/object:Gem::Version
71
+ hash: 27
72
+ segments:
73
+ - 2
74
+ - 5
75
+ - 0
76
+ version: 2.5.0
77
+ type: :development
78
+ version_requirements: *id004
79
+ - !ruby/object:Gem::Dependency
80
+ name: vcr
81
+ prerelease: false
82
+ requirement: &id005 !ruby/object:Gem::Requirement
83
+ none: false
84
+ requirements:
85
+ - - ">="
86
+ - !ruby/object:Gem::Version
87
+ hash: 1
88
+ segments:
89
+ - 1
90
+ - 5
91
+ - 1
92
+ version: 1.5.1
93
+ type: :development
94
+ version_requirements: *id005
95
+ - !ruby/object:Gem::Dependency
96
+ name: webmock
97
+ prerelease: false
98
+ requirement: &id006 !ruby/object:Gem::Requirement
99
+ none: false
100
+ requirements:
101
+ - - ">="
102
+ - !ruby/object:Gem::Version
103
+ hash: 11
104
+ segments:
105
+ - 1
106
+ - 6
107
+ - 2
108
+ version: 1.6.2
109
+ type: :development
110
+ version_requirements: *id006
111
+ description: Programmatic access to Amazon's Elastic Map Reduce service.
112
+ email:
113
+ executables: []
114
+
115
+ extensions: []
116
+
117
+ extra_rdoc_files: []
118
+
119
+ files:
120
+ - .gitignore
121
+ - .rspec
122
+ - .rvmrc
123
+ - Gemfile
124
+ - README.mediawiki
125
+ - Rakefile
126
+ - elasticity.gemspec
127
+ - lib/elasticity.rb
128
+ - lib/elasticity/aws_request.rb
129
+ - lib/elasticity/emr.rb
130
+ - lib/elasticity/job_flow.rb
131
+ - lib/elasticity/version.rb
132
+ - spec/fixtures/vcr_cassettes/describe_jobflows/all_jobflows.yml
133
+ - spec/fixtures/vcr_cassettes/terminate_jobflows/one_jobflow.yml
134
+ - spec/lib/elasticity/aws_request_spec.rb
135
+ - spec/lib/elasticity/emr_spec.rb
136
+ - spec/lib/elasticity/job_flow_spec.rb
137
+ - spec/spec_helper.rb
138
+ has_rdoc: true
139
+ homepage: http://www.github.com/rslifka/elasticity
140
+ licenses: []
141
+
142
+ post_install_message:
143
+ rdoc_options: []
144
+
145
+ require_paths:
146
+ - lib
147
+ required_ruby_version: !ruby/object:Gem::Requirement
148
+ none: false
149
+ requirements:
150
+ - - ">="
151
+ - !ruby/object:Gem::Version
152
+ hash: 3
153
+ segments:
154
+ - 0
155
+ version: "0"
156
+ required_rubygems_version: !ruby/object:Gem::Requirement
157
+ none: false
158
+ requirements:
159
+ - - ">="
160
+ - !ruby/object:Gem::Version
161
+ hash: 3
162
+ segments:
163
+ - 0
164
+ version: "0"
165
+ requirements: []
166
+
167
+ rubyforge_project: elasticity
168
+ rubygems_version: 1.4.1
169
+ signing_key:
170
+ specification_version: 3
171
+ summary: Programmatic access to Amazon's Elastic Map Reduce service.
172
+ test_files:
173
+ - spec/fixtures/vcr_cassettes/describe_jobflows/all_jobflows.yml
174
+ - spec/fixtures/vcr_cassettes/terminate_jobflows/one_jobflow.yml
175
+ - spec/lib/elasticity/aws_request_spec.rb
176
+ - spec/lib/elasticity/emr_spec.rb
177
+ - spec/lib/elasticity/job_flow_spec.rb
178
+ - spec/spec_helper.rb