elasticity 0.3.5 → 0.4

data/README.mediawiki

@@ -6,6 +6,8 @@ Each Elasticity call provides some level of wrapping to save you the trouble of
 
 Alternatively, you may opt for "direct" access to the API where you specify the params and Elasticity takes care of the signing for you, responding with the XML from Amazon.  Direct access is described below the API catalog.
 
+'''BACKLOG''': Have a look at the backlog  [https://www.pivotaltracker.com/projects/272429 here] (Pivotal Tracker).
+
 '''CREDITS''': AWS signing was used from [http://www.rightscale.com/ RightScale's] amazing [https://github.com/rightscale/right_aws right_aws gem] which works extraordinarily well!  If you need access to any AWS service (EC2, S3, etc.), have a look.
 
 == AddInstanceGroups ==
@@ -29,7 +31,29 @@ DescribeJobFlows returns detailed information as to the state of all jobs.  Curr
 
 == ModifyInstanceGroups ==
 
-''(not yet supported)''
+A job flow contains several "instance groups" of various types.  These instances are where the work for your EMR task occurs.  After a job flow has been created, you can find these instance groups in the AWS web UI by clicking on a job flow and then clicking on the "Instance Groups" tab.
+
+<pre>
+  > emr = Elasticity::EMR.new(ENV["aws_access_key_id"], ENV["aws_secret_key"])
+  > emr.modify_instance_groups({"ig-2T1HNUO61BG3O" => 3})
+  >
+</pre>
+
+If there's an error, you'll receive an ArgumentError containing the message from Amazon.  For example if you attempt to modify an instance group that's part of a terminated job flow:
+
+<pre>
+  > emr = Elasticity::EMR.new(ENV["aws_access_key_id"], ENV["aws_secret_key"])
+  > emr.modify_instance_groups({"ig-some_terminated_group" => 3})
+  ArgumentError: An instance group may only be modified when the job flow is running or waiting
+</pre>
+
+Or if you attempt to increase the instance count of the MASTER instance group:
+
+<pre>
+  > emr = Elasticity::EMR.new(ENV["aws_access_key_id"], ENV["aws_secret_key"])
+  > emr.modify_instance_groups({"ig-some_terminated_group" => 3})
+  ArgumentError: A master instance group may not be modified
+</pre>
 
 == RunJobFlow ==
 

data/lib/elasticity/emr.rb

@@ -15,6 +15,30 @@ module Elasticity
       JobFlow.from_members_nodeset(xml_doc.xpath("/DescribeJobFlowsResponse/DescribeJobFlowsResult/JobFlows/member"))
     end
 
+    # Set the number of instances in the specified instance groups to the
+    # specified counts.  Note that this modifies the *request* count, which
+    # is not the same as the *running* count.  I.e. you request instances
+    # and then wait for them to be created.
+    #
+    # Takes a {} of instance group IDs => desired instance count.
+    #
+    # {"ig-1" => 40, "ig-2" => 5", ...}
+    def modify_instance_groups(instance_group_config)
+      params = {"Operation" => "ModifyInstanceGroups"}
+      instance_group_config.keys.each_with_index do |instance_group, index|
+        params.merge!(
+          "InstanceGroups.member.#{index+1}.InstanceGroupId" => instance_group,
+          "InstanceGroups.member.#{index+1}.InstanceCount" => instance_group_config[instance_group]
+        )
+      end
+      begin
+        aws_result = @aws_request.aws_emr_request(params)
+        yield aws_result if block_given?
+      rescue RestClient::BadRequest => e
+        raise ArgumentError, parse_error_response(e.http_body)
+      end
+    end
+
     # Terminate the specified jobflow.  Amazon does not define a return value
     # for this operation, so you'll need to poll #describe_jobflows to see
     # the state of the jobflow.  Raises ArgumentError if the specified job
@@ -39,5 +63,13 @@ module Elasticity
       @aws_request.aws_emr_request(params)
     end
 
+    private
+
+    def parse_error_response(error_xml)
+      xml_doc = Nokogiri::XML(error_xml)
+      xml_doc.remove_namespaces!
+      xml_doc.xpath("/ErrorResponse/Error/Message").text
+    end
+
   end
 end

data/lib/elasticity/version.rb

@@ -1,3 +1,3 @@
 module Elasticity
-  VERSION = "0.3.5"
+  VERSION = "0.4"
 end

data/spec/fixtures/vcr_cassettes/modify_instance_groups/set_instances_to_3.yml

@@ -0,0 +1,32 @@
+--- 
+- !ruby/struct:VCR::HTTPInteraction 
+  request: !ruby/struct:VCR::Request 
+    method: :get
+    uri: !ruby/regexp /^http:\/\/elasticmapreduce.amazonaws.com:80\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&InstanceGroups.member.1.InstanceCount=2&InstanceGroups.member.1.InstanceGroupId=ig-2T1HNUO61BG3O&Operation=ModifyInstanceGroups/
+    body: 
+    headers: 
+      accept: 
+      - "*/*; q=0.5, application/xml"
+      accept-encoding: 
+      - gzip, deflate
+  response: !ruby/struct:VCR::Response 
+    status: !ruby/struct:VCR::ResponseStatus 
+      code: 200
+      message: OK
+    headers: 
+      x-amzn-requestid: 
+      - 4ef75373-659c-11e0-bdf6-e3d62a364c28
+      content-type: 
+      - text/xml
+      date: 
+      - Wed, 13 Apr 2011 07:04:43 GMT
+      content-length: 
+      - "231"
+    body: |
+      <ModifyInstanceGroupsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
+        <ResponseMetadata>
+          <RequestId>4ef75373-659c-11e0-bdf6-e3d62a364c28</RequestId>
+        </ResponseMetadata>
+      </ModifyInstanceGroupsResponse>
+
+    http_version: "1.1"

data/spec/lib/elasticity/emr_spec.rb

@@ -82,6 +82,7 @@ describe Elasticity::EMR do
     describe "unit tests" do
 
       context "when the jobflow exists" do
+
         before do
           @terminate_jobflows_xml = <<-RESPONSE
             <TerminateJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
@@ -91,6 +92,7 @@ describe Elasticity::EMR do
             </TerminateJobFlowsResponse>
           RESPONSE
         end
+
         it "should terminate the specific jobflow" do
           aws_request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_key")
           aws_request.should_receive(:aws_emr_request).with({
@@ -101,6 +103,7 @@ describe Elasticity::EMR do
           emr = Elasticity::EMR.new("aws_access_key_id", "aws_secret_key")
           emr.terminate_jobflows("j-1")
         end
+
         context "when a block is given" do
           it "should yield the XML result" do
             aws_request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_key")
@@ -117,10 +120,11 @@ describe Elasticity::EMR do
             xml_result.should == @terminate_jobflows_xml
           end
         end
+
       end
 
       context "when the jobflow does not exist" do
-        it "should terminate the specific jobflow" do
+        it "should raise an ArgumentError" do
           aws_request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_key")
           aws_request.should_receive(:aws_emr_request).and_raise(RestClient::BadRequest)
           Elasticity::AwsRequest.should_receive(:new).and_return(aws_request)
@@ -134,6 +138,90 @@ describe Elasticity::EMR do
     end
   end
 
+  describe "#modify_instance_groups" do
+
+    describe "integration happy path" do
+      context "when the instance group exists" do
+        use_vcr_cassette "modify_instance_groups/set_instances_to_3", :record => :none
+        it "should terminate the specified jobflow" do
+          emr = Elasticity::EMR.new(ENV["aws_access_key_id"], ENV["aws_secret_key"])
+          instance_group_config = {"ig-2T1HNUO61BG3O" => 2}
+          emr.modify_instance_groups(instance_group_config)
+        end
+      end
+    end
+
+    describe "unit tests" do
+
+      context "when the instance group exists" do
+        it "should modify the specified instance group" do
+          aws_request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_key")
+          aws_request.should_receive(:aws_emr_request).with({
+            "Operation" => "ModifyInstanceGroups",
+            "InstanceGroups.member.1.InstanceGroupId" => "ig-1",
+            "InstanceGroups.member.1.InstanceCount" => 2
+          })
+          Elasticity::AwsRequest.should_receive(:new).and_return(aws_request)
+          emr = Elasticity::EMR.new("aws_access_key_id", "aws_secret_key")
+          emr.modify_instance_groups({"ig-1" => 2})
+        end
+      end
+
+      context "when a block is given" do
+        before do
+          @modify_instance_groups_xml = <<-RESPONSE
+            <ModifyInstanceGroupsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
+              <ResponseMetadata>
+                <RequestId>4ef75373-659c-11e0-bdf6-e3d62a364c28</RequestId>
+              </ResponseMetadata>
+            </ModifyInstanceGroupsResponse>
+          RESPONSE
+        end
+        it "should yield the XML result" do
+          aws_request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_key")
+          aws_request.should_receive(:aws_emr_request).and_return(@modify_instance_groups_xml)
+          Elasticity::AwsRequest.should_receive(:new).and_return(aws_request)
+          emr = Elasticity::EMR.new("aws_access_key_id", "aws_secret_key")
+          xml_result = nil
+          emr.modify_instance_groups({"ig-1" => 2})  do |xml|
+            xml_result = xml
+          end
+          xml_result.should == @modify_instance_groups_xml
+        end
+      end
+
+
+      context "when there is an error" do
+
+        before do
+          @error_message = "1 validation error detected: Value null at 'instanceGroups.1.member.instanceCount' failed to satisfy constraint: Member must not be null"
+          @error_xml = <<-ERROR
+            <ErrorResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
+              <Error>
+                <Message>#{@error_message}</Message>
+              </Error>
+            </ErrorResponse>
+          ERROR
+        end
+
+        it "should raise an ArgumentError with the error message" do
+          aws_request = Elasticity::AwsRequest.new("aws_access_key_id", "aws_secret_key")
+          @exception = RestClient::BadRequest.new
+          @exception.should_receive(:http_body).and_return(@error_xml)
+          aws_request.should_receive(:aws_emr_request).and_raise(@exception)
+          Elasticity::AwsRequest.should_receive(:new).and_return(aws_request)
+          emr = Elasticity::EMR.new("aws_access_key_id", "aws_secret_key")
+          lambda {
+            emr.modify_instance_groups({"ig-1" => 2})
+          }.should raise_error(ArgumentError, @error_message)
+        end
+
+      end
+
+    end
+
+  end
+
   describe "#direct" do
 
     describe "integration happy path" do

metadata

@@ -1,13 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: elasticity
 version: !ruby/object:Gem::Version 
-  hash: 25
+  hash: 3
   prerelease: 
   segments: 
   - 0
-  - 3
-  - 5
-  version: 0.3.5
+  - 4
+  version: "0.4"
 platform: ruby
 authors: 
 - Robert Slifka
@@ -15,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-11 00:00:00 -07:00
+date: 2011-04-13 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -131,6 +130,7 @@ files:
 - lib/elasticity/version.rb
 - spec/fixtures/vcr_cassettes/describe_jobflows/all_jobflows.yml
 - spec/fixtures/vcr_cassettes/direct/terminate_jobflow.yml
+- spec/fixtures/vcr_cassettes/modify_instance_groups/set_instances_to_3.yml
 - spec/fixtures/vcr_cassettes/terminate_jobflows/one_jobflow.yml
 - spec/lib/elasticity/aws_request_spec.rb
 - spec/lib/elasticity/emr_spec.rb
@@ -173,6 +173,7 @@ summary: Programmatic access to Amazon's Elastic Map Reduce service.
 test_files: 
 - spec/fixtures/vcr_cassettes/describe_jobflows/all_jobflows.yml
 - spec/fixtures/vcr_cassettes/direct/terminate_jobflow.yml
+- spec/fixtures/vcr_cassettes/modify_instance_groups/set_instances_to_3.yml
 - spec/fixtures/vcr_cassettes/terminate_jobflows/one_jobflow.yml
 - spec/lib/elasticity/aws_request_spec.rb
 - spec/lib/elasticity/emr_spec.rb