elasticity 1.4.1 → 1.5

data/spec/fixtures/vcr_cassettes/set_termination_protection/protect_multiple_job_flows.yml

@@ -1,32 +1,38 @@
---- 
-- !ruby/struct:VCR::HTTPInteraction 
-  request: !ruby/struct:VCR::Request 
-    method: :get
-    uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com\/\?AWSAccessKeyId=\S+&JobFlowIds.member.1=j-1B4D1XP0C0A35&JobFlowIds.member.2=j-1YG2MYL0HVYS5&Operation=SetTerminationProtection&.*TerminationProtected=true/
+---
+http_interactions:
+- request:
+    method: get
+    uri: !ruby/regexp /^https/
     body:
-    headers: 
-      accept: 
+      encoding: US-ASCII
+      string: ""
+    headers:
+      Accept:
       - "*/*; q=0.5, application/xml"
-      accept-encoding: 
+      Accept-Encoding:
       - gzip, deflate
-  response: !ruby/struct:VCR::Response 
-    status: !ruby/struct:VCR::ResponseStatus 
+  response:
+    status:
       code: 200
       message: OK
-    headers: 
-      x-amzn-requestid: 
-      - 755ebe8a-6923-11e0-a9c2-c126f1bb4493
-      content-type: 
-      - text/xml
-      date: 
-      - Sun, 17 Apr 2011 18:49:43 GMT
-      content-length: 
+    headers:
+      Content-Length:
       - "239"
-    body: |
-      <SetTerminationProtectionResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
-        <ResponseMetadata>
-          <RequestId>755ebe8a-6923-11e0-a9c2-c126f1bb4493</RequestId>
-        </ResponseMetadata>
-      </SetTerminationProtectionResponse>
+      Date:
+      - Sun, 17 Apr 2011 18:49:43 GMT
+      Content-Type:
+      - text/xml
+      X-Amzn-Requestid:
+      - 755ebe8a-6923-11e0-a9c2-c126f1bb4493
+    body:
+      encoding: US-ASCII
+      string: |
+        <SetTerminationProtectionResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
+          <ResponseMetadata>
+            <RequestId>755ebe8a-6923-11e0-a9c2-c126f1bb4493</RequestId>
+          </ResponseMetadata>
+        </SetTerminationProtectionResponse>
 
     http_version: "1.1"
+  recorded_at: Sat, 03 Mar 2012 23:00:38 GMT
+recorded_with: VCR 2.0.0

data/spec/fixtures/vcr_cassettes/terminate_jobflows/one_jobflow.yml

@@ -1,32 +1,38 @@
---- 
-- !ruby/struct:VCR::HTTPInteraction 
-  request: !ruby/struct:VCR::Request 
-    method: :get
-    uri: !ruby/regexp /^https:\/\/elasticmapreduce.amazonaws.com\/\?AWSAccessKeyId=\S+&JobFlowIds.member.1=j-1MZ5TVWFJRSKN&Operation=TerminateJobFlows/
+---
+http_interactions:
+- request:
+    method: get
+    uri: !ruby/regexp /^https/
     body:
-    headers: 
-      accept: 
+      encoding: US-ASCII
+      string: ""
+    headers:
+      Accept:
       - "*/*; q=0.5, application/xml"
-      accept-encoding: 
+      Accept-Encoding:
       - gzip, deflate
-  response: !ruby/struct:VCR::Response 
-    status: !ruby/struct:VCR::ResponseStatus 
+  response:
+    status:
       code: 200
       message: OK
-    headers: 
-      x-amzn-requestid: 
-      - 83e0154d-63c6-11e0-bc41-ababd98a870b
-      content-type: 
-      - text/xml
-      date: 
-      - Sun, 10 Apr 2011 23:01:48 GMT
-      content-length: 
+    headers:
+      Content-Length:
       - "225"
-    body: |
-      <TerminateJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
-        <ResponseMetadata>
-          <RequestId>83e0154d-63c6-11e0-bc41-ababd98a870b</RequestId>
-        </ResponseMetadata>
-      </TerminateJobFlowsResponse>
+      Date:
+      - Sun, 10 Apr 2011 23:01:48 GMT
+      Content-Type:
+      - text/xml
+      X-Amzn-Requestid:
+      - 83e0154d-63c6-11e0-bc41-ababd98a870b
+    body:
+      encoding: US-ASCII
+      string: |
+        <TerminateJobFlowsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
+          <ResponseMetadata>
+            <RequestId>83e0154d-63c6-11e0-bc41-ababd98a870b</RequestId>
+          </ResponseMetadata>
+        </TerminateJobFlowsResponse>
 
     http_version: "1.1"
+  recorded_at: Sat, 03 Mar 2012 23:00:44 GMT
+recorded_with: VCR 2.0.0

data/spec/lib/elasticity/emr_spec.rb

@@ -266,6 +266,7 @@ describe Elasticity::EMR do
               <JobFlows>
                 <member>
                   <ExecutionStatusDetail>
+                    <CreationDateTime>2011-04-04T17:41:51Z</CreationDateTime>
                     <State>TERMINATED</State>
                   </ExecutionStatusDetail>
                   <JobFlowId>j-p</JobFlowId>
@@ -274,6 +275,7 @@ describe Elasticity::EMR do
                 <member>
                   <ExecutionStatusDetail>
                     <State>TERMINATED</State>
+                    <CreationDateTime>2011-04-04T17:41:51Z</CreationDateTime>
                   </ExecutionStatusDetail>
                   <JobFlowId>j-h</JobFlowId>
                   <Name>Hive Job</Name>
@@ -327,6 +329,7 @@ describe Elasticity::EMR do
               <member>
                 <ExecutionStatusDetail>
                   <State>TERMINATED</State>
+                  <CreationDateTime>2011-04-04T17:41:51Z</CreationDateTime>
                 </ExecutionStatusDetail>
                 <JobFlowId>j-3UN6WX5RRO2AG</JobFlowId>
                 <Name>The One Job Flow</Name>

data/spec/lib/elasticity/hive_job_spec.rb

@@ -57,7 +57,6 @@ describe Elasticity::HiveJob do
                     "--run-hive-script",
                     "--args",
                     "-f", "s3n://slif-hive/test.q",
-                    "-d", "XREFS=s3n://slif-test/xrefs",
                     "-d", "OUTPUT=s3n://slif-test/output"
                 ],
               },
@@ -71,8 +70,7 @@ describe Elasticity::HiveJob do
       hive.log_uri = "s3n://slif-test/output/logs"
       hive.action_on_failure = "CONTINUE"
       jobflow_id = hive.run('s3n://slif-hive/test.q', {
-        'OUTPUT' => 's3n://slif-test/output',
-        'XREFS' => 's3n://slif-test/xrefs'
+        'OUTPUT' => 's3n://slif-test/output'
       })
       jobflow_id.should == "new_jobflow_id"
     end
@@ -84,12 +82,8 @@ describe Elasticity::HiveJob do
     it "should kick off the sample Amazion EMR Hive application" do
       hive = Elasticity::HiveJob.new(AWS_ACCESS_KEY_ID, AWS_SECRET_KEY)
       hive.ec2_key_name = "sharethrough_dev"
-      jobflow_id = hive.run("s3n://elasticmapreduce/samples/hive-ads/libs/model-build.q", {
-        "LIBS"   => "s3n://elasticmapreduce/samples/hive-ads/libs",
-        "INPUT"  => "s3n://elasticmapreduce/samples/hive-ads/tables",
-        "OUTPUT" => "s3n://slif-elasticity/hive-ads/output/2011-04-19"
-      })
-      jobflow_id.should == "j-1UUVYMHBLKEGN"
+      jobflow_id = hive.run("s3n://elasticmapreduce/samples/hive-ads/libs/model-build.q")
+      jobflow_id.should == "j-2I4HV6S3SDGD9"
     end
   end
 

data/spec/lib/elasticity/pig_job_spec.rb

@@ -26,6 +26,22 @@ describe Elasticity::PigJob do
         pig.instance_count = 1
       }.should raise_error(ArgumentError, "Instance count cannot be set to less than 2 (requested 1)")
     end
+
+    it "should recalculate @parallels" do
+      pig = Elasticity::PigJob.new("access", "secret")
+      lambda {
+        pig.instance_count = 10
+      }.should change(pig, :parallels)
+    end
+  end
+
+  describe "#slave_instance_type=" do
+    it "should recalculate @parallels" do
+      pig = Elasticity::PigJob.new("access", "secret")
+      lambda {
+        pig.slave_instance_type = "c1.xlarge"
+      }.should change(pig, :parallels)
+    end
   end
 
   describe "calculated value of parallels" do
@@ -207,5 +223,4 @@ describe Elasticity::PigJob do
     end
   end
 
-
 end

data/spec/spec_helper.rb

@@ -1,27 +1,3 @@
-if !ENV["AWS_ACCESS_KEY_ID"] || !ENV["AWS_SECRET_KEY"]
-  puts "\n\n\e[32m**********************************************************************************************"
-  puts "Please set \e[0;1mAWS_ACCESS_KEY_ID\e[32m and \e[0;1mAWS_SECRET_KEY\e[32m in your environment to run the tests."
-  puts ""
-  puts "These keys can be found on your AWS Account > Security Credentials page, at the following URL:"
-  puts ""
-  puts "  \e[0;1mhttps://aws-portal.amazon.com/gp/aws/developer/account/index.html?ie=UTF8&action=access-key"
-  puts ""
-  bourne = !ENV['SHELL'].nil? && ENV['SHELL'] =~ /(bash|ksh|zsh)/
-  puts "\e[0mbash, zsh, ksh: #{"\e[35;1mThis is you!\e[0m" if bourne}"
-  puts "  export AWS_ACCESS_KEY_ID=01234"
-  puts "  export AWS_SECRET_KEY=56789"
-  puts ""
-  puts "\e[0mcsh, tcsh: #{"\e[35;1mThis is you!\e[0m" unless bourne}"
-  puts "  setenv AWS_ACCESS_KEY_ID 01234"
-  puts "  setenv AWS_SECRET_KEY 56789"
-  puts ""
-  puts "\e[32m**********************************************************************************************\n\n\n"
-  exit
-end
-
-AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"]
-AWS_SECRET_KEY = ENV["AWS_SECRET_KEY"]
-
 require 'rubygems'
 require 'bundler/setup'
 
@@ -33,11 +9,48 @@ ENV["RAILS_ENV"] ||= 'test'
 
 $:.unshift File.dirname(__FILE__)
 
-VCR.config do |c|
+uri_regexp_matcher = lambda do |real_request, recorded_request|
+  real_request.uri =~ recorded_request.uri
+end
+
+VCR.configure do |c|
   c.cassette_library_dir = 'spec/fixtures/vcr_cassettes'
-  c.stub_with :webmock
+  c.hook_into :webmock
+  c.default_cassette_options = {
+    :match_requests_on => [:method, uri_regexp_matcher]
+  }
+  c.before_http_request do |_|
+    require_aws_credentials unless VCR.current_cassette.record_mode == :none
+  end
 end
 
 RSpec.configure do |c|
   c.extend VCR::RSpec::Macros
+end
+
+AWS_ACCESS_KEY_ID = ENV['AWS_ACCESS_KEY_ID'] ||= 'default'
+AWS_SECRET_KEY = ENV['AWS_SECRET_KEY'] ||= 'default'
+
+def require_aws_credentials
+  if AWS_ACCESS_KEY_ID == 'default' && AWS_SECRET_KEY == 'default'
+    puts "\n\e[33m**********************************************************************************************"
+    puts "\e[32mIf you want to record new cassettes, you'll need to provide a set of AWS credentials so"
+    puts "Elasticity can interact with EMR.  These keys can be found on your AWS Account > Security"
+    puts "Credentials page, at the following URL:"
+    puts ""
+    puts "  \e[0;1mhttps://aws-portal.amazon.com/gp/aws/developer/account/index.html?ie=UTF8&action=access-key"
+    puts ""
+    bourne = !ENV['SHELL'].nil? && ENV['SHELL'] =~ /(bash|ksh|zsh)/
+    if bourne
+      puts "\e[32mbash, zsh, ksh:"
+      puts "  export AWS_ACCESS_KEY_ID=01234"
+      puts "  export AWS_SECRET_KEY=56789"
+    else
+      puts "\e[32mcsh, tcsh:"
+      puts "  setenv AWS_ACCESS_KEY_ID 01234"
+      puts "  setenv AWS_SECRET_KEY 56789"
+    end
+    puts "\e[33m**********************************************************************************************\n"
+    exit
+  end
 end

metadata

@@ -1,13 +1,8 @@
 --- !ruby/object:Gem::Specification 
 name: elasticity
 version: !ruby/object:Gem::Version 
-  hash: 5
   prerelease: 
-  segments: 
-  - 1
-  - 4
-  - 1
-  version: 1.4.1
+  version: "1.5"
 platform: ruby
 authors: 
 - Robert Slifka
@@ -15,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-12-17 00:00:00 -08:00
+date: 2012-03-05 00:00:00 -08:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -26,9 +21,6 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
   type: :runtime
   version_requirements: *id001
@@ -40,117 +32,54 @@ dependencies:
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: autotest-fsevent
+  name: rake
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
         version: "0"
   type: :development
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: autotest-growl
+  name: rspec
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+        version: 2.8.0
   type: :development
   version_requirements: *id004
 - !ruby/object:Gem::Dependency 
-  name: rake
+  name: vcr
   prerelease: false
   requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
+        version: "2.0"
   type: :development
   version_requirements: *id005
 - !ruby/object:Gem::Dependency 
-  name: rspec
+  name: webmock
   prerelease: false
   requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
-    - - ">="
+    - - ~>
       - !ruby/object:Gem::Version 
-        hash: 27
-        segments: 
-        - 2
-        - 5
-        - 0
-        version: 2.5.0
+        version: 1.8.0
   type: :development
   version_requirements: *id006
-- !ruby/object:Gem::Dependency 
-  name: vcr
-  prerelease: false
-  requirement: &id007 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 1
-        segments: 
-        - 1
-        - 5
-        - 1
-        version: 1.5.1
-  type: :development
-  version_requirements: *id007
-- !ruby/object:Gem::Dependency 
-  name: webmock
-  prerelease: false
-  requirement: &id008 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 11
-        segments: 
-        - 1
-        - 6
-        - 2
-        version: 1.6.2
-  type: :development
-  version_requirements: *id008
-- !ruby/object:Gem::Dependency 
-  name: ZenTest
-  prerelease: false
-  requirement: &id009 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 0
-        version: "0"
-  type: :development
-  version_requirements: *id009
-description: Programmatic access to Amazon's Elastic Map Reduce service.
+description: Programmatic access to Amazon's Elastic Map Reduce service, driven by the Sharethrough team's requirements for belting out EMR jobs.
 email: 
 executables: []
 
@@ -163,6 +92,7 @@ files:
 - .gitignore
 - .rspec
 - .rvmrc
+- .travis.yml
 - Gemfile
 - HISTORY.md
 - LICENSE
@@ -215,23 +145,17 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
       version: "0"
 requirements: []
 
 rubyforge_project: 
-rubygems_version: 1.4.1
+rubygems_version: 1.5.2
 signing_key: 
 specification_version: 3
 summary: Programmatic access to Amazon's Elastic Map Reduce service.