elasticity 1.1 → 1.1.1

data/HISTORY.mediawiki

@@ -10,3 +10,9 @@
 
 * Added HiveJob, a simplified way to launch basic Hive job flows.
 * Added HISTORY.
+
+=== 1.1.1 ===
+
+* HiveJob critical bug fixed, now it works :)
+* Added log_uri and action_on_failure as options to HiveJob.
+* Added integration tests to HiveJob.

data/lib/elasticity/hive_job.rb

@@ -4,16 +4,19 @@ module Elasticity
   # having to understand the entirety of the EMR API.
   class HiveJob
 
+    attr_accessor :action_on_failure
     attr_accessor :aws_access_key_id
     attr_accessor :aws_secret_access_key
     attr_accessor :ec2_key_name
     attr_accessor :name
     attr_accessor :hadoop_version
     attr_accessor :instance_count
+    attr_accessor :log_uri
     attr_accessor :master_instance_type
     attr_accessor :slave_instance_type
 
     def initialize(aws_access_key_id, aws_secret_access_key)
+      @action_on_failure = "TERMINATE_JOB_FLOW"
       @aws_access_key_id = aws_access_key_id
       @aws_secret_access_key = aws_secret_access_key
       @ec2_key_name = "default"
@@ -23,7 +26,7 @@ module Elasticity
       @name = "Elasticity Hive Job"
       @slave_instance_type = "m1.small"
 
-      @aws_request = Elasticity::AwsRequest.new(aws_access_key_id, aws_secret_access_key)
+      @emr = Elasticity::EMR.new(aws_access_key_id, aws_secret_access_key)
     end
 
     # Run the specified Hive script with the specified variables.
@@ -68,7 +71,7 @@ module Elasticity
             :name => "Setup Hive"
           },
             {
-              :action_on_failure => "TERMINATE_JOB_FLOW",
+              :action_on_failure => @action_on_failure,
               :hadoop_jar_step => {
                 :jar => "s3://elasticmapreduce/libs/script-runner/script-runner.jar",
                 :args => script_arguments,
@@ -77,7 +80,10 @@ module Elasticity
             }
         ]
       }
-      @aws_request.run_job_flow(jobflow_config)
+
+      jobflow_config.merge!(:log_uri => @log_uri) if @log_uri
+      
+      @emr.run_job_flow(jobflow_config)
     end
 
   end

data/lib/elasticity/version.rb

@@ -1,3 +1,3 @@
 module Elasticity
-  VERSION = "1.1"
+  VERSION = "1.1.1"
 end

data/spec/fixtures/vcr_cassettes/hive_job/hive_ads.yml

@@ -0,0 +1,35 @@
+---
+- !ruby/struct:VCR::HTTPInteraction
+  request: !ruby/struct:VCR::Request
+    method: :get
+    uri: !ruby/regexp /^http:\/\/elasticmapreduce.amazonaws.com:80\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&Instances\.Ec2KeyName=sharethrough_dev&Instances\.HadoopVersion=0\.20&Instances\.InstanceCount=2&Instances\.MasterInstanceType=m1\.small&Instances\.SlaveInstanceType=m1\.small&Name=Elasticity%20Hive%20Job&Operation=RunJobFlow&.*&Steps\.member\.1\.ActionOnFailure=TERMINATE_JOB_FLOW&Steps\.member\.1\.HadoopJarStep\.Args\.member\.1=s3:\/\/elasticmapreduce\/libs\/hive\/hive-script&Steps\.member\.1\.HadoopJarStep\.Args\.member\.2=--base-path&Steps\.member\.1\.HadoopJarStep\.Args\.member\.3=s3:\/\/elasticmapreduce\/libs\/hive\/&Steps\.member\.1\.HadoopJarStep\.Args\.member\.4=--install-hive&Steps\.member\.1\.HadoopJarStep\.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner\.jar&Steps\.member\.1\.Name=Setup%20Hive&Steps\.member\.2\.ActionOnFailure=TERMINATE_JOB_FLOW&Steps\.member\.2\.HadoopJarStep\.Args\.member\.1=s3:\/\/elasticmapreduce\/libs\/hive\/hive-script&Steps\.member\.2\.HadoopJarStep\.Args\.member\.10=-d&Steps\.member\.2\.HadoopJarStep\.Args\.member\.11=LIBS=s3n:\/\/elasticmapreduce\/samples\/hive-ads\/libs&Steps\.member\.2\.HadoopJarStep\.Args\.member\.2=--run-hive-script&Steps\.member\.2\.HadoopJarStep\.Args\.member\.3=--args&Steps\.member\.2\.HadoopJarStep\.Args\.member\.4=-f&Steps\.member\.2\.HadoopJarStep\.Args\.member\.5=s3n:\/\/elasticmapreduce\/samples\/hive-ads\/libs\/model-build\.q&Steps\.member\.2\.HadoopJarStep\.Args\.member\.6=-d&Steps\.member\.2\.HadoopJarStep\.Args\.member\.7=INPUT=s3n:\/\/elasticmapreduce\/samples\/hive-ads\/tables&Steps\.member\.2\.HadoopJarStep\.Args\.member\.8=-d&Steps\.member\.2\.HadoopJarStep\.Args\.member\.9=OUTPUT=s3n:\/\/slif-elasticity\/hive-ads\/output\/2011-04-19&Steps\.member\.2\.HadoopJarStep\.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner\.jar&Steps\.member\.2\.Name=Run%20Hive%20Script/
+    body:
+    headers:
+      accept:
+      - "*/*; q=0.5, application/xml"
+      accept-encoding:
+      - gzip, deflate
+  response: !ruby/struct:VCR::Response
+    status: !ruby/struct:VCR::ResponseStatus
+      code: 200
+      message: OK
+    headers:
+      x-amzn-requestid:
+      - 87d43cfd-6fb2-11e0-bc88-bf902ad8b20b
+      content-type:
+      - text/xml
+      date:
+      - Tue, 26 Apr 2011 03:08:58 GMT
+      content-length:
+      - "297"
+    body: |
+      <RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
+        <RunJobFlowResult>
+          <JobFlowId>j-1UUVYMHBLKEGN</JobFlowId>
+        </RunJobFlowResult>
+        <ResponseMetadata>
+          <RequestId>87d43cfd-6fb2-11e0-bc88-bf902ad8b20b</RequestId>
+        </ResponseMetadata>
+      </RunJobFlowResponse>
+
+    http_version: "1.1"

data/spec/lib/elasticity/emr_spec.rb

@@ -2,9 +2,6 @@ require 'spec_helper'
 
 describe Elasticity::EMR do
 
-  AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"]
-  AWS_SECRET_KEY = ENV["AWS_SECRET_KEY"]
-
   describe "#add_instance_groups" do
 
     describe "integration happy path" do

data/spec/lib/elasticity/hive_job_spec.rb

@@ -14,16 +14,19 @@ describe Elasticity::HiveJob do
       hive.master_instance_type.should == "m1.small"
       hive.name.should == "Elasticity Hive Job"
       hive.slave_instance_type.should == "m1.small"
+      hive.action_on_failure.should == "TERMINATE_JOB_FLOW"
+      hive.log_uri.should == nil
     end
 
   end
 
   describe "#run" do
 
-    it "run the script with the specified variables and return the jobflow_id" do
-      aws = Elasticity::AwsRequest.new("", "")
+    it "should run the script with the specified variables and return the jobflow_id" do
+      aws = Elasticity::EMR.new("", "")
       aws.should_receive(:run_job_flow).with({
         :name => "Elasticity Hive Job",
+        :log_uri => "s3n://slif-test/output/logs",
         :instances => {
           :ec2_key_name => "default",
           :hadoop_version => "0.20",
@@ -46,7 +49,7 @@ describe Elasticity::HiveJob do
             :name => "Setup Hive"
           },
             {
-              :action_on_failure => "TERMINATE_JOB_FLOW",
+              :action_on_failure => "CONTINUE",
               :hadoop_jar_step => {
                 :jar => "s3://elasticmapreduce/libs/script-runner/script-runner.jar",
                 :args => [
@@ -62,9 +65,11 @@ describe Elasticity::HiveJob do
             }
         ]
       }).and_return("new_jobflow_id")
-      Elasticity::AwsRequest.should_receive(:new).with("access", "secret").and_return(aws)
+      Elasticity::EMR.should_receive(:new).with("access", "secret").and_return(aws)
 
       hive = Elasticity::HiveJob.new("access", "secret")
+      hive.log_uri = "s3n://slif-test/output/logs"
+      hive.action_on_failure = "CONTINUE"
       jobflow_id = hive.run('s3n://slif-hive/test.q', {
         'OUTPUT' => 's3n://slif-test/output',
         'XREFS' => 's3n://slif-test/xrefs'
@@ -74,4 +79,18 @@ describe Elasticity::HiveJob do
 
   end
 
+  describe "integration happy path" do
+    use_vcr_cassette "hive_job/hive_ads", :record => :none
+    it "should kick off the sample Amazion EMR Hive application" do
+      hive = Elasticity::HiveJob.new(AWS_ACCESS_KEY_ID, AWS_SECRET_KEY)
+      hive.ec2_key_name = "sharethrough_dev"
+      jobflow_id = hive.run("s3n://elasticmapreduce/samples/hive-ads/libs/model-build.q", {
+        "LIBS"   => "s3n://elasticmapreduce/samples/hive-ads/libs",
+        "INPUT"  => "s3n://elasticmapreduce/samples/hive-ads/tables",
+        "OUTPUT" => "s3n://slif-elasticity/hive-ads/output/2011-04-19"
+      })
+      jobflow_id.should == "j-1UUVYMHBLKEGN"
+    end
+  end
+
 end

data/spec/spec_helper.rb

@@ -19,6 +19,8 @@ if !ENV["AWS_ACCESS_KEY_ID"] || !ENV["AWS_SECRET_KEY"]
   exit
 end
 
+AWS_ACCESS_KEY_ID = ENV["AWS_ACCESS_KEY_ID"]
+AWS_SECRET_KEY = ENV["AWS_SECRET_KEY"]
 
 require 'rubygems'
 require 'bundler/setup'

metadata

@@ -1,12 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: elasticity
 version: !ruby/object:Gem::Version 
-  hash: 13
+  hash: 17
   prerelease: 
   segments: 
   - 1
   - 1
-  version: "1.1"
+  - 1
+  version: 1.1.1
 platform: ruby
 authors: 
 - Robert Slifka
@@ -14,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-24 00:00:00 -07:00
+date: 2011-04-25 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -180,6 +181,7 @@ files:
 - spec/fixtures/vcr_cassettes/add_jobflow_steps/add_multiple_steps.yml
 - spec/fixtures/vcr_cassettes/describe_jobflows/all_jobflows.yml
 - spec/fixtures/vcr_cassettes/direct/terminate_jobflow.yml
+- spec/fixtures/vcr_cassettes/hive_job/hive_ads.yml
 - spec/fixtures/vcr_cassettes/modify_instance_groups/set_instances_to_3.yml
 - spec/fixtures/vcr_cassettes/run_jobflow/word_count.yml
 - spec/fixtures/vcr_cassettes/set_termination_protection/nonexistent_job_flows.yml
@@ -231,6 +233,7 @@ test_files:
 - spec/fixtures/vcr_cassettes/add_jobflow_steps/add_multiple_steps.yml
 - spec/fixtures/vcr_cassettes/describe_jobflows/all_jobflows.yml
 - spec/fixtures/vcr_cassettes/direct/terminate_jobflow.yml
+- spec/fixtures/vcr_cassettes/hive_job/hive_ads.yml
 - spec/fixtures/vcr_cassettes/modify_instance_groups/set_instances_to_3.yml
 - spec/fixtures/vcr_cassettes/run_jobflow/word_count.yml
 - spec/fixtures/vcr_cassettes/set_termination_protection/nonexistent_job_flows.yml