elasticity 1.2.1 → 1.2.2

data/HISTORY.mediawiki

@@ -1,3 +1,7 @@
+=== 1.2.2 ===
+
+* HiveJob and PigJob now support configuring Hadoop options via .add_hadoop_bootstrap_action().
+
 === 1.2.1 ===
 
 * Shipping up E_PARALLELS Pig variable with each invocation; reasonable default value for PARALLEL based on the number and type of instances configured.

data/README.mediawiki

@@ -26,7 +26,17 @@ Elasticity currently provides simplified access to launching Hive and Pig job fl
   @slave_instance_type = "m1.small"
 </pre>
 
-These are all accessible from HiveJob and PigJob instances.  See the PigJob description for an example.
+These are all accessible from HiveJob and PigJob.  See the PigJob description for an example.
+
+=== Bootstrap Actions ===
+
+You can also configure Hadoop options with add_hadoop_bootstrap_action().
+
+<pre>
+  pig = Elasticity::PigJob.new(ENV["AWS_ACCESS_KEY_ID"], ENV["AWS_SECRET_KEY"])
+  pig.add_hadoop_bootstrap_action("-m", "mapred.job.reuse.jvm.num.tasks=120")
+  ...
+</pre>
 
 == Hive ==
 

data/lib/elasticity/pig_job.rb

@@ -76,6 +76,7 @@ module Elasticity
       }
 
       jobflow_config.merge!(:log_uri => @log_uri) if @log_uri
+      jobflow_config.merge!(get_bootstrap_actions)
 
       @emr.run_job_flow(jobflow_config)
     end

data/lib/elasticity/simple_job.rb

@@ -27,6 +27,24 @@ module Elasticity
       @emr = Elasticity::EMR.new(aws_access_key_id, aws_secret_access_key)
     end
 
+    def add_hadoop_bootstrap_action(option, value)
+      @hadoop_actions ||= []
+      @hadoop_actions << {
+        :name => "Elasticity Bootstrap Action (Configure Hadoop)",
+        :script_bootstrap_action => {
+          :path => "s3n://elasticmapreduce/bootstrap-actions/configure-hadoop",
+          :args => [option, value]
+        }
+      }
+    end
+
+    private
+
+    def get_bootstrap_actions
+      return {} unless @hadoop_actions && !@hadoop_actions.empty?
+      { :bootstrap_actions => @hadoop_actions }
+    end
+
   end
 
 end

data/lib/elasticity/version.rb

@@ -1,3 +1,3 @@
 module Elasticity
-  VERSION = "1.2.1"
+  VERSION = "1.2.2"
 end

data/spec/fixtures/vcr_cassettes/pig_job/apache_log_reports_with_bootstrap.yml

@@ -0,0 +1,35 @@
+--- 
+- !ruby/struct:VCR::HTTPInteraction 
+  request: !ruby/struct:VCR::Request 
+    method: :get
+    uri: !ruby/regexp /^http:\/\/elasticmapreduce.amazonaws.com:80\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&BootstrapActions.member.1.Name=Elasticity%20Bootstrap%20Action%20\(Configure%20Hadoop\)&BootstrapActions.member.1.ScriptBootstrapAction.Args.member.1=-m&BootstrapActions.member.1.ScriptBootstrapAction.Args.member.2=mapred.job.reuse.jvm.num.tasks=120&BootstrapActions.member.1.ScriptBootstrapAction.Path=s3n:\/\/elasticmapreduce\/bootstrap-actions\/configure-hadoop&Instances.Ec2KeyName=sharethrough_dev&Instances.HadoopVersion=0.20&Instances.InstanceCount=2&Instances.MasterInstanceType=m1.small&Instances.SlaveInstanceType=m1.small&Name=Elasticity%20Pig%20Job&Operation=RunJobFlow&.*&Steps.member.1.ActionOnFailure=TERMINATE_JOB_FLOW&Steps.member.1.HadoopJarStep.Args.member.1=s3:\/\/elasticmapreduce\/libs\/pig\/pig-script&Steps.member.1.HadoopJarStep.Args.member.2=--base-path&Steps.member.1.HadoopJarStep.Args.member.3=s3:\/\/elasticmapreduce\/libs\/pig\/&Steps.member.1.HadoopJarStep.Args.member.4=--install-pig&Steps.member.1.HadoopJarStep.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner.jar&Steps.member.1.Name=Setup%20Pig&Steps.member.2.ActionOnFailure=TERMINATE_JOB_FLOW&Steps.member.2.HadoopJarStep.Args.member.1=s3:\/\/elasticmapreduce\/libs\/pig\/pig-script&Steps.member.2.HadoopJarStep.Args.member.10=s3n:\/\/elasticmapreduce\/samples\/pig-apache\/do-reports.pig&Steps.member.2.HadoopJarStep.Args.member.2=--run-pig-script&Steps.member.2.HadoopJarStep.Args.member.3=--args&Steps.member.2.HadoopJarStep.Args.member.4=-p&Steps.member.2.HadoopJarStep.Args.member.5=INPUT=s3n:\/\/elasticmapreduce\/samples\/pig-apache\/input&Steps.member.2.HadoopJarStep.Args.member.6=-p&Steps.member.2.HadoopJarStep.Args.member.7=OUTPUT=s3n:\/\/slif-elasticity\/pig-apache\/output\/2011-05-10&Steps.member.2.HadoopJarStep.Args.member.8=-p&Steps.member.2.HadoopJarStep.Args.member.9=E_PARALLELS=1&Steps.member.2.HadoopJarStep.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner.jar&Steps.member.2.Name=Run%20Pig%20Script/
+    body:
+    headers: 
+      accept: 
+      - "*/*; q=0.5, application/xml"
+      accept-encoding: 
+      - gzip, deflate
+  response: !ruby/struct:VCR::Response 
+    status: !ruby/struct:VCR::ResponseStatus 
+      code: 200
+      message: OK
+    headers: 
+      x-amzn-requestid: 
+      - 10533247-7ad7-11e0-86ad-3b6ddfd3d4b9
+      content-type: 
+      - text/xml
+      date: 
+      - Tue, 10 May 2011 07:28:12 GMT
+      content-length: 
+      - "297"
+    body: |
+      <RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
+        <RunJobFlowResult>
+          <JobFlowId>j-1UK43AWRT3QHD</JobFlowId>
+        </RunJobFlowResult>
+        <ResponseMetadata>
+          <RequestId>10533247-7ad7-11e0-86ad-3b6ddfd3d4b9</RequestId>
+        </ResponseMetadata>
+      </RunJobFlowResponse>
+
+    http_version: "1.1"

data/spec/lib/elasticity/pig_job_spec.rb

@@ -81,80 +81,131 @@ describe Elasticity::PigJob do
 
   describe "#run" do
 
-    it "should run the script with the specified variables and return the jobflow_id" do
-      aws = Elasticity::EMR.new("", "")
-      aws.should_receive(:run_job_flow).with({
-        :name => "Elasticity Pig Job",
-        :log_uri => "s3n://slif-test/output/logs",
-        :instances => {
-          :ec2_key_name => "default",
-          :hadoop_version => "0.20",
-          :instance_count => 8,
-          :master_instance_type => "m1.small",
-          :slave_instance_type => "m1.xlarge",
-        },
-        :steps => [
-          {
-            :action_on_failure => "TERMINATE_JOB_FLOW",
-            :hadoop_jar_step => {
-              :jar => "s3://elasticmapreduce/libs/script-runner/script-runner.jar",
-              :args => [
-                "s3://elasticmapreduce/libs/pig/pig-script",
-                  "--base-path",
-                  "s3://elasticmapreduce/libs/pig/",
-                  "--install-pig"
-              ],
-            },
-            :name => "Setup Pig"
+    context "when no bootstrap actions are specified" do
+
+      it "should run the script with the specified variables and return the jobflow_id" do
+        aws = Elasticity::EMR.new("", "")
+        aws.should_receive(:run_job_flow).with({
+          :name => "Elasticity Pig Job",
+          :log_uri => "s3n://slif-test/output/logs",
+          :instances => {
+            :ec2_key_name => "default",
+            :hadoop_version => "0.20",
+            :instance_count => 8,
+            :master_instance_type => "m1.small",
+            :slave_instance_type => "m1.xlarge",
           },
+          :steps => [
             {
-              :action_on_failure => "CONTINUE",
+              :action_on_failure => "TERMINATE_JOB_FLOW",
               :hadoop_jar_step => {
                 :jar => "s3://elasticmapreduce/libs/script-runner/script-runner.jar",
                 :args => [
                   "s3://elasticmapreduce/libs/pig/pig-script",
-                    "--run-pig-script",
-                    "--args",
-                    "-p", "OUTPUT=s3n://slif-pig-test/output",
-                    "-p", "XREFS=s3n://slif-pig-test/xrefs",
-                    "-p", "E_PARALLELS=26",
-                    "s3n://slif-pig-test/test.pig"
+                    "--base-path",
+                    "s3://elasticmapreduce/libs/pig/",
+                    "--install-pig"
                 ],
               },
-              :name => "Run Pig Script"
-            }
-        ]
-      }).and_return("new_jobflow_id")
-
-      Elasticity::EMR.should_receive(:new).with("access", "secret").and_return(aws)
-      pig = Elasticity::PigJob.new("access", "secret")
-
-      pig.log_uri = "s3n://slif-test/output/logs"
-      pig.action_on_failure = "CONTINUE"
-      pig.instance_count = 8
-      pig.slave_instance_type = "m1.xlarge"
+              :name => "Setup Pig"
+            },
+              {
+                :action_on_failure => "CONTINUE",
+                :hadoop_jar_step => {
+                  :jar => "s3://elasticmapreduce/libs/script-runner/script-runner.jar",
+                  :args => [
+                    "s3://elasticmapreduce/libs/pig/pig-script",
+                      "--run-pig-script",
+                      "--args",
+                      "-p", "OUTPUT=s3n://slif-pig-test/output",
+                      "-p", "XREFS=s3n://slif-pig-test/xrefs",
+                      "-p", "E_PARALLELS=26",
+                      "s3n://slif-pig-test/test.pig"
+                  ],
+                },
+                :name => "Run Pig Script"
+              }
+          ]
+        }).and_return("new_jobflow_id")
+
+        Elasticity::EMR.should_receive(:new).with("access", "secret").and_return(aws)
+        pig = Elasticity::PigJob.new("access", "secret")
+
+        pig.log_uri = "s3n://slif-test/output/logs"
+        pig.action_on_failure = "CONTINUE"
+        pig.instance_count = 8
+        pig.slave_instance_type = "m1.xlarge"
+
+        jobflow_id = pig.run('s3n://slif-pig-test/test.pig', {
+          'OUTPUT' => 's3n://slif-pig-test/output',
+          'XREFS' => 's3n://slif-pig-test/xrefs'
+        })
+        jobflow_id.should == "new_jobflow_id"
+      end
+    end
 
-      jobflow_id = pig.run('s3n://slif-pig-test/test.pig', {
-        'OUTPUT' => 's3n://slif-pig-test/output',
-        'XREFS' => 's3n://slif-pig-test/xrefs'
-      })
-      jobflow_id.should == "new_jobflow_id"
+    context "when bootstrap actions are specified" do
+      it "should run the script wth the proper job configuration" do
+        aws = Elasticity::EMR.new("", "")
+        aws.should_receive(:run_job_flow).with(hash_including({
+          :bootstrap_actions => [
+            {
+              :name => "Elasticity Bootstrap Action (Configure Hadoop)",
+              :script_bootstrap_action => {
+                :path => "s3n://elasticmapreduce/bootstrap-actions/configure-hadoop",
+                :args => ["-m", "foo=111"]
+              }
+            },
+            {
+              :name => "Elasticity Bootstrap Action (Configure Hadoop)",
+              :script_bootstrap_action => {
+                :path => "s3n://elasticmapreduce/bootstrap-actions/configure-hadoop",
+                :args => ["-m", "bar=222"]
+              }
+            }
+          ],
+        }))
+
+        Elasticity::EMR.should_receive(:new).with("access", "secret").and_return(aws)
+        pig = Elasticity::PigJob.new("access", "secret")
+        pig.add_hadoop_bootstrap_action("-m", "foo=111")
+        pig.add_hadoop_bootstrap_action("-m", "bar=222")
+        pig.run('s3n://slif-pig-test/test.pig')
+      end
     end
 
   end
 
   describe "integration happy path" do
-    use_vcr_cassette "pig_job/apache_log_reports", :record => :none
-    it "should kick off the sample Amazion EMR Pig application" do
-      pig = Elasticity::PigJob.new(AWS_ACCESS_KEY_ID, AWS_SECRET_KEY)
-      pig.log_uri = "s3n://slif-elasticity/pig-apache/logs"
-      pig.ec2_key_name = "sharethrough_dev"
-      jobflow_id = pig.run("s3n://elasticmapreduce/samples/pig-apache/do-reports.pig", {
-        "INPUT"  => "s3n://elasticmapreduce/samples/pig-apache/input",
-        "OUTPUT" => "s3n://slif-elasticity/pig-apache/output/2011-05-04"
-      })
-      jobflow_id.should == "j-1HB7A3TBRT3VS"
+
+    context "with bootstrap actions" do
+      use_vcr_cassette "pig_job/apache_log_reports_with_bootstrap", :record => :none
+      it "should kick off the sample Amazion EMR Pig application" do
+        pig = Elasticity::PigJob.new(AWS_ACCESS_KEY_ID, AWS_SECRET_KEY)
+        pig.ec2_key_name = "sharethrough_dev"
+        pig.add_hadoop_bootstrap_action("-m", "mapred.job.reuse.jvm.num.tasks=120")
+        jobflow_id = pig.run("s3n://elasticmapreduce/samples/pig-apache/do-reports.pig", {
+          "INPUT" => "s3n://elasticmapreduce/samples/pig-apache/input",
+          "OUTPUT" => "s3n://slif-elasticity/pig-apache/output/2011-05-10"
+        })
+        jobflow_id.should == "j-1UK43AWRT3QHD"
+      end
+    end
+
+    context "without bootstrap actions" do
+      use_vcr_cassette "pig_job/apache_log_reports", :record => :none
+      it "should kick off the sample Amazion EMR Pig application" do
+        pig = Elasticity::PigJob.new(AWS_ACCESS_KEY_ID, AWS_SECRET_KEY)
+        pig.log_uri = "s3n://slif-elasticity/pig-apache/logs"
+        pig.ec2_key_name = "sharethrough_dev"
+        jobflow_id = pig.run("s3n://elasticmapreduce/samples/pig-apache/do-reports.pig", {
+          "INPUT" => "s3n://elasticmapreduce/samples/pig-apache/input",
+          "OUTPUT" => "s3n://slif-elasticity/pig-apache/output/2011-05-04"
+        })
+        jobflow_id.should == "j-1HB7A3TBRT3VS"
+      end
     end
   end
 
+
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: elasticity
 version: !ruby/object:Gem::Version 
-  hash: 29
+  hash: 27
   prerelease: 
   segments: 
   - 1
   - 2
-  - 1
-  version: 1.2.1
+  - 2
+  version: 1.2.2
 platform: ruby
 authors: 
 - Robert Slifka
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-05-07 00:00:00 -07:00
+date: 2011-05-10 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -186,6 +186,7 @@ files:
 - spec/fixtures/vcr_cassettes/hive_job/hive_ads.yml
 - spec/fixtures/vcr_cassettes/modify_instance_groups/set_instances_to_3.yml
 - spec/fixtures/vcr_cassettes/pig_job/apache_log_reports.yml
+- spec/fixtures/vcr_cassettes/pig_job/apache_log_reports_with_bootstrap.yml
 - spec/fixtures/vcr_cassettes/run_jobflow/word_count.yml
 - spec/fixtures/vcr_cassettes/set_termination_protection/nonexistent_job_flows.yml
 - spec/fixtures/vcr_cassettes/set_termination_protection/protect_multiple_job_flows.yml
@@ -240,6 +241,7 @@ test_files:
 - spec/fixtures/vcr_cassettes/hive_job/hive_ads.yml
 - spec/fixtures/vcr_cassettes/modify_instance_groups/set_instances_to_3.yml
 - spec/fixtures/vcr_cassettes/pig_job/apache_log_reports.yml
+- spec/fixtures/vcr_cassettes/pig_job/apache_log_reports_with_bootstrap.yml
 - spec/fixtures/vcr_cassettes/run_jobflow/word_count.yml
 - spec/fixtures/vcr_cassettes/set_termination_protection/nonexistent_job_flows.yml
 - spec/fixtures/vcr_cassettes/set_termination_protection/protect_multiple_job_flows.yml