elasticity 1.1.1 → 1.2

data/HISTORY.mediawiki

@@ -1,18 +1,22 @@
-=== 1.0 ===
+=== 1.2 ===
 
-* Released!
+* Added PigJob!
 
-=== 1.0.1 ===
+=== 1.1.1 ===
 
-* Added LICENSE.
+* HiveJob critical bug fixed, now it works :)
+* Added log_uri and action_on_failure as options to HiveJob.
+* Added integration tests to HiveJob.
 
 === 1.1 ===
 
 * Added HiveJob, a simplified way to launch basic Hive job flows.
 * Added HISTORY.
 
-=== 1.1.1 ===
+=== 1.0.1 ===
 
-* HiveJob critical bug fixed, now it works :)
-* Added log_uri and action_on_failure as options to HiveJob.
-* Added integration tests to HiveJob.
+* Added LICENSE.
+
+=== 1.0 ===
+
+* Released!

data/README.mediawiki

@@ -14,7 +14,19 @@ All you have to do is <code>require 'elasticity'</code> and you're all set!
 
 = Simplified API Reference =
 
-Elasticity currently provides simplified access to launching Hive job flows.  Eventually I'd like to add several simplified wrappers, but I only need Hive at the moment :)
+Elasticity currently provides simplified access to launching Hive and Pig job flows, specifying several default values that you may optionally override:
+
+<pre>
+  @action_on_failure = "TERMINATE_JOB_FLOW"
+  @ec2_key_name = "default"
+  @hadoop_version = "0.20"
+  @instance_count = 2
+  @master_instance_type = "m1.small"
+  @name = "Elasticity Job"
+  @slave_instance_type = "m1.small"
+</pre>
+
+These are all accessible from HiveJob and PigJob instances.  See the PigJob description for an example.
 
 == Hive ==
 
@@ -30,6 +42,22 @@ HiveJob allows you to quickly launch Hive jobs without having to understand the
   > "j-129V5AQFMKO1C"
 </pre>
 
+== Pig ==
+
+Like HiveJob, PigJob allows you to quickly launch Pig jobs :)
+
+<pre>
+  pig = Elasticity::PigJob.new(ENV["AWS_ACCESS_KEY_ID"], ENV["AWS_SECRET_KEY"])
+  pig.log_uri = "s3n://slif-elasticity/pig-apache/logs"
+  pig.ec2_key_name = "slif_dev"
+  pig.run("s3n://elasticmapreduce/samples/pig-apache/do-reports.pig", {
+    "INPUT"  => "s3n://elasticmapreduce/samples/pig-apache/input",
+    "OUTPUT" => "s3n://slif-elasticity/pig-apache/output/2011-05-04"
+  })
+  
+  > "j-16PZ24OED71C6"
+</pre>
+
 = Amazon API Reference =
 
 Elasticity wraps all of the EMR API calls.  Please see the Amazon guide for details on these operations because the default values aren't obvious (e.g. the meaning of <code>DescribeJobFlows</code> without parameters).

data/lib/elasticity.rb

@@ -5,9 +5,12 @@ require 'nokogiri'
 
 require 'elasticity/aws_request'
 require 'elasticity/emr'
-require 'elasticity/hive_job'
 require 'elasticity/job_flow'
 require 'elasticity/job_flow_step'
 
+require 'elasticity/simple_job'
+require 'elasticity/hive_job'
+require 'elasticity/pig_job'
+
 module Elasticity
 end

data/lib/elasticity/hive_job.rb

@@ -2,31 +2,11 @@ module Elasticity
 
   # HiveJob allows you quickly easily kick off a Hive jobflow without
   # having to understand the entirety of the EMR API.
-  class HiveJob
-
-    attr_accessor :action_on_failure
-    attr_accessor :aws_access_key_id
-    attr_accessor :aws_secret_access_key
-    attr_accessor :ec2_key_name
-    attr_accessor :name
-    attr_accessor :hadoop_version
-    attr_accessor :instance_count
-    attr_accessor :log_uri
-    attr_accessor :master_instance_type
-    attr_accessor :slave_instance_type
+  class HiveJob < Elasticity::SimpleJob
 
     def initialize(aws_access_key_id, aws_secret_access_key)
-      @action_on_failure = "TERMINATE_JOB_FLOW"
-      @aws_access_key_id = aws_access_key_id
-      @aws_secret_access_key = aws_secret_access_key
-      @ec2_key_name = "default"
-      @hadoop_version = "0.20"
-      @instance_count = 2
-      @master_instance_type = "m1.small"
+      super
       @name = "Elasticity Hive Job"
-      @slave_instance_type = "m1.small"
-
-      @emr = Elasticity::EMR.new(aws_access_key_id, aws_secret_access_key)
     end
 
     # Run the specified Hive script with the specified variables.

data/lib/elasticity/pig_job.rb

@@ -0,0 +1,67 @@
+module Elasticity
+
+  class PigJob < Elasticity::SimpleJob
+
+    def initialize(aws_access_key_id, aws_secret_access_key)
+      super
+      @name = "Elasticity Pig Job"
+    end
+
+    # Run the specified Pig script with the specified variables.
+    #
+    #   pig = Elasticity::PigJob.new("access", "secret")
+    #   jobflow_id = pig.run('s3n://slif-pig-test/test.pig', {
+    #     'SCRIPTS' => 's3n://slif-pig-test/scripts',
+    #     'OUTPUT'  => 's3n://slif-pig-test/output',
+    #     'XREFS'   => 's3n://slif-pig-test/xrefs'
+    #   })
+    #
+    # The variables are accessible within your Pig scripts by using the
+    # standard ${NAME} syntax.
+    def run(pig_script, pig_variables={})
+      script_arguments = ["s3://elasticmapreduce/libs/pig/pig-script", "--run-pig-script", "--args"]
+      pig_variables.keys.sort.each do |variable_name|
+        script_arguments.concat(["-p", "#{variable_name}=#{pig_variables[variable_name]}"])
+      end
+      script_arguments << pig_script
+      jobflow_config = {
+        :name => @name,
+        :instances => {
+          :ec2_key_name => @ec2_key_name,
+          :hadoop_version => @hadoop_version,
+          :instance_count => @instance_count,
+          :master_instance_type => @master_instance_type,
+          :slave_instance_type => @slave_instance_type,
+        },
+        :steps => [
+          {
+            :action_on_failure => "TERMINATE_JOB_FLOW",
+            :hadoop_jar_step => {
+              :jar => "s3://elasticmapreduce/libs/script-runner/script-runner.jar",
+              :args => [
+                "s3://elasticmapreduce/libs/pig/pig-script",
+                  "--base-path", "s3://elasticmapreduce/libs/pig/",
+                  "--install-pig"
+              ],
+            },
+            :name => "Setup Pig"
+          },
+            {
+              :action_on_failure => @action_on_failure,
+              :hadoop_jar_step => {
+                :jar => "s3://elasticmapreduce/libs/script-runner/script-runner.jar",
+                :args => script_arguments,
+              },
+              :name => "Run Pig Script"
+            }
+        ]
+      }
+
+      jobflow_config.merge!(:log_uri => @log_uri) if @log_uri
+
+      @emr.run_job_flow(jobflow_config)
+    end
+    
+  end
+  
+end

data/lib/elasticity/simple_job.rb

@@ -0,0 +1,32 @@
+module Elasticity
+
+  class SimpleJob
+
+    attr_accessor :action_on_failure
+    attr_accessor :aws_access_key_id
+    attr_accessor :aws_secret_access_key
+    attr_accessor :ec2_key_name
+    attr_accessor :name
+    attr_accessor :hadoop_version
+    attr_accessor :instance_count
+    attr_accessor :log_uri
+    attr_accessor :master_instance_type
+    attr_accessor :slave_instance_type
+
+    def initialize(aws_access_key_id, aws_secret_access_key)
+      @action_on_failure = "TERMINATE_JOB_FLOW"
+      @aws_access_key_id = aws_access_key_id
+      @aws_secret_access_key = aws_secret_access_key
+      @ec2_key_name = "default"
+      @hadoop_version = "0.20"
+      @instance_count = 2
+      @master_instance_type = "m1.small"
+      @name = "Elasticity Job"
+      @slave_instance_type = "m1.small"
+
+      @emr = Elasticity::EMR.new(aws_access_key_id, aws_secret_access_key)
+    end
+
+  end
+
+end

data/lib/elasticity/version.rb

@@ -1,3 +1,3 @@
 module Elasticity
-  VERSION = "1.1.1"
+  VERSION = "1.2"
 end

data/spec/fixtures/vcr_cassettes/pig_job/apache_log_reports.yml

@@ -0,0 +1,35 @@
+---
+- !ruby/struct:VCR::HTTPInteraction 
+  request: !ruby/struct:VCR::Request 
+    method: :get
+    uri: !ruby/regexp /^http:\/\/elasticmapreduce\.amazonaws\.com:80\/\?AWSAccessKeyId=AKIAI7HEMMNKGT6VFFSA&Instances\.Ec2KeyName=sharethrough_dev&Instances\.HadoopVersion=0\.20&Instances\.InstanceCount=2&Instances\.MasterInstanceType=m1\.small&Instances\.SlaveInstanceType=m1\.small&LogUri=s3n:\/\/slif-elasticity\/pig-apache\/logs&Name=Elasticity%20Pig%20Job&Operation=RunJobFlow&.*&Steps\.member\.1\.ActionOnFailure=TERMINATE_JOB_FLOW&Steps\.member\.1\.HadoopJarStep\.Args\.member\.1=s3:\/\/elasticmapreduce\/libs\/pig\/pig-script&Steps\.member\.1\.HadoopJarStep\.Args\.member\.2=--base-path&Steps\.member\.1\.HadoopJarStep\.Args\.member\.3=s3:\/\/elasticmapreduce\/libs\/pig\/&Steps\.member\.1\.HadoopJarStep\.Args\.member\.4=--install-pig&Steps\.member\.1\.HadoopJarStep\.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner\.jar&Steps\.member\.1\.Name=Setup%20Pig&Steps\.member\.2\.ActionOnFailure=TERMINATE_JOB_FLOW&Steps\.member\.2\.HadoopJarStep\.Args\.member\.1=s3:\/\/elasticmapreduce\/libs\/pig\/pig-script&Steps\.member\.2\.HadoopJarStep\.Args\.member\.2=--run-pig-script&Steps\.member\.2\.HadoopJarStep\.Args\.member\.3=--args&Steps\.member\.2\.HadoopJarStep\.Args\.member\.4=-p&Steps\.member\.2\.HadoopJarStep\.Args\.member\.5=INPUT=s3n:\/\/elasticmapreduce\/samples\/pig-apache\/input&Steps\.member\.2\.HadoopJarStep\.Args\.member\.6=-p&Steps\.member\.2\.HadoopJarStep\.Args\.member\.7=OUTPUT=s3n:\/\/slif-elasticity\/pig-apache\/output\/2011-05-04&Steps\.member\.2\.HadoopJarStep\.Args\.member\.8=s3n:\/\/elasticmapreduce\/samples\/pig-apache\/do-reports\.pig&Steps\.member\.2\.HadoopJarStep\.Jar=s3:\/\/elasticmapreduce\/libs\/script-runner\/script-runner\.jar&Steps\.member\.2\.Name=Run%20Pig%20Script/
+    body:
+    headers: 
+      accept: 
+      - "*/*; q=0.5, application/xml"
+      accept-encoding: 
+      - gzip, deflate
+  response: !ruby/struct:VCR::Response 
+    status: !ruby/struct:VCR::ResponseStatus 
+      code: 200
+      message: OK
+    headers: 
+      x-amzn-requestid: 
+      - b237df84-761d-11e0-b625-05a26eeda1d8
+      content-type: 
+      - text/xml
+      date: 
+      - Wed, 04 May 2011 07:11:13 GMT
+      content-length: 
+      - "297"
+    body: |
+      <RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
+        <RunJobFlowResult>
+          <JobFlowId>j-16PZ24OED71C6</JobFlowId>
+        </RunJobFlowResult>
+        <ResponseMetadata>
+          <RequestId>b237df84-761d-11e0-b625-05a26eeda1d8</RequestId>
+        </ResponseMetadata>
+      </RunJobFlowResponse>
+
+    http_version: "1.1"

data/spec/lib/elasticity/pig_job_spec.rb

@@ -0,0 +1,96 @@
+require 'spec_helper'
+
+describe Elasticity::PigJob do
+
+  describe ".new" do
+
+    it "should have good defaults" do
+      hive = Elasticity::PigJob.new("access", "secret")
+      hive.aws_access_key_id.should == "access"
+      hive.aws_secret_access_key.should == "secret"
+      hive.ec2_key_name.should == "default"
+      hive.hadoop_version.should == "0.20"
+      hive.instance_count.should == 2
+      hive.master_instance_type.should == "m1.small"
+      hive.name.should == "Elasticity Pig Job"
+      hive.slave_instance_type.should == "m1.small"
+      hive.action_on_failure.should == "TERMINATE_JOB_FLOW"
+      hive.log_uri.should == nil
+    end
+
+  end
+
+  describe "#run" do
+
+    it "should run the script with the specified variables and return the jobflow_id" do
+      aws = Elasticity::EMR.new("", "")
+      aws.should_receive(:run_job_flow).with({
+        :name => "Elasticity Pig Job",
+        :log_uri => "s3n://slif-test/output/logs",
+        :instances => {
+          :ec2_key_name => "default",
+          :hadoop_version => "0.20",
+          :instance_count => 2,
+          :master_instance_type => "m1.small",
+          :slave_instance_type => "m1.small",
+        },
+        :steps => [
+          {
+            :action_on_failure => "TERMINATE_JOB_FLOW",
+            :hadoop_jar_step => {
+              :jar => "s3://elasticmapreduce/libs/script-runner/script-runner.jar",
+              :args => [
+                "s3://elasticmapreduce/libs/pig/pig-script",
+                  "--base-path",
+                  "s3://elasticmapreduce/libs/pig/",
+                  "--install-pig"
+              ],
+            },
+            :name => "Setup Pig"
+          },
+            {
+              :action_on_failure => "CONTINUE",
+              :hadoop_jar_step => {
+                :jar => "s3://elasticmapreduce/libs/script-runner/script-runner.jar",
+                :args => [
+                  "s3://elasticmapreduce/libs/pig/pig-script",
+                    "--run-pig-script",
+                    "--args",
+                    "-p", "OUTPUT=s3n://slif-pig-test/output",
+                    "-p", "XREFS=s3n://slif-pig-test/xrefs",
+                    "s3n://slif-pig-test/test.pig"
+                ],
+              },
+              :name => "Run Pig Script"
+            }
+        ]
+      }).and_return("new_jobflow_id")
+      Elasticity::EMR.should_receive(:new).with("access", "secret").and_return(aws)
+
+      pig = Elasticity::PigJob.new("access", "secret")
+      pig.log_uri = "s3n://slif-test/output/logs"
+      pig.action_on_failure = "CONTINUE"
+      jobflow_id = pig.run('s3n://slif-pig-test/test.pig', {
+        'OUTPUT' => 's3n://slif-pig-test/output',
+        'XREFS' => 's3n://slif-pig-test/xrefs'
+      })
+      jobflow_id.should == "new_jobflow_id"
+    end
+
+  end
+
+  describe "integration happy path" do
+    use_vcr_cassette "pig_job/apache_log_reports", :record => :none
+    it "should kick off the sample Amazion EMR Pig application" do
+      pig = Elasticity::PigJob.new(AWS_ACCESS_KEY_ID, AWS_SECRET_KEY)
+      pig.log_uri = "s3n://slif-elasticity/pig-apache/logs"
+      pig.ec2_key_name = "sharethrough_dev"
+      jobflow_id = pig.run("s3n://elasticmapreduce/samples/pig-apache/do-reports.pig", {
+        "INPUT"  => "s3n://elasticmapreduce/samples/pig-apache/input",
+        "OUTPUT" => "s3n://slif-elasticity/pig-apache/output/2011-05-04"
+      })
+      jobflow_id.should == "j-16PZ24OED71C6"
+    end
+  end
+
+end

metadata

@@ -1,13 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: elasticity
 version: !ruby/object:Gem::Version 
-  hash: 17
+  hash: 11
   prerelease: 
   segments: 
   - 1
-  - 1
-  - 1
-  version: 1.1.1
+  - 2
+  version: "1.2"
 platform: ruby
 authors: 
 - Robert Slifka
@@ -15,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-25 00:00:00 -07:00
+date: 2011-05-04 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -175,6 +174,8 @@ files:
 - lib/elasticity/hive_job.rb
 - lib/elasticity/job_flow.rb
 - lib/elasticity/job_flow_step.rb
+- lib/elasticity/pig_job.rb
+- lib/elasticity/simple_job.rb
 - lib/elasticity/version.rb
 - spec/fixtures/vcr_cassettes/add_instance_groups/one_group_successful.yml
 - spec/fixtures/vcr_cassettes/add_instance_groups/one_group_unsuccessful.yml
@@ -183,6 +184,7 @@ files:
 - spec/fixtures/vcr_cassettes/direct/terminate_jobflow.yml
 - spec/fixtures/vcr_cassettes/hive_job/hive_ads.yml
 - spec/fixtures/vcr_cassettes/modify_instance_groups/set_instances_to_3.yml
+- spec/fixtures/vcr_cassettes/pig_job/apache_log_reports.yml
 - spec/fixtures/vcr_cassettes/run_jobflow/word_count.yml
 - spec/fixtures/vcr_cassettes/set_termination_protection/nonexistent_job_flows.yml
 - spec/fixtures/vcr_cassettes/set_termination_protection/protect_multiple_job_flows.yml
@@ -192,6 +194,7 @@ files:
 - spec/lib/elasticity/hive_job_spec.rb
 - spec/lib/elasticity/job_flow_spec.rb
 - spec/lib/elasticity/job_flow_step_spec.rb
+- spec/lib/elasticity/pig_job_spec.rb
 - spec/spec_helper.rb
 has_rdoc: true
 homepage: http://www.github.com/rslifka/elasticity
@@ -235,6 +238,7 @@ test_files:
 - spec/fixtures/vcr_cassettes/direct/terminate_jobflow.yml
 - spec/fixtures/vcr_cassettes/hive_job/hive_ads.yml
 - spec/fixtures/vcr_cassettes/modify_instance_groups/set_instances_to_3.yml
+- spec/fixtures/vcr_cassettes/pig_job/apache_log_reports.yml
 - spec/fixtures/vcr_cassettes/run_jobflow/word_count.yml
 - spec/fixtures/vcr_cassettes/set_termination_protection/nonexistent_job_flows.yml
 - spec/fixtures/vcr_cassettes/set_termination_protection/protect_multiple_job_flows.yml
@@ -244,4 +248,5 @@ test_files:
 - spec/lib/elasticity/hive_job_spec.rb
 - spec/lib/elasticity/job_flow_spec.rb
 - spec/lib/elasticity/job_flow_step_spec.rb
+- spec/lib/elasticity/pig_job_spec.rb
 - spec/spec_helper.rb