elasticity 1.0.1 → 1.1

data/.autotest

@@ -0,0 +1,2 @@
+require 'autotest/fsevent'
+require 'autotest/growl'

data/HISTORY.mediawiki

@@ -0,0 +1,12 @@
+=== 1.0 ===
+
+* Released!
+
+=== 1.0.1 ===
+
+* Added LICENSE.
+
+=== 1.1 ===
+
+* Added HiveJob, a simplified way to launch basic Hive job flows.
+* Added HISTORY.

data/README.mediawiki

@@ -1,4 +1,4 @@
-Elasticity provides programmatic access to Amazon's Elastic Map Reduce service.  The aim is to conveniently wrap the API operations in a manner that makes working with EMR job flows from Ruby more enjoyable.  At the very least, using Elasticity allows you to easily experiment with the EMR API :)
+Elasticity provides programmatic access to Amazon's Elastic Map Reduce service.  The aim is to conveniently wrap the API operations in a manner that makes working with EMR job flows from Ruby more productive and more enjoyable, without having to understand the nuts and bolts of the EMR REST API.  At the very least, using Elasticity allows you to easily experiment with the EMR API :)
 
 '''BACKLOG''': Have a look at the [https://www.pivotaltracker.com/projects/272429 backlog] to see where this is headed.
 
@@ -12,7 +12,25 @@ Elasticity provides programmatic access to Amazon's Elastic Map Reduce service.
 
 All you have to do is <code>require 'elasticity'</code> and you're all set!
 
-= API Reference =
+= Simplified API Reference =
+
+Elasticity currently provides simplified access to launching Hive job flows.  Eventually I'd like to add several simplified wrappers, but I only need Hive at the moment :)
+
+== Hive ==
+
+HiveJob allows you to quickly launch Hive jobs without having to understand the ins and outs of the EMR API.  Specify only the Hive script location and (optionally) variables to make available to the Hive script.
+
+<pre>
+  hive = Elasticity::HiveJob.new(ENV["AWS_ACCESS_KEY_ID"], ENV["AWS_SECRET_KEY"])
+  hive.run("s3n://slif-hive/test.q", {
+    "LIB"    => "s3n://slif-test/lib",
+    "OUTPUT" => "s3n://slif-test/output"
+  })
+  
+  > "j-129V5AQFMKO1C"
+</pre>
+
+= Amazon API Reference =
 
 Elasticity wraps all of the EMR API calls.  Please see the Amazon guide for details on these operations because the default values aren't obvious (e.g. the meaning of <code>DescribeJobFlows</code> without parameters).
 

data/elasticity.gemspec

@@ -16,10 +16,13 @@ Gem::Specification.new do |s|
   s.add_dependency("rest-client")
   s.add_dependency("nokogiri")
 
+  s.add_development_dependency("autotest-fsevent")
+  s.add_development_dependency("autotest-growl")
   s.add_development_dependency("rake")
   s.add_development_dependency("rspec",   ">= 2.5.0")
   s.add_development_dependency("vcr",     ">= 1.5.1")
   s.add_development_dependency("webmock", ">= 1.6.2")
+  s.add_development_dependency("ZenTest")
 
   s.files         = `git ls-files`.split("\n")
   s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")

data/lib/elasticity.rb

@@ -5,6 +5,7 @@ require 'nokogiri'
 
 require 'elasticity/aws_request'
 require 'elasticity/emr'
+require 'elasticity/hive_job'
 require 'elasticity/job_flow'
 require 'elasticity/job_flow_step'
 

data/lib/elasticity/hive_job.rb

@@ -0,0 +1,85 @@
+module Elasticity
+
+  # HiveJob allows you quickly easily kick off a Hive jobflow without
+  # having to understand the entirety of the EMR API.
+  class HiveJob
+
+    attr_accessor :aws_access_key_id
+    attr_accessor :aws_secret_access_key
+    attr_accessor :ec2_key_name
+    attr_accessor :name
+    attr_accessor :hadoop_version
+    attr_accessor :instance_count
+    attr_accessor :master_instance_type
+    attr_accessor :slave_instance_type
+
+    def initialize(aws_access_key_id, aws_secret_access_key)
+      @aws_access_key_id = aws_access_key_id
+      @aws_secret_access_key = aws_secret_access_key
+      @ec2_key_name = "default"
+      @hadoop_version = "0.20"
+      @instance_count = 2
+      @master_instance_type = "m1.small"
+      @name = "Elasticity Hive Job"
+      @slave_instance_type = "m1.small"
+
+      @aws_request = Elasticity::AwsRequest.new(aws_access_key_id, aws_secret_access_key)
+    end
+
+    # Run the specified Hive script with the specified variables.
+    #
+    #   hive = Elasticity::HiveJob.new("access", "secret")
+    #   jobflow_id = hive.run('s3n://slif-hive/test.q', {
+    #     'SCRIPTS' => 's3n://slif-test/scripts',
+    #     'OUTPUT'  => 's3n://slif-test/output',
+    #     'XREFS'   => 's3n://slif-test/xrefs'
+    #   })
+    #
+    # The variables are accessible within your Hive scripts by using the
+    # standard ${NAME} syntax.  E.g.
+    #
+    #   ADD JAR ${SCRIPTS}/jsonserde.jar;
+    def run(hive_script, hive_variables={})
+      script_arguments = ["s3://elasticmapreduce/libs/hive/hive-script", "--run-hive-script", "--args"]
+      script_arguments.concat(["-f", hive_script])
+      hive_variables.each do |variable_name, value|
+        script_arguments.concat(["-d", "#{variable_name}=#{value}"])
+      end
+      jobflow_config = {
+        :name => @name,
+        :instances => {
+          :ec2_key_name => @ec2_key_name,
+          :hadoop_version => @hadoop_version,
+          :instance_count => @instance_count,
+          :master_instance_type => @master_instance_type,
+          :slave_instance_type => @slave_instance_type,
+        },
+        :steps => [
+          {
+            :action_on_failure => "TERMINATE_JOB_FLOW",
+            :hadoop_jar_step => {
+              :jar => "s3://elasticmapreduce/libs/script-runner/script-runner.jar",
+              :args => [
+                "s3://elasticmapreduce/libs/hive/hive-script",
+                  "--base-path", "s3://elasticmapreduce/libs/hive/",
+                  "--install-hive"
+              ],
+            },
+            :name => "Setup Hive"
+          },
+            {
+              :action_on_failure => "TERMINATE_JOB_FLOW",
+              :hadoop_jar_step => {
+                :jar => "s3://elasticmapreduce/libs/script-runner/script-runner.jar",
+                :args => script_arguments,
+              },
+              :name => "Run Hive Script"
+            }
+        ]
+      }
+      @aws_request.run_job_flow(jobflow_config)
+    end
+
+  end
+
+end

data/lib/elasticity/version.rb

@@ -1,3 +1,3 @@
 module Elasticity
-  VERSION = "1.0.1"
+  VERSION = "1.1"
 end

data/spec/lib/elasticity/hive_job_spec.rb

@@ -0,0 +1,77 @@
+require 'spec_helper'
+
+describe Elasticity::HiveJob do
+
+  describe ".new" do
+
+    it "should have good defaults" do
+      hive = Elasticity::HiveJob.new("access", "secret")
+      hive.aws_access_key_id.should == "access"
+      hive.aws_secret_access_key.should == "secret"
+      hive.ec2_key_name.should == "default"
+      hive.hadoop_version.should == "0.20"
+      hive.instance_count.should == 2
+      hive.master_instance_type.should == "m1.small"
+      hive.name.should == "Elasticity Hive Job"
+      hive.slave_instance_type.should == "m1.small"
+    end
+
+  end
+
+  describe "#run" do
+
+    it "run the script with the specified variables and return the jobflow_id" do
+      aws = Elasticity::AwsRequest.new("", "")
+      aws.should_receive(:run_job_flow).with({
+        :name => "Elasticity Hive Job",
+        :instances => {
+          :ec2_key_name => "default",
+          :hadoop_version => "0.20",
+          :instance_count => 2,
+          :master_instance_type => "m1.small",
+          :slave_instance_type => "m1.small",
+        },
+        :steps => [
+          {
+            :action_on_failure => "TERMINATE_JOB_FLOW",
+            :hadoop_jar_step => {
+              :jar => "s3://elasticmapreduce/libs/script-runner/script-runner.jar",
+              :args => [
+                "s3://elasticmapreduce/libs/hive/hive-script",
+                  "--base-path",
+                  "s3://elasticmapreduce/libs/hive/",
+                  "--install-hive"
+              ],
+            },
+            :name => "Setup Hive"
+          },
+            {
+              :action_on_failure => "TERMINATE_JOB_FLOW",
+              :hadoop_jar_step => {
+                :jar => "s3://elasticmapreduce/libs/script-runner/script-runner.jar",
+                :args => [
+                  "s3://elasticmapreduce/libs/hive/hive-script",
+                    "--run-hive-script",
+                    "--args",
+                    "-f", "s3n://slif-hive/test.q",
+                    "-d", "XREFS=s3n://slif-test/xrefs",
+                    "-d", "OUTPUT=s3n://slif-test/output"
+                ],
+              },
+              :name => "Run Hive Script"
+            }
+        ]
+      }).and_return("new_jobflow_id")
+      Elasticity::AwsRequest.should_receive(:new).with("access", "secret").and_return(aws)
+
+      hive = Elasticity::HiveJob.new("access", "secret")
+      jobflow_id = hive.run('s3n://slif-hive/test.q', {
+        'OUTPUT' => 's3n://slif-test/output',
+        'XREFS' => 's3n://slif-test/xrefs'
+      })
+      jobflow_id.should == "new_jobflow_id"
+    end
+
+  end
+
+end

metadata

@@ -1,13 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: elasticity
 version: !ruby/object:Gem::Version 
-  hash: 21
+  hash: 13
   prerelease: 
   segments: 
   - 1
-  - 0
   - 1
-  version: 1.0.1
+  version: "1.1"
 platform: ruby
 authors: 
 - Robert Slifka
@@ -15,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-04-22 00:00:00 -07:00
+date: 2011-04-24 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -47,7 +46,7 @@ dependencies:
   type: :runtime
   version_requirements: *id002
 - !ruby/object:Gem::Dependency 
-  name: rake
+  name: autotest-fsevent
   prerelease: false
   requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
@@ -61,9 +60,37 @@ dependencies:
   type: :development
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: rspec
+  name: autotest-growl
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -75,11 +102,11 @@ dependencies:
         - 0
         version: 2.5.0
   type: :development
-  version_requirements: *id004
+  version_requirements: *id006
 - !ruby/object:Gem::Dependency 
   name: vcr
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  requirement: &id007 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -91,11 +118,11 @@ dependencies:
         - 1
         version: 1.5.1
   type: :development
-  version_requirements: *id005
+  version_requirements: *id007
 - !ruby/object:Gem::Dependency 
   name: webmock
   prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  requirement: &id008 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -107,7 +134,21 @@ dependencies:
         - 2
         version: 1.6.2
   type: :development
-  version_requirements: *id006
+  version_requirements: *id008
+- !ruby/object:Gem::Dependency 
+  name: ZenTest
+  prerelease: false
+  requirement: &id009 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id009
 description: Programmatic access to Amazon's Elastic Map Reduce service.
 email: 
 executables: []
@@ -117,10 +158,12 @@ extensions: []
 extra_rdoc_files: []
 
 files: 
+- .autotest
 - .gitignore
 - .rspec
 - .rvmrc
 - Gemfile
+- HISTORY.mediawiki
 - LICENSE
 - README.mediawiki
 - Rakefile
@@ -128,6 +171,7 @@ files:
 - lib/elasticity.rb
 - lib/elasticity/aws_request.rb
 - lib/elasticity/emr.rb
+- lib/elasticity/hive_job.rb
 - lib/elasticity/job_flow.rb
 - lib/elasticity/job_flow_step.rb
 - lib/elasticity/version.rb
@@ -143,6 +187,7 @@ files:
 - spec/fixtures/vcr_cassettes/terminate_jobflows/one_jobflow.yml
 - spec/lib/elasticity/aws_request_spec.rb
 - spec/lib/elasticity/emr_spec.rb
+- spec/lib/elasticity/hive_job_spec.rb
 - spec/lib/elasticity/job_flow_spec.rb
 - spec/lib/elasticity/job_flow_step_spec.rb
 - spec/spec_helper.rb
@@ -193,6 +238,7 @@ test_files:
 - spec/fixtures/vcr_cassettes/terminate_jobflows/one_jobflow.yml
 - spec/lib/elasticity/aws_request_spec.rb
 - spec/lib/elasticity/emr_spec.rb
+- spec/lib/elasticity/hive_job_spec.rb
 - spec/lib/elasticity/job_flow_spec.rb
 - spec/lib/elasticity/job_flow_step_spec.rb
 - spec/spec_helper.rb