elasticity 2.5.1 → 2.5.2

data/HISTORY.md

@@ -1,7 +1,11 @@
+## 2.5.2 - November 29, 2012
+
++ Configuration of Hive installations via ```hive_site``` is now supported.  See the README.md for details.
+
 ## 2.5.1 - November 28, 2012
 
 + When ```JobFlow#placement``` is specified, instances are created in that availability zone.  Previously, this setting was only used to derive the EMR API endpoint to connect to (i.e. the region).
-+ Updated development depdencies.
++ Updated development dependencies.
 
 ## 2.5 - September 29, 2012
 

data/README.md

@@ -1,5 +1,3 @@
-(2012-11-23) Hey all, I'm still out here.  Elasticity remains part of our daily workflow at [Sharethrough](http://www.sharethrough.com/) and after a year of work, our usage patterns have stabilized.  Submit an issue if you have suggestions!
-
 Elasticity provides programmatic access to Amazon's Elastic Map Reduce service.  The aim is to conveniently abstract away the complex EMR REST API and make working with job flows more productive and more enjoyable.
 
 [![Build Status](https://secure.travis-ci.org/rslifka/elasticity.png)](http://travis-ci.org/rslifka/elasticity) REE, 1.8.7, 1.9.2, 1.9.3
@@ -323,6 +321,19 @@ If that's the case, or if you'd just like to terminate a running jobflow before
 jobflow.shutdown
 ```
 
+# Elasticity Configuration
+
+Elasticity supports a wide range of configuration options :) all of which are shown below.
+
+```ruby
+Elasticity.configure do |config|
+
+  # If using Hive, it will be configured via the directives here
+  config.hive_site = 's3://bucket/hive-site.xml'
+  
+end
+```
+
 # Amazon EMR Documentation
 
 Elasticity wraps all of the EMR API calls.  Please see the Amazon guide for details on these operations because the default values aren't obvious (e.g. the meaning of <code>DescribeJobFlows</code> without parameters).

data/lib/elasticity.rb

@@ -27,4 +27,25 @@ require 'elasticity/pig_step'
 require 'elasticity/streaming_step'
 
 module Elasticity
+
+  class << self
+    attr_reader :configuration
+
+    def configuration
+      @configuration ||= Configuration.new
+    end
+
+    def default_configuration
+      @configuration = Configuration.new
+    end
+
+    def configure
+      yield(configuration)
+    end
+  end
+
+  class Configuration
+    attr_accessor :hive_site
+  end
+
 end

data/lib/elasticity/hive_step.rb

@@ -12,11 +12,11 @@ module Elasticity
     def initialize(script)
       @name = "Elasticity Hive Step (#{script})"
       @script = script
-      @variables = { }
+      @variables = {}
       @action_on_failure = 'TERMINATE_JOB_FLOW'
     end
 
-    def to_aws_step(job_flow)
+    def to_aws_step(_)
       args = %w(s3://elasticmapreduce/libs/hive/hive-script --base-path s3://elasticmapreduce/libs/hive/ --hive-versions latest --run-hive-script --args)
       args.concat(['-f', @script])
       @variables.keys.sort.each do |name|
@@ -36,15 +36,40 @@ module Elasticity
       true
     end
 
-    def self.aws_installation_step
-      {
-        :action_on_failure => 'TERMINATE_JOB_FLOW',
-        :hadoop_jar_step => {
-          :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
-          :args => %w(s3://elasticmapreduce/libs/hive/hive-script --base-path s3://elasticmapreduce/libs/hive/ --install-hive --hive-versions latest)
-        },
-        :name => 'Elasticity - Install Hive'
-      }
+    def self.aws_installation_step_name
+      'Elasticity - Install Hive'
+    end
+
+    def self.aws_installation_steps
+      steps = [
+        {
+          :action_on_failure => 'TERMINATE_JOB_FLOW',
+          :hadoop_jar_step => {
+            :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
+            :args => %w(s3://elasticmapreduce/libs/hive/hive-script --base-path s3://elasticmapreduce/libs/hive/ --install-hive --hive-versions latest)
+          },
+          :name => aws_installation_step_name
+        }
+      ]
+      if Elasticity.configuration.hive_site
+        steps << {
+          :action_on_failure => 'TERMINATE_JOB_FLOW',
+          :hadoop_jar_step => {
+            :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
+            :args => [
+              's3://elasticmapreduce/libs/hive/hive-script',
+              '--base-path',
+              's3://elasticmapreduce/libs/hive/',
+              '--install-hive-site',
+              "--hive-site=#{Elasticity.configuration.hive_site}",
+              '--hive-versions',
+              'latest'
+            ],
+          },
+          :name => 'Elasticity - Configure Hive via Hive Site'
+        }
+      end
+      steps
     end
 
   end

data/lib/elasticity/job_flow.rb

@@ -98,7 +98,7 @@ module Elasticity
       if is_jobflow_running?
         jobflow_steps = []
         if jobflow_step.requires_installation? && !@installed_steps.include?(jobflow_step.class)
-          jobflow_steps << jobflow_step.aws_installation_step
+          jobflow_steps.concat(jobflow_step.aws_installation_steps)
         end
         jobflow_steps << jobflow_step.to_aws_step(self)
         emr.add_jobflow_steps(@jobflow_id, {:steps => jobflow_steps})
@@ -172,7 +172,7 @@ module Elasticity
       steps = []
       @jobflow_steps.each do |step|
         if step.class.send(:requires_installation?) && !@installed_steps.include?(step.class)
-          steps << step.class.send(:aws_installation_step)
+          steps.concat(step.class.send(:aws_installation_steps))
           @installed_steps << step.class
         end
         steps << step.to_aws_step(self)

data/lib/elasticity/job_flow_status.rb

@@ -35,7 +35,7 @@ module Elasticity
 
       step_names = jobflow.steps.map(&:name)
       Elasticity::JobFlowStep.steps_requiring_installation.each do |step|
-        jobflow.installed_steps << step if step_names.include?(step.aws_installation_step[:name])
+        jobflow.installed_steps << step if step_names.include?(step.aws_installation_step_name)
       end
 
       jobflow.created_at = Time.parse(xml_element.xpath('./ExecutionStatusDetail/CreationDateTime').text.strip)

data/lib/elasticity/job_flow_step.rb

@@ -12,8 +12,12 @@ module Elasticity
       self.class.requires_installation?
     end
 
-    def aws_installation_step
-      self.class.aws_installation_step
+    def aws_installation_steps
+      self.class.aws_installation_steps
+    end
+
+    def aws_installation_step_name
+      self.class.aws_installation_step_name
     end
 
     module ClassMethods
@@ -22,10 +26,14 @@ module Elasticity
         false
       end
 
-      def aws_installation_step
+      def aws_installation_steps
         raise RuntimeError, '.aws_installation_step is required to be defined when a step requires installation (e.g. Pig, Hive).'
       end
 
+      def aws_installation_step_name
+        raise RuntimeError, '.aws_installation_step_name is required to be defined when a step requires installation (e.g. Pig, Hive).'
+      end
+
     end
 
     def self.included(base)

data/lib/elasticity/pig_step.rb

@@ -12,7 +12,7 @@ module Elasticity
     def initialize(script)
       @name = "Elasticity Pig Step (#{script})"
       @script = script
-      @variables = { }
+      @variables = {}
       @action_on_failure = 'TERMINATE_JOB_FLOW'
     end
 
@@ -37,20 +37,26 @@ module Elasticity
       true
     end
 
-    def self.aws_installation_step
-      {
-        :action_on_failure => 'TERMINATE_JOB_FLOW',
-        :hadoop_jar_step => {
-          :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
-          :args => [
-            's3://elasticmapreduce/libs/pig/pig-script',
+    def self.aws_installation_step_name
+      'Elasticity - Install Pig'
+    end
+
+    def self.aws_installation_steps
+      [
+        {
+          :action_on_failure => 'TERMINATE_JOB_FLOW',
+          :hadoop_jar_step => {
+            :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
+            :args => [
+              's3://elasticmapreduce/libs/pig/pig-script',
               '--base-path',
               's3://elasticmapreduce/libs/pig/',
               '--install-pig'
-          ],
-        },
-        :name => 'Elasticity - Install Pig'
-      }
+            ],
+          },
+          :name => aws_installation_step_name
+        }
+      ]
     end
 
     private

data/lib/elasticity/version.rb

@@ -1,3 +1,3 @@
 module Elasticity
-  VERSION = '2.5.1'
+  VERSION = '2.5.2'
 end

data/spec/lib/elasticity/hive_step_spec.rb

@@ -55,10 +55,10 @@ describe Elasticity::HiveStep do
     end
   end
 
-  describe '.aws_installation_step' do
+  describe '.aws_installation_steps' do
 
-    it 'should provide a means to install Hive' do
-      Elasticity::HiveStep.aws_installation_step.should == {
+    let(:install_hive_step) do
+      {
         :action_on_failure => 'TERMINATE_JOB_FLOW',
         :hadoop_jar_step => {
           :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
@@ -75,6 +75,42 @@ describe Elasticity::HiveStep do
       }
     end
 
+    let(:configure_hive_step) do
+      {
+        :action_on_failure => 'TERMINATE_JOB_FLOW',
+        :hadoop_jar_step => {
+          :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
+          :args => [
+            's3://elasticmapreduce/libs/hive/hive-script',
+            '--base-path',
+            's3://elasticmapreduce/libs/hive/',
+            '--install-hive-site',
+            '--hive-site=s3://TEST/hive-site.xml',
+            '--hive-versions',
+            'latest'
+          ],
+        },
+        :name => 'Elasticity - Configure Hive via Hive Site'
+      }
+    end
+
+    context 'when a hive site configuration file is not specified' do
+      it 'should specify how to install Hive' do
+        Elasticity::HiveStep.aws_installation_steps.should == [install_hive_step]
+      end
+    end
+
+    context 'when a hive site configuration file is specified' do
+      before do
+        Elasticity.configure do |config|
+          config.hive_site = 's3://TEST/hive-site.xml'
+        end
+      end
+      it 'should specify how to install Hive' do
+        Elasticity::HiveStep.aws_installation_steps.should == [install_hive_step, configure_hive_step]
+      end
+    end
+
   end
 
 end

data/spec/lib/elasticity/job_flow_spec.rb

@@ -161,10 +161,7 @@ describe Elasticity::JobFlow do
 
           it 'should submit the installation step and the step' do
             emr.should_receive(:add_jobflow_steps).with('RUNNING_JOBFLOW_ID', {
-              :steps => [
-                Elasticity::HiveStep.aws_installation_step,
-                  additional_step.to_aws_step(running_jobflow)
-              ]
+              :steps => Elasticity::HiveStep.aws_installation_steps << additional_step.to_aws_step(running_jobflow)
             })
             running_jobflow.add_step(additional_step)
           end
@@ -209,12 +206,12 @@ describe Elasticity::JobFlow do
       end
       let(:aws_steps) do
         [
-          Elasticity::HiveStep.aws_installation_step,
+          Elasticity::HiveStep.aws_installation_steps,
             jobflow_steps[0].to_aws_step(jobflow_with_steps),
-            Elasticity::PigStep.aws_installation_step,
+            Elasticity::PigStep.aws_installation_steps,
             jobflow_steps[1].to_aws_step(jobflow_with_steps),
             jobflow_steps[2].to_aws_step(jobflow_with_steps),
-        ]
+        ].flatten
       end
 
       it 'should incorporate the installation and run steps into the jobflow config' do
@@ -265,8 +262,8 @@ describe Elasticity::JobFlow do
         let(:hadoop_bootstrap_actions) do
           [
             Elasticity::HadoopBootstrapAction.new('OPTION1', 'VALUE1'),
-              Elasticity::HadoopBootstrapAction.new('OPTION1', 'VALUE2'),
-              Elasticity::HadoopBootstrapAction.new('OPTION2', 'VALUE3')
+            Elasticity::HadoopBootstrapAction.new('OPTION1', 'VALUE2'),
+            Elasticity::HadoopBootstrapAction.new('OPTION2', 'VALUE3')
           ]
         end
         let(:jobflow_with_bootstrap_actions) do
@@ -306,12 +303,12 @@ describe Elasticity::JobFlow do
             :instance_type => 'm1.small',
             :market => 'ON_DEMAND',
           },
-            {
-              :instance_count => 1,
-              :instance_role => 'CORE',
-              :instance_type => 'm1.small',
-              :market => 'ON_DEMAND'
-            },
+          {
+            :instance_count => 1,
+            :instance_role => 'CORE',
+            :instance_type => 'm1.small',
+            :market => 'ON_DEMAND'
+          },
         ]
       end
 

data/spec/lib/elasticity/job_flow_step_spec.rb

@@ -33,18 +33,33 @@ describe Elasticity::JobFlowStep do
     end
   end
 
-  describe '.aws_installation_step' do
+  describe '#aws_installation_step_name' do
+    it 'should delegate to the class method' do
+      FakeStep.should_receive(:aws_installation_step_name).and_return('AWS_INSTALLATION_STEP_NAME')
+      subject.aws_installation_step_name.should == 'AWS_INSTALLATION_STEP_NAME'
+    end
+  end
+
+  describe '.aws_installation_step_name' do
     it 'should raise an error by default' do
       expect {
-        FakeStep.aws_installation_step
-      }.to raise_error(RuntimeError, '.aws_installation_step is required to be defined when a step requires installation (e.g. Pig, Hive).')
+        FakeStep.aws_installation_step_name
+      }.to raise_error(RuntimeError, '.aws_installation_step_name is required to be defined when a step requires installation (e.g. Pig, Hive).')
     end
   end
 
-  describe '#aws_installation_step' do
+  describe '#aws_installation_steps' do
     it 'should delegate to the class method' do
-      FakeStep.should_receive(:aws_installation_step).and_return('AWS_INSTALLATION_STEP')
-      subject.aws_installation_step.should == 'AWS_INSTALLATION_STEP'
+      FakeStep.should_receive(:aws_installation_steps).and_return('AWS_INSTALLATION_STEPS')
+      subject.aws_installation_steps.should == 'AWS_INSTALLATION_STEPS'
+    end
+  end
+
+  describe '.aws_installation_steps' do
+    it 'should raise an error by default' do
+      expect {
+        FakeStep.aws_installation_steps
+      }.to raise_error(RuntimeError, '.aws_installation_step is required to be defined when a step requires installation (e.g. Pig, Hive).')
     end
   end
 

data/spec/lib/elasticity/pig_step_spec.rb

@@ -8,7 +8,7 @@ describe Elasticity::PigStep do
 
   its(:name) { should == 'Elasticity Pig Step (script.pig)' }
   its(:script) { should == 'script.pig' }
-  its(:variables) { should == { } }
+  its(:variables) { should == {} }
   its(:action_on_failure) { should == 'TERMINATE_JOB_FLOW' }
 
   describe '#to_aws_step' do
@@ -20,9 +20,9 @@ describe Elasticity::PigStep do
       step[:hadoop_jar_step][:jar].should == 's3://elasticmapreduce/libs/script-runner/script-runner.jar'
       step[:hadoop_jar_step][:args].should start_with([
         's3://elasticmapreduce/libs/pig/pig-script',
-          '--run-pig-script',
-          '--args',
-          '-p'
+        '--run-pig-script',
+        '--args',
+        '-p'
       ])
       step[:hadoop_jar_step][:args][4] =~ /^E_PARALLELS=\d+$/
     end
@@ -68,9 +68,9 @@ describe Elasticity::PigStep do
         step = ps_with_variables.to_aws_step(Elasticity::JobFlow.new('access', 'secret'))
         step[:hadoop_jar_step][:args][3..9].should == [
           '-p', 'VAR1=VALUE1',
-            '-p', 'VAR2=VALUE2',
-            '-p', 'E_PARALLELS=1',
-            'script.pig'
+          '-p', 'VAR2=VALUE2',
+          '-p', 'E_PARALLELS=1',
+          'script.pig'
         ]
       end
     end
@@ -83,22 +83,26 @@ describe Elasticity::PigStep do
     end
   end
 
-  describe '.aws_installation_step' do
+  describe '.aws_installation_steps' do
+
     it 'should provide a means to install Pig' do
-      Elasticity::PigStep.aws_installation_step.should == {
-        :action_on_failure => 'TERMINATE_JOB_FLOW',
-        :hadoop_jar_step => {
-          :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
-          :args => [
-            's3://elasticmapreduce/libs/pig/pig-script',
+      Elasticity::PigStep.aws_installation_steps.should == [
+        {
+          :action_on_failure => 'TERMINATE_JOB_FLOW',
+          :hadoop_jar_step => {
+            :jar => 's3://elasticmapreduce/libs/script-runner/script-runner.jar',
+            :args => [
+              's3://elasticmapreduce/libs/pig/pig-script',
               '--base-path',
               's3://elasticmapreduce/libs/pig/',
               '--install-pig'
-          ],
-        },
-        :name => 'Elasticity - Install Pig'
-      }
+            ],
+          },
+          :name => 'Elasticity - Install Pig'
+        }
+      ]
     end
+
   end
 
 end

data/spec/spec_helper.rb

@@ -1,10 +1,15 @@
-require 'rubygems'
-require 'bundler/setup'
-require 'elasticity'
+ENV['RAILS_ENV'] ||= 'test'
 
+require 'elasticity'
 require 'timecop'
 require 'fakefs/spec_helpers'
 
-ENV['RAILS_ENV'] ||= 'test'
-
 Dir[File.join(File.dirname(__FILE__), 'support', '**', '*.rb')].each { |f| require f }
+
+RSpec.configure do |config|
+
+  config.before(:each) do
+    Elasticity.default_configuration
+  end
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: elasticity
 version: !ruby/object:Gem::Version
-  version: 2.5.1
+  version: 2.5.2
   prerelease: 
 platform: ruby
 authors: