elasticity 6.0.5 → 6.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d313d736b8c2be8bda97c7cba0c200c611b5fc59
-  data.tar.gz: e0c4afadd4d192770e3c79a67f5c62147425c1dd
+  metadata.gz: 9544e6d7a9f5c8b361f07657a2902a0006415c93
+  data.tar.gz: 2477d97b8f823248ae6f1d6061f092aa0e5e0747
 SHA512:
-  metadata.gz: 2c3c35bdeb9408756f085cc9fc8550c6ca6165fec7134af6a8f639279128cb6bf2a84a067766891ead282e6532adbb5a1a2fa7aff6fa62ed65261171f8fb7f80
-  data.tar.gz: cbab3121a67c1b7f0d3370f6ee662b7dd308fa0d8c57122e68db1f2b0b1e5a87119533968baadfae8c3e4a5bbd76a94ae28a49f362e276fb42d68e34e9d58e8f
+  metadata.gz: 356394fc7e44ec694f8c59f60a90b1f1daaa3a0c2f4b40f55532c89bfcbfce11b59ac70d385936dccb9fb08ee726f645a6b8e010c464dbd4ee147a5f2e174ed0
+  data.tar.gz: ffc3be6f66d3218e5a167afb3f4eb4431f0b86f6966d93f62c3ff86d54f4021485d100d04b16e73333970a2f708f2f4f7015977a339c7515bc790a9a1d2d1bf2

data/.ruby-version

@@ -1,2 +1 @@
 2.0.0-p598
-

data/.travis.yml

@@ -3,3 +3,8 @@ rvm:
   - 2.0
   - 2.1
   - 2.2
+  - 2.3.0
+before_install:
+  - gem update --system
+  - gem --version
+  - gem install bundler

data/HISTORY.md

@@ -1,3 +1,7 @@
+## 6.0.6 - February 29, 2016
+
+- - Including PR [#20](https://github.com/rslifka/elasticity/pull/120) - "EMR Applications and support for release_label".  Thank you [@robert2d](https://github.com/robert2d)!
+- 
 ## 6.0.5 - August 28, 2015
 
 - - Including PR [#119](https://github.com/rslifka/elasticity/pull/119) - "Error handling for API rate limiting".  Thank you [@robert2d](https://github.com/robert2d)!

data/README.md

@@ -2,7 +2,7 @@
 
 Elasticity provides programmatic access to Amazon's Elastic Map Reduce service.  The aim is to conveniently abstract away the complex EMR REST API and make working with job flows more productive and more enjoyable.
 
-[![Build Status](https://secure.travis-ci.org/rslifka/elasticity.png)](http://travis-ci.org/rslifka/elasticity) 2.0.x, 2.1.x, 2.2.x
+[![Build Status](https://secure.travis-ci.org/rslifka/elasticity.png)](http://travis-ci.org/rslifka/elasticity) 2.0.x, 2.1.x, 2.2.x, 2.3.x
 
 Elasticity provides two ways to access EMR:
 
@@ -115,10 +115,14 @@ jobflow.service_role                      = nil
 
 jobflow.action_on_failure                 = 'TERMINATE_JOB_FLOW'
 jobflow.keep_job_flow_alive_when_no_steps = false
-jobflow.ami_version                       = 'latest'
 jobflow.log_uri                           = nil
 jobflow.enable_debugging                  = false # Requires a log_uri to enable
 
+# >= 4.0.0 release label is now the default
+jobflow.release_label                     = '4.3.0'
+# < 4.0.0 ... Haven't used this before? just set the release label then.
+jobflow.ami_version                       = 'latest'
+
 jobflow.tags                              = {name: "app-name", department: 'marketing'}
 jobflow.ec2_key_name                      = nil
 jobflow.visible_to_all_users              = false
@@ -129,6 +133,25 @@ jobflow.master_instance_type              = 'm1.small'
 jobflow.slave_instance_type               = 'm1.small'
 ```
 
+### EMR Applications (optional needs release_label >= 4.0.0)
+With the release of EMR 4.0.0 you can now supply applications which EMR will install for you on boot(rather than a manual bootstrap action. Which you can still use if required). You must set the `release_label` for the jobflow(>=4.0.0)
+
+```ruby
+jobflow.release_label = '4.3.0' 
+# the simple way
+jobflow.add_application("Spark") # Pig, Hive, Mahout
+# more verbose
+spark = Elasticity::Application.new({
+  name: 'Spark',
+  arguments: '--webui-port 18080',
+  version: '1.0.1',
+  additional_info: '' # This option is for advanced users only. This is meta information about third-party applications that third-party vendors use for testing purposes.
+})
+jobflow.add_application(spark)
+```
+
+Further reading: http://docs.aws.amazon.com/ElasticMapReduce/latest/ReleaseGuide/emr-configure-apps.html
+
 ## 4 - Configure Instance Groups (optional)
 
 Technically this is optional since Elasticity creates MASTER and CORE instance groups for you (one m1.small instance in each).  If you'd like your jobs to finish in an appreciable amount of time, you'll want to at least add a few instances to the CORE group :)

data/lib/elasticity.rb

@@ -15,6 +15,7 @@ require 'elasticity/emr'
 require 'elasticity/sync_to_s3'
 
 require 'elasticity/bootstrap_action'
+require 'elasticity/application'
 require 'elasticity/hadoop_bootstrap_action'
 require 'elasticity/hadoop_file_bootstrap_action'
 require 'elasticity/ganglia_bootstrap_action'

data/lib/elasticity/application.rb

@@ -0,0 +1,27 @@
+module Elasticity
+  class Application
+
+    attr_accessor :name
+    attr_accessor :arguments
+    attr_accessor :version
+    attr_accessor :additional_info
+
+    def initialize(args)
+      @name = args[:name]
+      @arguments = args[:arguments]
+      @version = args[:version]
+      @additional_info = args[:additional_info]
+    end
+
+    def to_hash
+      application = {
+        name: @name
+      }
+      application[:args] = @arguments unless @arguments.nil? || @arguments.empty?
+      application[:version] = @version if @version
+      application[:additional_info] = @additional_info if @additional_info
+      application
+    end
+
+  end
+end

data/lib/elasticity/aws_request_v4.rb

@@ -107,4 +107,4 @@ module Elasticity
 
   end
 
-end
+end

data/lib/elasticity/job_flow.rb

@@ -17,6 +17,7 @@ module Elasticity
     attr_accessor :master_instance_type
     attr_accessor :slave_instance_type
     attr_accessor :ami_version
+    attr_accessor :release_label
     attr_accessor :keep_job_flow_alive_when_no_steps
     attr_accessor :ec2_subnet_id
     attr_accessor :placement
@@ -26,11 +27,11 @@ module Elasticity
     attr_accessor :job_flow_role
     attr_accessor :service_role
     attr_accessor :jobflow_id
+    attr_accessor :aws_applications
 
     def initialize
       @action_on_failure = 'TERMINATE_JOB_FLOW'
       @name = 'Elasticity Job Flow'
-      @ami_version = 'latest'
       @keep_job_flow_alive_when_no_steps = false
       self.placement = 'us-east-1a'
       @enable_debugging = false
@@ -38,6 +39,7 @@ module Elasticity
       @visible_to_all_users = false
 
       @bootstrap_actions = []
+      @aws_applications = []
       @jobflow_steps = []
       @installed_steps = []
 
@@ -106,6 +108,13 @@ module Elasticity
       @bootstrap_actions << bootstrap_action
     end
 
+    def add_application(application)
+      raise JobFlowRunningError, 'To add applications, please create a new job flow.' if is_jobflow_running?
+      application = Application.new(name: application) if application.is_a?(String)
+      fail "application is not an Elasticity::Application" unless application.is_a?(Application)
+      @aws_applications << application
+    end
+
     def set_master_instance_group(instance_group)
       instance_group.role = 'MASTER'
       @instance_groups[:master] = instance_group
@@ -184,6 +193,7 @@ module Elasticity
 
     def jobflow_config
       config = jobflow_preamble
+      validate_and_apply_ami_or_release_version(config)
       steps = jobflow_steps
       steps.insert(0, Elasticity::SetupHadoopDebuggingStep.new.to_aws_step(self)) if @enable_debugging
       config[:steps] = steps
@@ -191,10 +201,23 @@ module Elasticity
       config[:tags] = jobflow_tags if @tags
       config[:job_flow_role] = @job_flow_role if @job_flow_role
       config[:service_role] = @service_role if @service_role
-      config[:bootstrap_actions] = @bootstrap_actions.map{|a| a.to_aws_bootstrap_action} unless @bootstrap_actions.empty?
+      config[:bootstrap_actions] = @bootstrap_actions.map(&:to_aws_bootstrap_action) unless @bootstrap_actions.empty?
+      config[:applications] = @aws_applications.map(&:to_hash) if valid_aws_applications?
       config
     end
 
+    def valid_aws_applications?
+      !@aws_applications.empty?
+    end
+
+    def validate_and_apply_ami_or_release_version(config)
+      fail "Please use an EMR release_label not ami_version" if !@aws_applications.empty? && !@ami_version.nil?
+      fail "Please set the EMR release_label" if !@aws_applications.empty? && @release_label.nil?
+      config[:ami_version] = 'latest' if @ami_version.nil? && release_label.nil?
+      config[:ami_version] = @ami_version unless @ami_version.nil?
+      config[:release_label] = @release_label unless @release_label.nil?
+    end
+
     def jobflow_tags
       @tags.map do |key, value|
         {
@@ -207,7 +230,6 @@ module Elasticity
     def jobflow_preamble
       preamble = {
         :name => @name,
-        :ami_version => @ami_version,
         :visible_to_all_users => @visible_to_all_users,
         :instances => {
           :keep_job_flow_alive_when_no_steps => @keep_job_flow_alive_when_no_steps,

data/lib/elasticity/version.rb

@@ -1,3 +1,3 @@
 module Elasticity
-  VERSION = '6.0.5'
+  VERSION = '6.0.6'
 end

data/spec/lib/elasticity/application_spec.rb

@@ -0,0 +1,24 @@
+describe Elasticity::Application do
+
+  let(:config) do
+    {
+      name: 'name',
+      version: '1.0.0',
+      arguments: 'arguments',
+      additional_info: 'additional_info'
+    }
+  end
+
+  subject { described_class.new(config) }
+
+  describe '#to_hash' do
+    it 'has all configuration' do
+      expect(subject.to_hash).to eq(
+        name: "name",
+        args: "arguments",
+        version: "1.0.0",
+        additional_info: "additional_info"
+      )
+    end
+  end
+end

data/spec/lib/elasticity/job_flow_spec.rb

@@ -13,7 +13,9 @@ describe Elasticity::JobFlow do
       expect(subject.log_uri).to eql(nil)
       expect(subject.master_instance_type).to eql('m1.small')
       expect(subject.slave_instance_type).to eql('m1.small')
-      expect(subject.ami_version).to eql('latest')
+      expect(subject.ami_version).to eql(nil)
+      expect(subject.release_label).to eql(nil)
+      expect(subject.aws_applications).to eql([])
       expect(subject.keep_job_flow_alive_when_no_steps).to eql(false)
       expect(subject.ec2_subnet_id).to eql(nil)
       expect(subject.placement).to eql('us-east-1a')
@@ -455,6 +457,50 @@ describe Elasticity::JobFlow do
 
     end
 
+    describe 'aws_applications' do
+      let(:application) { Elasticity::Application.new({}) }
+      let(:emr) { double(Elasticity::EMR, run_job_flow: true) }
+
+      before do
+        subject.stub(:emr).and_return(emr)
+      end
+
+      context 'with applications' do
+        context 'no release_label' do
+          it 'fails' do
+            subject.instance_variable_set(:@aws_applications, [application])
+            expect { subject.run }.to raise_error(RuntimeError, 'Please set the EMR release_label')
+          end
+        end
+
+        context 'ami_version set' do
+          it 'fails' do
+            subject.instance_variable_set(:@aws_applications, [application])
+            subject.instance_variable_set(:@ami_version, 'latest')
+            expect { subject.run }.to raise_error(RuntimeError, 'Please use an EMR release_label not ami_version')
+          end
+        end
+
+        context 'release_label set' do
+          it 'succeeds' do
+            subject.instance_variable_set(:@aws_applications, [application])
+            subject.instance_variable_set(:@release_label, '4.0.0')
+            subject.run
+            expect(subject.send(:jobflow_config)[:release_label]).to eql('4.0.0')
+          end
+        end
+      end
+
+      context 'with no applications' do
+        it 'sets default ami version and config doesn\'t contain applications' do
+          subject.run
+          config = subject.send(:jobflow_config)
+          expect(config[:ami_version]).to eql('latest')
+          expect(config[:applications]).to eql(nil)
+        end
+      end
+    end
+
   end
 
   describe '#jobflow_instance_groups' do
@@ -515,7 +561,6 @@ describe Elasticity::JobFlow do
     let(:basic_preamble) do
       {
         :name => 'Elasticity Job Flow',
-        :ami_version => 'latest',
         :visible_to_all_users => false,
         :instances => {
           :keep_job_flow_alive_when_no_steps => false,
@@ -801,5 +846,4 @@ describe Elasticity::JobFlow do
     end
 
   end
-
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elasticity
 version: !ruby/object:Gem::Version
-  version: 6.0.5
+  version: 6.0.6
 platform: ruby
 authors:
 - Robert Slifka
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-28 00:00:00.000000000 Z
+date: 2016-01-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -142,6 +142,7 @@ files:
 - Rakefile
 - elasticity.gemspec
 - lib/elasticity.rb
+- lib/elasticity/application.rb
 - lib/elasticity/aws_request_v4.rb
 - lib/elasticity/aws_session.rb
 - lib/elasticity/aws_utils.rb
@@ -167,6 +168,7 @@ files:
 - lib/elasticity/version.rb
 - spec/factories/cluster_status_factory.rb
 - spec/factories/cluster_step_status_factory.rb
+- spec/lib/elasticity/application_spec.rb
 - spec/lib/elasticity/aws_request_v4_spec.rb
 - spec/lib/elasticity/aws_session_spec.rb
 - spec/lib/elasticity/aws_utils_spec.rb
@@ -212,13 +214,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 2.0.14
 signing_key: 
 specification_version: 4
 summary: Streamlined, programmatic access to Amazon's Elastic Map Reduce service.
 test_files:
 - spec/factories/cluster_status_factory.rb
 - spec/factories/cluster_step_status_factory.rb
+- spec/lib/elasticity/application_spec.rb
 - spec/lib/elasticity/aws_request_v4_spec.rb
 - spec/lib/elasticity/aws_session_spec.rb
 - spec/lib/elasticity/aws_utils_spec.rb