elasticity 6.0.11 → 6.0.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2f61a0e2148b8a7f75fe8c28516fbc777a57c566
-  data.tar.gz: b2913d92acd6c595eb9ef2637be9649f2a42c996
+  metadata.gz: f96e8b15d696c580e01a6957b7981806190c86ef
+  data.tar.gz: aa9fc8328096b4fc7cf0649b0bb3f82eaf416e8f
 SHA512:
-  metadata.gz: f524779e497a2a325d7e48ee5a08333e67f9e2c28d897bcab9beaa2f21d715b1e85b2830f6abddab3c3320db15e308fbf87bff2ba99d9af803ba16603b3dc7f9
-  data.tar.gz: 3d81e2b0e7f23b10ee8cd937e843e5ffd16a739a75c9300a13b505746d535e15e9e38f2d699fd0381d95d044df97d840c74c74e78d4c660729cdd08530b64f0c
+  metadata.gz: 709475923787784aaf6e38c3825a730cf462966b3cf7c10c1f26314d259743e21b462a2d3f36e4d4e077a79b16ec62da65d058ffbd8bdb78ccf915f994a3da36
+  data.tar.gz: 4b5a0731398a0e7cf22f144c853a99d76fbc27ed3cbbf0d18c9ed46d1a4803e76d9872833fb86a96fb4cb23393b0315b74d84eee418e45838ad48a33ace0d2df

data/HISTORY.md

@@ -1,3 +1,9 @@
+## 6.0.12 - February 7, 2017
+
+- Including PR [#140](https://github.com/rslifka/elasticity/pull/140) - "Update fog". Thank you [@BenFradet ](https://github.com/BenFradet )!
+- Including PR [#138](https://github.com/rslifka/elasticity/pull/138) - "Make application configuration available". Thank you [@henryhu666](https://github.com/henryhu666)!
+- Including PR [#136](https://github.com/rslifka/elasticity/pull/136) - "added security configuration attribute". Thank you [@devsaik](https://github.com/devsaik)!
+
 ## 6.0.11 - February 7, 2017
 
 - Including PR [#134](https://github.com/rslifka/elasticity/pull/134) - "Spark step".  Thank you [@BenFradet](https://github.com/BenFradet)!

data/elasticity.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |s|
   s.description           = %q{Streamlined, programmatic access to Amazon's Elastic Map Reduce service, driven by the Sharethrough team's requirements for belting out EMR jobs.}
 
   s.add_dependency('rest-client', '~> 1.0')
-  s.add_dependency('fog', '~> 1.0')
+  s.add_dependency('fog-aws', '~> 1.0')
   s.add_dependency('unf', '~> 0.1')
 
   s.add_development_dependency('factory_girl', '~> 4.0')

data/lib/elasticity.rb

@@ -3,7 +3,7 @@ require 'time'
 
 require 'rest_client'
 require 'nokogiri'
-require 'fog'
+require 'fog/aws'
 
 require 'elasticity/version'
 

data/lib/elasticity/aws_request_v4.rb

@@ -51,7 +51,12 @@ module Elasticity
     end
 
     def payload
-      AwsUtils.convert_ruby_to_aws_v4(@ruby_service_hash).to_json
+      configurations = @ruby_service_hash.delete(:configurations)
+      service_hash = AwsUtils.convert_ruby_to_aws_v4(@ruby_service_hash)
+      return service_hash.to_json if configurations.nil?
+      @ruby_service_hash[:configurations] = configurations
+      service_hash['Configurations'] = configurations
+      service_hash.to_json
     end
 
     private

data/lib/elasticity/aws_utils.rb

@@ -33,4 +33,4 @@ module Elasticity
 
   end
 
-end
+end

data/lib/elasticity/job_flow.rb

@@ -10,6 +10,7 @@ module Elasticity
 
     attr_accessor :action_on_failure
     attr_accessor :ec2_key_name
+    attr_accessor :security_configuration
     attr_accessor :name
     attr_accessor :instance_count
     attr_accessor :log_uri
@@ -28,6 +29,7 @@ module Elasticity
     attr_accessor :service_role
     attr_accessor :jobflow_id
     attr_accessor :aws_applications
+    attr_accessor :aws_configurations
     attr_accessor :additional_info
     attr_accessor :additional_master_security_groups
     attr_accessor :additional_slave_security_groups
@@ -43,6 +45,7 @@ module Elasticity
 
       @bootstrap_actions = []
       @aws_applications = []
+      @aws_configurations = []
       @jobflow_steps = []
       @installed_steps = []
 
@@ -118,6 +121,11 @@ module Elasticity
       @aws_applications << application
     end
 
+    def add_configuration(configuration)
+      raise JobFlowRunningError, 'To add configurations, please create a new job flow.' if is_jobflow_running?
+      @aws_configurations << configuration
+    end
+
     def set_master_instance_group(instance_group)
       instance_group.role = 'MASTER'
       @instance_groups[:master] = instance_group
@@ -210,15 +218,17 @@ module Elasticity
       config = jobflow_preamble
       validate_and_apply_ami_or_release_version(config)
       steps = jobflow_steps
-      steps.insert(0, Elasticity::SetupHadoopDebuggingStep.new.to_aws_step(self)) if @enable_debugging
+      steps.insert(0, Elasticity::SetupHadoopDebuggingStep.new(@region).to_aws_step(self)) if @enable_debugging
       config[:steps] = steps
       config[:log_uri] = @log_uri if @log_uri
       config[:tags] = jobflow_tags if @tags
       config[:job_flow_role] = @job_flow_role if @job_flow_role
       config[:service_role] = @service_role if @service_role
       config[:additional_info] = @additional_info if @additional_info
+      config[:security_configuration] = @security_configuration if @security_configuration
       config[:bootstrap_actions] = @bootstrap_actions.map(&:to_aws_bootstrap_action) unless @bootstrap_actions.empty?
       config[:applications] = @aws_applications.map(&:to_hash) if valid_aws_applications?
+      config[:configurations] = @aws_configurations unless @aws_configurations.empty?
       config
     end
 

data/lib/elasticity/setup_hadoop_debugging_step.rb

@@ -2,10 +2,10 @@ module Elasticity
 
   class SetupHadoopDebuggingStep < CustomJarStep
 
-    def initialize
+    def initialize(region)
       @name = 'Elasticity Setup Hadoop Debugging'
-      @jar = 's3://elasticmapreduce/libs/script-runner/script-runner.jar'
-      @arguments = ['s3://elasticmapreduce/libs/state-pusher/0.1/fetch']
+      @jar = "s3://#{region}.elasticmapreduce/libs/script-runner/script-runner.jar"
+      @arguments = ["s3://#{region}.elasticmapreduce/libs/state-pusher/0.1/fetch"]
       @action_on_failure = 'TERMINATE_JOB_FLOW'
     end
 

data/lib/elasticity/version.rb

@@ -1,3 +1,3 @@
 module Elasticity
-  VERSION = '6.0.11'
+  VERSION = '6.0.12'
 end

data/spec/lib/elasticity/aws_request_v4_spec.rb

@@ -86,8 +86,29 @@ describe Elasticity::AwsRequestV4 do
   end
 
   describe '#payload' do
-    it 'should create the proper payload' do
-      subject.payload.should == '{"JobFlowIds":["TEST_JOBFLOW_ID"]}'
+    context 'when no configurations are given' do
+      it 'should create the proper payload' do
+        subject.payload.should == '{"JobFlowIds":["TEST_JOBFLOW_ID"]}'
+      end
+    end
+
+    context 'when configurations are given' do
+      subject do
+        Elasticity::AwsRequestV4.new(
+          Elasticity::AwsSession.new,
+          {:operation => 'DescribeJobFlows', :job_flow_ids => ['TEST_JOBFLOW_ID'],
+           :configurations => [
+            'Classification' => 'capacity-scheduler',
+            'Properties' => {
+              'yarn.scheduler.capacity.resource-calculator' =>
+                'org.apache.hadoop.yarn.util.resource.DominantResourceCalculator'
+            }]
+          }
+        )
+      end
+      it 'should create the proper payload' do
+        subject.payload.should == '{"JobFlowIds":["TEST_JOBFLOW_ID"],"Configurations":[{"Classification":"capacity-scheduler","Properties":{"yarn.scheduler.capacity.resource-calculator":"org.apache.hadoop.yarn.util.resource.DominantResourceCalculator"}}]}'
+      end
     end
   end
 
@@ -123,7 +144,7 @@ describe Elasticity::AwsRequestV4 do
 
   describe '.aws_v4_signature' do
     it 'should create the proper signature' do
-      subject.send(:aws_v4_signature).should == '9fcb4107e8346b2b92a4a2c56de98ed80fdd87d3f9514e728adce76390c5b267'
+      subject.send(:aws_v4_signature).should == '71d7f2adf5b01990b09d7d10910b9a0312ded30bacf05ea8e8a05cc72ab389d1'
     end
   end
 

data/spec/lib/elasticity/aws_utils_spec.rb

@@ -56,4 +56,4 @@ describe Elasticity::AwsUtils do
     end
   end
 
-end
+end

data/spec/lib/elasticity/job_flow_spec.rb

@@ -8,6 +8,7 @@ describe Elasticity::JobFlow do
     it 'should set the fields appropriately' do
       expect(subject.action_on_failure).to eql('TERMINATE_JOB_FLOW')
       expect(subject.ec2_key_name).to eql(nil)
+      expect(subject.security_configuration).to eql(nil)
       expect(subject.name).to eql('Elasticity Job Flow')
       expect(subject.instance_count).to eql(2)
       expect(subject.log_uri).to eql(nil)
@@ -16,6 +17,7 @@ describe Elasticity::JobFlow do
       expect(subject.ami_version).to eql(nil)
       expect(subject.release_label).to eql(nil)
       expect(subject.aws_applications).to eql([])
+      expect(subject.aws_configurations).to eql([])
       expect(subject.keep_job_flow_alive_when_no_steps).to eql(false)
       expect(subject.ec2_subnet_id).to eql(nil)
       expect(subject.placement).to eql('us-east-1a')
@@ -77,6 +79,18 @@ describe Elasticity::JobFlow do
     end
   end
 
+  describe '#security_configuration=' do
+    context 'when set' do
+      before do
+        subject.security_configuration = 'security configuration'
+      end
+      it 'security_configuration is a string' do
+        expect(subject.security_configuration).to eq('security configuration')
+      end
+    end
+  end
+
+
   describe '#enable_debugging=' do
 
     context 'when a log_uri is present' do
@@ -212,6 +226,31 @@ describe Elasticity::JobFlow do
 
   end
 
+  describe '#add_configuration' do
+    let(:configuration) { {'Classification' => 'spark'} }
+    context 'when the jobflow is not yet started' do
+      it 'should not raise an error' do
+        expect {
+          subject.add_bootstrap_action(configuration)
+        }.to_not raise_error
+      end
+    end
+
+    context 'when the jobflow is already started' do
+      before do
+        Elasticity::EMR.any_instance.stub(:run_job_flow => '_')
+        subject.add_step(Elasticity::CustomJarStep.new('_'))
+        subject.run
+      end
+      it 'should raise an error' do
+        expect {
+          subject.add_configuration(configuration)
+        }.to raise_error(Elasticity::JobFlowRunningError, 'To add configurations, please create a new job flow.')
+      end
+    end
+
+  end
+
   describe '#add_step' do
 
     context 'when the jobflow is already running' do
@@ -312,7 +351,7 @@ describe Elasticity::JobFlow do
         before do
           jobflow_with_steps.log_uri = '_'
           jobflow_with_steps.enable_debugging = true
-          aws_steps.insert(0, Elasticity::SetupHadoopDebuggingStep.new.to_aws_step(jobflow_with_steps))
+          aws_steps.insert(0, Elasticity::SetupHadoopDebuggingStep.new('us-east-1').to_aws_step(jobflow_with_steps))
         end
         it 'should incorporate the step to setup Hadoop debugging' do
           jobflow_with_steps.send(:jobflow_config).should be_a_hash_including({:steps => aws_steps})
@@ -515,6 +554,27 @@ describe Elasticity::JobFlow do
       end
     end
 
+    describe 'aws_configurations' do
+      let(:configuration) {
+        { 'Classification' => 'spark',
+          'Properties' => { 'prop' => 'valule'} }
+      }
+
+      let(:emr) { double(Elasticity::EMR, run_job_flow: true) }
+
+      before do
+        subject.stub(:emr).and_return(emr)
+      end
+
+      context 'with configurations' do
+        it 'sets the configurations as a part of its config' do
+          subject.instance_variable_set(:@aws_configurations, [configuration])
+          config = subject.send(:jobflow_config)
+          expect(config[:configurations]).to eql([configuration])
+        end
+      end
+
+    end
   end
 
   describe '#jobflow_instance_groups' do

data/spec/lib/elasticity/setup_hadoop_debugging_step_spec.rb

@@ -1,14 +1,18 @@
 describe Elasticity::SetupHadoopDebuggingStep do
 
+  subject do
+    Elasticity::SetupHadoopDebuggingStep.new('us-east-1')
+  end
+
   it 'should be a CustomJarStep' do
     expect(subject).to be_a(Elasticity::CustomJarStep)
   end
 
   it 'should set the appropriate fields' do
     expect(subject.name).to eql('Elasticity Setup Hadoop Debugging')
-    expect(subject.jar).to eql('s3://elasticmapreduce/libs/script-runner/script-runner.jar')
-    expect(subject.arguments).to eql(['s3://elasticmapreduce/libs/state-pusher/0.1/fetch'])
+    expect(subject.jar).to eql('s3://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jar')
+    expect(subject.arguments).to eql(['s3://us-east-1.elasticmapreduce/libs/state-pusher/0.1/fetch'])
     expect(subject.action_on_failure).to eql('TERMINATE_JOB_FLOW')
   end
 
-end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elasticity
 version: !ruby/object:Gem::Version
-  version: 6.0.11
+  version: 6.0.12
 platform: ruby
 authors:
 - Robert Slifka
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-06 00:00:00.000000000 Z
+date: 2017-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -25,7 +25,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
-  name: fog
+  name: fog-aws
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"