elasticity 6.0.10 → 6.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dbcf92882a9214132462533d34770d92deffa247
-  data.tar.gz: 93d0188ecd69661ead048656600d3ee602ab9ce7
+  metadata.gz: 2f61a0e2148b8a7f75fe8c28516fbc777a57c566
+  data.tar.gz: b2913d92acd6c595eb9ef2637be9649f2a42c996
 SHA512:
-  metadata.gz: 5742f6c8ac1fbbc2df514b9711374d9e4ec3723e6513ddfe8a61395aababcf3361e6ba6cbac3f5d7346ed5a0a8b7f5cacdbe97a2a57638c08122cdf16be0d23c
-  data.tar.gz: 51bde4a0262b62b7ab7fdf19c0a33255700fef3eb3c50f3bb27f182a78ae73fecc2ed87518de4929ed984645240f5e83b5a73fdc6e2670f5ba58194d0b9609f5
+  metadata.gz: f524779e497a2a325d7e48ee5a08333e67f9e2c28d897bcab9beaa2f21d715b1e85b2830f6abddab3c3320db15e308fbf87bff2ba99d9af803ba16603b3dc7f9
+  data.tar.gz: 3d81e2b0e7f23b10ee8cd937e843e5ffd16a739a75c9300a13b505746d535e15e9e38f2d699fd0381d95d044df97d840c74c74e78d4c660729cdd08530b64f0c

data/HISTORY.md

@@ -1,3 +1,8 @@
+## 6.0.11 - February 7, 2017
+
+- Including PR [#134](https://github.com/rslifka/elasticity/pull/134) - "Spark step".  Thank you [@BenFradet](https://github.com/BenFradet)!
+- Including PR [#133](https://github.com/rslifka/elasticity/pull/133) - "Scalding step".  Thank you [@BenFradet](https://github.com/BenFradet)!
+
 ## 6.0.10 - January 16, 2017
 
 - Including PR [#132](https://github.com/rslifka/elasticity/pull/132) - "Fix size_in_gb not being parsed correctly for EBS volumes".  Thank you [@jbeemster](https://github.com/jbeemster)!

data/README.md

@@ -141,7 +141,7 @@ jobflow.additional_slave_security_groups  = ['sg-1111', 'sg-2222']
 With the release of EMR 4.0.0 you can now supply applications which EMR will install for you on boot(rather than a manual bootstrap action. Which you can still use if required). You must set the `release_label` for the jobflow(>=4.0.0)
 
 ```ruby
-jobflow.release_label = '4.3.0' 
+jobflow.release_label = '4.3.0'
 # the simple way
 jobflow.add_application("Spark") # Pig, Hive, Mahout
 # more verbose
@@ -341,7 +341,41 @@ jobflow.add_step(copy_step)
 
 # For AMI < 4.x you need to specifify legacy argument
 copy_step = Elasticity::S3DistCpStep.new(true)
+```
+
+### Adding a Scalding Step
+
+```ruby
+scalding_step = Elasticity::ScaldingStep.new('jar_location', 'main_class_fqcn', { 'arg1' => 'value1' })
+
+jobflow.add_step(scalding_step)
+```
+
+This will result in the following command line arguments:
+
+```bash
+main_class_fqcn --hdfs --arg1 value1
+```
+
+### Adding a Spark Step
+
+```ruby
+spark_step = Elasticity::SparkStep.new('jar_location', 'main_class_fqcn')
 
+# Specifying arguments relative to Spark
+spark_step.spark_arguments = { 'driver-memory' => '2G' }
+# Specifying arguments relative to your application
+spark_step.app_arguments = { 'arg1' => 'value1' }
+```
+
+This will be equivalent to the following script:
+
+```bash
+spark-submit \
+  --driver-memory 2G \
+  --class main_class_fqcn \
+  jar_location \
+  --arg1 value1
 ```
 
 ## 7 - Upload Assets (optional)
@@ -422,7 +456,7 @@ Elasticity.configure do |config|
   # AWS credentials
   config.access_key = ENV['AWS_ACCESS_KEY_ID']
   config.secret_key = ENV['AWS_SECRET_ACCESS_KEY']
-  
+
   # if you use federated Identity Management
   #config.security_token  = ENV['AWS_SECURITY_TOKEN']
 

data/lib/elasticity.rb

@@ -35,6 +35,8 @@ require 'elasticity/pig_step'
 require 'elasticity/streaming_step'
 require 'elasticity/script_step'
 require 'elasticity/s3distcp_step'
+require 'elasticity/scalding_step'
+require 'elasticity/spark_step'
 
 module Elasticity
 

data/lib/elasticity/scalding_step.rb

@@ -0,0 +1,32 @@
+module Elasticity
+
+  class ScaldingStep
+
+    include Elasticity::JobFlowStep
+
+    attr_accessor :name
+    attr_accessor :action_on_failure
+    attr_accessor :jar
+    attr_accessor :arguments
+
+    def initialize(jar, main_class, args)
+      @name = 'Elasticity Scalding Step'
+      @action_on_failure = 'TERMINATE_JOB_FLOW'
+      @jar = jar
+      @arguments = [ main_class, '--hdfs' ]
+      args.each do |arg, value|
+        @arguments << "--#{arg}" << value
+      end
+    end
+
+    def to_aws_step(job_flow)
+      step = Elasticity::CustomJarStep.new(@jar)
+      step.name = @name
+      step.action_on_failure = @action_on_failure
+      step.arguments = @arguments
+      step.to_aws_step(job_flow)
+    end
+
+  end
+
+end

data/lib/elasticity/spark_step.rb

@@ -0,0 +1,44 @@
+module Elasticity
+
+  class SparkStep
+
+    include Elasticity::JobFlowStep
+
+    attr_accessor :name
+    attr_accessor :main_class
+    attr_accessor :jar
+    attr_accessor :spark_arguments
+    attr_accessor :app_arguments
+    attr_accessor :action_on_failure
+
+    def initialize(jar, main_class)
+      @name = 'Elasticity Spark Step'
+      @main_class = main_class
+      @jar = jar
+      @spark_arguments = {}
+      @app_arguments = {}
+      @action_on_failure = 'TERMINATE_JOB_FLOW'
+    end
+
+    def to_aws_step(_)
+      args = %W(spark-submit --class #{@main_class})
+      spark_arguments.each do |arg, value|
+        args << "--#{arg}" << value
+      end
+      args.push(@jar)
+      app_arguments.each do |arg, value|
+        args << "--#{arg}" << value
+      end
+      {
+        :name => @name,
+        :action_on_failure => @action_on_failure,
+        :hadoop_jar_step => {
+          :jar => 'command-runner.jar',
+          :args => args
+        }
+      }
+    end
+
+  end
+
+end

data/lib/elasticity/version.rb

@@ -1,3 +1,3 @@
 module Elasticity
-  VERSION = '6.0.10'
+  VERSION = '6.0.11'
 end

data/spec/lib/elasticity/aws_request_v4_spec.rb

@@ -123,7 +123,7 @@ describe Elasticity::AwsRequestV4 do
 
   describe '.aws_v4_signature' do
     it 'should create the proper signature' do
-      subject.send(:aws_v4_signature).should == 'aee8cb25939b4262213612d3410b3033107f97134beadc91563b532c54cf3ab3'
+      subject.send(:aws_v4_signature).should == '9fcb4107e8346b2b92a4a2c56de98ed80fdd87d3f9514e728adce76390c5b267'
     end
   end
 

data/spec/lib/elasticity/scalding_step_spec.rb

@@ -0,0 +1,40 @@
+describe Elasticity::ScaldingStep do
+
+  subject do
+    Elasticity::ScaldingStep.new('jar', 'class', { 'key' => 'value' })
+  end
+
+  it { should be_a Elasticity::JobFlowStep }
+
+  describe '.initialize' do
+    it 'should set the fields appropriately' do
+      expect(subject.name).to eql('Elasticity Scalding Step')
+      expect(subject.action_on_failure).to eql('TERMINATE_JOB_FLOW')
+      expect(subject.jar).to eql('jar')
+      expect(subject.arguments).to eql(['class', '--hdfs', '--key', 'value'])
+    end
+  end
+
+  describe '#to_aws_step' do
+
+    it { should respond_to(:to_aws_step).with(1).argument }
+
+    it 'should convert to aws step format' do
+      subject.to_aws_step(Elasticity::JobFlow.new).should == {
+        :name => 'Elasticity Scalding Step',
+        :action_on_failure => 'TERMINATE_JOB_FLOW',
+        :hadoop_jar_step => {
+          :jar => 'jar',
+          :args => %w(class --hdfs --key value)
+        }
+      }
+    end
+  end
+
+  describe '.requires_installation?' do
+    it 'should not require installation' do
+      expect(Elasticity::ScaldingStep.requires_installation?).to be false
+    end
+  end
+
+end

data/spec/lib/elasticity/spark_step_spec.rb

@@ -0,0 +1,67 @@
+describe Elasticity::SparkStep do
+
+  subject do
+    Elasticity::SparkStep.new('jar', 'class')
+  end
+
+  it { should be_a Elasticity::JobFlowStep }
+
+  describe '.initialize' do
+    it 'should set the fields appropriately' do
+      expect(subject.name).to eql('Elasticity Spark Step')
+      expect(subject.jar).to eql('jar')
+      expect(subject.main_class).to eql('class')
+      expect(subject.spark_arguments).to eql({})
+      expect(subject.app_arguments).to eql({})
+      expect(subject.action_on_failure).to eql('TERMINATE_JOB_FLOW')
+    end
+  end
+
+  describe '#to_aws_step' do
+
+    it { should respond_to(:to_aws_step).with(1).argument }
+
+    context 'when there are no arguments provided' do
+      let(:ss_with_no_args) { Elasticity::SparkStep.new('jar', 'class') }
+
+      it 'should convert to aws step format' do
+        ss_with_no_args.to_aws_step(Elasticity::JobFlow.new).should == {
+          :name => 'Elasticity Spark Step',
+          :action_on_failure => 'TERMINATE_JOB_FLOW',
+          :hadoop_jar_step => {
+            :jar => 'command-runner.jar',
+            :args => %w(spark-submit --class class jar)
+          }
+        }
+      end
+    end
+
+    context 'when there are arguments provided' do
+      let(:ss_with_args) do
+        Elasticity::SparkStep.new('jar', 'class').tap do |ss|
+          ss.spark_arguments = { 'key1' => 'value1' }
+          ss.app_arguments = { 'key2' => 'value2' }
+        end
+      end
+
+      it 'should convert to aws step format' do
+        ss_with_args.to_aws_step(Elasticity::JobFlow.new).should == {
+          :name => 'Elasticity Spark Step',
+          :action_on_failure => 'TERMINATE_JOB_FLOW',
+          :hadoop_jar_step => {
+            :jar => 'command-runner.jar',
+            :args => %w(spark-submit --class class --key1 value1 jar --key2 value2)
+          }
+        }
+      end
+    end
+
+  end
+
+  describe '.requires_installation?' do
+    it 'should not require installation' do
+      expect(Elasticity::SparkStep.requires_installation?).to be false
+    end
+  end
+
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: elasticity
 version: !ruby/object:Gem::Version
-  version: 6.0.10
+  version: 6.0.11
 platform: ruby
 authors:
 - Robert Slifka
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-01-15 00:00:00.000000000 Z
+date: 2017-02-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -161,8 +161,10 @@ files:
 - lib/elasticity/looper.rb
 - lib/elasticity/pig_step.rb
 - lib/elasticity/s3distcp_step.rb
+- lib/elasticity/scalding_step.rb
 - lib/elasticity/script_step.rb
 - lib/elasticity/setup_hadoop_debugging_step.rb
+- lib/elasticity/spark_step.rb
 - lib/elasticity/streaming_step.rb
 - lib/elasticity/sync_to_s3.rb
 - lib/elasticity/version.rb
@@ -188,8 +190,10 @@ files:
 - spec/lib/elasticity/looper_spec.rb
 - spec/lib/elasticity/pig_step_spec.rb
 - spec/lib/elasticity/s3distcp_step_spec.rb
+- spec/lib/elasticity/scalding_step_spec.rb
 - spec/lib/elasticity/script_step_spec.rb
 - spec/lib/elasticity/setup_hadoop_debugging_step_spec.rb
+- spec/lib/elasticity/spark_step_spec.rb
 - spec/lib/elasticity/streaming_step_spec.rb
 - spec/lib/elasticity/sync_to_s3_spec.rb
 - spec/spec_helper.rb
@@ -241,8 +245,10 @@ test_files:
 - spec/lib/elasticity/looper_spec.rb
 - spec/lib/elasticity/pig_step_spec.rb
 - spec/lib/elasticity/s3distcp_step_spec.rb
+- spec/lib/elasticity/scalding_step_spec.rb
 - spec/lib/elasticity/script_step_spec.rb
 - spec/lib/elasticity/setup_hadoop_debugging_step_spec.rb
+- spec/lib/elasticity/spark_step_spec.rb
 - spec/lib/elasticity/streaming_step_spec.rb
 - spec/lib/elasticity/sync_to_s3_spec.rb
 - spec/spec_helper.rb