rp-emr 1.0.3

data/spec/rp/emr/instances_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+describe RP::EMR::Instances do
+  describe "#to_hash" do
+    let(:instances_args) { {} }
+
+    let(:instances) do
+      RP::EMR::Instances.new(instances_args)
+    end
+
+    it "returns a hash" do
+      expect(instances.to_hash).to eq({})
+    end
+
+    context "with stuff specified" do
+      let(:instances_args) { {master_instance_type: 'master_instance_type'} }
+
+      it "adds stuff to hash" do
+        expect(instances.to_hash).to eq(master_instance_type: 'master_instance_type')
+      end
+    end
+  end
+end

data/spec/rp/emr/job_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe RP::EMR::Job do
+  describe "#to_hash" do
+    let(:instances_args) { {} }
+
+    let(:instances) do
+      RP::EMR::Instances.new(instances_args)
+    end
+
+    let(:job_args) { {} }
+
+    let(:job) do
+      RP::EMR::Job.new(job_args) do |j|
+        j.instances = instances.to_hash
+      end
+    end
+
+    it "returns a hash" do
+      expect(job.to_hash).to eq(ami_version: 'latest', visible_to_all_users: true)
+    end
+
+    context "with stuff specified" do
+      let(:job_args) { {ami_version: 'ami_version'} }
+
+      it "adds stuff to hash" do
+        expect(job.to_hash[:ami_version]).to eq('ami_version')
+      end
+    end
+  end
+end

data/spec/rp/emr/step/pig_spec.rb

@@ -0,0 +1,136 @@
+require 'spec_helper'
+
+describe RP::EMR::Step::Pig do
+  describe "#to_hash" do
+    let(:written) { [] }
+
+    before(:each) do
+      allow_any_instance_of(AWS::S3::S3Object).to receive(:write) do |data|
+        written << data
+      end
+    end
+
+    after(:each) do
+      script.delete
+    end
+
+    let(:script_contents) { 'script_contents' }
+
+    let(:script) do
+      script = Tempfile.new('pig_step_test')
+      script.write(script_contents)
+      script.close
+      script
+    end
+
+    let(:expected_script_path) do
+      hash = Digest::MD5.hexdigest(script_contents)
+      "scripts/emr_gem/#{File.basename(script.path, '.pig')}_#{hash}.pig"
+    end
+
+    let(:step_args) { {} }
+
+    let(:step) do
+      RP::EMR::Step::Pig.new(step_args) do |s|
+        s.name = 'step_name'
+        s.script_path = script.path
+        s.script_bucket = 'script_bucket'
+      end
+    end
+
+    it "writes script to expected path" do
+      expect_any_instance_of(AWS::S3::ObjectCollection).to receive(:[]).with(expected_script_path).and_call_original
+
+      step.to_hash
+    end
+
+    it "uploads pig script contents" do
+      pending "Strange behavior with test setup?"
+      step.to_hash
+
+      expect(written).to eq(['script_contents'])
+    end
+
+    it "returns hash" do
+      expect(step.to_hash).to eq(
+        :name=>"step_name", 
+        :hadoop_jar_step=>{
+          :jar=>"s3://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jar",
+          :args=>[
+            "s3://us-east-1.elasticmapreduce/libs/pig/pig-script", 
+            "--base-path", "s3://us-east-1.elasticmapreduce/libs/pig/", 
+            "--pig-versions", "0.11.1.1", 
+            "--run-pig-script", 
+            "--args", 
+            "-f", "s3://script_bucket/#{expected_script_path}",
+          ]
+        }
+      )
+    end
+
+    context "with args" do
+      let(:step_args) { {args: ['foo', 'bar']} }
+
+      it "adds to the args array" do
+        expect(step.to_hash[:hadoop_jar_step][:args]).to eq([
+          "s3://us-east-1.elasticmapreduce/libs/pig/pig-script", 
+          "--base-path", "s3://us-east-1.elasticmapreduce/libs/pig/", 
+          "--pig-versions", "0.11.1.1", 
+          "--run-pig-script", 
+          "foo", "bar",
+          "--args", 
+          "-f", "s3://script_bucket/#{expected_script_path}",
+        ])
+      end
+    end
+
+    context "with pig_params" do
+      let(:step_args) { {pig_params: {foo: 'bar'}} }
+
+      it "adds to the args array" do
+        expect(step.to_hash[:hadoop_jar_step][:args]).to eq([
+          "s3://us-east-1.elasticmapreduce/libs/pig/pig-script", 
+          "--base-path", "s3://us-east-1.elasticmapreduce/libs/pig/", 
+          "--pig-versions", "0.11.1.1", 
+          "--run-pig-script", 
+          "--args", 
+          "-f", "s3://script_bucket/#{expected_script_path}",
+          "-p", "foo=bar",
+        ])
+      end
+    end
+
+    context "with pig_version" do
+      let(:step_args) { {pig_version: 'pig_version'} }
+
+      it "adds to the args array" do
+        expect(step.to_hash[:hadoop_jar_step][:args]).to eq([
+          "s3://us-east-1.elasticmapreduce/libs/pig/pig-script", 
+          "--base-path", "s3://us-east-1.elasticmapreduce/libs/pig/", 
+          "--pig-versions", "pig_version", 
+          "--run-pig-script", 
+          "--args", 
+          "-f", "s3://script_bucket/#{expected_script_path}",
+        ])
+      end
+    end
+
+    context "with action_on_failure" do
+      let(:step_args) { {action_on_failure: 'action_on_failure'} }
+
+      it "sets the step action on failure" do
+        expect(step.to_hash[:action_on_failure]).to eq('action_on_failure')
+      end
+    end
+
+    context "with dry_run" do
+      let(:step_args) { {dry_run: true} }
+
+      it "doesn't upload pig script" do
+        step.to_hash
+
+        expect(written).to eq([])
+      end
+    end
+  end
+end

data/spec/rp/emr/step/s3_dist_cp_step_spec.rb

@@ -0,0 +1,83 @@
+require 'spec_helper'
+
+describe RP::EMR::Step::S3DistCp do
+  describe "#to_hash" do
+    let(:step_args) { {} }
+
+    let(:step) do
+      RP::EMR::Step::S3DistCp.new(step_args)
+    end
+
+    it "returns hash" do
+      expect(step.to_hash).to eq(
+        :name=>"S3DistCp",
+        :hadoop_jar_step=>{
+          :jar=>"/home/hadoop/lib/emr-s3distcp-1.0.jar",
+          :args=>[]
+        }
+      )
+    end
+
+    context "with hash args" do
+      let(:step_args) do
+        {
+          src: 'src',
+          dest: 'dest',
+          groupBy: '.*',
+          outputCodec: nil,
+        }
+      end
+
+      it "returns hash" do
+        expect(step.to_hash[:hadoop_jar_step][:args]).to eq([
+          '--src', 'src',
+          '--dest', 'dest',
+          '--groupBy', '.*',
+        ])
+      end
+    end
+
+    context "with numeric hash args" do
+      let(:step_args) do
+        {targetSize: 100}
+      end
+
+      it "converts to strings" do
+        expect(step.to_hash[:hadoop_jar_step][:args]).to eq([
+          '--targetSize', '100',
+        ])
+      end
+    end
+
+    context "with boolean args" do
+      let(:step_args) do 
+        {
+          deleteOnSuccess: true,
+          numberFiles: false,
+        }
+      end
+
+      it "returns hash" do
+        expect(step.to_hash[:hadoop_jar_step][:args]).to eq([
+          '--deleteOnSuccess',
+        ])
+      end
+    end
+
+    context "with action_on_failure" do
+      let(:step_args) { {action_on_failure: 'action_on_failure'} }
+
+      it "sets the step action on failure" do
+        expect(step.to_hash[:action_on_failure]).to eq('action_on_failure')
+      end
+    end
+
+    context "with name" do
+      let(:step_args) { {name: 'name'} }
+
+      it "sets the step name" do
+        expect(step.to_hash[:name]).to eq('name')
+      end
+    end
+  end
+end

data/spec/rp/emr/step/setup_debugging_spec.rb

@@ -0,0 +1,29 @@
+require 'spec_helper'
+
+describe RP::EMR::Step::SetupDebugging do
+  describe "#to_hash" do
+    let(:step_args) { {} }
+
+    let(:step) do
+      RP::EMR::Step::SetupDebugging.new(step_args)
+    end
+
+    it "returns hash" do
+      expect(step.to_hash).to eq(
+        :name=>"Setup Hadoop Debugging",
+        :hadoop_jar_step=>{
+          :jar=>"s3://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jar", 
+          :args=>["s3://us-east-1.elasticmapreduce/libs/state-pusher/0.1/fetch"]
+        }
+      )
+    end
+
+    context "with action_on_failure" do
+      let(:step_args) { {action_on_failure: 'action_on_failure'} }
+
+      it "sets the step action on failure" do
+        expect(step.to_hash[:action_on_failure]).to eq('action_on_failure')
+      end
+    end
+  end
+end

data/spec/rp/emr/step/setup_pig_spec.rb

@@ -0,0 +1,47 @@
+require 'spec_helper'
+
+describe RP::EMR::Step::SetupPig do
+  describe "#to_hash" do
+    let(:step_args) { {} }
+
+    let(:step) do
+      RP::EMR::Step::SetupPig.new(step_args)
+    end
+
+    it "returns hash" do
+      expect(step.to_hash).to eq(
+        :name=>"Setup Pig",
+        :hadoop_jar_step=>{
+          :jar=>"s3://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jar", 
+          :args=>[
+            "s3://us-east-1.elasticmapreduce/libs/pig/pig-script", 
+            "--base-path", "s3://us-east-1.elasticmapreduce/libs/pig/", 
+            "--install-pig", 
+            "--pig-versions", "0.11.1.1"
+          ]
+        }
+      )
+    end
+
+    context "with pig_version" do
+      let(:step_args) { {pig_version: 'pig_version'} }
+
+      it "sets the pig version" do
+        expect(step.to_hash[:hadoop_jar_step][:args]).to eq([
+          "s3://us-east-1.elasticmapreduce/libs/pig/pig-script", 
+          "--base-path", "s3://us-east-1.elasticmapreduce/libs/pig/", 
+          "--install-pig", 
+          "--pig-versions", "pig_version"
+        ])
+      end
+    end
+
+    context "with action_on_failure" do
+      let(:step_args) { {action_on_failure: 'action_on_failure'} }
+
+      it "sets the step action on failure" do
+        expect(step.to_hash[:action_on_failure]).to eq('action_on_failure')
+      end
+    end
+  end
+end

data/spec/rp/emr/step_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+
+describe RP::EMR::Step do
+  describe "#to_hash" do
+    let(:step_args) { {} }
+
+    let(:step) do
+      RP::EMR::Step.new(step_args) do |s|
+        s.name = 'name'
+      end
+    end
+
+    it "returns a hash" do
+      expect(step.to_hash).to eq(name: 'name')
+    end
+
+    context "with action_on_failure" do
+      let(:step_args) { {action_on_failure: 'action_on_failure'} }
+
+      it "sets action on failure" do
+        expect(step.to_hash[:action_on_failure]).to eq('action_on_failure')
+      end
+    end
+
+    context "with hadoop_jar_step" do
+      let(:step_args) { {hadoop_jar_step: 'hadoop_jar_step'} }
+
+      it "sets hadoop jar step" do
+        expect(step.to_hash[:hadoop_jar_step]).to eq('hadoop_jar_step')
+      end
+    end
+  end
+end

data/spec/rp/emr_spec.rb

@@ -0,0 +1,5 @@
+require 'spec_helper'
+
+describe RP::EMR do
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,10 @@
+require 'rubygems'
+require 'rspec'
+require 'pry'
+
+require 'rp/emr'
+
+RSpec.configure do |config|
+  config.order = "random"
+end
+

metadata

@@ -0,0 +1,221 @@
+--- !ruby/object:Gem::Specification
+name: rp-emr
+version: !ruby/object:Gem::Version
+  version: 1.0.3
+platform: ruby
+authors:
+- Ryan Michael
+- Andrew Harrison
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: assembler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fuubar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Framework for launching EMR job flows
+email:
+- ryanmichael@otherinbox.com
+- andrew.harrison@returnpath.com
+executables:
+- emr
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/emr
+- lib/rp/emr.rb
+- lib/rp/emr/bootstrap_action.rb
+- lib/rp/emr/cli.rb
+- lib/rp/emr/instance_group.rb
+- lib/rp/emr/instance_groups.rb
+- lib/rp/emr/instances.rb
+- lib/rp/emr/job.rb
+- lib/rp/emr/step.rb
+- lib/rp/emr/step/pig.rb
+- lib/rp/emr/step/s3_dist_cp.rb
+- lib/rp/emr/step/setup_debugging.rb
+- lib/rp/emr/step/setup_hive.rb
+- lib/rp/emr/step/setup_pig.rb
+- lib/rp/emr/version.rb
+- rp-emr.gemspec
+- spec/rp/emr/bootstrap_action_spec.rb
+- spec/rp/emr/instance_group_spec.rb
+- spec/rp/emr/instance_groups_spec.rb
+- spec/rp/emr/instances_spec.rb
+- spec/rp/emr/job_spec.rb
+- spec/rp/emr/step/pig_spec.rb
+- spec/rp/emr/step/s3_dist_cp_step_spec.rb
+- spec/rp/emr/step/setup_debugging_spec.rb
+- spec/rp/emr/step/setup_pig_spec.rb
+- spec/rp/emr/step_spec.rb
+- spec/rp/emr_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.1
+signing_key: 
+specification_version: 4
+summary: EMR Helpers
+test_files:
+- spec/rp/emr/bootstrap_action_spec.rb
+- spec/rp/emr/instance_group_spec.rb
+- spec/rp/emr/instance_groups_spec.rb
+- spec/rp/emr/instances_spec.rb
+- spec/rp/emr/job_spec.rb
+- spec/rp/emr/step/pig_spec.rb
+- spec/rp/emr/step/s3_dist_cp_step_spec.rb
+- spec/rp/emr/step/setup_debugging_spec.rb
+- spec/rp/emr/step/setup_pig_spec.rb
+- spec/rp/emr/step_spec.rb
+- spec/rp/emr_spec.rb
+- spec/spec_helper.rb
+has_rdoc: