rp-emr 1.0.3

data/lib/rp/emr/instance_groups.rb

@@ -0,0 +1,66 @@
+module RP
+  module EMR
+    class InstanceGroups
+      extend Assembler
+
+      assemble_from(
+        default_instance_type: 't1.micro',
+
+        master_instance_type: nil,
+        master_instance_count: 1,
+        master_market: 'ON_DEMAND',
+        master_bid_price: nil,
+
+        core_instance_type: nil,
+        core_instance_count: 1,
+        core_market: 'ON_DEMAND',
+        core_bid_price: nil,
+
+        task_instance_type: nil,
+        task_instance_count: 1,
+        task_market: 'ON_DEMAND',
+        task_bid_price: nil,
+      )
+
+      def to_a
+        [
+          master_instance_group.to_hash, 
+          core_instance_group.to_hash, 
+          task_instance_group.to_hash,
+        ].reject { |h| h[:instance_count] == 0 }
+      end
+
+      private
+
+      def master_instance_group
+        RP::EMR::InstanceGroup.new do |ig|
+          ig.instance_role = 'MASTER'
+          ig.instance_type = master_instance_type || default_instance_type
+          ig.instance_count = master_instance_count
+          ig.market = master_market
+          ig.bid_price = master_bid_price
+        end
+      end
+
+      def core_instance_group
+        RP::EMR::InstanceGroup.new do |ig|
+          ig.instance_role = 'CORE'
+          ig.instance_type = core_instance_type || default_instance_type
+          ig.instance_count = core_instance_count
+          ig.market = core_market
+          ig.bid_price = core_bid_price
+        end
+      end
+
+      def task_instance_group
+        RP::EMR::InstanceGroup.new do |ig|
+          ig.instance_role = 'TASK'
+          ig.instance_type = task_instance_type || default_instance_type
+          ig.instance_count = task_instance_count
+          ig.market = task_market
+          ig.bid_price = task_bid_price
+        end
+      end
+    end
+  end
+end

data/lib/rp/emr/instances.rb

@@ -0,0 +1,36 @@
+module RP
+  module EMR
+    class Instances
+      extend Assembler
+
+      assemble_from(
+        # Optional params
+        master_instance_type: nil,
+        slave_instance_type: nil,
+        instance_count: nil,
+        instance_groups: nil,
+        ec2_key_name: nil,
+        placement: nil,
+        keep_job_flow_alive_when_no_steps: nil,
+        termination_protected: nil,
+        hadoop_version: nil,
+        ec2_subnet_id: nil,
+      )
+
+      def to_hash
+        {
+          master_instance_type: master_instance_type,
+          slave_instance_type: slave_instance_type,
+          instance_count: instance_count,
+          instance_groups: instance_groups,
+          ec2_key_name: ec2_key_name,
+          placement: placement,
+          keep_job_flow_alive_when_no_steps: keep_job_flow_alive_when_no_steps,
+          termination_protected: termination_protected,
+          hadoop_version: hadoop_version,
+          ec2_subnet_id: ec2_subnet_id,
+        }.reject { |k,v| !v || (v.respond_to?(:empty?) && v.empty?) }
+      end
+    end
+  end
+end

data/lib/rp/emr/job.rb

@@ -0,0 +1,42 @@
+module RP
+  module EMR
+    class Job
+      extend Assembler
+
+      assemble_from(
+        # Required params
+        :instances,
+
+        # Optional params
+        steps: nil,
+        log_uri: nil,
+        additional_info: nil,
+        ami_version: :latest,
+        bootstrap_actions: nil,
+        supported_products: nil,
+        new_supported_products: nil,
+        visible_to_all_users: true,
+        job_flow_role: nil,
+        service_role: nil,
+        tags: nil,
+      )
+
+      def to_hash
+        {
+          instances: instances,
+          log_uri: log_uri,
+          additional_info: additional_info,
+          ami_version: ami_version.to_s,
+          steps: steps,
+          bootstrap_actions: bootstrap_actions,
+          supported_products: supported_products,
+          new_supported_products: new_supported_products,
+          visible_to_all_users: visible_to_all_users,
+          job_flow_role: job_flow_role,
+          service_role: service_role,
+          tags: tags,
+        }.reject { |k,v| !v || (v.respond_to?(:empty?) && v.empty?) }
+      end
+    end
+  end
+end

data/lib/rp/emr/step/pig.rb

@@ -0,0 +1,84 @@
+require 'digest/md5'
+
+module RP
+  module EMR
+    class Step
+      class Pig
+        extend Assembler
+
+        assemble_from(
+          :name,
+          :script_path,
+          :script_bucket,
+          args: [],
+          pig_params: {},
+          pig_version: '0.11.1.1',
+          action_on_failure: nil,
+          dry_run: false,
+        )
+
+        def to_hash
+          @hash ||= begin
+            upload_script! unless dry_run
+            step.to_hash
+          end
+        end
+
+        private
+
+        def upload_script!
+          # puts "Uploading to s3://#{script_bucket}/#{script_key}"
+          s3.buckets[script_bucket].objects[script_key].write(script)
+        end
+
+        def script
+          @script ||= File.open(script_path, 'r').read
+        end
+
+        def script_key
+          @script_key ||= begin
+            hash = Digest::MD5.hexdigest(script)
+            "scripts/emr_gem/#{File.basename(script_path, '.pig')}_#{hash}.pig"
+          end
+        end
+
+        def script_url
+          "s3://#{script_bucket}/#{script_key}"
+        end
+
+        def step
+          RP::EMR::Step.new(
+            name: name,
+            action_on_failure: action_on_failure,
+            hadoop_jar_step: {
+              jar: 's3://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jar',
+              args: hadoop_jar_base_args + args + formatted_params,
+            }
+          )
+        end
+
+        def hadoop_jar_base_args
+          [
+            's3://us-east-1.elasticmapreduce/libs/pig/pig-script',
+            '--base-path', 's3://us-east-1.elasticmapreduce/libs/pig/',
+            '--pig-versions', pig_version,
+            '--run-pig-script',
+          ]
+        end
+
+        def formatted_params
+          [
+            '--args',
+            '-f', script_url,
+          ] + pig_params.
+            reject { |k, v| v.nil? }.
+            flat_map { |k, v| ['-p', "#{k}=#{v}"] }
+        end
+
+        def s3
+          AWS::S3.new
+        end
+      end
+    end
+  end
+end

data/lib/rp/emr/step/s3_dist_cp.rb

@@ -0,0 +1,93 @@
+module RP
+  module EMR
+    class Step
+      # Create a S3DistCp step
+      # http://docs.aws.amazon.com/ElasticMapReduce/latest/DeveloperGuide/UsingEMR_s3distcp.html
+      #
+      # Handles annoying details like argument escaping
+      #
+      # Accepts all the input parameters listed in the documentation as of version 1.0.7.
+      # 
+      # @example
+      #   step = S3DistCpStep.new do |s|
+      #     s.src = 's3://bucket/input/prefix/'   # Note this is NOT Hadoop's glob syntax
+      #     s.dest = 's3://bucket/output/path'
+      #     s.srcPattern = 's3://bucket/input/prefix/[foo|bar].*\.eml'     # Input regex - see Java's regex docs
+      #     s.groupBy = '.*([a-z0-9]{2}).tsv'     # Note that you need a capture group
+      #     s.targetSize = 120.megabytes
+      #     s.compression = 'snappy'
+      #     s.deleteOnSuccess = true
+      #   end
+      #
+      #   step.to_hash                            # => Ruby hash ready for use in :steps key of a job
+      #
+      class S3DistCp
+        extend Assembler
+
+        DEFAULT_S3_DISTCP_JAR = '/home/hadoop/lib/emr-s3distcp-1.0.jar'
+
+        HASH_FIELDS = [
+          :src,
+          :dest,
+          :groupBy,
+          :targetSize,
+          :outputCodec,
+          :multipartUploadChunkSize,
+          :startingIndex,
+          :outputManifest,
+          :previousManifest,
+          :s3Endpoint,
+          :srcPattern,
+        ]
+
+        BOOLEAN_FIELDS = [
+          :s3ServerSideEncryption,
+          :deleteOnSuccess,
+          :disableMultipartUpload,
+          :numberFiles,
+          :copyFromManifest,
+        ]
+
+        assemble_from(name: 'S3DistCp',
+                      action_on_failure: nil,
+                      s3_distcp_jar: DEFAULT_S3_DISTCP_JAR)
+        assemble_from(Hash[HASH_FIELDS.map { |f| [f, nil] }])
+        assemble_from(Hash[BOOLEAN_FIELDS.map { |f| [f, false] }])
+
+        def to_hash
+          step.to_hash
+        end
+
+        private
+
+        def step
+          RP::EMR::Step.new(
+            name: name,
+            action_on_failure: action_on_failure,
+            hadoop_jar_step: {
+              jar: s3_distcp_jar,
+              args: hash_field_args + boolean_fields_args,
+            }
+          )
+        end
+
+        def hash_field_args
+          HASH_FIELDS.each do |f|
+            raise ArgumentError, "I don't know how to handle whitespace" if send(f) =~ / /
+          end
+
+          HASH_FIELDS.
+            map { |f| [f, send(f)] }.
+            reject { |k, v| v.nil? }.
+            flat_map { |k, v| ["--#{k}", v.to_s] }
+        end
+
+        def boolean_fields_args
+          BOOLEAN_FIELDS.
+            reject { |f| !send(f) }.
+            map { |f| "--#{f}" }
+        end
+      end
+    end
+  end
+end

data/lib/rp/emr/step/setup_debugging.rb

@@ -0,0 +1,28 @@
+module RP
+  module EMR
+    class Step
+      class SetupDebugging
+        extend Assembler
+
+        assemble_from action_on_failure: nil
+
+        def to_hash
+          step.to_hash
+        end
+
+        private
+
+        def step
+          RP::EMR::Step.new(
+            name: "Setup Hadoop Debugging",
+            action_on_failure: action_on_failure,
+            hadoop_jar_step: {
+              jar: 's3://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jar',
+              args: ['s3://us-east-1.elasticmapreduce/libs/state-pusher/0.1/fetch'],
+            }
+          )
+        end
+      end
+    end
+  end
+end

data/lib/rp/emr/step/setup_hive.rb

@@ -0,0 +1,36 @@
+module RP
+  module EMR
+    class Step
+      class SetupHive
+        extend Assembler
+
+        assemble_from(
+          hive_version: 'latest',
+          action_on_failure: nil,
+        )
+
+        def to_hash
+          step.to_hash
+        end
+
+        private
+
+        def step
+          RP::EMR::Step.new(
+            name: "Setup Hive",
+            action_on_failure: action_on_failure,
+            hadoop_jar_step: {
+              jar: 's3://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jar',
+              args: [
+                's3://us-east-1.elasticmapreduce/libs/hive/hive-script',
+                '--base-path',  's3://us-east-1.elasticmapreduce/libs/hive/',
+                '--install-hive',
+                '--hive-versions', hive_version,
+              ]
+            }
+          )
+        end
+      end
+    end
+  end
+end

data/lib/rp/emr/step/setup_pig.rb

@@ -0,0 +1,36 @@
+module RP
+  module EMR
+    class Step
+      class SetupPig
+        extend Assembler
+
+        assemble_from(
+          pig_version: '0.11.1.1',
+          action_on_failure: nil,
+        )
+
+        def to_hash
+          step.to_hash
+        end
+
+        private
+
+        def step
+          RP::EMR::Step.new(
+            name: "Setup Pig",
+            action_on_failure: action_on_failure,
+            hadoop_jar_step: {
+              jar: 's3://us-east-1.elasticmapreduce/libs/script-runner/script-runner.jar',
+              args: [
+                's3://us-east-1.elasticmapreduce/libs/pig/pig-script',
+                '--base-path',  's3://us-east-1.elasticmapreduce/libs/pig/',
+                '--install-pig',
+                '--pig-versions', pig_version,
+              ]
+            }
+          )
+        end
+      end
+    end
+  end
+end

data/lib/rp/emr/step.rb

@@ -0,0 +1,21 @@
+module RP
+  module EMR
+    class Step
+      extend Assembler
+
+      assemble_from(
+        :name,
+        action_on_failure: nil,
+        hadoop_jar_step: nil,
+      )
+
+      def to_hash
+        {
+          name: name,
+          action_on_failure: action_on_failure,
+          hadoop_jar_step: hadoop_jar_step,
+        }.reject { |k,v| !v || (v.respond_to?(:empty?) && v.empty?) }
+      end
+    end
+  end
+end

data/lib/rp/emr/version.rb

@@ -0,0 +1,5 @@
+module RP
+  module EMR
+    VERSION = "1.0.3"
+  end
+end

data/lib/rp/emr.rb

@@ -0,0 +1,26 @@
+require 'active_support'
+require 'active_support/core_ext/numeric'
+require 'assembler'
+require 'aws-sdk'
+require 'ostruct'
+require 'pp'
+require 'thor'
+
+
+module RP
+  module EMR
+  end
+end
+
+require_relative 'emr/cli'
+require_relative 'emr/instance_group'
+require_relative 'emr/instances'
+require_relative 'emr/step'
+require_relative 'emr/job'
+require_relative 'emr/instance_groups'
+require_relative 'emr/bootstrap_action'
+require_relative 'emr/step/pig'
+require_relative 'emr/step/s3_dist_cp'
+require_relative 'emr/step/setup_debugging'
+require_relative 'emr/step/setup_pig'
+require_relative 'emr/step/setup_hive'

data/rp-emr.gemspec

@@ -0,0 +1,31 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rp/emr/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "rp-emr"
+  spec.version       = RP::EMR::VERSION
+  spec.authors       = ["Ryan Michael", "Andrew Harrison"]
+  spec.email         = ["ryanmichael@otherinbox.com", 'andrew.harrison@returnpath.com']
+  spec.summary       = %q{EMR Helpers}
+  spec.description   = %q{Framework for launching EMR job flows}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "activesupport"
+  spec.add_dependency "aws-sdk"
+  spec.add_dependency "assembler"
+  spec.add_dependency "thor"
+
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "fuubar"
+end

data/spec/rp/emr/bootstrap_action_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+describe RP::EMR::BootstrapAction do
+  describe "#to_hash" do
+    let(:action) do
+      RP::EMR::BootstrapAction.new do |a|
+        a.name = 'name'
+        a.path = 'path'
+        a.args = ['args']
+      end
+    end
+
+    it "returns hash" do
+      expect(action.to_hash).to eq(
+        name: 'name',
+        script_bootstrap_action: {
+          path: 'path',
+          args: ['args'],
+        },
+      )
+    end
+  end
+end

data/spec/rp/emr/instance_group_spec.rb

@@ -0,0 +1,51 @@
+require 'spec_helper'
+
+describe RP::EMR::InstanceGroup do
+  describe "#to_hash" do
+    let(:group_args) { {} }
+
+    let(:group) do
+      RP::EMR::InstanceGroup.new(group_args) do |ig|
+        ig.instance_role = 'instance_role'
+        ig.instance_type = 'instance_type'
+        ig.instance_count = 'instance_count'
+      end
+    end
+
+    it "returns a hash" do
+      expect(group.to_hash).to eq(
+        :instance_role=>"instance_role",
+        :instance_type=>"instance_type",
+        :instance_count=>"instance_count",
+      )
+    end
+
+    context "with name" do
+      let(:group_args) { {name: 'name'} }
+
+      it "sets name" do
+        expect(group.to_hash[:name]).to eq('name')
+      end
+    end
+
+    context "with market" do
+      let(:group_args) { {market: 'market'} }
+
+      it "sets name" do
+        expect(group.to_hash[:market]).to eq('market')
+      end
+    end
+
+    context "with bid_price" do
+      let(:group_args) { {bid_price: 1.0} }
+
+      it "sets market" do
+        expect(group.to_hash[:market]).to eq('SPOT')
+      end
+
+      it "sets bid_price" do
+        expect(group.to_hash[:bid_price]).to eq('1.0')
+      end
+    end
+  end
+end

data/spec/rp/emr/instance_groups_spec.rb

@@ -0,0 +1,106 @@
+require 'spec_helper'
+
+describe RP::EMR::InstanceGroups do
+  describe "#to_a" do
+    let(:group_args) { {} }
+
+    let(:groups) do
+      RP::EMR::InstanceGroups.new(group_args)
+    end
+
+    it "returns hash" do
+      expect(groups.to_a).to eq([
+        {
+          :market=>"ON_DEMAND",
+          :instance_role=>"MASTER",
+          :instance_type=>"t1.micro",
+          :instance_count=>1,
+        },
+        {
+          :market=>"ON_DEMAND",
+          :instance_role=>"CORE",
+          :instance_type=>"t1.micro",
+          :instance_count=>1,
+        },
+        {
+          :market=>"ON_DEMAND",
+          :instance_role=>"TASK",
+          :instance_type=>"t1.micro",
+          :instance_count=>1,
+        },
+      ])
+    end
+
+    context "with default_instance_type" do
+      let(:group_args) { {default_instance_type: 'default_instance_type'} }
+
+      it "sets instance type" do
+        expect(groups.to_a.map { |h| h[:instance_type] }.uniq).to eq(['default_instance_type'])
+      end
+    end
+
+    context "with instance_type" do
+      let(:group_args) do
+        {
+          master_instance_type: 'master_instance_type',
+          core_instance_type: 'core_instance_type',
+          task_instance_type: 'task_instance_type',
+        }
+      end
+
+      it "sets instance type" do
+        expect(groups.to_a.map { |h| h[:instance_type] }).to eq([
+          'master_instance_type',
+          'core_instance_type',
+          'task_instance_type',
+        ])
+      end
+    end
+
+    context "with instance_count" do
+      let(:group_args) do
+        {
+          master_instance_count: 1,
+          core_instance_count: 2,
+          task_instance_count: 3,
+        }
+      end
+
+      it "sets instance count" do
+        expect(groups.to_a.map { |h| h[:instance_count] }).to eq([1, 2, 3])
+      end
+    end
+
+    context "with bid_price" do
+      let(:group_args) do
+        {
+          master_bid_price: 1,
+          core_bid_price: 2,
+          task_bid_price: 3,
+        }
+      end
+
+      it "sets bid price" do
+        expect(groups.to_a.map { |h| h[:bid_price] }).to eq(['1', '2', '3'])
+      end
+
+      it "sets market" do
+        expect(groups.to_a.map { |h| h[:market] }.uniq).to eq(['SPOT'])
+      end
+    end
+
+    context "with market" do
+      let(:group_args) do
+        {
+          master_market: 'master_market',
+          core_market: 'core_market',
+          task_market: 'task_market',
+        }
+      end
+
+      it "sets market" do
+        expect(groups.to_a.map { |h| h[:market] }).to eq(['master_market', 'core_market', 'task_market'])
+      end
+    end
+  end
+end