rubadoop 0.7.8

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e877fa9607f54419f1a7cd9d2033439465bc7fd1
+  data.tar.gz: 99cd20b45eb293eae96a2836ae0e207b83d38ff3
+SHA512:
+  metadata.gz: 22c74443aabfe4b73edba04d89ccd4ce2ddd9eaaa8c3f7db389165a359f566a4d83f5608d051ed1af92700b5ce9b45177a18678155785f8677ee5d13320659ac
+  data.tar.gz: 570d9085f2dc997b908d09d9a12ec3d5041f85e6ae95cb260f3753de6ecab3bb23c2293bcfdd111ad7bbcd5551c9a548746891a00bd1093cf4279ca5307abd54

data/Gemfile

@@ -0,0 +1,7 @@
+source "http://rubygems.org"
+
+gemspec
+
+group :test do
+  gem 'simplecov', require: false
+end

data/Gemfile.lock

@@ -0,0 +1,43 @@
+PATH
+  remote: .
+  specs:
+    rubadoop (0.7.7)
+      activesupport (>= 3)
+      aws-sdk (>= 1.5.8)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    activesupport (4.0.0)
+      i18n (~> 0.6, >= 0.6.4)
+      minitest (~> 4.2)
+      multi_json (~> 1.3)
+      thread_safe (~> 0.1)
+      tzinfo (~> 0.3.37)
+    atomic (1.1.14)
+    aws-sdk (1.24.0)
+      json (~> 1.4)
+      nokogiri (>= 1.4.4, < 1.6.0)
+      uuidtools (~> 2.1)
+    i18n (0.6.5)
+    json (1.8.1)
+    minitest (4.7.5)
+    multi_json (1.8.2)
+    nokogiri (1.5.10)
+    rdoc (4.0.0)
+    simplecov (0.7.1)
+      multi_json (~> 1.0)
+      simplecov-html (~> 0.7.1)
+    simplecov-html (0.7.1)
+    thread_safe (0.1.3)
+      atomic
+    tzinfo (0.3.38)
+    uuidtools (2.1.4)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  rdoc
+  rubadoop!
+  simplecov

data/README.rdoc

@@ -0,0 +1,7 @@
+Rubadoop provides:
+
+* Streaming support structure
+* Hadoop Definition DSL
+* Amazon::EMR Definition DSL
+* Oozie Definition DSL
+

data/Rakefile

@@ -0,0 +1,25 @@
+require 'rake'
+require 'rake/testtask'
+require 'rdoc/task'
+
+task default: :test
+
+task :build do
+  exec 'gem build rubadoop.gemspec'
+end
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+end
+
+desc 'Generate documentation.'
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'Rubadoop'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README.rdoc')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/lib/rubadoop.rb

@@ -0,0 +1,26 @@
+require 'active_support/dependencies/autoload'
+require 'active_support/concern'
+require 'active_support/core_ext/class/attribute'
+require 'active_support/inflector'
+
+module Rubadoop
+  extend ActiveSupport::Autoload
+
+  autoload :BaseDsl
+  autoload :MapReduce
+
+  module Emr
+    extend ActiveSupport::Autoload
+
+    autoload :JobflowBuilder
+  end
+
+  module Oozie
+    extend ActiveSupport::Autoload
+
+    autoload :WorkflowBuilder
+  end
+
+end
+
+require 'rubadoop/version'

data/lib/rubadoop/base_dsl.rb

@@ -0,0 +1,31 @@
+require 'active_support/core_ext/hash/indifferent_access'
+
+module Rubadoop
+  class BaseDsl
+    attr_reader :params
+
+    def initialize(params)
+
+      unless params.singleton_methods.member? :[]
+        params = params ? params.dup.with_indifferent_access : {}.with_indifferent_access
+
+        def params.[](param)
+          value = super(param)
+          raise "Missing param: #{param}" if value.nil?
+          value
+        end
+      end
+
+      @params = params
+    end
+
+    def has_param?(param)
+      params.has_key? param
+    end
+
+    def optional_param(param, default_value)
+      params[param] = default_value unless has_param? param
+    end
+  end
+end
+

data/lib/rubadoop/emr.rb

@@ -0,0 +1,23 @@
+module Rubadoop
+  module Emr
+    JobFlowStates = {
+        :starting => 'STARTING',
+        :running => 'RUNNING',
+        :waiting => 'WAITING',
+        :shutting_down => 'SHUTTING_DOWN',
+        :terminated => 'TERMINATED',
+        :ended => 'ENDED'
+    }
+    InstanceRoles = {
+        :master => 'MASTER',
+        :core   => 'CORE',
+        :task   => 'TASK'
+    }
+    StepActionOnFailure = {
+        :terminate_job_flow => 'TERMINATE_JOB_FLOW',
+        :cancel_and_wait => 'CANCEL_AND_WAIT',
+        :continue => 'CONTINUE'
+    }
+
+  end
+end

data/lib/rubadoop/emr/jobflow_builder.rb

@@ -0,0 +1,36 @@
+module Rubadoop
+  module Emr
+    module JobflowBuilder
+      extend ActiveSupport::Autoload
+
+      autoload :JobSpec
+      autoload :Step
+      autoload :BootstrapAction
+
+      class << self
+        def new_job_spec(__params__ = {}, &block)
+          job_spec = JobSpec.new(__params__)
+
+          if block_given?
+            if block.arity == 1
+              yield job_spec
+            else
+              job_spec.instance_eval &block
+            end
+          end
+
+          job_spec
+        end
+
+        def load_job_spec(__params__ = {}, __spec_code__)
+          new_job_spec(__params__) do |dsl|
+            dsl.instance_eval __spec_code__
+          end
+        end
+
+      end
+
+    end
+  end
+end
+

data/lib/rubadoop/emr/jobflow_builder/bootstrap_action.rb

@@ -0,0 +1,27 @@
+module Rubadoop
+  module Emr
+    module JobflowBuilder
+      class BootstrapAction < Rubadoop::BaseDsl
+        attr_accessor :args, :name, :path
+
+        def initialize(params, path)
+          super(params)
+          @path = path
+        end
+
+        def arg(*value)
+          @args ||= []
+          @args.concat value
+        end
+
+        def to_h
+          built = {}
+          built[:name] = @name || "Bootstrap Action"
+          built[:script_bootstrap_action] = {:path => @path}
+          built[:script_bootstrap_action][:args] = @args unless @args.nil?
+          built
+        end
+      end
+    end
+  end
+end

data/lib/rubadoop/emr/jobflow_builder/job_spec.rb

@@ -0,0 +1,77 @@
+module Rubadoop
+  module Emr
+    module JobflowBuilder
+      class JobSpec < Rubadoop::BaseDsl
+        attr_accessor :additional_info, :ami_version, :bootstrap_actions, :instances, :log_uri, :name, :steps, :supported_products
+        attr_accessor :termination_protected, :visible_to_all_users
+
+        STREAMING_JAR_LOCATION = '/home/hadoop/contrib/streaming/hadoop-streaming.jar'
+
+        def with_bootstrap_action(path, &block)
+          builder = BootstrapAction.new(params, path)
+          if block_given?
+            if block.arity == 1
+              yield builder
+            else
+              builder.instance_eval &block
+            end
+          end
+          (@bootstrap_actions ||= []) << builder.to_h
+        end
+
+        def with_instances(master_type, core_type, core_count)
+          @instances ||= {}
+          @instances[:master_instance_type] = master_type
+          @instances[:slave_instance_type] = core_type
+          @instances[:instance_count] = core_count + 1
+        end
+
+        def keep_alive(value)
+          raise 'keep_alive value must be true/false' unless !!value == value
+          @instances ||= {}
+          @instances[:keep_job_flow_alive_when_no_steps] = value
+        end
+
+        def add_jar_step(name, jar, main_class = nil, &block)
+          builder = Step::Java.new(params, jar, main_class)
+          builder.name = name
+          if block_given?
+            if block.arity == 1
+              yield builder
+            else
+              builder.instance_eval &block
+            end
+          end
+          (@steps ||= []) << builder.to_h
+        end
+
+        def add_streaming_step(name, &block)
+          builder = Step::Streaming.new(params, STREAMING_JAR_LOCATION)
+          builder.name = name
+          if block_given?
+            if block.arity == 1
+              yield builder
+            else
+              builder.instance_eval &block
+            end
+          end
+          (@steps ||= []) << builder.to_h
+        end
+
+        def to_create_command
+          built = {}
+          [:additional_info, :ami_version, :log_uri, :name, :bootstrap_actions, :instances, :supported_products].each do |attr|
+            attr_value = self.instance_variable_get("@#{attr}")
+            built[attr] = attr_value unless attr_value.nil?
+          end
+
+          built
+        end
+
+        def to_steps_command
+          @steps
+        end
+      end
+    end
+  end
+end

data/lib/rubadoop/emr/jobflow_builder/step.rb

@@ -0,0 +1,41 @@
+module Rubadoop
+  module Emr
+    module JobflowBuilder
+      module Step
+
+        private
+          module StepBehavior
+            attr_accessor :name, :action_on_failure
+
+            def initialize(params, jar, main_class = nil)
+              super(params)
+              @name = name
+              @jar = jar
+              @main_class = main_class if main_class
+            end
+
+            def to_h
+              built = {}
+              built[:name] = @name unless @name.nil?
+              built[:hadoop_jar_step] = {}
+              built[:hadoop_jar_step][:jar] = @jar
+              built[:hadoop_jar_step][:main_class] = @main_class unless @main_class.nil?
+              built[:hadoop_jar_step][:args] = to_hadoop_cli(skip_jar: true)
+              built[:action_on_failure] = @action_on_failure unless @action_on_failure.nil?
+              built
+            end
+          end
+
+        public
+        class Streaming < Rubadoop::MapReduce::CallStreaming
+          include StepBehavior
+        end
+
+        class Java < Rubadoop::MapReduce::CallJava
+          include StepBehavior
+        end
+
+      end
+    end
+  end
+end

data/lib/rubadoop/map_reduce.rb

@@ -0,0 +1,23 @@
+require 'rubadoop/map_reduce/io'
+require 'rubadoop/map_reduce/utils'
+
+module Rubadoop
+  module MapReduce
+    extend ActiveSupport::Autoload
+
+    autoload :CallJava
+    autoload :CallStreaming
+    autoload :JobConfEnvironment
+    autoload :Mappable
+    autoload :Mapper
+    autoload :Reducable
+    autoload :Reducer
+    autoload :Identity
+    autoload :TestAssist
+
+    extend Io
+    extend Utils
+
+  end
+end
+

data/lib/rubadoop/map_reduce/call_java.rb

@@ -0,0 +1,112 @@
+module Rubadoop
+  module MapReduce
+    class CallJava < BaseDsl
+      attr_accessor :jar, :main_class, :envs, :args, :confs, :files, :archives
+
+      class << self
+        def new_java_call(params = {}, &block)
+          builder = CallJava.new(params)
+          if block_given?
+            if block.arity == 1
+              yield builder
+            else
+              builder.instance_eval &block
+            end
+          end
+          builder
+        end
+      end
+
+      def initialize(params = {})
+        super(params)
+      end
+
+      def env(name, value)
+        @envs ||= {}
+        @envs[name.to_sym] = value
+      end
+
+      def conf(name, value)
+        @confs ||= {}
+        @confs[name.to_sym] = value
+      end
+
+      def conf_concat(name, value)
+        @confs ||= {}
+        prev_value = @confs[name.to_sym]
+        if prev_value
+          if prev_value.kind_of?(Array)
+            prev_value << value
+          else
+            @confs[name.to_sym] = [prev_value, value]
+          end
+        else
+          @confs[name.to_sym] = value
+        end
+      end
+
+      def arg(*value)
+        @args ||= []
+        @args.concat value
+      end
+
+      def file(location, symlink)
+        @files ||= []
+        @files << "#{location}##{symlink}"
+      end
+
+      def archive(location, symlink)
+        @archives ||= []
+        @archives << "#{location}##{symlink}"
+      end
+
+      def to_hadoop_cli(opts = {})
+        validate
+        cmd = []
+        unless opts[:skip_jar]
+          cmd.concat ["hadoop", "jar", jar]
+          cmd << @main_class unless @main_class.nil?
+        end
+
+        @confs.each { |key, value|
+          if value.kind_of?(Array)
+            value.each { |entry|
+              cmd. << "-D#{key}=#{entry}"
+            }
+          else
+            cmd. << "-D#{key}=#{value}"
+          end
+        } if @confs
+        @envs.each { |key, value|
+          cmd.concat ['-cmdenv', "#{key}=#{value}"]
+        } if @envs
+        @files.each { |entry|
+          cmd.concat ['-cacheFile', "#{entry}"]
+        } if @files
+        @archives.each { |entry|
+          cmd.concat ['-cacheArchive', "#{entry}"]
+        } if @archives
+        unless opts[:skip_args]
+          cmd.concat @args if @args
+        end
+        cmd
+      end
+
+      def to_h
+        validate
+        built = {}
+        [:@jar, :@main_class, :@envs, :@args, :@confs, :@files, :@archives].each { |entry|
+          built[entry.to_s.delete("@").to_sym] = instance_variable_get(entry)
+        }
+        built
+      end
+
+      def validate
+        [:@jar].each { |property|
+          raise "Missing #{property}" if instance_variable_get(property).nil?
+        }
+      end
+
+    end
+  end
+end