rubadoop 0.7.8

data/lib/rubadoop/map_reduce/utils.rb

@@ -0,0 +1,29 @@
+module Rubadoop
+  module MapReduce
+    module Utils
+      def key_value_split(line)
+        line.split("\t", 2)
+      end
+
+      def log_counter(group, counter, amount)
+        ::Rubadoop::MapReduce.out.counter(group, counter, amount)
+      end
+
+      def log_status(status)
+        ::Rubadoop::MapReduce.out.status(status)
+      end
+
+      def log_error(message)
+        ::Rubadoop::MapReduce.out.error(message)
+      end
+
+      def out_entry(value)
+        ::Rubadoop::MapReduce.out.entry(value)
+      end
+
+      def out_map_entry(key, value)
+        ::Rubadoop::MapReduce.out.map_entry(key, value)
+      end
+    end
+  end
+end

data/lib/rubadoop/oozie/workflow_builder.rb

@@ -0,0 +1,42 @@
+module Rubadoop
+  module Oozie
+    module WorkflowBuilder
+      extend ActiveSupport::Autoload
+
+      autoload :JobProperties
+
+      class << self
+        def new_job_properties(params = {}, &block)
+          job_spec = JobProperties.new(params)
+
+          if block_given?
+            if block.arity == 1
+              yield job_spec
+            else
+              job_spec.instance_eval &block
+            end
+          end
+
+          unless params.has_key? :skip_aws_keys
+            require 'aws-sdk'
+            aws_config = AWS.config.credentials
+            job_spec.prop 'fs.s3n.awsAccessKeyId', aws_config[:access_key_id]
+            job_spec.prop 'fs.s3.awsAccessKeyId', aws_config[:access_key_id]
+            job_spec.prop 'fs.s3n.awsSecretAccessKey', aws_config[:secret_access_key]
+            job_spec.prop 'fs.s3.awsSecretAccessKey', aws_config[:secret_access_key]
+          end
+
+          job_spec
+        end
+
+        def load_job_properties(__params__ = {}, __spec_code__)
+          new_job_properties(__params__) do |dsl|
+            dsl.instance_eval __spec_code__
+          end
+        end
+
+
+      end
+    end
+  end
+end

data/lib/rubadoop/oozie/workflow_builder/job_properties.rb

@@ -0,0 +1,19 @@
+module Rubadoop
+  module Oozie
+    module WorkflowBuilder
+      class JobProperties < Rubadoop::BaseDsl
+
+        def prop(name, value)
+          @props ||= {}
+          @props[name] = value
+        end
+
+        def to_h
+          @props ||= {}
+          @props
+        end
+      end
+
+    end
+  end
+end

data/lib/rubadoop/version.rb

@@ -0,0 +1,3 @@
+module Rubadoop
+  VERSION = "0.7.8"
+end

data/test/rubadoop/base_dsl_test.rb

@@ -0,0 +1,27 @@
+require 'test_helper'
+
+module Rubadoop
+  class BaseDslTest < MiniTest::Spec
+
+    def test_params_missing
+      test = self
+      BaseDsl.new(Hash[a: true, 'dude' => 'rubadoop']).instance_eval do
+        test.assert params[:a]
+        test.assert_raises RuntimeError do
+          params[:nope]
+        end
+      end
+    end
+
+    def test_params_with_indifferent_access
+      test = self
+      BaseDsl.new(Hash[a: true, 'dude' => 'rubadoop']).instance_eval do
+        test.assert params[:a]
+        test.assert params['a']
+        test.assert_equal 'rubadoop', params[:dude]
+        test.assert_equal 'rubadoop', params['dude']
+      end
+    end
+  end
+end
+

data/test/rubadoop/emr/jobflow_builder_test.rb

@@ -0,0 +1,184 @@
+require 'test_helper'
+
+module Rubadoop
+  module Emr
+    class JobflowBuilderTest < MiniTest::Spec
+
+      def test_base
+        command = JobflowBuilder.new_job_spec do; end.to_create_command
+        assert_equal command, Hash.new
+
+        command = JobflowBuilder.new_job_spec do
+          my_name = "testink"
+          @name = my_name
+        end.to_create_command
+        assert_equal command, Hash[name: 'testink']
+      end
+
+      def test_param
+        command = JobflowBuilder.new_job_spec(name: 'testink') do
+          @name = params[:name]
+        end.to_create_command
+        assert_equal command, Hash[name: 'testink']
+      end
+
+      def test_missing_param
+        assert_raises RuntimeError do
+          JobflowBuilder.new_job_spec(name: 'testink') do
+            @name = params[:namey]
+          end.to_create_command
+        end
+
+        assert_raises RuntimeError do
+          JobflowBuilder.new_job_spec(name: 'testink') do |s|
+            s.name = s.params[:namey]
+          end.to_create_command
+        end
+
+        assert_raises NameError do
+          JobflowBuilder.new_job_spec(name: 'testink') do |s|
+            s.name = params[:namey]
+          end.to_create_command
+        end
+      end
+
+      def test_with_instances
+        command = JobflowBuilder.new_job_spec(name: 'something') do
+          optional_param :instance_type, 'c1.medium'
+          optional_param :instance_count, 1
+
+          with_instances 'c1.medium', params[:instance_type], params[:instance_count]
+        end.to_create_command
+        assert_equal command, {:instances=>{:master_instance_type=>"c1.medium", :slave_instance_type=>"c1.medium", :instance_count=>2}}
+      end
+
+      def test_bootstrap
+
+        command = JobflowBuilder.new_job_spec do
+          with_bootstrap_action "bootstrap location" do |b|
+            b.name = 'bootstrap name'
+            b.args = ['arg1', 'arg2']
+          end
+        end.to_create_command
+        assert_equal command, Hash[bootstrap_actions:[{name:"bootstrap name",
+                                              script_bootstrap_action:{path:"bootstrap location",
+                                                                       args:["arg1", "arg2"]}}]]
+
+        command = JobflowBuilder.new_job_spec do
+          with_bootstrap_action "bootstrap location"
+        end.to_create_command
+        assert_equal command, Hash[bootstrap_actions:[{name:"Bootstrap Action",
+                                              script_bootstrap_action:{path:"bootstrap location"}}]]
+
+        command = JobflowBuilder.new_job_spec do
+          with_bootstrap_action "bootstrap location" do
+            @name = 'bootstrap name'
+            arg "arg1", "arg2"
+          end
+        end.to_create_command
+        assert_equal command, Hash[bootstrap_actions:[{name:"bootstrap name",
+                                                       script_bootstrap_action:{path:"bootstrap location",
+                                                                                args:["arg1", "arg2"]}}]]
+      end
+
+      def test_bootstrap_param
+        command = JobflowBuilder.new_job_spec(bs_name: "total bs") do
+          with_bootstrap_action "bootstrap location" do |b|
+            b.name = b.params[:bs_name]
+          end
+        end.to_create_command
+        assert_equal command, Hash[bootstrap_actions:[{name:"total bs",
+                                                       script_bootstrap_action:{path:"bootstrap location"}}]]
+
+        command = JobflowBuilder.new_job_spec(bs_name: "total bs") do
+          with_bootstrap_action "bootstrap location" do
+            @name = params[:bs_name]
+          end
+        end.to_create_command
+        assert_equal command, Hash[bootstrap_actions:[{name:"total bs",
+                                                       script_bootstrap_action:{path:"bootstrap location"}}]]
+      end
+
+      def test_jar_steps
+        command = JobflowBuilder.new_job_spec do
+          add_jar_step 'jar name', 'jar jar', 'Binks' do |s|
+            s.args = ["Meesa", "Stupid"]
+            s.action_on_failure = 'CANCEL_AND_WAIT'
+          end
+        end.to_steps_command
+        assert_equal [{name:"jar name", hadoop_jar_step:{jar:"jar jar", main_class:"Binks",
+            args:["Meesa", "Stupid"]},
+            action_on_failure:"CANCEL_AND_WAIT"}], command
+      end
+
+      def test_streaming_steps
+        command = JobflowBuilder.new_job_spec do
+          add_streaming_step "streaming name" do |s|
+            s.mapper = 'mapit'
+            s.reducer = 'reduceit'
+            s.input = 'fromhere'
+            s.output = 'tothere'
+            s.args = [ 'arg1', 'arg2',]
+            s.action_on_failure = 'CANCEL_AND_WAIT'
+          end
+        end.to_steps_command
+        assert_equal [{name:"streaming name", hadoop_jar_step:{jar: JobflowBuilder::JobSpec::STREAMING_JAR_LOCATION,
+            args:["-input", "fromhere", "-output","tothere", "-mapper", "mapit", "-reducer", "reduceit", "arg1", "arg2"]},
+            action_on_failure:"CANCEL_AND_WAIT"}], command
+      end
+
+      def test_streaming_steps2
+        command = JobflowBuilder.new_job_spec do
+          add_streaming_step "streaming name" do
+            @mapper = 'mapit'
+            @reducer = 'reduceit'
+            @input = 'fromhere'
+            @output = 'tothere'
+            @args = [ 'arg1', 'arg2',]
+            @action_on_failure = 'CANCEL_AND_WAIT'
+          end
+        end.to_steps_command
+        assert_equal [{name:"streaming name", hadoop_jar_step:{jar: JobflowBuilder::JobSpec::STREAMING_JAR_LOCATION,
+            args:["-input", "fromhere", "-output","tothere", "-mapper", "mapit", "-reducer", "reduceit", "arg1", "arg2"]},
+            action_on_failure:"CANCEL_AND_WAIT"}], command
+      end
+
+
+      def test_steps_params
+        command = JobflowBuilder.new_job_spec(arg1: 'aack') do
+          add_jar_step 'jar name', 'jar jar', 'Binks' do |s|
+            s.arg s.params[:arg1]
+            s.action_on_failure = 'CANCEL_AND_WAIT'
+          end
+        end.to_steps_command
+        assert_equal [{name:"jar name", hadoop_jar_step:{jar:"jar jar", main_class:"Binks",
+                                                         args:["aack"]},
+                       action_on_failure:"CANCEL_AND_WAIT"}], command
+
+        command = JobflowBuilder.new_job_spec(arg1: 'aack') do
+          add_jar_step 'jar name', 'jar jar', 'Binks' do
+            arg params[:arg1]
+            @action_on_failure = 'CANCEL_AND_WAIT'
+          end
+        end.to_steps_command
+        assert_equal [{name:"jar name", hadoop_jar_step:{jar:"jar jar", main_class:"Binks",
+                                                         args:["aack"]},
+                       action_on_failure:"CANCEL_AND_WAIT"}], command
+      end
+
+      def test_keepalive
+        command = JobflowBuilder.new_job_spec do |job|
+          job.keep_alive true
+        end.to_create_command
+        assert_equal command, Hash[instances: {keep_job_flow_alive_when_no_steps: true}]
+
+        assert_raises RuntimeError do
+          JobflowBuilder.new_job_spec do
+            keep_alive "false" #if this doesn't check boolean, result would be true
+          end
+        end
+      end
+
+    end
+  end
+end

data/test/rubadoop/map_reduce/call_java_test.rb

@@ -0,0 +1,122 @@
+require 'test_helper'
+
+module Rubadoop
+  module MapReduce
+
+    class CallJavaTest < MiniTest::Spec
+
+      def test_simple
+        call = CallJava.new_java_call() do |s|
+          s.jar = 'hadoop-pie.jar'
+        end
+        assert_equal "hadoop jar hadoop-pie.jar", call.to_hadoop_cli.join(' ')
+      end
+
+      def test_env
+        call = CallJava.new_java_call() do |s|
+          s.jar = 'hadoop-pie.jar'
+          s.env 'abcd', 'efgh'
+        end
+        assert_equal "hadoop jar hadoop-pie.jar -cmdenv abcd=efgh", call.to_hadoop_cli.join(' ')
+      end
+
+      def test_conf
+        call = CallJava.new_java_call() do |s|
+          s.jar = 'hadoop-pie.jar'
+          s.conf 'abcd', 'efgh'
+        end
+        assert_equal "hadoop jar hadoop-pie.jar -Dabcd=efgh", call.to_hadoop_cli.join(' ')
+      end
+
+      def test_conf_multi
+        call = CallJava.new_java_call() do |s|
+          s.jar = 'hadoop-pie.jar'
+          s.conf 'abcd', 'efgh'
+          s.conf_concat 'abcd', 'ijkl'
+          s.conf_concat 'abcd', 'mnop'
+        end
+        assert_equal "hadoop jar hadoop-pie.jar -Dabcd=efgh -Dabcd=ijkl -Dabcd=mnop", call.to_hadoop_cli.join(' ')
+
+        call = CallJava.new_java_call() do |s|
+          s.jar = 'hadoop-pie.jar'
+          s.conf_concat 'abcd', 'mnop'
+        end
+        assert_equal "hadoop jar hadoop-pie.jar -Dabcd=mnop", call.to_hadoop_cli.join(' ')
+      end
+
+      def test_arg
+        call = CallJava.new_java_call() do
+          @jar = 'hadoop-pie.jar'
+          arg 'abcd', 'efgh'
+        end
+        assert_equal "hadoop jar hadoop-pie.jar abcd efgh", call.to_hadoop_cli.join(' ')
+      end
+
+      def test_archive
+        call = CallJava.new_java_call() do
+          @jar = 'hadoop-pie.jar'
+          archive 'dungeon', 'd'
+        end
+        assert_equal "hadoop jar hadoop-pie.jar -cacheArchive dungeon#d", call.to_hadoop_cli.join(' ')
+      end
+
+      def test_files
+        call = CallJava.new_java_call() do
+          @jar = 'hadoop-pie.jar'
+          file 'phile', 'f'
+        end
+        assert_equal "hadoop jar hadoop-pie.jar -cacheFile phile#f", call.to_hadoop_cli.join(' ')
+      end
+
+      def test_order
+        call = CallJava.new_java_call() do
+          @jar = 'hadoop-pie.jar'
+          @main_class = 'com.java.package.DoItLive'
+          env 'e1', 'dudio'
+          env 'e2', 'dudi-rio'
+          file 'phile', 'f'
+          conf 'c1', 'conf'
+          conf 'c2', 'cronf'
+          arg 'seriously'
+          archive 'dungeon', 'd'
+          file 'phile2', 'f2'
+        end
+        assert_equal "hadoop jar hadoop-pie.jar com.java.package.DoItLive -Dc1=conf -Dc2=cronf -cmdenv e1=dudio -cmdenv e2=dudi-rio -cacheFile phile#f -cacheFile phile2#f2 -cacheArchive dungeon#d seriously", call.to_hadoop_cli.join(' ')
+      end
+
+      def test_validation
+        assert_raises RuntimeError do
+          CallJava.new_java_call() do
+            @main_class = 'com.java.package.DoItLive'
+          end.to_hadoop_cli
+        end
+
+        assert_raises RuntimeError do
+          CallJava.new_java_call() do
+            @main_class = 'com.java.package.DoItLive'
+          end.to_h
+        end
+      end
+
+      def test_hash
+        call = CallJava.new_java_call() do
+          @jar = 'hadoop-pie.jar'
+          @main_class = 'com.java.package.DoItLive'
+          env 'e1', 'dudio'
+          env 'e2', 'dudi-rio'
+          file 'phile', 'f'
+          conf 'c1', 'conf'
+          conf 'c2', 'cronf'
+          arg 'seriously'
+          archive 'dungeon', 'd'
+          file 'phile', 'f'
+          @poopsy = 'poo'
+        end
+        assert_equal Hash[jar: "hadoop-pie.jar", archives: ["dungeon#d"],
+                          main_class: "com.java.package.DoItLive", envs: {e1: "dudio", e2: "dudi-rio"},
+                          files: ["phile#f", "phile#f"], confs: {c1: "conf", c2: "cronf"}, args: ["seriously"]], call.to_h
+      end
+
+    end
+  end
+end

data/test/rubadoop/map_reduce/call_streaming_test.rb

@@ -0,0 +1,81 @@
+require 'test_helper'
+
+module Rubadoop
+  module MapReduce
+
+    class CallStreamingTest < MiniTest::Spec
+
+      def test_simple
+        streaming = CallStreaming.new_streaming_call do |s|
+          s.jar = 'hadoop-streaming.jar'
+          s.input = 'input'
+          s.output = 'output'
+          s.mapper = '/bin/cat'
+          s.reducer = '/bin/wc'
+        end
+        assert_equal "hadoop jar hadoop-streaming.jar -input input -output output -mapper /bin/cat -reducer /bin/wc", streaming.to_hadoop_cli.join(' ')
+      end
+
+      def test_hash
+        streaming = CallStreaming.new_streaming_call do |s|
+          s.jar = 'hadoop-streaming.jar'
+          s.inputformat = 'green'
+          s.input = 'input'
+          s.outputformat = 'blue'
+          s.output = 'output'
+          s.mapper = '/bin/cat'
+          s.reducer = '/bin/wc'
+          @test = 'noshow'
+        end
+        assert_equal Hash[jar: "hadoop-streaming.jar", main_class: nil, envs: nil, args: nil, confs: nil,  files: nil, archives: nil,
+                          mapper: "/bin/cat", reducer: "/bin/wc", inputformat: "green", input: "input", outputformat: "blue", output: "output"], streaming.to_h
+      end
+
+      def test_cli
+        streaming = CallStreaming.new_streaming_call do |s|
+          s.jar = 'hadoop-streaming.jar'
+          s.inputformat = 'green'
+          s.input = 'input'
+          s.outputformat = 'blue'
+          s.output = 'output'
+          s.mapper = '/bin/cat'
+          s.reducer = '/bin/wc'
+          @test = 'noshow'
+        end
+        assert_equal ["hadoop", "jar", "hadoop-streaming.jar", "-inputformat", "green", "-input", "input", "-outputformat", "blue",
+                      "-output", "output", "-mapper", "/bin/cat", "-reducer", "/bin/wc"], streaming.to_hadoop_cli
+      end
+
+      def test_validation
+        assert_raises RuntimeError do
+          CallStreaming.new_streaming_call do |s|
+            s.jar = 'hadoop-streaming.jar'
+            s.input = 'input'
+            s.output = 'output'
+            s.reducer = '/bin/wc'
+          end.to_hadoop_cli
+        end
+
+        assert_raises RuntimeError do
+          CallStreaming.new_streaming_call() do
+            @jar = 'hadoop-streaming.jar'
+            @input = 'input'
+            @mapper = '/bin/cat'
+            @reducer = '/bin/wc'
+          end.to_h
+        end
+      end
+
+      def test_validation_inherited
+        assert_raises RuntimeError do
+          CallStreaming.new_streaming_call() do |s|
+            s.input = 'input'
+            s.output = 'output'
+            s.mapper = '/bin/cat'
+            s.reducer = '/bin/wc'
+          end.to_hadoop_cli
+        end
+      end
+    end
+  end
+end