rubadoop 0.7.8

data/lib/rubadoop/map_reduce/call_streaming.rb

@@ -0,0 +1,55 @@
+module Rubadoop
+  module MapReduce
+    class CallStreaming < CallJava
+      attr_accessor :mapper, :reducer, :input, :output, :inputformat, :outputformat, :combiner
+
+      class << self
+        def new_streaming_call(params = {}, &block)
+          builder = CallStreaming.new(params)
+          if block_given?
+            if block.arity == 1
+              yield builder
+            else
+              builder.instance_eval &block
+            end
+          end
+          builder
+        end
+      end
+
+      def initialize(params = {})
+        super(params)
+      end
+
+      def to_hadoop_cli(opts = {})
+        cmd = super(opts.merge(skip_args: true))
+        cmd.concat ['-inputformat', "#{@inputformat}"] if @inputformat
+        cmd.concat ['-input', "#{@input}"] if @input
+        cmd.concat ['-outputformat', "#{@outputformat}"] if @outputformat
+        cmd.concat ['-output', "#{@output}"] if @output
+        cmd.concat ['-mapper', "#{@mapper}"] if @mapper
+        cmd.concat ['-reducer', "#{@reducer}"] if @reducer
+        cmd.concat ['-combiner', "#{@combiner}"] if @combiner
+        cmd.concat @args if @args
+        cmd
+      end
+
+      def to_h
+        built = super
+        [:@mapper, :@reducer, :@combiner, :@inputformat, :@input, :@outputformat, :@output].each { |entry|
+          value = instance_variable_get(entry)
+          built[entry.to_s.delete("@").to_sym] = value unless value.nil?
+        }
+        built
+      end
+
+      def validate
+        super
+        [:@mapper, :@reducer, :@input, :@output].each { |property|
+          raise "Missing #{property}" if instance_variable_get(property).nil?
+        }
+      end
+
+    end
+  end
+end

data/lib/rubadoop/map_reduce/identity.rb

@@ -0,0 +1,30 @@
+module Rubadoop
+  module MapReduce
+    module Identity
+
+      class Mapper < MapReduce::Mapper
+        def initialize(config = {})
+          super(config)
+
+          mapper do |line|
+            MapReduce.out.entry(line)
+          end
+        end
+      end
+
+      class Reducer < MapReduce::Reducer
+        def initialize(config = {})
+          super(config)
+
+          reducer do |key, values|
+            values.each { |value|
+              MapReduce.out.entry "#{key}\t#{value}" unless value.nil?
+              MapReduce.out.entry key if value.nil?
+            }
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/rubadoop/map_reduce/io.rb

@@ -0,0 +1,128 @@
+module Rubadoop
+  module MapReduce
+    module Io
+      attr_writer :io_in, :io_out, :io_err
+      attr_writer :out
+
+      def out
+        @out ||= HadoopOut.new
+      end
+      def io_in;  @io_in  || STDIN  end
+      def io_out; @io_out || STDOUT end
+      def io_err; @io_err || STDERR end
+
+      class << self
+        def set_silent_output
+          ::Rubadoop::MapReduce.out = EmptyOut.new
+        end
+        def set_standard_output
+          ::Rubadoop::MapReduce.out = StandardOut.new
+        end
+        def set_rails_output
+          ::Rubadoop::MapReduce.out = RailsOut.new
+        end
+      end
+
+    end
+
+    private
+
+    class CollectCounters
+      def counters
+        @counters ||= Hash[]
+      end
+
+      def counter(group, counter, amount)
+        c_group = counters[group] ||= Hash[]
+        if c_group[counter]
+          c_group[counter] += amount
+        else
+          c_group[counter] = amount
+        end
+      end
+    end
+
+    public
+
+    class HadoopOut
+      def counter(group, counter, amount)
+        MapReduce.io_err.puts "reporter:counter:#{group},#{counter},#{amount}"
+      end
+      def status(status)
+        MapReduce.io_err.puts "reporter:status:#{status}"
+      end
+      def map_entry(key, value)
+        MapReduce.io_out.puts "#{key}\t#{value}"
+      end
+      def entry(value)
+        MapReduce.io_out.puts value
+      end
+      def error(message)
+        MapReduce.io_err.puts message
+      end
+    end
+
+    class StandardOut < CollectCounters
+      def counter(group, counter, amount)
+        super
+        MapReduce.io_out.puts "Counter: #{group}.#{counter} +#{amount}"
+      end
+      def status(status)
+        MapReduce.io_out.puts "reporter:status:#{status}"
+      end
+      def map_entry(key, value)
+        MapReduce.io_out.puts "MapEntry: #{key}\t#{value}"
+      end
+      def entry(value)
+        MapReduce.io_out.puts "Entry: #{value}"
+      end
+      def error(message)
+        MapReduce.io_err.puts message
+      end
+    end
+
+    class RailsOut < CollectCounters
+      def counter(group, counter, amount)
+        super
+        Rails.logger.debug "Counter: #{group}.#{counter} +#{amount}"
+      end
+      def status(status)
+        Rails.logger.info "Status: #{status}"
+      end
+      def map_entry(key, value)
+        Rails.logger.debug "MapEntry: #{key}\t#{value}"
+      end
+      def entry(value)
+        Rails.logger.debug "Entry: #{value}"
+      end
+      def error(message)
+        Rails.logger.error "Error: #{message}"
+      end
+    end
+
+    class EmptyOut < CollectCounters
+      def status(status); end
+      def map_entry(key, value); end
+      def entry(value); end
+      def error(message)
+        MapReduce.io_err.puts message
+      end
+    end
+
+    class TestOut < CollectCounters
+      def entries
+        @entries ||= Array[]
+      end
+      def map_entry(key, value)
+        entries << "#{key}\t#{value}"
+      end
+      def entry(value)
+        entries << value
+      end
+      def error(message)
+        MapReduce.io_err.puts message
+      end
+    end
+
+  end
+end

data/lib/rubadoop/map_reduce/job_conf_environment.rb

@@ -0,0 +1,9 @@
+module Rubadoop
+  module MapReduce
+    module JobConfEnvironment
+      def job_conf_environment(name)
+        ENV[name.gsub(/[^0-9A-Za-z]/, '_')]
+      end
+    end
+  end
+end

data/lib/rubadoop/map_reduce/mappable.rb

@@ -0,0 +1,59 @@
+module Rubadoop
+  module MapReduce
+    module Mappable
+      extend ActiveSupport::Concern
+
+      included do
+        include Rubadoop::MapReduce::Utils
+      end
+
+      # Process Hadoop input as a Mapper. Yields values line-by-line to supplied block.
+      # Supports returning array of full lines if no block supplied for testing only.
+      def mapper
+
+        lines = []
+        MapReduce.io_in.each_line do |line|
+          line.chomp!
+          if @input_ignore_key
+            key, line = key_value_split(line)
+            line = key if line.nil? && !key.nil?
+          end
+          if block_given?
+            yield line
+          else
+            lines << line
+          end
+        end
+        lines unless block_given?
+      end
+
+      def mapper_batched(batch_size = 50)
+        batch = []
+        batches = []
+
+        mapper do |line|
+          batch << line
+          if batch.size >= batch_size
+            if block_given?
+              yield batch
+            else
+              batches << batch
+            end
+            batch = []
+          end
+        end
+
+        if batch.size > 0
+          if block_given?
+            yield batch
+          else
+            batches << batch
+          end
+        end
+
+        batches unless block_given?
+      end
+
+    end
+  end
+end

data/lib/rubadoop/map_reduce/mapper.rb

@@ -0,0 +1,15 @@
+module Rubadoop
+  module MapReduce
+    class Mapper
+      include Mappable
+
+      # * *Params* :
+      #   - +input_ignore_key+ -> Ignore Key Parameter. Required for any Hadoop InputFormat other than TextInputFormat
+      def initialize(config = {})
+        @input_ignore_key = config[:input_ignore_key]
+      end
+
+      alias :process :mapper
+    end
+  end
+end

data/lib/rubadoop/map_reduce/reducable.rb

@@ -0,0 +1,74 @@
+module Rubadoop
+  module MapReduce
+    module Reducable
+      extend ActiveSupport::Concern
+
+      included do
+        include Rubadoop::MapReduce::Utils
+      end
+
+      def reducer(&block)
+        # Use blocks for memory efficiency, the alternative is for testing only
+        # Entries come in per line: key\tvalue1 \n key\tvalue2 \n etc..
+        # This will pivot that to: key [values1, value2] etc..
+
+        lines = []
+        last_key = nil
+        enum = nil
+
+        line = MapReduce.io_in.gets
+        if line.nil?
+          return lines unless block_given?
+          return nil
+        end
+        line.chomp!
+        key, value = key_value_split(line)
+
+        last_key ||= key
+        loop do
+          if enum && !enum.flushed
+            enum.each { |v| }
+            break if (line == nil)
+          end
+
+          enum = TrackingEnumerator.new { |y|
+            y << value
+            loop do
+              line = MapReduce.io_in.gets
+              if line.nil?
+                enum.flushed = true
+                break
+              end
+              line.chomp!
+
+              key, value = key_value_split(line)
+              if key != last_key
+                enum.flushed = true
+                break
+              end
+              y << value
+            end
+            last_key = key
+          }
+
+          enum.flushed = false
+          if block_given?
+            block.call(last_key, enum)
+          else
+            lines << {key: last_key, values: enum.to_a} # must execute enum immediately here
+          end
+          break if (line == nil)
+        end
+
+        lines unless block_given?
+      end
+
+    end
+
+    private
+      class TrackingEnumerator < Enumerator
+        attr_accessor :flushed
+      end
+
+  end
+end

data/lib/rubadoop/map_reduce/reducer.rb

@@ -0,0 +1,12 @@
+module Rubadoop
+  module MapReduce
+    class Reducer
+      include Reducable
+
+      def initialize(config = {})
+      end
+
+      alias :process :reducer
+    end
+  end
+end

data/lib/rubadoop/map_reduce/test_assist.rb

@@ -0,0 +1,65 @@
+module Rubadoop
+  module MapReduce
+    module TestAssist
+
+      def run_test_mapper(input)
+        run_with_test_io prepare_input(input) do
+          yield
+        end
+      end
+
+      def run_test_reducer(input)
+        run_with_test_io prepare_input(input) do
+          yield
+        end
+      end
+
+      private
+
+      def prepare_input(input)
+        case
+          when input.nil?
+            run_input = ''
+          when input.is_a?(Array)
+            in_io = StringIO.new
+            input.each do |item|
+              in_io.puts item
+            end
+            in_io.rewind
+            run_input = in_io
+          when input.is_a?(Hash)
+            in_io = StringIO.new
+            input.each do |key, value|
+              if value.is_a? Array
+                value.each do |entry|
+                  in_io << key << "\t" << entry << "\n"
+                end
+              else
+                in_io << key << "\t" << value << "\n"
+              end
+            end
+            in_io.rewind
+            run_input = in_io
+          else
+            run_input = input
+        end
+      end
+
+      def run_with_test_io input_io
+        mr_out = ::Rubadoop::MapReduce.out
+        io_in = ::Rubadoop::MapReduce.io_in
+        io_out = ::Rubadoop::MapReduce.io_out
+        io_err = ::Rubadoop::MapReduce.io_err
+        ::Rubadoop::MapReduce.io_in = input_io if input_io
+        test_io = TestOut.new
+        ::Rubadoop::MapReduce.out = test_io
+        yield
+        ::Rubadoop::MapReduce.out = mr_out
+        ::Rubadoop::MapReduce.io_in = io_in
+        ::Rubadoop::MapReduce.io_out = io_out
+        ::Rubadoop::MapReduce.io_err = io_err
+        test_io
+      end
+    end
+  end
+end