rubadoop 0.7.8

data/test/rubadoop/map_reduce/identity_test.rb

@@ -0,0 +1,40 @@
+require 'test_helper'
+
+module Rubadoop
+  module MapReduce
+
+    class IdentityTest < MiniTest::Spec
+      def test_mapper
+        MapReduce.io_in = StringIO.new("abcd\nefgh")
+        output = StringIO.new
+        MapReduce.io_out = output
+        Identity::Mapper.new
+        assert_equal "abcd\nefgh\n", output.string
+      end
+
+      def test_mapper_ignorekey
+        MapReduce.io_in = StringIO.new("1\tabcd\n2\tefgh")
+        output = StringIO.new
+        MapReduce.io_out = output
+        Identity::Mapper.new(input_ignore_key: true)
+        assert_equal "abcd\nefgh\n", output.string
+      end
+
+      def test_reducer
+        MapReduce.io_in = StringIO.new("k1\tv1\nk2\tv2\n")
+        output = StringIO.new
+        MapReduce.io_out = output
+        Identity::Reducer.new
+        assert_equal "k1\tv1\nk2\tv2\n", output.string
+      end
+
+      def test_reducer_dummy
+        MapReduce.io_in = StringIO.new("abcd\nefgh")
+        output = StringIO.new
+        MapReduce.io_out = output
+        Identity::Reducer.new
+        assert_equal "abcd\nefgh\n", output.string
+      end
+    end
+  end
+end

data/test/rubadoop/map_reduce/io_test.rb

@@ -0,0 +1,51 @@
+require 'test_helper'
+
+module Rubadoop
+  module MapReduce
+    class IoTest < MiniTest::Spec
+      include ::Rubadoop::MapReduce::Utils
+
+      def test_silent_output
+        ::Rubadoop::MapReduce::Io.set_silent_output
+
+        out_entry('silent')
+      end
+
+      def test_standard_output
+        ::Rubadoop::MapReduce::Io.set_standard_output
+
+        out_entry('standard')
+      end
+
+      def test_test_out
+        ::Rubadoop::MapReduce.out = TestOut.new
+
+        out_entry('poop')
+        log_counter 'Block', 'Rock', 1
+        log_counter 'Block', 'Stock', 0
+        log_counter 'Block', 'Rock', 5
+        out_map_entry('k1', 'v1')
+
+        assert_equal 6, ::Rubadoop::MapReduce.out.counters['Block']['Rock']
+        assert_equal 0, ::Rubadoop::MapReduce.out.counters['Block']['Stock']
+        assert_equal ['poop', "k1\tv1"], ::Rubadoop::MapReduce.out.entries
+      end
+
+      def test_counter_collection
+        [StandardOut, EmptyOut].each do |output_class|
+          ::Rubadoop::MapReduce.out = output_class.new
+
+          out_entry('poop')
+          log_counter 'Block', 'Rock', 1
+          log_counter 'Block', 'Stock', 0
+          log_counter 'Block', 'Rock', 5
+          out_map_entry('k1', 'v1')
+
+          assert_equal 6, ::Rubadoop::MapReduce.out.counters['Block']['Rock']
+          assert_equal 0, ::Rubadoop::MapReduce.out.counters['Block']['Stock']
+        end
+      end
+
+    end
+  end
+end

data/test/rubadoop/map_reduce/job_conf_environment_test.rb

@@ -0,0 +1,28 @@
+require 'test_helper'
+
+module Rubadoop
+  module MapReduce
+    class JobConfEnvironmentTest < MiniTest::Spec
+      include ::Rubadoop::MapReduce::JobConfEnvironment
+
+      def test_simple
+
+        ENV['map_input_file'] = 'goop'
+
+        assert_equal 'goop', job_conf_environment('map.input.file')
+
+        ENV.delete 'map_input_file'
+      end
+
+      def test_missing
+
+        ENV['map.input.file'] = 'poop'
+
+        assert_equal nil, job_conf_environment('map.input.file')
+
+        ENV.delete 'map.input.file'
+      end
+
+    end
+  end
+end

data/test/rubadoop/map_reduce/mappable_test.rb

@@ -0,0 +1,62 @@
+require 'test_helper'
+
+module Rubadoop
+  module MapReduce
+    class MappableTest < MiniTest::Spec
+      include TestAssist
+      include Mappable
+
+      def test_mapper
+        input = (1..500).to_a
+        result = run_test_mapper(input) do
+          mapper do |line|
+            out_entry line if line.to_i % 100 == 0
+          end
+        end
+        assert_equal 5, result.entries.size
+        assert_equal "100", result.entries[0]
+        assert_equal "200", result.entries[1]
+        assert_equal "300", result.entries[2]
+        assert_equal "400", result.entries[3]
+        assert_equal "500", result.entries[4]
+      end
+
+      def test_mapper_batched
+        input = (1..501).to_a
+        result = run_test_mapper(input) do
+          mapper_batched(100) do |batch|
+            out_entry batch
+          end
+        end
+        assert_equal 6, result.entries.size
+        assert_equal (1..100).to_a.map(&:to_s), result.entries[0]
+        assert_equal (101..200).to_a.map(&:to_s), result.entries[1]
+        assert_equal (201..300).to_a.map(&:to_s), result.entries[2]
+        assert_equal (301..400).to_a.map(&:to_s), result.entries[3]
+        assert_equal (401..500).to_a.map(&:to_s), result.entries[4]
+        assert_equal (501..501).to_a.map(&:to_s), result.entries[5]
+      end
+
+      def test_mapper_batched_edge
+        input = (1..10).to_a
+        result = run_test_mapper(input) do
+          mapper_batched(10) do |batch|
+            out_entry batch
+          end
+        end
+        assert_equal 1, result.entries.size
+        assert_equal (1..10).to_a.map(&:to_s), result.entries[0]
+      end
+
+      def test_mapper_batched_empty
+        result = run_test_mapper(nil) do
+          mapper_batched(10) do |batch|
+            out_entry batch
+          end
+        end
+        assert_equal 0, result.entries.size
+      end
+    end
+  end
+end
+

data/test/rubadoop/map_reduce/mapper_test.rb

@@ -0,0 +1,76 @@
+require 'test_helper'
+
+module Rubadoop
+  module MapReduce
+
+    class TestMapper < MapReduce::Mapper
+    end
+
+    class MapperTest < MiniTest::Spec
+
+      def test_split_block
+        mapper = TestMapper.new
+
+        MapReduce.io_in = StringIO.new("abcd\nefgh")
+        l = 0
+        mapper.process { |line|
+          case l
+            when 0; assert_equal "abcd", line
+            when 1; assert_equal "efgh", line
+            else; fail "unexpected line: #{l}=#{key}"
+          end
+          l += 1
+        }
+        assert_equal 2, l
+      end
+
+      def test_split_various
+        mapper = TestMapper.new
+
+        MapReduce.io_in = StringIO.new("abcd\nefgh")
+        assert_equal ["abcd", "efgh"], mapper.process
+
+        MapReduce.io_in = StringIO.new("abcd\n\n\n")
+        assert_equal ["abcd", "", ""], mapper.process
+
+        MapReduce.io_in = StringIO.new("0\n1")
+        assert_equal ["0", "1"], mapper.process
+
+        MapReduce.io_in = StringIO.new("0\n1\n")
+        assert_equal ["0", "1"], mapper.process
+
+        MapReduce.io_in = StringIO.new("0\n1\n ")
+        assert_equal ["0", "1", " "], mapper.process
+
+        MapReduce.io_in = StringIO.new("0\t1\n")
+        assert_equal ["0\t1"], mapper.process
+
+        MapReduce.io_in = StringIO.new("")
+        assert_equal [], mapper.process
+      end
+
+      def test_mapper_ignorekey
+        mapper = TestMapper.new(input_ignore_key: true)
+
+        MapReduce.io_in = StringIO.new("0\t1\n2\t3")
+        assert_equal ["1", "3"], mapper.process
+
+        MapReduce.io_in = StringIO.new("0\n1\n2\n3")
+        assert_equal ["0", "1", "2", "3"], mapper.process
+
+        MapReduce.io_in = StringIO.new("\t1\n2\t3\n")
+        assert_equal ["1", "3"], mapper.process
+      end
+
+      def test_mapper_function_name
+        mapper = TestMapper.new()
+
+        MapReduce.io_in = StringIO.new("0\n1\n2")
+        assert_equal ["0", "1", "2"], mapper.mapper
+
+        MapReduce.io_in = StringIO.new("0\n1\n2")
+        assert_equal ["0", "1", "2"], mapper.process
+      end
+    end
+  end
+end

data/test/rubadoop/map_reduce/reducable_test.rb

@@ -0,0 +1,12 @@
+require 'test_helper'
+
+module Rubadoop
+  module MapReduce
+    class ReducableTest < MiniTest::Spec
+
+      #TODO
+
+    end
+  end
+end
+

data/test/rubadoop/map_reduce/reducer_test.rb

@@ -0,0 +1,137 @@
+require 'test_helper'
+
+module Rubadoop
+
+  class TestReducer < MapReduce::Reducer
+  end
+
+  class ReducerTest < MiniTest::Spec
+
+    def test_splitting
+      reducer = TestReducer.new
+
+      MapReduce.io_in = StringIO.new("1\t1\n1\t2\n2\t2")
+      l = 0
+      reducer.process { |key, values|
+        case l
+          when 0; assert_equal "1", key; assert_equal ["1", "2"], values.to_a
+          when 1; assert_equal "2", key; assert_equal ["2"], values.to_a
+          else; fail "unexpected line: #{l}=#{key}"
+        end
+        l += 1
+      }
+      assert_equal 2, l
+
+      MapReduce.io_in = StringIO.new("1\t1\n1\t2\n2\t2")
+      r = reducer.process
+      assert_equal 2, r.size
+      assert_equal "1", r[0][:key]; assert_equal ["1", "2"], r[0][:values]
+      assert_equal "2", r[1][:key]; assert_equal ["2"], r[1][:values]
+
+      MapReduce.io_in = StringIO.new("1\t1")
+      r = reducer.process
+      assert_equal 1, r.size
+      assert_equal "1", r[0][:key]; assert_equal ["1"], r[0][:values]
+
+      MapReduce.io_in = StringIO.new("")
+      r = reducer.process
+      assert_equal 0, r.size
+
+      MapReduce.io_in = StringIO.new("abcd\nefgh")
+      r = reducer.process
+      assert_equal 2, r.size
+      assert_equal "abcd", r[0][:key]; assert_equal [nil], r[0][:values]
+      assert_equal "efgh", r[1][:key]; assert_equal [nil], r[1][:values]
+
+      MapReduce.io_in = StringIO.new("1\t1\n1\t2\n2\t2")
+      r = reducer.process
+      assert_equal 2, r.size
+      assert_equal "1", r[0][:key]; assert_equal ["1", "2"], r[0][:values]
+      assert_equal "2", r[1][:key]; assert_equal ["2"], r[1][:values]
+
+      MapReduce.io_in = StringIO.new("1\t1\n1\t2\n2\t2\n3\t3\n3\t1\n3\t9\n4\t1")
+      r = reducer.process
+      assert_equal 4, r.size
+      assert_equal "1", r[0][:key]; assert_equal ["1", "2"], r[0][:values]
+      assert_equal "2", r[1][:key]; assert_equal ["2"], r[1][:values]
+      assert_equal "3", r[2][:key]; assert_equal ["3", "1", "9"], r[2][:values]
+      assert_equal "4", r[3][:key]; assert_equal ["1"], r[3][:values]
+
+    end
+
+    def test_skipping
+      reducer = TestReducer.new
+
+      MapReduce.io_in = StringIO.new("1\t1\n1\t2\n2\t2\n3\t3\n3\t1\n3\t9\n4\t1")
+      l = 0
+      reducer.process { |key, values|
+        case l
+          when 0; assert_equal "1", key; assert_equal ["1", "2"], values.to_a
+          when 1; assert_equal "2", key; assert_equal ["2"], values.to_a
+          when 2; assert_equal "3", key; #assert_equal ["3", "1", "9"], values.to_a
+          when 3; assert_equal "4", key; assert_equal ["1"], values.to_a
+          else; fail "unexpected line: #{l}=#{key}"
+        end
+        l += 1
+      }
+      assert_equal 4, l
+
+      MapReduce.io_in = StringIO.new("1\t1\n1\t2\n2\t2\n3\t3\n3\t1\n3\t9\n4\t1")
+      l = 0
+      reducer.process { |key, values|
+        case l
+          when 0; assert_equal "1", key; assert_equal ["1", "2"], values.to_a
+          when 1; assert_equal "2", key; assert_equal ["2"], values.to_a
+          when 2; assert_equal "3", key; assert_equal "3", values.next()
+          when 3; assert_equal "4", key; assert_equal ["1"], values.to_a
+          else; fail "unexpected line: #{l}=#{key}"
+        end
+        l += 1
+      }
+      assert_equal 4, l
+
+      MapReduce.io_in = StringIO.new("1\t1\n1\t2\n2\t2\n3\t3\n3\t1\n3\t9\n4\t1")
+      l = 0
+      reducer.process { |key, values|
+        case l
+          when 0; assert_equal "1", key;
+          when 1; assert_equal "2", key; assert_equal ["2"], values.to_a
+          when 2; assert_equal "3", key;
+          when 3; assert_equal "4", key; assert_equal ["1"], values.to_a
+          else; fail "unexpected line: #{l}=#{key}"
+        end
+        l += 1
+      }
+      assert_equal 4, l
+
+      MapReduce.io_in = StringIO.new("1\t1\n1\t2\n2\t2\n3\t3\n3\t1\n3\t9\n4\t1")
+      l = 0
+      reducer.process { |key, values|
+        case l
+          when 0; assert_equal "1", key; assert_equal ["1", "2"], values.to_a
+          when 1; assert_equal "2", key; assert_equal ["2"], values.to_a
+          when 2; assert_equal "3", key; #assert_equal ["3", "1", "9"], values.to_a
+          when 3; assert_equal "4", key; #assert_equal ["1"], values.to_a
+          else; fail "unexpected line: #{l}=#{key}"
+        end
+        l += 1
+      }
+      assert_equal 4, l
+
+      MapReduce.io_in = StringIO.new("1\t1\n1\t2\n2\t2\n3\t3\n3\t1\n3\t9\n4\t1")
+      l = 0
+      reducer.process { |key, values|
+        case l
+          when 0; assert_equal "1", key; #assert_equal ["1", "2"], values.to_a
+          when 1; assert_equal "2", key; #assert_equal ["2"], values.to_a
+          when 2; assert_equal "3", key; #assert_equal ["3", "1", "9"], values.to_a
+          when 3; assert_equal "4", key; #assert_equal ["1"], values.to_a
+          else; fail "unexpected line: #{l}=#{key}"
+        end
+        l += 1
+      }
+      assert_equal 4, l
+    end
+  end
+end
+

data/test/rubadoop/map_reduce/test_assist_test.rb

@@ -0,0 +1,76 @@
+require 'test_helper'
+
+module Rubadoop
+  module MapReduce
+    class TestAssistTest < MiniTest::Spec
+      include ::Rubadoop::MapReduce::TestAssist
+
+      def test_wordcount_mapper
+        input = ['The quick brown fox', 'jumped over', 'the lazy dog']
+        result = run_test_mapper(input) do
+          SimpleWordCount.new.execute
+        end
+        assert_equal 8, result.entries.size
+        assert_equal "the\t2", result.entries[0]
+        assert_equal "dog\t1", result.entries[7]
+      end
+
+      def test_summing_reducer
+        input = Hash['the' => [3, 4, 5], 'fox' => [5]]
+        result = run_test_reducer(input) do
+          SummingReducer.new.execute
+        end
+        assert_equal 2, result.entries.size
+        assert_equal "the\t12", result.entries[0]
+        assert_equal "fox\t5", result.entries[1]
+      end
+
+      def test_no_input
+        result = run_test_mapper(nil) do
+          ::Rubadoop::MapReduce.out.entry('cookoo')
+        end
+        assert_equal 1, result.entries.size
+        assert_equal "cookoo", result.entries[0]
+      end
+
+    end
+
+    class SimpleWordCount
+      include ::Rubadoop::MapReduce::Utils
+      include ::Rubadoop::MapReduce::Mappable
+
+      def execute
+        words = Hash[]
+
+        mapper do |line|
+          line.split(/\s+/).each do |word|
+            word.downcase!
+            if words[word]
+              words[word] += 1
+            else
+              words[word] = 1
+            end
+          end
+        end
+
+        words.each do |word, count|
+          out_map_entry(word, count)
+        end
+      end
+    end
+
+    class SummingReducer
+      include ::Rubadoop::MapReduce::Utils
+      include ::Rubadoop::MapReduce::Reducable
+
+      def execute
+        reducer do |key, counts|
+          sum = 0
+          counts.each { |c| sum += c.to_i }
+          out_map_entry(key, sum)
+        end
+      end
+    end
+
+  end
+end