mrtoolkit 0.1.2

data/test/utest.rb

@@ -0,0 +1,471 @@
+require 'test/unit'
+require 'mrtoolkit'
+require 'regression'
+
+##############################
+# Log example.
+#
+# Reformats the date and time into one field.
+# Reducer adds an extra column
+
+JobBase.testing(true)
+
+class LogMap < MapBase
+  def declare
+    field :date
+    field :time
+    field :url
+  
+    emit :date_time
+    emit :url
+  end
+
+  def process(input, output)
+    output.date_time = input.date + "T" + input.time
+    output.url = input.url
+    output
+  end
+  
+end
+
+class LogReduce < ReduceBase
+  def declare
+    field :date_time
+    field :url
+
+    emit :date_time
+    emit :url
+    emit :junk
+  end
+
+  def process(input, output)
+    output = copy_struct input, output
+    output.junk = "x"
+    output
+  end
+end
+
+class LogJob < JobBase
+  def job
+    mapper LogMap
+    reducer LogReduce
+    infiles "test-in/test1-in"
+    outfiles "test-out"
+  end
+end
+
+class TestMRToolkit < Test::Unit::TestCase
+  def test_log
+    LogJob.run_command
+    out = File.read("test-out")
+    expected = "2008-10-01T10:30:00\t1.2.3.4\tx\n" + 
+               "2008-10-02T11:30:00\t1.2.3.5\tx\n"
+    assert_equal(expected, out)
+  end
+end
+
+
+##########################################
+#
+# Computs count, total, and sum of squares.
+
+class SumMap < MapBase
+  def declare
+    field :value
+  
+    emit :count
+    emit :total
+    emit :sum_of_squares
+  end
+
+  def process(input, output)
+    v = input.value.to_f
+    output.count = 1
+    output.total = v
+    output.sum_of_squares = v * v
+    output
+  end
+  
+end
+
+# This could be done with canned reducer
+class MySumReduce < ReduceBase
+  def declare
+    field :count
+    field :total
+    field :sum_of_squares
+
+    emit :count
+    emit :total
+    emit :sum_of_squares
+  end
+
+  def process_begin(dummy, output)
+    @count = 0
+    @total = 0
+    @sum_of_squares = 0
+    nil
+  end
+  def process(input, output)
+    @count += input.count.to_f
+    @total += input.total.to_f
+    @sum_of_squares += input.sum_of_squares.to_f
+    nil
+  end
+  def process_end(dummy, output)
+    output.count = @count
+    output.total = @total
+    output.sum_of_squares = @sum_of_squares
+    output
+  end
+end
+
+class SumJob < JobBase
+  def job
+    mapper SumMap
+    reducer MySumReduce
+    infiles "test-in/test2-in"
+    outfiles "test-out"
+  end
+end
+
+class TestMRToolkit < Test::Unit::TestCase
+  def test_sum
+    SumJob.run_command
+    out = File.read("test-out")
+    expected = "4.0\t43.0\t1005.0\n"
+    assert_equal(expected, out)
+  end
+end
+
+
+######################################
+#
+# Grops times into one-minute buckets
+# Calculates counts for each bucket
+
+require 'parsedate'
+
+class MinMap < MapBase
+  def declare
+    field :dt
+    field :tm
+  
+    emit :minute
+    emit :count
+  end
+
+  def process(input, output)
+    res = ParseDate.parsedate(input.dt + " " + input.tm)
+    t = Time.local(*res)
+    min = t.min + 60 * (t.hour + 24 * t.wday)
+    output.count = 1
+    output.minute = min
+    output
+  end
+  
+end
+
+class MyMinReduce < ReduceBase
+  def declare
+    field :minute
+    field :count
+
+    emit :min
+    emit :count
+  end
+
+  def process_init(input, output)
+    @count = 0
+    nil
+  end
+  def process_each(input, output)
+    @count += 1
+    nil
+  end
+  def process_term(input, output)
+    output.min = @last
+    output.count = @count
+    output
+  end
+end
+
+class MyMinJob < JobBase
+  def job
+    mapper MinMap
+    reducer MyMinReduce
+    infiles "test-in/test3-in"
+    outfiles "test-out"
+  end
+end
+
+class TestMRToolkit < Test::Unit::TestCase
+  def test_min
+    MyMinJob.run_command
+    out = File.read("test-out")
+    expected = "8460\t1\n" +
+               "8461\t1\n" +
+               "8470\t3\n"
+    assert_equal(expected, out)
+  end
+end
+
+#################################
+# 
+# This is the previous one, but with a standard reducer.
+
+class CollectJob < JobBase
+  def job
+    mapper MinMap
+    reducer CopyReduce, 1
+    infiles "test-in/test3-in"
+    outfiles "test-out"
+  end
+end
+
+class TestMRToolkit < Test::Unit::TestCase
+  def test_collect
+    CollectJob.run_command
+    out = File.read("test-out")
+    expected = "8460\n" +
+               "8461\n" +
+               "8470\n" +
+               "8470\n" +
+               "8470\n"
+    assert_equal(expected, out)
+  end
+end
+
+#################################
+# 
+# This is the previous one, but with adifferent
+# standard reducer.  This produces the same output
+# as the custom reducer.
+
+class UniqueJob < JobBase
+  def job
+    mapper MinMap
+    reducer UniqueReduce
+    infiles "test-in/test3-in"
+    outfiles "test-out"
+  end
+end
+
+class TestMRToolkit < Test::Unit::TestCase
+  def test_unique
+    UniqueJob.run_command
+    out = File.read("test-out")
+    expected = "8460\n" +
+               "8461\n" +
+               "8470\n"
+    assert_equal(expected, out)
+  end
+end
+
+###############################
+#
+# Exercises SumReduce, which sums a variable
+# set of columns.
+
+
+class GSumJob < JobBase
+  def job
+    mapper CopyMap, 3
+    reducer SumReduce, 3
+    infiles "test-in/test6-in"
+    outfiles "test-out"
+  end
+end
+
+class TestMRToolkit < Test::Unit::TestCase
+  def test_gsum
+    GSumJob.run_command
+    out = File.read("test-out")
+    expected = "12.0\t9.0\t8.0\n"
+    assert_equal(expected, out)
+  end
+end
+
+class SelectJob < JobBase
+  def job
+    mapper SelectMap, /^10[23]/
+    reducer CopyReduce
+    infiles "test-in/test5-in"
+    outfiles "test-out"
+  end
+end
+
+class TestMRToolkit < Test::Unit::TestCase
+  def test_select
+    SelectJob.run_command
+    out = File.read("test-out")
+    expected = "102\n102\n102\n102\n103\n"
+    assert_equal(expected, out)
+  end
+end
+
+class SampleJob < JobBase
+  def job
+    mapper CopyMap, 3
+    reducer SampleReduce, 10
+    infiles "test-in/test7-in"
+    outfiles "test-out"
+  end
+end
+
+class TestMRToolkit < Test::Unit::TestCase
+  def test_sample
+    srand 1234
+    SampleJob.run_command
+    out = File.read("test-out")
+    expected = "5\n20\n7\n12\n2\n8\n3\n16\n17\n18\n"
+    assert_equal(expected, out)
+  end
+end
+
+class MaxJob < JobBase
+  def job
+    mapper CopyMap, 3
+    reducer MaxReduce, 3
+    infiles "test-in/test4-in"
+    outfiles "test-out"
+  end
+end
+
+class TestMRToolkit < Test::Unit::TestCase
+  def test_max
+    MaxJob.run_command
+    out = File.read("test-out")
+    expected = "4\t10\n3\t3\n2\t2\n"
+    assert_equal(expected, out)
+  end
+end
+
+class MinJob < JobBase
+  def job
+    mapper CopyMap, 3
+    reducer MinReduce, 3
+    infiles "test-in/test4-in"
+    outfiles "test-out"
+  end
+end
+
+class TestMRToolkit < Test::Unit::TestCase
+  def test_min
+    MinJob.run_command
+    out = File.read("test-out")
+    expected = "3\t3\n2\t2\n1\t1\n"
+    assert_equal(expected, out)
+  end
+end
+
+class UniqueSumJob < JobBase
+  def job
+    mapper CopyMap, 2
+    reducer UniqueSumReduce
+    infiles "test-in/test5-in"
+    outfiles "test-out"
+  end
+end
+
+class TestMRToolkit < Test::Unit::TestCase
+  def test_unique_sum
+    UniqueSumJob.run_command
+    out = File.read("test-out")
+    expected = "100\t3\n101\t2\n102\t4\n103\t1\n104\t2\n"
+    assert_equal(expected, out)
+  end
+end
+
+class UniqueCountJob < JobBase
+  def job
+    mapper CopyMap
+    reducer UniqueCountReduce
+    infiles "test-in/test5-in"
+    outfiles "test-out"
+  end
+end
+
+class TestMRToolkit < Test::Unit::TestCase
+  def test_unique_count
+    UniqueCountJob.run_command
+    out = File.read("test-out")
+    expected = "100\t3\n101\t2\n102\t4\n103\t1\n104\t2\n"
+    assert_equal(expected, out)
+  end
+end
+
+class MaxUniqueSumJob < JobBase
+  def job
+    mapper CopyMap, 3
+    reducer MaxUniqueSumReduce, 3
+    infiles "test-in/test5-in"
+    outfiles "test-out"
+  end
+end
+
+class TestMRToolkit < Test::Unit::TestCase
+  def test_max_unique_sum
+    MaxUniqueSumJob.run_command
+    out = File.read("test-out")
+    expected = "102\t4\n100\t3\n101\t2\n"
+    assert_equal(expected, out)
+  end
+end
+
+class UniqueIndexedSumJob < JobBase
+  def job
+    mapper CopyMap, 3
+    reducer UniqueIndexedSumReduce, 3
+    infiles "test-in/test8-in"
+    outfiles "test-out"
+  end
+end
+
+class TestMRToolkit < Test::Unit::TestCase
+  def test_unique_indexed_sum
+    UniqueIndexedSumJob.run_command
+    out = File.read("test-out")
+    expected = "100\t1000\t3\n100\t1001\t1\n200\t1000\t2\n200\t1001\t1\n"
+    assert_equal(expected, out)
+  end
+end
+
+class UniqueFirstJob < JobBase
+  def job
+    mapper CopyMap, 4
+    reducer UniqueFirstReduce, 3, 1
+    infiles "test-in/test9-in"
+    outfiles "test-out"
+  end
+end
+
+class TestMRToolkit < Test::Unit::TestCase
+  def test_unique_first
+    UniqueFirstJob.run_command
+    out = File.read("test-out")
+    expected = "a\ta\ta\nx1\ty1\tz1\n"
+    assert_equal(expected, out)
+  end
+end
+
+
+class TestRegression < Test::Unit::TestCase
+  def test_regress
+    x = [1, 2, 3]
+    y = [1, 2, 3]
+    reg = LinearRegression.new(x, y)
+    assert_equal([1, 2, 3], reg.fit(x))
+    x = [1, 2, 3, 4]
+    y = [1, 5, 5, 9]
+    reg = LinearRegression.new(x, y)
+    assert_equal(2, reg.slope)
+    assert_equal(0, reg.offset)
+    y = [1, 5, 5, 9]
+    reg = LinearRegression.new(x, y)
+    assert_equal(2, reg.slope)
+    assert_equal(0, reg.offset)
+  end
+end
+
+

metadata

@@ -0,0 +1,104 @@
+--- !ruby/object:Gem::Specification 
+name: mrtoolkit
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 2
+  version: 0.1.2
+platform: ruby
+authors: 
+- cchayden
+- vadimj
+- jashmenn
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-05-17 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: 
+email: nate@natemurray.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- .document
+- .gitignore
+- Makefile
+- README.rdoc
+- Rakefile
+- VERSION.yml
+- examples/Rakefile
+- examples/Readme
+- examples/hour.rb
+- examples/import-logs
+- examples/import.rb
+- examples/ip-result.rb
+- examples/ip-size.rb
+- examples/ip-ua.rb
+- examples/ip.rb
+- examples/section.rb
+- examples/top-file.rb
+- lib/mrtoolkit.rb
+- lib/regression.rb
+- lib/stream_runner.rb
+- mrtoolkit.gemspec
+- standalone/hadoop
+- test/Rakefile
+- test/test-in/test1-in
+- test/test-in/test2-in
+- test/test-in/test3-in
+- test/test-in/test4-in
+- test/test-in/test5-in
+- test/test-in/test6-in
+- test/test-in/test7-in
+- test/test-in/test8-in
+- test/test-in/test9-in
+- test/utest.rb
+has_rdoc: true
+homepage: http://github.com/jashmenn/mrtoolkit
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: Simplify the creation of Hadoop Map/Reduce jobs
+test_files: 
+- test/utest.rb
+- examples/hour.rb
+- examples/import.rb
+- examples/ip-result.rb
+- examples/ip-size.rb
+- examples/ip-ua.rb
+- examples/ip.rb
+- examples/section.rb
+- examples/top-file.rb