cascading.jruby 0.0.4

data/test/test_assembly.rb

@@ -0,0 +1,321 @@
+require 'test/unit'
+require 'cascading'
+
+def compare_with_references(test_name)
+  result = compare_files("test/references/#{test_name}.txt", "output/#{test_name}/part-00000")
+  assert_nil(result)
+end
+
+# Convenience for basic assembly tests; not valid for applications
+def assembly(name, &block)
+  assembly = Assembly.new(name, nil)
+  assembly.instance_eval(&block)
+  assembly
+end
+
+class TC_Assembly < Test::Unit::TestCase
+  include Operations
+
+  def mock_assembly(&block)
+    flow 'test' do
+      source 'test', tap('test/data/data1.txt')
+      $assembly = assembly 'test', &block
+    end
+    $assembly
+  end
+
+  def test_create_assembly_simple
+    assembly = assembly "assembly1" do
+      # Empty assembly
+    end
+
+    assert_not_nil assembly
+    pipe = assembly.tail_pipe
+    assert pipe.is_a? Java::CascadingPipe::Pipe
+  end
+
+  def test_each_identity
+    assembly = mock_assembly do
+      each 'offset', :filter => identity
+    end
+
+    flow = Flow.get('test')
+    assert_not_nil flow
+
+    assert_not_nil flow.find_child('test')
+    assert_equal assembly, flow.find_child('test')
+    assert_not_nil Flow.get('test').find_child('test')
+  end
+
+  def test_create_each
+    # You can't apply an Each to 0 fields
+    assert_raise CascadingException do
+      assembly = mock_assembly do
+        each(:filter => identity)
+      end
+      assert assembly.tail_pipe.is_a? Java::CascadingPipe::Each
+    end
+
+    assembly = mock_assembly do
+      each('offset', :output => 'offset_copy',
+           :filter => Java::CascadingOperation::Identity.new(fields('offset_copy')))
+    end
+    pipe = assembly.tail_pipe
+
+    assert pipe.is_a? Java::CascadingPipe::Each
+
+    assert_equal 'offset', pipe.getArgumentSelector().get(0)
+    assert_equal 'offset_copy', pipe.getOutputSelector().get(0)
+  end
+
+  # For now, replaced these tests with the trivial observation that you can't
+  # follow a Tap with an Every.  Eventually, should support testing within a
+  # group_by block.
+  def test_create_every
+    assert_raise CascadingException do
+      assembly = mock_assembly do
+        every(:aggregator => count_function)
+      end
+      pipe = assembly.tail_pipe
+      assert pipe.is_a? Java::CascadingPipe::Every
+    end
+
+    assert_raise CascadingException do
+      assembly = mock_assembly do
+        every(:aggregator => count_function("field1", "field2"))
+      end
+      assert assembly.tail_pipe.is_a? Java::CascadingPipe::Every
+    end
+
+    assert_raise CascadingException do
+      assembly = mock_assembly do
+        every("Field1", :aggregator => count_function)
+      end
+      assert assembly.tail_pipe.is_a? Java::CascadingPipe::Every
+      assert_equal "Field1", assembly.tail_pipe.getArgumentSelector().get(0)
+    end
+
+    assert_raise CascadingException do
+      assembly = mock_assembly do
+        every('line', :aggregator => count_function, :output=>'line_count')
+      end
+      assert assembly.tail_pipe.is_a? Java::CascadingPipe::Every
+      assert_equal 'line', assembly.tail_pipe.getArgumentSelector().get(0)
+      assert_equal 'line_count', assembly.tail_pipe.getOutputSelector().get(0)
+    end
+  end
+
+  def test_create_group_by
+    assembly = mock_assembly do
+      group_by('line')
+    end
+
+    assert assembly.tail_pipe.is_a? Java::CascadingPipe::GroupBy
+    grouping_fields = assembly.tail_pipe.getGroupingSelectors()['test']
+    assert_equal 'line', grouping_fields.get(0)
+
+    assembly = mock_assembly do
+      group_by('line')
+    end
+
+    assert assembly.tail_pipe.is_a? Java::CascadingPipe::GroupBy
+    grouping_fields = assembly.tail_pipe.getGroupingSelectors()['test']
+    assert_equal 'line', grouping_fields.get(0)
+  end
+
+  def test_create_group_by_many_fields
+    assembly = mock_assembly do
+      group_by(['offset', 'line'])
+    end
+
+    assert assembly.tail_pipe.is_a? Java::CascadingPipe::GroupBy
+    grouping_fields = assembly.tail_pipe.getGroupingSelectors()['test']
+    assert_equal 'offset', grouping_fields.get(0)
+    assert_equal 'line', grouping_fields.get(1)
+  end
+
+  def test_create_group_by_with_sort
+    assembly = mock_assembly do
+      group_by('offset', 'line', :sort_by => ['line'])
+    end
+
+    assert assembly.tail_pipe.is_a? Java::CascadingPipe::GroupBy
+    grouping_fields = assembly.tail_pipe.getGroupingSelectors()['test']
+    sorting_fields = assembly.tail_pipe.getSortingSelectors()['test']
+
+    assert_equal 2, grouping_fields.size
+    assert_equal 1, sorting_fields.size
+
+    assert_equal 'offset', grouping_fields.get(0)
+    assert_equal 'line', grouping_fields.get(1)
+    assert assembly.tail_pipe.isSorted()
+    assert !assembly.tail_pipe.isSortReversed()
+    assert_equal 'line', sorting_fields.get(0)
+  end
+
+  def test_create_group_by_with_sort_reverse
+    assembly = mock_assembly do
+      group_by('offset', 'line', :sort_by => ['line'], :reverse => true)
+    end
+
+    assert assembly.tail_pipe.is_a? Java::CascadingPipe::GroupBy
+    grouping_fields = assembly.tail_pipe.getGroupingSelectors()['test']
+    sorting_fields = assembly.tail_pipe.getSortingSelectors()['test']
+
+    assert_equal 2, grouping_fields.size
+    assert_equal 1, sorting_fields.size
+
+    assert_equal 'offset', grouping_fields.get(0)
+    assert_equal 'line', grouping_fields.get(1)
+    assert assembly.tail_pipe.isSorted()
+    assert assembly.tail_pipe.isSortReversed()
+    assert_equal 'line', sorting_fields.get(0)
+  end
+
+  def test_create_group_by_reverse
+    assembly = mock_assembly do
+      group_by('offset', 'line', :reverse => true)
+    end
+
+    assert assembly.tail_pipe.is_a? Java::CascadingPipe::GroupBy
+    grouping_fields = assembly.tail_pipe.getGroupingSelectors()['test']
+    sorting_fields = assembly.tail_pipe.getSortingSelectors()['test']
+
+    assert_equal 2, grouping_fields.size
+    assert_equal 2, sorting_fields.size
+
+    assert_equal 'offset', grouping_fields.get(0)
+    assert_equal 'line', grouping_fields.get(1)
+    assert assembly.tail_pipe.isSorted()
+    assert assembly.tail_pipe.isSortReversed()
+    assert_equal 'offset', sorting_fields.get(0)
+    assert_equal 'line', sorting_fields.get(1)
+  end
+
+  def test_branch_unique
+    assembly = mock_assembly do
+      branch 'branch1' do
+      end
+    end
+
+    assert_equal 1, assembly.children.size
+
+  end
+
+  def test_branch_empty
+    assembly = mock_assembly do
+      branch 'branch1' do
+      end
+
+      branch 'branch2' do
+        branch 'branch3' do
+        end
+      end
+    end
+
+    assert_equal 2, assembly.children.size
+    assert_equal 0, assembly.children['branch1'].children.size
+    assert_equal 1, assembly.children['branch2'].children.size
+  end
+
+  def test_branch_single
+    assembly = mock_assembly do
+      branch 'branch1' do
+        branch 'branch2' do
+          each 'line', :function => identity
+        end
+      end
+    end
+
+    assert_equal 1, assembly.children.size
+    assert_equal 1, assembly.children['branch1'].children.size
+    assert_equal 0, assembly.children['branch1'].children['branch2'].children.size
+  end
+
+  # Fixed this test, but it isn't even valid.  You shouldn't be able to follow
+  # an Each with an Every.
+  def test_full_assembly
+    assert_raise CascadingException do
+      assembly = mock_assembly do
+        each('offset', :output => 'offset_copy',
+             :filter => Java::CascadingOperation::Identity.new(fields('offset_copy')))
+        every(:aggregator => count_function)
+      end
+
+      pipe = assembly.tail_pipe
+
+      assert pipe.is_a? Java::CascadingPipe::Every
+    end
+  end
+
+end
+
+
+class TC_AssemblyScenarii < Test::Unit::TestCase
+
+  def test_splitter
+    flow = flow "splitter" do
+      source "copy", tap("test/data/data1.txt")
+      sink "copy", tap('output/splitter', :sink_mode => :replace)
+
+      assembly "copy" do
+        split "line", :pattern => /[.,]*\s+/, :into=>["name", "score1", "score2", "id"], :output => ["name", "score1", "score2", "id"]
+        assert_size_equals 4
+        assert_not_null
+        debug :print_fields => true
+      end
+    end.complete
+  end
+
+  def test_join1
+    cascade 'splitter' do
+      flow 'splitter' do
+        source "data1", tap("test/data/data1.txt")
+        source "data2", tap("test/data/data2.txt")
+        sink "joined", tap('output/joined', :sink_mode => :replace)
+
+        assembly1 = assembly "data1" do
+          split "line", :pattern => /[.,]*\s+/, :into=>["name", "score1", "score2", "id"], :output => ["name", "score1", "score2", "id"]
+          assert_size_equals 4
+          assert_not_null
+          debug :print_fields => true
+        end
+
+        assembly2 = assembly "data2" do
+          split "line", :pattern => /[.,]*\s+/, :into=>["name",  "id", "town"], :output => ["name",  "id", "town"]
+          assert_size_equals 3
+          assert_not_null
+          debug :print_fields => true
+        end
+
+        assembly "joined" do
+          join assembly1.name, assembly2.name, :on => ["name", "id"], :declared_fields => ["name", "score1", "score2", "id", "name2", "id2", "town"]
+          assert_size_equals 7
+          assert_not_null
+        end
+      end
+    end.complete
+  end
+
+  def test_join2
+    flow = flow "splitter" do
+      source "data1", tap("test/data/data1.txt")
+      source "data2", tap("test/data/data2.txt")
+      sink "joined", tap('output/joined', :sink_mode => :replace)
+
+      assembly "data1" do
+        split "line", :pattern => /[.,]*\s+/, :into=>["name", "score1", "score2", "id"], :output => ["name", "score1", "score2", "id"]
+        debug :print_fields => true
+      end
+
+      assembly "data2" do
+        split "line", :pattern => /[.,]*\s+/, :into=>["name",  "code", "town"], :output => ["name",  "code", "town"]
+        debug :print_fields => true
+      end
+
+      assembly "joined" do
+        join :on => {"data1"=>["name", "id"], "data2"=>["name", "code"]}, :declared_fields => ["name", "score1", "score2", "id", "name2", "code", "town"]
+      end
+     end.complete
+   end
+end

data/test/test_cascading.rb

@@ -0,0 +1,49 @@
+require 'test/unit'
+require 'cascading'
+
+class TC_Cascading < Test::Unit::TestCase
+  def test_fields_field
+    result = fields(all_fields)
+    assert result == all_fields
+  end
+
+  def test_fields_single
+    declared = "Field1"
+
+    result = fields(declared)
+
+    assert result.size == 1
+
+    assert_equal declared, result.get(0) 
+  end
+
+  def test_fields_multiple
+    declared = ["Field1", "Field2", "Field3"]
+
+    result = fields(declared)
+
+    assert result.size == 3
+
+    assert_equal declared[0], result.get(0)
+    assert_equal declared[1], result.get(1)
+    assert_equal declared[2], result.get(2) 
+  end
+
+  def test_tap
+    tap = tap('/temp')
+    assert_equal '/temp', tap.getPath().toString()
+    assert tap.is_a? Java::CascadingTap::Hfs  
+
+    tap = tap('/temp', :kind => :dfs)
+    assert_equal '/temp', tap.getPath().toString()
+    assert tap.is_a? Java::CascadingTap::Dfs
+
+    tap = tap('/temp', :kind => :lfs)
+    assert_equal '/temp', tap.getPath().toString()
+    assert tap.is_a? Java::CascadingTap::Lfs
+
+    tap = tap('/temp', :kind => :hfs)
+    assert_equal '/temp', tap.getPath().toString()
+    assert tap.is_a? Java::CascadingTap::Hfs
+  end
+end

data/test/test_flow.rb

@@ -0,0 +1,15 @@
+require 'test/unit'
+require 'cascading'
+
+class TC_Flow < Test::Unit::TestCase
+  def test_assembly
+    flow = flow 'My Flow1' do
+      assembly "Test1" do
+      end
+    end
+
+    assert_equal 1, flow.children.size
+    assert_equal flow.children["Test1"], flow.find_child("Test1")
+    assert_equal flow.last_child, flow.find_child("Test1")
+  end
+end

metadata

@@ -0,0 +1,137 @@
+--- !ruby/object:Gem::Specification 
+name: cascading.jruby
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+    - 0
+    - 0
+    - 4
+  version: 0.0.4
+platform: ruby
+authors: 
+  - Matt Walker
+  - "Gr\xC3\xA9goire Marabout"
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-04-05 00:00:00 -05:00
+default_executable: 
+dependencies: []
+
+description: cascading.jruby is a small DSL above Cascading, written in JRuby
+email: mwalker@etsy.com
+executables: 
+  - make_job
+extensions: []
+
+extra_rdoc_files: 
+  - History.txt
+  - LICENSE.txt
+  - bin/make_job
+  - samples/data/data2.txt
+  - samples/data/data_join1.txt
+  - samples/data/data_join2.txt
+  - samples/data/data_join3.txt
+  - spec/resource/join_input.txt
+  - spec/resource/test_input.txt
+  - test/data/data1.txt
+  - test/data/data2.txt
+files: 
+  - HACKING.md
+  - History.txt
+  - LICENSE.txt
+  - README.md
+  - Rakefile
+  - bin/make_job
+  - lib/cascading.rb
+  - lib/cascading/assembly.rb
+  - lib/cascading/base.rb
+  - lib/cascading/cascade.rb
+  - lib/cascading/cascading.rb
+  - lib/cascading/cascading_exception.rb
+  - lib/cascading/expr_stub.rb
+  - lib/cascading/ext/array.rb
+  - lib/cascading/flow.rb
+  - lib/cascading/operations.rb
+  - lib/cascading/scope.rb
+  - samples/branch.rb
+  - samples/cascading.rb
+  - samples/copy.rb
+  - samples/data/data2.txt
+  - samples/data/data_join1.txt
+  - samples/data/data_join2.txt
+  - samples/data/data_join3.txt
+  - samples/join.rb
+  - samples/logwordcount.rb
+  - samples/project.rb
+  - samples/rename.rb
+  - samples/scorenames.rb
+  - samples/splitter.rb
+  - samples/union.rb
+  - spec/cascading_spec.rb
+  - spec/expr_spec.rb
+  - spec/primary_key_spec.rb
+  - spec/resource/join_input.txt
+  - spec/resource/test_input.txt
+  - spec/scope_spec.rb
+  - spec/spec.opts
+  - spec/spec_helper.rb
+  - spec/spec_util.rb
+  - src/cascading/jruby/Main.java
+  - src/cascading/jruby/runner.rb
+  - tags
+  - tasks/ann.rake
+  - tasks/ant.rake
+  - tasks/bones.rake
+  - tasks/gem.rake
+  - tasks/git.rake
+  - tasks/notes.rake
+  - tasks/post_load.rake
+  - tasks/rdoc.rake
+  - tasks/rubyforge.rake
+  - tasks/samples.rake
+  - tasks/setup.rb
+  - tasks/spec.rake
+  - tasks/svn.rake
+  - tasks/test.rake
+  - test/data/data1.txt
+  - test/data/data2.txt
+  - test/test_assembly.rb
+  - test/test_cascading.rb
+  - test/test_flow.rb
+has_rdoc: true
+homepage: http://github.com/etsy/cascading.jruby
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+  - --main
+  - README.md
+require_paths: 
+  - lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+          - 0
+        version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+          - 0
+        version: "0"
+requirements: []
+
+rubyforge_project: cascading.jruby
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: A JRuby DSL for Cascading
+test_files: 
+  - test/test_assembly.rb
+  - test/test_cascading.rb
+  - test/test_flow.rb