cascading.jruby 0.0.7 → 0.0.8

data/test/test_assembly.rb

@@ -1,41 +1,29 @@
 require 'test/unit'
 require 'cascading'
-
-def compare_with_references(test_name)
-  result = compare_files("test/references/#{test_name}.txt", "output/#{test_name}/part-00000")
-  assert_nil(result)
-end
-
-# Convenience for basic assembly tests; not valid for applications
-def assembly(name, &block)
-  assembly = Assembly.new(name, nil)
-  assembly.instance_eval(&block)
-  assembly
-end
+require 'test/mock_assemblies'
 
 class TC_Assembly < Test::Unit::TestCase
-  def mock_assembly(&block)
-    assembly = nil
-    flow 'test' do
-      source 'test', tap('test/data/data1.txt')
-      assembly = assembly 'test', &block
-    end
-    assembly
-  end
+  include MockAssemblies
 
   def test_create_assembly_simple
-    assembly = assembly "assembly1" do
-      # Empty assembly
+    assembly = nil
+    flow 'test_create_assembly_simple' do
+      assembly = assembly 'assembly1' do
+        # Empty assembly
+      end
     end
 
     assert_not_nil assembly
+    assert_equal assembly.name, 'assembly1'
+    assert_equal 0, assembly.children.size
+
     pipe = assembly.tail_pipe
-    assert pipe.is_a? Java::CascadingPipe::Pipe
+    assert_equal Java::CascadingPipe::Pipe, pipe.class
   end
 
   def test_each_identity
     assembly = mock_assembly do
-      each 'offset', :filter => identity
+      each 'offset', :function => identity
     end
 
     flow = assembly.parent
@@ -47,148 +35,424 @@ class TC_Assembly < Test::Unit::TestCase
   def test_create_each
     # You can apply an Each to 0 fields
     assembly = mock_assembly do
-      each(:filter => identity)
+      each(:function => identity)
     end
-    assert assembly.tail_pipe.is_a? Java::CascadingPipe::Each
+    assert_equal Java::CascadingPipe::Each, assembly.tail_pipe.class
 
     # In which case, it has empty argument and output selectors
     assert_equal 0, assembly.tail_pipe.argument_selector.size
     assert_equal 0, assembly.tail_pipe.output_selector.size
 
     assembly = mock_assembly do
-      each('offset', :output => 'offset_copy',
-           :filter => Java::CascadingOperation::Identity.new(fields('offset_copy')))
+      each 'offset', :output => 'offset_copy', :function => Java::CascadingOperation::Identity.new(fields('offset_copy'))
     end
     pipe = assembly.tail_pipe
 
-    assert pipe.is_a? Java::CascadingPipe::Each
+    assert_equal Java::CascadingPipe::Each, pipe.class
 
-    assert_equal 'offset', pipe.argument_selector.get(0)
-    assert_equal 'offset_copy', pipe.output_selector.get(0)
+    assert_equal ['offset'], pipe.argument_selector.to_a
+    assert_equal ['offset_copy'], pipe.output_selector.to_a
   end
 
-  # For now, replaced these tests with the trivial observation that you can't
-  # follow a Tap with an Every.  Eventually, should support testing within a
-  # group_by block.
-  def test_create_every
-    assert_raise CascadingException do
+  def test_every_cannot_follow_tap
+    # Assembly#count is no longer defined; instead, it has moved to
+    # Aggregations#count
+    assert_raise NameError do
       assembly = mock_assembly do
-        every(:aggregator => count_function)
+        count
       end
       pipe = assembly.tail_pipe
-      assert pipe.is_a? Java::CascadingPipe::Every
+      assert Java::CascadingPipe::Every, pipe.class
     end
+  end
 
-    assert_raise CascadingException do
+  def test_create_every
       assembly = mock_assembly do
-        every(:aggregator => count_function("field1", "field2"))
+        group_by 'line' do
+          count_aggregator = Java::CascadingOperationAggregator::Count.new(fields('count'))
+          every 'line', :aggregator => count_aggregator, :output => 'count'
+        end
       end
-      assert assembly.tail_pipe.is_a? Java::CascadingPipe::Every
-    end
+      assert Java::CascadingPipe::Every, assembly.tail_pipe.class
+      assert_equal ['line'], assembly.tail_pipe.argument_selector.to_a
+      assert_equal ['count'], assembly.tail_pipe.output_selector.to_a
 
-    assert_raise CascadingException do
       assembly = mock_assembly do
-        every("Field1", :aggregator => count_function)
+        group_by 'line' do
+          count
+        end
       end
-      assert assembly.tail_pipe.is_a? Java::CascadingPipe::Every
-      assert_equal "Field1", assembly.tail_pipe.argument_selector.get(0)
-    end
+      assert Java::CascadingPipe::Every, assembly.tail_pipe.class
 
-    assert_raise CascadingException do
-      assembly = mock_assembly do
-        every('line', :aggregator => count_function, :output=>'line_count')
-      end
-      assert assembly.tail_pipe.is_a? Java::CascadingPipe::Every
-      assert_equal 'line', assembly.tail_pipe.argument_selector.get(0)
-      assert_equal 'line_count', assembly.tail_pipe.output_selector.get(0)
-    end
+      # NOTE: this is not valid when we optimize using CountBy
+      #assert_equal last_grouping_fields, assembly.tail_pipe.argument_selector
+      assert_equal fields('count'), assembly.tail_pipe.argument_selector
+
+      assert_equal all_fields, assembly.tail_pipe.output_selector
   end
 
   def test_create_group_by
     assembly = mock_assembly do
-      group_by('line')
+      group_by 'line'
     end
 
-    assert assembly.tail_pipe.is_a? Java::CascadingPipe::GroupBy
+    assert_equal Java::CascadingPipe::GroupBy, assembly.tail_pipe.class
     grouping_fields = assembly.tail_pipe.key_selectors['test']
-    assert_equal 'line', grouping_fields.get(0)
+    assert_equal ['line'], grouping_fields.to_a
+
+    assert_equal ['offset', 'line'], assembly.scope.values_fields.to_a
+    assert_equal ['line'], assembly.scope.grouping_fields.to_a
 
     assembly = mock_assembly do
-      group_by('line')
+      group_by 'offset'
     end
 
-    assert assembly.tail_pipe.is_a? Java::CascadingPipe::GroupBy
+    assert_equal Java::CascadingPipe::GroupBy, assembly.tail_pipe.class
     grouping_fields = assembly.tail_pipe.key_selectors['test']
-    assert_equal 'line', grouping_fields.get(0)
+    assert_equal ['offset'], grouping_fields.to_a
+
+    assert_equal ['offset', 'line'], assembly.scope.values_fields.to_a
+    assert_equal ['offset'], assembly.scope.grouping_fields.to_a
   end
 
   def test_create_group_by_many_fields
     assembly = mock_assembly do
-      group_by(['offset', 'line'])
+      group_by 'offset', 'line'
     end
 
-    assert assembly.tail_pipe.is_a? Java::CascadingPipe::GroupBy
+    assert_equal Java::CascadingPipe::GroupBy, assembly.tail_pipe.class
     grouping_fields = assembly.tail_pipe.key_selectors['test']
-    assert_equal 'offset', grouping_fields.get(0)
-    assert_equal 'line', grouping_fields.get(1)
+    assert_equal ['offset', 'line'], grouping_fields.to_a
+
+    assert_equal ['offset', 'line'], assembly.scope.values_fields.to_a
+    assert_equal ['offset', 'line'], assembly.scope.grouping_fields.to_a
   end
 
   def test_create_group_by_with_sort
     assembly = mock_assembly do
-      group_by('offset', 'line', :sort_by => ['line'])
+      group_by 'offset', 'line', :sort_by => 'line'
     end
 
-    assert assembly.tail_pipe.is_a? Java::CascadingPipe::GroupBy
+    assert_equal Java::CascadingPipe::GroupBy, assembly.tail_pipe.class
     grouping_fields = assembly.tail_pipe.key_selectors['test']
     sorting_fields = assembly.tail_pipe.sorting_selectors['test']
 
-    assert_equal 2, grouping_fields.size
-    assert_equal 1, sorting_fields.size
+    assert assembly.tail_pipe.is_sorted
+    assert !assembly.tail_pipe.is_sort_reversed
 
-    assert_equal 'offset', grouping_fields.get(0)
-    assert_equal 'line', grouping_fields.get(1)
-    assert assembly.tail_pipe.isSorted()
-    assert !assembly.tail_pipe.isSortReversed()
-    assert_equal 'line', sorting_fields.get(0)
+    assert_equal ['offset', 'line'], grouping_fields.to_a
+    assert_equal ['line'], sorting_fields.to_a
+
+    assert_equal ['offset', 'line'], assembly.scope.values_fields.to_a
+    assert_equal ['offset', 'line'], assembly.scope.grouping_fields.to_a
   end
 
   def test_create_group_by_with_sort_reverse
     assembly = mock_assembly do
-      group_by('offset', 'line', :sort_by => ['line'], :reverse => true)
+      group_by 'offset', 'line', :sort_by => 'line', :reverse => true
     end
 
-    assert assembly.tail_pipe.is_a? Java::CascadingPipe::GroupBy
+    assert_equal Java::CascadingPipe::GroupBy, assembly.tail_pipe.class
     grouping_fields = assembly.tail_pipe.key_selectors['test']
     sorting_fields = assembly.tail_pipe.sorting_selectors['test']
 
-    assert_equal 2, grouping_fields.size
-    assert_equal 1, sorting_fields.size
+    assert assembly.tail_pipe.is_sorted
+    assert assembly.tail_pipe.is_sort_reversed
+
+    assert_equal ['offset', 'line'], grouping_fields.to_a
+    assert_equal ['line'], sorting_fields.to_a
 
-    assert_equal 'offset', grouping_fields.get(0)
-    assert_equal 'line', grouping_fields.get(1)
-    assert assembly.tail_pipe.isSorted()
-    assert assembly.tail_pipe.isSortReversed()
-    assert_equal 'line', sorting_fields.get(0)
+    assert_equal ['offset', 'line'], assembly.scope.values_fields.to_a
+    assert_equal ['offset', 'line'], assembly.scope.grouping_fields.to_a
   end
 
   def test_create_group_by_reverse
     assembly = mock_assembly do
-      group_by('offset', 'line', :reverse => true)
+      group_by 'offset', 'line', :reverse => true
     end
 
-    assert assembly.tail_pipe.is_a? Java::CascadingPipe::GroupBy
+    assert_equal Java::CascadingPipe::GroupBy, assembly.tail_pipe.class
     grouping_fields = assembly.tail_pipe.key_selectors['test']
     sorting_fields = assembly.tail_pipe.sorting_selectors['test']
 
-    assert_equal 2, grouping_fields.size
-    assert_equal 2, sorting_fields.size
+    assert !assembly.tail_pipe.is_sorted
+    assert assembly.tail_pipe.is_sort_reversed
+
+    assert_equal ['offset', 'line'], grouping_fields.to_a
+    assert_nil sorting_fields
+
+    assert_equal ['offset', 'line'], assembly.scope.values_fields.to_a
+    assert_equal ['offset', 'line'], assembly.scope.grouping_fields.to_a
+  end
+
+  def test_create_union
+    assembly = mock_branched_assembly do
+      union 'test1', 'test2', :on => 'line'
+    end
+
+    assert_equal Java::CascadingPipe::GroupBy, assembly.tail_pipe.class
+
+    left_grouping_fields = assembly.tail_pipe.key_selectors['test1']
+    assert_equal ['line'], left_grouping_fields.to_a
+
+    right_grouping_fields = assembly.tail_pipe.key_selectors['test2']
+    assert_equal ['line'], right_grouping_fields.to_a
+
+    assert_equal ['offset', 'line'], assembly.scope.values_fields.to_a
+    assert_equal ['line'], assembly.scope.grouping_fields.to_a
+
+    assembly = mock_branched_assembly do
+      union 'test1', 'test2', :on => 'offset'
+    end
+
+    assert_equal Java::CascadingPipe::GroupBy, assembly.tail_pipe.class
+    left_grouping_fields = assembly.tail_pipe.key_selectors['test1']
+    assert_equal ['offset'], left_grouping_fields.to_a
+    right_grouping_fields = assembly.tail_pipe.key_selectors['test2']
+    assert_equal ['offset'], right_grouping_fields.to_a
+
+    assert_equal ['offset', 'line'], assembly.scope.values_fields.to_a
+    assert_equal ['offset'], assembly.scope.grouping_fields.to_a
+
+    assembly = mock_branched_assembly do
+      union 'test1', 'test2'
+    end
+
+    assert_equal Java::CascadingPipe::GroupBy, assembly.tail_pipe.class
+    left_grouping_fields = assembly.tail_pipe.key_selectors['test1']
+    assert_equal ['offset'], left_grouping_fields.to_a
+    right_grouping_fields = assembly.tail_pipe.key_selectors['test2']
+    assert_equal ['offset'], right_grouping_fields.to_a
+
+    assert_equal ['offset', 'line'], assembly.scope.values_fields.to_a
+    assert_equal ['offset'], assembly.scope.grouping_fields.to_a
+  end
+
+  def test_create_union_many_fields
+    assembly = mock_branched_assembly do
+      union 'test1', 'test2', :on => ['offset', 'line']
+    end
+
+    assert_equal Java::CascadingPipe::GroupBy, assembly.tail_pipe.class
+
+    left_grouping_fields = assembly.tail_pipe.key_selectors['test1']
+    assert_equal ['offset', 'line'], left_grouping_fields.to_a
+
+    right_grouping_fields = assembly.tail_pipe.key_selectors['test2']
+    assert_equal ['offset', 'line'], right_grouping_fields.to_a
+
+    assert_equal ['offset', 'line'], assembly.scope.values_fields.to_a
+    assert_equal ['offset', 'line'], assembly.scope.grouping_fields.to_a
+  end
+
+  def test_create_union_with_sort
+    assembly = mock_branched_assembly do
+      union 'test1', 'test2', :on => ['offset', 'line'], :sort_by => 'line'
+    end
+
+    assert_equal Java::CascadingPipe::GroupBy, assembly.tail_pipe.class
+    left_grouping_fields = assembly.tail_pipe.key_selectors['test1']
+    right_grouping_fields = assembly.tail_pipe.key_selectors['test2']
+    left_sorting_fields = assembly.tail_pipe.sorting_selectors['test1']
+    right_sorting_fields = assembly.tail_pipe.sorting_selectors['test2']
+
+    assert assembly.tail_pipe.is_sorted
+    assert !assembly.tail_pipe.is_sort_reversed
+
+    assert_equal ['offset', 'line'], left_grouping_fields.to_a
+    assert_equal ['offset', 'line'], right_grouping_fields.to_a
+    assert_equal ['line'], left_sorting_fields.to_a
+    assert_equal ['line'], right_sorting_fields.to_a
+
+    assert_equal ['offset', 'line'], assembly.scope.values_fields.to_a
+    assert_equal ['offset', 'line'], assembly.scope.grouping_fields.to_a
+  end
+
+  def test_create_union_with_sort_reverse
+    assembly = mock_branched_assembly do
+      union 'test1', 'test2', :on => ['offset', 'line'], :sort_by => 'line', :reverse => true
+    end
+
+    assert_equal Java::CascadingPipe::GroupBy, assembly.tail_pipe.class
+    left_grouping_fields = assembly.tail_pipe.key_selectors['test1']
+    right_grouping_fields = assembly.tail_pipe.key_selectors['test2']
+    left_sorting_fields = assembly.tail_pipe.sorting_selectors['test1']
+    right_sorting_fields = assembly.tail_pipe.sorting_selectors['test2']
+
+    assert assembly.tail_pipe.is_sorted
+    assert assembly.tail_pipe.is_sort_reversed
+
+    assert_equal ['offset', 'line'], left_grouping_fields.to_a
+    assert_equal ['offset', 'line'], right_grouping_fields.to_a
+    assert_equal ['line'], left_sorting_fields.to_a
+    assert_equal ['line'], right_sorting_fields.to_a
+
+    assert_equal ['offset', 'line'], assembly.scope.values_fields.to_a
+    assert_equal ['offset', 'line'], assembly.scope.grouping_fields.to_a
+  end
+
+  def test_create_union_reverse
+    assembly = mock_branched_assembly do
+      union 'test1', 'test2', :on => ['offset', 'line'], :reverse => true
+    end
+
+    assert_equal Java::CascadingPipe::GroupBy, assembly.tail_pipe.class
+    left_grouping_fields = assembly.tail_pipe.key_selectors['test1']
+    right_grouping_fields = assembly.tail_pipe.key_selectors['test2']
+    left_sorting_fields = assembly.tail_pipe.sorting_selectors['test1']
+    right_sorting_fields = assembly.tail_pipe.sorting_selectors['test2']
+
+    assert assembly.tail_pipe.is_sorted # FIXME: Missing constructor in wip-255
+    assert assembly.tail_pipe.is_sort_reversed
+
+    assert_equal ['offset', 'line'], left_grouping_fields.to_a
+    assert_equal ['offset', 'line'], right_grouping_fields.to_a
+    assert_equal ['offset', 'line'], left_sorting_fields.to_a # FIXME: Missing constructor in wip-255
+    assert_equal ['offset', 'line'], right_sorting_fields.to_a # FIXME: Missing constructor in wip-255
+
+    assert_equal ['offset', 'line'], assembly.scope.values_fields.to_a
+    assert_equal ['offset', 'line'], assembly.scope.grouping_fields.to_a
+  end
+
+  def test_union_undefined_inputs
+    assert_raise RuntimeError, "Could not find assembly 'doesnotexist' in union" do
+      flow 'test_union_undefined_inputs' do
+        source 'data1', tap('test/data/data1.txt')
+
+        assembly 'data1' do
+          pass
+        end
+
+        assembly 'union' do
+          union 'doesnotexist', 'data1'
+        end
+
+        sink 'union', tap('output/test_union_undefined_inputs')
+      end
+    end
+  end
+
+  def test_create_join
+    assembly = mock_two_input_assembly do
+      join 'test1', 'test2', :on => 'name'
+    end
+
+    assert_equal Java::CascadingPipe::CoGroup, assembly.tail_pipe.class
+
+    left_grouping_fields = assembly.tail_pipe.key_selectors['test1']
+    assert_equal ['name'], left_grouping_fields.to_a
+
+    right_grouping_fields = assembly.tail_pipe.key_selectors['test2']
+    assert_equal ['name'], right_grouping_fields.to_a
+
+    assert_equal ['name', 'score1', 'score2', 'id', 'name_', 'id_', 'town'], assembly.scope.values_fields.to_a
+    assert_equal ['name', 'name_'], assembly.scope.grouping_fields.to_a
+
+    assembly = mock_two_input_assembly do
+      join 'test1', 'test2', :on => 'id'
+    end
+
+    assert_equal Java::CascadingPipe::CoGroup, assembly.tail_pipe.class
+    left_grouping_fields = assembly.tail_pipe.key_selectors['test1']
+    assert_equal ['id'], left_grouping_fields.to_a
+    right_grouping_fields = assembly.tail_pipe.key_selectors['test2']
+    assert_equal ['id'], right_grouping_fields.to_a
+
+    assert_equal ['name', 'score1', 'score2', 'id', 'name_', 'id_', 'town'], assembly.scope.values_fields.to_a
+    assert_equal ['id', 'id_'], assembly.scope.grouping_fields.to_a
+  end
+
+  def test_create_join_many_fields
+    assembly = mock_two_input_assembly do
+      join 'test1', 'test2', :on => ['name', 'id']
+    end
+
+    assert_equal Java::CascadingPipe::CoGroup, assembly.tail_pipe.class
+    left_grouping_fields = assembly.tail_pipe.key_selectors['test1']
+    assert_equal ['name', 'id'], left_grouping_fields.to_a
+    right_grouping_fields = assembly.tail_pipe.key_selectors['test2']
+    assert_equal ['name', 'id'], right_grouping_fields.to_a
 
-    assert_equal 'offset', grouping_fields.get(0)
-    assert_equal 'line', grouping_fields.get(1)
-    assert assembly.tail_pipe.isSorted()
-    assert assembly.tail_pipe.isSortReversed()
-    assert_equal 'offset', sorting_fields.get(0)
-    assert_equal 'line', sorting_fields.get(1)
+    assert_equal ['name', 'score1', 'score2', 'id', 'name_', 'id_', 'town'], assembly.scope.values_fields.to_a
+    assert_equal ['name', 'id', 'name_', 'id_'], assembly.scope.grouping_fields.to_a
+  end
+
+  def test_create_join_with_declared_fields
+    assembly = mock_two_input_assembly do
+      join 'test1', 'test2', :on => 'name', :declared_fields => ['a', 'b', 'c', 'd', 'e', 'f', 'g']
+    end
+
+    assert_equal Java::CascadingPipe::CoGroup, assembly.tail_pipe.class
+
+    left_grouping_fields = assembly.tail_pipe.key_selectors['test1']
+    assert_equal ['name'], left_grouping_fields.to_a
+
+    right_grouping_fields = assembly.tail_pipe.key_selectors['test2']
+    assert_equal ['name'], right_grouping_fields.to_a
+
+    assert_equal ['a', 'b', 'c', 'd', 'e', 'f', 'g'], assembly.scope.values_fields.to_a
+    assert_equal ['name', 'name_'], assembly.scope.grouping_fields.to_a
+  end
+
+  def test_join_with_block
+    assembly = mock_two_input_assembly do
+      join 'test1', 'test2', :on => 'name' do
+        count
+      end
+    end
+
+    assert_equal Java::CascadingPipe::Every, assembly.tail_pipe.class
+
+    assert_equal ['name', 'name_', 'count'], assembly.scope.values_fields.to_a
+    assert_equal ['name', 'name_', 'count'], assembly.scope.grouping_fields.to_a
+  end
+
+  def test_join_undefined_inputs
+    assert_raise RuntimeError, "Could not find assembly 'doesnotexist' in join" do
+      flow 'test_join_undefined_inputs' do
+        source 'data1', tap('test/data/data1.txt')
+
+        assembly 'data1' do
+          pass
+        end
+
+        assembly 'join' do
+          join 'doesnotexist', 'data1', :on => 'name'
+        end
+
+        sink 'join', tap('output/test_join_undefined_inputs')
+      end
+    end
+  end
+
+  def test_join_without_on
+    assert_raise RuntimeError, 'join requires :on parameter' do
+      mock_two_input_assembly do
+        join 'test1', 'test2'
+      end
+    end
+  end
+
+  def test_join_invalid_on
+    assert_raise RuntimeError, "Unsupported data type for :on in join: 'Fixnum'" do
+      mock_two_input_assembly do
+        join 'test1', 'test2', :on => 1
+      end
+    end
+  end
+
+  def test_join_empty_on
+    assert_raise RuntimeError, 'join requres non-empty :on parameter' do
+      mock_two_input_assembly do
+        join 'test1', 'test2', :on => []
+      end
+    end
+
+    assert_raise RuntimeError, 'join requres non-empty :on parameter' do
+      mock_two_input_assembly do
+        join 'test1', 'test2', :on => {}
+      end
+    end
   end
 
   def test_branch_unique
@@ -198,7 +462,6 @@ class TC_Assembly < Test::Unit::TestCase
     end
 
     assert_equal 1, assembly.children.size
-
   end
 
   def test_branch_empty
@@ -231,20 +494,74 @@ class TC_Assembly < Test::Unit::TestCase
     assert_equal 0, assembly.children['branch1'].children['branch2'].children.size
   end
 
-  # Fixed this test, but it isn't even valid.  You shouldn't be able to follow
-  # an Each with an Every.
-  def test_full_assembly
-    assert_raise CascadingException do
-      assembly = mock_assembly do
-        each('offset', :output => 'offset_copy',
-             :filter => Java::CascadingOperation::Identity.new(fields('offset_copy')))
-        every(:aggregator => count_function)
-      end
+  def test_sub_assembly
+    assembly = mock_assembly do
+      sub_assembly Java::CascadingPipeAssembly::Discard.new(tail_pipe, fields('offset'))
+    end
+    assert_equal ['line'], assembly.scope.values_fields.to_a
 
-      pipe = assembly.tail_pipe
+    assembly = mock_assembly do
+      sub_assembly Java::CascadingPipeAssembly::Retain.new(tail_pipe, fields('offset'))
+    end
+    assert_equal ['offset'], assembly.scope.values_fields.to_a
 
-      assert pipe.is_a? Java::CascadingPipe::Every
+    assembly = mock_assembly do
+      sub_assembly Java::CascadingPipeAssembly::Rename.new(tail_pipe, fields(['offset', 'line']), fields(['byte', 'line']))
     end
+    assert_equal ['byte', 'line'], assembly.scope.values_fields.to_a
+
+    assembly = mock_assembly do
+      sub_assembly Java::CascadingPipeAssembly::Unique.new(tail_pipe, fields('line'))
+    end
+    assert_equal ['offset', 'line'], assembly.scope.values_fields.to_a
+    assert_equal ['offset', 'line'], assembly.scope.grouping_fields.to_a
+  end
+
+  def test_count_by_sub_assembly
+    assembly = mock_branched_assembly do
+      pipes, _ = populate_incoming_scopes(['test1', 'test2'])
+
+      aggregate_by = Java::CascadingPipeAssembly::AggregateBy.new(
+        name,
+        pipes.to_java(Java::CascadingPipe::Pipe),
+        fields('line'),
+        [Java::CascadingPipeAssembly::CountBy.new(fields('count'))].to_java(Java::CascadingPipeAssembly::AggregateBy)
+      )
+
+      sub_assembly aggregate_by, pipes, @incoming_scopes
+    end
+    assert_equal ['line', 'count'], assembly.scope.values_fields.to_a
+    assert_equal ['line', 'count'], assembly.scope.grouping_fields.to_a
+  end
+
+  def test_average_by_sub_assembly
+    assembly = mock_assembly do
+      aggregate_by = Java::CascadingPipeAssembly::AggregateBy.new(
+        name,
+        [tail_pipe].to_java(Java::CascadingPipe::Pipe),
+        fields('line'),
+        [Java::CascadingPipeAssembly::AverageBy.new(fields('offset'), fields('average'))].to_java(Java::CascadingPipeAssembly::AggregateBy)
+      )
+
+      sub_assembly aggregate_by
+    end
+    assert_equal ['line', 'average'], assembly.scope.values_fields.to_a
+    assert_equal ['line', 'average'], assembly.scope.grouping_fields.to_a
+  end
+
+  def test_sum_by_sub_assembly
+    assembly = mock_assembly do
+      aggregate_by = Java::CascadingPipeAssembly::AggregateBy.new(
+        name,
+        [tail_pipe].to_java(Java::CascadingPipe::Pipe),
+        fields('line'),
+        [Java::CascadingPipeAssembly::SumBy.new(fields('offset'), fields('sum'), Java::double.java_class)].to_java(Java::CascadingPipeAssembly::AggregateBy)
+      )
+
+      sub_assembly aggregate_by
+    end
+    assert_equal ['line', 'sum'], assembly.scope.values_fields.to_a
+    assert_equal ['line', 'sum'], assembly.scope.grouping_fields.to_a
   end
 
   def test_empty_where
@@ -252,10 +569,10 @@ class TC_Assembly < Test::Unit::TestCase
       split 'line', ['name', 'score1', 'score2', 'id'], :pattern => /[.,]*\s+/, :output => ['name', 'score1', 'score2', 'id']
       where
     end
-    assert assembly.tail_pipe.is_a? Java::CascadingPipe::Each
+    assert_equal Java::CascadingPipe::Each, assembly.tail_pipe.class
 
     # Empty where compiles away
-    assert assembly.tail_pipe.operation.is_a? Java::CascadingOperationRegex::RegexSplitter
+    assert_equal Java::CascadingOperationRegex::RegexSplitter, assembly.tail_pipe.operation.class
   end
 
   def test_where
@@ -263,8 +580,8 @@ class TC_Assembly < Test::Unit::TestCase
       split 'line', ['name', 'score1', 'score2', 'id'], :pattern => /[.,]*\s+/, :output => ['name', 'score1', 'score2', 'id']
       where 'score1:double < score2:double'
     end
-    assert assembly.tail_pipe.is_a? Java::CascadingPipe::Each
-    assert assembly.tail_pipe.operation.is_a? Java::CascadingOperationExpression::ExpressionFilter
+    assert_equal Java::CascadingPipe::Each, assembly.tail_pipe.class
+    assert_equal Java::CascadingOperationExpression::ExpressionFilter, assembly.tail_pipe.operation.class
   end
 
   def test_where_with_expression
@@ -272,8 +589,8 @@ class TC_Assembly < Test::Unit::TestCase
       split 'line', ['name', 'score1', 'score2', 'id'], :pattern => /[.,]*\s+/, :output => ['name', 'score1', 'score2', 'id']
       where :expression => 'score1:double < score2:double'
     end
-    assert assembly.tail_pipe.is_a? Java::CascadingPipe::Each
-    assert assembly.tail_pipe.operation.is_a? Java::CascadingOperationExpression::ExpressionFilter
+    assert_equal Java::CascadingPipe::Each, assembly.tail_pipe.class
+    assert_equal Java::CascadingOperationExpression::ExpressionFilter, assembly.tail_pipe.operation.class
   end
 
   def test_where_with_import
@@ -282,8 +599,8 @@ class TC_Assembly < Test::Unit::TestCase
       names = ['SMITH', 'JONES', 'BROWN']
       where "import java.util.Arrays;\nArrays.asList(new String[] { \"#{names.join('", "')}\" }).contains(name:string)"
     end
-    assert assembly.tail_pipe.is_a? Java::CascadingPipe::Each
-    assert assembly.tail_pipe.operation.is_a? Java::CascadingOperationExpression::ExpressionFilter
+    assert_equal Java::CascadingPipe::Each, assembly.tail_pipe.class
+    assert_equal Java::CascadingOperationExpression::ExpressionFilter, assembly.tail_pipe.operation.class
   end
 
   def test_smoke_test_debug_scope
@@ -291,92 +608,16 @@ class TC_Assembly < Test::Unit::TestCase
       flow 'smoke' do
         source 'input', tap('test/data/data1.txt')
         assembly 'input' do
-          pass
+          debug_scope
+          group_by 'line' do
+            count
+            sum 'offset', :type => :long
+            debug_scope
+          end
           debug_scope
         end
-        sink 'input', tap('output/smoke_test_debug_scope')
+        sink 'input', tap('output/test_smoke_test_debug_scope')
       end
     end
   end
 end
-
-class TC_AssemblyScenarii < Test::Unit::TestCase
-  def test_smoke_test_sequence_file_scheme
-    cascade 'smoke' do
-      flow 'smoke' do
-        source 'input', tap('test/data/data1.txt')
-        assembly 'input' do
-          pass
-        end
-        compress_output :default, :block
-        sink 'input', tap('output/smoke_test_sequence_file_scheme', :scheme => sequence_file_scheme)
-      end
-    end.complete
-  end
-
-  def test_splitter
-    flow = flow "splitter" do
-      source "copy", tap("test/data/data1.txt")
-      sink "copy", tap('output/splitter', :sink_mode => :replace)
-
-      assembly "copy" do
-        split "line", :pattern => /[.,]*\s+/, :into=>["name", "score1", "score2", "id"], :output => ["name", "score1", "score2", "id"]
-        assert_size_equals 4
-        assert_not_null
-        debug :print_fields => true
-      end
-    end.complete
-  end
-
-  def test_join1
-    cascade 'splitter' do
-      flow 'splitter' do
-        source "data1", tap("test/data/data1.txt")
-        source "data2", tap("test/data/data2.txt")
-        sink "joined", tap('output/joined', :sink_mode => :replace)
-
-        assembly1 = assembly "data1" do
-          split "line", :pattern => /[.,]*\s+/, :into=>["name", "score1", "score2", "id"], :output => ["name", "score1", "score2", "id"]
-          assert_size_equals 4
-          assert_not_null
-          debug :print_fields => true
-        end
-
-        assembly2 = assembly "data2" do
-          split "line", :pattern => /[.,]*\s+/, :into=>["name",  "id", "town"], :output => ["name",  "id", "town"]
-          assert_size_equals 3
-          assert_not_null
-          debug :print_fields => true
-        end
-
-        assembly "joined" do
-          join assembly1.name, assembly2.name, :on => ["name", "id"], :declared_fields => ["name", "score1", "score2", "id", "name2", "id2", "town"]
-          assert_size_equals 7
-          assert_not_null
-        end
-      end
-    end.complete
-  end
-
-  def test_join2
-    flow = flow "splitter" do
-      source "data1", tap("test/data/data1.txt")
-      source "data2", tap("test/data/data2.txt")
-      sink "joined", tap('output/joined', :sink_mode => :replace)
-
-      assembly "data1" do
-        split "line", :pattern => /[.,]*\s+/, :into=>["name", "score1", "score2", "id"], :output => ["name", "score1", "score2", "id"]
-        debug :print_fields => true
-      end
-
-      assembly "data2" do
-        split "line", :pattern => /[.,]*\s+/, :into=>["name",  "code", "town"], :output => ["name",  "code", "town"]
-        debug :print_fields => true
-      end
-
-      assembly "joined" do
-        join :on => {"data1"=>["name", "id"], "data2"=>["name", "code"]}, :declared_fields => ["name", "score1", "score2", "id", "name2", "code", "town"]
-      end
-     end.complete
-   end
-end