cascading.jruby 0.0.7 → 0.0.8

data/samples/group_by.rb

@@ -0,0 +1,61 @@
+#! /usr/bin/env jruby
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'cascading'
+
+cascade 'group_by' do
+  flow 'group_by' do
+    source 'input', tap('samples/data/data_group_by.txt')
+
+    assembly 'input' do
+      split 'line', ['id', 'city'], :output => ['id', 'city']
+
+      branch 'group_by' do
+        group_by 'city', :sort_by => 'city' do
+          count
+          sum 'id', :type => :int
+        end
+      end
+
+      branch 'empty_group_by' do
+        group_by 'city', :sort_by => 'city' do
+        end
+      end
+
+      branch 'blockless_group_by' do
+        group_by 'city', :sort_by => 'city'
+      end
+
+      branch 'aggregate_by' do
+        group_by 'city' do
+          count
+          sum 'id', :type => :int
+        end
+      end
+
+      # These compile into GroupBy unless we relax Aggregations#can_aggregate?
+      # to allow empty Aggregations#aggregate_bys, which does not make sense
+      #branch 'empty_aggregate_by' do
+      #  group_by 'city' do
+      #  end
+      #end
+
+      #branch 'blockless_aggregate_by' do
+      #  group_by 'city'
+      #end
+
+      branch 'unique' do
+        sub_assembly Java::CascadingPipeAssembly::Unique.new(tail_pipe, fields('city'))
+      end
+    end
+
+    sink 'group_by', tap('output/group_by', :sink_mode => :replace)
+    sink 'empty_group_by', tap('output/empty_group_by', :sink_mode => :replace)
+    sink 'blockless_group_by', tap('output/blockless_group_by', :sink_mode => :replace)
+    sink 'aggregate_by', tap('output/aggregate_by', :sink_mode => :replace)
+    sink 'empty_aggregate_by', tap('output/empty_aggregate_by', :sink_mode => :replace)
+    sink 'blockless_aggregate_by', tap('output/blockless_aggregate_by', :sink_mode => :replace)
+    sink 'unique', tap('output/unique', :sink_mode => :replace)
+  end
+#end.draw(ARGV[0], local_properties('build/sample'))
+end.complete(local_properties('build/sample'))

data/samples/logwordcount.rb

@@ -5,7 +5,9 @@ require 'cascading'
 
 cascade 'logwordcount' do
   flow 'logwordcount' do
-    source 'input', tap('http://www.gutenberg.org/files/20417/20417-8.txt')
+    # This works just as well, but will get you blocked by Project Gutenberg
+    #source 'input', tap('http://www.gutenberg.org/files/20417/20417-8.txt')
+    source 'input', tap('samples/data/gutenberg/the_outline_of_science_vol_1')
 
     assembly 'input' do
       # TODO: create a helper for RegexSplitGenerator

data/samples/scorenames.rb

@@ -6,7 +6,8 @@ require 'cascading'
 cascade 'scorenames' do
   flow 'scorenames' do
     # You don't have to curl and cache inputs: tap can fetch via HTTP
-    source 'input', tap('http://www.census.gov/genealogy/names/dist.all.last')
+    #source 'input', tap('http://www.census.gov/genealogy/names/dist.all.last')
+    source 'input', tap('samples/data/genealogy/names/dist.all.last')
 
     assembly 'input' do
       split 'line', ['name', 'val1', 'val2', 'id']

data/samples/sub_assembly.rb

@@ -0,0 +1,30 @@
+#! /usr/bin/env jruby
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'cascading'
+
+cascade 'sub_assembly' do
+  flow 'sub_assembly' do
+    source 'input', tap('samples/data/data2.txt')
+
+    assembly 'input' do
+      split 'line', ['name', 'score1', 'score2', 'id'], :output => ['name', 'score1', 'score2', 'id']
+      assert Java::CascadingOperationAssertion::AssertSizeEquals.new(4)
+      sub_assembly Java::CascadingPipeAssembly::Discard.new(tail_pipe, fields('id'))
+      assert Java::CascadingOperationAssertion::AssertSizeEquals.new(3)
+
+      sub_assembly Java::CascadingPipeAssembly::Unique.new(tail_pipe, fields('name'))
+      assert Java::CascadingOperationAssertion::AssertSizeEquals.new(3)
+
+      sub_assembly Java::CascadingPipeAssembly::Retain.new(tail_pipe, fields(['score1', 'score2']))
+      assert Java::CascadingOperationAssertion::AssertSizeEquals.new(2)
+
+      sub_assembly Java::CascadingPipeAssembly::Rename.new(tail_pipe, fields(['score1', 'score2']), fields(['score_a', 'score_b']))
+      assert Java::CascadingOperationAssertion::AssertSizeEquals.new(2)
+
+      debug_scope
+    end
+
+    sink 'input', tap('output/sub_assembly', :sink_mode => :replace)
+  end
+end.complete(local_properties('build/sample'))

data/samples/union.rb

@@ -5,7 +5,9 @@ require 'cascading'
 
 cascade 'union' do
   flow 'union' do
-    source 'input', tap('http://www.census.gov/genealogy/names/dist.all.last')
+    # You don't have to curl and cache inputs: tap can fetch via HTTP
+    #source 'input', tap('http://www.census.gov/genealogy/names/dist.all.last')
+    source 'input', tap('samples/data/genealogy/names/dist.all.last')
 
     assembly 'input' do
       split 'line', ['name', 'score1', 'score2', 'id']

data/spec/scope_spec.rb

@@ -4,35 +4,11 @@ context Cascading::Scope do
   it 'should match Cascading fields names from source tap scheme' do
     test_assembly do
       # Pass that uses our scope instead of all_fields
-      operation = Java::CascadingOperation::Identity.new 
-      make_each(Java::CascadingPipe::Each, tail_pipe, scope.values_fields, operation)
-
+      each scope.values_fields, :function => Java::CascadingOperation::Identity.new
       check_scope :values_fields => ['offset', 'line']
     end
   end
 
-  it 'should match Cascading fields names after CoGroup' do
-    test_join_assembly do
-      # Pass that uses our scope instead of all_fields
-      operation = Java::CascadingOperation::Identity.new 
-      make_each(Java::CascadingPipe::Each, tail_pipe, scope.values_fields, operation)
-
-      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_']
-    end
-  end
-
-  it 'should match Cascading fields names after Every' do
-    test_join_assembly do
-      sum :mapping => {'x' => 'x_sum'}, :type => :int
-
-      # Pass that uses our grouping fields instead of all_fields
-      operation = Java::CascadingOperation::Identity.new 
-      make_each(Java::CascadingPipe::Each, tail_pipe, fields(['x', 'x_sum']), operation)
-
-      check_scope :values_fields => ['x', 'x_sum']
-    end
-  end
-
   it 'should pick up names from source tap scheme' do
     test_assembly do
       pass
@@ -65,14 +41,25 @@ context Cascading::Scope do
 
   it 'should propagate names through CoGroup' do
     test_join_assembly do
+      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
+        :grouping_fields => ['x', 'x_']
+    end
+  end
+
+  it 'should propagate names through CoGroup with no Aggregations' do
+    post_join_block = lambda do
+      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
+        :grouping_fields => ['x', 'x_']
     end
+
+    test_join_assembly(:post_join_block => post_join_block)
   end
 
   it 'should pass grouping fields to Every' do
     test_join_assembly do
       sum :mapping => {'x' => 'x_sum'}, :type => :int
       check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
-        :grouping_fields => ['x', 'x_sum']
+        :grouping_fields => ['x', 'x_', 'x_sum']
       assert_group_size_equals 1
     end
   end
@@ -81,88 +68,82 @@ context Cascading::Scope do
     test_join_assembly do
       sum :mapping => {'x' => 'x_sum'}, :type => :int
       check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
-        :grouping_fields => ['x', 'x_sum']
+        :grouping_fields => ['x', 'x_', 'x_sum']
       assert_group_size_equals 1
 
       sum :mapping => {'y' => 'y_sum'}, :type => :int
       check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
-        :grouping_fields => ['x', 'x_sum', 'y_sum']
+        :grouping_fields => ['x', 'x_', 'x_sum', 'y_sum']
       assert_group_size_equals 1
     end
   end
 
   it 'should propagate names through Every' do
-    test_join_assembly do
+    post_join_block = lambda do
+      check_scope :values_fields => ['x', 'x_', 'x_sum', 'y_sum']
+      assert_size_equals 4
+    end
+
+    test_join_assembly :post_join_block => post_join_block do
       sum :mapping => {'x' => 'x_sum'}, :type => :int
       check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
-        :grouping_fields => ['x', 'x_sum']
+        :grouping_fields => ['x', 'x_', 'x_sum']
       assert_group_size_equals 1
 
       sum :mapping => {'y' => 'y_sum'}, :type => :int
       check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
-        :grouping_fields => ['x', 'x_sum', 'y_sum']
+        :grouping_fields => ['x', 'x_', 'x_sum', 'y_sum']
       assert_group_size_equals 1
-
-      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
-        :grouping_fields => ['x', 'x_sum', 'y_sum']
-      assert_size_equals 3
-
-      check_scope :values_fields => ['x', 'x_sum', 'y_sum']
     end
   end
 
   it 'should pass values fields to Each immediately following CoGroup and remove grouping fields' do
-    test_join_assembly do
+    post_join_block = lambda do
       assert_size_equals 10
       check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_']
     end
+    test_join_assembly(:post_join_block => post_join_block)
   end
 
   it 'should fail to pass grouping fields to Every immediately following Each' do
+    post_join_block = lambda do
+      pass
+      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_']
+      sum :mapping => {'x' => 'x_sum'}, :type => :int
+    end
+
     lambda do # Composition fails
-      test_join_assembly do
-        pass
-        check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_']
-        begin
-          sum :mapping => {'x' => 'x_sum'}, :type => :int
-        rescue CascadingException => e
-          raise e.cause
-        end
-      end
-    end.should raise_error java.lang.IllegalStateException, 'Every cannot follow a Tap or an Each'
+      test_join_assembly(:post_join_block => post_join_block)
+    # sum doesn't exist outside of Aggregations (where block of join is
+    # evaluated)
+    end.should raise_error NoMethodError
   end
 
   it 'should propagate values fields and field names into branch' do
-    test_join_assembly(:branches => ['data_tuple']) do
+    post_join_block = lambda do
       branch 'data_tuple' do
         check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
-          :grouping_fields => ['x']
+          :grouping_fields => ['x', 'x_']
         assert_size_equals 10
       end
     end
+
+    test_join_assembly(:branches => ['data_tuple'], :post_join_block => post_join_block)
   end
 
-  it 'should fail to propagate grouping fields to branch' do
-    lambda do # Execution fails
-      begin
-        test_join_assembly(:branches => ['attempt_group']) do
-          branch 'attempt_group' do
-            check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
-              :grouping_fields => ['x']
-            sum :mapping => {'x' => 'x_sum'}, :type => :int
-          end
-        end
-      rescue CascadingException => e
-        raise e.cause
+  it 'should propagate names through GroupBy' do
+    test_assembly do
+      group_by 'line' do
+        count
       end
-    end.should raise_error Java::CascadingFlowPlanner::PlannerException, "[attempt_group][sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)] Every instances may not split after a GroupBy or CoGroup pipe, found: Every(attempt_group)[Sum[decl:'x_sum'][args:1]] after: CoGroup(left*right)[by:left:[{1}:'x']right:[{1}:'x']]"
+      check_scope :values_fields => ['line', 'count']
+    end
   end
 
-  it 'should propagate names through GroupBy' do
+  it 'should propagate names through blockless GroupBy' do
     test_assembly do
       group_by 'line'
-      check_scope :values_fields => ['offset', 'line'],
-        :grouping_fields => ['line']
+      check_scope :values_fields => ['offset', 'line'], :grouping_fields => ['line']
     end
   end
 end

data/spec/spec_util.rb

@@ -19,6 +19,7 @@ end
 module Cascading
   class Flow; include ScopeTests; end
   class Assembly; include ScopeTests; end
+  class Aggregations; include ScopeTests; end
 end
 
 def test_flow(&block)
@@ -50,6 +51,7 @@ end
 
 def test_join_assembly(params = {}, &block)
   branches = params[:branches] || []
+  post_join_block = params[:post_join_block]
 
   test_flow do
     source 'left', tap('spec/resource/join_input.txt', :kind => :lfs, :scheme => text_line_scheme)
@@ -75,11 +77,9 @@ def test_join_assembly(params = {}, &block)
       # Empty scope because there is no 'join' source or assembly
       check_scope :values_fields => []
 
-      left_join 'left', 'right', :on => ['x']
-      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
-        :grouping_fields => ['x']
+      left_join 'left', 'right', :on => ['x'], &block
 
-      instance_eval(&block)
+      instance_eval &post_join_block if post_join_block
     end
 
     sink 'join', tap("#{OUTPUT_DIR}/join_out.txt", :kind => :lfs, :sink_mode => :replace)

data/tags

@@ -4,11 +4,14 @@
 !_TAG_PROGRAM_NAME	Exuberant Ctags	//
 !_TAG_PROGRAM_URL	http://ctags.sourceforge.net	/official site/
 !_TAG_PROGRAM_VERSION	5.8	//
+Aggregations	lib/cascading/aggregations.rb	/^  class Aggregations$/;"	c	class:Cascading
+Aggregations	spec/spec_util.rb	/^  class Aggregations; include ScopeTests; end$/;"	c	class:Cascading
 Array	lib/cascading/ext/array.rb	/^class Array$/;"	c
 Assembly	lib/cascading/assembly.rb	/^  class Assembly < Cascading::Node$/;"	c	class:Cascading
 Assembly	spec/spec_util.rb	/^  class Assembly; include ScopeTests; end$/;"	c	class:Cascading
 Cascade	lib/cascading/cascade.rb	/^  class Cascade < Cascading::Node$/;"	c	class:Cascading
 Cascading	lib/cascading.rb	/^module Cascading$/;"	m
+Cascading	lib/cascading/aggregations.rb	/^module Cascading$/;"	m
 Cascading	lib/cascading/assembly.rb	/^module Cascading$/;"	m
 Cascading	lib/cascading/base.rb	/^module Cascading$/;"	m
 Cascading	lib/cascading/cascade.rb	/^module Cascading$/;"	m
@@ -18,7 +21,7 @@ Cascading	lib/cascading/expr_stub.rb	/^module Cascading$/;"	m
 Cascading	lib/cascading/flow.rb	/^module Cascading$/;"	m
 Cascading	lib/cascading/operations.rb	/^module Cascading$/;"	m
 Cascading	lib/cascading/scope.rb	/^module Cascading$/;"	m
-Cascading	samples/cascading.rb	/^module Cascading$/;"	m
+Cascading	lib/cascading/sub_assembly.rb	/^module Cascading$/;"	m
 Cascading	spec/spec_util.rb	/^module Cascading$/;"	m
 CascadingException	lib/cascading/cascading_exception.rb	/^  class CascadingException < StandardError$/;"	c	class:Cascading
 ExprArgException	lib/cascading/expr_stub.rb	/^  class ExprArgException < StandardError; end$/;"	c
@@ -34,12 +37,13 @@ Operations	lib/cascading/operations.rb	/^  module Operations$/;"	m	class:Cascadi
 Registerable	lib/cascading/base.rb	/^  module Registerable$/;"	m	class:Cascading
 Scope	lib/cascading/scope.rb	/^  class Scope$/;"	c	class:Cascading
 ScopeTests	spec/spec_util.rb	/^module ScopeTests$/;"	m
+SubAssembly	lib/cascading/sub_assembly.rb	/^  class SubAssembly$/;"	c	class:Cascading
 TC_Assembly	test/test_assembly.rb	/^class TC_Assembly < Test::Unit::TestCase$/;"	c
-TC_AssemblyScenarii	test/test_assembly.rb	/^class TC_AssemblyScenarii < Test::Unit::TestCase$/;"	c
 TC_Cascade	test/test_cascade.rb	/^class TC_Cascade < Test::Unit::TestCase$/;"	c
 TC_Cascading	test/test_cascading.rb	/^class TC_Cascading < Test::Unit::TestCase$/;"	c
 TC_Exceptions	test/test_exceptions.rb	/^class TC_Exceptions < Test::Unit::TestCase$/;"	c
 TC_Flow	test/test_flow.rb	/^class TC_Flow < Test::Unit::TestCase$/;"	c
+TC_LocalExecution	test/test_local_execution.rb	/^class TC_LocalExecution < Test::Unit::TestCase$/;"	c
 TC_Operations	test/test_operations.rb	/^class TC_Operations < Test::Unit::TestCase$/;"	c
 add	lib/cascading/base.rb	/^    def add(name, instance)$/;"	f	class:Cascading.Registerable
 add_archive_to_distributed_cache	lib/cascading/flow.rb	/^    def add_archive_to_distributed_cache(file)$/;"	f
@@ -50,37 +54,34 @@ add_to_distributed_cache	lib/cascading/flow.rb	/^    def add_to_distributed_cach
 aggregator_function	lib/cascading/operations.rb	/^    def aggregator_function(args, aggregator_klass)$/;"	f	class:Cascading.Operations
 all	lib/cascading/base.rb	/^    def all$/;"	f	class:Cascading.Registerable
 all_fields	lib/cascading/cascading.rb	/^  def all_fields$/;"	f	class:Cascading
+apply_aggregations	lib/cascading/assembly.rb	/^    def apply_aggregations(group, incoming_scopes, &block)$/;"	f	class:Cascading.Assembly
 assembly	lib/cascading/flow.rb	/^    def assembly(name, &block)$/;"	f	class:Cascading.Flow
-assembly	test/test_assembly.rb	/^def assembly(name, &block)$/;"	f
 assert	lib/cascading/assembly.rb	/^    def assert(*args)$/;"	f	class:Cascading.Assembly
-assert_group	lib/cascading/assembly.rb	/^    def assert_group(*args)$/;"	f	class:Cascading.Assembly
-assert_group_size_equals	lib/cascading/assembly.rb	/^    def assert_group_size_equals(*args)$/;"	f	class:Cascading.Assembly
+assert_group	lib/cascading/aggregations.rb	/^    def assert_group(*args)$/;"	f	class:Cascading.Aggregations
+assert_group_size_equals	lib/cascading/aggregations.rb	/^    def assert_group_size_equals(*args)$/;"	f	class:Cascading.Aggregations
 assert_not_null	lib/cascading/assembly.rb	/^    def assert_not_null(*args)$/;"	f	class:Cascading.Assembly
 assert_size_equals	lib/cascading/assembly.rb	/^    def assert_size_equals(*args)$/;"	f	class:Cascading.Assembly
-average	lib/cascading/assembly.rb	/^    def average(*args); composite_aggregator(args, :average_function); end$/;"	f	class:Cascading.Assembly
-average_function	lib/cascading/operations.rb	/^    def average_function(*args)$/;"	f	class:Cascading.Operations
-bind_names	lib/cascading/assembly.rb	/^    def bind_names(*new_names)$/;"	f	class:Cascading.Assembly
+average	lib/cascading/aggregations.rb	/^    def average(*args)$/;"	f	class:Cascading.Aggregations
 branch	lib/cascading/assembly.rb	/^    def branch(name, &block)$/;"	f	class:Cascading.Assembly
 cascade	lib/cascading/cascading.rb	/^  def cascade(name, &block)$/;"	f	class:Cascading
 cascading.jruby	src/cascading/jruby/Main.java	/^package cascading.jruby;$/;"	p
 cast	lib/cascading/assembly.rb	/^    def cast(type_map)$/;"	f	class:Cascading.Assembly
 cause	lib/cascading/cascading_exception.rb	/^    def cause(depth = @depth)$/;"	f	class:Cascading.CascadingException
 check_scope	spec/spec_util.rb	/^  def check_scope(params = {})$/;"	f	class:ScopeTests
-compare_with_references	test/test_assembly.rb	/^def compare_with_references(test_name)$/;"	f
 compile	lib/cascading/expr_stub.rb	/^    def compile$/;"	f	class:Cascading
 complete	lib/cascading/cascade.rb	/^    def complete(properties = nil)$/;"	f	class:Cascading.Cascade
 complete	lib/cascading/flow.rb	/^    def complete(properties = nil)$/;"	f
-composite_aggregator	lib/cascading/assembly.rb	/^    def composite_aggregator(args, function)$/;"	f	class:Cascading.Assembly
+composite_aggregator	lib/cascading/aggregations.rb	/^    def composite_aggregator(args, function)$/;"	f	class:Cascading.Aggregations
 compress_output	lib/cascading/flow.rb	/^    def compress_output(codec, type)$/;"	f	class:Cascading.Flow
 connect	lib/cascading/flow.rb	/^    def connect(properties = nil)$/;"	f
 copy	lib/cascading/assembly.rb	/^    def copy(*args)$/;"	f	class:Cascading.Assembly
 copy	lib/cascading/scope.rb	/^    def copy$/;"	f	class:Cascading.Scope
 copy_fields	lib/cascading/cascading.rb	/^  def copy_fields(fields)$/;"	f	class:Cascading
-count	lib/cascading/assembly.rb	/^    def count(*args)$/;"	f	class:Cascading.Assembly
-count_function	lib/cascading/operations.rb	/^    def count_function(*args)$/;"	f	class:Cascading.Operations
+count	lib/cascading/aggregations.rb	/^    def count(name = 'count')$/;"	f	class:Cascading.Aggregations
 date_formatter	lib/cascading/operations.rb	/^    def date_formatter(fields, format, timezone=nil)$/;"	f	class:Cascading.Operations.to_java_comparable_array
 date_parser	lib/cascading/operations.rb	/^    def date_parser(field, format)$/;"	f	class:Cascading.Operations.to_java_comparable_array
 debug	lib/cascading/assembly.rb	/^    def debug(*args)$/;"	f	class:Cascading.Assembly
+debug_scope	lib/cascading/aggregations.rb	/^    def debug_scope$/;"	f	class:Cascading.Aggregations
 debug_scope	lib/cascading/assembly.rb	/^    def debug_scope$/;"	f	class:Cascading.Assembly
 debug_scope	lib/cascading/flow.rb	/^    def debug_scope(name = nil)$/;"	f	class:Cascading.Flow
 dedup_field_names	lib/cascading/cascading.rb	/^  def dedup_field_names(*names)$/;"	f	class:Cascading
@@ -94,7 +95,6 @@ describe	lib/cascading/flow.rb	/^    def describe(offset = '')$/;"	f	class:Casca
 difference_fields	lib/cascading/cascading.rb	/^  def difference_fields(*fields)$/;"	f	class:Cascading
 discard	lib/cascading/assembly.rb	/^    def discard(*args)$/;"	f	class:Cascading.Assembly
 distinct	lib/cascading/assembly.rb	/^    def distinct(*args)$/;"	f	class:Cascading.Assembly
-do_every_block_and_rename_fields	lib/cascading/assembly.rb	/^    def do_every_block_and_rename_fields(group_fields, incoming_scopes, &block)$/;"	f	class:Cascading.Assembly
 draw	lib/cascading/cascade.rb	/^    def draw(dir, properties = nil)$/;"	f	class:Cascading.Cascade
 each	lib/cascading/assembly.rb	/^    def each(*args)$/;"	f	class:Cascading.Assembly
 empty_scope	lib/cascading/scope.rb	/^    def self.empty_scope(name)$/;"	F	class:Cascading.Scope
@@ -104,12 +104,12 @@ eval	lib/cascading/expr_stub.rb	/^    def eval(actual_args)$/;"	f	class:Cascadin
 eval_expression	lib/cascading/assembly.rb	/^    def eval_expression(*args)$/;"	f	class:Cascading.Assembly
 evaluate	lib/cascading/expr_stub.rb	/^    def evaluate(values)$/;"	f	class:Cascading
 evaluator	lib/cascading/expr_stub.rb	/^    def evaluator$/;"	f	class:Cascading
-every	lib/cascading/assembly.rb	/^    def every(*args)$/;"	f	class:Cascading.Assembly
-every_applied?	lib/cascading/assembly.rb	/^    def every_applied?$/;"	f	class:Cascading.Assembly
+every	lib/cascading/aggregations.rb	/^    def every(*args)$/;"	f	class:Cascading.Aggregations
 expr	lib/cascading/cascading.rb	/^  def expr(expression, params = {})$/;"	f	class:Cascading
 expr	lib/cascading/expr_stub.rb	/^    def self.expr(expression, params = {})$/;"	F	class:Cascading
 expression_filter	lib/cascading/operations.rb	/^    def expression_filter(*args)$/;"	f	class:Cascading.Operations.to_java_comparable_array
 expression_function	lib/cascading/operations.rb	/^    def expression_function(*args)$/;"	f	class:Cascading.Operations
+extract_field_map	lib/cascading/aggregations.rb	/^    def extract_field_map(args)$/;"	f	class:Cascading.Aggregations
 extract_options	lib/cascading/ext/array.rb	/^  def extract_options$/;"	f	class:Array
 extract_options!	lib/cascading/ext/array.rb	/^  def extract_options!$/;"	f	class:Array
 fetch_cause	lib/cascading/cascading_exception.rb	/^    def fetch_cause(ne, depth)$/;"	f	class:Cascading.CascadingException
@@ -118,8 +118,10 @@ fields	lib/cascading/cascading.rb	/^  def fields(fields)$/;"	f	class:Cascading
 filter	lib/cascading/assembly.rb	/^    def filter(*args)$/;"	f	class:Cascading.Assembly
 filter_not_null	lib/cascading/assembly.rb	/^    def filter_not_null(*args)$/;"	f	class:Cascading.Assembly
 filter_null	lib/cascading/assembly.rb	/^    def filter_null(*args)$/;"	f	class:Cascading.Assembly
+finalize	lib/cascading/aggregations.rb	/^    def finalize$/;"	f	class:Cascading.Aggregations
+finalize	lib/cascading/sub_assembly.rb	/^    def finalize$/;"	f	class:Cascading.SubAssembly
 find_child	lib/cascading/base.rb	/^    def find_child(name)$/;"	f	class:Cascading.Node
-first	lib/cascading/assembly.rb	/^    def first(*args); composite_aggregator(args, :first_function); end$/;"	f	class:Cascading.Assembly
+first	lib/cascading/aggregations.rb	/^    def first(*args); composite_aggregator(args, :first_function); end$/;"	f	class:Cascading.Aggregations
 first_function	lib/cascading/operations.rb	/^    def first_function(*args)$/;"	f	class:Cascading.Operations
 flow	lib/cascading/cascade.rb	/^    def flow(name, &block)$/;"	f	class:Cascading.Cascade
 flow	lib/cascading/cascading.rb	/^  def flow(name, &block)$/;"	f	class:Cascading
@@ -129,42 +131,46 @@ group_by	lib/cascading/assembly.rb	/^    def group_by(*args, &block)$/;"	f	class
 grouping_fields	lib/cascading/scope.rb	/^    def grouping_fields$/;"	f	class:Cascading.Scope
 identity	lib/cascading/operations.rb	/^    def identity$/;"	f	class:Cascading.Operations
 in_directory	tasks/setup.rb	/^def in_directory( dir, &block )$/;"	f
+initialize	lib/cascading/aggregations.rb	/^    def initialize(assembly, group, incoming_scopes)$/;"	f	class:Cascading.Aggregations
 initialize	lib/cascading/assembly.rb	/^    def initialize(name, parent, outgoing_scopes = {})$/;"	f	class:Cascading.Assembly
 initialize	lib/cascading/base.rb	/^    def initialize(name, parent)$/;"	f	class:Cascading.Node
 initialize	lib/cascading/cascade.rb	/^    def initialize(name)$/;"	f	class:Cascading.Cascade
 initialize	lib/cascading/cascading_exception.rb	/^    def initialize(native_exception, message)$/;"	f	class:Cascading.CascadingException
 initialize	lib/cascading/expr_stub.rb	/^    def initialize(expression)$/;"	f	class:Cascading.ExprStub
 initialize	lib/cascading/flow.rb	/^    def initialize(name, parent)$/;"	f	class:Cascading.Flow
-initialize	lib/cascading/scope.rb	/^    def initialize(scope, params = {})$/;"	f	class:Cascading.Scope
+initialize	lib/cascading/scope.rb	/^    def initialize(scope)$/;"	f	class:Cascading.Scope
+initialize	lib/cascading/sub_assembly.rb	/^    def initialize(assembly, sub_assembly)$/;"	f	class:Cascading.SubAssembly
 inner_join	lib/cascading/assembly.rb	/^    def inner_join(*args, &block)$/;"	f	class:Cascading.Assembly
 insert	lib/cascading/assembly.rb	/^    def insert(args)$/;"	f	class:Cascading.Assembly
 insert_function	lib/cascading/operations.rb	/^    def insert_function(*args)$/;"	f	class:Cascading.Operations
 join	lib/cascading/assembly.rb	/^    def join(*args, &block)$/;"	f	class:Cascading.Assembly
 join_fields	lib/cascading/assembly.rb	/^    def join_fields(*args)$/;"	f	class:Cascading.Assembly
-last	lib/cascading/assembly.rb	/^    def last(*args); composite_aggregator(args, :last_function); end$/;"	f	class:Cascading.Assembly
+last	lib/cascading/aggregations.rb	/^    def last(*args); composite_aggregator(args, :last_function); end$/;"	f	class:Cascading.Aggregations
 last_function	lib/cascading/operations.rb	/^    def last_function(*args)$/;"	f	class:Cascading.Operations
 last_grouping_fields	lib/cascading/cascading.rb	/^  def last_grouping_fields$/;"	f	class:Cascading
 left_join	lib/cascading/assembly.rb	/^    def left_join(*args, &block)$/;"	f	class:Cascading.Assembly
 libpath	lib/cascading.rb	/^  def self.libpath( *args )$/;"	F	class:Cascading
 local_properties	lib/cascading/cascading.rb	/^  def local_properties(base_dir)$/;"	f
 main	src/cascading/jruby/Main.java	/^    public static void main(String[] args) {$/;"	m	class:Main
-make_each	lib/cascading/assembly.rb	/^    def make_each(type, *parameters)$/;"	f	class:Cascading.Assembly
-make_every	lib/cascading/assembly.rb	/^    def make_every(type, *parameters)$/;"	f	class:Cascading.Assembly
 make_flows	lib/cascading/cascade.rb	/^    def make_flows(flows, properties)$/;"	f	class:Cascading.Cascade
-make_pipe	lib/cascading/assembly.rb	/^    def make_pipe(type, parameters, grouping_key_fields = [], incoming_scopes = [scope])$/;"	f	class:Cascading.Assembly
+make_pipe	lib/cascading/aggregations.rb	/^    def make_pipe(type, parameters)$/;"	f	class:Cascading.Aggregations
+make_pipe	lib/cascading/assembly.rb	/^    def make_pipe(type, parameters)$/;"	f	class:Cascading.Assembly
 make_pipes	lib/cascading/flow.rb	/^    def make_pipes$/;"	f
 make_tap_parameter	lib/cascading/flow.rb	/^    def make_tap_parameter(taps)$/;"	f
 manifest	tasks/setup.rb	/^def manifest$/;"	f
 match_rows	lib/cascading/assembly.rb	/^    def match_rows(*args)$/;"	f	class:Cascading.Assembly
-max	lib/cascading/assembly.rb	/^    def max(*args); composite_aggregator(args, :max_function); end$/;"	f	class:Cascading.Assembly
+max	lib/cascading/aggregations.rb	/^    def max(*args); composite_aggregator(args, :max_function); end$/;"	f	class:Cascading.Aggregations
 max_function	lib/cascading/operations.rb	/^    def max_function(*args)$/;"	f	class:Cascading.Operations
-min	lib/cascading/assembly.rb	/^    def min(*args); composite_aggregator(args, :min_function); end$/;"	f	class:Cascading.Assembly
+min	lib/cascading/aggregations.rb	/^    def min(*args); composite_aggregator(args, :min_function); end$/;"	f	class:Cascading.Aggregations
 min_function	lib/cascading/operations.rb	/^    def min_function(*args)$/;"	f	class:Cascading.Operations
 mock_assembly	test/test_assembly.rb	/^  def mock_assembly(&block)$/;"	f	class:TC_Assembly
-multi_tap	lib/cascading/cascading.rb	/^  def multi_tap(*taps)$/;"	f
+mock_branched_assembly	test/test_assembly.rb	/^  def mock_branched_assembly(&block)$/;"	f	class:TC_Assembly
+mock_two_input_assembly	test/test_assembly.rb	/^  def mock_two_input_assembly(&block)$/;"	f	class:TC_Assembly
+multi_sink_tap	lib/cascading/cascading.rb	/^  def multi_sink_tap(*taps)$/;"	f
+multi_source_tap	lib/cascading/cascading.rb	/^  def multi_source_tap(*taps)$/;"	f
 names_and_types	lib/cascading/expr_stub.rb	/^    def names_and_types$/;"	f	class:Cascading
 outer_join	lib/cascading/assembly.rb	/^    def outer_join(*args, &block)$/;"	f	class:Cascading.Assembly
-outgoing_scope	lib/cascading/scope.rb	/^    def self.outgoing_scope(flow_element, incoming_scopes, grouping_key_fields, every_applied)$/;"	F	class:Cascading.Scope
+outgoing_scope	lib/cascading/scope.rb	/^    def self.outgoing_scope(flow_element, incoming_scopes)$/;"	F	class:Cascading.Scope
 outgoing_scope_for	lib/cascading/scope.rb	/^    def self.outgoing_scope_for(flow_element, incoming_scopes)$/;"	F	class:Cascading.Scope
 paragraphs_of	tasks/setup.rb	/^def paragraphs_of( path, *paragraphs )$/;"	f
 parent_flow	lib/cascading/assembly.rb	/^    def parent_flow$/;"	f	class:Cascading.Assembly
@@ -172,6 +178,7 @@ parse	lib/cascading/assembly.rb	/^    def parse(*args)$/;"	f	class:Cascading.Ass
 parse_date	lib/cascading/assembly.rb	/^    def parse_date(*args)$/;"	f	class:Cascading.Assembly
 pass	lib/cascading/assembly.rb	/^    def pass(*args)$/;"	f	class:Cascading.Assembly
 path	lib/cascading.rb	/^  def self.path( *args )$/;"	F	class:Cascading
+path	lib/cascading/sub_assembly.rb	/^    def path(pipe, tail_pipes)$/;"	f	class:Cascading.SubAssembly
 project	lib/cascading/assembly.rb	/^    def project(*args)$/;"	f	class:Cascading.Assembly
 qualified_name	lib/cascading/base.rb	/^    def qualified_name$/;"	f	class:Cascading.Node
 quiet	tasks/setup.rb	/^def quiet( &block )$/;"	f
@@ -191,9 +198,9 @@ reset	lib/cascading/base.rb	/^    def reset$/;"	f	class:Cascading.Registerable
 results_fields	lib/cascading/cascading.rb	/^  def results_fields$/;"	f	class:Cascading
 right_join	lib/cascading/assembly.rb	/^    def right_join(*args, &block)$/;"	f	class:Cascading.Assembly
 root	lib/cascading/base.rb	/^    def root$/;"	f	class:Cascading.Node
-sample_properties	samples/cascading.rb	/^  def sample_properties$/;"	f	class:Cascading
 scope	lib/cascading/assembly.rb	/^    def scope$/;"	f	class:Cascading.Assembly
 scope	lib/cascading/flow.rb	/^    def scope(name = nil)$/;"	f	class:Cascading.Flow
+scope_fields_to_s	lib/cascading/scope.rb	/^    def scope_fields_to_s(accessor)$/;"	f	class:Cascading.Scope
 search_field_name	lib/cascading/cascading.rb	/^  def search_field_name(names, candidate)$/;"	f	class:Cascading
 sequence_file_scheme	lib/cascading/cascading.rb	/^  def sequence_file_scheme(*fields)$/;"	f
 set_spill_threshold	lib/cascading/flow.rb	/^    def set_spill_threshold(threshold)$/;"	f
@@ -205,11 +212,10 @@ specific_to_java	lib/cascading/expr_stub.rb	/^    def specific_to_java(value, ty
 split	lib/cascading/assembly.rb	/^    def split(*args)$/;"	f	class:Cascading.Assembly
 split_hash	lib/cascading/expr_stub.rb	/^    def split_hash(h)$/;"	f	class:Cascading
 split_rows	lib/cascading/assembly.rb	/^    def split_rows(*args)$/;"	f	class:Cascading.Assembly
-sum	lib/cascading/assembly.rb	/^    def sum(*args)$/;"	f	class:Cascading.Assembly
-sum_function	lib/cascading/operations.rb	/^    def sum_function(*args)$/;"	f	class:Cascading.Operations
+sub_assembly	lib/cascading/assembly.rb	/^    def sub_assembly(sub_assembly)$/;"	f	class:Cascading.Assembly
+sum	lib/cascading/aggregations.rb	/^    def sum(*args)$/;"	f	class:Cascading.Aggregations
 tap	lib/cascading/cascading.rb	/^  def tap(*args)$/;"	f
 tap_scope	lib/cascading/scope.rb	/^    def self.tap_scope(tap, name)$/;"	F	class:Cascading.Scope
-test_aggregator_function_ignore_exception	test/test_operations.rb	/^  def test_aggregator_function_ignore_exception$/;"	f	class:TC_Operations
 test_aggregator_function_ignore_tuples	test/test_operations.rb	/^  def test_aggregator_function_ignore_tuples$/;"	f	class:TC_Operations
 test_aggregator_function_ignore_values	test/test_operations.rb	/^  def test_aggregator_function_ignore_values$/;"	f	class:TC_Operations
 test_ambiguous_assembly_names	test/test_flow.rb	/^  def test_ambiguous_assembly_names$/;"	f	class:TC_Flow
@@ -221,28 +227,48 @@ test_branch_unique	test/test_assembly.rb	/^  def test_branch_unique$/;"	f
 test_cascade	test/test_cascade.rb	/^  def test_cascade$/;"	f	class:TC_Cascade
 test_cascading_exception	test/test_exceptions.rb	/^  def test_cascading_exception$/;"	f	class:TC_Exceptions
 test_create_assembly_simple	test/test_assembly.rb	/^  def test_create_assembly_simple$/;"	f	class:TC_Assembly
-test_create_each	test/test_assembly.rb	/^  def test_create_each$/;"	f
-test_create_every	test/test_assembly.rb	/^  def test_create_every$/;"	f
-test_create_group_by	test/test_assembly.rb	/^  def test_create_group_by$/;"	f
-test_create_group_by_many_fields	test/test_assembly.rb	/^  def test_create_group_by_many_fields$/;"	f
-test_create_group_by_reverse	test/test_assembly.rb	/^  def test_create_group_by_reverse$/;"	f
-test_create_group_by_with_sort	test/test_assembly.rb	/^  def test_create_group_by_with_sort$/;"	f
-test_create_group_by_with_sort_reverse	test/test_assembly.rb	/^  def test_create_group_by_with_sort_reverse$/;"	f
-test_each_identity	test/test_assembly.rb	/^  def test_each_identity$/;"	f
+test_create_each	test/test_assembly.rb	/^  def test_create_each$/;"	f	class:TC_Assembly
+test_create_every	test/test_assembly.rb	/^  def test_create_every$/;"	f	class:TC_Assembly
+test_create_group_by	test/test_assembly.rb	/^  def test_create_group_by$/;"	f	class:TC_Assembly
+test_create_group_by_many_fields	test/test_assembly.rb	/^  def test_create_group_by_many_fields$/;"	f	class:TC_Assembly
+test_create_group_by_reverse	test/test_assembly.rb	/^  def test_create_group_by_reverse$/;"	f	class:TC_Assembly
+test_create_group_by_with_sort	test/test_assembly.rb	/^  def test_create_group_by_with_sort$/;"	f	class:TC_Assembly
+test_create_group_by_with_sort_reverse	test/test_assembly.rb	/^  def test_create_group_by_with_sort_reverse$/;"	f	class:TC_Assembly
+test_create_join	test/test_assembly.rb	/^  def test_create_join$/;"	f
+test_create_join_many_fields	test/test_assembly.rb	/^  def test_create_join_many_fields$/;"	f
+test_create_join_with_declared_fields	test/test_assembly.rb	/^  def test_create_join_with_declared_fields$/;"	f
+test_create_union	test/test_assembly.rb	/^  def test_create_union$/;"	f	class:TC_Assembly
+test_create_union_many_fields	test/test_assembly.rb	/^  def test_create_union_many_fields$/;"	f	class:TC_Assembly
+test_create_union_reverse	test/test_assembly.rb	/^  def test_create_union_reverse$/;"	f	class:TC_Assembly
+test_create_union_with_sort	test/test_assembly.rb	/^  def test_create_union_with_sort$/;"	f	class:TC_Assembly
+test_create_union_with_sort_reverse	test/test_assembly.rb	/^  def test_create_union_with_sort_reverse$/;"	f	class:TC_Assembly
+test_each_identity	test/test_assembly.rb	/^  def test_each_identity$/;"	f	class:TC_Assembly
 test_empty_where	test/test_assembly.rb	/^  def test_empty_where$/;"	f
+test_every_cannot_follow_tap	test/test_assembly.rb	/^  def test_every_cannot_follow_tap$/;"	f	class:TC_Assembly
 test_fields_field	test/test_cascading.rb	/^  def test_fields_field$/;"	f	class:TC_Cascading
 test_fields_multiple	test/test_cascading.rb	/^  def test_fields_multiple$/;"	f	class:TC_Cascading
 test_fields_single	test/test_cascading.rb	/^  def test_fields_single$/;"	f	class:TC_Cascading
 test_flow	spec/spec_util.rb	/^def test_flow(&block)$/;"	f
 test_flow	test/test_flow.rb	/^  def test_flow$/;"	f	class:TC_Flow
-test_full_assembly	test/test_assembly.rb	/^  def test_full_assembly$/;"	f
-test_join1	test/test_assembly.rb	/^  def test_join1$/;"	f	class:TC_AssemblyScenarii
-test_join2	test/test_assembly.rb	/^  def test_join2$/;"	f
+test_group_by_with_block	test/test_assembly.rb	/^  def test_group_by_with_block$/;"	f	class:TC_Assembly
+test_join1	test/test_local_execution.rb	/^  def test_join1$/;"	f	class:TC_LocalExecution.test_smoke_test_multi_source_tap.test_smoke_test_sequence_file_scheme.test_splitter.test_smoke_test_multi_source_tap
+test_join2	test/test_local_execution.rb	/^  def test_join2$/;"	f	class:TC_LocalExecution.test_smoke_test_multi_source_tap.test_smoke_test_sequence_file_scheme.test_splitter.test_smoke_test_multi_source_tap.test_join1
 test_join_assembly	spec/spec_util.rb	/^def test_join_assembly(params = {}, &block)$/;"	f
+test_join_empty_on	test/test_assembly.rb	/^  def test_join_empty_on$/;"	f
+test_join_invalid_on	test/test_assembly.rb	/^  def test_join_invalid_on$/;"	f
+test_join_undefined_inputs	test/test_assembly.rb	/^  def test_join_undefined_inputs$/;"	f
+test_join_with_block	test/test_assembly.rb	/^  def test_join_with_block$/;"	f
+test_join_without_on	test/test_assembly.rb	/^  def test_join_without_on$/;"	f
 test_smoke_test_debug_scope	test/test_assembly.rb	/^  def test_smoke_test_debug_scope$/;"	f
-test_smoke_test_sequence_file_scheme	test/test_assembly.rb	/^  def test_smoke_test_sequence_file_scheme$/;"	f	class:TC_AssemblyScenarii
-test_splitter	test/test_assembly.rb	/^  def test_splitter$/;"	f	class:TC_AssemblyScenarii.test_smoke_test_sequence_file_scheme
+test_smoke_test_multi_source_tap	test/test_local_execution.rb	/^  def test_smoke_test_multi_source_tap$/;"	f	class:TC_LocalExecution
+test_smoke_test_multi_source_tap	test/test_local_execution.rb	/^  def test_smoke_test_multi_source_tap$/;"	f	class:TC_LocalExecution.test_smoke_test_multi_source_tap.test_smoke_test_sequence_file_scheme.test_splitter
+test_smoke_test_sequence_file_scheme	test/test_local_execution.rb	/^  def test_smoke_test_sequence_file_scheme$/;"	f	class:TC_LocalExecution.test_smoke_test_multi_source_tap
+test_splitter	test/test_local_execution.rb	/^  def test_splitter$/;"	f	class:TC_LocalExecution.test_smoke_test_multi_source_tap.test_smoke_test_sequence_file_scheme
+test_sub_assembly	test/test_assembly.rb	/^  def test_sub_assembly$/;"	f
 test_tap	test/test_cascading.rb	/^  def test_tap$/;"	f	class:TC_Cascading
+test_union	test/test_local_execution.rb	/^  def test_union$/;"	f	class:TC_LocalExecution.test_smoke_test_multi_source_tap.test_smoke_test_sequence_file_scheme.test_splitter.test_smoke_test_multi_source_tap.test_join1.test_join2
+test_union_undefined_inputs	test/test_assembly.rb	/^  def test_union_undefined_inputs$/;"	f	class:TC_Assembly
+test_union_with_block	test/test_assembly.rb	/^  def test_union_with_block$/;"	f	class:TC_Assembly
 test_values	lib/cascading/expr_stub.rb	/^    def test_values$/;"	f	class:Cascading
 test_where	test/test_assembly.rb	/^  def test_where$/;"	f
 test_where_with_expression	test/test_assembly.rb	/^  def test_where_with_expression$/;"	f
@@ -253,9 +279,8 @@ to_s	lib/cascading/assembly.rb	/^    def to_s$/;"	f	class:Cascading.Assembly
 to_s	lib/cascading/expr_stub.rb	/^    def to_s$/;"	f	class:Cascading
 to_s	lib/cascading/scope.rb	/^    def to_s$/;"	f	class:Cascading.Scope
 trace_causes	lib/cascading/cascading_exception.rb	/^    def trace_causes(ne, depth)$/;"	f	class:Cascading.CascadingException
-union	lib/cascading/assembly.rb	/^    def union(*args)$/;"	f	class:Cascading.Assembly
+union	lib/cascading/assembly.rb	/^    def union(*args, &block)$/;"	f	class:Cascading.Assembly
 union_fields	lib/cascading/cascading.rb	/^  def union_fields(*fields)$/;"	f	class:Cascading
-union_pipes	lib/cascading/assembly.rb	/^    def union_pipes(*args)$/;"	f	class:Cascading.Assembly
 valid?	tasks/setup.rb	/^  def valid?$/;"	f	class:Object
 validate	lib/cascading/expr_stub.rb	/^    def validate(actual_args = {})$/;"	f	class:Cascading
 validate_fields	lib/cascading/expr_stub.rb	/^    def validate_fields(fields)$/;"	f	class:Cascading