cascading.jruby 0.0.7 → 0.0.8

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+  - jruby-18mode
+notifications:
+  recipients:
+    - mwalker@etsy.com

data/Gemfile

@@ -0,0 +1,6 @@
+source :rubygems
+
+group :test do
+  gem 'rake', '0.8.7'
+  gem 'rspec', '1.1.11'
+end

data/Gemfile.lock

@@ -0,0 +1,12 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    rake (0.8.7)
+    rspec (1.1.11)
+
+PLATFORMS
+  java
+
+DEPENDENCIES
+  rake (= 0.8.7)
+  rspec (= 1.1.11)

data/HACKING.md

@@ -2,13 +2,21 @@
 
 Some hacking info on `cascading.jruby`:
 
-`cascading.jruby` can be packaged as a gem. To do so, you must generate the necessary packaging files:
+For local development, install with (requires [bundler](http://gembundler.com/)):
 
-    jruby -S rake gem
+    bundle install
 
-will produce the gem in the pkg/ sub-directory. After that, just cd to this directory and:
+To run the tests (will download Cascading and Hadoop jars):
 
-    jruby -S rake install cascading.jruby-xxx.gem
+    jruby -S bundle exec rake
+
+To create the gem:
+
+    jruby -S bundle exec rake gem
+
+To install it locally:
+
+    jruby -S gem install pkg/cascading.jruby-xxx.gem
 
 The `Cascading::Operations` module is mixed-in the `Cascading::Assembly` class to provide some shortcuts for common operations.
 

data/History.txt

@@ -1,3 +1,11 @@
+0.0.8 - AggregateBy and upgrade to Cascading 2.0.0 wip-286
+
+This release upgrades to Cascading 2.0.0 wip-286, but again does not implement
+local mode.  It does, however, replace group_by/union which contain only count,
+sum, and average with the corresponding composite AggregateBy.  Various
+refactorings were required to support this, including the addition of limited
+support for SubAssemblies.
+
 0.0.7 - Upgrade to Cascading 2.0.0 wip-255
 
 This release upgrades to Cascading 2.0.0 wip-255, but implements neither local

data/README.md

@@ -1,8 +1,8 @@
-# Cascading.JRuby
+# Cascading.JRuby [![Build Status](https://secure.travis-ci.org/mrwalker/cascading.jruby.png)](http://travis-ci.org/mrwalker/cascading.jruby)
 
 `cascading.jruby` is a small DSL above [Cascading](http://www.cascading.org/).
 
-It requires Hadoop (>= 0.20.2) and Cascading (>= 2.0.0) to be set via the environment variables: `HADOOP_HOME` and `CASCADING_HOME`
+It requires Hadoop (>= 0.20.2) and [Cascading 2.0.0-wip-286](http://files.concurrentinc.com/cascading/2.0/cascading-2.0.0-wip-286-hadoop-0.20.2%2B.tgz) to be set via the environment variables: `HADOOP_HOME` and `CASCADING_HOME`
 
 It has been tested on JRuby versions 1.2.0, 1.4.0, 1.5.3, and 1.6.5.
 

data/Rakefile

@@ -18,8 +18,9 @@ end
 ensure_in_path 'lib'
 
 require 'cascading'
+require 'bundler/setup'
 
-task :default => 'test:run'
+task :default => 'test'
 
 task :run do
   # ensure_in_path "samples"
@@ -28,7 +29,7 @@ task :run do
 end
 
 desc 'Remove gem and Java build files'
-task :clean => ['ant:clean', 'gem:clean'] do
+task :clean => ['ant:clean', 'gem:clean', 'samples:clean'] do
   puts 'Build files removed'
 end
 

data/TODO

@@ -1,13 +1,13 @@
 Documentation
 
-Assembly -> Each/Every refactor
-Look into totally eliminating registries
+Import local_job/Job from Etsy, which can eliminate registries
 
 Bug fixes on github
 Enforce more runtime rules at composition time
 Standardize helper contracts
 Possibly combine unit tests...into unit tests because RSpec sucks and swallows stack traces
 
+(Jading)
 Split out runner
 Make runner implement Tool
 Create build tool for job jar

data/ivy.xml

@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="ISO-8859-1"?>
+<ivy-module version="2.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+       xsi:noNamespaceSchemaLocation="http://ant.apache.org/ivy/schemas/ivy.xsd">
+  <info organisation="com.etsy" module="cascading.jruby" status="integration" />
+ 
+  <configurations>
+    <conf name="default" visibility="public" description="runtime dependencies and master artifact can be used with this conf" extends="runtime,master" />
+    <conf name="master" visibility="public" description="contains only the artifact published by this module itself, with no transitive dependencies" />
+    <conf name="compile" visibility="public" description="this is the default scope, used if none is specified. Compile dependencies are available in all classpaths." />
+    <conf name="provided" visibility="public" description="this is much like compile, but indicates you expect the JDK or a container to provide it. It is only available on the compilation classpath, and is not transitive." />
+    <conf name="runtime" visibility="public" description="this scope indicates that the dependency is not required for compilation, but is for execution. It is in the runtime and test classpaths, but not the compile classpath." extends="compile" />
+    <conf name="test" visibility="private" description="this scope indicates that the dependency is not required for normal use of the application, and is only available for the test compilation and execution phases." extends="runtime" />
+    <conf name="system" visibility="public" description="this scope is similar to provided except that you have to provide the JAR which contains it explicitly. The artifact is always available and is not looked up in a repository." />
+    <conf name="sources" visibility="public" description="this configuration contains the source artifact of this module, if any." />
+    <conf name="javadoc" visibility="public" description="this configuration contains the javadoc artifact of this module, if any." />
+    <conf name="optional" visibility="public" description="contains all optional dependencies" />
+  </configurations>
+
+  <dependencies>
+    <dependency org="cascading" name="cascading-core" rev="2.0.0-wip-286" conf="default" />
+    <dependency org="cascading" name="cascading-local" rev="2.0.0-wip-286" conf="default" />
+    <dependency org="cascading" name="cascading-hadoop" rev="2.0.0-wip-286" conf="default" />
+    <dependency org="org.jruby" name="jruby" rev="1.6.5" conf="default" />
+  </dependencies>
+</ivy-module>

data/ivysettings.xml

@@ -0,0 +1,7 @@
+<ivysettings>
+  <include url="http://conjars.org/repo/ivysettings.xml"/>
+
+  <resolvers>
+    <ibiblio name="public" m2compatible="true"/>
+  </resolvers>
+</ivysettings>

data/lib/cascading.rb

@@ -6,7 +6,7 @@ require 'java'
 
 module Cascading
   # :stopdoc:
-  VERSION = '0.0.7'
+  VERSION = '0.0.8'
   LIBPATH = ::File.expand_path(::File.dirname(__FILE__)) + ::File::SEPARATOR
   PATH = ::File.dirname(LIBPATH) + ::File::SEPARATOR
   CASCADING_HOME = ENV['CASCADING_HOME']

data/lib/cascading/aggregations.rb

@@ -0,0 +1,190 @@
+require 'cascading/operations'
+require 'cascading/scope'
+require 'cascading/ext/array'
+
+module Cascading
+  # Rules enforced by Aggregations:
+  #   Contains either 1 Buffer or >= 1 Aggregator (explicitly checked)
+  #   No GroupBys, CoGroups, Joins, or Merges (methods for these pipes do not exist on Aggregations)
+  #   No Eaches (Aggregations#each does not exist)
+  #   Aggregations may not branch (Aggregations#branch does not exist)
+  #
+  # Externally enforced rules:
+  #   May be empty (in which case, Aggregations is not instantiated)
+  #   Must follow a GroupBy or CoGroup (not a Join or Merge)
+  #
+  # Optimizations:
+  #   If the leading Group is a GroupBy and all subsequent Everies are
+  #   Aggregators that have a corresponding AggregateBy, Aggregations can
+  #   replace the GroupBy/Aggregator pipe with a single composite AggregateBy.
+  class Aggregations
+    include Operations
+
+    attr_reader :assembly, :tail_pipe, :scope, :aggregate_bys
+
+    def initialize(assembly, group, incoming_scopes)
+      @assembly = assembly
+      @tail_pipe = group
+      @scope = Scope.outgoing_scope(tail_pipe, incoming_scopes)
+
+      # AggregateBy optimization only applies to GroupBy
+      @aggregate_bys = tail_pipe.is_group_by ? [] : nil
+    end
+
+    def debug_scope
+      puts "Current scope of aggregations for '#{assembly.name}':\n  #{scope}\n----------\n"
+    end
+
+    def make_pipe(type, parameters)
+      pipe = type.new(*parameters)
+
+      # Enforce 1 Buffer or >= 1 Aggregator rule
+      if tail_pipe.kind_of?(Java::CascadingPipe::Every)
+        raise 'Buffer must be sole aggregation' if tail_pipe.buffer? || (tail_pipe.aggregator? && pipe.buffer?)
+      end
+
+      @tail_pipe = pipe
+      @scope = Scope.outgoing_scope(tail_pipe, [scope])
+    end
+    private :make_pipe
+
+    # We can replace these aggregations with the corresponding composite
+    # AggregateBy if the leading Group was a GroupBy and all subsequent
+    # Aggregators had a corresponding AggregateBy (which we've encoded in the
+    # list of aggregate_bys being a non-empty array).
+    def can_aggregate_by?
+      !aggregate_bys.nil? && !aggregate_bys.empty?
+    end
+
+    # "Fix" out values fields after a sequence of Everies.  This is a field
+    # name metadata fix which is why the Identity is not planned into the
+    # resulting Cascading pipe.  Without it, all values fields would propagate
+    # through non-empty aggregations, which doesn't match Cascading's planner's
+    # behavior.
+    def finalize
+      discard_each = Java::CascadingPipe::Each.new(tail_pipe, all_fields, Java::CascadingOperation::Identity.new)
+      @scope = Scope.outgoing_scope(discard_each, [scope])
+    end
+
+    # Builds an every pipe and adds it to the current list of aggregations.
+    # Note that this list may be either exactly 1 Buffer or any number of
+    # Aggregators.
+    def every(*args)
+      options = args.extract_options!
+
+      in_fields = fields(args)
+      out_fields = fields(options[:output])
+      operation = options[:aggregator] || options[:buffer]
+
+      if options[:aggregate_by] && aggregate_bys
+        aggregate_bys << options[:aggregate_by]
+      else
+        @aggregate_bys = nil
+      end
+
+      parameters = [tail_pipe, in_fields, operation, out_fields].compact
+      make_pipe(Java::CascadingPipe::Every, parameters)
+    end
+
+    def assert_group(*args)
+      options = args.extract_options!
+
+      assertion = args[0]
+      assertion_level = options[:level] || Java::CascadingOperation::AssertionLevel::STRICT
+
+      parameters = [tail_pipe, assertion_level, assertion]
+      make_pipe(Java::CascadingPipe::Every, parameters)
+    end
+
+    def assert_group_size_equals(*args)
+      options = args.extract_options!
+
+      assertion = Java::CascadingOperationAssertion::AssertGroupSizeEquals.new(args[0])
+      assert_group(assertion, options)
+    end
+
+    # Builds a series of every pipes for aggregation.
+    #
+    # Args can either be a list of fields to aggregate and an options hash or
+    # a hash that maps input field name to output field name (similar to
+    # insert) and an options hash.
+    #
+    # Options include:
+    #   * <tt>:ignore</tt> a Java Array of Objects (for min and max) or Tuples
+    #     (for first and last) of values for the aggregator to ignore
+    #
+    # <tt>function</tt> is a symbol that is the method to call to construct the Cascading Aggregator.
+    def composite_aggregator(args, function)
+      field_map, options = extract_field_map(args)
+
+      field_map.each do |in_field, out_field|
+        agg = self.send(function, out_field, options)
+        every(in_field, :aggregator => agg, :output => all_fields)
+      end
+      raise "Composite aggregator '#{function.to_s.gsub('_function', '')}' invoked on 0 fields" if field_map.empty?
+    end
+
+    def min(*args); composite_aggregator(args, :min_function); end
+    def max(*args); composite_aggregator(args, :max_function); end
+    def first(*args); composite_aggregator(args, :first_function); end
+    def last(*args); composite_aggregator(args, :last_function); end
+
+    # Counts elements of a group.  May optionally specify the name of the
+    # output count field (defaults to 'count').
+    def count(name = 'count')
+      count_aggregator = Java::CascadingOperationAggregator::Count.new(fields(name))
+      count_by = Java::CascadingPipeAssembly::CountBy.new(fields(name))
+      every(last_grouping_fields, :aggregator => count_aggregator, :output => all_fields, :aggregate_by => count_by)
+    end
+
+    # Sums one or more fields.  Fields to be summed may either be provided as
+    # the arguments to sum (in which case they will be aggregated into a field
+    # of the same name in the given order), or via a hash using the :mapping
+    # parameter (in which case they will be aggregated from the field named by
+    # the key into the field named by the value after being sorted).  The type
+    # of the output sum may be controlled with the :type parameter.
+    def sum(*args)
+      options = args.extract_options!
+      type = JAVA_TYPE_MAP[options[:type]]
+
+      mapping = options[:mapping] ? options[:mapping].sort : args.zip(args)
+      mapping.each do |in_field, out_field|
+        sum_aggregator = Java::CascadingOperationAggregator::Sum.new(*[fields(out_field), type].compact)
+        # NOTE: SumBy requires a type in wip-286, unlike Sum (see Sum.java line 42 for default)
+        sum_by = Java::CascadingPipeAssembly::SumBy.new(fields(in_field), fields(out_field), type || Java::double.java_class)
+        every(in_field, :aggregator => sum_aggregator, :output => all_fields, :aggregate_by => sum_by)
+      end
+      raise "sum invoked on 0 fields (note :mapping must be provided to explicitly rename fields)" if mapping.empty?
+    end
+
+    # Averages one or more fields.  The contract of average is identical to
+    # that of other composite aggregators, but it accepts no options.
+    def average(*args)
+      field_map, _ = extract_field_map(args)
+
+      field_map.each do |in_field, out_field|
+        average_aggregator = Java::CascadingOperationAggregator::Average.new(fields(out_field))
+        average_by = Java::CascadingPipeAssembly::AverageBy.new(fields(in_field), fields(out_field))
+        every(in_field, :aggregator => average_aggregator, :output => all_fields, :aggregate_by => average_by)
+      end
+      raise "average invoked on 0 fields" if field_map.empty?
+    end
+
+    private
+
+    # Extracts a field mapping, input field => output field, by accepting a
+    # hash in the first argument.  If no hash is provided, then maps arguments
+    # onto themselves which names outputs the same as inputs.  Additionally
+    # extracts options from args.
+    def extract_field_map(args)
+      if !args.empty? && args.first.kind_of?(Hash)
+        field_map = args.shift.sort
+        options = args.extract_options!
+      else
+        options = args.extract_options!
+        field_map = args.zip(args)
+      end
+      [field_map, options]
+    end
+  end
+end

data/lib/cascading/assembly.rb

@@ -1,21 +1,23 @@
+# -*- coding: utf-8 -*-
 # Copyright 2009, Grégoire Marabout. All Rights Reserved.
 #
 # This is free software. Please see the LICENSE and COPYING files for details.
 
 require 'cascading/base'
 require 'cascading/operations'
+require 'cascading/aggregations'
+require 'cascading/sub_assembly'
 require 'cascading/ext/array'
 
 module Cascading
   class Assembly < Cascading::Node
     include Operations
 
-    attr_accessor :head_pipe, :tail_pipe, :incoming_scopes, :outgoing_scopes
+    attr_reader :head_pipe, :tail_pipe
 
     def initialize(name, parent, outgoing_scopes = {})
       super(name, parent)
 
-      @every_applied = false
       @outgoing_scopes = outgoing_scopes
       if parent.kind_of?(Assembly)
         @head_pipe = Java::CascadingPipe::Pipe.new(name, parent.tail_pipe)
@@ -26,13 +28,13 @@ module Cascading
         @head_pipe = Java::CascadingPipe::Pipe.new(name)
         @outgoing_scopes[name] ||= Scope.empty_scope(name)
       end
-      @tail_pipe = @head_pipe
+      @tail_pipe = head_pipe
       @incoming_scopes = [scope]
     end
 
     def describe(offset = '')
-      incoming_scopes_desc = "#{incoming_scopes.map{ |incoming_scope| incoming_scope.values_fields.to_a.inspect }.join(', ')}"
-      incoming_scopes_desc = "(#{incoming_scopes_desc})" unless incoming_scopes.size == 1
+      incoming_scopes_desc = "#{@incoming_scopes.map{ |incoming_scope| incoming_scope.values_fields.to_a.inspect }.join(', ')}"
+      incoming_scopes_desc = "(#{incoming_scopes_desc})" unless @incoming_scopes.size == 1
       description =  "#{offset}#{name}:assembly :: #{incoming_scopes_desc} -> #{scope.values_fields.to_a.inspect}"
       description += "\n#{child_names.map{ |child| children[child].describe("#{offset}  ") }.join("\n")}" unless children.empty?
       description
@@ -51,108 +53,122 @@ module Cascading
       puts "Current scope for '#{name}':\n  #{scope}\n----------\n"
     end
 
-    def make_each(type, *parameters)
-      make_pipe(type, parameters)
-      @every_applied = false
-    end
+    def make_pipe(type, parameters)
+      @tail_pipe = type.new(*parameters)
+      @outgoing_scopes[name] = Scope.outgoing_scope(tail_pipe, [scope])
 
-    def make_every(type, *parameters)
-      make_pipe(type, parameters, scope.grouping_key_fields)
-      @every_applied = true
+      tail_pipe
     end
+    private :make_pipe
 
-    def every_applied?
-      @every_applied
+    def populate_incoming_scopes(assembly_names, group_fields_args = {})
+      # NOTE: this overrides the existing incoming_scopes, which changes the
+      # way describe will function on this assembly
+      pipes, @incoming_scopes, group_fields = [], [], []
+      assembly_names.each do |assembly_name|
+        assembly = parent_flow.find_child(assembly_name)
+        raise "Could not find assembly '#{assembly_name}' from '#{name}'" unless assembly
+
+        pipes << assembly.tail_pipe
+        @incoming_scopes << assembly.scope
+        group_fields << fields(group_fields_args[assembly_name]) if group_fields_args[assembly_name]
+      end
+      [pipes, group_fields]
     end
+    private :populate_incoming_scopes
 
-    def do_every_block_and_rename_fields(group_fields, incoming_scopes, &block)
-      return unless block
+    def apply_aggregations(group, incoming_scopes, &block)
+      aggregations = Aggregations.new(self, group, incoming_scopes)
+      aggregations.instance_eval(&block) if block_given?
 
-      # TODO: this should really be instance evaled on an object
-      # that only allows aggregation and buffer operations.
-      instance_eval &block
+      # Sorting of any type means that we cannot use the AggregateBy optimization
+      if aggregations.can_aggregate_by? && !group.is_sorted && !group.is_sort_reversed
+        grouping_fields = group.key_selectors.values.first
+        group.key_selectors.values.each do |key_fields|
+          raise "Grouping fields mismatch: #{grouping_fields} expected; #{key_fields} found from #{group.key_selectors}" unless key_fields == grouping_fields
+        end
 
-      bind_names scope.grouping_fields.to_a if every_applied?
-    end
+        aggregate_by = sub_assembly(Java::CascadingPipeAssembly::AggregateBy.new(
+          name,
+          group.previous,
+          grouping_fields,
+          aggregations.aggregate_bys.to_java(Java::CascadingPipeAssembly::AggregateBy)
+        ), group.previous, incoming_scopes)
 
-    def make_pipe(type, parameters, grouping_key_fields = [], incoming_scopes = [scope])
-      @tail_pipe = type.new(*parameters)
-      @outgoing_scopes[name] = Scope.outgoing_scope(@tail_pipe, incoming_scopes, grouping_key_fields, every_applied?)
+        aggregate_by
+      else
+        aggregations.finalize if block_given?
+        @tail_pipe = aggregations.tail_pipe
+        @outgoing_scopes[name] = aggregations.scope
+
+        group
+      end
     end
+    private :apply_aggregations
 
     def to_s
-      "#{name} : head pipe : #{@head_pipe} - tail pipe: #{@tail_pipe}"
+      "#{name} : head pipe : #{head_pipe} - tail pipe: #{tail_pipe}"
     end
 
-    # Builds a join (CoGroup) pipe. Requires a list of assembly names to join.
+    # Builds a join (CoGroup) pipe. Requires a list of assembly names to join
+    # and :on to specify the group_fields.
     def join(*args, &block)
       options = args.extract_options!
 
-      pipes, @incoming_scopes = [], []
-      args.each do |assembly_name|
-        assembly = parent_flow.find_child(assembly_name)
-        raise "Could not find assembly '#{assembly_name}' in join" unless assembly
+      pipes, _ = populate_incoming_scopes(args)
 
-        pipes << assembly.tail_pipe
-        incoming_scopes << @outgoing_scopes[assembly.name]
-      end
+      group_fields_args = options[:on]
+      raise 'join requires :on parameter' unless group_fields_args
 
-      group_fields_args = options.delete(:on)
       if group_fields_args.kind_of?(String)
         group_fields_args = [group_fields_args]
       end
-      group_fields_names = group_fields_args.to_a
+
       group_fields = []
       if group_fields_args.kind_of?(Array)
         pipes.size.times do
           group_fields << fields(group_fields_args)
         end
       elsif group_fields_args.kind_of?(Hash)
-        pipes, @incoming_scopes = [], []
-        keys = group_fields_args.keys.sort
-        keys.each do |assembly_name|
-          v = group_fields_args[assembly_name]
-          assembly = parent_flow.find_child(assembly_name)
-          raise "Could not find assembly '#{assembly_name}' in join" unless assembly
-
-          pipes << assembly.tail_pipe
-          incoming_scopes << @outgoing_scopes[assembly.name]
-          group_fields << fields(v)
-          group_fields_names = group_fields_args[keys.first].to_a
-        end
+        pipes, group_fields = populate_incoming_scopes(group_fields_args.keys.sort, group_fields_args)
+      else
+        raise "Unsupported data type for :on in join: '#{group_fields_args.class}'"
       end
 
+      raise 'join requires non-empty :on parameter' if group_fields_args.empty?
       group_fields = group_fields.to_java(Java::CascadingTuple::Fields)
-      incoming_fields = incoming_scopes.map{ |s| s.values_fields }
+      incoming_fields = @incoming_scopes.map{ |s| s.values_fields }
       declared_fields = fields(options[:declared_fields] || dedup_fields(*incoming_fields))
-      joiner = options.delete(:joiner)
-
-      if declared_fields
-        case joiner
-        when :inner, "inner", nil
-          joiner = Java::CascadingPipeJoiner::InnerJoin.new
-        when :left,  "left"
-          joiner = Java::CascadingPipeJoiner::LeftJoin.new
-        when :right, "right"
-          joiner = Java::CascadingPipeJoiner::RightJoin.new
-        when :outer, "outer"
-          joiner = Java::CascadingPipeJoiner::OuterJoin.new
-        when Array
-          joiner = joiner.map do |t|
-            case t
-            when true,  1, :inner then true
-            when false, 0, :outer then false
-            else fail "invalid mixed joiner entry: #{t}"
-            end
+      joiner = options[:joiner]
+
+      case joiner
+      when :inner, 'inner', nil
+        joiner = Java::CascadingPipeJoiner::InnerJoin.new
+      when :left,  'left'
+        joiner = Java::CascadingPipeJoiner::LeftJoin.new
+      when :right, 'right'
+        joiner = Java::CascadingPipeJoiner::RightJoin.new
+      when :outer, 'outer'
+        joiner = Java::CascadingPipeJoiner::OuterJoin.new
+      when Array
+        joiner = joiner.map do |t|
+          case t
+          when true,  1, :inner then true
+          when false, 0, :outer then false
+          else fail "invalid mixed joiner entry: #{t}"
           end
-          joiner = Java::CascadingPipeJoiner::MixedJoin.new(joiner.to_java(:boolean))
         end
+        joiner = Java::CascadingPipeJoiner::MixedJoin.new(joiner.to_java(:boolean))
       end
-
-      parameters = [pipes.to_java(Java::CascadingPipe::Pipe), group_fields, declared_fields, joiner].compact
-      grouping_key_fields = group_fields[0] # Left key group wins
-      make_pipe(Java::CascadingPipe::CoGroup, parameters, grouping_key_fields, incoming_scopes)
-      do_every_block_and_rename_fields(group_fields_names, incoming_scopes, &block)
+      result_group_fields = dedup_fields(*group_fields)
+      parameters = [
+        pipes.to_java(Java::CascadingPipe::Pipe),
+        group_fields,
+        declared_fields,
+        result_group_fields,
+        joiner
+      ]
+      apply_aggregations(Java::CascadingPipe::CoGroup.new(*parameters), @incoming_scopes, &block)
     end
     alias co_group join
 
@@ -193,66 +209,76 @@ module Cascading
       assembly
     end
 
-    # Builds a new _group_by_ pipe. The fields used for grouping are specified in the args
-    # array.
+    # Builds a new GroupBy pipe that groups on the fields given in args.
+    # Any block passed to this method should contain only Everies.
     def group_by(*args, &block)
       options = args.extract_options!
-
       group_fields = fields(args)
-
-      sort_fields = fields(options[:sort_by] || args)
+      sort_fields = fields(options[:sort_by])
       reverse = options[:reverse]
 
-      parameters = [@tail_pipe, group_fields, sort_fields, reverse].compact
-      make_pipe(Java::CascadingPipe::GroupBy, parameters, group_fields)
-      do_every_block_and_rename_fields(args, [scope], &block)
+      parameters = [tail_pipe, group_fields, sort_fields, reverse].compact
+      apply_aggregations(Java::CascadingPipe::GroupBy.new(*parameters), [scope], &block)
     end
 
-    # Unifies several pipes sharing the same field structure.
-    # This actually creates a GroupBy pipe.
-    # It expects a list of assembly names as parameter.
-    def union_pipes(*args)
-      pipes, @incoming_scopes = [], []
-      args[0].each do |assembly_name|
-        assembly = parent_flow.find_child(assembly_name)
-        pipes << assembly.tail_pipe
-        incoming_scopes << @outgoing_scopes[assembly.name]
-      end
+    # Unifies multiple incoming pipes sharing the same field structure using a
+    # GroupBy.  Accepts :on like join and :sort_by and :reverse like group_by,
+    # as well as a block which may be used for a sequence of Every
+    # aggregations.
+    #
+    # By default, groups only on the first field (see line 189 of GroupBy.java)
+    def union(*args, &block)
+      options = args.extract_options!
+      group_fields = fields(options[:on])
+      sort_fields = fields(options[:sort_by])
+      reverse = options[:reverse]
 
-      # Groups only on the 1st field (see line 186 of GroupBy.java)
-      grouping_key_fields = fields(incoming_scopes.first.values_fields.get(0))
-      make_pipe(Java::CascadingPipe::GroupBy, [pipes.to_java(Java::CascadingPipe::Pipe)], grouping_key_fields, incoming_scopes)
-      # TODO: Shouldn't union_pipes accept an every block?
-      #do_every_block_and_rename_fields(args, incoming_scopes, &block)
+      pipes, _ = populate_incoming_scopes(args)
+
+      # Must provide group_fields to ensure field name propagation
+      group_fields = fields(@incoming_scopes.first.values_fields.get(0)) unless group_fields
+
+      # FIXME: GroupBy is missing a constructor for union in wip-255
+      sort_fields = group_fields if !sort_fields && !reverse.nil?
+
+      parameters = [pipes.to_java(Java::CascadingPipe::Pipe), group_fields, sort_fields, reverse].compact
+      apply_aggregations(Java::CascadingPipe::GroupBy.new(*parameters), @incoming_scopes, &block)
     end
+    alias :union_pipes :union
 
-    # Builds an basic _every_ pipe, and adds it to the current assembly.
-    def every(*args)
-      options = args.extract_options!
+    # Allows you to plugin c.p.SubAssemblies to a cascading.jruby Assembly
+    # under certain assumptions.  Note the default is to extend the tail pipe
+    # of this Assembly using a linear SubAssembly.  See SubAssembly class for
+    # details.
+    def sub_assembly(sub_assembly, pipes = [tail_pipe], incoming_scopes = [scope])
+      sub_assembly = SubAssembly.new(self, sub_assembly)
+      sub_assembly.finalize(pipes, incoming_scopes)
 
-      in_fields = fields(args)
-      out_fields = fields(options[:output])
-      operation = options[:aggregator] || options[:buffer]
+      @tail_pipe = sub_assembly.tail_pipe
+      @outgoing_scopes[name] = sub_assembly.scope
 
-      parameters = [@tail_pipe, in_fields, operation, out_fields].compact
-      make_every(Java::CascadingPipe::Every, *parameters)
+      sub_assembly
     end
 
     # Builds a basic _each_ pipe, and adds it to the current assembly.
     # --
     # Example:
-    #     each "line", :filter=>regex_splitter(["name", "val1", "val2", "id"],
-    #                  :pattern => /[.,]*\s+/),
-    #                  :output=>["id", "name", "val1", "val2"]
+    #     each 'line', :function => regex_splitter(['name', 'val1', 'val2', 'id'], :pattern => /[.,]*\s+/), :output => ['id', 'name', 'val1', 'val2']
     def each(*args)
       options = args.extract_options!
 
       in_fields = fields(args)
       out_fields = fields(options[:output])
+
       operation = options[:filter] || options[:function]
+      raise 'c.p.Each does not support applying an output selector to a c.o.Filter' if options[:filter] && options[:output]
+
+      parameters = [tail_pipe, in_fields, operation, out_fields].compact
+      each = make_pipe(Java::CascadingPipe::Each, parameters)
+      raise ':function specified but c.o.Filter provided' if options[:function] && each.is_filter
+      raise ':filter specified but c.o.Function provided' if options[:filter] && each.is_function
 
-      parameters = [@tail_pipe, in_fields, operation, out_fields].compact
-      make_each(Java::CascadingPipe::Each, *parameters)
+      each
     end
 
     # Restricts the current assembly to the specified fields.
@@ -273,14 +299,6 @@ module Cascading
       project(*keep_fields.to_a)
     end
 
-    # Assign new names to initial fields in positional order.
-    # --
-    # Example:
-    #     bind_names "field1", "field2"
-    def bind_names(*new_names)
-      each all_fields, :function => Java::CascadingOperation::Identity.new(fields(new_names))
-    end
-
     # Renames fields according to the mapping provided.
     # --
     # Example:
@@ -318,14 +336,9 @@ module Cascading
       options = args.extract_options!
       assertion = args[0]
       assertion_level = options[:level] || Java::CascadingOperation::AssertionLevel::STRICT
-      make_each(Java::CascadingPipe::Each, @tail_pipe, assertion_level, assertion)
-    end
 
-    def assert_group(*args)
-      options = args.extract_options!
-      assertion = args[0]
-      assertion_level = options[:level] || Java::CascadingOperation::AssertionLevel::STRICT
-      make_every(Java::CascadingPipe::Every, @tail_pipe, assertion_level, assertion)
+      parameters = [tail_pipe, assertion_level, assertion]
+      make_pipe(Java::CascadingPipe::Each, parameters)
     end
 
     # Builds a debugging pipe.
@@ -362,67 +375,6 @@ module Cascading
       assert(assertion, options)
     end
 
-    def assert_group_size_equals(*args)
-      options = args.extract_options!
-      assertion = Java::CascadingOperationAssertion::AssertGroupSizeEquals.new(args[0])
-      assert_group(assertion, options)
-    end
-
-    # Builds a series of every pipes for aggregation.
-    #
-    # Args can either be a list of fields to aggregate and an options hash or
-    # a hash that maps input field name to output field name (similar to
-    # insert) and an options hash.
-    #
-    # Options include:
-    #   * <tt>:ignore</tt> a Java Array of Objects (for min and max) or Tuples
-    #     (for first and last) of values for the aggregator to ignore
-    #
-    # <tt>function</tt> is a symbol that is the method to call to construct the Cascading Aggregator.
-    def composite_aggregator(args, function)
-      if !args.empty? && args.first.kind_of?(Hash)
-        field_map = args.shift.sort
-        options = args.extract_options!
-      else
-        options = args.extract_options!
-        field_map = args.zip(args)
-      end
-      field_map.each do |in_field, out_field|
-        agg = self.send(function, out_field, options)
-        every(in_field, :aggregator => agg, :output => all_fields)
-      end
-      puts "WARNING: composite aggregator '#{function.to_s.gsub('_function', '')}' invoked on 0 fields; will be ignored" if field_map.empty?
-    end
-
-    def min(*args); composite_aggregator(args, :min_function); end
-    def max(*args); composite_aggregator(args, :max_function); end
-    def first(*args); composite_aggregator(args, :first_function); end
-    def last(*args); composite_aggregator(args, :last_function); end
-    def average(*args); composite_aggregator(args, :average_function); end
-
-    # Counts elements of a group.  First unnamed parameter is the name of the
-    # output count field (defaults to 'count' if it is not provided).
-    def count(*args)
-      options = args.extract_options!
-      name = args[0] || 'count'
-      every(last_grouping_fields, :aggregator => count_function(name, options), :output => all_fields)
-    end
-
-    # Fields to be summed may either be provided as an array, in which case
-    # they will be aggregated into the same field in the given order, or as a
-    # hash, in which case they will be aggregated from the field named by the
-    # key into the field named by the value after being sorted.
-    def sum(*args)
-      options = args.extract_options!
-      type = JAVA_TYPE_MAP[options[:type]]
-      raise "No type specified for sum" unless type
-
-      mapping = options[:mapping] ? options[:mapping].sort : args.zip(args)
-      mapping.each do |in_field, out_field|
-        every(in_field, :aggregator => sum_function(out_field, :type => type), :output => all_fields)
-      end
-    end
-
     # Builds a _parse_ pipe. This pipe will parse the fields specified in input (first unamed arguments),
     # using a specified regex pattern.
     #
@@ -437,7 +389,7 @@ module Cascading
         fields = args || all_fields
         pattern = options[:pattern]
         output = options[:output] || all_fields
-        each(fields, :filter => regex_parser(pattern, options), :output => output)
+        each(fields, :function => regex_parser(pattern, options), :output => output)
     end
 
     # Builds a pipe that splits a field into other fields, using a specified regular expression.
@@ -692,14 +644,6 @@ module Cascading
       pass
     end
 
-    # Builds a pipe that will unify (merge) pipes. The method accepts the list of pipes as argument.
-    # Tuples unified must share the same fields.
-    def union(*args)
-      options = args.extract_options!
-      pipes = args
-      union_pipes pipes
-    end
-
     def join_fields(*args)
       options = args.extract_options!
       output = options[:output] || all_fields