cascading.jruby 0.0.10 → 1.0.0

data/lib/cascading/assembly.rb

@@ -1,15 +1,50 @@
 require 'cascading/base'
 require 'cascading/operations'
+require 'cascading/identity_operations'
+require 'cascading/filter_operations'
+require 'cascading/regex_operations'
+require 'cascading/text_operations'
 require 'cascading/aggregations'
 require 'cascading/sub_assembly'
 require 'cascading/ext/array'
 
 module Cascading
+  # An Assembly is a sequence of Cascading pipes (Each, GroupBy, CoGroup,
+  # Every, and SubAssembly).  This class will serve as your primary mechanism
+  # for doing work within a flow and contains all the functions and filters you
+  # will apply to a pipe (Eaches), as well as group_by, union, and join.  For
+  # aggregators and buffers, please see Aggregations.
+  #
+  # Function and filter DSL rules:
+  # * Use positional arguments for required parameters
+  # * Use options = {} for optional parameters
+  # * Use *args sparingly, specifically when you need to accept a varying length list of fields
+  # * If you require both a varying length list of fields and optional parameters, then see the Array#extract_options! extension
+  # * If you choose to name a required parameter, add it to options = {} and throw an exception if the caller does not provide it
+  # * If you have a require parameter that is provided by one of a set of options names, throw an exception if the caller does not provide at least one value (see :function and :filter in Assembly#each for an example)
+  #
+  # Function and filter DSL standard optional parameter names:
+  # [input] c.p.Each argument selector
+  # [into] c.o.Operation field declaration
+  # [output] c.p.Each output selector
+  #
+  # A note on aliases: when a DSL method uniquely wraps a single Cascading
+  # operation, we attempt to provide an alias that matches the Cascading
+  # operation.  However, Cascading operations are often nouns rather than verbs,
+  # and the latter are preferable for a dataflow DSL.
   class Assembly < Cascading::Node
-    include Operations
-
     attr_reader :head_pipe, :tail_pipe
 
+    # Do not use this constructor directly; instead, use Flow#assembly or
+    # Assembly#branch to build assemblies.
+    #
+    # Builds an Assembly given a name, parent, and optional outgoing_scopes
+    # (necessary only for branching).
+    #
+    # An assembly's name is quite important as it will determine:
+    # * The sources from which it will read, if any
+    # * The name to be used in joins or unions downstream
+    # * The name to be used to sink the output of the assembly downstream
     def initialize(name, parent, outgoing_scopes = {})
       super(name, parent)
 
@@ -27,6 +62,11 @@ module Cascading
       @incoming_scopes = [scope]
     end
 
+    # Produces a textual description of this Assembly.  The description details
+    # the structure of the Assembly, its input and output fields and any
+    # children (branches).  The offset parameter allows for this describe to be
+    # nested within a calling context, which lets us indent the structural
+    # hierarchy of a job.
     def describe(offset = '')
       incoming_scopes_desc = "#{@incoming_scopes.map{ |incoming_scope| incoming_scope.values_fields.to_a.inspect }.join(', ')}"
       incoming_scopes_desc = "(#{incoming_scopes_desc})" unless @incoming_scopes.size == 1
@@ -35,199 +75,231 @@ module Cascading
       description
     end
 
+    # Rather than the immediate parent, this method returns the parent flow of
+    # this Assembly.  If this is a branch, we must traverse the parents of
+    # parent assemblies.
     def parent_flow
       return parent if parent.kind_of?(Flow)
       parent.parent_flow
     end
 
+    # Accesses the outgoing scope of this Assembly at the point at which it is
+    # called.  This is useful for grabbing the values_fields at any point in
+    # the construction of the Assembly.  See Scope for details.
     def scope
       @outgoing_scopes[name]
     end
 
+    # Prints information about the scope of this Assembly at the point at which
+    # it is called.  This allows you to trace the propagation of field names
+    # through your job and is handy for debugging.  See Scope for details.
     def debug_scope
       puts "Current scope for '#{name}':\n  #{scope}\n----------\n"
     end
 
-    def make_pipe(type, parameters)
-      @tail_pipe = type.new(*parameters)
-      @outgoing_scopes[name] = Scope.outgoing_scope(tail_pipe, [scope])
-
-      tail_pipe
-    end
-    private :make_pipe
-
-    def populate_incoming_scopes(assembly_names, group_fields_args = {})
-      # NOTE: this overrides the existing incoming_scopes, which changes the
-      # way describe will function on this assembly
-      pipes, @incoming_scopes, group_fields = [], [], []
-      assembly_names.each do |assembly_name|
-        assembly = parent_flow.find_child(assembly_name)
-        raise "Could not find assembly '#{assembly_name}' from '#{name}'" unless assembly
-
-        pipes << assembly.tail_pipe
-        @incoming_scopes << assembly.scope
-        group_fields << fields(group_fields_args[assembly_name]) if group_fields_args[assembly_name]
-      end
-      [pipes, group_fields]
-    end
-    private :populate_incoming_scopes
-
-    def apply_aggregations(group, incoming_scopes, &block)
-      aggregations = Aggregations.new(self, group, incoming_scopes)
-      aggregations.instance_eval(&block) if block_given?
-
-      # Sorting of any type means that we cannot use the AggregateBy optimization
-      if aggregations.can_aggregate_by? && !group.is_sorted && !group.is_sort_reversed
-        grouping_fields = group.key_selectors.values.first
-        group.key_selectors.values.each do |key_fields|
-          raise "Grouping fields mismatch: #{grouping_fields} expected; #{key_fields} found from #{group.key_selectors}" unless key_fields == grouping_fields
-        end
-
-        aggregate_by = sub_assembly(Java::CascadingPipeAssembly::AggregateBy.new(
-          name,
-          group.previous,
-          grouping_fields,
-          aggregations.aggregate_bys.to_java(Java::CascadingPipeAssembly::AggregateBy)
-        ), group.previous, incoming_scopes)
-
-        aggregate_by
-      else
-        aggregations.finalize if block_given?
-        @tail_pipe = aggregations.tail_pipe
-        @outgoing_scopes[name] = aggregations.scope
-
-        group
-      end
-    end
-    private :apply_aggregations
-
+    # Prints detail about this Assembly including its name, head pipe, and tail
+    # pipe.
     def to_s
       "#{name} : head pipe : #{head_pipe} - tail pipe: #{tail_pipe}"
     end
 
-    def prepare_join(*args, &block)
-      options = args.extract_options!
-
-      pipes, _ = populate_incoming_scopes(args)
-
-      group_fields_args = options[:on]
-      raise 'join requires :on parameter' unless group_fields_args
-
-      if group_fields_args.kind_of?(String)
-        group_fields_args = [group_fields_args]
-      end
-
-      group_fields = []
-      if group_fields_args.kind_of?(Array)
-        pipes.size.times do
-          group_fields << fields(group_fields_args)
-        end
-      elsif group_fields_args.kind_of?(Hash)
-        pipes, group_fields = populate_incoming_scopes(group_fields_args.keys.sort, group_fields_args)
-      else
-        raise "Unsupported data type for :on in join: '#{group_fields_args.class}'"
-      end
-
-      raise 'join requires non-empty :on parameter' if group_fields_args.empty?
-      group_fields = group_fields.to_java(Java::CascadingTuple::Fields)
-      incoming_fields = @incoming_scopes.map{ |s| s.values_fields }
-      declared_fields = fields(options[:declared_fields] || dedup_fields(*incoming_fields))
-      joiner = options[:joiner]
-      is_hash_join = options[:hash] || false
-
-      case joiner
-      when :inner, 'inner', nil
-        joiner = Java::CascadingPipeJoiner::InnerJoin.new
-      when :left,  'left'
-        joiner = Java::CascadingPipeJoiner::LeftJoin.new
-      when :right, 'right'
-        joiner = Java::CascadingPipeJoiner::RightJoin.new
-      when :outer, 'outer'
-        joiner = Java::CascadingPipeJoiner::OuterJoin.new
-      when Array
-        joiner = joiner.map do |t|
-          case t
-          when true,  1, :inner then true
-          when false, 0, :outer then false
-          else fail "invalid mixed joiner entry: #{t}"
-          end
-        end
-        joiner = Java::CascadingPipeJoiner::MixedJoin.new(joiner.to_java(:boolean))
-      end
-
-      if is_hash_join
-        raise ArgumentError, "hash joins don't support aggregations" if block_given?
-        parameters = [
-          pipes.to_java(Java::CascadingPipe::Pipe),
-          group_fields,
-          declared_fields,
-          joiner
-        ]
-        group_assembly = Java::CascadingPipe::HashJoin.new(*parameters)
-      else
-        result_group_fields = dedup_fields(*group_fields)
-        parameters = [
-          pipes.to_java(Java::CascadingPipe::Pipe),
-          group_fields,
-          declared_fields,
-          result_group_fields,
-          joiner
-        ]
-        group_assembly = Java::CascadingPipe::CoGroup.new(*parameters)
-      end
-      apply_aggregations(group_assembly, @incoming_scopes, &block)
-    end
-    private :prepare_join
-
     # Builds a HashJoin pipe. This should be used carefully, as the right side
-    # of the join is accumulated entirely in memory. Requires a list of assembly
-    # names to join and :on to specify the join_fields.
-    def hash_join(*args, &block)
-      options = args.extract_options!
+    # of the join is accumulated entirely in memory. Requires a list of
+    # assembly names to join and :on to specify the join_fields.  Note that a
+    # hash_join "takes over" the Assembly in which it is built, so it is
+    # typically the first statement within the block of the assembly or branch.
+    # Additionally, a hash join does not accept a block for aggregations like
+    # other joins; this restriction is enforced here, but comes directly from
+    # Cascading.
+    #
+    # The named options are:
+    # [on] The keys of the join, an array of strings if they are the same in
+    #      all inputs, or a hash mapping assembly names to key names if they
+    #      differ across inputs.
+    # [declared_fields] By default, a deduplicated array of incoming field
+    #                   names (see Cascading::dedup_fields).  Specifies the
+    #                   names of the fields that will be available to
+    #                   aggregations or post-join if no aggregations are
+    #                   specified.
+    # [joiner] A specification of the c.p.j.Joiner to use.  Values like :inner
+    #          and 'inner', :right and 'right' are accepted, as well as an
+    #          array specifying mixed joins.  Typically, this is not provided,
+    #          but one of the higher level join methods on Assembly is used
+    #          directly (like Assembly#inner_join or Assembly#right_join).
+    #
+    # Example:
+    #     assembly 'join_left_right' do
+    #       hash_join 'left', 'right', :on => ['key1', 'key2'], :joiner => :inner
+    #     end
+    def hash_join(*args_with_options)
+      raise ArgumentError, "HashJoin doesn't support aggregations so the block provided to hash_join will be ignored" if block_given?
+
+      options, assembly_names = args_with_options.extract_options!, args_with_options
       options[:hash] = true
-      args << options
-      prepare_join(*args, &block)
+      prepare_join(assembly_names, options)
     end
 
     # Builds a join (CoGroup) pipe. Requires a list of assembly names to join
-    # and :on to specify the group_fields.
-    def join(*args, &block)
-      options = args.extract_options!
+    # and :on to specify the group_fields.  Note that a join "takes over" the
+    # Assembly in which it is built, so it is typically the first statement
+    # within the block of the assembly or branch.  The block passed to this
+    # method will be evaluated in the context of Aggregations, not Assembly.
+    #
+    # The named options are:
+    # [on] The keys of the join, an array of strings if they are the same in
+    #      all inputs, or a hash mapping assembly names to key names if they
+    #      differ across inputs.
+    # [declared_fields] By default, a deduplicated array of incoming field
+    #                   names (see Cascading::dedup_fields).  Specifies the
+    #                   names of the fields that will be available to
+    #                   aggregations or post-join if no aggregations are
+    #                   specified.
+    # [joiner] A specification of the c.p.j.Joiner to use.  Values like :inner
+    #          and 'inner', :right and 'right' are accepted, as well as an
+    #          array specifying mixed joins.  Typically, this is not provided,
+    #          but one of the higher level join methods on Assembly is used
+    #          directly (like Assembly#inner_join or Assembly#right_join).
+    #
+    # Example:
+    #     assembly 'join_left_right' do
+    #       join 'left', 'right', :on => ['key1', 'key2'], :joiner => :inner do
+    #         sum 'val1', 'val2', :type => :long
+    #       end
+    #     end
+    def join(*args_with_options, &block)
+      options, assembly_names = args_with_options.extract_options!, args_with_options
       options[:hash] = false
-      args << options
-      prepare_join(*args, &block)
+      prepare_join(assembly_names, options, &block)
     end
     alias co_group join
 
-    def inner_join(*args, &block)
-      options = args.extract_options!
+    # Builds an inner join (CoGroup) pipe. Requires a list of assembly names to
+    # join and :on to specify the group_fields.
+    #
+    # The named options are:
+    # [on] The keys of the join, an array of strings if they are the same in
+    #      all inputs, or a hash mapping assembly names to key names if they
+    #      differ across inputs.
+    # [declared_fields] By default, a deduplicated array of incoming field
+    #                   names (see Cascading::dedup_fields).  Specifies the
+    #                   names of the fields that will be available to
+    #                   aggregations or post-join if no aggregations are
+    #                   specified.
+    #
+    # Example:
+    #     assembly 'join_left_right' do
+    #       inner_join 'left', 'right', :on => ['key1', 'key2']
+    #         sum 'val1', 'val2', :type => :long
+    #       end
+    #     end
+    def inner_join(*args_with_options, &block)
+      options = args_with_options.extract_options!
       options[:joiner] = :inner
-      args << options
-      join(*args, &block)
+      args_with_options << options
+      join(*args_with_options, &block)
     end
 
-    def left_join(*args, &block)
-      options = args.extract_options!
+    # Builds a left join (CoGroup) pipe. Requires a list of assembly names to
+    # join and :on to specify the group_fields.
+    #
+    # The named options are:
+    # [on] The keys of the join, an array of strings if they are the same in
+    #      all inputs, or a hash mapping assembly names to key names if they
+    #      differ across inputs.
+    # [declared_fields] By default, a deduplicated array of incoming field
+    #                   names (see Cascading::dedup_fields).  Specifies the
+    #                   names of the fields that will be available to
+    #                   aggregations or post-join if no aggregations are
+    #                   specified.
+    #
+    # Example:
+    #     assembly 'join_left_right' do
+    #       left_join 'left', 'right', :on => ['key1', 'key2'] do
+    #         sum 'val1', 'val2', :type => :long
+    #       end
+    #     end
+    def left_join(*args_with_options, &block)
+      options = args_with_options.extract_options!
       options[:joiner] = :left
-      args << options
-      join(*args, &block)
+      args_with_options << options
+      join(*args_with_options, &block)
     end
 
-    def right_join(*args, &block)
-      options = args.extract_options!
+    # Builds a right join (CoGroup) pipe. Requires a list of assembly names to
+    # join and :on to specify the group_fields.
+    #
+    # The named options are:
+    # [on] The keys of the join, an array of strings if they are the same in
+    #      all inputs, or a hash mapping assembly names to key names if they
+    #      differ across inputs.
+    # [declared_fields] By default, a deduplicated array of incoming field
+    #                   names (see Cascading::dedup_fields).  Specifies the
+    #                   names of the fields that will be available to
+    #                   aggregations or post-join if no aggregations are
+    #                   specified.
+    #
+    # Example:
+    #     assembly 'join_left_right' do
+    #       right_join 'left', 'right', :on => ['key1', 'key2'] do
+    #         sum 'val1', 'val2', :type => :long
+    #       end
+    #     end
+    def right_join(*args_with_options, &block)
+      options = args_with_options.extract_options!
       options[:joiner] = :right
-      args << options
-      join(*args, &block)
+      args_with_options << options
+      join(*args_with_options, &block)
     end
 
-    def outer_join(*args, &block)
-      options = args.extract_options!
+    # Builds an outer join (CoGroup) pipe. Requires a list of assembly names to
+    # join and :on to specify the group_fields.
+    #
+    # The named options are:
+    # [on] The keys of the join, an array of strings if they are the same in
+    #      all inputs, or a hash mapping assembly names to key names if they
+    #      differ across inputs.
+    # [declared_fields] By default, a deduplicated array of incoming field
+    #                   names (see Cascading::dedup_fields).  Specifies the
+    #                   names of the fields that will be available to
+    #                   aggregations or post-join if no aggregations are
+    #                   specified.
+    #
+    # Example:
+    #     assembly 'join_left_right' do
+    #       outer_join 'left', 'right', :on => ['key1', 'key2'] do
+    #         sum 'val1', 'val2', :type => :long
+    #       end
+    #     end
+    def outer_join(*args_with_options, &block)
+      options = args_with_options.extract_options!
       options[:joiner] = :outer
-      args << options
-      join(*args, &block)
+      args_with_options << options
+      join(*args_with_options, &block)
     end
 
-    # Builds a new branch.
+    # Builds a child Assembly that branches this Assembly given a name and
+    # block.
+    #
+    # An assembly's name is quite important as it will determine:
+    # * The sources from which it will read, if any
+    # * The name to be used in joins or unions downstream
+    # * The name to be used to sink the output of the assembly downstream
+    #
+    # Many branches may be built within an assembly.  The result of a branch is
+    # the same as the Flow#assembly constructor, an Assembly object.
+    #
+    # Example:
+    #     assembly 'some_work' do
+    #       ...
+    #
+    #       branch 'more_work' do
+    #         ...
+    #       end
+    #
+    #       branch 'yet_more_work' do
+    #         ...
+    #       end
+    #     end
     def branch(name, &block)
       raise "Could not build branch '#{name}'; block required" unless block_given?
       assembly = Assembly.new(name, self, @outgoing_scopes)
@@ -236,11 +308,27 @@ module Cascading
       assembly
     end
 
-    # Builds a new GroupBy pipe that groups on the fields given in args.
-    # Any block passed to this method should contain only Everies.
-    def group_by(*args, &block)
-      options = args.extract_options!
-      group_fields = fields(args)
+    # Builds a new GroupBy pipe that groups on the fields given in
+    # args_with_options. The block passed to this method will be evaluated in
+    # the context of Aggregations, not Assembly.
+    #
+    # The named options are:
+    # [sort_by] Optional keys for within-group sort.
+    # [reverse] Boolean that can reverse the order of within-group sorting
+    #           (only makes sense given :sort_by keys).
+    #
+    # Example:
+    #     assembly 'total' do
+    #       ...
+    #       insert 'const' => 1
+    #       group_by 'const' do
+    #         count
+    #         sum 'val1', 'val2', :type => :long
+    #       end
+    #       discard 'const'
+    #     end
+    def group_by(*args_with_options, &block)
+      options, group_fields = args_with_options.extract_options!, fields(args_with_options)
       sort_fields = fields(options[:sort_by])
       reverse = options[:reverse]
 
@@ -251,16 +339,31 @@ module Cascading
     # Unifies multiple incoming pipes sharing the same field structure using a
     # GroupBy.  Accepts :on like join and :sort_by and :reverse like group_by,
     # as well as a block which may be used for a sequence of Every
-    # aggregations.
+    # aggregations.  The block passed to this method will be evaluated in the
+    # context of Aggregations, not Assembly.
     #
     # By default, groups only on the first field (see line 189 of GroupBy.java)
-    def union(*args, &block)
-      options = args.extract_options!
+    #
+    # The named options are:
+    # [on] The keys of the union, which defaults to the first field in the
+    #      first input assembly.
+    # [sort_by] Optional keys for sorting.
+    # [reverse] Boolean that can reverse the order of sorting
+    #           (only makes sense given :sort_by keys).
+    #
+    # Example:
+    #     assembly 'union_left_right' do
+    #       union 'left', 'right' do
+    #         sum 'val1', 'val2', :type => :long
+    #       end
+    #     end
+    def union(*args_with_options, &block)
+      options, assembly_names = args_with_options.extract_options!, args_with_options
       group_fields = fields(options[:on])
       sort_fields = fields(options[:sort_by])
       reverse = options[:reverse]
 
-      pipes, _ = populate_incoming_scopes(args)
+      pipes, _ = populate_incoming_scopes(assembly_names)
 
       # Must provide group_fields to ensure field name propagation
       group_fields = fields(@incoming_scopes.first.values_fields.get(0)) unless group_fields
@@ -273,10 +376,15 @@ module Cascading
     end
     alias :union_pipes :union
 
-    # Allows you to plugin c.p.SubAssemblies to a cascading.jruby Assembly
-    # under certain assumptions.  Note the default is to extend the tail pipe
-    # of this Assembly using a linear SubAssembly.  See SubAssembly class for
-    # details.
+    # Allows you to plugin c.p.SubAssemblies to an Assembly under certain
+    # assumptions.  Note the default is to extend the tail pipe of this
+    # Assembly using a linear SubAssembly.  See SubAssembly class for details.
+    #
+    # Example:
+    #     assembly 'id_rows' do
+    #       ...
+    #       sub_assembly Java::CascadingPipeAssembly::Discard.new(tail_pipe, fields('id'))
+    #     end
     def sub_assembly(sub_assembly, pipes = [tail_pipe], incoming_scopes = [scope])
       sub_assembly = SubAssembly.new(self, sub_assembly)
       sub_assembly.finalize(pipes, incoming_scopes)
@@ -287,17 +395,24 @@ module Cascading
       sub_assembly
     end
 
-    # Builds a basic _each_ pipe, and adds it to the current assembly.
-    # --
+    # Builds a basic each pipe and adds it to the current Assembly.  Default
+    # arguments are all_fields, a default inherited from c.o.Each.  Exactly one
+    # of :function and :filter must be specified and filters do not support an
+    # :output selector.
+    #
+    # The named options are:
+    # [filter] A Cascading Filter, mutually exclusive with :function.
+    # [function] A Cascading Function, mutually exclusive with :filter.
+    # [output] c.p.Each output selector, only valid with :function.
+    #
     # Example:
-    #     each 'line', :function => regex_splitter(['name', 'val1', 'val2', 'id'], :pattern => /[.,]*\s+/), :output => ['id', 'name', 'val1', 'val2']
-    def each(*args)
-      options = args.extract_options!
-
-      in_fields = fields(args)
-      out_fields = fields(options[:output])
-
+    #    each fields(input_fields), :function => Java::CascadingOperation::Identity.new
+    #    each 'field1', 'field2', :function => Java::CascadingOperation::Identity.new
+    def each(*args_with_options)
+      options, in_fields = args_with_options.extract_options!, fields(args_with_options)
+      out_fields = fields(options[:output]) # Default Fields.RESULTS from c.o.Each
       operation = options[:filter] || options[:function]
+      raise 'each requires either :filter or :function' unless operation
       raise 'c.p.Each does not support applying an output selector to a c.o.Filter' if options[:filter] && options[:output]
 
       parameters = [tail_pipe, in_fields, operation, out_fields].compact
@@ -308,468 +423,156 @@ module Cascading
       each
     end
 
-    # Restricts the current assembly to the specified fields.
-    # --
-    # Example:
-    #     project "field1", "field2"
-    def project(*args)
-      each fields(args), :function => Java::CascadingOperation::Identity.new
-    end
-
-    # Removes the specified fields from the current assembly.
-    # --
-    # Example:
-    #     discard "field1", "field2"
-    def discard(*args)
-      discard_fields = fields(args)
-      keep_fields = difference_fields(scope.values_fields, discard_fields)
-      project(*keep_fields.to_a)
-    end
-
-    # Renames fields according to the mapping provided.
-    # --
-    # Example:
-    #     rename "old_name" => "new_name"
-    def rename(name_map)
-      old_names = scope.values_fields.to_a
-      new_names = old_names.map{ |name| name_map[name] || name }
-      invalid = name_map.keys.sort - old_names
-      raise "invalid names: #{invalid.inspect}" unless invalid.empty?
-
-      each all_fields, :function => Java::CascadingOperation::Identity.new(fields(new_names))
-    end
-
-    def cast(type_map)
-      names = type_map.keys.sort
-      types = JAVA_TYPE_MAP.values_at(*type_map.values_at(*names))
-      fields = fields(names)
-      types = types.to_java(java.lang.Class)
-      each fields, :function => Java::CascadingOperation::Identity.new(fields, types)
-    end
-
-    def copy(*args)
-      options = args.extract_options!
-      from = args[0] || all_fields
-      into = args[1] || options[:into] || all_fields
-      each fields(from), :function => Java::CascadingOperation::Identity.new(fields(into)), :output => all_fields
-    end
-
-    # A pipe that does nothing.
-    def pass(*args)
-      each all_fields, :function => Java::CascadingOperation::Identity.new
-    end
+    include Operations
+    include IdentityOperations
+    include FilterOperations
+    include RegexOperations
+    include TextOperations
 
-    def assert(*args)
-      options = args.extract_options!
-      assertion = args[0]
+    # Builds an each assertion pipe given a c.o.a.Assertion and adds it to the
+    # current Assembly.
+    #
+    # The named options are:
+    # [level] The assertion level; defaults to strict.
+    def assert(assertion, options = {})
       assertion_level = options[:level] || Java::CascadingOperation::AssertionLevel::STRICT
 
       parameters = [tail_pipe, assertion_level, assertion]
       make_pipe(Java::CascadingPipe::Each, parameters)
     end
 
-    # Builds a debugging pipe.
-    #
-    # Without arguments, it generate a simple debug pipe, that prints all tuple to the standard
-    # output.
-    #
-    # The other named options are:
-    # * <tt>:print_fields</tt> a boolean. If is set to true, then it prints every 10 tuples.
-    #
-    def debug(*args)
-      options = args.extract_options!
-      print_fields = options[:print_fields] || true
-      parameters = [print_fields].compact
-      debug = Java::CascadingOperation::Debug.new(*parameters)
-      debug.print_tuple_every = options[:tuple_interval] || 1
-      debug.print_fields_every = options[:fields_interval] || 10
-      each(all_fields, :filter => debug)
-    end
-
-    # Builds a pipe that assert the size of the tuple is the size specified in parameter.
-    #
-    # The method accept an unique uname argument : a number indicating the size expected.
-    def assert_size_equals(*args)
-      options = args.extract_options!
-      assertion = Java::CascadingOperationAssertion::AssertSizeEquals.new(args[0])
+    # Builds a pipe that asserts the size of the tuple is the specified size.
+    def assert_size_equals(size, options = {})
+      assertion = Java::CascadingOperationAssertion::AssertSizeEquals.new(size)
       assert(assertion, options)
     end
 
-    # Builds a pipe that assert the none of the fields in the tuple are null.
-    def assert_not_null(*args)
-      options = args.extract_options!
+    # Builes a pipe that asserts none of the fiels in the tuple are null.
+    def assert_not_null(options = {})
       assertion = Java::CascadingOperationAssertion::AssertNotNull.new
       assert(assertion, options)
     end
 
-    # Builds a _parse_ pipe. This pipe will parse the fields specified in input (first unamed arguments),
-    # using a specified regex pattern.
-    #
-    # If provided, the unamed arguments must be the fields to be parsed. If not provided, then all incoming
-    # fields are used.
-    #
-    # The named options are:
-    # * <tt>:pattern</tt> a string or regex. Specifies the regular expression used for parsing the argument fields.
-    # * <tt>:output</tt> a string or array of strings. Specifies the outgoing fields (all fields will be output by default)
-    def parse(*args)
-        options = args.extract_options!
-        fields = args || all_fields
-        pattern = options[:pattern]
-        output = options[:output] || all_fields
-        each(fields, :function => regex_parser(pattern, options), :output => output)
-    end
+    private
 
-    # Builds a pipe that splits a field into other fields, using a specified regular expression.
-    #
-    # The first unnamed argument is the field to be split.
-    # The second unnamed argument is an array of strings indicating the fields receiving the result of the split.
-    #
-    # The named options are:
-    # * <tt>:pattern</tt> a string or regex. Specifies the regular expression used for splitting the argument fields.
-    # * <tt>:output</tt> a string or array of strings. Specifies the outgoing fields (all fields will be output by default)
-    def split(*args)
-      options = args.extract_options!
-      fields = options[:into] || args[1]
-      pattern = options[:pattern] || /[.,]*\s+/
-      output = options[:output] || all_fields
-      each(args[0], :function => regex_splitter(fields, :pattern => pattern), :output=>output)
-    end
-
-    # Builds a pipe that splits a field into new rows, using a specified regular expression.
-    #
-    # The first unnamed argument is the field to be split.
-    # The second unnamed argument is the field receiving the result of the split.
-    #
-    # The named options are:
-    # * <tt>:pattern</tt> a string or regex. Specifies the regular expression used for splitting the argument fields.
-    # * <tt>:output</tt> a string or array of strings. Specifies the outgoing fields (all fields will be output by default)
-    def split_rows(*args)
-      options = args.extract_options!
-      fields = options[:into] || args[1]
-      pattern = options[:pattern] || /[.,]*\s+/
-      output = options[:output] || all_fields
-      each(args[0], :function => regex_split_generator(fields, :pattern => pattern), :output=>output)
-    end
-
-    # Builds a pipe that emits a new row for each regex group matched in a field, using a specified regular expression.
-    #
-    # The first unnamed argument is the field to be matched against.
-    # The second unnamed argument is the field receiving the result of the match.
-    #
-    # The named options are:
-    # * <tt>:pattern</tt> a string or regex. Specifies the regular expression used for matching the argument fields.
-    # * <tt>:output</tt> a string or array of strings. Specifies the outgoing fields (all fields will be output by default)
-    def match_rows(*args)
-      options = args.extract_options!
-      fields = options[:into] || args[1]
-      pattern = options[:pattern] || /[\w]+/
-      output = options[:output] || all_fields
-      each(args[0], :function => regex_generator(fields, :pattern => pattern), :output=>output)
-    end
-
-    # Builds a pipe that parses the specified field as a date using hte provided format string.
-    # The unamed argument specifies the field to format.
-    #
-    # The named options are:
-    # * <tt>:into</tt> a string. It specifies the receiving field. By default, it will be named after
-    # the input argument.
-    # * <tt>:pattern</tt> a string. Specifies the date format.
-    # * <tt>:output</tt> a string or array of strings. Specifies the outgoing fields (all fields will be output by default)
-    def parse_date(*args)
-      options = args.extract_options!
-      field = options[:into] || "#{args[0]}_parsed"
-      output = options[:output] || all_fields
-      pattern = options[:pattern] || "yyyy/MM/dd"
-
-      each args[0], :function => date_parser(field, pattern), :output => output
-    end
+    def make_pipe(type, parameters)
+      @tail_pipe = type.new(*parameters)
+      @outgoing_scopes[name] = Scope.outgoing_scope(tail_pipe, [scope])
 
-    # Builds a pipe that format a date using a specified format pattern.
-    #
-    # The unamed argument specifies the field to format.
-    #
-    # The named options are:
-    # * <tt>:into</tt> a string. It specifies the receiving field. By default, it will be named after
-    # the input argument.
-    # * <tt>:pattern</tt> a string. Specifies the date format.
-    # * <tt>:timezone</tt> a string.  Specifies the timezone (defaults to UTC).
-    # * <tt>:output</tt> a string or array of strings. Specifies the outgoing fields (all fields will be output by default)
-    def format_date(*args)
-      options = args.extract_options!
-      field = options[:into] || "#{args[0]}_formatted"
-      pattern = options[:pattern] || "yyyy/MM/dd"
-      output = options[:output] || all_fields
-
-      each args[0], :function => date_formatter(field, pattern, options[:timezone]), :output => output
+      tail_pipe
     end
 
-    # Builds a pipe that perform a query/replace based on a regular expression.
-    #
-    # The first unamed argument specifies the input field.
-    #
-    # The named options are:
-    # * <tt>:pattern</tt> a string or regex. Specifies the pattern to look for in the input field. This non-optional argument
-    # can also be specified as a second _unamed_ argument.
-    # * <tt>:replacement</tt> a string. Specifies the replacement.
-    # * <tt>:output</tt> a string or array of strings. Specifies the outgoing fields (all fields will be output by default)
-    def replace(*args)
-      options = args.extract_options!
-
-      pattern = options[:pattern] || args[1]
-      replacement = options[:replacement] || args[2]
-      into = options[:into] || "#{args[0]}_replaced"
-      output = options[:output] || all_fields
-
-      each args[0], :function => regex_replace(into, pattern, replacement), :output => output
-    end
+    def populate_incoming_scopes(assembly_names, group_fields_args = {})
+      # NOTE: this overrides the existing incoming_scopes, which changes the
+      # way describe will function on this assembly
+      pipes, @incoming_scopes, group_fields = [], [], []
+      assembly_names.each do |assembly_name|
+        assembly = parent_flow.find_child(assembly_name)
+        raise "Could not find assembly '#{assembly_name}' from '#{name}'" unless assembly
 
-    # Builds a pipe that inserts values into the current tuple.
-    #
-    # The method takes a hash as parameter. This hash contains as keys the names of the fields to insert
-    # and as values, the values they must contain. For example:
-    #
-    #       insert {"who" => "Grégoire", "when" => Time.now.strftime("%Y-%m-%d") }
-    #
-    # will insert two new fields: a field _who_ containing the string "Grégoire", and a field _when_ containing
-    # the formatted current date.
-    # The methods outputs all fields.
-    # The named options are:
-    def insert(args)
-      args.keys.sort.each do |field_name|
-        value = args[field_name]
-
-        if value.kind_of?(ExprStub)
-          value.validate_scope(scope)
-          each all_fields, :function => expression_function(field_name, :expression => value.expression, :parameters => value.types), :output => all_fields
-        else
-          each all_fields, :function => insert_function([field_name], :values => [value]), :output => all_fields
-        end
+        pipes << assembly.tail_pipe
+        @incoming_scopes << assembly.scope
+        group_fields << fields(group_fields_args[assembly_name]) if group_fields_args[assembly_name]
       end
+      [pipes, group_fields]
     end
 
-    # Builds a pipe that filters the tuples based on an expression or a pattern (but not both !).
-    #
-    # The first unamed argument, if provided, is a filtering expression (using the Janino syntax).
-    #
-    # The named options are:
-    # * <tt>:pattern</tt> a string. Specifies a regular expression pattern used to filter the tuples. If this
-    # option is provided, then the filter is regular expression-based. This is incompatible with the _expression_ option.
-    # * <tt>:expression</tt> a string. Specifies a Janino expression used to filter the tuples. This option has the
-    # same effect than providing it as first unamed argument. If this option is provided, then the filter is Janino
-    # expression-based. This is incompatible with the _pattern_ option.
-    # * <tt>:validate</tt> a boolean.  Passed into Cascading#expr to enable or disable
-    # expression validation.  Defaults to true.
-    # * <tt>:validate_with</tt> a hash.  Actual arguments used by Cascading#expr for
-    # expression validation.  Defaults to {}.
-    def filter(*args)
-      options = args.extract_options!
-      from = options.delete(:from) || all_fields
-      expression = options.delete(:expression) || args.shift
-      regex = options.delete(:pattern)
-      validate = options.has_key?(:validate) ? options.delete(:validate) : true
-      validate_with = options.has_key?(:validate_with) ? options.delete(:validate_with) : {}
-
-      if expression
-        stub = expr(expression, { :validate => validate, :validate_with => validate_with })
-        types, expression = stub.types, stub.expression
-
-        stub.validate_scope(scope)
-        each from, :filter => expression_filter(
-          :parameters => types,
-          :expression => expression
-        )
-      elsif regex
-        each from, :filter => regex_filter(regex, options)
-      end
-    end
+    def apply_aggregations(group, incoming_scopes, &block)
+      aggregations = Aggregations.new(self, group, incoming_scopes)
+      aggregations.instance_eval(&block) if block_given?
 
-    def filter_null(*args)
-      options = args.extract_options!
-      each(args, :filter => Java::CascadingOperationFilter::FilterNull.new)
-    end
-    alias reject_null filter_null
+      # Sorting of any type means that we cannot use the AggregateBy optimization
+      if aggregations.can_aggregate_by? && !group.is_sorted && !group.is_sort_reversed
+        grouping_fields = group.key_selectors.values.first
+        group.key_selectors.values.each do |key_fields|
+          raise "Grouping fields mismatch: #{grouping_fields} expected; #{key_fields} found from #{group.key_selectors}" unless key_fields == grouping_fields
+        end
 
-    def filter_not_null(*args)
-      options = args.extract_options!
-      each(args, :filter => Java::CascadingOperationFilter::FilterNotNull.new)
-    end
-    alias where_null filter_not_null
+        aggregate_by = sub_assembly(Java::CascadingPipeAssembly::AggregateBy.new(
+          name,
+          group.previous,
+          grouping_fields,
+          aggregations.aggregate_bys.to_java(Java::CascadingPipeAssembly::AggregateBy)
+        ), group.previous, incoming_scopes)
 
-    # Builds a pipe that rejects the tuples based on an expression.
-    #
-    # The first unamed argument, if provided, is a filtering expression (using the Janino syntax).
-    #
-    # The named options are:
-    # * <tt>:expression</tt> a string. Specifies a Janino expression used to filter the tuples. This option has the
-    # same effect than providing it as first unamed argument. If this option is provided, then the filter is Janino
-    # expression-based.
-    # * <tt>:validate</tt> a boolean.  Passed into Cascading#expr to enable or disable
-    # expression validation.  Defaults to true.
-    # * <tt>:validate_with</tt> a hash.  Actual arguments used by Cascading#expr for
-    # expression validation.  Defaults to {}.
-    def reject(*args)
-      options = args.extract_options
-      raise "Regex not allowed" if options && options[:pattern]
-
-      filter(*args)
-    end
+        aggregate_by
+      else
+        aggregations.finalize if block_given?
+        @tail_pipe = aggregations.tail_pipe
+        @outgoing_scopes[name] = aggregations.scope
 
-    # Builds a pipe that includes just the tuples matching an expression.
-    #
-    # The first unamed argument, if provided, is a filtering expression (using the Janino syntax).
-    #
-    # The named options are:
-    # * <tt>:expression</tt> a string. Specifies a Janino expression used to select the tuples. This option has the
-    # same effect than providing it as first unamed argument. If this option is provided, then the filter is Janino
-    # expression-based.
-    # * <tt>:validate</tt> a boolean.  Passed into Cascading#expr to enable or disable
-    # expression validation.  Defaults to true.
-    # * <tt>:validate_with</tt> a hash.  Actual arguments used by Cascading#expr for
-    # expression validation.  Defaults to {}.
-    def where(*args)
-      options = args.extract_options
-      raise "Regex not allowed" if options && options[:pattern]
-
-      if options[:expression]
-        _, imports, expr = options[:expression].match(/^((?:\s*import.*;\s*)*)(.*)$/).to_a
-        options[:expression] = "#{imports}!(#{expr})"
-      elsif args[0]
-        _, imports, expr = args[0].match(/^((?:\s*import.*;\s*)*)(.*)$/).to_a
-        args[0] = "#{imports}!(#{expr})"
+        group
       end
-
-      filter(*args)
     end
 
-    # Builds a pipe that evaluates the specified Janino expression and insert it in a new field in the tuple.
-    #
-    # The named options are:
-    # * <tt>:from</tt> a string or array of strings. Specifies the input fields.
-    # * <tt>:express</tt> a string. The janino expression.
-    # * <tt>:into</tt> a string. Specified the name of the field to insert with the result of the evaluation.
-    # * <tt>:parameters</tt> a hash. Specifies the type mapping for the parameters. See Cascading::Operations.expression_function.
-    def eval_expression(*args)
-      options = args.extract_options!
-
-      into = options.delete(:into)
-      from = options.delete(:from) || all_fields
-      output = options.delete(:output) || all_fields
-      options[:expression] ||= args.shift
-      options[:parameters] ||= args.shift
-
-      each from, :function => expression_function(into, options), :output=>output
-    end
+    def prepare_join(assembly_names, options, &block)
+      pipes, _ = populate_incoming_scopes(assembly_names)
 
-    # Builds a pipe that returns distinct tuples based on the provided fields.
-    #
-    # The method accepts optional unamed argument specifying the fields to base the distinct on
-    # (all fields, by default).
-    def distinct(*args)
-      raise "Distinct is badly broken"
-      fields = args[0] || all_fields
-      group_by *fields
-      pass
-    end
-
-    def join_fields(*args)
-      options = args.extract_options!
-      output = options[:output] || all_fields
+      group_fields_args = options[:on]
+      raise 'join requires :on parameter' unless group_fields_args
 
-      each args, :function => field_joiner(options), :output => output
-    end
+      if group_fields_args.kind_of?(String)
+        group_fields_args = [group_fields_args]
+      end
 
-    # Ungroups, or unpivots, a tuple (see Cascading's UnGroup at http://docs.cascading.org/cascading/2.0/javadoc/cascading/operation/function/UnGroup.html).
-    #
-    # You must provide :key and you must provide only one of :value_selectors
-    # and :num_values.
-    #
-    # The named options are:
-    # * <tt>:key</tt> required array of field names to replicate on every
-    #   output row in an ungrouped group.
-    # * <tt>:value_selectors</tt> an array of field names to ungroup.  Each
-    #   field will be ungrouped into an output tuple along with the key fields
-    #   in the order provided.
-    # * <tt>:num_values</tt> an integer specifying the number of fields to
-    #   ungroup into each output tuple (excluding the key fields).  All input
-    #   fields will be ungrouped.
-    # * <tt>:input</tt> an array of field names that specifies the fields to
-    #   input to UnGroup.  Defaults to all_fields.
-    # * <tt>:into</tt> an array of field names.  Default set by UnGroup.
-    # * <tt>:output</tt> an array of field names that specifies the fields to
-    #   produce as output of UnGroup.  Defaults to all_fields.
-    def ungroup(*args)
-      options = args.extract_options!
-      input = options[:input] || all_fields
-      into = fields(options[:into])
-      output = options[:output] || all_fields
-      key = fields(options[:key])
-
-      raise 'You must provide exactly one of :value_selectors or :num_values to ungroup' unless options.has_key?(:value_selectors) ^ options.has_key?(:num_values)
-      value_selectors = options[:value_selectors].map{ |vs| fields(vs) }.to_java(Java::CascadingTuple::Fields) if options.has_key?(:value_selectors)
-      num_values = options[:num_values] if options.has_key?(:num_values)
-
-      parameters = [into, key, value_selectors, num_values].compact
-      each input, :function => Java::CascadingOperationFunction::UnGroup.new(*parameters), :output => output
-    end
+      group_fields = []
+      if group_fields_args.kind_of?(Array)
+        pipes.size.times do
+          group_fields << fields(group_fields_args)
+        end
+      elsif group_fields_args.kind_of?(Hash)
+        pipes, group_fields = populate_incoming_scopes(group_fields_args.keys.sort, group_fields_args)
+      else
+        raise "Unsupported data type for :on in join: '#{group_fields_args.class}'"
+      end
 
-    # Inserts one of two values into the dataflow based upon the result of the
-    # supplied filter on the input fields.  This is primarily useful for
-    # creating indicators from filters.
-    #
-    # Parameters:
-    # * <tt>input</tt> name of field to apply the filter.
-    # * <tt>filter</tt> Cascading Filter to apply.
-    # * <tt>keep_value</tt> Java value to produce when the filter would keep
-    #   the given input.
-    # * <tt>remove_value</tt> Java value to produce when the filter would
-    #   remove the given input.
-    #
-    # The named options are:
-    # * <tt>:into</tt> an output field name, defaulting to 'filter_value'.
-    # * <tt>:output</tt> an array of field names that specifies the fields to
-    #   retain in the output tuple.  Defaults to all_fields.
-    def set_value(input, filter, keep_value, remove_value, params = {})
-      into = fields(params[:into] || 'filter_value')
-      output = params[:output] || all_fields
-      each input, :function => Java::CascadingOperationFunction::SetValue.new(into, filter, keep_value, remove_value), :output => output
-    end
+      raise 'join requires non-empty :on parameter' if group_fields_args.empty?
+      group_fields = group_fields.to_java(Java::CascadingTuple::Fields)
+      incoming_fields = @incoming_scopes.map{ |s| s.values_fields }
+      declared_fields = fields(options[:declared_fields] || dedup_fields(*incoming_fields))
+      joiner = options[:joiner]
+      is_hash_join = options[:hash] || false
 
-    # Efficient way of inserting a null indicator for any field, even one that
-    # cannot be coerced to a string.  This is accomplished using Cascading's
-    # FilterNull and SetValue operators rather than Janino.  1 is produced if
-    # the field is null and 0 otherwise.
-    #
-    # Parameters:
-    # * <tt>input</tt> name of field to check for null.
-    #
-    # The named options are:
-    # * <tt>:into</tt> an output field name, defaulting to 'is_null'.
-    # * <tt>:output</tt> an array of field names that specifies the fields to
-    #   retain in the output tuple.  Defaults to all_fields.
-    def null_indicator(input, params = {})
-      into = fields(params[:into] || 'is_null')
-      output = params[:output] || all_fields
-      set_value input, Java::CascadingOperationFilter::FilterNull.new, 1.to_java, 0.to_java, :into => into, :output => output
-    end
+      case joiner
+      when :inner, 'inner', nil
+        joiner = Java::CascadingPipeJoiner::InnerJoin.new
+      when :left,  'left'
+        joiner = Java::CascadingPipeJoiner::LeftJoin.new
+      when :right, 'right'
+        joiner = Java::CascadingPipeJoiner::RightJoin.new
+      when :outer, 'outer'
+        joiner = Java::CascadingPipeJoiner::OuterJoin.new
+      when Array
+        joiner = joiner.map do |t|
+          case t
+          when true,  1, :inner then true
+          when false, 0, :outer then false
+          else fail "invalid mixed joiner entry: #{t}"
+          end
+        end
+        joiner = Java::CascadingPipeJoiner::MixedJoin.new(joiner.to_java(:boolean))
+      end
 
-    # Given a field and a regex, returns an indicator that is 1 if the string
-    # contains at least 1 match and 0 otherwise.
-    #
-    # Parameters:
-    # * <tt>input</tt> field name or names that specifies the fields over which
-    #   to perform the match.
-    # * <tt>pattern</tt> regex to apply to the input.
-    #
-    # The named options are:
-    # * <tt>:into</tt> an output field name, defaulting to 'regex_contains'.
-    # * <tt>:output</tt> an array of field names that specifies the fields to
-    #   retain in the output tuple.  Defaults to all_fields.
-    def regex_contains(input, pattern, params = {})
-      input = fields(input)
-      pattern = pattern.to_s # Supports JRuby regexes
-      into = fields(params[:into] || 'regex_contains')
-      output = params[:output] || all_fields
-      set_value input, Java::CascadingOperationRegex::RegexFilter.new(pattern), 1.to_java, 0.to_java, :into => into, :output => output
+      if is_hash_join
+        parameters = [
+          pipes.to_java(Java::CascadingPipe::Pipe),
+          group_fields,
+          declared_fields,
+          joiner
+        ]
+        group_assembly = Java::CascadingPipe::HashJoin.new(*parameters)
+      else
+        result_group_fields = dedup_fields(*group_fields)
+        parameters = [
+          pipes.to_java(Java::CascadingPipe::Pipe),
+          group_fields,
+          declared_fields,
+          result_group_fields,
+          joiner
+        ]
+        group_assembly = Java::CascadingPipe::CoGroup.new(*parameters)
+      end
+      apply_aggregations(group_assembly, @incoming_scopes, &block)
     end
   end
 end