cascading.jruby 0.0.10 → 1.0.0

data/lib/cascading/base.rb

@@ -1,7 +1,22 @@
 module Cascading
+  # A Node is a Cascade, Flow, or Assembly, all of which are composite
+  # structures that describe the hierarchical structure of your job.  A Cascade
+  # may contain many Flows and a Flow and Assembly may contain many Assemblies
+  # (branches in the case of the Assembly).  Nodes are named, contain parent
+  # and child pointers, and keep track of their children both by name and by
+  # insertion order.
+  #
+  # Nodes must be uniquely named within the scope of their parent so that they
+  # unambiguously looked up for connecting pipes within a flow.  However, we
+  # only ensure that children are uniquely named upon insertion; full
+  # uniqueness isn't required until Node#find_child is called (this allows for
+  # name reuse in a few limited circumstances that was important when migrating
+  # the Etsy workload to enforce these constraints).
   class Node
     attr_accessor :name, :parent, :children, :child_names, :last_child
 
+    # A Node requires a name and a parent when it is constructed.  Children are
+    # added later with Node#add_child.
     def initialize(name, parent)
       @name = name
       @parent = parent
@@ -23,10 +38,15 @@ module Cascading
       node
     end
 
+    # The qualified name of a node is formed from the name of all nodes in the
+    # path from the root to that node.
     def qualified_name
       parent ? "#{parent.qualified_name}.#{name}" : name
     end
 
+    # Produces a textual description of this Node.  This method is overridden
+    # by all classes inheriting Node, so it serves mainly as a template for
+    # describing a node with children.
     def describe(offset = '')
       "#{offset}#{name}:node\n#{child_names.map{ |child| children[child].describe("#{offset}  ") }.join("\n")}"
     end
@@ -44,6 +64,8 @@ module Cascading
       all_children_with_name.first
     end
 
+    # Returns the root Node, the topmost parent of the hierarchy (typically a
+    # Cascade or Flow).
     def root
       return self unless parent
       parent.root

data/lib/cascading/cascade.rb

@@ -2,6 +2,13 @@ require 'cascading/base'
 require 'yaml'
 
 module Cascading
+  # A Cascade wraps a c.c.Cascade.  A Cascade is composed of Flows, which are
+  # constructed using the Cascade#flow method within the block passed to the
+  # Cascading::cascade constructor.  Many flows may be nested within a Cascade.
+  #
+  # Note that you are not required to use a Cascade to wrap your job.  Instead,
+  # you could start with a top-level Flow, which you might prefer if you have
+  # no need of a c.c.Cascade's make-like semantics wrt sinks.
   class Cascade < Cascading::Node
     extend Registerable
 
@@ -10,46 +17,72 @@ module Cascading
     # Do not use this constructor directly; instead, use Cascading::cascade to
     # build cascades.
     #
-    # Builds a cascade given the specified name.  Optionally accepts
-    # :properties which will be used as the default properties for all child
-    # flows.  Properties must be a Ruby Hash with string keys and values and
-    # will be copied before being passed into each flow in the cascade.  See
-    # Cascading::Flow#initialize for details on how flows handle properties.
-    # Optionally accepts a :mode which will be used as the default mode for all
-    # child flows.  See Cascading::Mode.parse for details.
-    def initialize(name, params = {})
-      @properties = params[:properties] || {}
-      @mode = params[:mode]
+    # Builds a Cascade given a name.
+    #
+    # The named options are:
+    # [properties] Properties hash which will be used as the default properties
+    #              for all child flows.  Properties must be a Ruby Hash with
+    #              string keys and values and will be copied before being
+    #              passed into each flow in the cascade.  See Flow#initialize
+    #              for details on how flows handle properties.
+    # [mode] Mode which will be used as the default mode for all child flows.
+    #        See Mode.parse for details.
+    def initialize(name, options = {})
+      @properties = options[:properties] || {}
+      @mode = options[:mode]
       super(name, nil) # A Cascade cannot have a parent
       self.class.add(name, self)
     end
 
-    # Builds a child flow given a name and block.  Optionally accepts
-    # :properties which will override the default properties stroed in this
-    # cascade.  Optionally accepts a :mode, which will override the default
-    # mode stored in this cascade.
-    def flow(name, params = {}, &block)
+    # Builds a child Flow in this Cascade given a name and block.
+    #
+    # The named options are:
+    # [properties] Properties hash which will override the default properties
+    #              stored in this cascade.
+    # [mode] Mode which will override the default mode stored in this cascade.
+    #
+    # Example:
+    #     cascade 'wordcount', :mode => :local do
+    #       flow 'first_step' do
+    #         ...
+    #       end
+    #
+    #       flow 'second_step' do
+    #         ...
+    #       end
+    #     end
+    def flow(name, options = {}, &block)
       raise "Could not build flow '#{name}'; block required" unless block_given?
 
-      params[:properties] ||= properties.dup
-      params[:mode] ||= mode
+      options[:properties] ||= properties.dup
+      options[:mode] ||= mode
 
-      flow = Flow.new(name, self, params)
+      flow = Flow.new(name, self, options)
       add_child(flow)
       flow.instance_eval(&block)
       flow
     end
 
+    # Produces a textual description of this Cascade.  The description details
+    # the structure of the Cascade, the sources and sinks of each Flow, and the
+    # input and output fields of each Assembly.  The offset parameter allows
+    # for this describe to be nested within a calling context, which lets us
+    # indent the structural hierarchy of a job.
     def describe(offset = '')
       "#{offset}#{name}:cascade\n#{child_names.map{ |child| children[child].describe("#{offset}  ") }.join("\n")}"
     end
 
+    # Writes out the DOT file describing the structure of this Cascade.
+    #
+    # NOTE: will be at Job in later version and also present on Flow
     def draw(dir)
       @children.each do |name, flow|
         flow.connect.writeDOT("#{dir}/#{name}.dot")
       end
     end
 
+    # Builds a map, keyed by flow name, of the sink metadata for each child
+    # flow.  Currently, this contains only the field names of each sink.
     def sink_metadata
       @children.inject({}) do |sink_fields, (name, flow)|
         sink_fields[name] = flow.sink_metadata
@@ -57,12 +90,16 @@ module Cascading
       end
     end
 
+    # Writes the mapping produced by Cascade#sink_metadata to a file at the
+    # given path in YAML.
     def write_sink_metadata(file_name)
       File.open(file_name, 'w') do |file|
         YAML.dump(sink_metadata, file)
       end
     end
 
+    # Connects this Cascade, producing a c.c.Cascade, which is then completed,
+    # executing it.  Child flows are connected, so no parameters are required.
     def complete
       begin
         Java::CascadingCascade::CascadeConnector.new.connect(name, make_flows(@children)).complete

data/lib/cascading/cascading.rb

@@ -1,6 +1,33 @@
+require 'cascading/cascade'
+require 'cascading/flow'
 require 'cascading/expr_stub'
 
+# The Cascading module contains all of the cascading.jruby DSL.  Inserting the
+# following into your script:
+#     require 'rubygems'
+#     require 'cascading'
+# includes this module at the top level, making all of its features available.
+#
+# To build a dataflow like the one in the README.md or
+# {samples}[http://github.com/mrwalker/cascading.jruby/tree/master/samples],
+# start by looking at Cascade or Flow.  These are the
+# highest level structures you'll use to put together your job.
+#
+# Within a flow, you'll connect sources to sinks by way of Assembly, which
+# refers to "pipe assemblies" from Cascading.  Within an Assembly, you'll use
+# functions and filters (see Operations, IdentityOperations, RegexOperations,
+# FilterOperations, and TextOperations) as well as Assembly#group_by,
+# Assembly#union, and Assembly#join.  You can provide those last pipes with a
+# block that can select operations from Aggregations.
+#
+# Finally, you'll want to address the execution of your job, whether it be
+# locally testing or running remotely on a Hadoop cluster.  See the Mode class
+# for the available modes, and parameterize your script such that it can operate
+# in Cascading local mode locally and in Hadoop mode when run in a jar produced
+# with {Jading}[http://github.com/mrwalker/jading].
 module Cascading
+  # Mapping that defines a convenient syntax for specifying Java classes, used
+  # in Janino expressions and elsewhere.
   JAVA_TYPE_MAP = {
     :int => java.lang.Integer.java_class, :long => java.lang.Long.java_class,
     :bool => java.lang.Boolean.java_class, :double => java.lang.Double.java_class,
@@ -24,44 +51,84 @@ module Cascading
   # directly building their own cascades and flows so that jading can send them
   # default properties.
 
-  # Builds a top-level cascade given a name and a block.  Optionally accepts a
-  # :mode, as explained in Cascading::Cascade#initialize.
-  def cascade(name, params = {}, &block)
+  # Builds a top-level Cascade given a name and a block.
+  #
+  # The named options are:
+  # [properties] See Cascade#initialize
+  # [mode] See Cascade#initialize
+  #
+  # Example:
+  #     cascade 'wordcount', :mode => :local do
+  #       flow 'first_step' do
+  #         ...
+  #       end
+  #
+  #       flow 'second_step' do
+  #         ...
+  #       end
+  #     end
+  def cascade(name, options = {}, &block)
     raise "Could not build cascade '#{name}'; block required" unless block_given?
-    raise 'Cascading::cascade does not accept the :properties param only the global $jobconf_properties' if params[:properties]
+    raise 'Cascading::cascade does not accept the :properties param only the global $jobconf_properties' if options[:properties]
 
-    params[:properties] = $jobconf_properties.dup if $jobconf_properties
+    options[:properties] = $jobconf_properties.dup if defined?($jobconf_properties) && $jobconf_properties
 
-    cascade = Cascade.new(name, params)
+    cascade = Cascade.new(name, options)
     cascade.instance_eval(&block)
     cascade
   end
 
-  # Builds a top-level flow given a name and block for applications built of
-  # flows with no cascades.  Optionally accepts a :mode, as explained in
-  # Cascading::Flow#initialize.
-  def flow(name, params = {}, &block)
+  # Builds a top-level Flow given a name and block for applications built of
+  # flows with no cascades.
+  #
+  # The named options are:
+  # [properties] See Flow#initialize
+  # [mode] See Flow#initialize
+  #
+  # Example:
+  #     flow 'wordcount', :mode => :local do
+  #       assembly 'first_step' do
+  #         ...
+  #       end
+  #
+  #       assembly 'second_step' do
+  #         ...
+  #       end
+  #     end
+  def flow(name, options = {}, &block)
     raise "Could not build flow '#{name}'; block required" unless block_given?
-    raise 'Cascading::flow does not accept the :properties param only the global $jobconf_properties' if params[:properties]
+    raise 'Cascading::flow does not accept the :properties param only the global $jobconf_properties' if options[:properties]
 
-    params[:properties] = $jobconf_properties.dup if $jobconf_properties
+    options[:properties] = $jobconf_properties.dup if defined?($jobconf_properties) && $jobconf_properties
 
-    flow = Flow.new(name, nil, params)
+    flow = Flow.new(name, nil, options)
     flow.instance_eval(&block)
     flow
   end
 
+  # Produces a textual description of all Cascades in the global registry.  The
+  # description details the structure of the Cascades, the sources and sinks of
+  # each Flow, and the input and output fields of each Assembly.
+  #
+  # NOTE: will be moved to Job in later version
   def describe
     Cascade.all.map{ |cascade| cascade.describe }.join("\n")
   end
   alias desc describe
 
   # See ExprStub.expr
-  def expr(expression, params = {})
-    ExprStub.expr(expression, params)
+  def expr(expression, options = {})
+    ExprStub.expr(expression, options)
   end
 
-  # Creates a cascading.tuple.Fields instance from a string or an array of strings.
+  # Utility method for creating Cascading c.t.Fields from a field name (string)
+  # or list of field names (array of strings).  If the input fields is already a
+  # c.t.Fields or nil, it is passed through.  This allows for flexible use of
+  # the method at multiple layers in the DSL.
+  #
+  # Example:
+  #     cascading_fields = fields(['first', 'second', 'third'])
+  #     # cascading_fields.to_a == ['first', 'second', 'third']
   def fields(fields)
     if fields.nil?
       return nil
@@ -76,27 +143,45 @@ module Cascading
     return Java::CascadingTuple::Fields.new([fields].flatten.map{ |f| f.kind_of?(Fixnum) ? java.lang.Integer.new(f) : f }.to_java(java.lang.Comparable))
   end
 
+  # Convenience method wrapping c.t.Fields::ALL
   def all_fields
     Java::CascadingTuple::Fields::ALL
   end
 
-  def union_fields(*fields)
-    fields(fields.inject([]){ |acc, arr| acc | arr.to_a })
-  end
-
-  def difference_fields(*fields)
-    fields(fields[1..-1].inject(fields.first.to_a){ |acc, arr| acc - arr.to_a })
+  # Convenience method wrapping c.t.Fields::VALUES
+  def last_grouping_fields
+    Java::CascadingTuple::Fields::VALUES
   end
 
-  def copy_fields(fields)
-    fields.select(all_fields)
+  # Computes fields formed by removing remove_fields from base_fields.  Operates
+  # only on named fields, not positional fields.
+  #
+  # Example:
+  #     base_fields = fields(['a', 'b', 'c'])
+  #     remove_fields = fields(['b'])
+  #     result_fields = difference_fields(base_fields, remove_fields)
+  #     # results_fields.to_a == ['a', 'c']
+  def difference_fields(base_fields, remove_fields)
+    fields(base_fields.to_a - remove_fields.to_a)
   end
 
+  # Combines fields deduplicating them with trailing underscores as necessary.
+  # This is used in joins to avoid requiring the caller to unique fields before
+  # they are joined.
   def dedup_fields(*fields)
     raise 'Can only be applied to declarators' unless fields.all?{ |f| f.is_declarator? }
     fields(dedup_field_names(*fields.map{ |f| f.to_a }))
   end
 
+  # Helper used by dedup_fields that operates on arrays of field names rather
+  # than fields objects.
+  #
+  # Example:
+  #     left_names = ['a', 'b']
+  #     mid_names = ['a', 'c']
+  #     right_names = ['a', 'd']
+  #     deduped_names = dedup_field_names(left_names, mid_names, right_names)
+  #     # deduped_names == ['a', 'b', 'a_', 'c', 'a__', 'd']
   def dedup_field_names(*names)
     names.inject([]) do |acc, arr|
       acc + arr.map{ |e| search_field_name(acc, e) }
@@ -106,30 +191,22 @@ module Cascading
   def search_field_name(names, candidate)
     names.include?(candidate) ? search_field_name(names, "#{candidate}_") : candidate
   end
-
-  def last_grouping_fields
-    Java::CascadingTuple::Fields::VALUES
-  end
-
-  def results_fields
-    Java::CascadingTuple::Fields::RESULTS
-  end
+  private :search_field_name
 
   # Creates a TextLine scheme (can be used in both Cascading local and hadoop
-  # modes).  Positional args are used if <tt>:source_fields</tt> is not
-  # provided.
+  # modes).  Positional args are used if :source_fields is not provided.
   #
   # The named options are:
-  # * <tt>:source_fields</tt> a string or array of strings.  Specifies the
-  #   fields to be read from a source with this scheme.  Defaults to ['offset', 'line'].
-  # * <tt>:sink_fields</tt> a string or array of strings. Specifies the fields
-  #   to be written to a sink with this scheme.  Defaults to all_fields.
-  # * <tt>:compression</tt> a symbol, either <tt>:enable</tt> or
-  #   <tt>:disable</tt>, that governs the TextLine scheme's compression.  Defaults
-  #   to the default TextLine compression (only applies to c.s.h.TextLine).
-  def text_line_scheme(*args)
-    options = args.extract_options!
-    source_fields = fields(options[:source_fields] || (args.empty? ? ['offset', 'line'] : args))
+  # [source_fields] Fields to be read from a source with this scheme.  Defaults
+  #                 to ['offset', 'line'].
+  # [sink_fields] Fields to be written to a sink with this scheme.  Defaults to
+  #               all_fields.
+  # [compression] A symbol, either :enable or :disable, that
+  #               governs the TextLine scheme's compression.  Defaults to the
+  #               default TextLine compression (only applies to c.s.h.TextLine).
+  def text_line_scheme(*args_with_options)
+    options, source_fields = args_with_options.extract_options!, args_with_options
+    source_fields = fields(options[:source_fields] || (source_fields.empty? ? ['offset', 'line'] : source_fields))
     sink_fields = fields(options[:sink_fields]) || all_fields
     sink_compression = case options[:compression]
       when :enable  then Java::CascadingSchemeHadoop::TextLine::Compress::ENABLE
@@ -153,17 +230,30 @@ module Cascading
     }
   end
 
+  # Convenience access to MultiTap.multi_source_tap.  This constructor is more
+  # "DSL-like" because it allows you to pass taps directly as actual args rather
+  # than in an array:
+  #     multi_source_tap tap1, tap2, tap3, ..., tapn
+  #
+  # See MultiTap.multi_source_tap for more details.
   def multi_source_tap(*taps)
     MultiTap.multi_source_tap(taps)
   end
 
+  # Convenience access to MultiTap.multi_sink_tap.  This constructor is more
+  # "DSL-like" because it allows you to pass taps directly as actual args rather
+  # than in an array:
+  #     multi_sink_tap tap1, tap2, tap3, ..., tapn
+  #
+  # See MultiTap.multi_sink_tap for more details.
   def multi_sink_tap(*taps)
     MultiTap.multi_sink_tap(taps)
   end
 
-  # Creates a Cascading::Tap given a path and optional :scheme and :sink_mode.
-  def tap(path, params = {})
-    Tap.new(path, params)
+  # Convenience constructor for a Tap, that accepts the same options as that
+  # class' constructor.  See Tap for more details.
+  def tap(path, options = {})
+    Tap.new(path, options)
   end
 
   # Constructs properties to be passed to Flow#complete or Cascade#complete

data/lib/cascading/expr_stub.rb

@@ -3,15 +3,15 @@ module Cascading
     attr_accessor :expression, :types, :input_expression
 
     # ExprStub requires a Janino expression decorated with field types.  For
-    # example: '"Found: " + (x:int + y:int) + " " + z:string'.  Type names are
-    # defined in Cascading::JAVA_TYPE_MAP.
+    # example:
+    #     expr('"Found: " + (x:int + y:int) + " " + z:string')
+    # Type names are defined in Cascading::JAVA_TYPE_MAP.
     def initialize(expression)
       @input_expression = expression
       @expression = expression.dup
       @types = {}
 
       # Simple regexp based parser for types
-
       JAVA_TYPE_MAP.each do |sym, klass|
         @expression.gsub!(/[A-Za-z0-9_]+:#{sym.to_s}/) do |match|
           name = match.split(/:/).first.gsub(/\s+/, "")
@@ -21,21 +21,38 @@ module Cascading
       end
     end
 
+    # Extract Java names and types from @types hash.  Cascading constructors
+    # often require two separate Java Arrays in this fashion.
+    def names_and_types
+      names, types = split_hash(@types)
+      [names.to_java(java.lang.String), types.to_java(java.lang.Class)]
+    end
+
+    # Prints the original input expression.
     def to_s
       @input_expression
     end
 
     # Convenience constructor for an ExprStub that optionally performs
     # validation.  Takes a string to use as a Janino expression and an optional
-    # params hash.  By default, the param :validate is set to true (performs
-    # expression validation using default actual argument values) and the param
-    # :validate_with is set to {} (which doesn't override any of the default
-    # actual argument values used for validation).
-    def self.expr(expression, params = {})
-      params = { :validate => true, :validate_with => {} }.merge(params)
+    # options hash.
+    #
+    # The named options are:
+    # [validate] A boolean indicating whether expression validation using
+    #            default actual argument values should be performed.  Defaults
+    #            to true.
+    # [validate_with] A hash mapping field names (or symbols) to the value that
+    #                 should be used for validation.  Strings default to nil,
+    #                 so if you have previously filtered nulls you might use a
+    #                 marker value like 'nulls_filtered'.  Defaults to {}.
+    #
+    # Example:
+    #     insert 'x_eq_y' => expr('x:string.equals(y:string)', :validate_with => { :x => 'nulls_filtered' })
+    def self.expr(expression, options = {})
+      options = { :validate => true, :validate_with => {} }.merge(options)
       expr_stub = expression.kind_of?(ExprStub) ? expression : ExprStub.new(expression).compile
-      expr_stub.validate(params[:validate_with]) if params[:validate]
-      puts "Expression validation is disabled for '#{expression}'" unless params[:validate]
+      expr_stub.validate(options[:validate_with]) if options[:validate]
+      puts "Expression validation is disabled for '#{expression}'" unless options[:validate]
       expr_stub
     end
 
@@ -68,6 +85,9 @@ module Cascading
       self.eval(test_values.merge(actual_args))
     end
 
+    # Given a scope, validates that the fields required by this ExprStub are
+    # available in the values fields of the scope.  Returns those values fields
+    # which are unused in the expression.
     def validate_scope(scope)
       validate_fields(scope.values_fields.to_a)
     end
@@ -113,12 +133,6 @@ module Cascading
       end
     end
 
-    # Extract Java names and types from @types hash
-    def names_and_types
-      names, types = split_hash(@types)
-      [names.to_java(java.lang.String), types.to_java(java.lang.Class)]
-    end
-
     # Makes best effort to convert Ruby numbers into the Java numeric type
     # exepcted by a Janino expression. However, if the conversion fails, it
     # returns the original value so that the exception thrown will be from