cascading.jruby 0.0.10 → 1.0.0

data/lib/cascading/ext/array.rb

@@ -1,8 +1,25 @@
+# Extensions to Arrays in support of variable length lists of field names.  This
+# is not pretty, but supports DSL features like:
+#     group_by 'field1', 'field2', :sort_by => 'field3' do
+#       ...
+#     end
+#
+# The most obvious limitation of the approach is that function definitions of
+# the form f(*args_with_options) are not self-documenting.  To compensate for
+# this, documentation of all arguments and optional parameters must be provided
+# on the DSL method.
 class Array
+  # Use this extension to extract the optional parameters from a
+  # *args_with_options argument.
+  # So if you have a function:
+  #     def f(*args_with_options)
+  # You can destructively process the args_with_options as follows:
+  #     options, just_args = args_with_options.extract_options!, args_with_options
   def extract_options!
      last.is_a?(::Hash) ? pop : {}
   end
   
+  # Non-destructive form of Array#extract_options!
   def extract_options
      last.is_a?(::Hash) ? last : {}
   end

data/lib/cascading/filter_operations.rb

@@ -0,0 +1,101 @@
+module Cascading
+  # Module of filtering operations.  Unlike some of the other functional
+  # operations modules, this one does not just wrap operations defined by
+  # Cascading in cascading.operation.filter.  Instead, it provides some useful
+  # high-level DSL pipes which map many Cascading operations into a smaller
+  # number of DSL statements.
+  #
+  # Still, some are direct wrappers:
+  # filter\_null:: {FilterNull}[http://docs.cascading.org/cascading/2.1/javadoc/cascading/operation/filter/FilterNull.html]
+  # filter\_not\_null:: {FilterNotNull}[http://docs.cascading.org/cascading/2.1/javadoc/cascading/operation/filter/FilterNotNull.html]
+  module FilterOperations
+    # Filter the current assembly based on an expression or regex, but not both.
+    #
+    # The named options are:
+    # [expression] A Janino expression used to filter.  Has access to all :input
+    #              fields.
+    # [validate] Boolean passed to Cascading#expr to enable or disable
+    #            expression validation.  Defaults to true.
+    # [validate_with] Hash mapping field names to actual arguments used by
+    #                 Cascading#expr for expression validation.  Defaults to {}.
+    # [regex] A regular expression used to filter.
+    # [remove_match] Boolean indicating if regex matches should be removed or
+    #                kept.  Defaults to false, which is a bit counterintuitive.
+    # [match_each_element] Boolean indicating if regex should match entire
+    #                      incoming tuple (joined with tabs) or each field
+    #                      individually.  Defaults to false.
+    #
+    # Example:
+    #     filter :input => 'field1', :regex => /\t/, :remove_match => true
+    #     filter :expression => 'field1:long > 0 && "".equals(field2:string)'
+    def filter(options = {})
+      input_fields = options[:input] || all_fields
+      expression = options[:expression]
+      regex = options[:regex]
+
+      if expression
+        validate = options.has_key?(:validate) ? options[:validate] : true
+        validate_with = options[:validate_with] || {}
+
+        stub = expr(expression, { :validate => validate, :validate_with => validate_with })
+        stub.validate_scope(scope)
+
+        names, types = stub.names_and_types
+        each input_fields, :filter => Java::CascadingOperationExpression::ExpressionFilter.new(
+            stub.expression,
+            names,
+            types
+          )
+      elsif regex
+        parameters = [regex.to_s, options[:remove_match], options[:match_each_element]].compact
+        each input_fields, :filter => Java::CascadingOperationRegex::RegexFilter.new(*parameters)
+      else
+        raise 'filter requires one of :expression or :regex'
+      end
+    end
+
+    # Rejects tuples from the current assembly based on a Janino expression.
+    # This is just a wrapper for FilterOperations.filter.
+    #
+    # Example:
+    #     reject 'field1:long > 0 && "".equals(field2:string)'
+    def reject(expression, options = {})
+      options[:expression] = expression
+      filter(options)
+    end
+
+    # Keeps tuples from the current assembly based on a Janino expression.  This
+    # is a wrapper for FilterOperations.filter.
+    #
+    # Note that this is accomplished by inverting the given expression, and best
+    # attempt is made to support import statements prior to the expression.  If
+    # this support should break, simply negate your expression and use
+    # FilterOperations.reject.
+    #
+    # Example:
+    #     where 'field1:long > 0 && "".equals(field2:string)'
+    def where(expression, options = {})
+      _, imports, expr = expression.match(/^((?:\s*import.*;\s*)*)(.*)$/).to_a
+      options[:expression] = "#{imports}!(#{expr})"
+      filter(options)
+    end
+
+    # Rejects tuples from the current assembly if any input field is null.
+    #
+    # Example:
+    #     filter_null 'field1', 'field2'
+    def filter_null(*input_fields)
+      each(input_fields, :filter => Java::CascadingOperationFilter::FilterNull.new)
+    end
+    alias reject_null filter_null
+
+    # Rejects tuples from the current assembly if any input field is not null.
+    #
+    # Example:
+    #     filter_not_null 'field1', 'field2'
+    def filter_not_null(*input_fields)
+      each(input_fields, :filter => Java::CascadingOperationFilter::FilterNotNull.new)
+    end
+    alias where_null filter_not_null
+  end
+end

data/lib/cascading/flow.rb

@@ -1,6 +1,10 @@
 require 'cascading/assembly'
 
 module Cascading
+  # A Flow wraps a c.f.Flow.  A Flow is composed of Assemblies, which are
+  # constructed using the Flow#assembly method within the block passed to the
+  # Cascading::flow or Cascade#flow constructor.  Many Assemblies may be nested
+  # within a Flow.
   class Flow < Cascading::Node
     extend Registerable
 
@@ -10,23 +14,46 @@ module Cascading
     # Do not use this constructor directly.  Instead, use Cascading::flow to
     # build top-level flows and Cascade#flow to build flows within a Cascade.
     #
-    # Builds a flow given a name and a parent node (a cascade or nil).
-    # Optionally accepts :properties which allows external configuration of
-    # this flow.  The flow will side-effect the properties during composition,
-    # then pass the modified properties along to the FlowConnector for
-    # execution. See Cascading::Cascade#initialize for details on how
-    # properties are propagated through cascades.  Optionally accepts a :mode
-    # which will determine the execution mode of this flow.  See
-    # Cascading::Mode.parse for details.
-    def initialize(name, parent, params = {})
+    # Builds a Flow given a name and a parent node (a Cascade or nil).
+    #
+    # The named options are:
+    # [properties] Properties hash which allows external configuration of this
+    #              flow.  The flow will side-effect the properties during
+    #              composition, then pass the modified properties along to the
+    #              FlowConnector for execution.  See Cascade#initialize for
+    #              details on how properties are propagated through cascades.
+    # [mode] Mode which will determine the execution mode of this flow.  See
+    #        Mode.parse for details.
+    def initialize(name, parent, options = {})
       @sources, @sinks, @incoming_scopes, @outgoing_scopes, @listeners = {}, {}, {}, {}, []
-      @properties = params[:properties] || {}
-      @mode = Mode.parse(params[:mode])
+      @properties = options[:properties] || {}
+      @mode = Mode.parse(options[:mode])
       @flow_scope = Scope.flow_scope(name)
       super(name, parent)
       self.class.add(name, self)
     end
 
+    # Builds a child Assembly in this Flow given a name and block.
+    #
+    # An assembly's name is quite important as it will determine:
+    # * The sources from which it will read, if any
+    # * The name to be used in joins or unions downstream
+    # * The name to be used to sink the output of the assembly downstream
+    #
+    # Many assemblies may be built within a flow.  The Assembly#branch method
+    # is used for creating nested assemblies and produces objects of the same
+    # type as this constructor.
+    #
+    # Example:
+    #     flow 'wordcount', :mode => :local do
+    #       assembly 'first_step' do
+    #         ...
+    #       end
+    #
+    #       assembly 'second_step' do
+    #         ...
+    #       end
+    #     end
     def assembly(name, &block)
       raise "Could not build assembly '#{name}'; block required" unless block_given?
       assembly = Assembly.new(name, self, @outgoing_scopes)
@@ -49,6 +76,11 @@ module Cascading
       sinks[name] = tap
     end
 
+    # Produces a textual description of this Flow.  The description details the
+    # structure of the Flow, its sources and sinks, and the input and output
+    # fields of each Assembly.  The offset parameter allows for this describe
+    # to be nested within a calling context, which lets us indent the
+    # structural hierarchy of a job.
     def describe(offset = '')
       description =  "#{offset}#{name}:flow\n"
       description += "#{sources.keys.map{ |source| "#{offset}  #{source}:source :: #{incoming_scopes[source].values_fields.to_a.inspect}" }.join("\n")}\n"
@@ -57,18 +89,28 @@ module Cascading
       description
     end
 
+    # Accesses the outgoing scope of this Flow at the point at which it is
+    # called by default, or for the child specified by the given name, if
+    # specified.  This is useful for grabbing the values_fields at any point in
+    # the construction of the Flow.  See Scope for details.
     def scope(name = nil)
       raise 'Must specify name if no children have been defined yet' unless name || last_child
       name ||= last_child.name
       @outgoing_scopes[name]
     end
 
+    # Prints information about the scope of this Flow at the point at which it
+    # is called by default, or for the child specified by the given name, if
+    # specified.  This allows you to trace the propagation of field names
+    # through your job and is handy for debugging.  See Scope for details.
     def debug_scope(name = nil)
       scope = scope(name)
       name ||= last_child.name
       puts "Scope for '#{name}':\n  #{scope}"
     end
 
+    # Builds a map, keyed by sink name, of the sink metadata for each sink.
+    # Currently, this contains only the field names of each sink.
     def sink_metadata
       @sinks.keys.inject({}) do |sink_metadata, sink_name|
         raise "Cannot sink undefined assembly '#{sink_name}'" unless @outgoing_scopes[sink_name]
@@ -79,7 +121,16 @@ module Cascading
       end
     end
 
-    # TODO: support all codecs, support list of codecs
+    # Property modifier that sets the codec and type of the compression for all
+    # sinks in this flow.  Currently only supports o.a.h.i.c.DefaultCodec and
+    # o.a.h.i.c.GzipCodec, and the the NONE, RECORD, or BLOCK compressions
+    # types defined in o.a.h.i.SequenceFile.
+    #
+    # codec may be symbols like :default or :gzip and type may be symbols like
+    # :none, :record, or :block.
+    #
+    # Example:
+    #     compress_output :default, :block
     def compress_output(codec, type)
       properties['mapred.output.compress'] = 'true'
       properties['mapred.output.compression.codec'] = case codec
@@ -95,22 +146,28 @@ module Cascading
         end
     end
 
+    # Set the cascading.spill.list.threshold property in this flow's
+    # properties.  See c.t.c.SpillableProps for details.
     def set_spill_threshold(threshold)
-      properties['cascading.cogroup.spill.threshold'] = threshold.to_s
+      properties['cascading.spill.list.threshold'] = threshold.to_s
     end
 
+    # Adds the given path to the mapred.cache.files list property.
     def add_file_to_distributed_cache(file)
       add_to_distributed_cache(file, "mapred.cache.files")
     end
 
+    # Adds the given path to the mapred.cache.archives list property.
     def add_archive_to_distributed_cache(file)
       add_to_distributed_cache(file, "mapred.cache.archives")
     end
 
+    # Appends a FlowListener to the list of listeners for this flow.
     def add_listener(listener)
       @listeners << listener
     end
 
+    # Handles locating a file cached from S3 on local disk.  TODO: remove
     def emr_local_path_for_distributed_cache_file(file)
       # NOTE this needs to be *appended* to the property mapred.local.dir
       if file =~ /^s3n?:\/\//
@@ -122,16 +179,9 @@ module Cascading
       end
     end
 
-    def add_to_distributed_cache(file, property)
-      v = properties[property]
-
-      if v
-        properties[property] = [v.split(/,/), file].flatten.join(",")
-      else
-        properties[property] = file
-      end
-    end
-
+    # Connects this Flow, producing a c.f.Flow without completing it (the Flow
+    # is not executed).  This method is used by Cascade to connect its child
+    # Flows.  To connect and complete a Flow, see Flow#complete.
     def connect
       puts "Connecting flow '#{name}' with properties:"
       properties.keys.sort.each do |key|
@@ -141,6 +191,7 @@ module Cascading
       # FIXME: why do I have to do this in 2.0 wip-255?
       Java::CascadingProperty::AppProps.setApplicationName(properties, name)
       Java::CascadingProperty::AppProps.setApplicationVersion(properties, '0.0.0')
+      Java::CascadingProperty::AppProps.setApplicationJarClass(properties, Java::CascadingFlow::Flow.java_class)
 
       sources = make_tap_parameter(@sources, :head_pipe)
       sinks = make_tap_parameter(@sinks, :tail_pipe)
@@ -148,6 +199,9 @@ module Cascading
       mode.connect_flow(properties, name, sources, sinks, pipes)
     end
 
+    # Completes this Flow after connecting it.  This results in execution of
+    # the c.f.Flow built from this Flow.  Use this method when executing a
+    # top-level Flow.
     def complete
       begin
         flow = connect
@@ -160,6 +214,16 @@ module Cascading
 
     private
 
+    def add_to_distributed_cache(file, property)
+      v = properties[property]
+
+      if v
+        properties[property] = [v.split(/,/), file].flatten.join(",")
+      else
+        properties[property] = file
+      end
+    end
+
     def make_tap_parameter(taps, pipe_accessor)
       taps.inject({}) do |map, (name, tap)|
         assembly = find_child(name)

data/lib/cascading/identity_operations.rb

@@ -0,0 +1,82 @@
+module Cascading
+  # Module of pipe assemblies that wrap the Cascading Identity operation.  These
+  # are split out only to group similar functionality.
+  module IdentityOperations
+    # Restricts the current assembly to the specified fields in the order in
+    # which they are specified (can be used to reorder fields).
+    #
+    # Example:
+    #     project 'field1', 'field2'
+    def project(*input_fields)
+      each fields(input_fields), :function => Java::CascadingOperation::Identity.new
+    end
+
+    # Removes the specified fields from the current assembly.
+    #
+    # Example:
+    #     discard 'field1', 'field2'
+    def discard(*input_fields)
+      discard_fields = fields(input_fields)
+      keep_fields = difference_fields(scope.values_fields, discard_fields)
+      project(*keep_fields.to_a)
+    end
+
+    # Renames fields according to the mapping provided, preserving the original
+    # field order.  Throws an exception if non-existent fields are specified.
+    # 
+    # Example:
+    #     rename 'field1' => 'fieldA', 'field2' => 'fieldB'
+    #
+    # Produces: ['fieldA', 'fieldB'], assuming those were the only 2 input
+    # fields.
+    def rename(name_map)
+      original_fields = scope.values_fields.to_a
+      invalid = name_map.keys - original_fields
+      raise "Invalid field names in rename: #{invalid.inspect}" unless invalid.empty?
+
+      renamed_fields = original_fields.map{ |name| name_map[name] || name }
+
+      each original_fields, :function => Java::CascadingOperation::Identity.new(fields(renamed_fields))
+    end
+
+    # Coerces fields to the Java type selected from Cascading::JAVA_TYPE_MAP.
+    #
+    # Example:
+    #     cast 'field1' => :int, 'field2' => :double
+    def cast(type_map)
+      input_fields = type_map.keys.sort
+      types = JAVA_TYPE_MAP.values_at(*type_map.values_at(*input_fields))
+      input_fields = fields(input_fields)
+      types = types.to_java(java.lang.Class)
+      each input_fields, :function => Java::CascadingOperation::Identity.new(input_fields, types)
+    end
+
+    # A field copy (not a pipe copy).  Renames fields according to name_map,
+    # appending them to the fields in the assembly in the same order as the
+    # original fields from which they are copied.  Throws an exception if
+    # non-existent fields are specified.
+    #
+    # Example:
+    #     copy 'field1' => 'fieldA', 'field2' => 'fieldB'
+    #
+    # Produces: ['field1', 'field2', 'fieldA', 'fieldB'], assuming those were
+    # the only input fields.
+    def copy(name_map)
+      original_fields = scope.values_fields.to_a
+      invalid = name_map.keys - original_fields
+      raise "Invalid field names in copy: #{invalid.inspect}" unless invalid.empty?
+
+      # Original fields in name_map in their original order
+      input_fields = original_fields - (original_fields - name_map.keys)
+      into_fields = name_map.values_at(*input_fields)
+
+      each input_fields, :function => Java::CascadingOperation::Identity.new(fields(into_fields)), :output => all_fields
+    end
+
+    # A pipe copy (not a field copy).  Can be used within a branch to copy a
+    # pipe.
+    def pass
+      each all_fields, :function => Java::CascadingOperation::Identity.new
+    end
+  end
+end

data/lib/cascading/mode.rb

@@ -1,21 +1,25 @@
 module Cascading
-  # A Cascading::Mode encapsulates the idea of the execution mode for your
-  # flows.  The default is Hadoop mode, but you can request that your code run
-  # in Cascading local mode.  If you subsequently use a tap or a scheme that
-  # has no local implementation, the mode will be converted back to Hadoop
-  # mode.
+  # A Mode encapsulates the idea of the execution mode for your flows.  The
+  # default is Hadoop mode, but you can request that your code run in Cascading
+  # local mode.  If you subsequently use a tap or a scheme that has no local
+  # implementation, the mode will be converted back to Hadoop mode.
   class Mode
     attr_reader :local
 
-    # Hadoop mode is the default.  You must explicitly request Cascading local
-    # mode with values 'local' or :local.
+    # Parses a specification of which mode, Cascading local mode or Hadoop mode,
+    # to execute in.  Defaults to Hadoop mode.  You may explicitly request
+    # Cascading local mode with values 'local' or :local.  If you pass a Mode
+    # object to this method, it will be passed through.
     def self.parse(mode)
       case mode
+      when Mode then mode
       when 'local', :local then Mode.new(true)
       else Mode.new(false)
       end
     end
 
+    # Constructs a Mode given a flag indicating if it should be Cascading local
+    # mode.
     def initialize(local)
       @local = local
     end
@@ -34,9 +38,9 @@ module Cascading
     end
 
     # Builds a c.f.Flow given properties, name, sources, sinks, and pipes from
-    # a Cascading::Flow.  The current mode is adjusted based on the taps and
-    # schemes of the sources and sinks, then the correct taps are selected
-    # before building the flow.
+    # a Flow.  The current mode is adjusted based on the taps and schemes of
+    # the sources and sinks, then the correct taps are selected before building
+    # the flow.
     def connect_flow(properties, name, sources, sinks, pipes)
       update_local_mode(sources, sinks)
       sources = select_taps(sources)