cascading.jruby 0.0.10 → 1.0.0

data/lib/cascading/operations.rb

@@ -1,116 +1,118 @@
 module Cascading
-  # The Cascading::Operations module is deprecated.  The original idea from long
-  # ago is that it would be useful to mixin operator wrappers to places other
-  # than Cascading::Assembly, but this is not true.  Instead, put Eaches in
-  # Cascading::Assembly, Everies in Cascading::Aggregations, and any more
-  # generally useful utility code directly in the Cascading module
-  # (cascading/cascading.rb).
-  #
-  # Further, the entire *args pattern should be deprecated as it leads to
-  # functions that can only be understood by reading their code.  Instead,
-  # idiomatic Ruby (positional required params and a params hash for optional
-  # args) should be used.  See Cascading::Assembly#set_value for an example.
   module Operations
-    def identity
-      Java::CascadingOperation::Identity.new
-    end
-
-    def aggregator_function(args, aggregator_klass)
-      options = args.extract_options!
-      ignore = options[:ignore]
-
-      parameters = [Cascading.fields(args), ignore].compact
-      aggregator_klass.new(*parameters)
-    end
-
-    def first_function(*args)
-      aggregator_function(args, Java::CascadingOperationAggregator::First)
-    end
-
-    def min_function(*args)
-      aggregator_function(args, Java::CascadingOperationAggregator::Min)
-    end
-
-    def max_function(*args)
-      aggregator_function(args, Java::CascadingOperationAggregator::Max)
-    end
-
-    def last_function(*args)
-      aggregator_function(args, Java::CascadingOperationAggregator::Last)
-    end
-
-    def regex_parser(*args)
-      options = args.extract_options!
-
-      pattern = args[0].to_s
-      fields = Cascading.fields(options[:fields])
-      groups = options[:groups].to_java(:int) if options[:groups]
-      parameters = [fields, pattern, groups].compact
-
-      Java::CascadingOperationRegex::RegexParser.new(*parameters)
-    end
-
-    def regex_splitter(*args)
-      options = args.extract_options!
-
-      fields = Cascading.fields(args)
-      pattern = options[:pattern].to_s
-      parameters = [fields, pattern].compact
-      Java::CascadingOperationRegex::RegexSplitter.new(*parameters)
-    end
-
-    def regex_split_generator(*args)
-      options = args.extract_options!
-
-      fields = Cascading.fields(args)
-      pattern = options[:pattern].to_s
-      parameters = [fields, pattern].compact
-      Java::CascadingOperationRegex::RegexSplitGenerator.new(*parameters)
-    end
-
-    def regex_generator(*args)
-      options = args.extract_options!
-
-      fields = Cascading.fields(args)
-      pattern = options[:pattern].to_s
-      parameters = [fields, pattern].compact
-      Java::CascadingOperationRegex::RegexGenerator.new(*parameters)
-    end
-
-    def expression_function(*args)
-      options = args.extract_options!
-
-      fields = Cascading.fields(args)
-      expression = options[:expression].to_s
-      parameters = options[:parameters]
-      parameter_names = []
-      parameter_types = []
-      if parameters.is_a? ::Hash
-        parameters.each do |name, type|
-          parameter_names << name
-          parameter_types << type
+    # Debugs the current assembly at runtime, printing every tuple and fields
+    # every 10 tuples by default.
+    #
+    # The named options are:
+    # [prefix] String to prefix prints with.
+    # [print_fields] Boolean controlling field printing, defaults to false.
+    # [tuple_interval] Integer specifying interval between printed tuples
+    # [fields_interval] Integer specifying interval between printing fields
+    #
+    # Example:
+    #     debug :prefix => 'DEBUG', :print_fields => true, :fields_interval => 1000
+    def debug(options = {})
+      input_fields = options[:input] || all_fields
+      prefix = options[:prefix]
+      print_fields = options[:print_fields]
+
+      debug = Java::CascadingOperation::Debug.new(*[prefix, print_fields].compact)
+
+      debug.print_tuple_every = options[:tuple_interval] || 1
+      debug.print_fields_every = options[:fields_interval] || 10
+
+      each(input_fields, :filter => debug)
+    end
+
+    # Inserts new fields into the current assembly.  Values may be constants or
+    # expressions (see Cascading::expr).  Fields will be inserted in
+    # lexicographic order (not necessarily the order provided).
+    #
+    # Example:
+    #     insert 'field1' => 'constant_string', 'field2' => 0, 'field3' => expr('fieldA:long + fieldB:long')
+    def insert(insert_map)
+      insert_map.keys.sort.each do |field_name|
+        value = insert_map[field_name]
+
+        if value.kind_of?(ExprStub)
+          value.validate_scope(scope)
+          names, types = value.names_and_types
+          each(
+            all_fields,
+            :function => Java::CascadingOperationExpression::ExpressionFunction.new(fields(field_name), value.expression, names, types),
+            :output => all_fields
+          )
+        else # value is a constant
+          each(
+            all_fields,
+            :function => Java::CascadingOperation::Insert.new(fields(field_name), to_java_comparable_array([value])),
+            :output => all_fields
+          )
         end
-        parameter_names = parameter_names.to_java(java.lang.String)
-        parameter_types = parameter_types.to_java(java.lang.Class)
-
-        arguments = [fields, expression, parameter_names, parameter_types].compact
-      elsif !parameters.nil?
-        arguments = [fields, expression, parameters.java_class].compact
-      else
-        arguments = [fields, expression, java.lang.String.java_class].compact
       end
-
-      Java::CascadingOperationExpression::ExpressionFunction.new(*arguments)
     end
 
-    def insert_function(*args)
-      options=args.extract_options!
-      fields = Cascading.fields(args)
-      values = options[:values]
-
-      parameters = [fields, to_java_comparable_array(values)].compact
-      Java::CascadingOperation::Insert.new(*parameters)
-    end
+    # Ungroups, or unpivots, a tuple (see Cascading's {UnGroup}[http://docs.cascading.org/cascading/2.1/javadoc/cascading/operation/function/UnGroup.html]).
+    #
+    # You must provide exactly one of :value_selectors and :num_values.
+    #
+    # The named options are:
+    # [value_selectors] Array of field names to ungroup. Each field will be
+    #                   ungrouped into an output tuple along with the key fields
+    #                   in the order provided.
+    # [num_values] Integer specifying the number of fields to ungroup into each
+    #              output tuple (excluding the key fields).  All input fields
+    #              will be ungrouped.
+    #
+    # Example:
+    #     ungroup 'key', ['new_key', 'val], :value_selectors => ['val1', 'val2', 'val3'], :output => ['new_key', 'val']
+    def ungroup(key, into_fields, options = {})
+      input_fields = options[:input] || all_fields
+      output = options[:output] || all_fields
+
+      raise 'You must provide exactly one of :value_selectors or :num_values to ungroup' unless options.has_key?(:value_selectors) ^ options.has_key?(:num_values)
+      value_selectors = options[:value_selectors].map{ |vs| fields(vs) }.to_java(Java::CascadingTuple::Fields) if options.has_key?(:value_selectors)
+      num_values = options[:num_values] if options.has_key?(:num_values)
+
+      parameters = [fields(into_fields), fields(key), value_selectors, num_values].compact
+      each input_fields, :function => Java::CascadingOperationFunction::UnGroup.new(*parameters), :output => output
+    end
+
+    # Inserts one of two values into the dataflow based upon the result of the
+    # supplied filter on the input_fields.  This is primarily useful for
+    # creating indicators from filters.  keep_value specifies the Java value to
+    # produce when the filter would keep the given input and remove_value
+    # specifies the Java value to produce when the filter would remove the given
+    # input.
+    #
+    # Example:
+    #     set_value 'field1', Java::CascadingOperationFilter::FilterNull.new, 1.to_java, 0.to_java, 'is_field1_null'
+    def set_value(input_fields, filter, keep_value, remove_value, into_field, options = {})
+      output = options[:output] || all_fields
+      each input_fields, :function => Java::CascadingOperationFunction::SetValue.new(fields(into_field), filter, keep_value, remove_value), :output => output
+    end
+
+    # Efficient way of inserting a null indicator for any field, even one that
+    # cannot be coerced to a string.  This is accomplished using Cascading's
+    # FilterNull and SetValue operators rather than Janino.  1 is produced if
+    # the field is null and 0 otherwise.
+    #
+    # Example:
+    #     null_indicator 'field1', 'is_field1_null'
+    def null_indicator(input_field, into_field, options = {})
+      set_value input_field, Java::CascadingOperationFilter::FilterNull.new, 1.to_java, 0.to_java, into_field, :output => options[:output]
+    end
+
+    # Given an input_field and a regex, returns an indicator that is 1 if the string
+    # contains at least 1 match and 0 otherwise.
+    #
+    # Example:
+    #     regex_contains 'field1', /\w+\s+\w+/, 'does_field1_contain_pair'
+    def regex_contains(input_field, regex, into_field, options = {})
+      set_value input_field, Java::CascadingOperationRegex::RegexFilter.new(pattern.to_s), 1.to_java, 0.to_java, into_field, :output => options[:output]
+    end
+
+    private
 
     def to_java_comparable_array(arr)
       (arr.map do |v|
@@ -130,72 +132,5 @@ module Cascading
           java.lang.String.new(v.to_s)
       end
     end
-
-    def expression_filter(*args)
-      options = args.extract_options!
-      expression = (args[0] || options[:expression]).to_s
-      parameters = options[:parameters]
-      parameter_names = []
-      parameter_types = []
-      if parameters.is_a? ::Hash
-        parameters.each do |name, type|
-          parameter_names << name
-          parameter_types << type
-        end
-        parameter_names = parameter_names.to_java(java.lang.String)
-        parameter_types = parameter_types.to_java(java.lang.Class)
-
-        arguments = [expression, parameter_names, parameter_types].compact
-      elsif !parameters.nil?
-        arguments = [expression, parameters.java_class].compact
-      else
-        arguments = [expression, java.lang.String.java_class].compact
-      end
-
-      Java::CascadingOperationExpression::ExpressionFilter.new(*arguments)
-    end
-
-    def date_parser(field, format)
-      fields = fields(field)
-      Java::CascadingOperationText::DateParser.new(fields, format)
-    end
-
-    def date_formatter(fields, format, timezone=nil)
-      fields = fields(fields)
-      timezone = Java::JavaUtil::TimeZone.get_time_zone(timezone) if timezone
-      arguments = [fields, format, timezone].compact
-      Java::CascadingOperationText::DateFormatter.new(*arguments)
-    end
-
-    def regex_filter(*args)
-      options = args.extract_options!
-
-      pattern = args[0]
-      remove_match = options[:remove_match]
-      match_each_element = options[:match_each_element]
-      parameters = [pattern.to_s, remove_match, match_each_element].compact
-      Java::CascadingOperationRegex::RegexFilter.new(*parameters)
-    end
-
-    def regex_replace(*args)
-      options = args.extract_options!
-
-      fields = fields(args[0])
-      pattern = args[1]
-      replacement = args[2]
-      replace_all = options[:replace_all]
-
-      parameters = [fields, pattern.to_s, replacement.to_s, replace_all].compact
-      Java::CascadingOperationRegex::RegexReplace.new(*parameters)
-    end
-
-    def field_joiner(*args)
-      options = args.extract_options!
-      delimiter = options[:delimiter] || ','
-      fields = fields(options[:into])
-
-      parameters = [fields, delimiter].compact
-      Java::CascadingOperationText::FieldJoiner.new(*parameters)
-    end
   end
 end

data/lib/cascading/regex_operations.rb

@@ -0,0 +1,133 @@
+module Cascading
+  # Module of pipe assemblies that wrap operations defined in the Cascading
+  # cascading.operations.regex package.  These are split out only to group
+  # similar functionality.
+  #
+  # All DSL regex pipes require an input_field, a regex, and either a single
+  # into_field or one or more into_fields.  Requiring a single input field
+  # allows us to raise an exception early if the wrong input is specified and
+  # avoids the non-intuitive situation where the first of many fields is
+  # silently taken as in Cascading.  Requiring a regex means you don't have to
+  # go looking for defaults in code.  And into_field(s) means we can propagate
+  # field names through the dataflow.
+  #
+  # Mapping of DSL pipes into Cascading regex operations:
+  # parse:: {RegexParser}[http://docs.cascading.org/cascading/2.1/javadoc/cascading/operation/regex/RegexParser.html]
+  # split:: {RegexSplitter}[http://docs.cascading.org/cascading/2.1/javadoc/cascading/operation/regex/RegexSplitter.html]
+  # split\_rows:: {RegexSplitGenerator}[http://docs.cascading.org/cascading/2.1/javadoc/cascading/operation/regex/RegexSplitGenerator.html]
+  # match\_rows:: {RegexGenerator}[http://docs.cascading.org/cascading/2.1/javadoc/cascading/operation/regex/RegexGenerator.html]
+  # replace:: {RegexReplace}[http://docs.cascading.org/cascading/2.1/javadoc/cascading/operation/regex/RegexReplace.html]
+  module RegexOperations
+    # Parses the given input_field using the specified regular expression to
+    # produce one output per group in that expression.
+    #
+    # The named options are:
+    # [groups] Array of integers specifying which groups to capture if you want
+    #          a subset of groups.
+    #
+    # Example:
+    #     parse 'field1', /(\w+)\s+(\w+)/, ['out1', 'out2'], :groups => [1, 2]
+    def parse(input_field, regex, into_fields, options = {})
+      groups = options[:groups].to_java(:int) if options[:groups]
+      output = options[:output] || all_fields # Overrides Cascading default
+
+      input_field = fields(input_field)
+      raise "input_field must declare exactly one field, was '#{input_field}'" unless input_field.size == 1
+
+      parameters = [fields(into_fields), regex.to_s, groups].compact
+      each(
+        input_field,
+        :function => Java::CascadingOperationRegex::RegexParser.new(*parameters),
+        :output => output
+      )
+    end
+    alias regex_parser parse
+
+    # Splits the given input_field into multiple fields using the specified
+    # regular expression.
+    #
+    # Example:
+    #     split 'line', /\s+/, ['out1', 'out2']
+    def split(input_field, regex, into_fields, options = {})
+      output = options[:output] || all_fields # Overrides Cascading default
+
+      input_field = fields(input_field)
+      raise "input_field must declare exactly one field, was '#{input_field}'" unless input_field.size == 1
+
+      each(
+        input_field,
+        :function => Java::CascadingOperationRegex::RegexSplitter.new(fields(into_fields), regex.to_s),
+        :output => output
+      )
+    end
+    alias regex_splitter split
+
+    # Splits the given input_field into new rows using the specified regular
+    # expression.
+    #
+    # Example:
+    #     split_rows 'line', /\s+/, 'word'
+    def split_rows(input_field, regex, into_field, options = {})
+      output = options[:output] || all_fields # Overrides Cascading default
+
+      input_field = fields(input_field)
+      raise "input_field must declare exactly one field, was '#{input_field}'" unless input_field.size == 1
+      into_field = fields(into_field)
+      raise "into_field must declare exactly one field, was '#{into_field}'" unless into_field.size == 1
+
+      each(
+        input_field,
+        :function => Java::CascadingOperationRegex::RegexSplitGenerator.new(into_field, regex.to_s),
+        :output => output
+      )
+    end
+    alias regex_split_generator split_rows
+
+    # Emits a new row for each regex group matched in input_field using the
+    # specified regular expression.
+    #
+    # Example:
+    #     match_rows 'line', /(\w+)\s+(\w+)/, 'word'
+    def match_rows(input_field, regex, into_field, options = {})
+      output = options[:output] || all_fields # Overrides Cascading default
+
+      input_field = fields(input_field)
+      raise "input_field must declare exactly one field, was '#{input_field}'" unless input_field.size == 1
+      into_field = fields(into_field)
+      raise "into_field must declare exactly one field, was '#{into_field}'" unless into_field.size == 1
+
+      each(
+        input_field,
+        :function => Java::CascadingOperationRegex::RegexGenerator.new(into_field, regex.to_s),
+        :output => output
+      )
+    end
+    alias regex_generator match_rows
+
+    # Performs a query/replace on the given input_field using the specified
+    # regular expression and replacement.
+    #
+    # The named options are:
+    # [replace_all] Boolean indicating if all matches should be replaced;
+    #               defaults to true (the Cascading default).
+    #
+    # Example:
+    #     replace 'line', /[.,]*\s+/, 'tab_separated_line', "\t"
+    def replace(input_field, regex, into_field, replacement, options = {})
+      output = options[:output] || all_fields # Overrides Cascading default
+
+      input_field = fields(input_field)
+      raise "input_field must declare exactly one field, was '#{input_field}'" unless input_field.size == 1
+      into_field = fields(into_field)
+      raise "into_field must declare exactly one field, was '#{into_field}'" unless into_field.size == 1
+
+      parameters = [into_field, regex.to_s, replacement.to_s, options[:replace_all]].compact
+      each(
+        input_field,
+        :function => Java::CascadingOperationRegex::RegexReplace.new(*parameters),
+        :output => output
+      )
+    end
+    alias regex_replace replace
+  end
+end

data/lib/cascading/scope.rb

@@ -1,23 +1,35 @@
 module Cascading
+  # Scope is a wrapper for a the private Cascading c.f.p.Scope object used to
+  # connect the dataflow graph by resolving fields.  cascading.jruby wraps this
+  # facility so that it may be used to propagate field names at composition
+  # time (not Cascading plan time) in the same way they will later be
+  # propagated by the planner.
   class Scope
     attr_accessor :scope
 
+    # Construct a Scope given the Cascading c.f.p.Scope to wrap.
     def initialize(scope)
       @scope = scope
     end
 
+    # Copy one Scope into another; relies upon the copy constructor of
+    # c.f.p.Scope.
     def copy
       Scope.new(Java::CascadingFlowPlanner::Scope.new(@scope))
     end
 
+    # Build a c.f.p.Scope for a Flow, which is empty except for its name.
     def self.flow_scope(name)
       Java::CascadingFlowPlanner::Scope.new(name)
     end
 
+    # Build an empty Scope, wrapping an empty c.f.p.Scope.
     def self.empty_scope(name)
       Scope.new(Java::CascadingFlowPlanner::Scope.new(name))
     end
 
+    # Build a Scope for a single source Tap.  The flow_scope is propagated
+    # through this call into a new Scope.
     def self.source_scope(name, tap, flow_scope)
       incoming_scopes = java.util.HashSet.new
       incoming_scopes.add(flow_scope)
@@ -27,28 +39,30 @@ module Cascading
       Scope.new(java_scope)
     end
 
+    # Build a Scope for an arbitrary flow element.  This is used to update the
+    # Scope at each stage in a pipe Assembly.
     def self.outgoing_scope(flow_element, incoming_scopes)
       java_scopes = incoming_scopes.compact.map{ |s| s.scope }
       Scope.new(outgoing_scope_for(flow_element, java.util.HashSet.new(java_scopes)))
     end
 
+    # The values fields of the Scope, which indicate the fields in the current
+    # dataflow tuple.
     def values_fields
       @scope.out_values_fields
     end
 
+    # The grouping fields of the Scope, which indicate the keys of an
+    # group/cogroup.
     def grouping_fields
       @scope.out_grouping_fields
     end
 
-    def scope_fields_to_s(accessor)
-      begin
-        fields = @scope.send(accessor)
-        fields.nil? ? 'null' : fields.to_s
-      rescue
-        'ERROR'
-      end
-    end
-
+    # Prints a detailed description of this Scope, including its type and
+    # various selectors, fields, and key fields.  Data is bubbled up directly
+    # from the Cascading c.f.p.Scope.  This output can be useful for debugging
+    # the propagation of fields through your job (see Flow#debug_scope and
+    # Assembly#debug_scope, which both rely upon this method).
     def to_s
       kind = 'Unknown'
       kind = 'Tap'   if @scope.tap?
@@ -77,6 +91,15 @@ END
 
     private
 
+    def scope_fields_to_s(accessor)
+      begin
+        fields = @scope.send(accessor)
+        fields.nil? ? 'null' : fields.to_s
+      rescue Exception => e
+        'ERROR'
+      end
+    end
+
     def self.outgoing_scope_for(flow_element, incoming_scopes)
       begin
         flow_element.outgoing_scope_for(incoming_scopes)