cascading.jruby 0.0.4 → 0.0.5

data/History.txt

@@ -0,0 +1,17 @@
+0.0.5 - Addressing Janino pain
+
+This release expands upon the ExprStub class adding composition time compilation
+and validation of Janino expressions.  This causes faulty Janino expressions to
+fail early rather than hours into a job.
+
+0.0.4 - Cleanup
+
+This release merges in a good deal of work from Etsy and addresses many internal
+structural issues with cascading.jruby node hierarchies.  Greatly expands unit
+tests and makes samples easily executable as a form of functional testing.
+
+0.0.3 - Etsy's first version of cascading.jruby
+
+First major commit of work from Etsy that introduces scope propagation to
+cascading.jruby scripts in support of field propagation and primary keys.
+Mainly a cleanup and testing release.

data/README.md

@@ -4,4 +4,6 @@
 
 It requires Hadoop (>= 0.18.3) and Cascading (>=1.0.1) to be set via the environment variables: `HADOOP_HOME` and `CASCADING_HOME`
 
+It has been tested on JRuby versions 1.2.0, 1.4.0, and 1.5.3.
+
 Copyright 2009, Grégoire Marabout.

data/lib/cascading/assembly.rb

@@ -10,7 +10,7 @@ module Cascading
   class Assembly < Cascading::Node
     include Operations
 
-    attr_accessor :tail_pipe, :head_pipe, :outgoing_scopes
+    attr_accessor :head_pipe, :tail_pipe, :incoming_scopes, :outgoing_scopes
 
     def initialize(name, parent, outgoing_scopes = {})
       super(name, parent)
@@ -27,6 +27,15 @@ module Cascading
         @outgoing_scopes[name] ||= Scope.empty_scope(name)
       end
       @tail_pipe = @head_pipe
+      @incoming_scopes = [scope]
+    end
+
+    def describe(offset = '')
+      incoming_scopes_desc = "#{incoming_scopes.map{ |incoming_scope| incoming_scope.values_fields.to_a.inspect }.join(', ')}"
+      incoming_scopes_desc = "(#{incoming_scopes_desc})" unless incoming_scopes.size == 1
+      description =  "#{offset}#{name}:assembly :: #{incoming_scopes_desc} -> #{scope.values_fields.to_a.inspect}"
+      description += "\n#{child_names.map{ |child| children[child].describe("#{offset}  ") }.join("\n")}" unless children.empty?
+      description
     end
 
     def parent_flow
@@ -90,7 +99,7 @@ module Cascading
       first_fields = first_fields - scope.grouping_fields.to_a
       if first_fields.size > 0
         first *first_fields
-        puts "Firsting: #{first_fields.inspect} in assembly: #{@name}"
+        puts "Firsting: #{first_fields.inspect} in assembly: #{name}"
       end
 
       bind_names scope.grouping_fields.to_a if every_applied?
@@ -102,14 +111,14 @@ module Cascading
     end
 
     def to_s
-      "#{@name} : head pipe : #{@head_pipe} - tail pipe: #{@tail_pipe}"
+      "#{name} : head pipe : #{@head_pipe} - tail pipe: #{@tail_pipe}"
     end
 
     # Builds a join (CoGroup) pipe. Requires a list of assembly names to join.
     def join(*args, &block)
       options = args.extract_options!
 
-      pipes, incoming_scopes = [], []
+      pipes, @incoming_scopes = [], []
       args.each do |assembly_name|
         assembly = parent_flow.find_child(assembly_name)
         raise "Could not find assembly '#{assembly_name}' in join" unless assembly
@@ -129,7 +138,7 @@ module Cascading
           group_fields << fields(group_fields_args)
         end
       elsif group_fields_args.kind_of?(Hash)
-        pipes, incoming_scopes = [], []
+        pipes, @incoming_scopes = [], []
         keys = group_fields_args.keys.sort
         keys.each do |assembly_name|
           v = group_fields_args[assembly_name]
@@ -233,7 +242,7 @@ module Cascading
     # This actually creates a GroupBy pipe.
     # It expects a list of assembly names as parameter.
     def union_pipes(*args)
-      pipes, incoming_scopes = [], []
+      pipes, @incoming_scopes = [], []
       args[0].each do |assembly_name|
         assembly = parent_flow.find_child(assembly_name)
         pipes << assembly.tail_pipe
@@ -281,9 +290,7 @@ module Cascading
     # Example:
     #     project "field1", "field2"
     def project(*args)
-      fields = fields(args)
-      operation = Java::CascadingOperation::Identity.new
-      make_each(Java::CascadingPipe::Each, @tail_pipe, fields, operation)
+      each fields(args), :function => Java::CascadingOperation::Identity.new
     end
 
     # Removes the specified fields from the current assembly.
@@ -301,9 +308,7 @@ module Cascading
     # Example:
     #     bind_names "field1", "field2"
     def bind_names(*new_names)
-      new_fields = fields(new_names)
-      operation = Java::CascadingOperation::Identity.new(new_fields)
-      make_each(Java::CascadingPipe::Each, @tail_pipe, all_fields, operation)
+      each all_fields, :function => Java::CascadingOperation::Identity.new(fields(new_names))
     end
 
     # Renames fields according to the mapping provided.
@@ -319,9 +324,7 @@ module Cascading
       old_key = scope.primary_key_fields.to_a
       new_key = old_key.map{ |name| name_map[name] || name }
 
-      new_fields = fields(new_names)
-      operation = Java::CascadingOperation::Identity.new(new_fields)
-      make_each(Java::CascadingPipe::Each, @tail_pipe, all_fields, operation)
+      each all_fields, :function => Java::CascadingOperation::Identity.new(fields(new_names))
       primary(*new_key)
     end
 
@@ -330,22 +333,19 @@ module Cascading
       types = JAVA_TYPE_MAP.values_at(*type_map.values_at(*names))
       fields = fields(names)
       types = types.to_java(java.lang.Class)
-      operation = Java::CascadingOperation::Identity.new(fields, types)
-      make_each(Java::CascadingPipe::Each, @tail_pipe, fields, operation)
+      each fields, :function => Java::CascadingOperation::Identity.new(fields, types)
     end
 
     def copy(*args)
       options = args.extract_options!
       from = args[0] || all_fields
       into = args[1] || options[:into] || all_fields
-      operation = Java::CascadingOperation::Identity.new(fields(into))
-      make_each(Java::CascadingPipe::Each, @tail_pipe, fields(from), operation, Java::CascadingTuple::Fields::ALL)
+      each fields(from), :function => Java::CascadingOperation::Identity.new(fields(into)), :output => all_fields
     end
 
     # A pipe that does nothing.
     def pass(*args)
-      operation = Java::CascadingOperation::Identity.new
-      make_each(Java::CascadingPipe::Each, @tail_pipe, all_fields, operation)
+      each all_fields, :function => Java::CascadingOperation::Identity.new
     end
 
     def assert(*args)
@@ -593,9 +593,8 @@ module Cascading
         value = args[field_name]
 
         if value.kind_of?(ExprStub)
-          each all_fields,
-            :function => expression_function(field_name, :expression => value.expression,
-                           :parameters => value.types), :output => all_fields
+          value.validate_scope(scope)
+          each all_fields, :function => expression_function(field_name, :expression => value.expression, :parameters => value.types), :output => all_fields
         else
           each all_fields, :function => insert_function([field_name], :values => [value]), :output => all_fields
         end
@@ -612,15 +611,23 @@ module Cascading
     # * <tt>:expression</tt> a string. Specifies a Janino expression used to filter the tuples. This option has the
     # same effect than providing it as first unamed argument. If this option is provided, then the filter is Janino
     # expression-based. This is incompatible with the _pattern_ option.
+    # * <tt>:validate</tt> a boolean.  Passed into Cascading#expr to enable or disable
+    # expression validation.  Defaults to true.
+    # * <tt>:validate_with</tt> a hash.  Actual arguments used by Cascading#expr for
+    # expression validation.  Defaults to {}.
     def filter(*args)
       options = args.extract_options!
       from = options.delete(:from) || all_fields
       expression = options.delete(:expression) || args.shift
       regex = options.delete(:pattern)
+      validate = options.has_key?(:validate) ? options.delete(:validate) : true
+      validate_with = options.has_key?(:validate_with) ? options.delete(:validate_with) : {}
+
       if expression
-        stub = ExprStub.new(expression)
+        stub = expr(expression, { :validate => validate, :validate_with => validate_with })
         types, expression = stub.types, stub.expression
 
+        stub.validate_scope(scope)
         each from, :filter => expression_filter(
           :parameters => types,
           :expression => expression
@@ -650,6 +657,10 @@ module Cascading
     # * <tt>:expression</tt> a string. Specifies a Janino expression used to filter the tuples. This option has the
     # same effect than providing it as first unamed argument. If this option is provided, then the filter is Janino
     # expression-based.
+    # * <tt>:validate</tt> a boolean.  Passed into Cascading#expr to enable or disable
+    # expression validation.  Defaults to true.
+    # * <tt>:validate_with</tt> a hash.  Actual arguments used by Cascading#expr for
+    # expression validation.  Defaults to {}.
     def reject(*args)
       options = args.extract_options
       raise "Regex not allowed" if options && options[:pattern]
@@ -665,6 +676,10 @@ module Cascading
     # * <tt>:expression</tt> a string. Specifies a Janino expression used to select the tuples. This option has the
     # same effect than providing it as first unamed argument. If this option is provided, then the filter is Janino
     # expression-based.
+    # * <tt>:validate</tt> a boolean.  Passed into Cascading#expr to enable or disable
+    # expression validation.  Defaults to true.
+    # * <tt>:validate_with</tt> a hash.  Actual arguments used by Cascading#expr for
+    # expression validation.  Defaults to {}.
     def where(*args)
       options = args.extract_options
       raise "Regex not allowed" if options && options[:pattern]

data/lib/cascading/base.rb

@@ -4,21 +4,28 @@
 
 module Cascading
   class Node
-    attr_accessor :name, :parent, :children, :last_child
+    attr_accessor :name, :parent, :children, :child_names, :last_child
 
     def initialize(name, parent)
       @name = name
       @parent = parent
       @children = {}
+      @child_names = []
       @last_child = nil
     end
 
     def add_child(node)
       @children[node.name] = node
+      @child_names << node.name
       @last_child = node
       node
     end
 
+    def describe(offset = '')
+      "#{offset}#{name}:node\n#{child_names.map{ |child| children[child].describe("#{offset}  ") }.join("\n")}"
+    end
+    alias desc describe
+
     def find_child(name)
       children.each do |child_name, child|
         return child if child_name == name

data/lib/cascading/cascade.rb

@@ -22,6 +22,10 @@ module Cascading
       flow
     end
 
+    def describe(offset = '')
+      "#{offset}#{name}:cascade\n#{child_names.map{ |child| children[child].describe("#{offset}  ") }.join("\n")}"
+    end
+
     def draw(dir, properties = nil)
       @children.each do |name, flow|
         flow.connect(properties).writeDOT("#{dir}/#{name}.dot")

data/lib/cascading/cascading.rb

@@ -25,9 +25,14 @@ module Cascading
     flow
   end
 
-  def expr(s)
-    return s if s.kind_of?(ExprStub)
-    ExprStub.new(s)
+  def describe
+    Cascade.all.map{ |cascade| cascade.describe }.join("\n")
+  end
+  alias desc describe
+
+  # See ExprStub.expr
+  def expr(expression, params = {})
+    ExprStub.expr(expression, params)
   end
 
   # Creates a cascading.tuple.Fields instance from a string or an array of strings.
@@ -42,7 +47,7 @@ module Cascading
       end
       raise "Fields cannot be nil: #{fields.inspect}" if fields.include?(nil)
     end
-    return Java::CascadingTuple::Fields.new([fields].flatten.map{ |f| f.kind_of?(Fixnum) && JRUBY_VERSION > '1.2.0' ? f.to_java(:int) : f }.to_java(java.lang.Comparable))
+    return Java::CascadingTuple::Fields.new([fields].flatten.map{ |f| f.kind_of?(Fixnum) ? java.lang.Integer.new(f) : f }.to_java(java.lang.Comparable))
   end
 
   def all_fields

data/lib/cascading/cascading_exception.rb

@@ -1,30 +1,35 @@
-# Wrapper meant for NativeExceptions that wrap exceptions from Cascading.  The
-# trouble is that the combined stack traces are so long, printing them case
-# actually omit locations in the cascading.jruby or application code that
-# matter, leaving you with no information about the source of the error.  This
-# class just swallows all the nested exceptions, printing their message, while
-# giving you a direct route into JRuby code to the cause of the problem.
-class CascadingException < StandardError
-  def initialize(native_exception, message)
-    @ne = native_exception
-    super("#{message}\n#{trace_causes(@ne, 1)}")
-  end
+# NativeException wrapper that prints the full nested stack trace of the Java
+# exception and all of its causes wrapped by the NativeException.
+# NativeException by default reveals only the first cause, which is
+# insufficient for tracing cascading.jruby errors into JRuby code or revealing
+# underlying Janino expression problems.
+module Cascading
+  class CascadingException < StandardError
+    def initialize(native_exception, message)
+      @ne = native_exception
+      super("#{message}\n#{trace_causes(@ne, 1)}")
+    end
 
-  def cause(depth)
-    fetch_cause(@ne, depth)
-  end
+    def cause(depth)
+      fetch_cause(@ne, depth)
+    end
 
-  private
+    private
 
-  def fetch_cause(ne, depth)
-    return ne if depth <= 1
-    fetch_cause(ne.cause, depth - 1)
-  end
+    def fetch_cause(ne, depth)
+      return ne if depth <= 1
+      fetch_cause(ne.cause, depth - 1)
+    end
 
-  def trace_causes(ne, depth)
-    return unless ne
-    trace = "Cause #{depth}: #{ne}\n"
-    trace += ne.stack_trace.map { |e| "  at #{e.class_name}.#{e.method_name}(#{e.file_name}:#{e.line_number})" }.join("\n") + "\n" if ne.respond_to?(:stack_trace)
-    trace += "#{trace_causes(ne.cause, depth + 1)}"
+    def trace_causes(ne, depth)
+      return unless ne
+      trace = "Cause #{depth}: #{ne.respond_to?(:java_class) ? ne.java_class : ne.class}: #{ne}\n"
+      if ne.respond_to?(:stack_trace)
+        trace += "#{ne.stack_trace.map{ |e| "  at #{e.class_name}.#{e.method_name}(#{e.file_name}:#{e.line_number})" }.join("\n")}\n"
+      elsif ne.respond_to?(:backtrace)
+        trace += "  #{ne.backtrace.join("\n  ")}\n"
+      end
+      trace += "#{trace_causes(ne.cause, depth + 1)}"
+    end
   end
 end

data/lib/cascading/expr_stub.rb

@@ -1,33 +1,164 @@
-class ExprStub
-  attr_accessor :expression, :types
+module Cascading
+  class ExprStub
+    attr_accessor :expression, :types, :input_expression
 
-  def initialize(st)
-    @expression = st.dup
-    @types = {}
+    # ExprStub requires a Janino expression decorated with field types.  For
+    # example: '"Found: " + (x:int + y:int) + " " + z:string'.  Type names are
+    # defined in Cascading::JAVA_TYPE_MAP.
+    def initialize(expression)
+      @input_expression = expression
+      @expression = expression.dup
+      @types = {}
 
-    # Simple regexp based parser for types
+      # Simple regexp based parser for types
 
-    JAVA_TYPE_MAP.each do |sym, klass|
-      @expression.gsub!(/[A-Za-z0-9_]+:#{sym.to_s}/) do |match|
-        name = match.split(/:/).first.gsub(/\s+/, "")
-        @types[name] = klass
-        match.gsub(/:#{sym.to_s}/, "")
+      JAVA_TYPE_MAP.each do |sym, klass|
+        @expression.gsub!(/[A-Za-z0-9_]+:#{sym.to_s}/) do |match|
+          name = match.split(/:/).first.gsub(/\s+/, "")
+          @types[name] = klass
+          match.gsub(/:#{sym.to_s}/, "")
+        end
       end
     end
-  end
 
-  def self.split_hash(h)
-    keys, values = h.keys.sort, []
-    keys.each do |key|
-      values << h[key]
+    def to_s
+      @input_expression
+    end
+
+    # Convenience constructor for an ExprStub that optionally performs
+    # validation.  Takes a string to use as a Janino expression and an optional
+    # params hash.  By default, the param :validate is set to true (performs
+    # expression validation using default actual argument values) and the param
+    # :validate_with is set to {} (which doesn't override any of the default
+    # actual argument values used for validation).
+    def self.expr(expression, params = {})
+      params = { :validate => true, :validate_with => {} }.merge(params)
+      expr_stub = expression.kind_of?(ExprStub) ? expression : ExprStub.new(expression).compile
+      expr_stub.validate(params[:validate_with]) if params[:validate]
+      puts "Expression validation is disabled for '#{expression}'" unless params[:validate]
+      expr_stub
+    end
+
+    # Scan, parse, and compile expression, then return this ExprStub upon
+    # success.  Throws an CascadingException upon failure.
+    def compile
+      evaluator
+      self
+    end
+
+    # Evaluates this ExprStub given a hash mapping argument names to argument
+    # values. Names may be strings or symbols. Throws an CascadingException
+    # upon failure.
+    def eval(actual_args)
+      actual_args = actual_args.inject({}) do |string_keys, (arg, value)|
+        string_keys[arg.to_s] = specific_to_java(value, @types[arg.to_s])
+        string_keys
+      end
+      args, values = split_hash(actual_args)
+      unused = validate_fields(args)
+      return self.eval(actual_args.reject{ |arg, value| unused.include?(arg) }) unless unused.empty?
+      evaluate(values)
+    end
+
+    # Evaluates this ExprStub with default values for each actual argument.
+    # Values may be overridden with the optional actual_args argument, which
+    # accepts a hash like ExprStub#eval. Throws an CascadingException upon
+    # failure.
+    def validate(actual_args = {})
+      self.eval(test_values.merge(actual_args))
+    end
+
+    def validate_scope(scope)
+      validate_fields(scope.values_fields.to_a)
+    end
+
+    # Throws an exception if any arguments required by this ExprStub are
+    # missing from fields.  Returns those fields which are unused. Throws an
+    # ExprArgException upon failure.
+    def validate_fields(fields)
+      names = @types.keys.sort
+      missing = names - fields
+      raise ExprArgException.new("Expression '#{@expression}' is missing these fields: #{missing.inspect}\nRequires: #{names.inspect}, found: #{fields.inspect}") unless missing.empty?
+      fields - names
+    end
+
+    private
+
+    def split_hash(h)
+      keys, values = h.sort.inject([[], []]) do |(keys, values), (key, value)|
+        [keys << key, values << value]
+      end
+      [keys, values]
+    end
+
+    # Evaluate this ExprStub given an array of actual arguments. Throws an
+    # CascadingException upon failure. GOTCHA: requires values to be in order
+    # of lexicographically sorted formal arguments.
+    def evaluate(values)
+      begin
+        evaluator.evaluate(values.to_java)
+      rescue NativeException => ne
+        raise CascadingException.new(ne, "Exception encountered while evaluating '#{@expression}' with arguments: #{values.inspect}")
+      end
     end
-    [keys, values]
-  end
 
-  def self.split_names_and_types(expr_types)
-    names, types = split_hash(expr_types)
-    names = names.to_java(java.lang.String)
-    types = types.to_java(java.lang.Class)
-    [names, types]
+    # Building an evaluator ensures that the expression scans, parses, and
+    # compiles
+    def evaluator
+      begin
+        names, types = names_and_types
+        Java::OrgCodehausJanino::ExpressionEvaluator.new(@expression, java.lang.Comparable.java_class, names, types)
+      rescue NativeException => ne
+        raise CascadingException.new(ne, "Exception encountered while compiling '#{@expression}'")
+      end
+    end
+
+    # Extract Java names and types from @types hash
+    def names_and_types
+      names, types = split_hash(@types)
+      [names.to_java(java.lang.String), types.to_java(java.lang.Class)]
+    end
+
+    # Makes best effort to convert Ruby numbers into the Java numeric type
+    # exepcted by a Janino expression. However, if the conversion fails, it
+    # returns the original value so that the exception thrown will be from
+    # Janino, not this code.
+    def specific_to_java(value, type)
+      # GOTCHA: Java's Float and Long have constructors that take strings and
+      # parse them.  If value is a string representation of a number, this code
+      # could coerce it to a number whereas invocation of the Janino expression
+      # would fail.  We therefore punt if value is a String.
+      return value if value.kind_of?(::String)
+      if type == java.lang.Float.java_class
+        return value if value.kind_of?(::Integer)
+        java.lang.Float.new(value) rescue value
+      elsif type == java.lang.Long.java_class && JRUBY_VERSION <= '1.2.0'
+        return value if value.kind_of?(::Float)
+        java.lang.Long.new(value) rescue value
+      elsif type == java.lang.Integer.java_class && JRUBY_VERSION > '1.2.0'
+        return value if value.kind_of?(::Float)
+        java.lang.Integer.new(value) rescue value
+      else
+        value
+      end
+    end
+
+    @@defaults = {
+      java.lang.Integer.java_class => JRUBY_VERSION > '1.2.0' ? java.lang.Integer.new(0) : 0,
+      java.lang.Boolean.java_class => false,
+      java.lang.Double.java_class => 0.0,
+      java.lang.Float.java_class => java.lang.Float.new(0.0),
+      java.lang.Long.java_class => JRUBY_VERSION > '1.2.0' ? 0 : java.lang.Long.new(0),
+      java.lang.String.java_class => nil,
+    }
+
+    def test_values
+      @types.sort.inject({}) do |test_values, (name, type)|
+        test_values[name] = @@defaults[type]
+        test_values
+      end
+    end
   end
+
+  class ExprArgException < StandardError; end
 end

data/lib/cascading/flow.rb

@@ -8,10 +8,10 @@ module Cascading
   class Flow < Cascading::Node
     extend Registerable
 
-    attr_accessor :properties, :sources, :sinks, :outgoing_scopes, :listeners
+    attr_accessor :properties, :sources, :sinks, :incoming_scopes, :outgoing_scopes, :listeners
 
     def initialize(name, parent)
-      @properties, @sources, @sinks, @outgoing_scopes, @listeners = {}, {}, {}, {}, []
+      @properties, @sources, @sinks, @incoming_scopes, @outgoing_scopes, @listeners = {}, {}, {}, {}, {}, []
       super(name, parent)
       self.class.add(name, self)
     end
@@ -29,9 +29,9 @@ module Cascading
     # reference a path.
     def sink(*args)
       if (args.size == 2)
-        @sinks[args[0]] = args[1]
+        sinks[args[0]] = args[1]
       elsif (args.size == 1)
-        @sinks[@name] =  args[0]
+        sinks[name] =  args[0]
       end
     end
 
@@ -40,14 +40,24 @@ module Cascading
     # reference a path.
     def source(*args)
       if (args.size == 2)
-        @sources[args[0]] = args[1]
-        @outgoing_scopes[args[0]] = Scope.tap_scope(args[1], args[0])
+        sources[args[0]] = args[1]
+        incoming_scopes[args[0]] = Scope.tap_scope(args[1], args[0])
+        outgoing_scopes[args[0]] = incoming_scopes[args[0]]
       elsif (args.size == 1)
-        @sources[@name] = args[0]
-        @outgoing_scopes[@name] = Scope.empty_scope(@name)
+        sources[name] = args[0]
+        incoming_scopes[name] = Scope.empty_scope(name)
+        outgoing_scopes[name] = incoming_scopes[name]
       end
     end
 
+    def describe(offset = '')
+      description =  "#{offset}#{name}:flow\n"
+      description += "#{sources.keys.map{ |source| "#{offset}  #{source}:source :: #{incoming_scopes[source].values_fields.to_a.inspect}" }.join("\n")}\n"
+      description += "#{child_names.map{ |child| children[child].describe("#{offset}  ") }.join("\n")}\n"
+      description += "#{sinks.keys.map{ |sink| "#{offset}  #{sink}:sink :: #{outgoing_scopes[sink].values_fields.to_a.inspect}" }.join("\n")}"
+      description
+    end
+
     def scope(name = nil)
       raise 'Must specify name if no children have been defined yet' unless name || last_child
       name ||= last_child.name

data/lib/cascading.rb

@@ -6,7 +6,7 @@ require 'java'
 
 module Cascading
   # :stopdoc:
-  VERSION = '0.0.4'
+  VERSION = '0.0.5'
   LIBPATH = ::File.expand_path(::File.dirname(__FILE__)) + ::File::SEPARATOR
   PATH = ::File.dirname(LIBPATH) + ::File::SEPARATOR
   CASCADING_HOME = ENV['CASCADING_HOME']

data/samples/copy.rb

@@ -10,7 +10,8 @@ cascade 'copy' do
 
     assembly 'input' do
       rename 'line' => 'value'
-      reject 'value:string.indexOf("R") == -1'
+      # We override validate_with because we know line will never be null
+      reject 'value:string.indexOf("R") == -1', :validate_with => { :value => 'nothinghere' }
     end
 
     sink 'input', tap('output/copy', :sink_mode => :replace)

data/spec/cascading_spec.rb

@@ -1,6 +1,11 @@
 require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
 
-describe Cascading do
+context Cascading do
+  it 'should handle string and integer field names' do
+    f = fields(['a', 1, 'b', 2])
+    f.to_a.should == ['a', 1, 'b', 2]
+  end
+
   it 'should dedup field names from multiple sources' do
     left_names = ['a', 'b', 'c', 'd', 'e']
     mid_names = ['a', 'f']