cascading.jruby 0.0.5 → 0.0.6

data/HACKING.md

@@ -4,7 +4,7 @@ Some hacking info on `cascading.jruby`:
 
 `cascading.jruby` can be packaged as a gem. To do so, you must generate the necessary packaging files:
 
-    ant build; jruby -S rake gem
+    jruby -S rake gem
 
 will produce the gem in the pkg/ sub-directory. After that, just cd to this directory and:
 

data/History.txt

@@ -1,3 +1,9 @@
+0.0.6 - Removing primary key
+
+The primary key feature was a source of great confusion at Etsy, so it's been
+removed.  This release also closes several issues, warns about potential node
+name conflicts, and updates the tests to work under JRuby 1.6.5.
+
 0.0.5 - Addressing Janino pain
 
 This release expands upon the ExprStub class adding composition time compilation

data/README.md

@@ -4,6 +4,6 @@
 
 It requires Hadoop (>= 0.18.3) and Cascading (>=1.0.1) to be set via the environment variables: `HADOOP_HOME` and `CASCADING_HOME`
 
-It has been tested on JRuby versions 1.2.0, 1.4.0, and 1.5.3.
+It has been tested on JRuby versions 1.2.0, 1.4.0, 1.5.3, and 1.6.5.
 
 Copyright 2009, Grégoire Marabout.

data/TODO

@@ -0,0 +1,14 @@
+Documentation
+
+Assembly -> Each/Every refactor
+Look into totally eliminating registries
+
+Bug fixes on github
+Enforce more runtime rules at composition time
+Standardize helper contracts
+Upgrade Cascading (already upgraded JRuby)
+Possibly combine unit tests...into unit tests because RSpec sucks and swallows stack traces
+
+Split out runner
+Make runner implement Tool
+Create build tool for job jar

data/lib/cascading/assembly.rb

@@ -51,16 +51,6 @@ module Cascading
       puts "Current scope for '#{name}':\n  #{scope}\n----------\n"
     end
 
-    def primary(*args)
-      options = args.extract_options!
-      if args.size > 0 && args[0] != nil
-        scope.primary_key_fields = fields(args)
-      else
-        scope.primary_key_fields = nil
-      end
-      scope.grouping_primary_key_fields = scope.primary_key_fields
-    end
-
     def make_each(type, *parameters)
       make_pipe(type, parameters)
       @every_applied = false
@@ -82,26 +72,6 @@ module Cascading
       # that only allows aggregation and buffer operations.
       instance_eval &block
 
-      # First all non-primary key fields from each pipe if its primary key is a
-      # subset of the grouping primary key
-      first_fields = incoming_scopes.map do |scope|
-        if scope.primary_key_fields
-          primary_key = scope.primary_key_fields.to_a
-          grouping_primary_key = scope.grouping_primary_key_fields.to_a
-          if (primary_key & grouping_primary_key) == primary_key
-            difference_fields(scope.values_fields, scope.primary_key_fields).to_a
-          end
-        end
-      end.compact.flatten
-      # assert first_fields == first_fields.uniq
-
-      # Do no first any fields explicitly aggregated over
-      first_fields = first_fields - scope.grouping_fields.to_a
-      if first_fields.size > 0
-        first *first_fields
-        puts "Firsting: #{first_fields.inspect} in assembly: #{name}"
-      end
-
       bind_names scope.grouping_fields.to_a if every_applied?
     end
 
@@ -321,11 +291,7 @@ module Cascading
       invalid = name_map.keys.sort - old_names
       raise "invalid names: #{invalid.inspect}" unless invalid.empty?
 
-      old_key = scope.primary_key_fields.to_a
-      new_key = old_key.map{ |name| name_map[name] || name }
-
       each all_fields, :function => Java::CascadingOperation::Identity.new(fields(new_names))
-      primary(*new_key)
     end
 
     def cast(type_map)
@@ -409,7 +375,8 @@ module Cascading
     # insert) and an options hash.
     #
     # Options include:
-    #   * <tt>:sql</tt> a boolean indicating whether the operation should act like the SQL equivalent
+    #   * <tt>:ignore</tt> a Java Array of Objects (for min and max) or Tuples
+    #     (for first and last) of values for the aggregator to ignore
     #
     # <tt>function</tt> is a symbol that is the method to call to construct the Cascading Aggregator.
     def composite_aggregator(args, function)
@@ -685,9 +652,11 @@ module Cascading
       raise "Regex not allowed" if options && options[:pattern]
 
       if options[:expression]
-        options[:expression] = "!(#{options[:expression]})"
+        _, imports, expr = options[:expression].match(/^((?:\s*import.*;\s*)*)(.*)$/).to_a
+        options[:expression] = "#{imports}!(#{expr})"
       elsif args[0]
-        args[0] = "!(#{args[0]})"
+        _, imports, expr = args[0].match(/^((?:\s*import.*;\s*)*)(.*)$/).to_a
+        args[0] = "#{imports}!(#{expr})"
       end
 
       filter(*args)

data/lib/cascading/base.rb

@@ -15,12 +15,19 @@ module Cascading
     end
 
     def add_child(node)
+      child = root.find_child(node.name)
+      warn "WARNING: adding '#{node.qualified_name}', but node named '#{node.name}' already exists at '#{child.qualified_name}'" if child
+
       @children[node.name] = node
       @child_names << node.name
       @last_child = node
       node
     end
 
+    def qualified_name
+      parent ? "#{parent.qualified_name}.#{name}" : name
+    end
+
     def describe(offset = '')
       "#{offset}#{name}:node\n#{child_names.map{ |child| children[child].describe("#{offset}  ") }.join("\n")}"
     end
@@ -32,7 +39,12 @@ module Cascading
         result = child.find_child(name)
         return result if result
       end
-      return nil
+      nil
+    end
+
+    def root
+      return self unless parent
+      parent.root
     end
   end
 
@@ -57,6 +69,7 @@ module Cascading
 
     def add(name, instance)
       @registered ||= {}
+      warn "WARNING: node named '#{name}' already registered in #{self}" if @registered[name]
       @registered[name] = instance
     end
 

data/lib/cascading/cascading.rb

@@ -20,6 +20,7 @@ module Cascading
 
   # For applications built of Flows with no Cascades
   def flow(name, &block)
+    raise "Could not build flow '#{name}'; block required" unless block_given?
     flow = Flow.new(name, nil)
     flow.instance_eval(&block)
     flow

data/lib/cascading/flow.rb

@@ -75,7 +75,6 @@ module Cascading
         raise "Cannot sink undefined assembly '#{sink_name}'" unless @outgoing_scopes[sink_name]
         sink_metadata[sink_name] = {
           :field_names => @outgoing_scopes[sink_name].values_fields.to_a,
-          :primary_key => @outgoing_scopes[sink_name].primary_key_fields.to_a
         }
         sink_metadata
       end
@@ -135,7 +134,16 @@ module Cascading
     end
 
     def connect(properties = nil)
-      properties = java.util.HashMap.new(properties || @properties)
+      # This ensures we have a hash, and that it is a Ruby Hash (because we
+      # also accept java.util.HashMap), then merges it with Flow properties
+      properties ||= {}
+      properties = java.util.HashMap.new(@properties.merge(Hash[*properties.to_a.flatten]))
+
+      puts "Connecting flow '#{name}' with properties:"
+      properties.key_set.to_a.sort.each do |key|
+        puts "#{key}=#{properties[key]}"
+      end
+
       Java::CascadingFlow::FlowConnector.new(properties).connect(
         name,
         make_tap_parameter(@sources),

data/lib/cascading/operations.rb

@@ -19,8 +19,14 @@ module Cascading
 
     def aggregator_function(args, aggregator_klass)
       options = args.extract_options!
-      ignore_values = options[:sql] ? [nil].to_java(java.lang.Object) : nil
-      parameters = [Cascading.fields(args), ignore_values].compact
+      ignore = options[:ignore]
+      raise "Option 'ignore' is only supported by min, max, first, and last" if ignore && ![
+        Java::CascadingOperationAggregator::First,
+        Java::CascadingOperationAggregator::Min,
+        Java::CascadingOperationAggregator::Max,
+        Java::CascadingOperationAggregator::Last,
+      ].include?(aggregator_klass)
+      parameters = [Cascading.fields(args), ignore].compact
       aggregator_klass.new(*parameters)
     end
 

data/lib/cascading/scope.rb

@@ -1,25 +1,14 @@
 module Cascading
   class Scope
-    attr_accessor :scope, :grouping_key_fields, :primary_key_fields, :grouping_primary_key_fields
-    @@scheme_keys = {}
+    attr_accessor :scope, :grouping_key_fields
 
     def initialize(scope, params = {})
       @scope = scope
       @grouping_key_fields = fields(params[:grouping_key_fields] || [])
-      @primary_key_fields = fields(params[:primary_key_fields])
-      @grouping_primary_key_fields = fields(params[:grouping_primary_key_fields])
     end
 
     def copy
-      Scope.new(Java::CascadingFlow::Scope.new(@scope),
-          :grouping_key_fields => @grouping_key_fields,
-          :primary_key_fields => @primary_key_fields,
-          :grouping_primary_key_fields => @grouping_primary_key_fields
-      )
-    end
-
-    def self.register_scheme_key(scheme, primary_key)
-      @@scheme_keys[scheme] = primary_key
+      Scope.new(Java::CascadingFlow::Scope.new(@scope), :grouping_key_fields => @grouping_key_fields)
     end
 
     def self.empty_scope(name)
@@ -30,26 +19,14 @@ module Cascading
       java_scope = outgoing_scope_for(tap, java.util.HashSet.new)
       # Taps and Pipes don't name their outgoing scopes like other FlowElements
       java_scope.name = name
-      scope = Scope.new(java_scope,
-          :primary_key_fields => @@scheme_keys[tap.scheme.class],
-          :grouping_primary_key_fields => @@scheme_keys[tap.scheme.class]
-      )
-      vf, gf = scope.values_fields.to_a, scope.grouping_fields.to_a
-      pk, gpk = scope.primary_key_fields.to_a, scope.grouping_primary_key_fields.to_a
-      raise "Primary key must be a subset of available fields (primary key: #{pk.inspect}, values fields: #{vf.inspect})" unless vf & pk == pk
-      raise "Grouping primary key must be a subset of available fields (grouping primary key: #{gpk.inspect}, grouping fields: #{gf.inspect})" unless gf & gpk == gpk
-      scope
+      Scope.new(java_scope)
     end
 
     def self.outgoing_scope(flow_element, incoming_scopes, grouping_key_fields, every_applied)
       java_scopes = incoming_scopes.compact.map{ |s| s.scope }
-      scope = Scope.new(outgoing_scope_for(flow_element, java.util.HashSet.new(java_scopes)),
+      Scope.new(outgoing_scope_for(flow_element, java.util.HashSet.new(java_scopes)),
           :grouping_key_fields => grouping_key_fields
       )
-      scope.grouping_primary_key_fields = fields(grouping_primary_key_fields(flow_element, incoming_scopes, scope))
-      scope.primary_key_fields = scope.grouping_primary_key_fields if every_applied
-      scope.primary_key_fields = fields(primary_key_fields(flow_element, incoming_scopes, scope)) unless every_applied
-      scope
     end
 
     def values_fields
@@ -80,11 +57,9 @@ Scope name: #{@scope.name}
     selector: #{@scope.out_grouping_selector}
     fields: #{grouping_fields}
     key fields: #{@grouping_key_fields}
-    primary key fields: #{@grouping_primary_key_fields}
   Out values
     selector: #{@scope.out_values_selector}
     fields: #{values_fields}
-    primary key fields: #{@primary_key_fields}
 END
     end
 
@@ -97,64 +72,5 @@ END
         raise CascadingException.new(e, 'Exception computing outgoing scope')
       end
     end
-
-    def self.primary_key_fields(flow_element, incoming_scopes, scope)
-      case flow_element
-        when Java::CascadingPipe::Each
-          # assert incoming_scopes.size == 1
-          project_primary_key(incoming_scopes.first.primary_key_fields,
-                              incoming_scopes.first.values_fields.to_a,
-                              scope.values_fields.to_a)
-        when Java::CascadingPipe::Every
-          # assert incoming_scopes.size == 1
-          incoming_scopes.first.primary_key_fields
-        when Java::CascadingPipe::GroupBy
-          if incoming_scopes.size == 1
-            incoming_scopes.first.primary_key_fields
-          else
-            # We must clear the primary key when unioning multiple inputs.  If
-            # the programmer wants to preserve the primary key, they must use
-            # the primary override.
-            nil
-          end
-        when Java::CascadingPipe::CoGroup
-          # FIXME: assume grouping_key_fields are the same for all
-          # incoming_scopes.  Need join to give me names from all incoming
-          # scopes to perform rename on primary key fields.
-          union_fields(*incoming_scopes.map{ |s| s.primary_key_fields })
-        else raise "No primary key rules for FlowElement of type #{flow_element}"
-      end
-    end
-
-    def self.project_primary_key(primary_key, old_fields, new_fields)
-      return nil if primary_key.nil?
-      primary_key = primary_key.to_a
-      primary_key if (primary_key & new_fields) == primary_key
-    end
-
-    def self.grouping_primary_key_fields(flow_element, incoming_scopes, scope)
-      case flow_element
-        when Java::CascadingPipe::Each
-          # assert incoming_scopes.size == 1
-          project_primary_key(incoming_scopes.first.grouping_primary_key_fields,
-                              incoming_scopes.first.grouping_fields.to_a,
-                              scope.grouping_fields.to_a)
-        when Java::CascadingPipe::Every
-          # assert incoming_scopes.size == 1
-          incoming_scopes.first.grouping_primary_key_fields
-        when Java::CascadingPipe::GroupBy
-          scope.grouping_key_fields
-        when Java::CascadingPipe::CoGroup
-          scope.grouping_key_fields
-        else raise "No primary key rules for FlowElement of type #{flow_element}"
-      end
-    end
-  end
-
-  # Register default primary keys
-  begin
-    Scope.register_scheme_key(Java::CascadingScheme::TextLine, ['offset'])
-  rescue NameError => ne
-    puts 'WARNING: Could not register primary key for TextLine Scheme as it was not on the class path'
   end
 end

data/lib/cascading.rb

@@ -6,7 +6,7 @@ require 'java'
 
 module Cascading
   # :stopdoc:
-  VERSION = '0.0.5'
+  VERSION = '0.0.6'
   LIBPATH = ::File.expand_path(::File.dirname(__FILE__)) + ::File::SEPARATOR
   PATH = ::File.dirname(LIBPATH) + ::File::SEPARATOR
   CASCADING_HOME = ENV['CASCADING_HOME']

data/spec/jruby_version_spec.rb

@@ -54,7 +54,7 @@ context Object do
       result = e.validate
       result.should == 0
     end
-  when '1.5.3'
+  when '1.5.3', '1.6.5'
     it 'should handle Fixnum -> Integer for ExprStub#eval' do
       e = ExprStub.new('x:int + y:int')
       result = e.eval(:x => 2, :y => 3)

data/spec/spec_util.rb

@@ -7,22 +7,12 @@ module ScopeTests
     scope = scope(*name_params)
     values_fields = params[:values_fields]
     grouping_fields = params[:grouping_fields] || values_fields
-    primary_key_fields = params[:primary_key_fields]
-    grouping_primary_key_fields = primary_key_fields
-    grouping_primary_key_fields = params[:grouping_primary_key_fields] if params.has_key?(:grouping_primary_key_fields)
 
     debug = params[:debug]
     debug_scope(*name_params) if debug
 
     scope.values_fields.to_a.should == values_fields
     scope.grouping_fields.to_a.should == grouping_fields
-    if params.has_key?(:primary_key_fields) # Must support nil values
-      scope.primary_key_fields.should == nil if primary_key_fields.nil?
-      scope.primary_key_fields.to_a.should == primary_key_fields unless primary_key_fields.nil?
-
-      scope.grouping_primary_key_fields.should == nil if grouping_primary_key_fields.nil?
-      scope.grouping_primary_key_fields.to_a.should == grouping_primary_key_fields unless grouping_primary_key_fields.nil?
-    end
   end
 end
 
@@ -135,54 +125,3 @@ def cascading_properties
   Java::CascadingFlow::MultiMapReducePlanner.set_job_conf(properties, job_conf)
   properties
 end
-
-def verify_assembly_output(assembly_name, params, &block)
-  `rm -rf spec_output`
-
-  Cascade.new("foo") do
-    flow("bar") do
-      source assembly_name, tap(params[:source], params.slice(:scheme)) 
-      assembly = assembly(assembly_name)
-      sink assembly_name, tap("spec_output", :kind => :lfs, :sink_mode => :replace)
-    end
-  end.complete(@properties)
-
-  output_data = nil
-
-  File.open("spec_output/part-00000") do |f|
-    output_data = f.readlines
-  end
-
-  if params[:length]
-    output_data.size.should == params[:length]
-  end
-
-  keys = assembly.scope.values_fields
-  if block_given?
-    output_data.each do |line| 
-      values = line.chomp.split(/\t/) 
-
-      yield(keys.zip(values).inject({}) do |map, kv|
-        map[kv[0].to_sym] = kv[1]
-        map
-      end)
-    end
-  end
-end
-
-def describe_job(job_file, &block)
-  context Object do
-    before(:each) do
-      @properties = cascading_properties
-      # Must artificially fill ARGV to prevent errors when creating multi-taps
-      # in ETL cascade
-      ARGV.clear
-      10.times do
-        ARGV << 'text_line_scheme' # Dummy value, required for 3rd arg
-      end
-      load "lib/jobs/#{job_file}/#{job_file}.rb"
-    end
-
-    self.class_eval(&block)
-  end
-end