cascading.jruby 0.0.9 → 0.0.10

data/History.txt

@@ -1,3 +1,18 @@
+0.0.10 - Introduce Jading, cleanup, and library standardization
+
+This release removes the code associated with running jobs on a Hadoop cluster
+as that functionality is now moved to Jading.  This separates the DSL library (a
+gem) from the repository required to build and package job jars (Jading).
+Additionally, this release removes a lot of the auto-generated Rake garbage
+inherited from the original version and greatly streamlines the packaged gem.
+Finally, it starts the precedent for making the contract of operations idiomatic
+Ruby, removing the unnecessary Operations module, and eventually eliminating
+global cascade and flow registries.  A non-backwards compatible change was made
+to the contract of complete and draw which allowed the mechnism for propagating
+properties through a job to be righted and made more Hadoop-idiomatic.
+
+This release removes all the cruft associated with running jobs
+
 0.0.9 - Cascading local mode and upgrade to Cascading 2.0.0
 
 This release upgrades to Cascading 2.0.0 (final) and introduces Cascading local

data/lib/cascading/assembly.rb

@@ -1,8 +1,3 @@
-# -*- coding: utf-8 -*-
-# Copyright 2009, Grégoire Marabout. All Rights Reserved.
-#
-# This is free software. Please see the LICENSE and COPYING files for details.
-
 require 'cascading/base'
 require 'cascading/operations'
 require 'cascading/aggregations'
@@ -110,9 +105,7 @@ module Cascading
       "#{name} : head pipe : #{head_pipe} - tail pipe: #{tail_pipe}"
     end
 
-    # Builds a join (CoGroup) pipe. Requires a list of assembly names to join
-    # and :on to specify the group_fields.
-    def join(*args, &block)
+    def prepare_join(*args, &block)
       options = args.extract_options!
 
       pipes, _ = populate_incoming_scopes(args)
@@ -140,6 +133,7 @@ module Cascading
       incoming_fields = @incoming_scopes.map{ |s| s.values_fields }
       declared_fields = fields(options[:declared_fields] || dedup_fields(*incoming_fields))
       joiner = options[:joiner]
+      is_hash_join = options[:hash] || false
 
       case joiner
       when :inner, 'inner', nil
@@ -160,15 +154,48 @@ module Cascading
         end
         joiner = Java::CascadingPipeJoiner::MixedJoin.new(joiner.to_java(:boolean))
       end
-      result_group_fields = dedup_fields(*group_fields)
-      parameters = [
-        pipes.to_java(Java::CascadingPipe::Pipe),
-        group_fields,
-        declared_fields,
-        result_group_fields,
-        joiner
-      ]
-      apply_aggregations(Java::CascadingPipe::CoGroup.new(*parameters), @incoming_scopes, &block)
+
+      if is_hash_join
+        raise ArgumentError, "hash joins don't support aggregations" if block_given?
+        parameters = [
+          pipes.to_java(Java::CascadingPipe::Pipe),
+          group_fields,
+          declared_fields,
+          joiner
+        ]
+        group_assembly = Java::CascadingPipe::HashJoin.new(*parameters)
+      else
+        result_group_fields = dedup_fields(*group_fields)
+        parameters = [
+          pipes.to_java(Java::CascadingPipe::Pipe),
+          group_fields,
+          declared_fields,
+          result_group_fields,
+          joiner
+        ]
+        group_assembly = Java::CascadingPipe::CoGroup.new(*parameters)
+      end
+      apply_aggregations(group_assembly, @incoming_scopes, &block)
+    end
+    private :prepare_join
+
+    # Builds a HashJoin pipe. This should be used carefully, as the right side
+    # of the join is accumulated entirely in memory. Requires a list of assembly
+    # names to join and :on to specify the join_fields.
+    def hash_join(*args, &block)
+      options = args.extract_options!
+      options[:hash] = true
+      args << options
+      prepare_join(*args, &block)
+    end
+
+    # Builds a join (CoGroup) pipe. Requires a list of assembly names to join
+    # and :on to specify the group_fields.
+    def join(*args, &block)
+      options = args.extract_options!
+      options[:hash] = false
+      args << options
+      prepare_join(*args, &block)
     end
     alias co_group join
 
@@ -650,5 +677,99 @@ module Cascading
 
       each args, :function => field_joiner(options), :output => output
     end
+
+    # Ungroups, or unpivots, a tuple (see Cascading's UnGroup at http://docs.cascading.org/cascading/2.0/javadoc/cascading/operation/function/UnGroup.html).
+    #
+    # You must provide :key and you must provide only one of :value_selectors
+    # and :num_values.
+    #
+    # The named options are:
+    # * <tt>:key</tt> required array of field names to replicate on every
+    #   output row in an ungrouped group.
+    # * <tt>:value_selectors</tt> an array of field names to ungroup.  Each
+    #   field will be ungrouped into an output tuple along with the key fields
+    #   in the order provided.
+    # * <tt>:num_values</tt> an integer specifying the number of fields to
+    #   ungroup into each output tuple (excluding the key fields).  All input
+    #   fields will be ungrouped.
+    # * <tt>:input</tt> an array of field names that specifies the fields to
+    #   input to UnGroup.  Defaults to all_fields.
+    # * <tt>:into</tt> an array of field names.  Default set by UnGroup.
+    # * <tt>:output</tt> an array of field names that specifies the fields to
+    #   produce as output of UnGroup.  Defaults to all_fields.
+    def ungroup(*args)
+      options = args.extract_options!
+      input = options[:input] || all_fields
+      into = fields(options[:into])
+      output = options[:output] || all_fields
+      key = fields(options[:key])
+
+      raise 'You must provide exactly one of :value_selectors or :num_values to ungroup' unless options.has_key?(:value_selectors) ^ options.has_key?(:num_values)
+      value_selectors = options[:value_selectors].map{ |vs| fields(vs) }.to_java(Java::CascadingTuple::Fields) if options.has_key?(:value_selectors)
+      num_values = options[:num_values] if options.has_key?(:num_values)
+
+      parameters = [into, key, value_selectors, num_values].compact
+      each input, :function => Java::CascadingOperationFunction::UnGroup.new(*parameters), :output => output
+    end
+
+    # Inserts one of two values into the dataflow based upon the result of the
+    # supplied filter on the input fields.  This is primarily useful for
+    # creating indicators from filters.
+    #
+    # Parameters:
+    # * <tt>input</tt> name of field to apply the filter.
+    # * <tt>filter</tt> Cascading Filter to apply.
+    # * <tt>keep_value</tt> Java value to produce when the filter would keep
+    #   the given input.
+    # * <tt>remove_value</tt> Java value to produce when the filter would
+    #   remove the given input.
+    #
+    # The named options are:
+    # * <tt>:into</tt> an output field name, defaulting to 'filter_value'.
+    # * <tt>:output</tt> an array of field names that specifies the fields to
+    #   retain in the output tuple.  Defaults to all_fields.
+    def set_value(input, filter, keep_value, remove_value, params = {})
+      into = fields(params[:into] || 'filter_value')
+      output = params[:output] || all_fields
+      each input, :function => Java::CascadingOperationFunction::SetValue.new(into, filter, keep_value, remove_value), :output => output
+    end
+
+    # Efficient way of inserting a null indicator for any field, even one that
+    # cannot be coerced to a string.  This is accomplished using Cascading's
+    # FilterNull and SetValue operators rather than Janino.  1 is produced if
+    # the field is null and 0 otherwise.
+    #
+    # Parameters:
+    # * <tt>input</tt> name of field to check for null.
+    #
+    # The named options are:
+    # * <tt>:into</tt> an output field name, defaulting to 'is_null'.
+    # * <tt>:output</tt> an array of field names that specifies the fields to
+    #   retain in the output tuple.  Defaults to all_fields.
+    def null_indicator(input, params = {})
+      into = fields(params[:into] || 'is_null')
+      output = params[:output] || all_fields
+      set_value input, Java::CascadingOperationFilter::FilterNull.new, 1.to_java, 0.to_java, :into => into, :output => output
+    end
+
+    # Given a field and a regex, returns an indicator that is 1 if the string
+    # contains at least 1 match and 0 otherwise.
+    #
+    # Parameters:
+    # * <tt>input</tt> field name or names that specifies the fields over which
+    #   to perform the match.
+    # * <tt>pattern</tt> regex to apply to the input.
+    #
+    # The named options are:
+    # * <tt>:into</tt> an output field name, defaulting to 'regex_contains'.
+    # * <tt>:output</tt> an array of field names that specifies the fields to
+    #   retain in the output tuple.  Defaults to all_fields.
+    def regex_contains(input, pattern, params = {})
+      input = fields(input)
+      pattern = pattern.to_s # Supports JRuby regexes
+      into = fields(params[:into] || 'regex_contains')
+      output = params[:output] || all_fields
+      set_value input, Java::CascadingOperationRegex::RegexFilter.new(pattern), 1.to_java, 0.to_java, :into => into, :output => output
+    end
   end
 end

data/lib/cascading/base.rb

@@ -1,7 +1,3 @@
-# Copyright 2009, Grégoire Marabout. All Rights Reserved.
-#
-# This is free software. Please see the LICENSE and COPYING files for details.
-
 module Cascading
   class Node
     attr_accessor :name, :parent, :children, :child_names, :last_child

data/lib/cascading/cascade.rb

@@ -1,7 +1,3 @@
-# Copyright 2009, Grégoire Marabout. All Rights Reserved.
-#
-# This is free software. Please see the LICENSE and COPYING files for details.
-
 require 'cascading/base'
 require 'yaml'
 
@@ -9,22 +5,35 @@ module Cascading
   class Cascade < Cascading::Node
     extend Registerable
 
-    attr_reader :mode
+    attr_reader :properties, :mode
 
-    # Builds a cascade given the specified name.  Optionally accepts a :mode
-    # which will be used as the default mode for all child flows.  See
-    # Cascading::Mode.parse for details.
+    # Do not use this constructor directly; instead, use Cascading::cascade to
+    # build cascades.
+    #
+    # Builds a cascade given the specified name.  Optionally accepts
+    # :properties which will be used as the default properties for all child
+    # flows.  Properties must be a Ruby Hash with string keys and values and
+    # will be copied before being passed into each flow in the cascade.  See
+    # Cascading::Flow#initialize for details on how flows handle properties.
+    # Optionally accepts a :mode which will be used as the default mode for all
+    # child flows.  See Cascading::Mode.parse for details.
     def initialize(name, params = {})
+      @properties = params[:properties] || {}
       @mode = params[:mode]
       super(name, nil) # A Cascade cannot have a parent
       self.class.add(name, self)
     end
 
-    # Builds a child flow given a name and block.  Optionally accepts a :mode,
-    # which will override the default mode stored in this cascade.
+    # Builds a child flow given a name and block.  Optionally accepts
+    # :properties which will override the default properties stroed in this
+    # cascade.  Optionally accepts a :mode, which will override the default
+    # mode stored in this cascade.
     def flow(name, params = {}, &block)
       raise "Could not build flow '#{name}'; block required" unless block_given?
+
+      params[:properties] ||= properties.dup
       params[:mode] ||= mode
+
       flow = Flow.new(name, self, params)
       add_child(flow)
       flow.instance_eval(&block)
@@ -35,9 +44,9 @@ module Cascading
       "#{offset}#{name}:cascade\n#{child_names.map{ |child| children[child].describe("#{offset}  ") }.join("\n")}"
     end
 
-    def draw(dir, properties = nil)
+    def draw(dir)
       @children.each do |name, flow|
-        flow.connect(properties).writeDOT("#{dir}/#{name}.dot")
+        flow.connect.writeDOT("#{dir}/#{name}.dot")
       end
     end
 
@@ -54,9 +63,9 @@ module Cascading
       end
     end
 
-    def complete(properties = nil)
+    def complete
       begin
-        Java::CascadingCascade::CascadeConnector.new.connect(name, make_flows(@children, properties)).complete
+        Java::CascadingCascade::CascadeConnector.new.connect(name, make_flows(@children)).complete
       rescue NativeException => e
         raise CascadingException.new(e, 'Error completing cascade')
       end
@@ -64,9 +73,9 @@ module Cascading
 
     private
 
-    def make_flows(flows, properties)
+    def make_flows(flows)
       flow_instances = flows.map do |name, flow|
-        cascading_flow = flow.connect(properties)
+        cascading_flow = flow.connect
         flow.listeners.each { |l| cascading_flow.addListener(l) }
         cascading_flow
       end

data/lib/cascading/cascading.rb

@@ -1,8 +1,3 @@
-# -*- coding: utf-8 -*-
-# Copyright 2009, Grégoire Marabout. All Rights Reserved.
-#
-# This is free software. Please see the LICENSE and COPYING files for details.
-
 require 'cascading/expr_stub'
 
 module Cascading
@@ -12,10 +7,31 @@ module Cascading
     :float => java.lang.Float.java_class, :string => java.lang.String.java_class,
   }
 
+  # FIXME: I consider $jobconf_properties to be a hack forced on us by the lack
+  # of properties handling in earlier versions of the gem.  Fully removing the
+  # hack would look like introducing a Job abstraction which instantiates user
+  # code, and allowing jading's runner to pass properties into that.  I've
+  # already taken the step to thread properties through cascades and flows
+  # rather than merge properties before connect, but we still require the
+  # global properties hack to integrate with external runner code (jading).
+  #
+  # Note that this would also mean we can get rid of the global "registries" of
+  # cascades and flows.  I've already eliminated most uses of these registries,
+  # but they are still required for the runner to find user code required in a
+  # previous step.  A Job abstraction would clean this up, as well.
+  #
+  # For now, it is important that people use these constructors rather than
+  # directly building their own cascades and flows so that jading can send them
+  # default properties.
+
   # Builds a top-level cascade given a name and a block.  Optionally accepts a
   # :mode, as explained in Cascading::Cascade#initialize.
   def cascade(name, params = {}, &block)
     raise "Could not build cascade '#{name}'; block required" unless block_given?
+    raise 'Cascading::cascade does not accept the :properties param only the global $jobconf_properties' if params[:properties]
+
+    params[:properties] = $jobconf_properties.dup if $jobconf_properties
+
     cascade = Cascade.new(name, params)
     cascade.instance_eval(&block)
     cascade
@@ -26,6 +42,10 @@ module Cascading
   # Cascading::Flow#initialize.
   def flow(name, params = {}, &block)
     raise "Could not build flow '#{name}'; block required" unless block_given?
+    raise 'Cascading::flow does not accept the :properties param only the global $jobconf_properties' if params[:properties]
+
+    params[:properties] = $jobconf_properties.dup if $jobconf_properties
+
     flow = Flow.new(name, nil, params)
     flow.instance_eval(&block)
     flow

data/lib/cascading/ext/array.rb

@@ -1,9 +1,3 @@
-# ext.rb : some extensions to basic types
-#
-# Copyright 2009, Grégoire Marabout. All Rights Reserved.
-#
-# This is free software. Please see the LICENSE and COPYING files for details.
-
 class Array
   def extract_options!
      last.is_a?(::Hash) ? pop : {}
@@ -12,4 +6,4 @@ class Array
   def extract_options
      last.is_a?(::Hash) ? last : {}
   end
-end
+end

data/lib/cascading/flow.rb

@@ -1,22 +1,26 @@
-# -*- coding: utf-8 -*-
-# Copyright 2009, Grégoire Marabout. All Rights Reserved.
-#
-# This is free software. Please see the LICENSE and COPYING files for details.
-
 require 'cascading/assembly'
 
 module Cascading
   class Flow < Cascading::Node
     extend Registerable
 
-    attr_accessor :properties, :sources, :sinks, :incoming_scopes, :outgoing_scopes, :listeners
-    attr_reader :mode
+    attr_accessor :sources, :sinks, :incoming_scopes, :outgoing_scopes, :listeners
+    attr_reader :properties, :mode
 
+    # Do not use this constructor directly.  Instead, use Cascading::flow to
+    # build top-level flows and Cascade#flow to build flows within a Cascade.
+    #
     # Builds a flow given a name and a parent node (a cascade or nil).
-    # Optionally accepts a :mode which will determine the execution mode of
-    # this flow.  See Cascading::Mode.parse for details.
+    # Optionally accepts :properties which allows external configuration of
+    # this flow.  The flow will side-effect the properties during composition,
+    # then pass the modified properties along to the FlowConnector for
+    # execution. See Cascading::Cascade#initialize for details on how
+    # properties are propagated through cascades.  Optionally accepts a :mode
+    # which will determine the execution mode of this flow.  See
+    # Cascading::Mode.parse for details.
     def initialize(name, parent, params = {})
-      @properties, @sources, @sinks, @incoming_scopes, @outgoing_scopes, @listeners = {}, {}, {}, {}, {}, []
+      @sources, @sinks, @incoming_scopes, @outgoing_scopes, @listeners = {}, {}, {}, {}, []
+      @properties = params[:properties] || {}
       @mode = Mode.parse(params[:mode])
       @flow_scope = Scope.flow_scope(name)
       super(name, parent)
@@ -128,14 +132,9 @@ module Cascading
       end
     end
 
-    def connect(properties = nil)
-      # This ensures we have a hash, and that it is a Ruby Hash (because we
-      # also accept java.util.HashMap), then merges it with Flow properties
-      properties ||= {}
-      properties = java.util.HashMap.new(@properties.merge(Hash[*properties.to_a.flatten]))
-
+    def connect
       puts "Connecting flow '#{name}' with properties:"
-      properties.key_set.to_a.sort.each do |key|
+      properties.keys.sort.each do |key|
         puts "#{key}=#{properties[key]}"
       end
 
@@ -149,9 +148,9 @@ module Cascading
       mode.connect_flow(properties, name, sources, sinks, pipes)
     end
 
-    def complete(properties = nil)
+    def complete
       begin
-        flow = connect(properties)
+        flow = connect
         @listeners.each { |l| flow.addListener(l) }
         flow.complete
       rescue NativeException => e

data/lib/cascading/mode.rb

@@ -41,7 +41,11 @@ module Cascading
       update_local_mode(sources, sinks)
       sources = select_taps(sources)
       sinks = select_taps(sinks)
-      flow_connector_class.new(properties).connect(name, sources, sinks, pipes)
+
+      # Report execution mode to stdout before connecting
+      puts "Connecting flow '#{name}' in #{local ? 'Cascading local mode' : 'Hadoop mode'}"
+
+      flow_connector_class.new(java.util.HashMap.new(properties)).connect(name, sources, sinks, pipes)
     end
 
     private

data/lib/cascading/operations.rb

@@ -1,8 +1,15 @@
-# Copyright 2009, Grégoire Marabout. All Rights Reserved.
-#
-# This is free software. Please see the LICENSE and COPYING files for details.
-
 module Cascading
+  # The Cascading::Operations module is deprecated.  The original idea from long
+  # ago is that it would be useful to mixin operator wrappers to places other
+  # than Cascading::Assembly, but this is not true.  Instead, put Eaches in
+  # Cascading::Assembly, Everies in Cascading::Aggregations, and any more
+  # generally useful utility code directly in the Cascading module
+  # (cascading/cascading.rb).
+  #
+  # Further, the entire *args pattern should be deprecated as it leads to
+  # functions that can only be understood by reading their code.  Instead,
+  # idiomatic Ruby (positional required params and a params hash for optional
+  # args) should be used.  See Cascading::Assembly#set_value for an example.
   module Operations
     def identity
       Java::CascadingOperation::Identity.new

data/lib/cascading/tap.rb

@@ -9,6 +9,10 @@ module Cascading
       @hadoop_tap = hadoop_tap
     end
 
+    def to_s
+      "Local: #{local_tap}, Hadoop: #{hadoop_tap}"
+    end
+
     def local?
       !local_tap.nil?
     end

data/lib/cascading.rb

@@ -1,12 +1,8 @@
-# Copyright 2009, Grégoire Marabout. All Rights Reserved.
-#
-# This is free software. Please see the LICENSE and COPYING files for details.
-
 require 'java'
 
 module Cascading
   # :stopdoc:
-  VERSION = '0.0.9'
+  VERSION = '0.0.10'
   LIBPATH = ::File.expand_path(::File.dirname(__FILE__)) + ::File::SEPARATOR
   PATH = ::File.dirname(LIBPATH) + ::File::SEPARATOR
   CASCADING_HOME = ENV['CASCADING_HOME']