cascading.jruby 0.0.8 → 0.0.9

data/HACKING.md

@@ -4,7 +4,7 @@ Some hacking info on `cascading.jruby`:
 
 For local development, install with (requires [bundler](http://gembundler.com/)):
 
-    bundle install
+    jruby -S bundle install
 
 To run the tests (will download Cascading and Hadoop jars):
 

data/History.txt

@@ -1,3 +1,9 @@
+0.0.9 - Cascading local mode and upgrade to Cascading 2.0.0
+
+This release upgrades to Cascading 2.0.0 (final) and introduces Cascading local
+mode.  Ambiguous node names are now prohibited and the insert helper correctly
+supports constants.
+
 0.0.8 - AggregateBy and upgrade to Cascading 2.0.0 wip-286
 
 This release upgrades to Cascading 2.0.0 wip-286, but again does not implement

data/README.md

@@ -2,7 +2,7 @@
 
 `cascading.jruby` is a small DSL above [Cascading](http://www.cascading.org/).
 
-It requires Hadoop (>= 0.20.2) and [Cascading 2.0.0-wip-286](http://files.concurrentinc.com/cascading/2.0/cascading-2.0.0-wip-286-hadoop-0.20.2%2B.tgz) to be set via the environment variables: `HADOOP_HOME` and `CASCADING_HOME`
+It requires Hadoop (>= 0.20.2) and [Cascading 2.0.0](http://files.cascading.org/cascading/2.0/cascading-2.0.0.tgz) to be set via the environment variables: `HADOOP_HOME` and `CASCADING_HOME`
 
 It has been tested on JRuby versions 1.2.0, 1.4.0, 1.5.3, and 1.6.5.
 

data/ivy.xml

@@ -17,9 +17,9 @@
   </configurations>
 
   <dependencies>
-    <dependency org="cascading" name="cascading-core" rev="2.0.0-wip-286" conf="default" />
-    <dependency org="cascading" name="cascading-local" rev="2.0.0-wip-286" conf="default" />
-    <dependency org="cascading" name="cascading-hadoop" rev="2.0.0-wip-286" conf="default" />
+    <dependency org="cascading" name="cascading-core" rev="2.0.0" conf="default" />
+    <dependency org="cascading" name="cascading-local" rev="2.0.0" conf="default" />
+    <dependency org="cascading" name="cascading-hadoop" rev="2.0.0" conf="default" />
     <dependency org="org.jruby" name="jruby" rev="1.6.5" conf="default" />
   </dependencies>
 </ivy-module>

data/lib/cascading/aggregations.rb

@@ -4,19 +4,20 @@ require 'cascading/ext/array'
 
 module Cascading
   # Rules enforced by Aggregations:
-  #   Contains either 1 Buffer or >= 1 Aggregator (explicitly checked)
-  #   No GroupBys, CoGroups, Joins, or Merges (methods for these pipes do not exist on Aggregations)
-  #   No Eaches (Aggregations#each does not exist)
-  #   Aggregations may not branch (Aggregations#branch does not exist)
+  # * Contains either 1 Buffer or >= 1 Aggregator (explicitly checked)
+  # * No GroupBys, CoGroups, Joins, or Merges (methods for these pipes do not
+  # exist on Aggregations)
+  # * No Eaches (Aggregations#each does not exist)
+  # * Aggregations may not branch (Aggregations#branch does not exist)
   #
   # Externally enforced rules:
-  #   May be empty (in which case, Aggregations is not instantiated)
-  #   Must follow a GroupBy or CoGroup (not a Join or Merge)
+  # * May be empty (in which case, Aggregations is not instantiated)
+  # * Must follow a GroupBy or CoGroup (not a Join or Merge)
   #
   # Optimizations:
-  #   If the leading Group is a GroupBy and all subsequent Everies are
-  #   Aggregators that have a corresponding AggregateBy, Aggregations can
-  #   replace the GroupBy/Aggregator pipe with a single composite AggregateBy.
+  # * If the leading Group is a GroupBy and all subsequent Everies are
+  # Aggregators that have a corresponding AggregateBy, Aggregations can replace
+  # the GroupBy/Aggregator pipe with a single composite AggregateBy
   class Aggregations
     include Operations
 
@@ -110,10 +111,10 @@ module Cascading
     # insert) and an options hash.
     #
     # Options include:
-    #   * <tt>:ignore</tt> a Java Array of Objects (for min and max) or Tuples
-    #     (for first and last) of values for the aggregator to ignore
-    #
-    # <tt>function</tt> is a symbol that is the method to call to construct the Cascading Aggregator.
+    # * <tt>:ignore</tt> a Java Array of Objects (for min and max) or Tuples
+    # (for first and last) of values for the aggregator to ignore
+    # * <tt>function</tt> is a symbol that is the method to call to construct
+    # the Cascading Aggregator.
     def composite_aggregator(args, function)
       field_map, options = extract_field_map(args)
 

data/lib/cascading/base.rb

@@ -14,9 +14,12 @@ module Cascading
       @last_child = nil
     end
 
+    # Children must be uniquely named within the scope of each Node.  This
+    # ensures, for example, two assemblies are not created within the same flow
+    # with the same name, causing joins, unions, and sinks on them to be
+    # ambiguous.
     def add_child(node)
-      child = root.find_child(node.name)
-      warn "WARNING: adding '#{node.qualified_name}', but node named '#{node.name}' already exists at '#{child.qualified_name}'" if child
+      raise AmbiguousNodeNameException.new("Attempted to add '#{node.qualified_name}', but node named '#{node.name}' already exists") if @children[node.name]
 
       @children[node.name] = node
       @child_names << node.name
@@ -33,21 +36,36 @@ module Cascading
     end
     alias desc describe
 
+    # In order to find a child, we require it to be uniquely named within this
+    # Node and its children.  This ensures, for example, branches in peer
+    # assemblies or branches and assemblies do not conflict in joins, unions,
+    # and sinks.
     def find_child(name)
-      children.each do |child_name, child|
-        return child if child_name == name
-        result = child.find_child(name)
-        return result if result
-      end
-      nil
+      all_children_with_name = find_all_children_with_name(name)
+      qualified_names = all_children_with_name.map{ |child| child.qualified_name }
+      raise AmbiguousNodeNameException.new("Ambiguous lookup of child by name '#{name}'; found '#{qualified_names.join("', '")}'") if all_children_with_name.size > 1
+
+      all_children_with_name.first
     end
 
     def root
       return self unless parent
       parent.root
     end
+
+    protected
+
+    def find_all_children_with_name(name)
+      child_names.map do |child_name|
+        children[child_name] if child_name == name
+      end.compact + child_names.map do |child_name|
+        children[child_name].find_all_children_with_name(name)
+      end.flatten
+    end
   end
 
+  class AmbiguousNodeNameException < StandardError; end
+
   # A module to add auto-registration capability
   module Registerable
     def all
@@ -69,7 +87,7 @@ module Cascading
 
     def add(name, instance)
       @registered ||= {}
-      warn "WARNING: node named '#{name}' already registered in #{self}" if @registered[name]
+      warn "WARNING: Node named '#{name}' already registered in #{self}" if @registered[name]
       @registered[name] = instance
     end
 

data/lib/cascading/cascade.rb

@@ -9,14 +9,23 @@ module Cascading
   class Cascade < Cascading::Node
     extend Registerable
 
-    def initialize(name)
+    attr_reader :mode
+
+    # Builds a cascade given the specified name.  Optionally accepts a :mode
+    # which will be used as the default mode for all child flows.  See
+    # Cascading::Mode.parse for details.
+    def initialize(name, params = {})
+      @mode = params[:mode]
       super(name, nil) # A Cascade cannot have a parent
       self.class.add(name, self)
     end
 
-    def flow(name, &block)
+    # Builds a child flow given a name and block.  Optionally accepts a :mode,
+    # which will override the default mode stored in this cascade.
+    def flow(name, params = {}, &block)
       raise "Could not build flow '#{name}'; block required" unless block_given?
-      flow = Flow.new(name, self)
+      params[:mode] ||= mode
+      flow = Flow.new(name, self, params)
       add_child(flow)
       flow.instance_eval(&block)
       flow

data/lib/cascading/cascading.rb

@@ -12,17 +12,21 @@ module Cascading
     :float => java.lang.Float.java_class, :string => java.lang.String.java_class,
   }
 
-  def cascade(name, &block)
+  # Builds a top-level cascade given a name and a block.  Optionally accepts a
+  # :mode, as explained in Cascading::Cascade#initialize.
+  def cascade(name, params = {}, &block)
     raise "Could not build cascade '#{name}'; block required" unless block_given?
-    cascade = Cascade.new(name)
+    cascade = Cascade.new(name, params)
     cascade.instance_eval(&block)
     cascade
   end
 
-  # For applications built of Flows with no Cascades
-  def flow(name, &block)
+  # Builds a top-level flow given a name and block for applications built of
+  # flows with no cascades.  Optionally accepts a :mode, as explained in
+  # Cascading::Flow#initialize.
+  def flow(name, params = {}, &block)
     raise "Could not build flow '#{name}'; block required" unless block_given?
-    flow = Flow.new(name, nil)
+    flow = Flow.new(name, nil, params)
     flow.instance_eval(&block)
     flow
   end
@@ -91,7 +95,9 @@ module Cascading
     Java::CascadingTuple::Fields::RESULTS
   end
 
-  # Creates a c.s.h.TextLine scheme.  Positional args are used if <tt>:source_fields</tt> is not provided.
+  # Creates a TextLine scheme (can be used in both Cascading local and hadoop
+  # modes).  Positional args are used if <tt>:source_fields</tt> is not
+  # provided.
   #
   # The named options are:
   # * <tt>:source_fields</tt> a string or array of strings.  Specifies the
@@ -100,7 +106,7 @@ module Cascading
   #   to be written to a sink with this scheme.  Defaults to all_fields.
   # * <tt>:compression</tt> a symbol, either <tt>:enable</tt> or
   #   <tt>:disable</tt>, that governs the TextLine scheme's compression.  Defaults
-  #   to the default TextLine compression.
+  #   to the default TextLine compression (only applies to c.s.h.TextLine).
   def text_line_scheme(*args)
     options = args.extract_options!
     source_fields = fields(options[:source_fields] || (args.empty? ? ['offset', 'line'] : args))
@@ -111,55 +117,40 @@ module Cascading
       else Java::CascadingSchemeHadoop::TextLine::Compress::DEFAULT
     end
 
-    Java::CascadingSchemeHadoop::TextLine.new(source_fields, sink_fields, sink_compression)
+    {
+      :local_scheme => Java::CascadingSchemeLocal::TextLine.new(source_fields, sink_fields),
+      :hadoop_scheme => Java::CascadingSchemeHadoop::TextLine.new(source_fields, sink_fields, sink_compression),
+    }
   end
 
-  # Creates a c.s.h.SequenceFile scheme instance from the specified fields.
+  # Creates a c.s.h.SequenceFile scheme instance from the specified fields.  A
+  # local SequenceFile scheme is not provided by Cascading, so this scheme
+  # cannot be used in Cascading local mode.
   def sequence_file_scheme(*fields)
-    unless fields.empty?
-      fields = fields(fields)
-      return Java::CascadingSchemeHadoop::SequenceFile.new(fields)
-    else
-      return Java::CascadingSchemeHadoop::SequenceFile.new(all_fields)
-    end
+    {
+      :local_scheme => nil,
+      :hadoop_scheme => Java::CascadingSchemeHadoop::SequenceFile.new(fields.empty? ? all_fields : fields(fields)),
+    }
   end
 
   def multi_source_tap(*taps)
-    Java::CascadingTap::MultiSourceTap.new(taps.to_java('cascading.tap.Tap'))
+    MultiTap.multi_source_tap(taps)
   end
 
   def multi_sink_tap(*taps)
-    Java::CascadingTap::MultiSinkTap.new(taps.to_java('cascading.tap.Tap'))
-  end
-
-  # Generic method for creating taps.
-  # It expects a ":kind" argument pointing to the type of tap to create.
-  def tap(*args)
-    opts = args.extract_options!
-    path = args.empty? ? opts[:path] : args[0]
-    scheme = opts[:scheme] || text_line_scheme
-    sink_mode = opts[:sink_mode] || :keep
-    sink_mode = case sink_mode
-      when :keep, 'keep'       then Java::CascadingTap::SinkMode::KEEP
-      when :replace, 'replace' then Java::CascadingTap::SinkMode::REPLACE
-      when :append, 'append'   then Java::CascadingTap::SinkMode::APPEND
-      else raise "Unrecognized sink mode '#{sink_mode}'"
-    end
-    fs = opts[:kind] || :hfs
-    klass = case fs
-      when :hfs, 'hfs' then Java::CascadingTapHadoop::Hfs
-      when :dfs, 'dfs' then Java::CascadingTapHadoop::Dfs
-      when :lfs, 'lfs' then Java::CascadingTapHadoop::Lfs
-      else raise "Unrecognized kind of tap '#{fs}'"
-    end
-    parameters = [scheme, path, sink_mode]
-    klass.new(*parameters)
+    MultiTap.multi_sink_tap(taps)
+  end
+
+  # Creates a Cascading::Tap given a path and optional :scheme and :sink_mode.
+  def tap(path, params = {})
+    Tap.new(path, params)
   end
 
   # Constructs properties to be passed to Flow#complete or Cascade#complete
-  # which will locate temporary Hadoop files in base_dir.  It is necessary
-  # to pass these properties only when executing local scripts via JRuby's main
-  # method, which confuses Cascading's attempt to find the containing jar.
+  # which will locate temporary Hadoop files in base_dir.  It is necessary to
+  # pass these properties only when executing scripts in Hadoop local mode via
+  # JRuby's main method, which confuses Cascading's attempt to find the
+  # containing jar.  When using Cascading local mode, these are unnecessary.
   def local_properties(base_dir)
     dirs = {
       'test.build.data' => "#{base_dir}/build",

data/lib/cascading/cascading_exception.rb

@@ -1,9 +1,9 @@
-# NativeException wrapper that prints the full nested stack trace of the Java
-# exception and all of its causes wrapped by the NativeException.
-# NativeException by default reveals only the first cause, which is
-# insufficient for tracing cascading.jruby errors into JRuby code or revealing
-# underlying Janino expression problems.
 module Cascading
+  # NativeException wrapper that prints the full nested stack trace of the Java
+  # exception and all of its causes wrapped by the NativeException.
+  # NativeException by default reveals only the first cause, which is
+  # insufficient for tracing cascading.jruby errors into JRuby code or
+  # revealing underlying Janino expression problems.
   class CascadingException < StandardError
     attr_accessor :ne, :depth
 

data/lib/cascading/flow.rb

@@ -10,9 +10,15 @@ module Cascading
     extend Registerable
 
     attr_accessor :properties, :sources, :sinks, :incoming_scopes, :outgoing_scopes, :listeners
+    attr_reader :mode
 
-    def initialize(name, parent)
+    # Builds a flow given a name and a parent node (a cascade or nil).
+    # Optionally accepts a :mode which will determine the execution mode of
+    # this flow.  See Cascading::Mode.parse for details.
+    def initialize(name, parent, params = {})
       @properties, @sources, @sinks, @incoming_scopes, @outgoing_scopes, @listeners = {}, {}, {}, {}, {}, []
+      @mode = Mode.parse(params[:mode])
+      @flow_scope = Scope.flow_scope(name)
       super(name, parent)
       self.class.add(name, self)
     end
@@ -25,30 +31,18 @@ module Cascading
       assembly
     end
 
-    # Create a new sink for this flow, with the specified name.
-    # "tap" can be either a tap (see Cascading.tap) or a string that will
-    # reference a path.
-    def sink(*args)
-      if (args.size == 2)
-        sinks[args[0]] = args[1]
-      elsif (args.size == 1)
-        sinks[name] =  args[0]
-      end
+    # Create a new source for this flow, using the specified name and
+    # Cascading::Tap
+    def source(name, tap)
+      sources[name] = tap
+      incoming_scopes[name] = Scope.source_scope(name, mode.source_tap(name, tap), @flow_scope)
+      outgoing_scopes[name] = incoming_scopes[name]
     end
 
-    # Create a new source for this flow, with the specified name.
-    # "tap" can be either a tap (see Cascading.tap) or a string that will
-    # reference a path.
-    def source(*args)
-      if (args.size == 2)
-        sources[args[0]] = args[1]
-        incoming_scopes[args[0]] = Scope.tap_scope(args[1], args[0])
-        outgoing_scopes[args[0]] = incoming_scopes[args[0]]
-      elsif (args.size == 1)
-        sources[name] = args[0]
-        incoming_scopes[name] = Scope.empty_scope(name)
-        outgoing_scopes[name] = incoming_scopes[name]
-      end
+    # Create a new sink for this flow, using the specified name and
+    # Cascading::Tap
+    def sink(name, tap)
+      sinks[name] = tap
     end
 
     def describe(offset = '')
@@ -149,12 +143,10 @@ module Cascading
       Java::CascadingProperty::AppProps.setApplicationName(properties, name)
       Java::CascadingProperty::AppProps.setApplicationVersion(properties, '0.0.0')
 
-      Java::CascadingFlowHadoop::HadoopFlowConnector.new(properties).connect(
-        name,
-        make_tap_parameter(@sources),
-        make_tap_parameter(@sinks),
-        make_pipes
-      )
+      sources = make_tap_parameter(@sources, :head_pipe)
+      sinks = make_tap_parameter(@sinks, :tail_pipe)
+      pipes = make_pipes
+      mode.connect_flow(properties, name, sources, sinks, pipes)
     end
 
     def complete(properties = nil)
@@ -169,12 +161,11 @@ module Cascading
 
     private
 
-    def make_tap_parameter(taps)
+    def make_tap_parameter(taps, pipe_accessor)
       taps.inject({}) do |map, (name, tap)|
         assembly = find_child(name)
         raise "Could not find assembly '#{name}' to connect to tap: #{tap}" unless assembly
-
-        map[assembly.tail_pipe.name] = tap
+        map[assembly.send(pipe_accessor).name] = tap
         map
       end
     end

data/lib/cascading/mode.rb

@@ -0,0 +1,78 @@
+module Cascading
+  # A Cascading::Mode encapsulates the idea of the execution mode for your
+  # flows.  The default is Hadoop mode, but you can request that your code run
+  # in Cascading local mode.  If you subsequently use a tap or a scheme that
+  # has no local implementation, the mode will be converted back to Hadoop
+  # mode.
+  class Mode
+    attr_reader :local
+
+    # Hadoop mode is the default.  You must explicitly request Cascading local
+    # mode with values 'local' or :local.
+    def self.parse(mode)
+      case mode
+      when 'local', :local then Mode.new(true)
+      else Mode.new(false)
+      end
+    end
+
+    def initialize(local)
+      @local = local
+    end
+
+    # Attempts to select the appropriate tap given the current mode.  If that
+    # tap does not exist, it fails over to the other tap with a warning.
+    def source_tap(name, tap)
+      warn "WARNING: No local tap for source '#{name}' in tap #{tap}" if local && !tap.local?
+      warn "WARNING: No Hadoop tap for source '#{name}' in tap #{tap}" if !local && !tap.hadoop?
+
+      if local
+        tap.local_tap || tap.hadoop_tap
+      else
+        tap.hadoop_tap || tap.local_tap
+      end
+    end
+
+    # Builds a c.f.Flow given properties, name, sources, sinks, and pipes from
+    # a Cascading::Flow.  The current mode is adjusted based on the taps and
+    # schemes of the sources and sinks, then the correct taps are selected
+    # before building the flow.
+    def connect_flow(properties, name, sources, sinks, pipes)
+      update_local_mode(sources, sinks)
+      sources = select_taps(sources)
+      sinks = select_taps(sinks)
+      flow_connector_class.new(properties).connect(name, sources, sinks, pipes)
+    end
+
+    private
+
+    # Updates this mode based upon your sources and sinks.  It's possible that
+    # you asked for Cascading local mode, but that request cannot be fulfilled
+    # because you used taps or schemes which have no local implementation.
+    def update_local_mode(sources, sinks)
+      local_supported = sources.all?{ |name, tap| tap.local? } && sinks.all?{ |name, tap| tap.local? }
+
+      if local && !local_supported
+        non_local_sources = sources.reject{ |name, tap| tap.local? }
+        non_local_sinks = sinks.reject{ |name, tap| tap.local? }
+        warn "WARNING: Cascading local mode requested but these sources: #{non_local_sources.inspect} and these sinks: #{non_local_sinks.inspect} do not support it"
+        @local = false
+      end
+
+      local
+    end
+
+    # Given a tap map, extracts the correct taps for the current mode
+    def select_taps(tap_map)
+      tap_map.inject({}) do |map, (name, tap)|
+        map[name] = tap.send(local ? :local_tap : :hadoop_tap)
+        map
+      end
+    end
+
+    # Chooses the correct FlowConnector class for the current mode
+    def flow_connector_class
+      local ? Java::CascadingFlowLocal::LocalFlowConnector : Java::CascadingFlowHadoop::HadoopFlowConnector
+    end
+  end
+end

data/lib/cascading/operations.rb

@@ -107,15 +107,21 @@ module Cascading
 
     def to_java_comparable_array(arr)
       (arr.map do |v|
-        case v.class
+        coerce_to_java(v)
+      end).to_java(java.lang.Comparable)
+    end
+
+    def coerce_to_java(v)
+      case v
         when Fixnum
-          java.lang.Integer.new(v)
+          java.lang.Long.new(v)
         when Float
           java.lang.Double.new(v)
+        when NilClass
+          nil
         else
           java.lang.String.new(v.to_s)
-        end
-      end).to_java(java.lang.Comparable)
+      end
     end
 
     def expression_filter(*args)

data/lib/cascading/scope.rb

@@ -10,12 +10,18 @@ module Cascading
       Scope.new(Java::CascadingFlowPlanner::Scope.new(@scope))
     end
 
+    def self.flow_scope(name)
+      Java::CascadingFlowPlanner::Scope.new(name)
+    end
+
     def self.empty_scope(name)
       Scope.new(Java::CascadingFlowPlanner::Scope.new(name))
     end
 
-    def self.tap_scope(tap, name)
-      java_scope = outgoing_scope_for(tap, java.util.HashSet.new)
+    def self.source_scope(name, tap, flow_scope)
+      incoming_scopes = java.util.HashSet.new
+      incoming_scopes.add(flow_scope)
+      java_scope = outgoing_scope_for(tap, incoming_scopes)
       # Taps and Pipes don't name their outgoing scopes like other FlowElements
       java_scope.name = name
       Scope.new(java_scope)

data/lib/cascading/sub_assembly.rb

@@ -4,12 +4,12 @@ module Cascading
   # Allows you to plugin c.p.SubAssemblies to a cascading.jruby Assembly.
   #
   # Assumptions:
-  #   * You will either use the tail_pipe of the calling Assembly, or overwrite
-  #   its incoming_scopes (as do join and union).
-  #   * Your subassembly will have only 1 tail pipe; branching is not
-  #   supported.  This allows you to continue operating upon the tail of the
-  #   SubAssembly within the calling Assembly.
-  #   * You will not use nested c.p.SubAssemblies.
+  # * You will either use the tail_pipe of the calling Assembly, or overwrite
+  # its incoming_scopes (as do join and union)
+  # * Your subassembly will have only 1 tail pipe; branching is not
+  # supported.  This allows you to continue operating upon the tail of the
+  # SubAssembly within the calling Assembly
+  # * You will not use nested c.p.SubAssemblies
   #
   # This is a low-level tool, so be careful.
   class SubAssembly

data/lib/cascading/tap.rb

@@ -0,0 +1,81 @@
+module Cascading
+  # A Cascading::BaseTap wraps up a pair of Cascading taps, one for Cascading
+  # local mode and the other for Hadoop mode.
+  class BaseTap
+    attr_reader :local_tap, :hadoop_tap
+
+    def initialize(local_tap, hadoop_tap)
+      @local_tap = local_tap
+      @hadoop_tap = hadoop_tap
+    end
+
+    def local?
+      !local_tap.nil?
+    end
+
+    def hadoop?
+      !hadoop_tap.nil?
+    end
+  end
+
+  # A Cascading::Tap represents a non-aggregate tap with a scheme, path, and
+  # optional sink_mode.  c.t.l.FileTap is used in Cascading local mode and
+  # c.t.h.Hfs is used in Hadoop mode.  Whether or not these can be created is
+  # governed by the :scheme parameter, which must contain at least one of
+  # :local_scheme or :hadoop_scheme.  Schemes like TextLine are supported in
+  # both modes (by Cascading), but SequenceFile is only supported in Hadoop
+  # mode.
+  class Tap < BaseTap
+    attr_reader :scheme, :path, :sink_mode
+
+    def initialize(path, params = {})
+      @path = path
+
+      @scheme = params[:scheme] || text_line_scheme
+      raise "Scheme must provide one of :local_scheme or :hadoop_scheme; received: '#{scheme.inspect}'" unless scheme[:local_scheme] || scheme[:hadoop_scheme]
+
+      @sink_mode = case params[:sink_mode] || :keep
+        when :keep, 'keep'       then Java::CascadingTap::SinkMode::KEEP
+        when :replace, 'replace' then Java::CascadingTap::SinkMode::REPLACE
+        when :append, 'append'   then Java::CascadingTap::SinkMode::APPEND
+        else raise "Unrecognized sink mode '#{params[:sink_mode]}'"
+      end
+
+      local_scheme = scheme[:local_scheme]
+      @local_tap = local_scheme ? Java::CascadingTapLocal::FileTap.new(local_scheme, path, sink_mode) : nil
+
+      hadoop_scheme = scheme[:hadoop_scheme]
+      @hadoop_tap = hadoop_scheme ? Java::CascadingTapHadoop::Hfs.new(hadoop_scheme, path, sink_mode) : nil
+    end
+  end
+
+  # A Cascading::MultiTap represents one of Cascading's aggregate taps and is
+  # built via static constructors that accept an array of Cascading::Taps.  In
+  # order for a mode (Cascading local or Hadoop) to be supported, all provided
+  # taps must support it.
+  class MultiTap < BaseTap
+    def initialize(local_tap, hadoop_tap)
+      super(local_tap, hadoop_tap)
+    end
+
+    def self.multi_source_tap(taps)
+      multi_tap(taps, Java::CascadingTap::MultiSourceTap)
+    end
+
+    def self.multi_sink_tap(taps)
+      multi_tap(taps, Java::CascadingTap::MultiSinkTap)
+    end
+
+    private
+
+    def self.multi_tap(taps, klass)
+      local_supported = taps.all?{ |tap| tap.local? }
+      local_tap = local_supported ? klass.new(taps.map{ |tap| tap.local_tap }.to_java('cascading.tap.Tap')) : nil
+
+      hadoop_supported = taps.all?{ |tap| tap.hadoop? }
+      hadoop_tap = hadoop_supported ? klass.new(taps.map{ |tap| tap.hadoop_tap }.to_java('cascading.tap.Tap')) : nil
+
+      MultiTap.new(local_tap, hadoop_tap)
+    end
+  end
+end

data/lib/cascading.rb

@@ -6,7 +6,7 @@ require 'java'
 
 module Cascading
   # :stopdoc:
-  VERSION = '0.0.8'
+  VERSION = '0.0.9'
   LIBPATH = ::File.expand_path(::File.dirname(__FILE__)) + ::File::SEPARATOR
   PATH = ::File.dirname(LIBPATH) + ::File::SEPARATOR
   CASCADING_HOME = ENV['CASCADING_HOME']
@@ -55,8 +55,10 @@ require 'cascading/cascading'
 require 'cascading/cascading_exception'
 require 'cascading/expr_stub'
 require 'cascading/flow'
+require 'cascading/mode'
 require 'cascading/operations'
 require 'cascading/scope'
+require 'cascading/tap'
 
 # include module to make them available at top package
 include Cascading