cascading.jruby 0.0.6 → 0.0.7

data/History.txt

@@ -1,3 +1,9 @@
+0.0.7 - Upgrade to Cascading 2.0.0 wip-255
+
+This release upgrades to Cascading 2.0.0 wip-255, but implements neither local
+mode nor aggregations through AggregateBy.  Local jobs continue to run in Hadoop
+mode, but we intend to release local mode support soon.
+
 0.0.6 - Removing primary key
 
 The primary key feature was a source of great confusion at Etsy, so it's been

data/README.md

@@ -2,7 +2,7 @@
 
 `cascading.jruby` is a small DSL above [Cascading](http://www.cascading.org/).
 
-It requires Hadoop (>= 0.18.3) and Cascading (>=1.0.1) to be set via the environment variables: `HADOOP_HOME` and `CASCADING_HOME`
+It requires Hadoop (>= 0.20.2) and Cascading (>= 2.0.0) to be set via the environment variables: `HADOOP_HOME` and `CASCADING_HOME`
 
 It has been tested on JRuby versions 1.2.0, 1.4.0, 1.5.3, and 1.6.5.
 

data/TODO

@@ -6,7 +6,6 @@ Look into totally eliminating registries
 Bug fixes on github
 Enforce more runtime rules at composition time
 Standardize helper contracts
-Upgrade Cascading (already upgraded JRuby)
 Possibly combine unit tests...into unit tests because RSpec sucks and swallows stack traces
 
 Split out runner

data/lib/cascading.rb

@@ -6,7 +6,7 @@ require 'java'
 
 module Cascading
   # :stopdoc:
-  VERSION = '0.0.6'
+  VERSION = '0.0.7'
   LIBPATH = ::File.expand_path(::File.dirname(__FILE__)) + ::File::SEPARATOR
   PATH = ::File.dirname(LIBPATH) + ::File::SEPARATOR
   CASCADING_HOME = ENV['CASCADING_HOME']
@@ -40,7 +40,6 @@ module Cascading
     search_me = ::File.expand_path(
         ::File.join(from, '**', '*.jar'))
     Dir.glob(search_me).sort.each do |jar|
-      #puts "required: #{jar}"
       require jar
     end
   end

data/lib/cascading/assembly.rb

@@ -130,13 +130,13 @@ module Cascading
       if declared_fields
         case joiner
         when :inner, "inner", nil
-          joiner = Java::CascadingPipeCogroup::InnerJoin.new
+          joiner = Java::CascadingPipeJoiner::InnerJoin.new
         when :left,  "left"
-          joiner = Java::CascadingPipeCogroup::LeftJoin.new
+          joiner = Java::CascadingPipeJoiner::LeftJoin.new
         when :right, "right"
-          joiner = Java::CascadingPipeCogroup::RightJoin.new
+          joiner = Java::CascadingPipeJoiner::RightJoin.new
         when :outer, "outer"
-          joiner = Java::CascadingPipeCogroup::OuterJoin.new
+          joiner = Java::CascadingPipeJoiner::OuterJoin.new
         when Array
           joiner = joiner.map do |t|
             case t
@@ -145,7 +145,7 @@ module Cascading
             else fail "invalid mixed joiner entry: #{t}"
             end
           end
-          joiner = Java::CascadingPipeCogroup::MixedJoin.new(joiner.to_java(:boolean))
+          joiner = Java::CascadingPipeJoiner::MixedJoin.new(joiner.to_java(:boolean))
         end
       end
 

data/lib/cascading/cascading.rb

@@ -90,23 +90,36 @@ module Cascading
     Java::CascadingTuple::Fields::RESULTS
   end
 
-  # Creates a c.s.TextLine scheme instance from the specified fields.
-  def text_line_scheme(*fields)
-    unless fields.empty?
-      fields = fields(fields)
-      return Java::CascadingScheme::TextLine.new(fields)
-    else
-      return Java::CascadingScheme::TextLine.new
+  # Creates a c.s.h.TextLine scheme.  Positional args are used if <tt>:source_fields</tt> is not provided.
+  #
+  # The named options are:
+  # * <tt>:source_fields</tt> a string or array of strings.  Specifies the
+  #   fields to be read from a source with this scheme.  Defaults to ['offset', 'line'].
+  # * <tt>:sink_fields</tt> a string or array of strings. Specifies the fields
+  #   to be written to a sink with this scheme.  Defaults to all_fields.
+  # * <tt>:compression</tt> a symbol, either <tt>:enable</tt> or
+  #   <tt>:disable</tt>, that governs the TextLine scheme's compression.  Defaults
+  #   to the default TextLine compression.
+  def text_line_scheme(*args)
+    options = args.extract_options!
+    source_fields = fields(options[:source_fields] || (args.empty? ? ['offset', 'line'] : args))
+    sink_fields = fields(options[:sink_fields]) || all_fields
+    sink_compression = case options[:compression]
+      when :enable  then Java::CascadingSchemeHadoop::TextLine::Compress::ENABLE
+      when :disable then Java::CascadingSchemeHadoop::TextLine::Compress::DISABLE
+      else Java::CascadingSchemeHadoop::TextLine::Compress::DEFAULT
     end
+
+    Java::CascadingSchemeHadoop::TextLine.new(source_fields, sink_fields, sink_compression)
   end
 
-  # Creates a c.s.SequenceFile scheme instance from the specified fields.
+  # Creates a c.s.h.SequenceFile scheme instance from the specified fields.
   def sequence_file_scheme(*fields)
     unless fields.empty?
       fields = fields(fields)
-      return Java::CascadingScheme::SequenceFile.new(fields)
+      return Java::CascadingSchemeHadoop::SequenceFile.new(fields)
     else
-      return Java::CascadingScheme::SequenceFile.new(all_fields)
+      return Java::CascadingSchemeHadoop::SequenceFile.new(all_fields)
     end
   end
 
@@ -129,12 +142,36 @@ module Cascading
     end
     fs = opts[:kind] || :hfs
     klass = case fs
-      when :hfs, 'hfs' then Java::CascadingTap::Hfs
-      when :dfs, 'dfs' then Java::CascadingTap::Dfs
-      when :lfs, 'lfs' then Java::CascadingTap::Lfs
+      when :hfs, 'hfs' then Java::CascadingTapHadoop::Hfs
+      when :dfs, 'dfs' then Java::CascadingTapHadoop::Dfs
+      when :lfs, 'lfs' then Java::CascadingTapHadoop::Lfs
       else raise "Unrecognized kind of tap '#{fs}'"
     end
     parameters = [scheme, path, sink_mode]
     klass.new(*parameters)
   end
+
+  # Constructs properties to be passed to Flow#complete or Cascade#complete
+  # which will locate temporary Hadoop files in base_dir.  It is necessary
+  # to pass these properties only when executing local scripts via JRuby's main
+  # method, which confuses Cascading's attempt to find the containing jar.
+  def local_properties(base_dir)
+    dirs = {
+      'test.build.data' => "#{base_dir}/build",
+      'hadoop.tmp.dir' => "#{base_dir}/tmp",
+      'hadoop.log.dir' => "#{base_dir}/log",
+    }
+    dirs.each{ |key, dir| `mkdir -p #{dir}` }
+
+    job_conf = Java::OrgApacheHadoopMapred::JobConf.new
+    job_conf.jar = dirs['test.build.data']
+    dirs.each{ |key, dir| job_conf.set(key, dir) }
+
+    job_conf.num_map_tasks = 1
+    job_conf.num_reduce_tasks = 1
+
+    properties = java.util.HashMap.new
+    Java::CascadingFlowHadoop::HadoopPlanner.copy_job_conf(properties, job_conf)
+    properties
+  end
 end

data/lib/cascading/cascading_exception.rb

@@ -5,13 +5,23 @@
 # underlying Janino expression problems.
 module Cascading
   class CascadingException < StandardError
+    attr_accessor :ne, :message, :depth
+
     def initialize(native_exception, message)
       @ne = native_exception
-      super("#{message}\n#{trace_causes(@ne, 1)}")
+      @message = message
+      trace, @depth = trace_causes(@ne, 1)
+      super("#{message}\n#{trace}")
     end
 
-    def cause(depth)
-      fetch_cause(@ne, depth)
+    # Fetch cause at depth.  If depth is not provided, root cause is returned.
+    def cause(depth = @depth)
+      if depth > @depth
+        warn "WARNING: Depth (#{depth}) greater than depth of cause stack (#{@depth}) requested"
+        nil
+      else
+        fetch_cause(@ne, depth)
+      end
     end
 
     private
@@ -22,14 +32,20 @@ module Cascading
     end
 
     def trace_causes(ne, depth)
-      return unless ne
+      return ['', depth - 1] unless ne
+
+      warn "ERROR: Exception does not respond to cause: #{ne}" unless ne.respond_to?(:cause)
+      cause_trace, cause_depth = trace_causes(ne.respond_to?(:cause) ? ne.cause : nil, depth + 1)
+
       trace = "Cause #{depth}: #{ne.respond_to?(:java_class) ? ne.java_class : ne.class}: #{ne}\n"
       if ne.respond_to?(:stack_trace)
         trace += "#{ne.stack_trace.map{ |e| "  at #{e.class_name}.#{e.method_name}(#{e.file_name}:#{e.line_number})" }.join("\n")}\n"
       elsif ne.respond_to?(:backtrace)
         trace += "  #{ne.backtrace.join("\n  ")}\n"
       end
-      trace += "#{trace_causes(ne.cause, depth + 1)}"
+      trace += cause_trace
+
+      [trace, cause_depth]
     end
   end
 end

data/lib/cascading/flow.rb

@@ -144,7 +144,11 @@ module Cascading
         puts "#{key}=#{properties[key]}"
       end
 
-      Java::CascadingFlow::FlowConnector.new(properties).connect(
+      # FIXME: why do I have to do this in 2.0 wip-255?
+      Java::CascadingFlow::FlowConnector.setApplicationName(properties, name)
+      Java::CascadingFlow::FlowConnector.setApplicationVersion(properties, '0.0.0')
+
+      Java::CascadingFlowHadoop::HadoopFlowConnector.new(properties).connect(
         name,
         make_tap_parameter(@sources),
         make_tap_parameter(@sinks),

data/lib/cascading/scope.rb

@@ -49,17 +49,20 @@ module Cascading
       <<-END
 Scope name: #{@scope.name}
   Kind: #{kind}
-  Argument selector: #{@scope.argument_selector}
-  Declared fields: #{@scope.declared_fields}
-  Grouping selectors: #{@scope.grouping_selectors}
+  Key selectors:     #{@scope.key_selectors}
   Sorting selectors: #{@scope.sorting_selectors}
+  Remainder fields:  #{@scope.remainder_fields}
+  Declared fields:   #{@scope.declared_fields}
+  Arguments
+    selector:   #{@scope.arguments_selector}
+    declarator: #{@scope.arguments_declarator}
   Out grouping
-    selector: #{@scope.out_grouping_selector}
-    fields: #{grouping_fields}
-    key fields: #{@grouping_key_fields}
+    selector:   #{@scope.out_grouping_selector}
+    fields:     #{grouping_fields} (#{@scope.out_grouping_fields})
+    key fields: #{@grouping_key_fields} (#{@scope.key_selectors})
   Out values
     selector: #{@scope.out_values_selector}
-    fields: #{values_fields}
+    fields:   #{values_fields} (#{@scope.out_values_fields})
 END
     end
 

data/samples/branch.rb

@@ -3,7 +3,6 @@
 $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'cascading'
-require 'samples/cascading'
 
 cascade 'branch' do
   flow 'branch' do
@@ -28,4 +27,4 @@ cascade 'branch' do
     sink 'branch1', tap('output/branch1', :sink_mode => :replace)
     sink 'branch2', tap('output/branch2', :sink_mode => :replace)
   end
-end.complete(sample_properties)
+end.complete(local_properties('build/sample'))

data/samples/copy.rb

@@ -2,7 +2,6 @@
 $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'cascading'
-require 'samples/cascading'
 
 cascade 'copy' do
   flow 'copy' do
@@ -16,4 +15,4 @@ cascade 'copy' do
 
     sink 'input', tap('output/copy', :sink_mode => :replace)
   end
-end.complete(sample_properties)
+end.complete(local_properties('build/sample'))

data/samples/join.rb

@@ -2,7 +2,6 @@
 $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'cascading'
-require 'samples/cascading'
 
 cascade 'join' do
   flow 'join' do
@@ -29,4 +28,4 @@ cascade 'join' do
 
     sink 'join', tap('output/join', :sink_mode => :replace)
   end
-end.complete(sample_properties)
+end.complete(local_properties('build/sample'))

data/samples/logwordcount.rb

@@ -2,7 +2,6 @@
 $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'cascading'
-require 'samples/cascading'
 
 cascade 'logwordcount' do
   flow 'logwordcount' do
@@ -19,4 +18,4 @@ cascade 'logwordcount' do
 
     sink 'input', tap('output/logwordcount', :sink_mode => :replace)
   end
-end.complete(sample_properties)
+end.complete(local_properties('build/sample'))

data/samples/project.rb

@@ -4,7 +4,6 @@ $: << File.join(File.dirname(__FILE__), '..', 'lib')
 # History: "project" (verb) used to be known as "restrict"
 
 require 'cascading'
-require 'samples/cascading'
 
 cascade 'project' do
   flow 'project' do
@@ -21,4 +20,4 @@ cascade 'project' do
 
     sink 'input', tap('output/project', :sink_mode => :replace)
   end
-end.complete(sample_properties)
+end.complete(local_properties('build/sample'))

data/samples/rename.rb

@@ -2,7 +2,6 @@
 $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'cascading'
-require 'samples/cascading'
 
 cascade 'rename' do
   flow 'rename' do
@@ -18,4 +17,4 @@ cascade 'rename' do
 
     sink 'input', tap('output/rename', :sink_mode => :replace)
   end
-end.complete(sample_properties)
+end.complete(local_properties('build/sample'))

data/samples/scorenames.rb

@@ -2,7 +2,6 @@
 $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'cascading'
-require 'samples/cascading'
 
 cascade 'scorenames' do
   flow 'scorenames' do
@@ -17,4 +16,4 @@ cascade 'scorenames' do
 
     sink 'input', tap('output/scorenames', :sink_mode => :replace)
   end
-end.complete(sample_properties)
+end.complete(local_properties('build/sample'))

data/samples/splitter.rb

@@ -2,7 +2,6 @@
 $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'cascading'
-require 'samples/cascading'
 
 cascade 'splitter' do
   flow 'splitter' do
@@ -17,4 +16,4 @@ cascade 'splitter' do
 
     sink 'input', tap('output/splitter', :sink_mode => :replace)
   end
-end.complete(sample_properties)
+end.complete(local_properties('build/sample'))

data/samples/union.rb

@@ -2,7 +2,6 @@
 $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'cascading'
-require 'samples/cascading'
 
 cascade 'union' do
   flow 'union' do
@@ -32,4 +31,4 @@ cascade 'union' do
 
     sink 'union', tap('output/union', :sink_mode => :replace)
   end
-end.complete(sample_properties)
+end.complete(local_properties('build/sample'))

data/spec/scope_spec.rb

@@ -27,10 +27,9 @@ context Cascading::Scope do
 
       # Pass that uses our grouping fields instead of all_fields
       operation = Java::CascadingOperation::Identity.new 
-      make_each(# FIXME: names of grouping fields are not what we'd expect!
-                Java::CascadingPipe::Each, tail_pipe, fields([0, 'x_sum']), operation)
+      make_each(Java::CascadingPipe::Each, tail_pipe, fields(['x', 'x_sum']), operation)
 
-      check_scope :values_fields => [0, 'x_sum']
+      check_scope :values_fields => ['x', 'x_sum']
     end
   end
 
@@ -108,11 +107,6 @@ context Cascading::Scope do
         :grouping_fields => ['x', 'x_sum', 'y_sum']
       assert_size_equals 3
 
-      # No rename service provided unless you use the block form of join!
-      check_scope :values_fields => [0, 'x_sum', 'y_sum']
-
-      # Mimic rename service
-      bind_names ['x', 'x_sum', 'y_sum']
       check_scope :values_fields => ['x', 'x_sum', 'y_sum']
     end
   end
@@ -132,7 +126,7 @@ context Cascading::Scope do
         begin
           sum :mapping => {'x' => 'x_sum'}, :type => :int
         rescue CascadingException => e
-          raise e.cause(3)
+          raise e.cause
         end
       end
     end.should raise_error java.lang.IllegalStateException, 'Every cannot follow a Tap or an Each'
@@ -159,9 +153,9 @@ context Cascading::Scope do
           end
         end
       rescue CascadingException => e
-        raise e.cause(4)
+        raise e.cause
       end
-    end.should raise_error java.lang.IllegalStateException, 'Every cannot follow a Tap or an Each'
+    end.should raise_error Java::CascadingFlowPlanner::PlannerException, "[attempt_group][sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)] Every instances may not split after a GroupBy or CoGroup pipe, found: Every(attempt_group)[Sum[decl:'x_sum'][args:1]] after: CoGroup(left*right)[by:left:[{1}:'x']right:[{1}:'x']]"
   end
 
   it 'should propagate names through GroupBy' do

data/spec/spec_util.rb

@@ -25,7 +25,7 @@ def test_flow(&block)
   cascade = cascade 'test_app' do
     flow 'test', &block
   end
-  cascade.complete(cascading_properties)
+  cascade.complete(local_properties(BUILD_DIR))
 end
 
 def test_assembly(params = {}, &block)
@@ -90,38 +90,3 @@ def test_join_assembly(params = {}, &block)
     end
   end
 end
-
-def cascading_properties
-  build_dir = "#{BUILD_DIR}/build"
-  `mkdir -p #{build_dir}`
-  tmp_dir = "#{BUILD_DIR}/tmp"
-  `mkdir -p #{tmp_dir}`
-  log_dir = "#{BUILD_DIR}/log"
-  `mkdir -p #{log_dir}`
-
-  # Local cluster settings
-  #java.lang.System.set_property("test.build.data", build_dir)
-  #java.lang.System.set_property("hadoop.tmp.dir", tmp_dir)
-  #java.lang.System.set_property("hadoop.log.dir", log_dir)
-  #conf = Java::OrgApacheHadoopConf::Configuration.new
-  #dfs = Java::OrgApacheHadoopDfs::MiniDFSCluster.new(conf, 4, true, nil);
-  #file_sys = dfs.file_system
-  #mr = Java::OrgApacheHadoopMapred::MiniMRCluster.new(4, file_sys.uri.to_string, 1)
-  #job_conf = mr.create_job_conf
-  #job_conf.set("mapred.child.java.opts", "-Xmx512m")
-  #job_conf.set("mapred.map.tasks.speculative.execution", "false")
-  #job_conf.set("mapred.reduce.tasks.speculative.execution", "false")
-
-  job_conf = Java::OrgApacheHadoopMapred::JobConf.new
-  job_conf.jar = build_dir
-  job_conf.set("test.build.data", build_dir)
-  job_conf.set("hadoop.tmp.dir", tmp_dir)
-  job_conf.set("hadoop.log.dir", log_dir)
-
-  job_conf.num_map_tasks = 4
-  job_conf.num_reduce_tasks = 1
-
-  properties = java.util.HashMap.new({})
-  Java::CascadingFlow::MultiMapReducePlanner.set_job_conf(properties, job_conf)
-  properties
-end

data/tags

@@ -36,8 +36,11 @@ Scope	lib/cascading/scope.rb	/^  class Scope$/;"	c	class:Cascading
 ScopeTests	spec/spec_util.rb	/^module ScopeTests$/;"	m
 TC_Assembly	test/test_assembly.rb	/^class TC_Assembly < Test::Unit::TestCase$/;"	c
 TC_AssemblyScenarii	test/test_assembly.rb	/^class TC_AssemblyScenarii < Test::Unit::TestCase$/;"	c
+TC_Cascade	test/test_cascade.rb	/^class TC_Cascade < Test::Unit::TestCase$/;"	c
 TC_Cascading	test/test_cascading.rb	/^class TC_Cascading < Test::Unit::TestCase$/;"	c
+TC_Exceptions	test/test_exceptions.rb	/^class TC_Exceptions < Test::Unit::TestCase$/;"	c
 TC_Flow	test/test_flow.rb	/^class TC_Flow < Test::Unit::TestCase$/;"	c
+TC_Operations	test/test_operations.rb	/^class TC_Operations < Test::Unit::TestCase$/;"	c
 add	lib/cascading/base.rb	/^    def add(name, instance)$/;"	f	class:Cascading.Registerable
 add_archive_to_distributed_cache	lib/cascading/flow.rb	/^    def add_archive_to_distributed_cache(file)$/;"	f
 add_child	lib/cascading/base.rb	/^    def add_child(node)$/;"	f	class:Cascading.Node
@@ -60,9 +63,8 @@ bind_names	lib/cascading/assembly.rb	/^    def bind_names(*new_names)$/;"	f	clas
 branch	lib/cascading/assembly.rb	/^    def branch(name, &block)$/;"	f	class:Cascading.Assembly
 cascade	lib/cascading/cascading.rb	/^  def cascade(name, &block)$/;"	f	class:Cascading
 cascading.jruby	src/cascading/jruby/Main.java	/^package cascading.jruby;$/;"	p
-cascading_properties	spec/spec_util.rb	/^def cascading_properties$/;"	f
 cast	lib/cascading/assembly.rb	/^    def cast(type_map)$/;"	f	class:Cascading.Assembly
-cause	lib/cascading/cascading_exception.rb	/^    def cause(depth)$/;"	f	class:Cascading.CascadingException
+cause	lib/cascading/cascading_exception.rb	/^    def cause(depth = @depth)$/;"	f	class:Cascading.CascadingException
 check_scope	spec/spec_util.rb	/^  def check_scope(params = {})$/;"	f	class:ScopeTests
 compare_with_references	test/test_assembly.rb	/^def compare_with_references(test_name)$/;"	f
 compile	lib/cascading/expr_stub.rb	/^    def compile$/;"	f	class:Cascading
@@ -89,7 +91,6 @@ describe	lib/cascading/base.rb	/^    def describe(offset = '')$/;"	f	class:Casca
 describe	lib/cascading/cascade.rb	/^    def describe(offset = '')$/;"	f	class:Cascading.Cascade
 describe	lib/cascading/cascading.rb	/^  def describe$/;"	f	class:Cascading
 describe	lib/cascading/flow.rb	/^    def describe(offset = '')$/;"	f	class:Cascading.Flow
-describe_job	spec/spec_util.rb	/^def describe_job(job_file, &block)$/;"	f
 difference_fields	lib/cascading/cascading.rb	/^  def difference_fields(*fields)$/;"	f	class:Cascading
 discard	lib/cascading/assembly.rb	/^    def discard(*args)$/;"	f	class:Cascading.Assembly
 distinct	lib/cascading/assembly.rb	/^    def distinct(*args)$/;"	f	class:Cascading.Assembly
@@ -145,6 +146,7 @@ last_function	lib/cascading/operations.rb	/^    def last_function(*args)$/;"	f	c
 last_grouping_fields	lib/cascading/cascading.rb	/^  def last_grouping_fields$/;"	f	class:Cascading
 left_join	lib/cascading/assembly.rb	/^    def left_join(*args, &block)$/;"	f	class:Cascading.Assembly
 libpath	lib/cascading.rb	/^  def self.libpath( *args )$/;"	F	class:Cascading
+local_properties	lib/cascading/cascading.rb	/^  def local_properties(base_dir)$/;"	f
 main	src/cascading/jruby/Main.java	/^    public static void main(String[] args) {$/;"	m	class:Main
 make_each	lib/cascading/assembly.rb	/^    def make_each(type, *parameters)$/;"	f	class:Cascading.Assembly
 make_every	lib/cascading/assembly.rb	/^    def make_every(type, *parameters)$/;"	f	class:Cascading.Assembly
@@ -159,7 +161,7 @@ max_function	lib/cascading/operations.rb	/^    def max_function(*args)$/;"	f	cla
 min	lib/cascading/assembly.rb	/^    def min(*args); composite_aggregator(args, :min_function); end$/;"	f	class:Cascading.Assembly
 min_function	lib/cascading/operations.rb	/^    def min_function(*args)$/;"	f	class:Cascading.Operations
 mock_assembly	test/test_assembly.rb	/^  def mock_assembly(&block)$/;"	f	class:TC_Assembly
-multi_tap	lib/cascading/cascading.rb	/^  def multi_tap(*taps)$/;"	f	class:Cascading
+multi_tap	lib/cascading/cascading.rb	/^  def multi_tap(*taps)$/;"	f
 names_and_types	lib/cascading/expr_stub.rb	/^    def names_and_types$/;"	f	class:Cascading
 outer_join	lib/cascading/assembly.rb	/^    def outer_join(*args, &block)$/;"	f	class:Cascading.Assembly
 outgoing_scope	lib/cascading/scope.rb	/^    def self.outgoing_scope(flow_element, incoming_scopes, grouping_key_fields, every_applied)$/;"	F	class:Cascading.Scope
@@ -171,6 +173,7 @@ parse_date	lib/cascading/assembly.rb	/^    def parse_date(*args)$/;"	f	class:Cas
 pass	lib/cascading/assembly.rb	/^    def pass(*args)$/;"	f	class:Cascading.Assembly
 path	lib/cascading.rb	/^  def self.path( *args )$/;"	F	class:Cascading
 project	lib/cascading/assembly.rb	/^    def project(*args)$/;"	f	class:Cascading.Assembly
+qualified_name	lib/cascading/base.rb	/^    def qualified_name$/;"	f	class:Cascading.Node
 quiet	tasks/setup.rb	/^def quiet( &block )$/;"	f
 regex_filter	lib/cascading/operations.rb	/^    def regex_filter(*args)$/;"	f	class:Cascading.Operations.to_java_comparable_array
 regex_generator	lib/cascading/operations.rb	/^    def regex_generator(*args)$/;"	f	class:Cascading.Operations
@@ -187,11 +190,12 @@ require_all_jars	lib/cascading.rb	/^  def self.require_all_jars(from = ::File.jo
 reset	lib/cascading/base.rb	/^    def reset$/;"	f	class:Cascading.Registerable
 results_fields	lib/cascading/cascading.rb	/^  def results_fields$/;"	f	class:Cascading
 right_join	lib/cascading/assembly.rb	/^    def right_join(*args, &block)$/;"	f	class:Cascading.Assembly
+root	lib/cascading/base.rb	/^    def root$/;"	f	class:Cascading.Node
 sample_properties	samples/cascading.rb	/^  def sample_properties$/;"	f	class:Cascading
 scope	lib/cascading/assembly.rb	/^    def scope$/;"	f	class:Cascading.Assembly
 scope	lib/cascading/flow.rb	/^    def scope(name = nil)$/;"	f	class:Cascading.Flow
 search_field_name	lib/cascading/cascading.rb	/^  def search_field_name(names, candidate)$/;"	f	class:Cascading
-sequence_file_scheme	lib/cascading/cascading.rb	/^  def sequence_file_scheme(*fields)$/;"	f	class:Cascading
+sequence_file_scheme	lib/cascading/cascading.rb	/^  def sequence_file_scheme(*fields)$/;"	f
 set_spill_threshold	lib/cascading/flow.rb	/^    def set_spill_threshold(threshold)$/;"	f
 sink	lib/cascading/flow.rb	/^    def sink(*args)$/;"	f	class:Cascading.Flow
 sink_metadata	lib/cascading/cascade.rb	/^    def sink_metadata$/;"	f	class:Cascading.Cascade
@@ -203,13 +207,19 @@ split_hash	lib/cascading/expr_stub.rb	/^    def split_hash(h)$/;"	f	class:Cascad
 split_rows	lib/cascading/assembly.rb	/^    def split_rows(*args)$/;"	f	class:Cascading.Assembly
 sum	lib/cascading/assembly.rb	/^    def sum(*args)$/;"	f	class:Cascading.Assembly
 sum_function	lib/cascading/operations.rb	/^    def sum_function(*args)$/;"	f	class:Cascading.Operations
-tap	lib/cascading/cascading.rb	/^  def tap(*args)$/;"	f	class:Cascading
+tap	lib/cascading/cascading.rb	/^  def tap(*args)$/;"	f
 tap_scope	lib/cascading/scope.rb	/^    def self.tap_scope(tap, name)$/;"	F	class:Cascading.Scope
+test_aggregator_function_ignore_exception	test/test_operations.rb	/^  def test_aggregator_function_ignore_exception$/;"	f	class:TC_Operations
+test_aggregator_function_ignore_tuples	test/test_operations.rb	/^  def test_aggregator_function_ignore_tuples$/;"	f	class:TC_Operations
+test_aggregator_function_ignore_values	test/test_operations.rb	/^  def test_aggregator_function_ignore_values$/;"	f	class:TC_Operations
+test_ambiguous_assembly_names	test/test_flow.rb	/^  def test_ambiguous_assembly_names$/;"	f	class:TC_Flow
+test_ambiguous_branch_names	test/test_flow.rb	/^  def test_ambiguous_branch_names$/;"	f	class:TC_Flow
 test_assembly	spec/spec_util.rb	/^def test_assembly(params = {}, &block)$/;"	f
-test_assembly	test/test_flow.rb	/^  def test_assembly$/;"	f	class:TC_Flow
 test_branch_empty	test/test_assembly.rb	/^  def test_branch_empty$/;"	f
 test_branch_single	test/test_assembly.rb	/^  def test_branch_single$/;"	f
 test_branch_unique	test/test_assembly.rb	/^  def test_branch_unique$/;"	f
+test_cascade	test/test_cascade.rb	/^  def test_cascade$/;"	f	class:TC_Cascade
+test_cascading_exception	test/test_exceptions.rb	/^  def test_cascading_exception$/;"	f	class:TC_Exceptions
 test_create_assembly_simple	test/test_assembly.rb	/^  def test_create_assembly_simple$/;"	f	class:TC_Assembly
 test_create_each	test/test_assembly.rb	/^  def test_create_each$/;"	f
 test_create_every	test/test_assembly.rb	/^  def test_create_every$/;"	f
@@ -219,18 +229,25 @@ test_create_group_by_reverse	test/test_assembly.rb	/^  def test_create_group_by_
 test_create_group_by_with_sort	test/test_assembly.rb	/^  def test_create_group_by_with_sort$/;"	f
 test_create_group_by_with_sort_reverse	test/test_assembly.rb	/^  def test_create_group_by_with_sort_reverse$/;"	f
 test_each_identity	test/test_assembly.rb	/^  def test_each_identity$/;"	f
+test_empty_where	test/test_assembly.rb	/^  def test_empty_where$/;"	f
 test_fields_field	test/test_cascading.rb	/^  def test_fields_field$/;"	f	class:TC_Cascading
 test_fields_multiple	test/test_cascading.rb	/^  def test_fields_multiple$/;"	f	class:TC_Cascading
 test_fields_single	test/test_cascading.rb	/^  def test_fields_single$/;"	f	class:TC_Cascading
 test_flow	spec/spec_util.rb	/^def test_flow(&block)$/;"	f
+test_flow	test/test_flow.rb	/^  def test_flow$/;"	f	class:TC_Flow
 test_full_assembly	test/test_assembly.rb	/^  def test_full_assembly$/;"	f
-test_join1	test/test_assembly.rb	/^  def test_join1$/;"	f
+test_join1	test/test_assembly.rb	/^  def test_join1$/;"	f	class:TC_AssemblyScenarii
 test_join2	test/test_assembly.rb	/^  def test_join2$/;"	f
 test_join_assembly	spec/spec_util.rb	/^def test_join_assembly(params = {}, &block)$/;"	f
-test_splitter	test/test_assembly.rb	/^  def test_splitter$/;"	f	class:TC_AssemblyScenarii
+test_smoke_test_debug_scope	test/test_assembly.rb	/^  def test_smoke_test_debug_scope$/;"	f
+test_smoke_test_sequence_file_scheme	test/test_assembly.rb	/^  def test_smoke_test_sequence_file_scheme$/;"	f	class:TC_AssemblyScenarii
+test_splitter	test/test_assembly.rb	/^  def test_splitter$/;"	f	class:TC_AssemblyScenarii.test_smoke_test_sequence_file_scheme
 test_tap	test/test_cascading.rb	/^  def test_tap$/;"	f	class:TC_Cascading
 test_values	lib/cascading/expr_stub.rb	/^    def test_values$/;"	f	class:Cascading
-text_line_scheme	lib/cascading/cascading.rb	/^  def text_line_scheme(*fields)$/;"	f	class:Cascading
+test_where	test/test_assembly.rb	/^  def test_where$/;"	f
+test_where_with_expression	test/test_assembly.rb	/^  def test_where_with_expression$/;"	f
+test_where_with_import	test/test_assembly.rb	/^  def test_where_with_import$/;"	f
+text_line_scheme	lib/cascading/cascading.rb	/^  def text_line_scheme(*args)$/;"	f	class:Cascading
 to_java_comparable_array	lib/cascading/operations.rb	/^    def to_java_comparable_array(arr)$/;"	f	class:Cascading.Operations
 to_s	lib/cascading/assembly.rb	/^    def to_s$/;"	f	class:Cascading.Assembly
 to_s	lib/cascading/expr_stub.rb	/^    def to_s$/;"	f	class:Cascading
@@ -244,7 +261,6 @@ validate	lib/cascading/expr_stub.rb	/^    def validate(actual_args = {})$/;"	f	c
 validate_fields	lib/cascading/expr_stub.rb	/^    def validate_fields(fields)$/;"	f	class:Cascading
 validate_scope	lib/cascading/expr_stub.rb	/^    def validate_scope(scope)$/;"	f	class:Cascading
 values_fields	lib/cascading/scope.rb	/^    def values_fields$/;"	f	class:Cascading.Scope
-verify_assembly_output	spec/spec_util.rb	/^def verify_assembly_output(assembly_name, params, &block)$/;"	f
 version	lib/cascading.rb	/^  def self.version$/;"	F	class:Cascading
 where	lib/cascading/assembly.rb	/^    def where(*args)$/;"	f	class:Cascading.Assembly
 write_sink_metadata	lib/cascading/cascade.rb	/^    def write_sink_metadata(file_name)$/;"	f	class:Cascading.Cascade

data/test/test_assembly.rb

@@ -45,13 +45,15 @@ class TC_Assembly < Test::Unit::TestCase
   end
 
   def test_create_each
-    # You can't apply an Each to 0 fields
-    assert_raise CascadingException do
-      assembly = mock_assembly do
-        each(:filter => identity)
-      end
-      assert assembly.tail_pipe.is_a? Java::CascadingPipe::Each
+    # You can apply an Each to 0 fields
+    assembly = mock_assembly do
+      each(:filter => identity)
     end
+    assert assembly.tail_pipe.is_a? Java::CascadingPipe::Each
+
+    # In which case, it has empty argument and output selectors
+    assert_equal 0, assembly.tail_pipe.argument_selector.size
+    assert_equal 0, assembly.tail_pipe.output_selector.size
 
     assembly = mock_assembly do
       each('offset', :output => 'offset_copy',
@@ -61,8 +63,8 @@ class TC_Assembly < Test::Unit::TestCase
 
     assert pipe.is_a? Java::CascadingPipe::Each
 
-    assert_equal 'offset', pipe.getArgumentSelector().get(0)
-    assert_equal 'offset_copy', pipe.getOutputSelector().get(0)
+    assert_equal 'offset', pipe.argument_selector.get(0)
+    assert_equal 'offset_copy', pipe.output_selector.get(0)
   end
 
   # For now, replaced these tests with the trivial observation that you can't
@@ -89,7 +91,7 @@ class TC_Assembly < Test::Unit::TestCase
         every("Field1", :aggregator => count_function)
       end
       assert assembly.tail_pipe.is_a? Java::CascadingPipe::Every
-      assert_equal "Field1", assembly.tail_pipe.getArgumentSelector().get(0)
+      assert_equal "Field1", assembly.tail_pipe.argument_selector.get(0)
     end
 
     assert_raise CascadingException do
@@ -97,8 +99,8 @@ class TC_Assembly < Test::Unit::TestCase
         every('line', :aggregator => count_function, :output=>'line_count')
       end
       assert assembly.tail_pipe.is_a? Java::CascadingPipe::Every
-      assert_equal 'line', assembly.tail_pipe.getArgumentSelector().get(0)
-      assert_equal 'line_count', assembly.tail_pipe.getOutputSelector().get(0)
+      assert_equal 'line', assembly.tail_pipe.argument_selector.get(0)
+      assert_equal 'line_count', assembly.tail_pipe.output_selector.get(0)
     end
   end
 
@@ -108,7 +110,7 @@ class TC_Assembly < Test::Unit::TestCase
     end
 
     assert assembly.tail_pipe.is_a? Java::CascadingPipe::GroupBy
-    grouping_fields = assembly.tail_pipe.getGroupingSelectors()['test']
+    grouping_fields = assembly.tail_pipe.key_selectors['test']
     assert_equal 'line', grouping_fields.get(0)
 
     assembly = mock_assembly do
@@ -116,7 +118,7 @@ class TC_Assembly < Test::Unit::TestCase
     end
 
     assert assembly.tail_pipe.is_a? Java::CascadingPipe::GroupBy
-    grouping_fields = assembly.tail_pipe.getGroupingSelectors()['test']
+    grouping_fields = assembly.tail_pipe.key_selectors['test']
     assert_equal 'line', grouping_fields.get(0)
   end
 
@@ -126,7 +128,7 @@ class TC_Assembly < Test::Unit::TestCase
     end
 
     assert assembly.tail_pipe.is_a? Java::CascadingPipe::GroupBy
-    grouping_fields = assembly.tail_pipe.getGroupingSelectors()['test']
+    grouping_fields = assembly.tail_pipe.key_selectors['test']
     assert_equal 'offset', grouping_fields.get(0)
     assert_equal 'line', grouping_fields.get(1)
   end
@@ -137,8 +139,8 @@ class TC_Assembly < Test::Unit::TestCase
     end
 
     assert assembly.tail_pipe.is_a? Java::CascadingPipe::GroupBy
-    grouping_fields = assembly.tail_pipe.getGroupingSelectors()['test']
-    sorting_fields = assembly.tail_pipe.getSortingSelectors()['test']
+    grouping_fields = assembly.tail_pipe.key_selectors['test']
+    sorting_fields = assembly.tail_pipe.sorting_selectors['test']
 
     assert_equal 2, grouping_fields.size
     assert_equal 1, sorting_fields.size
@@ -156,8 +158,8 @@ class TC_Assembly < Test::Unit::TestCase
     end
 
     assert assembly.tail_pipe.is_a? Java::CascadingPipe::GroupBy
-    grouping_fields = assembly.tail_pipe.getGroupingSelectors()['test']
-    sorting_fields = assembly.tail_pipe.getSortingSelectors()['test']
+    grouping_fields = assembly.tail_pipe.key_selectors['test']
+    sorting_fields = assembly.tail_pipe.sorting_selectors['test']
 
     assert_equal 2, grouping_fields.size
     assert_equal 1, sorting_fields.size
@@ -175,8 +177,8 @@ class TC_Assembly < Test::Unit::TestCase
     end
 
     assert assembly.tail_pipe.is_a? Java::CascadingPipe::GroupBy
-    grouping_fields = assembly.tail_pipe.getGroupingSelectors()['test']
-    sorting_fields = assembly.tail_pipe.getSortingSelectors()['test']
+    grouping_fields = assembly.tail_pipe.key_selectors['test']
+    sorting_fields = assembly.tail_pipe.sorting_selectors['test']
 
     assert_equal 2, grouping_fields.size
     assert_equal 2, sorting_fields.size
@@ -283,10 +285,34 @@ class TC_Assembly < Test::Unit::TestCase
     assert assembly.tail_pipe.is_a? Java::CascadingPipe::Each
     assert assembly.tail_pipe.operation.is_a? Java::CascadingOperationExpression::ExpressionFilter
   end
-end
 
+  def test_smoke_test_debug_scope
+    cascade 'smoke' do
+      flow 'smoke' do
+        source 'input', tap('test/data/data1.txt')
+        assembly 'input' do
+          pass
+          debug_scope
+        end
+        sink 'input', tap('output/smoke_test_debug_scope')
+      end
+    end
+  end
+end
 
 class TC_AssemblyScenarii < Test::Unit::TestCase
+  def test_smoke_test_sequence_file_scheme
+    cascade 'smoke' do
+      flow 'smoke' do
+        source 'input', tap('test/data/data1.txt')
+        assembly 'input' do
+          pass
+        end
+        compress_output :default, :block
+        sink 'input', tap('output/smoke_test_sequence_file_scheme', :scheme => sequence_file_scheme)
+      end
+    end.complete
+  end
 
   def test_splitter
     flow = flow "splitter" do

data/test/test_cascading.rb

@@ -32,18 +32,18 @@ class TC_Cascading < Test::Unit::TestCase
   def test_tap
     tap = tap('/temp')
     assert_equal '/temp', tap.getPath().toString()
-    assert tap.is_a? Java::CascadingTap::Hfs  
+    assert tap.is_a? Java::CascadingTapHadoop::Hfs
 
     tap = tap('/temp', :kind => :dfs)
     assert_equal '/temp', tap.getPath().toString()
-    assert tap.is_a? Java::CascadingTap::Dfs
+    assert tap.is_a? Java::CascadingTapHadoop::Dfs
 
     tap = tap('/temp', :kind => :lfs)
     assert_equal '/temp', tap.getPath().toString()
-    assert tap.is_a? Java::CascadingTap::Lfs
+    assert tap.is_a? Java::CascadingTapHadoop::Lfs
 
     tap = tap('/temp', :kind => :hfs)
     assert_equal '/temp', tap.getPath().toString()
-    assert tap.is_a? Java::CascadingTap::Hfs
+    assert tap.is_a? Java::CascadingTapHadoop::Hfs
   end
 end

data/test/test_exceptions.rb

@@ -0,0 +1,41 @@
+require 'test/unit'
+require 'cascading'
+
+class TC_Exceptions < Test::Unit::TestCase
+  def test_cascading_exception
+    ne3 = java.lang.IllegalArgumentException.new('Root cause')
+    ne2 = Java::CascadingPipe::OperatorException.new(Java::CascadingPipe::Pipe.new('dummy'), 'Exception thrown by Cascading', ne3)
+    ne1 = Java::Cascading::CascadingException.new('Exception Cascading hands us', ne2)
+    e = CascadingException.new(ne1, 'cascading.jruby wrapper exception')
+
+    assert_equal ne1, e.ne
+    assert_equal 'cascading.jruby wrapper exception', e.message
+    assert_equal 3, e.depth
+
+
+    assert_equal ne1, e.cause(1)
+    assert_equal 'Exception Cascading hands us', e.cause(1).message
+
+    assert_equal ne2, e.cause(2)
+    # Cascading inserts Operator#to_s, here
+    assert_match /Exception thrown by Cascading/, e.cause(2).message
+
+    assert_equal ne3, e.cause(3)
+    assert_equal 'Root cause', e.cause(3).message
+
+    # Shallower than depth 1 is the first cause
+    (-5..0).each do |i|
+      assert_equal ne1, e.cause(i)
+      assert_equal 'Exception Cascading hands us', e.cause(i).message
+    end
+
+    # Deeper than the root cause is nil
+    (4..10).each do |i|
+      assert_nil e.cause(i)
+    end
+
+    # cause without depth returns root cause
+    assert_equal ne3, e.cause
+    assert_equal 'Root cause', e.cause.message
+  end
+end

metadata

@@ -2,7 +2,7 @@
 name: cascading.jruby
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors: 
   - Matt Walker
@@ -11,7 +11,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-03-26 00:00:00 Z
+date: 2012-04-02 00:00:00 Z
 dependencies: []
 
 description: cascading.jruby is a small DSL above Cascading, written in JRuby
@@ -52,7 +52,6 @@ files:
   - lib/cascading/operations.rb
   - lib/cascading/scope.rb
   - samples/branch.rb
-  - samples/cascading.rb
   - samples/copy.rb
   - samples/data/data2.txt
   - samples/data/data_join1.txt
@@ -96,6 +95,7 @@ files:
   - test/test_assembly.rb
   - test/test_cascade.rb
   - test/test_cascading.rb
+  - test/test_exceptions.rb
   - test/test_flow.rb
   - test/test_operations.rb
 homepage: http://github.com/etsy/cascading.jruby
@@ -122,7 +122,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: cascading.jruby
-rubygems_version: 1.8.9
+rubygems_version: 1.8.21
 signing_key: 
 specification_version: 3
 summary: A JRuby DSL for Cascading
@@ -130,5 +130,6 @@ test_files:
   - test/test_assembly.rb
   - test/test_cascade.rb
   - test/test_cascading.rb
+  - test/test_exceptions.rb
   - test/test_flow.rb
   - test/test_operations.rb

data/samples/cascading.rb

@@ -1,41 +0,0 @@
-module Cascading
-  # Constructs properties to be passed to Flow#complete or Cascade#complete
-  # which will locate temporary Hadoop files in build/sample.  It is necessary
-  # to pass these properties only because the sample apps are invoked using
-  # JRuby's main method, which confuses the JobConf's attempt to find the
-  # containing jar.
-  def sample_properties
-    build_dir = 'build/sample/build'
-    `mkdir -p #{build_dir}`
-    tmp_dir = "build/sample/tmp"
-    `mkdir -p #{tmp_dir}`
-    log_dir = "build/sample/log"
-    `mkdir -p #{log_dir}`
-
-    # Local cluster settings
-    #java.lang.System.set_property("test.build.data", build_dir)
-    #java.lang.System.set_property("hadoop.tmp.dir", tmp_dir)
-    #java.lang.System.set_property("hadoop.log.dir", log_dir)
-    #conf = Java::OrgApacheHadoopConf::Configuration.new
-    #dfs = Java::OrgApacheHadoopDfs::MiniDFSCluster.new(conf, 4, true, nil);
-    #file_sys = dfs.file_system
-    #mr = Java::OrgApacheHadoopMapred::MiniMRCluster.new(4, file_sys.uri.to_string, 1)
-    #job_conf = mr.create_job_conf
-    #job_conf.set("mapred.child.java.opts", "-Xmx512m")
-    #job_conf.set("mapred.map.tasks.speculative.execution", "false")
-    #job_conf.set("mapred.reduce.tasks.speculative.execution", "false")
-
-    job_conf = Java::OrgApacheHadoopMapred::JobConf.new
-    job_conf.jar = build_dir
-    job_conf.set("test.build.data", build_dir)
-    job_conf.set("hadoop.tmp.dir", tmp_dir)
-    job_conf.set("hadoop.log.dir", log_dir)
-
-    job_conf.num_map_tasks = 4
-    job_conf.num_reduce_tasks = 1
-
-    properties = java.util.HashMap.new({})
-    Java::CascadingFlow::MultiMapReducePlanner.set_job_conf(properties, job_conf)
-    properties
-  end
-end