cascading.jruby 0.0.4

data/samples/data/data_join1.txt

@@ -0,0 +1,3 @@
+1			Grégoire			
+2			Mathias
+3			Stéphane

data/samples/data/data_join2.txt

@@ -0,0 +1,3 @@
+1			33
+2			30
+3			25

data/samples/data/data_join3.txt

@@ -0,0 +1,3 @@
+1				Cannes
+2				Boston
+3				Paris

data/samples/join.rb

@@ -0,0 +1,32 @@
+#! /usr/bin/env jruby
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'cascading'
+require 'samples/cascading'
+
+cascade 'join' do
+  flow 'join' do
+    source 'input1', tap('samples/data/data_join1.txt')
+    source 'input2', tap('samples/data/data_join2.txt')
+    source 'input3', tap('samples/data/data_join3.txt')
+
+    assembly 'input1' do
+      split 'line', ['id', 'name']
+    end
+
+    assembly 'input2' do
+      split 'line', ['id', 'age']
+    end
+
+    assembly 'input3' do
+      split 'line', ['id', 'city']
+    end
+
+    assembly 'join' do
+      join 'input1', 'input2', 'input3', :on => 'id'
+      project 'id', 'name', 'age', 'city'
+    end
+
+    sink 'join', tap('output/join', :sink_mode => :replace)
+  end
+end.complete(sample_properties)

data/samples/logwordcount.rb

@@ -0,0 +1,22 @@
+#! /usr/bin/env jruby
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'cascading'
+require 'samples/cascading'
+
+cascade 'logwordcount' do
+  flow 'logwordcount' do
+    source 'input', tap('http://www.gutenberg.org/files/20417/20417-8.txt')
+
+    assembly 'input' do
+      # TODO: create a helper for RegexSplitGenerator
+      each 'line', :function => regex_split_generator('word', :pattern => /[.,]*\s+/)
+      group_by 'word' do
+        count
+      end
+      group_by 'count', :reverse => true
+    end
+
+    sink 'input', tap('output/logwordcount', :sink_mode => :replace)
+  end
+end.complete(sample_properties)

data/samples/project.rb

@@ -0,0 +1,24 @@
+#! /usr/bin/env jruby
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+# History: "project" (verb) used to be known as "restrict"
+
+require 'cascading'
+require 'samples/cascading'
+
+cascade 'project' do
+  flow 'project' do
+    source 'input', tap('samples/data/data2.txt')
+
+    assembly 'input' do
+      split 'line', ['name', 'score1', 'score2', 'id'], :output => ['name', 'score1', 'score2', 'id']
+      assert Java::CascadingOperationAssertion::AssertSizeEquals.new(4)
+      project 'name', 'score1', 'score2'
+      assert Java::CascadingOperationAssertion::AssertSizeEquals.new(3)
+      project 'name', 'score2'
+      assert Java::CascadingOperationAssertion::AssertSizeEquals.new(2)
+    end
+
+    sink 'input', tap('output/project', :sink_mode => :replace)
+  end
+end.complete(sample_properties)

data/samples/rename.rb

@@ -0,0 +1,21 @@
+#! /usr/bin/env jruby
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'cascading'
+require 'samples/cascading'
+
+cascade 'rename' do
+  flow 'rename' do
+    source 'input', tap('samples/data/data2.txt')
+
+    assembly 'input' do
+      split 'line', ['name', 'score1', 'score2', 'id'], :output => ['name', 'score1', 'score2', 'id']
+      assert Java::CascadingOperationAssertion::AssertSizeEquals.new(4)
+      rename 'name' => 'new_name', 'score1' => 'new_score1', 'score2' => 'new_score2'
+      assert Java::CascadingOperationAssertion::AssertSizeEquals.new(4)
+      puts "Final field names: #{scope.values_fields.to_a.inspect}"
+    end
+
+    sink 'input', tap('output/rename', :sink_mode => :replace)
+  end
+end.complete(sample_properties)

data/samples/scorenames.rb

@@ -0,0 +1,20 @@
+#! /usr/bin/env jruby
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'cascading'
+require 'samples/cascading'
+
+cascade 'scorenames' do
+  flow 'scorenames' do
+    # You don't have to curl and cache inputs: tap can fetch via HTTP
+    source 'input', tap('http://www.census.gov/genealogy/names/dist.all.last')
+
+    assembly 'input' do
+      split 'line', ['name', 'val1', 'val2', 'id']
+      insert 'val3' => expr('val2:double < 40.0 ? val1:double : val2:double')
+      project 'name', 'val3', 'id'
+    end
+
+    sink 'input', tap('output/scorenames', :sink_mode => :replace)
+  end
+end.complete(sample_properties)

data/samples/splitter.rb

@@ -0,0 +1,20 @@
+#! /usr/bin/env jruby
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'cascading'
+require 'samples/cascading'
+
+cascade 'splitter' do
+  flow 'splitter' do
+    source 'input', tap('samples/data/data2.txt')
+
+    assembly 'input' do
+      split 'line', ['name', 'score1', 'score2', 'id'], :output => ['name', 'score1', 'score2', 'id']
+      group_by 'score1' do
+        count
+      end
+    end
+
+    sink 'input', tap('output/splitter', :sink_mode => :replace)
+  end
+end.complete(sample_properties)

data/samples/union.rb

@@ -0,0 +1,35 @@
+#! /usr/bin/env jruby
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'cascading'
+require 'samples/cascading'
+
+cascade 'union' do
+  flow 'union' do
+    source 'input', tap('http://www.census.gov/genealogy/names/dist.all.last')
+
+    assembly 'input' do
+      split 'line', ['name', 'score1', 'score2', 'id']
+
+      branch 'branch1' do
+        group_by 'score1', 'name' do
+          count
+        end
+        rename 'score1' => 'score'
+      end
+
+      branch 'branch2' do
+        group_by 'score2', 'name' do
+          count
+        end
+        rename 'score2' => 'score'
+      end
+    end
+
+    assembly 'union' do
+      union 'branch1', 'branch2'
+    end
+
+    sink 'union', tap('output/union', :sink_mode => :replace)
+  end
+end.complete(sample_properties)

data/spec/cascading_spec.rb

@@ -0,0 +1,100 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe Cascading do
+  it 'should dedup field names from multiple sources' do
+    left_names = ['a', 'b', 'c', 'd', 'e']
+    mid_names = ['a', 'f']
+    right_names = ['a', 'g']
+
+    field_names = dedup_field_names(left_names, mid_names, right_names)
+    field_names.should == [
+      'a', 'b', 'c', 'd', 'e',
+      'a_', 'f',
+      'a__', 'g'
+    ]
+  end
+
+  it 'should fail to resolve duplicate fields' do
+    incoming = fields(['line'])
+    declared = fields(['line'])
+    outgoing = all_fields
+    lambda do
+      begin
+        resolved = Java::CascadingTuple::Fields.resolve(outgoing, [incoming, declared].to_java(Java::CascadingTuple::Fields))
+      rescue NativeException => e
+        raise e.cause
+      end
+    end.should raise_error Java::CascadingTuple::TupleException, 'field name already exists: line'
+  end
+
+  it 'should find branches to sink' do
+    cascade 'branched_pass' do
+      flow 'branched_pass' do
+        source 'input', tap('spec/resource/test_input.txt', :kind => :lfs, :scheme => text_line_scheme)
+        assembly 'input' do
+          branch 'branched_input' do
+            project 'line'
+          end
+        end
+        sink 'branched_input', tap("#{OUTPUT_DIR}/branched_pass_out", :kind => :lfs, :sink_mode => :replace)
+      end
+    end.complete
+
+    ilc = `wc -l spec/resource/test_input.txt`.split(/\s+/).first
+    olc = `wc -l #{OUTPUT_DIR}/branched_pass_out/part-00000`.split(/\s+/).first
+    ilc.should == olc
+  end
+
+  it 'should create an isolated namespace per cascade' do
+    cascade 'double' do
+      flow 'double' do
+        source 'input', tap('spec/resource/test_input.txt', :kind => :lfs, :scheme => text_line_scheme)
+        assembly 'input' do # Dup name
+          insert 'doubled' => expr('line:string + "," + line:string')
+          project 'doubled'
+        end
+        sink 'input', tap("#{OUTPUT_DIR}/double_out", :kind => :lfs, :sink_mode => :replace)
+      end
+    end
+
+    cascade 'pass' do
+      flow 'pass' do
+        source 'input', tap('spec/resource/test_input.txt', :kind => :lfs, :scheme => text_line_scheme)
+        assembly 'input' do # Dup name
+          project 'line'
+        end
+        sink 'input', tap("#{OUTPUT_DIR}/pass_out", :kind => :lfs, :sink_mode => :replace)
+      end
+    end
+
+    Cascade.get('double').complete
+    Cascade.get('pass').complete
+    diff = `diff #{OUTPUT_DIR}/double_out/part-00000 #{OUTPUT_DIR}/pass_out/part-00000`
+    diff.should_not be_empty
+  end
+
+  it 'should support joins in branches' do
+    cascade 'branch_join' do
+      flow 'branch_join' do
+        source 'left', tap('spec/resource/join_input.txt', :kind => :lfs, :scheme => text_line_scheme)
+        source 'right', tap('spec/resource/join_input.txt', :kind => :lfs, :scheme => text_line_scheme)
+
+        assembly 'left' do
+          split 'line', ['x', 'y', 'z'], :pattern => /,/
+          project 'x', 'y', 'z'
+        end
+
+        assembly 'right' do
+          split 'line', ['x', 'y', 'z'], :pattern => /,/
+          project 'x', 'y', 'z'
+
+          branch 'branch_join' do
+            join 'left', 'right', :on => 'x'
+          end
+        end
+
+        sink 'branch_join', tap("#{OUTPUT_DIR}/branch_join_out.txt", :kind => :lfs, :sink_mode => :replace)
+      end
+    end.complete
+  end
+end

data/spec/expr_spec.rb

@@ -0,0 +1,10 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe Object do
+  it 'should allow expr syntax' do
+    test_assembly do
+      insert 'foo' => 1, 'bar' => expr('offset:int')
+      check_scope :values_fields => ['offset', 'line', 'bar', 'foo']
+    end
+  end
+end

data/spec/primary_key_spec.rb

@@ -0,0 +1,119 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe Cascading::Scope do
+  it 'should allow override of primary key' do
+    test_assembly do
+      split 'line', ['x', 'y'], :pattern => /,/
+      check_scope :values_fields => ['offset', 'line', 'x', 'y'],
+        :primary_key_fields => ['offset']
+    end
+  end
+
+  it 'should pass primary key through Each' do
+    test_assembly do
+      split 'line', ['x', 'y'], :pattern => /,/
+      check_scope :values_fields => ['offset', 'line', 'x', 'y'],
+        :primary_key_fields => ['offset']
+      pass
+      check_scope :values_fields => ['offset', 'line', 'x', 'y'],
+        :primary_key_fields => ['offset']
+    end
+  end
+
+  it 'should support renaming primary keys' do
+    test_assembly do
+      split 'line', ['x', 'y'], :pattern => /,/
+      check_scope :values_fields => ['offset', 'line', 'x', 'y'],
+        :primary_key_fields => ['offset']
+      rename 'offset' => 'primary_key', 'line' => 'data'
+      check_scope :values_fields => ['primary_key', 'data', 'x', 'y'],
+        :primary_key_fields => ['primary_key']
+    end
+  end
+
+  it 'should clear primary keys when a subset of their fields are discarded' do
+    test_assembly do
+      primary 'offset', 'line' # Make primary keys interesting
+      split 'line', ['x', 'y'], :pattern => /,/
+      check_scope :values_fields => ['offset', 'line', 'x', 'y'],
+        :primary_key_fields => ['offset', 'line']
+      project 'line', 'x', 'y'
+      check_scope :values_fields => ['line', 'x', 'y'],
+        :primary_key_fields => nil
+    end
+  end
+
+  it 'should pass primary key through branch' do
+    test_assembly do
+      split 'line', ['x', 'y'], :pattern => /,/
+      check_scope :values_fields => ['offset', 'line', 'x', 'y'],
+        :primary_key_fields => ['offset']
+
+      branch 'check_keys' do
+        check_scope :values_fields => ['offset', 'line', 'x', 'y'],
+          :primary_key_fields => ['offset']
+        pass
+        check_scope :values_fields => ['offset', 'line', 'x', 'y'],
+          :primary_key_fields => ['offset']
+      end
+
+      check_scope :values_fields => ['offset', 'line', 'x', 'y'],
+        :primary_key_fields => ['offset']
+      pass
+      check_scope :values_fields => ['offset', 'line', 'x', 'y'],
+        :primary_key_fields => ['offset']
+    end
+  end
+
+  it 'should pass primary key through GroupBy followed by Each' do
+    test_assembly do
+      group_by 'offset'
+      check_scope :values_fields => ['offset', 'line'],
+        :grouping_fields => ['offset'],
+        :primary_key_fields => ['offset']
+    end
+  end
+
+  it 'should pass primary key through GroupBy followed by Every' do
+    test_assembly do
+      split 'line', ['x', 'y'], :pattern => /,/
+      group_by 'offset', 'x' do
+        check_scope :values_fields => ['offset', 'line', 'x', 'y'],
+          :grouping_fields => ['offset', 'x'],
+          :primary_key_fields => ['offset'],
+          :grouping_primary_key_fields => ['offset', 'x']
+        count 'line'
+        check_scope :values_fields => ['offset', 'line', 'x', 'y'],
+          :grouping_fields => ['offset', 'x', 'line'],
+          :primary_key_fields => ['offset'],
+          :grouping_primary_key_fields => ['offset', 'x']
+        count 'y'
+        check_scope :values_fields => ['offset', 'line', 'x', 'y'],
+          :grouping_fields => ['offset', 'x', 'line', 'y'],
+          :primary_key_fields => ['offset', 'x'], # FIXME: why has the pk changed?
+          :grouping_primary_key_fields => ['offset', 'x']
+      end
+      check_scope :values_fields => ['offset', 'x', 'line', 'y'],
+        :primary_key_fields => ['offset', 'x']
+    end
+  end
+
+  it 'should not clear primary key when grouping on other fields' do
+    test_assembly do
+      group_by 'line'
+      check_scope :values_fields => ['offset', 'line'],
+        :grouping_fields => ['line'],
+        :primary_key_fields => ['offset'],
+        :grouping_primary_key_fields => ['line']
+    end
+  end
+
+  it 'should pass primary key through CoGroup' do
+    test_join_assembly do
+      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
+        :grouping_fields => ['x'],
+        :primary_key_fields => ['offset'],
+        :grouping_primary_key_fields => ['x']
+    end
+  end
+end 

data/spec/resource/join_input.txt

@@ -0,0 +1,3 @@
+1,4,world
+2,3,bar
+3,8,baz

data/spec/resource/test_input.txt

@@ -0,0 +1,4 @@
+hello,world
+foo,bar
+foo,bar
+biz,baz

data/spec/scope_spec.rb

@@ -0,0 +1,174 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe Cascading::Scope do
+  it 'should match Cascading fields names from source tap scheme' do
+    test_assembly do
+      # Pass that uses our scope instead of all_fields
+      operation = Java::CascadingOperation::Identity.new 
+      make_each(Java::CascadingPipe::Each, tail_pipe, scope.values_fields, operation)
+
+      check_scope :values_fields => ['offset', 'line']
+    end
+  end
+
+  it 'should match Cascading fields names after CoGroup' do
+    test_join_assembly do
+      # Pass that uses our scope instead of all_fields
+      operation = Java::CascadingOperation::Identity.new 
+      make_each(Java::CascadingPipe::Each, tail_pipe, scope.values_fields, operation)
+
+      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_']
+    end
+  end
+
+  it 'should match Cascading fields names after Every' do
+    test_join_assembly do
+      sum :mapping => {'x' => 'x_sum'}, :type => :int
+
+      # Pass that uses our grouping fields instead of all_fields
+      operation = Java::CascadingOperation::Identity.new 
+      make_each(# FIXME: names of grouping fields are not what we'd expect!
+                Java::CascadingPipe::Each, tail_pipe, fields([0, 'x_sum']), operation)
+
+      check_scope :values_fields => [0, 'x_sum']
+    end
+  end
+
+  it 'should pick up names from source tap scheme' do
+    test_assembly do
+      pass
+
+      check_scope :values_fields => ['offset', 'line']
+    end
+  end
+
+  it 'should propagate names through Each' do
+    test_assembly do
+      check_scope :values_fields => ['offset', 'line']
+      assert_size_equals 2
+
+      split 'line', ['x', 'y'], :pattern => /,/
+      check_scope :values_fields => ['offset', 'line', 'x', 'y']
+      assert_size_equals 4
+    end
+  end
+
+  it 'should allow field filtration at Each' do
+    test_assembly do
+      check_scope :values_fields => ['offset', 'line']
+      assert_size_equals 2
+
+      split 'line', ['x', 'y'], :pattern => /,/, :output => ['x', 'y']
+      check_scope :values_fields => ['x', 'y']
+      assert_size_equals 2
+    end
+  end
+
+  it 'should propagate names through CoGroup' do
+    test_join_assembly do
+    end
+  end
+
+  it 'should pass grouping fields to Every' do
+    test_join_assembly do
+      sum :mapping => {'x' => 'x_sum'}, :type => :int
+      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
+        :grouping_fields => ['x', 'x_sum']
+      assert_group_size_equals 1
+    end
+  end
+
+  it 'should pass grouping fields through chained Every' do
+    test_join_assembly do
+      sum :mapping => {'x' => 'x_sum'}, :type => :int
+      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
+        :grouping_fields => ['x', 'x_sum']
+      assert_group_size_equals 1
+
+      sum :mapping => {'y' => 'y_sum'}, :type => :int
+      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
+        :grouping_fields => ['x', 'x_sum', 'y_sum']
+      assert_group_size_equals 1
+    end
+  end
+
+  it 'should propagate names through Every' do
+    test_join_assembly do
+      sum :mapping => {'x' => 'x_sum'}, :type => :int
+      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
+        :grouping_fields => ['x', 'x_sum']
+      assert_group_size_equals 1
+
+      sum :mapping => {'y' => 'y_sum'}, :type => :int
+      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
+        :grouping_fields => ['x', 'x_sum', 'y_sum']
+      assert_group_size_equals 1
+
+      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
+        :grouping_fields => ['x', 'x_sum', 'y_sum']
+      assert_size_equals 3
+
+      # No rename service provided unless you use the block form of join!
+      check_scope :values_fields => [0, 'x_sum', 'y_sum']
+
+      # Mimic rename service
+      bind_names ['x', 'x_sum', 'y_sum']
+      check_scope :values_fields => ['x', 'x_sum', 'y_sum']
+    end
+  end
+
+  it 'should pass values fields to Each immediately following CoGroup and remove grouping fields' do
+    test_join_assembly do
+      assert_size_equals 10
+      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_']
+    end
+  end
+
+  it 'should fail to pass grouping fields to Every immediately following Each' do
+    lambda do # Composition fails
+      test_join_assembly do
+        pass
+        check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_']
+        begin
+          sum :mapping => {'x' => 'x_sum'}, :type => :int
+        rescue CascadingException => e
+          raise e.cause(3)
+        end
+      end
+    end.should raise_error java.lang.IllegalStateException, 'Every cannot follow a Tap or an Each'
+  end
+
+  it 'should propagate values fields and field names into branch' do
+    test_join_assembly(:branches => ['data_tuple']) do
+      branch 'data_tuple' do
+        check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
+          :grouping_fields => ['x']
+        assert_size_equals 10
+      end
+    end
+  end
+
+  it 'should fail to propagate grouping fields to branch' do
+    lambda do # Execution fails
+      begin
+        test_join_assembly(:branches => ['attempt_group']) do
+          branch 'attempt_group' do
+            check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
+              :grouping_fields => ['x']
+            sum :mapping => {'x' => 'x_sum'}, :type => :int
+          end
+        end
+      rescue CascadingException => e
+        raise e.cause(4)
+      end
+    end.should raise_error java.lang.IllegalStateException, 'Every cannot follow a Tap or an Each'
+  end
+
+  it 'should propagate names through GroupBy' do
+    test_assembly do
+      group_by 'line'
+      check_scope :values_fields => ['offset', 'line'],
+        :grouping_fields => ['line']
+    end
+  end
+end

data/spec/spec.opts

@@ -0,0 +1,6 @@
+--colour
+--format
+progress
+--loadby
+mtime
+--reverse

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+require 'spec'
+require 'rubygems'
+require 'cascading'
+
+require File.expand_path(File.dirname(__FILE__) + '/spec_util')