cascading.jruby 0.0.7 → 0.0.8

data/test/test_exceptions.rb

@@ -9,16 +9,16 @@ class TC_Exceptions < Test::Unit::TestCase
     e = CascadingException.new(ne1, 'cascading.jruby wrapper exception')
 
     assert_equal ne1, e.ne
-    assert_equal 'cascading.jruby wrapper exception', e.message
+    assert_match /^cascading\.jruby wrapper exception/, e.message
+    assert_match /^Exception summary for: cascading\.jruby wrapper exception/, e.message
     assert_equal 3, e.depth
 
-
     assert_equal ne1, e.cause(1)
     assert_equal 'Exception Cascading hands us', e.cause(1).message
 
     assert_equal ne2, e.cause(2)
     # Cascading inserts Operator#to_s, here
-    assert_match /Exception thrown by Cascading/, e.cause(2).message
+    assert_match /Exception thrown by Cascading$/, e.cause(2).message
 
     assert_equal ne3, e.cause(3)
     assert_equal 'Root cause', e.cause(3).message

data/test/test_local_execution.rb

@@ -0,0 +1,168 @@
+require 'test/unit'
+require 'cascading'
+
+class TC_LocalExecution < Test::Unit::TestCase
+  def test_smoke_test_multi_source_tap
+    cascade 'splitter' do
+      flow 'splitter' do
+        tap1 = tap 'test/data/data1.txt'
+        tap2 = tap 'test/data/data2.txt'
+        source 'data', multi_source_tap(tap1, tap2)
+
+        assembly 'data' do
+          pass
+        end
+
+        sink 'data', tap('output/test_smoke_test_multi_source_tap')
+      end
+    end.complete
+  end
+
+  def test_smoke_test_sequence_file_scheme
+    cascade 'smoke' do
+      flow 'smoke' do
+        source 'input', tap('test/data/data1.txt')
+        assembly 'input' do
+          pass
+        end
+        compress_output :default, :block
+        sink 'input', tap('output/test_smoke_test_sequence_file_scheme', :scheme => sequence_file_scheme)
+      end
+    end.complete
+  end
+
+  def test_splitter
+    flow = flow 'splitter' do
+      source 'copy', tap('test/data/data1.txt')
+
+      assembly 'copy' do
+        split 'line', :pattern => /[.,]*\s+/, :into=>['name', 'score1', 'score2', 'id'], :output => ['name', 'score1', 'score2', 'id']
+        assert_size_equals 4
+        assert_not_null
+        debug :print_fields => true
+      end
+
+      sink 'copy', tap('output/test_splitter', :sink_mode => :replace)
+    end.complete
+  end
+
+  def test_smoke_test_multi_source_tap
+    cascade 'multi_source_tap' do
+      flow 'multi_source_tap' do
+        tap1 = tap 'test/data/data1.txt'
+        tap2 = tap 'test/data/data2.txt'
+        source 'data', multi_source_tap(tap1, tap2)
+
+        assembly 'data' do
+          pass
+        end
+
+        sink 'data', tap('output/test_smoke_test_multi_source_tap')
+      end
+    end.complete
+  end
+
+  def test_join1
+    join_grouping_fields, join_values_fields = nil, nil
+    cascade 'splitter' do
+      flow 'splitter' do
+        source 'data1', tap('test/data/data1.txt')
+        source 'data2', tap('test/data/data2.txt')
+
+        assembly1 = assembly 'data1' do
+          split 'line', :pattern => /[.,]*\s+/, :into => ['name', 'score1', 'score2', 'id'], :output => ['name', 'score1', 'score2', 'id']
+          assert_size_equals 4
+          assert_not_null
+          debug :print_fields => true
+        end
+
+        assembly2 = assembly 'data2' do
+          split 'line', :pattern => /[.,]*\s+/, :into => ['name',  'id', 'town'], :output => ['name',  'id', 'town']
+          assert_size_equals 3
+          assert_not_null
+          debug :print_fields => true
+        end
+
+        assembly 'joined' do
+          join assembly1.name, assembly2.name, :on => ['name', 'id'], :declared_fields => ['name', 'score1', 'score2', 'id', 'name2', 'id2', 'town']
+          join_grouping_fields = scope.grouping_fields.to_a
+          join_values_fields = scope.values_fields.to_a
+
+          assert_size_equals 7
+          assert_not_null
+        end
+
+        sink 'joined', tap('output/test_join1', :sink_mode => :replace)
+      end
+    end.complete
+    assert_equal ['name', 'id', 'name_', 'id_'], join_grouping_fields
+    assert_equal ['name', 'score1', 'score2', 'id', 'name2', 'id2', 'town'], join_values_fields
+  end
+
+  def test_join2
+    join_grouping_fields, join_values_fields = nil, nil
+    flow = flow 'splitter' do
+      source 'data1', tap('test/data/data1.txt')
+      source 'data2', tap('test/data/data2.txt')
+
+      assembly 'data1' do
+        split 'line', :pattern => /[.,]*\s+/, :into => ['name', 'score1', 'score2', 'id'], :output => ['name', 'score1', 'score2', 'id']
+        debug :print_fields => true
+      end
+
+      assembly 'data2' do
+        split 'line', :pattern => /[.,]*\s+/, :into => ['name',  'code', 'town'], :output => ['name',  'code', 'town']
+        debug :print_fields => true
+      end
+
+      assembly 'joined' do
+        join :on => {'data1' => ['name', 'id'], 'data2' => ['name', 'code']}, :declared_fields => ['name', 'score1', 'score2', 'id', 'name2', 'code', 'town']
+        join_grouping_fields = scope.grouping_fields.to_a
+        join_values_fields = scope.values_fields.to_a
+      end
+
+      sink 'joined', tap('output/test_join2', :sink_mode => :replace)
+     end.complete
+     assert_equal ['name', 'id', 'name_', 'code'], join_grouping_fields
+     assert_equal ['name', 'score1', 'score2', 'id', 'name2', 'code', 'town'], join_values_fields
+   end
+
+  def test_union
+    union_grouping_fields, union_values_fields = nil, nil
+    cascade 'union' do
+      flow 'union' do
+        source 'data1', tap('test/data/data1.txt')
+        source 'data2', tap('test/data/data2.txt')
+
+        assembly 'data1' do
+          split 'line', :pattern => /[.,]*\s+/, :into => ['name', 'score1', 'score2', 'id'], :output => ['name', 'score1', 'score2', 'id']
+          assert_size_equals 4
+          assert_not_null
+
+          project 'name', 'id'
+          assert_size_equals 2
+        end
+
+        assembly 'data2' do
+          split 'line', :pattern => /[.,]*\s+/, :into => ['name',  'code', 'town'], :output => ['name',  'code', 'town']
+          assert_size_equals 3
+          assert_not_null
+
+          rename 'code' => 'id'
+          project 'name', 'id'
+          assert_size_equals 2
+        end
+
+        assembly 'union' do
+          union 'data1', 'data2'
+          union_grouping_fields = scope.grouping_fields.to_a
+          union_values_fields = scope.values_fields.to_a
+        end
+
+        sink 'union', tap('output/test_union', :sink_mode => :replace)
+     end
+    end.complete
+    assert_equal ['name'], union_grouping_fields
+    assert_equal ['name', 'id'], union_values_fields
+  end
+end

data/test/test_operations.rb

@@ -13,11 +13,4 @@ class TC_Operations < Test::Unit::TestCase
     first = first_function 'first_field', :ignore => [Java::CascadingTuple::Tuple.new(-1)].to_java(Java::CascadingTuple::Tuple)
     assert_not_nil first
   end
-
-  def test_aggregator_function_ignore_exception
-    assert_raise RuntimeError do
-      avg = average_function 'avg_field', :ignore => [nil].to_java(:string)
-      assert_not_nil avg
-    end
-  end
 end

metadata

@@ -2,7 +2,7 @@
 name: cascading.jruby
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.0.7
+  version: 0.0.8
 platform: ruby
 authors: 
   - Matt Walker
@@ -11,7 +11,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-04-02 00:00:00 Z
+date: 2012-05-15 00:00:00 Z
 dependencies: []
 
 description: cascading.jruby is a small DSL above Cascading, written in JRuby
@@ -25,6 +25,7 @@ extra_rdoc_files:
   - LICENSE.txt
   - bin/make_job
   - samples/data/data2.txt
+  - samples/data/data_group_by.txt
   - samples/data/data_join1.txt
   - samples/data/data_join2.txt
   - samples/data/data_join3.txt
@@ -33,6 +34,9 @@ extra_rdoc_files:
   - test/data/data1.txt
   - test/data/data2.txt
 files: 
+  - .travis.yml
+  - Gemfile
+  - Gemfile.lock
   - HACKING.md
   - History.txt
   - LICENSE.txt
@@ -40,7 +44,10 @@ files:
   - Rakefile
   - TODO
   - bin/make_job
+  - ivy.xml
+  - ivysettings.xml
   - lib/cascading.rb
+  - lib/cascading/aggregations.rb
   - lib/cascading/assembly.rb
   - lib/cascading/base.rb
   - lib/cascading/cascade.rb
@@ -51,18 +58,24 @@ files:
   - lib/cascading/flow.rb
   - lib/cascading/operations.rb
   - lib/cascading/scope.rb
+  - lib/cascading/sub_assembly.rb
   - samples/branch.rb
   - samples/copy.rb
   - samples/data/data2.txt
+  - samples/data/data_group_by.txt
   - samples/data/data_join1.txt
   - samples/data/data_join2.txt
   - samples/data/data_join3.txt
+  - samples/data/genealogy/names/dist.all.last
+  - samples/data/gutenberg/the_outline_of_science_vol_1
+  - samples/group_by.rb
   - samples/join.rb
   - samples/logwordcount.rb
   - samples/project.rb
   - samples/rename.rb
   - samples/scorenames.rb
   - samples/splitter.rb
+  - samples/sub_assembly.rb
   - samples/union.rb
   - spec/cascading_spec.rb
   - spec/expr_spec.rb
@@ -92,11 +105,14 @@ files:
   - tasks/test.rake
   - test/data/data1.txt
   - test/data/data2.txt
+  - test/mock_assemblies.rb
+  - test/test_aggregations.rb
   - test/test_assembly.rb
   - test/test_cascade.rb
   - test/test_cascading.rb
   - test/test_exceptions.rb
   - test/test_flow.rb
+  - test/test_local_execution.rb
   - test/test_operations.rb
 homepage: http://github.com/etsy/cascading.jruby
 licenses: []
@@ -112,6 +128,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        hash: 2
+        segments: 
+          - 0
         version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false
@@ -127,9 +146,11 @@ signing_key:
 specification_version: 3
 summary: A JRuby DSL for Cascading
 test_files: 
+  - test/test_aggregations.rb
   - test/test_assembly.rb
   - test/test_cascade.rb
   - test/test_cascading.rb
   - test/test_exceptions.rb
   - test/test_flow.rb
+  - test/test_local_execution.rb
   - test/test_operations.rb