cascading.jruby 0.0.8 → 0.0.9

data/samples/branch.rb

@@ -4,7 +4,7 @@ $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'cascading'
 
-cascade 'branch' do
+cascade 'branch', :mode => :local do
   flow 'branch' do
     source 'input', tap('samples/data/data2.txt')
 
@@ -27,4 +27,4 @@ cascade 'branch' do
     sink 'branch1', tap('output/branch1', :sink_mode => :replace)
     sink 'branch2', tap('output/branch2', :sink_mode => :replace)
   end
-end.complete(local_properties('build/sample'))
+end.complete

data/samples/copy.rb

@@ -3,7 +3,7 @@ $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'cascading'
 
-cascade 'copy' do
+cascade 'copy', :mode => :local do
   flow 'copy' do
     # You don't have to curl and cache inputs: tap can fetch via HTTP
     #source 'input', tap('http://www.census.gov/genealogy/names/dist.all.last')
@@ -17,4 +17,4 @@ cascade 'copy' do
 
     sink 'input', tap('output/copy', :sink_mode => :replace)
   end
-end.complete(local_properties('build/sample'))
+end.complete

data/samples/group_by.rb

@@ -3,7 +3,7 @@ $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'cascading'
 
-cascade 'group_by' do
+cascade 'group_by', :mode => :local do
   flow 'group_by' do
     source 'input', tap('samples/data/data_group_by.txt')
 
@@ -53,9 +53,9 @@ cascade 'group_by' do
     sink 'empty_group_by', tap('output/empty_group_by', :sink_mode => :replace)
     sink 'blockless_group_by', tap('output/blockless_group_by', :sink_mode => :replace)
     sink 'aggregate_by', tap('output/aggregate_by', :sink_mode => :replace)
-    sink 'empty_aggregate_by', tap('output/empty_aggregate_by', :sink_mode => :replace)
-    sink 'blockless_aggregate_by', tap('output/blockless_aggregate_by', :sink_mode => :replace)
+    #sink 'empty_aggregate_by', tap('output/empty_aggregate_by', :sink_mode => :replace)
+    #sink 'blockless_aggregate_by', tap('output/blockless_aggregate_by', :sink_mode => :replace)
     sink 'unique', tap('output/unique', :sink_mode => :replace)
   end
-#end.draw(ARGV[0], local_properties('build/sample'))
-end.complete(local_properties('build/sample'))
+#end.draw(ARGV[0])
+end.complete

data/samples/join.rb

@@ -3,7 +3,7 @@ $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'cascading'
 
-cascade 'join' do
+cascade 'join', :mode => :local do
   flow 'join' do
     source 'input1', tap('samples/data/data_join1.txt')
     source 'input2', tap('samples/data/data_join2.txt')
@@ -28,4 +28,4 @@ cascade 'join' do
 
     sink 'join', tap('output/join', :sink_mode => :replace)
   end
-end.complete(local_properties('build/sample'))
+end.complete

data/samples/logwordcount.rb

@@ -3,15 +3,14 @@ $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'cascading'
 
-cascade 'logwordcount' do
+cascade 'logwordcount', :mode => :local do
   flow 'logwordcount' do
     # This works just as well, but will get you blocked by Project Gutenberg
     #source 'input', tap('http://www.gutenberg.org/files/20417/20417-8.txt')
     source 'input', tap('samples/data/gutenberg/the_outline_of_science_vol_1')
 
     assembly 'input' do
-      # TODO: create a helper for RegexSplitGenerator
-      each 'line', :function => regex_split_generator('word', :pattern => /[.,]*\s+/)
+      split_rows 'line', 'word', :pattern => /[.,]*\s+/, :output => 'word'
       group_by 'word' do
         count
       end
@@ -20,4 +19,4 @@ cascade 'logwordcount' do
 
     sink 'input', tap('output/logwordcount', :sink_mode => :replace)
   end
-end.complete(local_properties('build/sample'))
+end.complete

data/samples/project.rb

@@ -5,7 +5,7 @@ $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'cascading'
 
-cascade 'project' do
+cascade 'project', :mode => :local do
   flow 'project' do
     source 'input', tap('samples/data/data2.txt')
 
@@ -20,4 +20,4 @@ cascade 'project' do
 
     sink 'input', tap('output/project', :sink_mode => :replace)
   end
-end.complete(local_properties('build/sample'))
+end.complete

data/samples/rename.rb

@@ -3,7 +3,7 @@ $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'cascading'
 
-cascade 'rename' do
+cascade 'rename', :mode => :local do
   flow 'rename' do
     source 'input', tap('samples/data/data2.txt')
 
@@ -17,4 +17,4 @@ cascade 'rename' do
 
     sink 'input', tap('output/rename', :sink_mode => :replace)
   end
-end.complete(local_properties('build/sample'))
+end.complete

data/samples/scorenames.rb

@@ -3,7 +3,7 @@ $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'cascading'
 
-cascade 'scorenames' do
+cascade 'scorenames', :mode => :local do
   flow 'scorenames' do
     # You don't have to curl and cache inputs: tap can fetch via HTTP
     #source 'input', tap('http://www.census.gov/genealogy/names/dist.all.last')
@@ -17,4 +17,4 @@ cascade 'scorenames' do
 
     sink 'input', tap('output/scorenames', :sink_mode => :replace)
   end
-end.complete(local_properties('build/sample'))
+end.complete

data/samples/splitter.rb

@@ -3,7 +3,7 @@ $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'cascading'
 
-cascade 'splitter' do
+cascade 'splitter', :mode => :local do
   flow 'splitter' do
     source 'input', tap('samples/data/data2.txt')
 
@@ -16,4 +16,4 @@ cascade 'splitter' do
 
     sink 'input', tap('output/splitter', :sink_mode => :replace)
   end
-end.complete(local_properties('build/sample'))
+end.complete

data/samples/sub_assembly.rb

@@ -3,7 +3,7 @@ $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'cascading'
 
-cascade 'sub_assembly' do
+cascade 'sub_assembly', :mode => :local do
   flow 'sub_assembly' do
     source 'input', tap('samples/data/data2.txt')
 
@@ -27,4 +27,4 @@ cascade 'sub_assembly' do
 
     sink 'input', tap('output/sub_assembly', :sink_mode => :replace)
   end
-end.complete(local_properties('build/sample'))
+end.complete

data/samples/union.rb

@@ -3,7 +3,7 @@ $: << File.join(File.dirname(__FILE__), '..', 'lib')
 
 require 'cascading'
 
-cascade 'union' do
+cascade 'union', :mode => :local do
   flow 'union' do
     # You don't have to curl and cache inputs: tap can fetch via HTTP
     #source 'input', tap('http://www.census.gov/genealogy/names/dist.all.last')
@@ -33,4 +33,4 @@ cascade 'union' do
 
     sink 'union', tap('output/union', :sink_mode => :replace)
   end
-end.complete(local_properties('build/sample'))
+end.complete

data/spec/cascading_spec.rb

@@ -33,56 +33,56 @@ context Cascading do
   end
 
   it 'should find branches to sink' do
-    cascade 'branched_pass' do
+    cascade 'branched_pass', :mode => :local do
       flow 'branched_pass' do
-        source 'input', tap('spec/resource/test_input.txt', :kind => :lfs, :scheme => text_line_scheme)
+        source 'input', tap('spec/resource/test_input.txt', :scheme => text_line_scheme)
         assembly 'input' do
           branch 'branched_input' do
             project 'line'
           end
         end
-        sink 'branched_input', tap("#{OUTPUT_DIR}/branched_pass_out", :kind => :lfs, :sink_mode => :replace)
+        sink 'branched_input', tap("#{OUTPUT_DIR}/branched_pass_out", :sink_mode => :replace)
       end
     end.complete
 
-    ilc = `wc -l spec/resource/test_input.txt`.split(/\s+/).first
-    olc = `wc -l #{OUTPUT_DIR}/branched_pass_out/part-00000`.split(/\s+/).first
-    ilc.should == olc
+    ilc = `wc -l spec/resource/test_input.txt`.strip.split(/\s+/).first
+    olc = `wc -l #{OUTPUT_DIR}/branched_pass_out`.strip.split(/\s+/).first
+    olc.should == ilc
   end
 
   it 'should create an isolated namespace per cascade' do
-    cascade 'double' do
+    cascade 'double', :mode => :local do
       flow 'double' do
-        source 'input', tap('spec/resource/test_input.txt', :kind => :lfs, :scheme => text_line_scheme)
+        source 'input', tap('spec/resource/test_input.txt', :scheme => text_line_scheme)
         assembly 'input' do # Dup name
           insert 'doubled' => expr('line:string + "," + line:string')
           project 'doubled'
         end
-        sink 'input', tap("#{OUTPUT_DIR}/double_out", :kind => :lfs, :sink_mode => :replace)
+        sink 'input', tap("#{OUTPUT_DIR}/double_out", :sink_mode => :replace)
       end
     end
 
-    cascade 'pass' do
+    cascade 'pass', :mode => :local do
       flow 'pass' do
-        source 'input', tap('spec/resource/test_input.txt', :kind => :lfs, :scheme => text_line_scheme)
+        source 'input', tap('spec/resource/test_input.txt', :scheme => text_line_scheme)
         assembly 'input' do # Dup name
           project 'line'
         end
-        sink 'input', tap("#{OUTPUT_DIR}/pass_out", :kind => :lfs, :sink_mode => :replace)
+        sink 'input', tap("#{OUTPUT_DIR}/pass_out", :sink_mode => :replace)
       end
     end
 
     Cascade.get('double').complete
     Cascade.get('pass').complete
-    diff = `diff #{OUTPUT_DIR}/double_out/part-00000 #{OUTPUT_DIR}/pass_out/part-00000`
+    diff = `diff #{OUTPUT_DIR}/double_out #{OUTPUT_DIR}/pass_out`
     diff.should_not be_empty
   end
 
   it 'should support joins in branches' do
-    cascade 'branch_join' do
+    cascade 'branch_join', :mode => :local do
       flow 'branch_join' do
-        source 'left', tap('spec/resource/join_input.txt', :kind => :lfs, :scheme => text_line_scheme)
-        source 'right', tap('spec/resource/join_input.txt', :kind => :lfs, :scheme => text_line_scheme)
+        source 'left', tap('spec/resource/join_input.txt', :scheme => text_line_scheme)
+        source 'right', tap('spec/resource/join_input.txt', :scheme => text_line_scheme)
 
         assembly 'left' do
           split 'line', ['x', 'y', 'z'], :pattern => /,/
@@ -98,7 +98,7 @@ context Cascading do
           end
         end
 
-        sink 'branch_join', tap("#{OUTPUT_DIR}/branch_join_out.txt", :kind => :lfs, :sink_mode => :replace)
+        sink 'branch_join', tap("#{OUTPUT_DIR}/branch_join_out.txt", :sink_mode => :replace)
       end
     end.complete
   end

data/spec/spec_util.rb

@@ -23,28 +23,28 @@ module Cascading
 end
 
 def test_flow(&block)
-  cascade = cascade 'test_app' do
+  cascade = cascade 'test_app', :mode => :local do
     flow 'test', &block
   end
-  cascade.complete(local_properties(BUILD_DIR))
+  cascade.complete
 end
 
 def test_assembly(params = {}, &block)
   branches = params[:branches] || []
 
   test_flow do
-    source 'input', tap('spec/resource/test_input.txt', :kind => :lfs, :scheme => text_line_scheme)
+    source 'input', tap('spec/resource/test_input.txt', :scheme => text_line_scheme)
 
     # Default Fields defined by TextLineScheme
     check_scope :source => 'input', :values_fields => ['offset', 'line']
 
     assembly 'input', &block
 
-    sink 'input', tap("#{OUTPUT_DIR}/out.txt", :kind => :lfs, :sink_mode => :replace)
+    sink 'input', tap("#{OUTPUT_DIR}/out.txt", :sink_mode => :replace)
 
     # Branches must be sunk so that they (and their assertions) will be run
     branches.each do |branch|
-      sink branch, tap("#{OUTPUT_DIR}/#{branch}_out.txt", :kind => :lfs, :sink_mode => :replace)
+      sink branch, tap("#{OUTPUT_DIR}/#{branch}_out.txt", :sink_mode => :replace)
     end
   end
 end
@@ -54,8 +54,8 @@ def test_join_assembly(params = {}, &block)
   post_join_block = params[:post_join_block]
 
   test_flow do
-    source 'left', tap('spec/resource/join_input.txt', :kind => :lfs, :scheme => text_line_scheme)
-    source 'right', tap('spec/resource/join_input.txt', :kind => :lfs, :scheme => text_line_scheme)
+    source 'left', tap('spec/resource/join_input.txt', :scheme => text_line_scheme)
+    source 'right', tap('spec/resource/join_input.txt', :scheme => text_line_scheme)
 
     # Default Fields defined by TextLineScheme
     check_scope :source => 'left', :values_fields => ['offset', 'line']
@@ -82,11 +82,11 @@ def test_join_assembly(params = {}, &block)
       instance_eval &post_join_block if post_join_block
     end
 
-    sink 'join', tap("#{OUTPUT_DIR}/join_out.txt", :kind => :lfs, :sink_mode => :replace)
+    sink 'join', tap("#{OUTPUT_DIR}/join_out.txt", :sink_mode => :replace)
 
     # Branches must be sunk so that they (and their assertions) will be run
     branches.each do |branch|
-      sink branch, tap("#{OUTPUT_DIR}/#{branch}_out.txt", :kind => :lfs, :sink_mode => :replace)
+      sink branch, tap("#{OUTPUT_DIR}/#{branch}_out.txt", :sink_mode => :replace)
     end
   end
 end