cascading.jruby 0.0.9 → 0.0.10

data/test/test_assembly.rb

@@ -62,7 +62,7 @@ class TC_Assembly < Test::Unit::TestCase
         count
       end
       pipe = assembly.tail_pipe
-      assert Java::CascadingPipe::Every, pipe.class
+      assert_equal Java::CascadingPipe::Every, pipe.class
     end
     assert_match /^undefined local variable or method `count' for #<Cascading::Assembly:.*>$/, ex.message
   end
@@ -74,7 +74,7 @@ class TC_Assembly < Test::Unit::TestCase
           every 'line', :aggregator => count_aggregator, :output => 'count'
         end
       end
-      assert Java::CascadingPipe::Every, assembly.tail_pipe.class
+      assert_equal Java::CascadingPipe::Every, assembly.tail_pipe.class
       assert_equal ['line'], assembly.tail_pipe.argument_selector.to_a
       assert_equal ['count'], assembly.tail_pipe.output_selector.to_a
 
@@ -83,7 +83,7 @@ class TC_Assembly < Test::Unit::TestCase
           count
         end
       end
-      assert Java::CascadingPipe::Every, assembly.tail_pipe.class
+      assert_equal Java::CascadingPipe::Every, assembly.tail_pipe.class
 
       # NOTE: this is not valid when we optimize using CountBy
       #assert_equal last_grouping_fields, assembly.tail_pipe.argument_selector
@@ -370,8 +370,10 @@ class TC_Assembly < Test::Unit::TestCase
     end
 
     assert_equal Java::CascadingPipe::CoGroup, assembly.tail_pipe.class
+
     left_grouping_fields = assembly.tail_pipe.key_selectors['test1']
     assert_equal ['name', 'id'], left_grouping_fields.to_a
+
     right_grouping_fields = assembly.tail_pipe.key_selectors['test2']
     assert_equal ['name', 'id'], right_grouping_fields.to_a
 
@@ -410,56 +412,142 @@ class TC_Assembly < Test::Unit::TestCase
   end
 
   def test_join_undefined_inputs
-    ex = assert_raise RuntimeError do
-      flow 'test_join_undefined_inputs' do
-        source 'data1', tap('test/data/data1.txt')
+    [:join, :hash_join].each do |join|
+      ex = assert_raise RuntimeError do
+        flow 'test_join_undefined_inputs' do
+          source 'data1', tap('test/data/data1.txt')
 
-        assembly 'data1' do
-          pass
-        end
+          assembly 'data1' do
+            pass
+          end
 
-        assembly 'join' do
-          join 'doesnotexist', 'data1', :on => 'name'
-        end
+          assembly 'join' do
+            send(join, 'doesnotexist', 'data1', :on => 'name')
+          end
 
-        sink 'join', tap('output/test_join_undefined_inputs')
+          sink 'join', tap('output/test_join_undefined_inputs')
+        end
       end
+      assert_equal "Could not find assembly 'doesnotexist' from 'join'", ex.message
     end
-    assert_equal "Could not find assembly 'doesnotexist' from 'join'", ex.message
   end
 
   def test_join_without_on
-    ex = assert_raise RuntimeError do
-      mock_two_input_assembly do
-        join 'test1', 'test2'
+    [:join, :hash_join].each do |join|
+      ex = assert_raise RuntimeError do
+        mock_two_input_assembly do
+          send(join, 'test1', 'test2')
+        end
       end
+      assert_equal 'join requires :on parameter', ex.message
     end
-    assert_equal 'join requires :on parameter', ex.message
   end
 
   def test_join_invalid_on
-    ex = assert_raise RuntimeError do
-      mock_two_input_assembly do
-        join 'test1', 'test2', :on => 1
+    [:join, :hash_join].each do |join|
+      ex = assert_raise RuntimeError do
+        mock_two_input_assembly do
+          send(join, 'test1', 'test2', :on => 1)
+        end
       end
+      assert_equal "Unsupported data type for :on in join: 'Fixnum'", ex.message
     end
-    assert_equal "Unsupported data type for :on in join: 'Fixnum'", ex.message
   end
 
   def test_join_empty_on
-    ex = assert_raise RuntimeError do
-      mock_two_input_assembly do
-        join 'test1', 'test2', :on => []
+    [:join, :hash_join].each do |join|
+      ex = assert_raise RuntimeError do
+        mock_two_input_assembly do
+          send(join, 'test1', 'test2', :on => [])
+        end
+      end
+      assert_equal "join requires non-empty :on parameter", ex.message
+
+      ex = assert_raise RuntimeError do
+        mock_two_input_assembly do
+          send(join, 'test1', 'test2', :on => {})
+        end
       end
+      assert_equal "join requires non-empty :on parameter", ex.message
     end
-    assert_equal "join requires non-empty :on parameter", ex.message
+  end
 
-    ex = assert_raise RuntimeError do
+  def test_create_hash_join
+    assembly = mock_two_input_assembly do
+      hash_join 'test1', 'test2', :on => 'name'
+    end
+
+    assert_equal Java::CascadingPipe::HashJoin, assembly.tail_pipe.class
+
+    left_grouping_fields = assembly.tail_pipe.key_selectors['test1']
+    assert_equal ['name'], left_grouping_fields.to_a
+
+    right_grouping_fields = assembly.tail_pipe.key_selectors['test2']
+    assert_equal ['name'], right_grouping_fields.to_a
+
+    assert_equal ['name', 'score1', 'score2', 'id', 'name_', 'id_', 'town'], assembly.scope.values_fields.to_a
+    # NOTE: Why does HashJoin produce different grouping fields than CoGroup?
+    assert_equal ['name'], assembly.scope.grouping_fields.to_a
+
+    assembly = mock_two_input_assembly do
+      hash_join 'test1', 'test2', :on => 'id'
+    end
+
+    assert_equal Java::CascadingPipe::HashJoin, assembly.tail_pipe.class
+    left_grouping_fields = assembly.tail_pipe.key_selectors['test1']
+    assert_equal ['id'], left_grouping_fields.to_a
+    right_grouping_fields = assembly.tail_pipe.key_selectors['test2']
+    assert_equal ['id'], right_grouping_fields.to_a
+    assert_equal ['name', 'score1', 'score2', 'id', 'name_', 'id_', 'town'], assembly.scope.values_fields.to_a
+    # NOTE: Why does HashJoin produce different grouping fields than CoGroup?
+    assert_equal ['id'], assembly.scope.grouping_fields.to_a
+  end
+
+  def create_hash_join_many_fields
+        assembly = mock_two_input_assembly do
+      hash_join 'test1', 'test2', :on => ['name', 'id']
+    end
+
+    assert_equal Java::CascadingPipe::HashJoin, assembly.tail_pipe.class
+
+    left_grouping_fields = assembly.tail_pipe.key_selectors['test1']
+    assert_equal ['name', 'id'], left_grouping_fields.to_a
+
+    right_grouping_fields = assembly.tail_pipe.key_selectors['test2']
+    assert_equal ['name', 'id'], right_grouping_fields.to_a
+
+    assert_equal ['name', 'score1', 'score2', 'id', 'name_', 'id_', 'town'], assembly.scope.values_fields.to_a
+    # NOTE: Why does HashJoin produce different grouping fields than CoGroup?
+    assert_equal ['name', 'id'], assembly.scope.grouping_fields.to_a
+  end
+
+  def create_hash_join_with_declared_fields
+    assembly = mock_two_input_assembly do
+      hash_join 'test1', 'test2', :on => 'name', :declared_fields => ['a', 'b', 'c', 'd', 'e', 'f', 'g']
+    end
+
+    assert_equal Java::CascadingPipe::HashJoin, assembly.tail_pipe.class
+
+    left_grouping_fields = assembly.tail_pipe.key_selectors['test1']
+    assert_equal ['name'], left_grouping_fields.to_a
+
+    right_grouping_fields = assembly.tail_pipe.key_selectors['test2']
+    assert_equal ['name'], right_grouping_fields.to_a
+
+    assert_equal ['a', 'b', 'c', 'd', 'e', 'f', 'g'], assembly.scope.values_fields.to_a
+    # NOTE: Why does HashJoin produce different grouping fields than CoGroup?
+    assert_equal ['name'], assembly.scope.grouping_fields.to_a
+  end
+
+  def test_hash_join_with_block
+    ex = assert_raise ArgumentError do
       mock_two_input_assembly do
-        join 'test1', 'test2', :on => {}
+        hash_join 'test1', 'test2', :on => 'name' do
+          count
+        end
       end
     end
-    assert_equal "join requires non-empty :on parameter", ex.message
+    assert_equal "hash joins don't support aggregations", ex.message
   end
 
   def test_branch_unique
@@ -610,6 +698,24 @@ class TC_Assembly < Test::Unit::TestCase
     assert_equal Java::CascadingOperationExpression::ExpressionFilter, assembly.tail_pipe.operation.class
   end
 
+  def test_smoke_test_describe
+    cascade 'smoke' do
+      flow 'smoke' do
+        source 'input', tap('test/data/data1.txt')
+        assembly 'input' do
+          puts "Describe at assembly start: '#{describe}'"
+          group_by 'line' do
+            count
+            sum 'offset', :type => :long
+            puts "Describe at group_by end (falls out to top-level Cascading::describe): '#{describe}'"
+          end
+          puts "Describe at assembly end: '#{describe}'"
+        end
+        sink 'input', tap('output/test_smoke_test_debug_scope')
+      end
+    end
+  end
+
   def test_smoke_test_debug_scope
     cascade 'smoke' do
       flow 'smoke' do

data/test/test_cascade.rb

@@ -45,6 +45,67 @@ class TC_Cascade < Test::Unit::TestCase
     assert_equal 'cascade.flow2.assembly4', cascade.last_child.last_child.qualified_name
   end
 
+  def test_default_properties
+    f1, f2 = nil, nil
+    cascade = cascade 'cascade' do
+      f1 = flow 'flow1' do
+      end
+
+      f2 = flow 'flow2' do
+      end
+    end
+
+    assert_equal({}, cascade.properties)
+    assert_equal({}, f1.properties)
+    assert_equal({}, f2.properties)
+  end
+
+  def test_set_properties
+    # Simulate the global properties set by jading
+    $jobconf_properties = {
+      'external_no_overwrite' => 'external_no_overwrite',
+      'external_overwrite' => 'external_overwrite',
+    }
+
+    f1, f2 = nil, nil
+    cascade = cascade 'cascade' do
+      properties['external_overwrite'] = 'overwritten'
+      properties['internal_no_overwrite'] = 'internal_no_overwrite'
+      properties['internal_overwrite'] = 'internal_overwrite'
+
+      f1 = flow 'flow1' do
+        properties['external_overwrite'] = 'overwritten_flow1'
+        properties['internal_overwrite'] = 'overwritten_flow1'
+      end
+
+      f2 = flow 'flow2' do
+        properties['external_overwrite'] = 'overwritten_flow2'
+        properties['internal_overwrite'] = 'overwritten_flow2'
+      end
+    end
+
+    assert_equal({
+      'external_no_overwrite' => 'external_no_overwrite',
+      'external_overwrite' => 'overwritten',
+      'internal_no_overwrite' => 'internal_no_overwrite',
+      'internal_overwrite' => 'internal_overwrite',
+    }, cascade.properties)
+
+    assert_equal({
+      'external_no_overwrite' => 'external_no_overwrite',
+      'external_overwrite' => 'overwritten_flow1',
+      'internal_no_overwrite' => 'internal_no_overwrite',
+      'internal_overwrite' => 'overwritten_flow1',
+    }, f1.properties)
+
+    assert_equal({
+      'external_no_overwrite' => 'external_no_overwrite',
+      'external_overwrite' => 'overwritten_flow2',
+      'internal_no_overwrite' => 'internal_no_overwrite',
+      'internal_overwrite' => 'overwritten_flow2',
+    }, f2.properties)
+  end
+
   def test_ambiguous_flow_names
     ex = assert_raise AmbiguousNodeNameException do
       f1, a1, a2, f2, a3, a4 = [nil] * 6
@@ -99,4 +160,23 @@ class TC_Cascade < Test::Unit::TestCase
     # NOTE: Looking up flows this way is not a very common practice, so this is
     # unlikely to cause issues
   end
+
+  def test_smoke_test_describe
+    cascade 'smoke' do
+      puts "Describe at cascade start: '#{describe}'"
+
+      flow 'smoke' do
+        source 'input', tap('test/data/data1.txt')
+        assembly 'input' do
+          group_by 'line' do
+            count
+            sum 'offset', :type => :long
+          end
+        end
+        sink 'input', tap('output/test_smoke_test_debug_scope')
+      end
+
+      puts "Describe at cascade end: '#{describe}'"
+    end
+  end
 end

data/test/test_flow.rb

@@ -113,4 +113,24 @@ class TC_Flow < Test::Unit::TestCase
     end
     assert_equal "Ambiguous lookup of child by name 'b'; found 'flow.b', 'flow.a.b'", ex.message
   end
+
+  def test_smoke_test_describe
+    cascade 'smoke' do
+      flow 'smoke' do
+        puts "Describe at flow start: '#{describe}'"
+        source 'input', tap('test/data/data1.txt')
+
+        puts "Describe before first assembly: '#{describe}'"
+        assembly 'input' do
+          group_by 'line' do
+            count
+            sum 'offset', :type => :long
+          end
+        end
+
+        sink 'input', tap('output/test_smoke_test_debug_scope')
+        puts "Describe at flow end: '#{describe}'"
+      end
+    end
+  end
 end

data/test/test_operations.rb

@@ -62,11 +62,12 @@ class TC_Operations < Test::Unit::TestCase
   end
 
   def test_coerce_to_java_other
-    result = coerce_to_java([1,2,3])
+    orig_val = [1,2,3]
+    result = coerce_to_java(orig_val)
 
     assert_equal Java::JavaLang::String, result.class
     assert_equal ''.to_java(java.lang.String).java_class, result.java_class
-    assert_equal '123'.to_java(java.lang.String), result
+    assert_equal orig_val.to_s.to_java(java.lang.String), result
   end
 
   def test_to_java_comparable_array

metadata

@@ -2,7 +2,7 @@
 name: cascading.jruby
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.0.9
+  version: 0.0.10
 platform: ruby
 authors: 
   - Matt Walker
@@ -11,41 +11,19 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-08-05 00:00:00 Z
+date: 2012-08-09 00:00:00 Z
 dependencies: []
 
 description: cascading.jruby is a small DSL above Cascading, written in JRuby
 email: mwalker@etsy.com
-executables: 
-  - make_job
+executables: []
+
 extensions: []
 
 extra_rdoc_files: 
   - History.txt
   - LICENSE.txt
-  - bin/make_job
-  - samples/data/data2.txt
-  - samples/data/data_group_by.txt
-  - samples/data/data_join1.txt
-  - samples/data/data_join2.txt
-  - samples/data/data_join3.txt
-  - spec/resource/join_input.txt
-  - spec/resource/test_input.txt
-  - test/data/data1.txt
-  - test/data/data2.txt
 files: 
-  - .travis.yml
-  - Gemfile
-  - Gemfile.lock
-  - HACKING.md
-  - History.txt
-  - LICENSE.txt
-  - README.md
-  - Rakefile
-  - TODO
-  - bin/make_job
-  - ivy.xml
-  - ivysettings.xml
   - lib/cascading.rb
   - lib/cascading/aggregations.rb
   - lib/cascading/assembly.rb
@@ -61,53 +39,8 @@ files:
   - lib/cascading/scope.rb
   - lib/cascading/sub_assembly.rb
   - lib/cascading/tap.rb
-  - samples/branch.rb
-  - samples/copy.rb
-  - samples/data/data2.txt
-  - samples/data/data_group_by.txt
-  - samples/data/data_join1.txt
-  - samples/data/data_join2.txt
-  - samples/data/data_join3.txt
-  - samples/data/genealogy/names/dist.all.last
-  - samples/data/gutenberg/the_outline_of_science_vol_1
-  - samples/group_by.rb
-  - samples/join.rb
-  - samples/logwordcount.rb
-  - samples/project.rb
-  - samples/rename.rb
-  - samples/scorenames.rb
-  - samples/splitter.rb
-  - samples/sub_assembly.rb
-  - samples/union.rb
-  - spec/cascading_spec.rb
-  - spec/expr_spec.rb
-  - spec/jruby_version_spec.rb
-  - spec/resource/join_input.txt
-  - spec/resource/test_input.txt
-  - spec/scope_spec.rb
-  - spec/spec.opts
-  - spec/spec_helper.rb
-  - spec/spec_util.rb
-  - src/cascading/jruby/Main.java
-  - src/cascading/jruby/runner.rb
-  - tags
-  - tasks/ann.rake
-  - tasks/ant.rake
-  - tasks/bones.rake
-  - tasks/gem.rake
-  - tasks/git.rake
-  - tasks/notes.rake
-  - tasks/post_load.rake
-  - tasks/rdoc.rake
-  - tasks/rubyforge.rake
-  - tasks/samples.rake
-  - tasks/setup.rb
-  - tasks/spec.rake
-  - tasks/svn.rake
-  - tasks/test.rake
-  - test/data/data1.txt
-  - test/data/data2.txt
-  - test/mock_assemblies.rb
+  - History.txt
+  - LICENSE.txt
   - test/test_aggregations.rb
   - test/test_assembly.rb
   - test/test_cascade.rb
@@ -130,9 +63,6 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
-        hash: 2
-        segments: 
-          - 0
         version: "0"
 required_rubygems_version: !ruby/object:Gem::Requirement 
   none: false

data/.travis.yml

@@ -1,6 +0,0 @@
-language: ruby
-rvm:
-  - jruby-18mode
-notifications:
-  recipients:
-    - mwalker@etsy.com

data/Gemfile

@@ -1,6 +0,0 @@
-source :rubygems
-
-group :test do
-  gem 'rake', '0.8.7'
-  gem 'rspec', '1.1.11'
-end

data/Gemfile.lock

@@ -1,12 +0,0 @@
-GEM
-  remote: http://rubygems.org/
-  specs:
-    rake (0.8.7)
-    rspec (1.1.11)
-
-PLATFORMS
-  java
-
-DEPENDENCIES
-  rake (= 0.8.7)
-  rspec (= 1.1.11)

data/HACKING.md

@@ -1,23 +0,0 @@
-# Hacking
-
-Some hacking info on `cascading.jruby`:
-
-For local development, install with (requires [bundler](http://gembundler.com/)):
-
-    jruby -S bundle install
-
-To run the tests (will download Cascading and Hadoop jars):
-
-    jruby -S bundle exec rake
-
-To create the gem:
-
-    jruby -S bundle exec rake gem
-
-To install it locally:
-
-    jruby -S gem install pkg/cascading.jruby-xxx.gem
-
-The `Cascading::Operations` module is mixed-in the `Cascading::Assembly` class to provide some shortcuts for common operations.
-
-The file cascading/cascading.rb defines global helper methods for cascading like tap creation, fields creation, etc. 

data/README.md

@@ -1,9 +0,0 @@
-# Cascading.JRuby [![Build Status](https://secure.travis-ci.org/mrwalker/cascading.jruby.png)](http://travis-ci.org/mrwalker/cascading.jruby)
-
-`cascading.jruby` is a small DSL above [Cascading](http://www.cascading.org/).
-
-It requires Hadoop (>= 0.20.2) and [Cascading 2.0.0](http://files.cascading.org/cascading/2.0/cascading-2.0.0.tgz) to be set via the environment variables: `HADOOP_HOME` and `CASCADING_HOME`
-
-It has been tested on JRuby versions 1.2.0, 1.4.0, 1.5.3, and 1.6.5.
-
-Copyright 2009, Grégoire Marabout.

data/Rakefile

@@ -1,46 +0,0 @@
-#! /usr/bin/env jruby
-
-# Look in the tasks/setup.rb file for the various options that can be
-# configured in this Rakefile. The .rake files in the tasks directory
-# are where the options are used.
-
-begin
-  require 'bones'
-  Bones.setup
-rescue LoadError
-  begin
-    load 'tasks/setup.rb'
-  rescue LoadError
-    raise RuntimeError, '### please install the "bones" gem ###'
-  end
-end
-
-ensure_in_path 'lib'
-
-require 'cascading'
-require 'bundler/setup'
-
-task :default => 'test'
-
-task :run do
-  # ensure_in_path "samples"
-  puts "Running #{ARGS[0]}"
-  require "samples/#{ARGS[0]}"
-end
-
-desc 'Remove gem and Java build files'
-task :clean => ['ant:clean', 'gem:clean', 'samples:clean'] do
-  puts 'Build files removed'
-end
-
-PROJ.name = 'cascading.jruby'
-PROJ.authors = ['Matt Walker', 'Grégoire Marabout']
-PROJ.email = 'mwalker@etsy.com'
-PROJ.url = 'http://github.com/etsy/cascading.jruby'
-PROJ.version = Cascading::VERSION
-PROJ.summary = 'A JRuby DSL for Cascading'
-PROJ.description = 'cascading.jruby is a small DSL above Cascading, written in JRuby'
-PROJ.rubyforge.name = 'cascading.jruby'
-PROJ.spec.opts << '--color'
-
-# EOF

data/TODO

@@ -1,13 +0,0 @@
-Documentation
-
-Import local_job/Job from Etsy, which can eliminate registries
-
-Bug fixes on github
-Enforce more runtime rules at composition time
-Standardize helper contracts
-Possibly combine unit tests...into unit tests because RSpec sucks and swallows stack traces
-
-(Jading)
-Split out runner
-Make runner implement Tool
-Create build tool for job jar