cascading.jruby 0.0.9 → 0.0.10

data/spec/resource/test_input.txt

@@ -1,4 +0,0 @@
-hello,world
-foo,bar
-foo,bar
-biz,baz

data/spec/scope_spec.rb

@@ -1,149 +0,0 @@
-require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
-
-context Cascading::Scope do
-  it 'should match Cascading fields names from source tap scheme' do
-    test_assembly do
-      # Pass that uses our scope instead of all_fields
-      each scope.values_fields, :function => Java::CascadingOperation::Identity.new
-      check_scope :values_fields => ['offset', 'line']
-    end
-  end
-
-  it 'should pick up names from source tap scheme' do
-    test_assembly do
-      pass
-
-      check_scope :values_fields => ['offset', 'line']
-    end
-  end
-
-  it 'should propagate names through Each' do
-    test_assembly do
-      check_scope :values_fields => ['offset', 'line']
-      assert_size_equals 2
-
-      split 'line', ['x', 'y'], :pattern => /,/
-      check_scope :values_fields => ['offset', 'line', 'x', 'y']
-      assert_size_equals 4
-    end
-  end
-
-  it 'should allow field filtration at Each' do
-    test_assembly do
-      check_scope :values_fields => ['offset', 'line']
-      assert_size_equals 2
-
-      split 'line', ['x', 'y'], :pattern => /,/, :output => ['x', 'y']
-      check_scope :values_fields => ['x', 'y']
-      assert_size_equals 2
-    end
-  end
-
-  it 'should propagate names through CoGroup' do
-    test_join_assembly do
-      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
-        :grouping_fields => ['x', 'x_']
-    end
-  end
-
-  it 'should propagate names through CoGroup with no Aggregations' do
-    post_join_block = lambda do
-      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
-        :grouping_fields => ['x', 'x_']
-    end
-
-    test_join_assembly(:post_join_block => post_join_block)
-  end
-
-  it 'should pass grouping fields to Every' do
-    test_join_assembly do
-      sum :mapping => {'x' => 'x_sum'}, :type => :int
-      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
-        :grouping_fields => ['x', 'x_', 'x_sum']
-      assert_group_size_equals 1
-    end
-  end
-
-  it 'should pass grouping fields through chained Every' do
-    test_join_assembly do
-      sum :mapping => {'x' => 'x_sum'}, :type => :int
-      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
-        :grouping_fields => ['x', 'x_', 'x_sum']
-      assert_group_size_equals 1
-
-      sum :mapping => {'y' => 'y_sum'}, :type => :int
-      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
-        :grouping_fields => ['x', 'x_', 'x_sum', 'y_sum']
-      assert_group_size_equals 1
-    end
-  end
-
-  it 'should propagate names through Every' do
-    post_join_block = lambda do
-      check_scope :values_fields => ['x', 'x_', 'x_sum', 'y_sum']
-      assert_size_equals 4
-    end
-
-    test_join_assembly :post_join_block => post_join_block do
-      sum :mapping => {'x' => 'x_sum'}, :type => :int
-      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
-        :grouping_fields => ['x', 'x_', 'x_sum']
-      assert_group_size_equals 1
-
-      sum :mapping => {'y' => 'y_sum'}, :type => :int
-      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
-        :grouping_fields => ['x', 'x_', 'x_sum', 'y_sum']
-      assert_group_size_equals 1
-    end
-  end
-
-  it 'should pass values fields to Each immediately following CoGroup and remove grouping fields' do
-    post_join_block = lambda do
-      assert_size_equals 10
-      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_']
-    end
-    test_join_assembly(:post_join_block => post_join_block)
-  end
-
-  it 'should fail to pass grouping fields to Every immediately following Each' do
-    post_join_block = lambda do
-      pass
-      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_']
-      sum :mapping => {'x' => 'x_sum'}, :type => :int
-    end
-
-    lambda do # Composition fails
-      test_join_assembly(:post_join_block => post_join_block)
-    # sum doesn't exist outside of Aggregations (where block of join is
-    # evaluated)
-    end.should raise_error NoMethodError
-  end
-
-  it 'should propagate values fields and field names into branch' do
-    post_join_block = lambda do
-      branch 'data_tuple' do
-        check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
-          :grouping_fields => ['x', 'x_']
-        assert_size_equals 10
-      end
-    end
-
-    test_join_assembly(:branches => ['data_tuple'], :post_join_block => post_join_block)
-  end
-
-  it 'should propagate names through GroupBy' do
-    test_assembly do
-      group_by 'line' do
-        count
-      end
-      check_scope :values_fields => ['line', 'count']
-    end
-  end
-
-  it 'should propagate names through blockless GroupBy' do
-    test_assembly do
-      group_by 'line'
-      check_scope :values_fields => ['offset', 'line'], :grouping_fields => ['line']
-    end
-  end
-end

data/spec/spec.opts

@@ -1,6 +0,0 @@
---colour
---format
-progress
---loadby
-mtime
---reverse

data/spec/spec_helper.rb

@@ -1,5 +0,0 @@
-require 'spec'
-require 'rubygems'
-require 'cascading'
-
-require File.expand_path(File.dirname(__FILE__) + '/spec_util')

data/spec/spec_util.rb

@@ -1,92 +0,0 @@
-OUTPUT_DIR = 'output'
-BUILD_DIR = 'build/spec'
-
-module ScopeTests
-  def check_scope(params = {})
-    name_params = [params[:source]].compact
-    scope = scope(*name_params)
-    values_fields = params[:values_fields]
-    grouping_fields = params[:grouping_fields] || values_fields
-
-    debug = params[:debug]
-    debug_scope(*name_params) if debug
-
-    scope.values_fields.to_a.should == values_fields
-    scope.grouping_fields.to_a.should == grouping_fields
-  end
-end
-
-module Cascading
-  class Flow; include ScopeTests; end
-  class Assembly; include ScopeTests; end
-  class Aggregations; include ScopeTests; end
-end
-
-def test_flow(&block)
-  cascade = cascade 'test_app', :mode => :local do
-    flow 'test', &block
-  end
-  cascade.complete
-end
-
-def test_assembly(params = {}, &block)
-  branches = params[:branches] || []
-
-  test_flow do
-    source 'input', tap('spec/resource/test_input.txt', :scheme => text_line_scheme)
-
-    # Default Fields defined by TextLineScheme
-    check_scope :source => 'input', :values_fields => ['offset', 'line']
-
-    assembly 'input', &block
-
-    sink 'input', tap("#{OUTPUT_DIR}/out.txt", :sink_mode => :replace)
-
-    # Branches must be sunk so that they (and their assertions) will be run
-    branches.each do |branch|
-      sink branch, tap("#{OUTPUT_DIR}/#{branch}_out.txt", :sink_mode => :replace)
-    end
-  end
-end
-
-def test_join_assembly(params = {}, &block)
-  branches = params[:branches] || []
-  post_join_block = params[:post_join_block]
-
-  test_flow do
-    source 'left', tap('spec/resource/join_input.txt', :scheme => text_line_scheme)
-    source 'right', tap('spec/resource/join_input.txt', :scheme => text_line_scheme)
-
-    # Default Fields defined by TextLineScheme
-    check_scope :source => 'left', :values_fields => ['offset', 'line']
-    check_scope :source => 'right', :values_fields => ['offset', 'line']
-
-    assembly 'left' do
-      check_scope :values_fields => ['offset', 'line']
-      split 'line', ['x', 'y', 'z'], :pattern => /,/
-      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z']
-    end
-
-    assembly 'right' do
-      check_scope :values_fields => ['offset', 'line']
-      split 'line', ['x', 'y', 'z'], :pattern => /,/
-      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z']
-    end
-
-    assembly 'join' do
-      # Empty scope because there is no 'join' source or assembly
-      check_scope :values_fields => []
-
-      left_join 'left', 'right', :on => ['x'], &block
-
-      instance_eval &post_join_block if post_join_block
-    end
-
-    sink 'join', tap("#{OUTPUT_DIR}/join_out.txt", :sink_mode => :replace)
-
-    # Branches must be sunk so that they (and their assertions) will be run
-    branches.each do |branch|
-      sink branch, tap("#{OUTPUT_DIR}/#{branch}_out.txt", :sink_mode => :replace)
-    end
-  end
-end

data/src/cascading/jruby/Main.java

@@ -1,38 +0,0 @@
-package cascading.jruby;
-
-import org.jruby.Ruby;
-import org.jruby.RubyInstanceConfig;
-
-public class Main {
-    private final static String JRUBY_HOME = "/opt/jruby";
-
-    /**
-     * Starts a Hadoop job by reading the specified JRuby script.
-     *
-     * @param args
-     */
-    public static void main(String[] args) {
-        String name = args[0]; // c.j script name
-        if (!name.startsWith("/"))
-            name = "/" + name;
-
-        // c.j script args
-        String[] newArgs = new String[args.length - 1];
-        System.arraycopy(args, 1, newArgs, 0, args.length - 1);
-        RubyInstanceConfig config = new RubyInstanceConfig();
-        config.setJRubyHome(JRUBY_HOME); // mwalker
-        config.processArguments(newArgs);
-
-        System.out.println("Arguments: ");
-        for (String arg : config.getArgv())
-            System.out.println(arg);
-
-        Ruby runtime = Ruby.newInstance(config);
-
-        System.out.println("Requiring '" + name + "'");
-        runtime.executeScript("require '" + name + "'", name);
-
-        System.out.println("Requiring 'cascading/jruby/runner'");
-        runtime.executeScript("require 'cascading/jruby/runner'", "runner"); // gfodor
-    }
-}

data/src/cascading/jruby/runner.rb

@@ -1,6 +0,0 @@
-puts "Found #{Cascading::Cascade.all.size} Cascades in global registry"
-
-Cascading::Cascade.all.each do |cascade|
-  puts "runner.rb running '#{cascade.name}' Cascade"
-  cascade.complete
-end