cascading.jruby 0.0.4

data/spec/spec_util.rb

@@ -0,0 +1,188 @@
+OUTPUT_DIR = 'output'
+BUILD_DIR = 'build/spec'
+
+module ScopeTests
+  def check_scope(params = {})
+    name_params = [params[:source]].compact
+    scope = scope(*name_params)
+    values_fields = params[:values_fields]
+    grouping_fields = params[:grouping_fields] || values_fields
+    primary_key_fields = params[:primary_key_fields]
+    grouping_primary_key_fields = primary_key_fields
+    grouping_primary_key_fields = params[:grouping_primary_key_fields] if params.has_key?(:grouping_primary_key_fields)
+
+    debug = params[:debug]
+    debug_scope(*name_params) if debug
+
+    scope.values_fields.to_a.should == values_fields
+    scope.grouping_fields.to_a.should == grouping_fields
+    if params.has_key?(:primary_key_fields) # Must support nil values
+      scope.primary_key_fields.should == nil if primary_key_fields.nil?
+      scope.primary_key_fields.to_a.should == primary_key_fields unless primary_key_fields.nil?
+
+      scope.grouping_primary_key_fields.should == nil if grouping_primary_key_fields.nil?
+      scope.grouping_primary_key_fields.to_a.should == grouping_primary_key_fields unless grouping_primary_key_fields.nil?
+    end
+  end
+end
+
+module Cascading
+  class Flow; include ScopeTests; end
+  class Assembly; include ScopeTests; end
+end
+
+def test_flow(&block)
+  cascade = cascade 'test_app' do
+    flow 'test', &block
+  end
+  cascade.complete(cascading_properties)
+end
+
+def test_assembly(params = {}, &block)
+  branches = params[:branches] || []
+
+  test_flow do
+    source 'input', tap('spec/resource/test_input.txt', :kind => :lfs, :scheme => text_line_scheme)
+
+    # Default Fields defined by TextLineScheme
+    check_scope :source => 'input', :values_fields => ['offset', 'line']
+
+    assembly 'input', &block
+
+    sink 'input', tap("#{OUTPUT_DIR}/out.txt", :kind => :lfs, :sink_mode => :replace)
+
+    # Branches must be sunk so that they (and their assertions) will be run
+    branches.each do |branch|
+      sink branch, tap("#{OUTPUT_DIR}/#{branch}_out.txt", :kind => :lfs, :sink_mode => :replace)
+    end
+  end
+end
+
+def test_join_assembly(params = {}, &block)
+  branches = params[:branches] || []
+
+  test_flow do
+    source 'left', tap('spec/resource/join_input.txt', :kind => :lfs, :scheme => text_line_scheme)
+    source 'right', tap('spec/resource/join_input.txt', :kind => :lfs, :scheme => text_line_scheme)
+
+    # Default Fields defined by TextLineScheme
+    check_scope :source => 'left', :values_fields => ['offset', 'line']
+    check_scope :source => 'right', :values_fields => ['offset', 'line']
+
+    assembly 'left' do
+      check_scope :values_fields => ['offset', 'line']
+      split 'line', ['x', 'y', 'z'], :pattern => /,/
+      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z']
+    end
+
+    assembly 'right' do
+      check_scope :values_fields => ['offset', 'line']
+      split 'line', ['x', 'y', 'z'], :pattern => /,/
+      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z']
+    end
+
+    assembly 'join' do
+      # Empty scope because there is no 'join' source or assembly
+      check_scope :values_fields => []
+
+      left_join 'left', 'right', :on => ['x']
+      check_scope :values_fields => ['offset', 'line', 'x', 'y', 'z', 'offset_', 'line_', 'x_', 'y_', 'z_'],
+        :grouping_fields => ['x']
+
+      instance_eval(&block)
+    end
+
+    sink 'join', tap("#{OUTPUT_DIR}/join_out.txt", :kind => :lfs, :sink_mode => :replace)
+
+    # Branches must be sunk so that they (and their assertions) will be run
+    branches.each do |branch|
+      sink branch, tap("#{OUTPUT_DIR}/#{branch}_out.txt", :kind => :lfs, :sink_mode => :replace)
+    end
+  end
+end
+
+def cascading_properties
+  build_dir = "#{BUILD_DIR}/build"
+  `mkdir -p #{build_dir}`
+  tmp_dir = "#{BUILD_DIR}/tmp"
+  `mkdir -p #{tmp_dir}`
+  log_dir = "#{BUILD_DIR}/log"
+  `mkdir -p #{log_dir}`
+
+  # Local cluster settings
+  #java.lang.System.set_property("test.build.data", build_dir)
+  #java.lang.System.set_property("hadoop.tmp.dir", tmp_dir)
+  #java.lang.System.set_property("hadoop.log.dir", log_dir)
+  #conf = Java::OrgApacheHadoopConf::Configuration.new
+  #dfs = Java::OrgApacheHadoopDfs::MiniDFSCluster.new(conf, 4, true, nil);
+  #file_sys = dfs.file_system
+  #mr = Java::OrgApacheHadoopMapred::MiniMRCluster.new(4, file_sys.uri.to_string, 1)
+  #job_conf = mr.create_job_conf
+  #job_conf.set("mapred.child.java.opts", "-Xmx512m")
+  #job_conf.set("mapred.map.tasks.speculative.execution", "false")
+  #job_conf.set("mapred.reduce.tasks.speculative.execution", "false")
+
+  job_conf = Java::OrgApacheHadoopMapred::JobConf.new
+  job_conf.jar = build_dir
+  job_conf.set("test.build.data", build_dir)
+  job_conf.set("hadoop.tmp.dir", tmp_dir)
+  job_conf.set("hadoop.log.dir", log_dir)
+
+  job_conf.num_map_tasks = 4
+  job_conf.num_reduce_tasks = 1
+
+  properties = java.util.HashMap.new({})
+  Java::CascadingFlow::MultiMapReducePlanner.set_job_conf(properties, job_conf)
+  properties
+end
+
+def verify_assembly_output(assembly_name, params, &block)
+  `rm -rf spec_output`
+
+  Cascade.new("foo") do
+    flow("bar") do
+      source assembly_name, tap(params[:source], params.slice(:scheme)) 
+      assembly = assembly(assembly_name)
+      sink assembly_name, tap("spec_output", :kind => :lfs, :sink_mode => :replace)
+    end
+  end.complete(@properties)
+
+  output_data = nil
+
+  File.open("spec_output/part-00000") do |f|
+    output_data = f.readlines
+  end
+
+  if params[:length]
+    output_data.size.should == params[:length]
+  end
+
+  keys = assembly.scope.values_fields
+  if block_given?
+    output_data.each do |line| 
+      values = line.chomp.split(/\t/) 
+
+      yield(keys.zip(values).inject({}) do |map, kv|
+        map[kv[0].to_sym] = kv[1]
+        map
+      end)
+    end
+  end
+end
+
+def describe_job(job_file, &block)
+  describe Object do
+    before(:each) do
+      @properties = cascading_properties
+      # Must artificially fill ARGV to prevent errors when creating multi-taps
+      # in ETL cascade
+      ARGV.clear
+      10.times do
+        ARGV << 'text_line_scheme' # Dummy value, required for 3rd arg
+      end
+      load "lib/jobs/#{job_file}/#{job_file}.rb"
+    end
+
+    self.class_eval(&block)
+  end
+end

data/src/cascading/jruby/Main.java

@@ -0,0 +1,38 @@
+package cascading.jruby;
+
+import org.jruby.Ruby;
+import org.jruby.RubyInstanceConfig;
+
+public class Main {
+    private final static String JRUBY_HOME = "/opt/jruby";
+
+    /**
+     * Starts a Hadoop job by reading the specified JRuby script.
+     *
+     * @param args
+     */
+    public static void main(String[] args) {
+        String name = args[0]; // c.j script name
+        if (!name.startsWith("/"))
+            name = "/" + name;
+
+        // c.j script args
+        String[] newArgs = new String[args.length - 1];
+        System.arraycopy(args, 1, newArgs, 0, args.length - 1);
+        RubyInstanceConfig config = new RubyInstanceConfig();
+        config.setJRubyHome(JRUBY_HOME); // mwalker
+        config.processArguments(newArgs);
+
+        System.out.println("Arguments: ");
+        for (String arg : config.getArgv())
+            System.out.println(arg);
+
+        Ruby runtime = Ruby.newInstance(config);
+
+        System.out.println("Requiring '" + name + "'");
+        runtime.executeScript("require '" + name + "'", name);
+
+        System.out.println("Requiring 'cascading/jruby/runner'");
+        runtime.executeScript("require 'cascading/jruby/runner'", "runner"); // gfodor
+    }
+}

data/src/cascading/jruby/runner.rb

@@ -0,0 +1,6 @@
+puts "Found #{Cascading::Cascade.all.size} Cascades in global registry"
+
+Cascading::Cascade.all.each do |cascade|
+  puts "runner.rb running '#{cascade.name}' Cascade"
+  cascade.complete
+end

data/tags

@@ -0,0 +1,238 @@
+!_TAG_FILE_FORMAT	2	/extended format; --format=1 will not append ;" to lines/
+!_TAG_FILE_SORTED	1	/0=unsorted, 1=sorted, 2=foldcase/
+!_TAG_PROGRAM_AUTHOR	Darren Hiebert	/dhiebert@users.sourceforge.net/
+!_TAG_PROGRAM_NAME	Exuberant Ctags	//
+!_TAG_PROGRAM_URL	http://ctags.sourceforge.net	/official site/
+!_TAG_PROGRAM_VERSION	5.8	//
+Array	lib/cascading/ext/array.rb	/^class Array$/;"	c
+Assembly	lib/cascading/assembly.rb	/^  class Assembly < Cascading::Node$/;"	c	class:Cascading
+Assembly	spec/spec_util.rb	/^  class Assembly; include ScopeTests; end$/;"	c	class:Cascading
+Cascade	lib/cascading/cascade.rb	/^  class Cascade < Cascading::Node$/;"	c	class:Cascading
+Cascading	lib/cascading.rb	/^module Cascading$/;"	m
+Cascading	lib/cascading/assembly.rb	/^module Cascading$/;"	m
+Cascading	lib/cascading/base.rb	/^module Cascading$/;"	m
+Cascading	lib/cascading/cascade.rb	/^module Cascading$/;"	m
+Cascading	lib/cascading/cascading.rb	/^module Cascading$/;"	m
+Cascading	lib/cascading/flow.rb	/^module Cascading$/;"	m
+Cascading	lib/cascading/operations.rb	/^module Cascading$/;"	m
+Cascading	lib/cascading/scope.rb	/^module Cascading$/;"	m
+Cascading	samples/cascading.rb	/^module Cascading$/;"	m
+Cascading	spec/spec_util.rb	/^module Cascading$/;"	m
+CascadingException	lib/cascading/cascading_exception.rb	/^class CascadingException < StandardError$/;"	c
+ExprStub	lib/cascading/expr_stub.rb	/^class ExprStub$/;"	c
+Flow	lib/cascading/flow.rb	/^  class Flow < Cascading::Node$/;"	c	class:Cascading
+Flow	spec/spec_util.rb	/^  class Flow; include ScopeTests; end$/;"	c	class:Cascading
+JRUBY_HOME	src/cascading/jruby/Main.java	/^	private final static String JRUBY_HOME = "\/opt\/jruby";$/;"	f	class:Main	file:
+Main	src/cascading/jruby/Main.java	/^public class Main {$/;"	c
+Node	lib/cascading/base.rb	/^  class Node$/;"	c	class:Cascading
+Object	tasks/setup.rb	/^class Object$/;"	c
+OpenStruct	tasks/setup.rb	/^class OpenStruct; undef :gem; end$/;"	c
+Operations	lib/cascading/operations.rb	/^  module Operations$/;"	m	class:Cascading
+Registerable	lib/cascading/base.rb	/^  module Registerable$/;"	m	class:Cascading
+Scope	lib/cascading/scope.rb	/^  class Scope$/;"	c	class:Cascading
+ScopeTests	spec/spec_util.rb	/^module ScopeTests$/;"	m
+TC_Assembly	test/test_assembly.rb	/^class TC_Assembly < Test::Unit::TestCase$/;"	c
+TC_AssemblyScenarii	test/test_assembly.rb	/^class TC_AssemblyScenarii < Test::Unit::TestCase$/;"	c
+TC_Cascading	test/test_cascading.rb	/^class TC_Cascading < Test::Unit::TestCase$/;"	c
+TC_Flow	test/test_flow.rb	/^class TC_Flow < Test::Unit::TestCase$/;"	c
+add	lib/cascading/base.rb	/^    def add(name, instance)$/;"	f	class:Cascading.Registerable
+add_archive_to_distributed_cache	lib/cascading/flow.rb	/^    def add_archive_to_distributed_cache(file)$/;"	f
+add_child	lib/cascading/base.rb	/^    def add_child(node)$/;"	f	class:Cascading.Node
+add_file_to_distributed_cache	lib/cascading/flow.rb	/^    def add_file_to_distributed_cache(file)$/;"	f
+add_listener	lib/cascading/flow.rb	/^    def add_listener(listener)$/;"	f
+add_to_distributed_cache	lib/cascading/flow.rb	/^    def add_to_distributed_cache(file, property)$/;"	f
+aggregator_function	lib/cascading/operations.rb	/^    def aggregator_function(args, aggregator_klass)$/;"	f	class:Cascading.Operations
+all	lib/cascading/base.rb	/^    def all$/;"	f	class:Cascading.Registerable
+all_fields	lib/cascading/cascading.rb	/^  def all_fields$/;"	f	class:Cascading
+assembly	lib/cascading/flow.rb	/^    def assembly(name, &block)$/;"	f	class:Cascading.Flow
+assembly	test/test_assembly.rb	/^def assembly(name, &block)$/;"	f
+assert	lib/cascading/assembly.rb	/^    def assert(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+assert_group	lib/cascading/assembly.rb	/^    def assert_group(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+assert_group_size_equals	lib/cascading/assembly.rb	/^    def assert_group_size_equals(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+assert_not_null	lib/cascading/assembly.rb	/^    def assert_not_null(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+assert_size_equals	lib/cascading/assembly.rb	/^    def assert_size_equals(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+average	lib/cascading/assembly.rb	/^    def average(*args); composite_aggregator(args, :average_function); end$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+average_function	lib/cascading/operations.rb	/^    def average_function(*args)$/;"	f	class:Cascading.Operations
+bind_names	lib/cascading/assembly.rb	/^    def bind_names(*new_names)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+branch	lib/cascading/assembly.rb	/^    def branch(name, &block)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+build_connect_parameter	lib/cascading/flow.rb	/^    def build_connect_parameter$/;"	f
+cascade	lib/cascading/cascading.rb	/^  def cascade(name, &block)$/;"	f	class:Cascading
+cascading.jruby	src/cascading/jruby/Main.java	/^package cascading.jruby;$/;"	p
+cascading_properties	spec/spec_util.rb	/^def cascading_properties$/;"	f
+cast	lib/cascading/assembly.rb	/^    def cast(type_map)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+cause	lib/cascading/cascading_exception.rb	/^  def cause(depth)$/;"	f	class:CascadingException
+check_scope	spec/spec_util.rb	/^  def check_scope(params = {})$/;"	f	class:ScopeTests
+compare_with_references	test/test_assembly.rb	/^def compare_with_references(test_name)$/;"	f
+complete	lib/cascading/cascade.rb	/^    def complete(properties = nil)$/;"	f	class:Cascading.Cascade
+complete	lib/cascading/flow.rb	/^    def complete(properties = nil)$/;"	f
+composite_aggregator	lib/cascading/assembly.rb	/^    def composite_aggregator(args, function)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+compress_output	lib/cascading/flow.rb	/^    def compress_output(codec, type)$/;"	f	class:Cascading.Flow
+connect	lib/cascading/flow.rb	/^    def connect(properties = nil)$/;"	f
+copy	lib/cascading/assembly.rb	/^    def copy(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+copy	lib/cascading/scope.rb	/^    def copy$/;"	f	class:Cascading.Scope
+copy_fields	lib/cascading/cascading.rb	/^  def copy_fields(fields)$/;"	f	class:Cascading
+count	lib/cascading/assembly.rb	/^    def count(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+count_function	lib/cascading/operations.rb	/^    def count_function(*args)$/;"	f	class:Cascading.Operations
+date_formatter	lib/cascading/operations.rb	/^    def date_formatter(fields, format, timezone=nil)$/;"	f	class:Cascading.Operations.to_java_comparable_array
+date_parser	lib/cascading/operations.rb	/^    def date_parser(field, format)$/;"	f	class:Cascading.Operations.to_java_comparable_array
+debug	lib/cascading/assembly.rb	/^    def debug(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+debug_scope	lib/cascading/assembly.rb	/^    def debug_scope$/;"	f	class:Cascading.Assembly
+debug_scope	lib/cascading/flow.rb	/^    def debug_scope(name = nil)$/;"	f	class:Cascading.Flow
+dedup_field_names	lib/cascading/cascading.rb	/^  def dedup_field_names(*names)$/;"	f	class:Cascading
+dedup_fields	lib/cascading/cascading.rb	/^  def dedup_fields(*fields)$/;"	f	class:Cascading
+depend_on	tasks/setup.rb	/^def depend_on( name, version = nil )$/;"	f
+describe_job	spec/spec_util.rb	/^def describe_job(job_file, &block)$/;"	f
+difference_fields	lib/cascading/cascading.rb	/^  def difference_fields(*fields)$/;"	f	class:Cascading
+discard	lib/cascading/assembly.rb	/^    def discard(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+distinct	lib/cascading/assembly.rb	/^    def distinct(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+do_every_block_and_rename_fields	lib/cascading/assembly.rb	/^    def do_every_block_and_rename_fields(group_fields, incoming_scopes, &block)$/;"	f	class:Cascading.Assembly
+draw	lib/cascading/cascade.rb	/^    def draw(dir, properties = nil)$/;"	f	class:Cascading.Cascade
+each	lib/cascading/assembly.rb	/^    def each(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+empty_scope	lib/cascading/scope.rb	/^    def self.empty_scope(name)$/;"	F	class:Cascading.Scope
+emr_local_path_for_distributed_cache_file	lib/cascading/flow.rb	/^    def emr_local_path_for_distributed_cache_file(file)$/;"	f
+ensure_in_path	tasks/setup.rb	/^def ensure_in_path( *args )$/;"	f
+eval_expression	lib/cascading/assembly.rb	/^    def eval_expression(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+every	lib/cascading/assembly.rb	/^    def every(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+every_applied?	lib/cascading/assembly.rb	/^    def every_applied?$/;"	f	class:Cascading.Assembly
+expr	lib/cascading/cascading.rb	/^  def expr(s)$/;"	f	class:Cascading
+expression_filter	lib/cascading/operations.rb	/^    def expression_filter(*args)$/;"	f	class:Cascading.Operations.to_java_comparable_array
+expression_function	lib/cascading/operations.rb	/^    def expression_function(*args)$/;"	f	class:Cascading.Operations
+extract_options	lib/cascading/ext/array.rb	/^  def extract_options$/;"	f	class:Array
+extract_options!	lib/cascading/ext/array.rb	/^  def extract_options!$/;"	f	class:Array
+fetch_cause	lib/cascading/cascading_exception.rb	/^  def fetch_cause(ne, depth)$/;"	f	class:CascadingException
+field_joiner	lib/cascading/operations.rb	/^    def field_joiner(*args)$/;"	f	class:Cascading.Operations.to_java_comparable_array
+fields	lib/cascading/cascading.rb	/^  def fields(fields)$/;"	f	class:Cascading
+filter	lib/cascading/assembly.rb	/^    def filter(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+filter_not_null	lib/cascading/assembly.rb	/^    def filter_not_null(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+filter_null	lib/cascading/assembly.rb	/^    def filter_null(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+find_child	lib/cascading/base.rb	/^    def find_child(name)$/;"	f	class:Cascading.Node
+first	lib/cascading/assembly.rb	/^    def first(*args); composite_aggregator(args, :first_function); end$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+first_function	lib/cascading/operations.rb	/^    def first_function(*args)$/;"	f	class:Cascading.Operations
+flow	lib/cascading/cascade.rb	/^    def flow(name, &block)$/;"	f	class:Cascading.Cascade
+flow	lib/cascading/cascading.rb	/^  def flow(name, &block)$/;"	f	class:Cascading
+format_date	lib/cascading/assembly.rb	/^    def format_date(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+get	lib/cascading/base.rb	/^    def get(key)$/;"	f	class:Cascading.Registerable
+group_by	lib/cascading/assembly.rb	/^    def group_by(*args, &block)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+grouping_fields	lib/cascading/scope.rb	/^    def grouping_fields$/;"	f	class:Cascading.Scope
+grouping_primary_key_fields	lib/cascading/scope.rb	/^    def self.grouping_primary_key_fields(flow_element, incoming_scopes, scope)$/;"	F	class:Cascading.Scope
+identity	lib/cascading/operations.rb	/^    def identity$/;"	f	class:Cascading.Operations
+in_directory	tasks/setup.rb	/^def in_directory( dir, &block )$/;"	f
+initialize	lib/cascading/assembly.rb	/^    def initialize(name, parent, outgoing_scopes = {})$/;"	f	class:Cascading.Assembly
+initialize	lib/cascading/base.rb	/^    def initialize(name, parent)$/;"	f	class:Cascading.Node
+initialize	lib/cascading/cascade.rb	/^    def initialize(name)$/;"	f	class:Cascading.Cascade
+initialize	lib/cascading/cascading_exception.rb	/^  def initialize(native_exception, message)$/;"	f	class:CascadingException
+initialize	lib/cascading/expr_stub.rb	/^  def initialize(st)$/;"	f	class:ExprStub
+initialize	lib/cascading/flow.rb	/^    def initialize(name, parent)$/;"	f	class:Cascading.Flow
+initialize	lib/cascading/scope.rb	/^    def initialize(scope, params = {})$/;"	f	class:Cascading.Scope
+inner_join	lib/cascading/assembly.rb	/^    def inner_join(*args, &block)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+insert	lib/cascading/assembly.rb	/^    def insert(args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+insert_function	lib/cascading/operations.rb	/^    def insert_function(*args)$/;"	f	class:Cascading.Operations
+join	lib/cascading/assembly.rb	/^    def join(*args, &block)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+join_fields	lib/cascading/assembly.rb	/^    def join_fields(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+last	lib/cascading/assembly.rb	/^    def last(*args); composite_aggregator(args, :last_function); end$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+last_function	lib/cascading/operations.rb	/^    def last_function(*args)$/;"	f	class:Cascading.Operations
+last_grouping_fields	lib/cascading/cascading.rb	/^  def last_grouping_fields$/;"	f	class:Cascading
+left_join	lib/cascading/assembly.rb	/^    def left_join(*args, &block)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+libpath	lib/cascading.rb	/^  def self.libpath( *args )$/;"	F	class:Cascading
+main	src/cascading/jruby/Main.java	/^	public static void main(String[] args) {$/;"	m	class:Main
+make_each	lib/cascading/assembly.rb	/^    def make_each(type, *parameters)$/;"	f	class:Cascading.Assembly
+make_every	lib/cascading/assembly.rb	/^    def make_every(type, *parameters)$/;"	f	class:Cascading.Assembly
+make_flows	lib/cascading/cascade.rb	/^    def make_flows(flows, properties)$/;"	f	class:Cascading.Cascade
+make_pipe	lib/cascading/assembly.rb	/^    def make_pipe(type, parameters, grouping_key_fields = [], incoming_scopes = [scope])$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+make_pipes	lib/cascading/flow.rb	/^    def make_pipes$/;"	f
+make_tap_parameter	lib/cascading/flow.rb	/^    def make_tap_parameter(taps)$/;"	f
+manifest	tasks/setup.rb	/^def manifest$/;"	f
+match_rows	lib/cascading/assembly.rb	/^    def match_rows(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+max	lib/cascading/assembly.rb	/^    def max(*args); composite_aggregator(args, :max_function); end$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+max_function	lib/cascading/operations.rb	/^    def max_function(*args)$/;"	f	class:Cascading.Operations
+min	lib/cascading/assembly.rb	/^    def min(*args); composite_aggregator(args, :min_function); end$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+min_function	lib/cascading/operations.rb	/^    def min_function(*args)$/;"	f	class:Cascading.Operations
+mock_assembly	test/test_assembly.rb	/^  def mock_assembly(&block)$/;"	f	class:TC_Assembly
+multi_tap	lib/cascading/cascading.rb	/^  def multi_tap(*taps)$/;"	f	class:Cascading
+outer_join	lib/cascading/assembly.rb	/^    def outer_join(*args, &block)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+outgoing_scope	lib/cascading/scope.rb	/^    def self.outgoing_scope(flow_element, incoming_scopes, grouping_key_fields, every_applied)$/;"	F	class:Cascading.Scope
+outgoing_scope_for	lib/cascading/scope.rb	/^    def self.outgoing_scope_for(flow_element, incoming_scopes)$/;"	F	class:Cascading.Scope
+paragraphs_of	tasks/setup.rb	/^def paragraphs_of( path, *paragraphs )$/;"	f
+parent_flow	lib/cascading/assembly.rb	/^    def parent_flow$/;"	f	class:Cascading.Assembly
+parse	lib/cascading/assembly.rb	/^    def parse(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+parse_date	lib/cascading/assembly.rb	/^    def parse_date(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+pass	lib/cascading/assembly.rb	/^    def pass(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+path	lib/cascading.rb	/^  def self.path( *args )$/;"	F	class:Cascading
+primary	lib/cascading/assembly.rb	/^    def primary(*args)$/;"	f	class:Cascading.Assembly
+primary_key_fields	lib/cascading/scope.rb	/^    def self.primary_key_fields(flow_element, incoming_scopes, scope)$/;"	F	class:Cascading.Scope
+project	lib/cascading/assembly.rb	/^    def project(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+project_primary_key	lib/cascading/scope.rb	/^    def self.project_primary_key(primary_key, old_fields, new_fields)$/;"	F	class:Cascading.Scope
+quiet	tasks/setup.rb	/^def quiet( &block )$/;"	f
+regex_filter	lib/cascading/operations.rb	/^    def regex_filter(*args)$/;"	f	class:Cascading.Operations.to_java_comparable_array
+regex_generator	lib/cascading/operations.rb	/^    def regex_generator(*args)$/;"	f	class:Cascading.Operations
+regex_parser	lib/cascading/operations.rb	/^    def regex_parser(*args)$/;"	f	class:Cascading.Operations
+regex_replace	lib/cascading/operations.rb	/^    def regex_replace(*args)$/;"	f	class:Cascading.Operations.to_java_comparable_array
+regex_split_generator	lib/cascading/operations.rb	/^    def regex_split_generator(*args)$/;"	f	class:Cascading.Operations
+regex_splitter	lib/cascading/operations.rb	/^    def regex_splitter(*args)$/;"	f	class:Cascading.Operations
+register_scheme_key	lib/cascading/scope.rb	/^    def self.register_scheme_key(scheme, primary_key)$/;"	F	class:Cascading.Scope
+registered	lib/cascading/base.rb	/^    def registered$/;"	f	class:Cascading.Registerable
+reject	lib/cascading/assembly.rb	/^    def reject(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+remove_desc_for_task	tasks/setup.rb	/^def remove_desc_for_task( names )$/;"	f
+rename	lib/cascading/assembly.rb	/^    def rename(name_map)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+replace	lib/cascading/assembly.rb	/^    def replace(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+require_all_jars	lib/cascading.rb	/^  def self.require_all_jars(from = ::File.join(::File.dirname(__FILE__), "..", "jars"))$/;"	F	class:Cascading
+require_all_libs_relative_to	lib/cascading.rb	/^  def self.require_all_libs_relative_to( fname, dir = nil )$/;"	F	class:Cascading
+reset	lib/cascading/base.rb	/^    def reset$/;"	f	class:Cascading.Registerable
+results_fields	lib/cascading/cascading.rb	/^  def results_fields$/;"	f	class:Cascading
+right_join	lib/cascading/assembly.rb	/^    def right_join(*args, &block)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+sample_properties	samples/cascading.rb	/^  def sample_properties$/;"	f	class:Cascading
+scope	lib/cascading/assembly.rb	/^    def scope$/;"	f	class:Cascading.Assembly
+scope	lib/cascading/flow.rb	/^    def scope(name = nil)$/;"	f	class:Cascading.Flow
+search_field_name	lib/cascading/cascading.rb	/^  def search_field_name(names, candidate)$/;"	f	class:Cascading
+sequence_file_scheme	lib/cascading/cascading.rb	/^  def sequence_file_scheme(*fields)$/;"	f	class:Cascading
+set_spill_threshold	lib/cascading/flow.rb	/^    def set_spill_threshold(threshold)$/;"	f
+sink	lib/cascading/flow.rb	/^    def sink(*args)$/;"	f	class:Cascading.Flow
+sink_metadata	lib/cascading/cascade.rb	/^    def sink_metadata$/;"	f	class:Cascading.Cascade
+sink_metadata	lib/cascading/flow.rb	/^    def sink_metadata$/;"	f	class:Cascading.Flow
+source	lib/cascading/flow.rb	/^    def source(*args)$/;"	f	class:Cascading.Flow
+split	lib/cascading/assembly.rb	/^    def split(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+split_hash	lib/cascading/expr_stub.rb	/^  def self.split_hash(h)$/;"	F
+split_names_and_types	lib/cascading/expr_stub.rb	/^  def self.split_names_and_types(expr_types)$/;"	F
+split_rows	lib/cascading/assembly.rb	/^    def split_rows(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+sum	lib/cascading/assembly.rb	/^    def sum(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+sum_function	lib/cascading/operations.rb	/^    def sum_function(*args)$/;"	f	class:Cascading.Operations
+tap	lib/cascading/cascading.rb	/^  def tap(*args)$/;"	f	class:Cascading
+tap_scope	lib/cascading/scope.rb	/^    def self.tap_scope(tap, name)$/;"	F	class:Cascading.Scope
+test_assembly	spec/spec_util.rb	/^def test_assembly(params = {}, &block)$/;"	f
+test_assembly	test/test_flow.rb	/^  def test_assembly$/;"	f	class:TC_Flow
+test_branch_empty	test/test_assembly.rb	/^  def test_branch_empty$/;"	f
+test_branch_single	test/test_assembly.rb	/^  def test_branch_single$/;"	f
+test_branch_unique	test/test_assembly.rb	/^  def test_branch_unique$/;"	f
+test_create_assembly_simple	test/test_assembly.rb	/^  def test_create_assembly_simple$/;"	f	class:TC_Assembly
+test_create_each	test/test_assembly.rb	/^  def test_create_each$/;"	f
+test_create_every	test/test_assembly.rb	/^  def test_create_every$/;"	f
+test_create_group_by	test/test_assembly.rb	/^  def test_create_group_by$/;"	f
+test_create_group_by_many_fields	test/test_assembly.rb	/^  def test_create_group_by_many_fields$/;"	f
+test_create_group_by_reverse	test/test_assembly.rb	/^  def test_create_group_by_reverse$/;"	f
+test_create_group_by_with_sort	test/test_assembly.rb	/^  def test_create_group_by_with_sort$/;"	f
+test_create_group_by_with_sort_reverse	test/test_assembly.rb	/^  def test_create_group_by_with_sort_reverse$/;"	f
+test_each_identity	test/test_assembly.rb	/^  def test_each_identity$/;"	f
+test_fields_field	test/test_cascading.rb	/^  def test_fields_field$/;"	f	class:TC_Cascading
+test_fields_multiple	test/test_cascading.rb	/^  def test_fields_multiple$/;"	f	class:TC_Cascading
+test_fields_single	test/test_cascading.rb	/^  def test_fields_single$/;"	f	class:TC_Cascading
+test_flow	spec/spec_util.rb	/^def test_flow(&block)$/;"	f
+test_full_assembly	test/test_assembly.rb	/^  def test_full_assembly$/;"	f
+test_join1	test/test_assembly.rb	/^  def test_join1$/;"	f
+test_join2	test/test_assembly.rb	/^  def test_join2$/;"	f
+test_join_assembly	spec/spec_util.rb	/^def test_join_assembly(params = {}, &block)$/;"	f
+test_splitter	test/test_assembly.rb	/^  def test_splitter$/;"	f	class:TC_AssemblyScenarii
+test_tap	test/test_cascading.rb	/^  def test_tap$/;"	f	class:TC_Cascading
+text_line_scheme	lib/cascading/cascading.rb	/^  def text_line_scheme(*fields)$/;"	f	class:Cascading
+to_java_comparable_array	lib/cascading/operations.rb	/^    def to_java_comparable_array(arr)$/;"	f	class:Cascading.Operations
+to_s	lib/cascading/assembly.rb	/^    def to_s$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+to_s	lib/cascading/scope.rb	/^    def to_s$/;"	f	class:Cascading.Scope
+trace_causes	lib/cascading/cascading_exception.rb	/^  def trace_causes(ne, depth)$/;"	f	class:CascadingException
+union	lib/cascading/assembly.rb	/^    def union(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+union_fields	lib/cascading/cascading.rb	/^  def union_fields(*fields)$/;"	f	class:Cascading
+union_pipes	lib/cascading/assembly.rb	/^    def union_pipes(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+valid?	tasks/setup.rb	/^  def valid?$/;"	f	class:Object
+values_fields	lib/cascading/scope.rb	/^    def values_fields$/;"	f	class:Cascading.Scope
+verify_assembly_output	spec/spec_util.rb	/^def verify_assembly_output(assembly_name, params, &block)$/;"	f
+version	lib/cascading.rb	/^  def self.version$/;"	F	class:Cascading
+where	lib/cascading/assembly.rb	/^    def where(*args)$/;"	f	class:Cascading.Assembly.do_every_block_and_rename_fields
+write_sink_metadata	lib/cascading/cascade.rb	/^    def write_sink_metadata(file_name)$/;"	f	class:Cascading.Cascade

data/tasks/ann.rake

@@ -0,0 +1,80 @@
+
+begin
+  require 'bones/smtp_tls'
+rescue LoadError
+  require 'net/smtp'
+end
+require 'time'
+
+namespace :ann do
+
+  # A prerequisites task that all other tasks depend upon
+  task :prereqs
+
+  file PROJ.ann.file do
+    ann = PROJ.ann
+    puts "Generating #{ann.file}"
+    File.open(ann.file,'w') do |fd|
+      fd.puts("#{PROJ.name} version #{PROJ.version}")
+      fd.puts("    by #{Array(PROJ.authors).first}") if PROJ.authors
+      fd.puts("    #{PROJ.url}") if PROJ.url.valid?
+      fd.puts("    (the \"#{PROJ.release_name}\" release)") if PROJ.release_name
+      fd.puts
+      fd.puts("== DESCRIPTION")
+      fd.puts
+      fd.puts(PROJ.description)
+      fd.puts
+      fd.puts(PROJ.changes.sub(%r/^.*$/, '== CHANGES'))
+      fd.puts
+      ann.paragraphs.each do |p|
+        fd.puts "== #{p.upcase}"
+        fd.puts
+        fd.puts paragraphs_of(PROJ.readme_file, p).join("\n\n")
+        fd.puts
+      end
+      fd.puts ann.text if ann.text
+    end
+  end
+
+  desc "Create an announcement file"
+  task :announcement => ['ann:prereqs', PROJ.ann.file]
+
+  desc "Send an email announcement"
+  task :email => ['ann:prereqs', PROJ.ann.file] do
+    ann = PROJ.ann
+    from = ann.email[:from] || Array(PROJ.authors).first || PROJ.email
+    to   = Array(ann.email[:to])
+
+    ### build a mail header for RFC 822
+    rfc822msg =  "From: #{from}\n"
+    rfc822msg << "To: #{to.join(',')}\n"
+    rfc822msg << "Subject: [ANN] #{PROJ.name} #{PROJ.version}"
+    rfc822msg << " (#{PROJ.release_name})" if PROJ.release_name
+    rfc822msg << "\n"
+    rfc822msg << "Date: #{Time.new.rfc822}\n"
+    rfc822msg << "Message-Id: "
+    rfc822msg << "<#{"%.8f" % Time.now.to_f}@#{ann.email[:domain]}>\n\n"
+    rfc822msg << File.read(ann.file)
+
+    params = [:server, :port, :domain, :acct, :passwd, :authtype].map do |key|
+      ann.email[key]
+    end
+
+    params[3] = PROJ.email if params[3].nil?
+
+    if params[4].nil?
+      STDOUT.write "Please enter your e-mail password (#{params[3]}): "
+      params[4] = STDIN.gets.chomp
+    end
+
+    ### send email
+    Net::SMTP.start(*params) {|smtp| smtp.sendmail(rfc822msg, from, to)}
+  end
+end  # namespace :ann
+
+desc 'Alias to ann:announcement'
+task :ann => 'ann:announcement'
+
+CLOBBER << PROJ.ann.file
+
+# EOF

data/tasks/ant.rake

@@ -0,0 +1,11 @@
+namespace :ant do
+  desc 'Builds Java source for inclusion in gem'
+  task :build do
+    `ant build`
+  end
+
+  desc 'Cleans Java build files'
+  task :clean do
+    `ant clean`
+  end
+end

data/tasks/bones.rake

@@ -0,0 +1,20 @@
+
+if HAVE_BONES
+
+namespace :bones do
+
+  desc 'Show the PROJ open struct'
+  task :debug do |t|
+    atr = if t.application.top_level_tasks.length == 2
+      t.application.top_level_tasks.pop
+    end
+
+    if atr then Bones::Debug.show_attr(PROJ, atr)
+    else Bones::Debug.show PROJ end
+  end
+
+end  # namespace :bones
+
+end  # HAVE_BONES
+
+# EOF