cascading.jruby 0.0.4

data/lib/cascading/operations.rb

@@ -0,0 +1,204 @@
+# Copyright 2009, Grégoire Marabout. All Rights Reserved.
+#
+# This is free software. Please see the LICENSE and COPYING files for details.
+
+module Cascading
+  module Operations
+    def identity
+      Java::CascadingOperation::Identity.new
+    end
+
+    def sum_function(*args)
+      options = args.extract_options!
+      raise "Need to specify args" if args.empty?
+      type = options[:type] || java.lang.Double.java_class
+      parameters = [Cascading.fields(args),type].compact.to_java
+
+      Java::CascadingOperationAggregator::Sum.new(*parameters)
+    end
+
+    def aggregator_function(args, aggregator_klass)
+      options = args.extract_options!
+      ignore_values = options[:sql] ? [nil].to_java(java.lang.Object) : nil
+      parameters = [Cascading.fields(args), ignore_values].compact
+      aggregator_klass.new(*parameters)
+    end
+
+    def count_function(*args)
+      aggregator_function(args, Java::CascadingOperationAggregator::Count)
+    end
+
+    def average_function(*args)
+      aggregator_function(args, Java::CascadingOperationAggregator::Average)
+    end
+
+    def first_function(*args)
+      aggregator_function(args, Java::CascadingOperationAggregator::First)
+    end
+
+    def min_function(*args)
+      aggregator_function(args, Java::CascadingOperationAggregator::Min)
+    end
+
+    def max_function(*args)
+      aggregator_function(args, Java::CascadingOperationAggregator::Max)
+    end
+
+    def last_function(*args)
+      aggregator_function(args, Java::CascadingOperationAggregator::Last)
+    end
+
+    def regex_parser(*args)
+      options = args.extract_options!
+
+      pattern = args[0].to_s
+      fields = Cascading.fields(options[:fields])
+      groups = options[:groups].to_java(:int) if options[:groups]
+      parameters = [fields, pattern, groups].compact
+
+      Java::CascadingOperationRegex::RegexParser.new(*parameters)
+    end
+
+    def regex_splitter(*args)
+      options = args.extract_options!
+
+      fields = Cascading.fields(args)
+      pattern = options[:pattern].to_s
+      parameters = [fields, pattern].compact
+      Java::CascadingOperationRegex::RegexSplitter.new(*parameters)
+    end
+
+    def regex_split_generator(*args)
+      options = args.extract_options!
+
+      fields = Cascading.fields(args)
+      pattern = options[:pattern].to_s
+      parameters = [fields, pattern].compact
+      Java::CascadingOperationRegex::RegexSplitGenerator.new(*parameters)
+    end
+
+    def regex_generator(*args)
+      options = args.extract_options!
+
+      fields = Cascading.fields(args)
+      pattern = options[:pattern].to_s
+      parameters = [fields, pattern].compact
+      Java::CascadingOperationRegex::RegexGenerator.new(*parameters)
+    end
+
+    def expression_function(*args)
+      options = args.extract_options!
+
+      fields = Cascading.fields(args)
+      expression = options[:expression].to_s
+      parameters = options[:parameters]
+      parameter_names = []
+      parameter_types = []
+      if parameters.is_a? ::Hash
+        parameters.each do |name, type|
+          parameter_names << name
+          parameter_types << type
+        end
+        parameter_names = parameter_names.to_java(java.lang.String)
+        parameter_types = parameter_types.to_java(java.lang.Class)
+
+        arguments = [fields, expression, parameter_names, parameter_types].compact
+      elsif !parameters.nil?
+        arguments = [fields, expression, parameters.java_class].compact
+      else
+        arguments = [fields, expression, java.lang.String.java_class].compact
+      end
+
+      Java::CascadingOperationExpression::ExpressionFunction.new(*arguments)
+    end
+
+    def insert_function(*args)
+      options=args.extract_options!
+      fields = Cascading.fields(args)
+      values = options[:values]
+
+      parameters = [fields, to_java_comparable_array(values)].compact
+      Java::CascadingOperation::Insert.new(*parameters)
+    end
+
+    def to_java_comparable_array(arr)
+      (arr.map do |v|
+        case v.class
+        when Fixnum
+          java.lang.Integer.new(v)
+        when Float
+          java.lang.Double.new(v)
+        else
+          java.lang.String.new(v.to_s)
+        end
+      end).to_java(java.lang.Comparable)
+    end
+
+    def expression_filter(*args)
+      options = args.extract_options!
+      expression = (args[0] || options[:expression]).to_s
+      parameters = options[:parameters]
+      parameter_names = []
+      parameter_types = []
+      if parameters.is_a? ::Hash
+        parameters.each do |name, type|
+          parameter_names << name
+          parameter_types << type
+        end
+        parameter_names = parameter_names.to_java(java.lang.String)
+        parameter_types = parameter_types.to_java(java.lang.Class)
+
+        arguments = [expression, parameter_names, parameter_types].compact
+      elsif !parameters.nil?
+        arguments = [expression, parameters.java_class].compact
+      else
+        arguments = [expression, java.lang.String.java_class].compact
+      end
+
+      Java::CascadingOperationExpression::ExpressionFilter.new(*arguments)
+    end
+
+    def date_parser(field, format)
+      fields = fields(field)
+      Java::CascadingOperationText::DateParser.new(fields, format)
+    end
+
+    def date_formatter(fields, format, timezone=nil)
+      fields = fields(fields)
+      timezone = Java::JavaUtil::TimeZone.get_time_zone(timezone) if timezone
+      arguments = [fields, format, timezone].compact
+      Java::CascadingOperationText::DateFormatter.new(*arguments)
+    end
+
+    def regex_filter(*args)
+      options = args.extract_options!
+
+      pattern = args[0]
+      remove_match = options[:remove_match]
+      match_each_element = options[:match_each_element]
+      parameters = [pattern.to_s, remove_match, match_each_element].compact
+      Java::CascadingOperationRegex::RegexFilter.new(*parameters)
+    end
+
+    def regex_replace(*args)
+      options = args.extract_options!
+
+      fields = fields(args[0])
+      pattern = args[1]
+      replacement = args[2]
+      replace_all = options[:replace_all]
+
+      parameters = [fields, pattern.to_s, replacement.to_s, replace_all].compact
+      Java::CascadingOperationRegex::RegexReplace.new(*parameters)
+    end
+
+    def field_joiner(*args)
+      options = args.extract_options!
+      delimiter = options[:delimiter] || ','
+      fields = fields(options[:into])
+
+      parameters = [fields, delimiter].compact
+      Java::CascadingOperationText::FieldJoiner.new(*parameters)
+    end
+  end
+end

data/lib/cascading/scope.rb

@@ -0,0 +1,160 @@
+module Cascading
+  class Scope
+    attr_accessor :scope, :grouping_key_fields, :primary_key_fields, :grouping_primary_key_fields
+    @@scheme_keys = {}
+
+    def initialize(scope, params = {})
+      @scope = scope
+      @grouping_key_fields = fields(params[:grouping_key_fields] || [])
+      @primary_key_fields = fields(params[:primary_key_fields])
+      @grouping_primary_key_fields = fields(params[:grouping_primary_key_fields])
+    end
+
+    def copy
+      Scope.new(Java::CascadingFlow::Scope.new(@scope),
+          :grouping_key_fields => @grouping_key_fields,
+          :primary_key_fields => @primary_key_fields,
+          :grouping_primary_key_fields => @grouping_primary_key_fields
+      )
+    end
+
+    def self.register_scheme_key(scheme, primary_key)
+      @@scheme_keys[scheme] = primary_key
+    end
+
+    def self.empty_scope(name)
+      Scope.new(Java::CascadingFlow::Scope.new(name))
+    end
+
+    def self.tap_scope(tap, name)
+      java_scope = outgoing_scope_for(tap, java.util.HashSet.new)
+      # Taps and Pipes don't name their outgoing scopes like other FlowElements
+      java_scope.name = name
+      scope = Scope.new(java_scope,
+          :primary_key_fields => @@scheme_keys[tap.scheme.class],
+          :grouping_primary_key_fields => @@scheme_keys[tap.scheme.class]
+      )
+      vf, gf = scope.values_fields.to_a, scope.grouping_fields.to_a
+      pk, gpk = scope.primary_key_fields.to_a, scope.grouping_primary_key_fields.to_a
+      raise "Primary key must be a subset of available fields (primary key: #{pk.inspect}, values fields: #{vf.inspect})" unless vf & pk == pk
+      raise "Grouping primary key must be a subset of available fields (grouping primary key: #{gpk.inspect}, grouping fields: #{gf.inspect})" unless gf & gpk == gpk
+      scope
+    end
+
+    def self.outgoing_scope(flow_element, incoming_scopes, grouping_key_fields, every_applied)
+      java_scopes = incoming_scopes.compact.map{ |s| s.scope }
+      scope = Scope.new(outgoing_scope_for(flow_element, java.util.HashSet.new(java_scopes)),
+          :grouping_key_fields => grouping_key_fields
+      )
+      scope.grouping_primary_key_fields = fields(grouping_primary_key_fields(flow_element, incoming_scopes, scope))
+      scope.primary_key_fields = scope.grouping_primary_key_fields if every_applied
+      scope.primary_key_fields = fields(primary_key_fields(flow_element, incoming_scopes, scope)) unless every_applied
+      scope
+    end
+
+    def values_fields
+      @scope.out_values_fields
+    end
+
+    def grouping_fields
+      keys = @grouping_key_fields.to_a
+      grouping_fields = @scope.out_grouping_fields.to_a
+      # Overwrite key fields only
+      fields(keys + grouping_fields[keys.size..-1])
+    end
+
+    def to_s
+      kind = 'Unknown'
+      kind = 'Tap'   if @scope.tap?
+      kind = 'Group' if @scope.group?
+      kind = 'Each'  if @scope.each?
+      kind = 'Every' if @scope.every?
+      <<-END
+Scope name: #{@scope.name}
+  Kind: #{kind}
+  Argument selector: #{@scope.argument_selector}
+  Declared fields: #{@scope.declared_fields}
+  Grouping selectors: #{@scope.grouping_selectors}
+  Sorting selectors: #{@scope.sorting_selectors}
+  Out grouping
+    selector: #{@scope.out_grouping_selector}
+    fields: #{grouping_fields}
+    key fields: #{@grouping_key_fields}
+    primary key fields: #{@grouping_primary_key_fields}
+  Out values
+    selector: #{@scope.out_values_selector}
+    fields: #{values_fields}
+    primary key fields: #{@primary_key_fields}
+END
+    end
+
+    private
+
+    def self.outgoing_scope_for(flow_element, incoming_scopes)
+      begin
+        flow_element.outgoing_scope_for(incoming_scopes)
+      rescue NativeException => e
+        raise CascadingException.new(e, 'Exception computing outgoing scope')
+      end
+    end
+
+    def self.primary_key_fields(flow_element, incoming_scopes, scope)
+      case flow_element
+        when Java::CascadingPipe::Each
+          # assert incoming_scopes.size == 1
+          project_primary_key(incoming_scopes.first.primary_key_fields,
+                              incoming_scopes.first.values_fields.to_a,
+                              scope.values_fields.to_a)
+        when Java::CascadingPipe::Every
+          # assert incoming_scopes.size == 1
+          incoming_scopes.first.primary_key_fields
+        when Java::CascadingPipe::GroupBy
+          if incoming_scopes.size == 1
+            incoming_scopes.first.primary_key_fields
+          else
+            # We must clear the primary key when unioning multiple inputs.  If
+            # the programmer wants to preserve the primary key, they must use
+            # the primary override.
+            nil
+          end
+        when Java::CascadingPipe::CoGroup
+          # FIXME: assume grouping_key_fields are the same for all
+          # incoming_scopes.  Need join to give me names from all incoming
+          # scopes to perform rename on primary key fields.
+          union_fields(*incoming_scopes.map{ |s| s.primary_key_fields })
+        else raise "No primary key rules for FlowElement of type #{flow_element}"
+      end
+    end
+
+    def self.project_primary_key(primary_key, old_fields, new_fields)
+      return nil if primary_key.nil?
+      primary_key = primary_key.to_a
+      primary_key if (primary_key & new_fields) == primary_key
+    end
+
+    def self.grouping_primary_key_fields(flow_element, incoming_scopes, scope)
+      case flow_element
+        when Java::CascadingPipe::Each
+          # assert incoming_scopes.size == 1
+          project_primary_key(incoming_scopes.first.grouping_primary_key_fields,
+                              incoming_scopes.first.grouping_fields.to_a,
+                              scope.grouping_fields.to_a)
+        when Java::CascadingPipe::Every
+          # assert incoming_scopes.size == 1
+          incoming_scopes.first.grouping_primary_key_fields
+        when Java::CascadingPipe::GroupBy
+          scope.grouping_key_fields
+        when Java::CascadingPipe::CoGroup
+          scope.grouping_key_fields
+        else raise "No primary key rules for FlowElement of type #{flow_element}"
+      end
+    end
+  end
+
+  # Register default primary keys
+  begin
+    Scope.register_scheme_key(Java::CascadingScheme::TextLine, ['offset'])
+  rescue NameError => ne
+    puts 'WARNING: Could not register primary key for TextLine Scheme as it was not on the class path'
+  end
+end

data/lib/cascading.rb

@@ -0,0 +1,63 @@
+# Copyright 2009, Grégoire Marabout. All Rights Reserved.
+#
+# This is free software. Please see the LICENSE and COPYING files for details.
+
+require 'java'
+
+module Cascading
+  # :stopdoc:
+  VERSION = '0.0.4'
+  LIBPATH = ::File.expand_path(::File.dirname(__FILE__)) + ::File::SEPARATOR
+  PATH = ::File.dirname(LIBPATH) + ::File::SEPARATOR
+  CASCADING_HOME = ENV['CASCADING_HOME']
+  HADOOP_HOME = ENV['HADOOP_HOME']
+
+  # :startdoc:
+
+  # Returns the version string for the library.
+  #
+  def self.version
+    VERSION
+  end
+
+  # Returns the library path for the module. If any arguments are given,
+  # they will be joined to the end of the libray path using
+  # <tt>File.join</tt>.
+  #
+  def self.libpath( *args )
+    args.empty? ? LIBPATH : ::File.join(LIBPATH, args.flatten)
+  end
+
+  # Returns the lpath for the module. If any arguments are given,
+  # they will be joined to the end of the path using
+  # <tt>File.join</tt>.
+  #
+  def self.path( *args )
+    args.empty? ? PATH : ::File.join(PATH, args.flatten)
+  end
+
+  def self.require_all_jars(from = ::File.join(::File.dirname(__FILE__), "..", "jars"))
+    search_me = ::File.expand_path(
+        ::File.join(from, '**', '*.jar'))
+    Dir.glob(search_me).sort.each do |jar|
+      #puts "required: #{jar}"
+      require jar
+    end
+  end
+end
+
+Cascading.require_all_jars(Cascading::HADOOP_HOME) if Cascading::HADOOP_HOME
+Cascading.require_all_jars(Cascading::CASCADING_HOME) if Cascading::CASCADING_HOME
+
+require 'cascading/assembly'
+require 'cascading/base'
+require 'cascading/cascade'
+require 'cascading/cascading'
+require 'cascading/cascading_exception'
+require 'cascading/expr_stub'
+require 'cascading/flow'
+require 'cascading/operations'
+require 'cascading/scope'
+
+# include module to make them available at top package
+include Cascading

data/samples/branch.rb

@@ -0,0 +1,31 @@
+#! /usr/bin/env jruby
+
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'cascading'
+require 'samples/cascading'
+
+cascade 'branch' do
+  flow 'branch' do
+    source 'input', tap('samples/data/data2.txt')
+
+    assembly 'input' do
+      split 'line', ['name', 'score1', 'score2', 'id'], :pattern => /[.,]*\s+/
+
+      branch 'branch1' do
+        group_by 'score1' do
+          count
+        end
+      end
+
+      branch 'branch2' do
+        group_by 'score2' do
+          count
+        end
+      end
+    end
+
+    sink 'branch1', tap('output/branch1', :sink_mode => :replace)
+    sink 'branch2', tap('output/branch2', :sink_mode => :replace)
+  end
+end.complete(sample_properties)

data/samples/cascading.rb

@@ -0,0 +1,41 @@
+module Cascading
+  # Constructs properties to be passed to Flow#complete or Cascade#complete
+  # which will locate temporary Hadoop files in build/sample.  It is necessary
+  # to pass these properties only because the sample apps are invoked using
+  # JRuby's main method, which confuses the JobConf's attempt to find the
+  # containing jar.
+  def sample_properties
+    build_dir = 'build/sample/build'
+    `mkdir -p #{build_dir}`
+    tmp_dir = "build/sample/tmp"
+    `mkdir -p #{tmp_dir}`
+    log_dir = "build/sample/log"
+    `mkdir -p #{log_dir}`
+
+    # Local cluster settings
+    #java.lang.System.set_property("test.build.data", build_dir)
+    #java.lang.System.set_property("hadoop.tmp.dir", tmp_dir)
+    #java.lang.System.set_property("hadoop.log.dir", log_dir)
+    #conf = Java::OrgApacheHadoopConf::Configuration.new
+    #dfs = Java::OrgApacheHadoopDfs::MiniDFSCluster.new(conf, 4, true, nil);
+    #file_sys = dfs.file_system
+    #mr = Java::OrgApacheHadoopMapred::MiniMRCluster.new(4, file_sys.uri.to_string, 1)
+    #job_conf = mr.create_job_conf
+    #job_conf.set("mapred.child.java.opts", "-Xmx512m")
+    #job_conf.set("mapred.map.tasks.speculative.execution", "false")
+    #job_conf.set("mapred.reduce.tasks.speculative.execution", "false")
+
+    job_conf = Java::OrgApacheHadoopMapred::JobConf.new
+    job_conf.jar = build_dir
+    job_conf.set("test.build.data", build_dir)
+    job_conf.set("hadoop.tmp.dir", tmp_dir)
+    job_conf.set("hadoop.log.dir", log_dir)
+
+    job_conf.num_map_tasks = 4
+    job_conf.num_reduce_tasks = 1
+
+    properties = java.util.HashMap.new({})
+    Java::CascadingFlow::MultiMapReducePlanner.set_job_conf(properties, job_conf)
+    properties
+  end
+end

data/samples/copy.rb

@@ -0,0 +1,18 @@
+#! /usr/bin/env jruby
+$: << File.join(File.dirname(__FILE__), '..', 'lib')
+
+require 'cascading'
+require 'samples/cascading'
+
+cascade 'copy' do
+  flow 'copy' do
+    source 'input', tap('http://www.census.gov/genealogy/names/dist.all.last')
+
+    assembly 'input' do
+      rename 'line' => 'value'
+      reject 'value:string.indexOf("R") == -1'
+    end
+
+    sink 'input', tap('output/copy', :sink_mode => :replace)
+  end
+end.complete(sample_properties)