cascading.jruby 0.0.4

data/lib/cascading/base.rb

@@ -0,0 +1,63 @@
+# Copyright 2009, Grégoire Marabout. All Rights Reserved.
+#
+# This is free software. Please see the LICENSE and COPYING files for details.
+
+module Cascading
+  class Node
+    attr_accessor :name, :parent, :children, :last_child
+
+    def initialize(name, parent)
+      @name = name
+      @parent = parent
+      @children = {}
+      @last_child = nil
+    end
+
+    def add_child(node)
+      @children[node.name] = node
+      @last_child = node
+      node
+    end
+
+    def find_child(name)
+      children.each do |child_name, child|
+        return child if child_name == name
+        result = child.find_child(name)
+        return result if result
+      end
+      return nil
+    end
+  end
+
+  # A module to add auto-registration capability
+  module Registerable
+    def all
+      @registered.nil? ? [] : @registered.values
+    end
+
+    def get(key)
+      if key.is_a? self
+        return key
+      else
+        @registered ||= {}
+        return @registered[key]
+      end
+    end
+
+    def reset
+      @registered.clear if @registered
+    end
+
+    def add(name, instance)
+      @registered ||= {}
+      @registered[name] = instance
+    end
+
+    private
+
+    def registered
+      @registered ||= {}
+      @registered
+    end
+  end
+end

data/lib/cascading/cascade.rb

@@ -0,0 +1,63 @@
+# Copyright 2009, Grégoire Marabout. All Rights Reserved.
+#
+# This is free software. Please see the LICENSE and COPYING files for details.
+
+require 'cascading/base'
+require 'yaml'
+
+module Cascading
+  class Cascade < Cascading::Node
+    extend Registerable
+
+    def initialize(name)
+      super(name, nil) # A Cascade cannot have a parent
+      self.class.add(name, self)
+    end
+
+    def flow(name, &block)
+      raise "Could not build flow '#{name}'; block required" unless block_given?
+      flow = Flow.new(name, self)
+      add_child(flow)
+      flow.instance_eval(&block)
+      flow
+    end
+
+    def draw(dir, properties = nil)
+      @children.each do |name, flow|
+        flow.connect(properties).writeDOT("#{dir}/#{name}.dot")
+      end
+    end
+
+    def sink_metadata
+      @children.inject({}) do |sink_fields, (name, flow)|
+        sink_fields[name] = flow.sink_metadata
+        sink_fields
+      end
+    end
+
+    def write_sink_metadata(file_name)
+      File.open(file_name, 'w') do |file|
+        YAML.dump(sink_metadata, file)
+      end
+    end
+
+    def complete(properties = nil)
+      begin
+        Java::CascadingCascade::CascadeConnector.new.connect(name, make_flows(@children, properties)).complete
+      rescue NativeException => e
+        raise CascadingException.new(e, 'Error completing cascade')
+      end
+    end
+
+    private
+
+    def make_flows(flows, properties)
+      flow_instances = flows.map do |name, flow|
+        cascading_flow = flow.connect(properties)
+        flow.listeners.each { |l| cascading_flow.addListener(l) }
+        cascading_flow
+      end
+      flow_instances.to_java(Java::CascadingFlow::Flow)
+    end
+  end
+end

data/lib/cascading/cascading.rb

@@ -0,0 +1,134 @@
+# Copyright 2009, Grégoire Marabout. All Rights Reserved.
+#
+# This is free software. Please see the LICENSE and COPYING files for details.
+
+require 'cascading/expr_stub'
+
+module Cascading
+  JAVA_TYPE_MAP = {
+    :int => java.lang.Integer.java_class, :long => java.lang.Long.java_class,
+    :bool => java.lang.Boolean.java_class, :double => java.lang.Double.java_class,
+    :float => java.lang.Float.java_class, :string => java.lang.String.java_class,
+  }
+
+  def cascade(name, &block)
+    raise "Could not build cascade '#{name}'; block required" unless block_given?
+    cascade = Cascade.new(name)
+    cascade.instance_eval(&block)
+    cascade
+  end
+
+  # For applications built of Flows with no Cascades
+  def flow(name, &block)
+    flow = Flow.new(name, nil)
+    flow.instance_eval(&block)
+    flow
+  end
+
+  def expr(s)
+    return s if s.kind_of?(ExprStub)
+    ExprStub.new(s)
+  end
+
+  # Creates a cascading.tuple.Fields instance from a string or an array of strings.
+  def fields(fields)
+    if fields.nil?
+      return nil
+    elsif fields.is_a? Java::CascadingTuple::Fields
+      return fields
+    elsif fields.is_a? ::Array
+      if fields.size == 1
+        return fields(fields[0])
+      end
+      raise "Fields cannot be nil: #{fields.inspect}" if fields.include?(nil)
+    end
+    return Java::CascadingTuple::Fields.new([fields].flatten.map{ |f| f.kind_of?(Fixnum) && JRUBY_VERSION > '1.2.0' ? f.to_java(:int) : f }.to_java(java.lang.Comparable))
+  end
+
+  def all_fields
+    Java::CascadingTuple::Fields::ALL
+  end
+
+  def union_fields(*fields)
+    fields(fields.inject([]){ |acc, arr| acc | arr.to_a })
+  end
+
+  def difference_fields(*fields)
+    fields(fields[1..-1].inject(fields.first.to_a){ |acc, arr| acc - arr.to_a })
+  end
+
+  def copy_fields(fields)
+    fields.select(all_fields)
+  end
+
+  def dedup_fields(*fields)
+    raise 'Can only be applied to declarators' unless fields.all?{ |f| f.is_declarator? }
+    fields(dedup_field_names(*fields.map{ |f| f.to_a }))
+  end
+
+  def dedup_field_names(*names)
+    names.inject([]) do |acc, arr|
+      acc + arr.map{ |e| search_field_name(acc, e) }
+    end
+  end
+
+  def search_field_name(names, candidate)
+    names.include?(candidate) ? search_field_name(names, "#{candidate}_") : candidate
+  end
+
+  def last_grouping_fields
+    Java::CascadingTuple::Fields::VALUES
+  end
+
+  def results_fields
+    Java::CascadingTuple::Fields::RESULTS
+  end
+
+  # Creates a c.s.TextLine scheme instance from the specified fields.
+  def text_line_scheme(*fields)
+    unless fields.empty?
+      fields = fields(fields)
+      return Java::CascadingScheme::TextLine.new(fields)
+    else
+      return Java::CascadingScheme::TextLine.new
+    end
+  end
+
+  # Creates a c.s.SequenceFile scheme instance from the specified fields.
+  def sequence_file_scheme(*fields)
+    unless fields.empty?
+      fields = fields(fields)
+      return Java::CascadingScheme::SequenceFile.new(fields)
+    else
+      return Java::CascadingScheme::SequenceFile.new(all_fields)
+    end
+  end
+
+  def multi_tap(*taps)
+    Java::CascadingTap::MultiTap.new(taps.to_java("cascading.tap.Tap"))
+  end
+
+  # Generic method for creating taps.
+  # It expects a ":kind" argument pointing to the type of tap to create.
+  def tap(*args)
+    opts = args.extract_options!
+    path = args.empty? ? opts[:path] : args[0]
+    scheme = opts[:scheme] || text_line_scheme
+    sink_mode = opts[:sink_mode] || :keep
+    sink_mode = case sink_mode
+      when :keep, 'keep'       then Java::CascadingTap::SinkMode::KEEP
+      when :replace, 'replace' then Java::CascadingTap::SinkMode::REPLACE
+      when :append, 'append'   then Java::CascadingTap::SinkMode::APPEND
+      else raise "Unrecognized sink mode '#{sink_mode}'"
+    end
+    fs = opts[:kind] || :hfs
+    klass = case fs
+      when :hfs, 'hfs' then Java::CascadingTap::Hfs
+      when :dfs, 'dfs' then Java::CascadingTap::Dfs
+      when :lfs, 'lfs' then Java::CascadingTap::Lfs
+      else raise "Unrecognized kind of tap '#{fs}'"
+    end
+    parameters = [scheme, path, sink_mode]
+    klass.new(*parameters)
+  end
+end

data/lib/cascading/cascading_exception.rb

@@ -0,0 +1,30 @@
+# Wrapper meant for NativeExceptions that wrap exceptions from Cascading.  The
+# trouble is that the combined stack traces are so long, printing them case
+# actually omit locations in the cascading.jruby or application code that
+# matter, leaving you with no information about the source of the error.  This
+# class just swallows all the nested exceptions, printing their message, while
+# giving you a direct route into JRuby code to the cause of the problem.
+class CascadingException < StandardError
+  def initialize(native_exception, message)
+    @ne = native_exception
+    super("#{message}\n#{trace_causes(@ne, 1)}")
+  end
+
+  def cause(depth)
+    fetch_cause(@ne, depth)
+  end
+
+  private
+
+  def fetch_cause(ne, depth)
+    return ne if depth <= 1
+    fetch_cause(ne.cause, depth - 1)
+  end
+
+  def trace_causes(ne, depth)
+    return unless ne
+    trace = "Cause #{depth}: #{ne}\n"
+    trace += ne.stack_trace.map { |e| "  at #{e.class_name}.#{e.method_name}(#{e.file_name}:#{e.line_number})" }.join("\n") + "\n" if ne.respond_to?(:stack_trace)
+    trace += "#{trace_causes(ne.cause, depth + 1)}"
+  end
+end

data/lib/cascading/expr_stub.rb

@@ -0,0 +1,33 @@
+class ExprStub
+  attr_accessor :expression, :types
+
+  def initialize(st)
+    @expression = st.dup
+    @types = {}
+
+    # Simple regexp based parser for types
+
+    JAVA_TYPE_MAP.each do |sym, klass|
+      @expression.gsub!(/[A-Za-z0-9_]+:#{sym.to_s}/) do |match|
+        name = match.split(/:/).first.gsub(/\s+/, "")
+        @types[name] = klass
+        match.gsub(/:#{sym.to_s}/, "")
+      end
+    end
+  end
+
+  def self.split_hash(h)
+    keys, values = h.keys.sort, []
+    keys.each do |key|
+      values << h[key]
+    end
+    [keys, values]
+  end
+
+  def self.split_names_and_types(expr_types)
+    names, types = split_hash(expr_types)
+    names = names.to_java(java.lang.String)
+    types = types.to_java(java.lang.Class)
+    [names, types]
+  end
+end

data/lib/cascading/ext/array.rb

@@ -0,0 +1,15 @@
+# ext.rb : some extensions to basic types
+#
+# Copyright 2009, Grégoire Marabout. All Rights Reserved.
+#
+# This is free software. Please see the LICENSE and COPYING files for details.
+
+class Array
+  def extract_options!
+     last.is_a?(::Hash) ? pop : {}
+  end
+  
+  def extract_options
+     last.is_a?(::Hash) ? last : {}
+  end
+end

data/lib/cascading/flow.rb

@@ -0,0 +1,168 @@
+# Copyright 2009, Grégoire Marabout. All Rights Reserved.
+#
+# This is free software. Please see the LICENSE and COPYING files for details.
+
+require 'cascading/assembly'
+
+module Cascading
+  class Flow < Cascading::Node
+    extend Registerable
+
+    attr_accessor :properties, :sources, :sinks, :outgoing_scopes, :listeners
+
+    def initialize(name, parent)
+      @properties, @sources, @sinks, @outgoing_scopes, @listeners = {}, {}, {}, {}, []
+      super(name, parent)
+      self.class.add(name, self)
+    end
+
+    def assembly(name, &block)
+      raise "Could not build assembly '#{name}'; block required" unless block_given?
+      assembly = Assembly.new(name, self, @outgoing_scopes)
+      add_child(assembly)
+      assembly.instance_eval(&block)
+      assembly
+    end
+
+    # Create a new sink for this flow, with the specified name.
+    # "tap" can be either a tap (see Cascading.tap) or a string that will
+    # reference a path.
+    def sink(*args)
+      if (args.size == 2)
+        @sinks[args[0]] = args[1]
+      elsif (args.size == 1)
+        @sinks[@name] =  args[0]
+      end
+    end
+
+    # Create a new source for this flow, with the specified name.
+    # "tap" can be either a tap (see Cascading.tap) or a string that will
+    # reference a path.
+    def source(*args)
+      if (args.size == 2)
+        @sources[args[0]] = args[1]
+        @outgoing_scopes[args[0]] = Scope.tap_scope(args[1], args[0])
+      elsif (args.size == 1)
+        @sources[@name] = args[0]
+        @outgoing_scopes[@name] = Scope.empty_scope(@name)
+      end
+    end
+
+    def scope(name = nil)
+      raise 'Must specify name if no children have been defined yet' unless name || last_child
+      name ||= last_child.name
+      @outgoing_scopes[name]
+    end
+
+    def debug_scope(name = nil)
+      scope = scope(name)
+      name ||= last_child.name
+      puts "Scope for '#{name}':\n  #{scope}"
+    end
+
+    def sink_metadata
+      @sinks.keys.inject({}) do |sink_metadata, sink_name|
+        raise "Cannot sink undefined assembly '#{sink_name}'" unless @outgoing_scopes[sink_name]
+        sink_metadata[sink_name] = {
+          :field_names => @outgoing_scopes[sink_name].values_fields.to_a,
+          :primary_key => @outgoing_scopes[sink_name].primary_key_fields.to_a
+        }
+        sink_metadata
+      end
+    end
+
+    # TODO: support all codecs, support list of codecs
+    def compress_output(codec, type)
+      properties['mapred.output.compress'] = 'true'
+      properties['mapred.output.compression.codec'] = case codec
+        when :default then Java::OrgApacheHadoopIoCompress::DefaultCodec.java_class.name
+        when :gzip then Java::OrgApacheHadoopIoCompress::GzipCodec.java_class.name
+        else raise "Codec #{codec} not yet supported by cascading.jruby"
+        end
+      properties['mapred.output.compression.type'] = case type
+        when :none   then Java::OrgApacheHadoopIo::SequenceFile::CompressionType::NONE.to_s
+        when :record then Java::OrgApacheHadoopIo::SequenceFile::CompressionType::RECORD.to_s
+        when :block  then Java::OrgApacheHadoopIo::SequenceFile::CompressionType::BLOCK.to_s
+        else raise "Compression type '#{type}' not supported"
+        end
+    end
+
+    def set_spill_threshold(threshold)
+      properties['cascading.cogroup.spill.threshold'] = threshold.to_s
+    end
+
+    def add_file_to_distributed_cache(file)
+      add_to_distributed_cache(file, "mapred.cache.files")
+    end
+
+    def add_archive_to_distributed_cache(file)
+      add_to_distributed_cache(file, "mapred.cache.archives")
+    end
+
+    def add_listener(listener)
+      @listeners << listener
+    end
+
+    def emr_local_path_for_distributed_cache_file(file)
+      # NOTE this needs to be *appended* to the property mapred.local.dir
+      if file =~ /^s3n?:\/\//
+        # EMR
+        "/taskTracker/archive/#{file.gsub(/^s3n?:\/\//, "")}"
+      else
+        # Local
+        file
+      end
+    end
+
+    def add_to_distributed_cache(file, property)
+      v = properties[property]
+
+      if v
+        properties[property] = [v.split(/,/), file].flatten.join(",")
+      else
+        properties[property] = file
+      end
+    end
+
+    def connect(properties = nil)
+      properties = java.util.HashMap.new(properties || @properties)
+      Java::CascadingFlow::FlowConnector.new(properties).connect(
+        name,
+        make_tap_parameter(@sources),
+        make_tap_parameter(@sinks),
+        make_pipes
+      )
+    end
+
+    def complete(properties = nil)
+      begin
+        flow = connect(properties)
+        @listeners.each { |l| flow.addListener(l) }
+        flow.complete
+      rescue NativeException => e
+        raise CascadingException.new(e, 'Error completing flow')
+      end
+    end
+
+    private
+
+    def make_tap_parameter(taps)
+      taps.inject({}) do |map, (name, tap)|
+        assembly = find_child(name)
+        raise "Could not find assembly '#{name}' to connect to tap: #{tap}" unless assembly
+
+        map[assembly.tail_pipe.name] = tap
+        map
+      end
+    end
+
+    def make_pipes
+      @sinks.inject([]) do |pipes, (name, sink)|
+        assembly = find_child(name)
+        raise "Could not find assembly '#{name}' to make pipe for sink: #{sink}" unless assembly
+        pipes << assembly.tail_pipe
+        pipes
+      end.to_java(Java::CascadingPipe::Pipe)
+    end
+  end
+end