streaming_join 0.1.0

data/CHANGELOG

@@ -0,0 +1,2 @@
+* 2011-08-19 - fsf
+- initial import

data/README

@@ -0,0 +1,23 @@
+streaming_join
+
+Ruby classes intended to be used in Hadoop Streaming reducers.  It
+has been tested with jruby 1.6+ and ruby 1.9.2+.
+
+Examples (found in the examples directory) use the test data and
+scenarios from here:
+
+http://en.wikipedia.org/wiki/Join_(SQL)
+
+The equivalent sql for each example is listed in its directory.  The
+supported join types are:
+
+inner_join
+cross_join
+left_outer_join
+right_outer_join
+full_outer_join
+merge_rows
+
+Please let me know if you find this software useful!
+
+--frank

data/lib/streaming_join/cross_join.rb

@@ -0,0 +1,30 @@
+require 'streaming_join/join'
+
+class CrossJoin < Join
+  def output
+    left,right = @join
+
+    left.each do |lk,lv|
+      right.each do |rk,rv|
+        o = "#{lk}#{@sep_out}#{lv}#{@sep_out}#{rk}#{@sep_out}#{rv}"
+        if block_given?
+          yield o
+        else
+          puts o
+        end
+      end
+    end
+  end
+
+  def process_stream(input = STDIN, &blk)
+    @join = [] # big memory, big prizes
+
+    input.each do |line|
+      key, side, value = line.chomp.split(@sep_in, 3)
+
+      (@join[side.to_i] ||= []) << [key, value]
+    end
+
+    output(&blk) if not @join.empty?
+  end
+end

data/lib/streaming_join/full_outer_join.rb

@@ -0,0 +1,59 @@
+require 'streaming_join/join'
+
+class FullOuterJoin < Join
+  def output key
+    report 'keys'
+
+    left,right = @join
+    if not left
+      report 'null left'
+    else
+      report 'left keys'
+    end
+    if not right
+      report 'null right'
+    else
+      report 'right keys'
+    end
+
+    #p left, right
+
+    # the number of columns on the sides can be passed in as env variables
+    # so that the full record with empty "" values can be displayed even if
+    # there is no match
+    cols_r = ENV['streaming_join_cols_right'].to_i
+    cols_l = ENV['streaming_join_cols_left'].to_i
+
+    left.each do |l|
+      if right
+        report 'left and right'
+        right.each do |r|
+          o = "#{key}#{@sep_out}#{l}#{@sep_out}#{r}"
+          block_given? ? (yield o) : (puts o)
+        end
+      elsif cols_r > 0
+        report 'null right'
+        o = "#{key}#{@sep_out}"
+        o << "#{l}#{@sep_out}#{Array.new(cols_r).join(@sep_out)}"
+        block_given? ? (yield o) : (puts o)
+      else
+        report 'null right'
+        o = "#{key}#{@sep_out}#{l}"
+        block_given? ? (yield o) : (puts o)
+      end
+    end if left
+
+    right.each do |r|
+      next if left
+      report 'null left'
+      if cols_l > 0
+        o = "#{key}#{@sep_out}"
+        o << "#{Array.new(cols_l).join(@sep_out)}#{@sep_out}#{r}"
+      else
+        o = "#{key}#{@sep_out}#{r}"
+      end
+
+      block_given? ? (yield o) : (puts o)
+    end if right
+  end
+end

data/lib/streaming_join/join.rb

@@ -0,0 +1,56 @@
+# base Join class (inner join)
+class Join
+  def initialize
+    @sep_in  = ENV['stream_map_output_field_separator'] || "\t"
+    @sep_out = ENV['streaming_join_output_separator'] || "\t"
+    @sep_out = $1.hex.chr if @sep_out =~ /\A(?:\\u?)?(\d+)\Z/
+  end
+
+  def report detail
+    STDERR.puts "reporter:counter:join,#{detail},1"
+  end
+
+  def output key
+    report 'keys'
+
+    left,right = @join
+    if not left
+      report 'null left'
+      return
+    elsif not right
+      report 'null right'
+      return
+    end
+
+    left.each do |l|
+      report 'left and right'
+      right.each do |r|
+        o = "#{key}#{@sep_out}#{l}#{@sep_out}#{r}"
+        if block_given?
+          yield o
+        else
+          puts o
+        end
+      end
+    end
+  end
+
+  def process_stream(input = STDIN, &blk)
+    last_key = key = nil
+    @join = []
+
+    input.each do |line|
+      key, side, value = line.chomp.split(@sep_in, 3)
+
+      if last_key and last_key != key
+        output(last_key, &blk)
+        @join = []
+      end
+
+      (@join[side.to_i] ||= []) << value
+      last_key = key
+    end
+
+    output(last_key, &blk) if key
+  end
+end

data/lib/streaming_join/left_outer_join.rb

@@ -0,0 +1,39 @@
+require 'streaming_join/join'
+
+class LeftOuterJoin < Join
+  def output key
+    report 'keys'
+
+    left,right = @join
+    if not left
+      report 'null left'
+      return
+    else
+      report 'left keys'
+    end
+
+    # the number of columns on the right can be passed in as an env variable
+    # so that the full record with empty "" values can be displayed even if
+    # there is no match
+    cols_r = ENV['streaming_join_cols_right'].to_i
+
+    left.each do |l|
+      if right
+        report 'left and right'
+        right.each do |r|
+          o = "#{key}#{@sep_out}#{l}#{@sep_out}#{r}"
+          block_given? ? (yield o) : (puts o)
+        end
+      elsif cols_r > 0
+        report 'null right'
+        o = "#{key}#{@sep_out}"
+        o << "#{l}#{@sep_out}#{Array.new(cols_r).join(@sep_out)}"
+        block_given? ? (yield o) : (puts o)
+      else
+        report 'null right'
+        o = "#{key}#{@sep_out}#{l}"
+        block_given? ? (yield o) : (puts o)
+      end
+    end
+  end
+end

data/lib/streaming_join/merge_rows.rb

@@ -0,0 +1,29 @@
+require 'streaming_join'
+
+class MergeRows < Join
+  def output key
+    report 'keys'
+
+    o = "#{key}#{@sep_out}#{@join.join(@sep_out)}"
+    block_given? ? (yield o) : (puts o)
+  end
+
+  def process_stream(input = STDIN)
+    last_key = key = nil
+    @join = []
+
+    input.each do |line|
+      key, value = line.chomp.split(@sep_in, 2)
+
+      if last_key and last_key != key
+        output last_key
+        @join = []
+      end
+
+      @join << value
+      last_key = key
+    end
+
+    output last_key if key
+  end
+end

data/lib/streaming_join/right_outer_join.rb

@@ -0,0 +1,39 @@
+require 'streaming_join/join'
+
+class RightOuterJoin < Join
+  def output key
+    report 'keys'
+
+    left,right = @join
+    if not right
+      report 'null right'
+      return
+    else
+      report 'right keys'
+    end
+
+    # the number of columns on the left can be passed in as an env variable
+    # so that the full record with empty "" values can be displayed even if
+    # there is no match
+    cols_l = ENV['streaming_join_cols_left'].to_i
+
+    right.each do |r|
+      if left
+        report 'left and right'
+        left.each do |l|
+          o = "#{key}#{@sep_out}#{l}#{@sep_out}#{r}"
+          block_given? ? (yield o) : (puts o)
+        end
+      elsif cols_l > 0
+        report 'null left'
+        o = "#{key}#{@sep_out}"
+        o << "#{Array.new(cols_l).join(@sep_out)}#{@sep_out}#{r}"
+        block_given? ? (yield o) : (puts o)
+      else
+        report 'null left'
+        o = "#{key}#{@sep_out}#{r}"
+        block_given? ? (yield o) : (puts o)
+      end
+    end
+  end
+end

data/lib/streaming_join.rb

@@ -0,0 +1,6 @@
+require 'streaming_join/join.rb'
+require 'streaming_join/left_outer_join.rb'
+require 'streaming_join/right_outer_join.rb'
+require 'streaming_join/full_outer_join.rb'
+require 'streaming_join/cross_join.rb'
+require 'streaming_join/merge_rows.rb'

metadata

@@ -0,0 +1,72 @@
+--- !ruby/object:Gem::Specification 
+name: streaming_join
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Frank Fejes
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-08-19 00:00:00 -05:00
+default_executable: 
+dependencies: []
+
+description: Classes to process various joins in Hadoop Streaming.
+email: frank@fejes.net
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README
+- CHANGELOG
+- lib/streaming_join.rb
+- lib/streaming_join/join.rb
+- lib/streaming_join/left_outer_join.rb
+- lib/streaming_join/right_outer_join.rb
+- lib/streaming_join/full_outer_join.rb
+- lib/streaming_join/cross_join.rb
+- lib/streaming_join/merge_rows.rb
+has_rdoc: true
+homepage: https://github.com/fsfiii/streaming_join
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: streaming_join
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Classes to process joins in Hadoop Streaming
+test_files: []
+