streaming_join 0.2.0 → 0.3.0

data/CHANGELOG

@@ -1,3 +1,14 @@
+* 2011-08-27 - fsfiii
+- update: Redesigned join logic so as to not read individual keyspaces
+          into memory since this simply won't scale.  Now only the left
+          side of a keyspace will be placed into memory (one keyspace at
+          a time, only) and the right side will be processed as values are
+          read in.  This now requires KeyFieldBasedPartioner so that the
+          values are sorted left side first.
+- clean: removed example/job and renamed example/job_full to example/job
+- new: add opts passing to Join class
+- new: add :report opt for Join class to control reporting of key stats
+
 * 2011-08-22 - fsfiii
 - new: add JoinMapper class, intended to be used as the mapper portion
 - new: added examples/job_full which runs an entire job with both map and

data/README

@@ -18,6 +18,64 @@ right_outer_join
 full_outer_join
 merge_rows
 
+API
+
+As with most map/reduce jobs, there are two basic components:
+
+Mapper
+
+The mapper in a streaming_join job outputs records of the form:
+
+key TAB side_index TAB value
+
+The "left" side uses side_index 0 while the right side would use 1.
+
+In a simple join, here is one example mapper output:
+
+1	0	Matsumoto
+2	0	Wall
+3	0	van Rossum
+
+Here is the other side's output:
+
+1	1	Ruby	1995
+2	1	Perl	1987
+4	1	Clojure	2007
+
+Reducer
+
+Depending on the join style, the reducer will emit combined records.  Using
+the above mapper output, an inner join would end up like:
+
+1	Matsumoto	Ruby
+2	Wall	Perl
+
+The code would look something like this:
+
+Mapper
+
+require 'streaming_join'
+
+j = JoinMapper.new
+j.add_side(/_left/, 0, 1)
+j.add_side(/_right/, 0, 1)
+j.process_stream
+
+Reducer
+
+require 'streaming_join'
+
+Join.new.process_stream
+
+See examples/job for more detail and comments.
+
+Current Limitations
+
+- As each key is processed in a reducer, the left side of that single
+  keyspace must fit into memory.  So, when in doubt, put the smaller
+  table on the left.
+- Only two tables can be joined in a single job.
+
 Please let me know if you find this software useful!
 
 --frank

data/lib/streaming_join/full_outer_join.rb

@@ -1,59 +1,29 @@
 require 'streaming_join/join'
 
 class FullOuterJoin < Join
-  def output key
-    report 'keys'
-
-    left,right = @join
-    if not left
-      report 'null left'
-    else
-      report 'left keys'
-    end
-    if not right
-      report 'null right'
-    else
-      report 'right keys'
-    end
-
-    #p left, right
-
-    # the number of columns on the sides can be passed in as env variables
-    # so that the full record with empty "" values can be displayed even if
-    # there is no match
-    cols_r = ENV['streaming_join_cols_right'].to_i
-    cols_l = ENV['streaming_join_cols_left'].to_i
+  def null_right key, left
+    o = "#{key}#{@sep_out}"
 
     left.each do |l|
-      if right
-        report 'left and right'
-        right.each do |r|
-          o = "#{key}#{@sep_out}#{l}#{@sep_out}#{r}"
-          block_given? ? (yield o) : (puts o)
-        end
-      elsif cols_r > 0
-        report 'null right'
-        o = "#{key}#{@sep_out}"
-        o << "#{l}#{@sep_out}#{Array.new(cols_r).join(@sep_out)}"
-        block_given? ? (yield o) : (puts o)
+      if @cols_r > 0
+        o << "#{l}#{@sep_out}#{Array.new(@cols_r).join(@sep_out)}"
       else
-        report 'null right'
         o = "#{key}#{@sep_out}#{l}"
-        block_given? ? (yield o) : (puts o)
       end
-    end if left
+    end
 
-    right.each do |r|
-      next if left
-      report 'null left'
-      if cols_l > 0
-        o = "#{key}#{@sep_out}"
-        o << "#{Array.new(cols_l).join(@sep_out)}#{@sep_out}#{r}"
-      else
-        o = "#{key}#{@sep_out}#{r}"
-      end
+    block_given? ? (yield o) : (puts o)
+  end
+
+  def null_left key, right
+    o = "#{key}#{@sep_out}"
+
+    if @cols_l > 0
+      o << "#{Array.new(@cols_l).join(@sep_out)}#{@sep_out}#{right}"
+    else
+      o = "#{key}#{@sep_out}#{right}"
+    end
 
-      block_given? ? (yield o) : (puts o)
-    end if right
+    block_given? ? (yield o) : (puts o)
   end
 end

data/lib/streaming_join/join.rb

@@ -1,56 +1,81 @@
 # base Join class (inner join)
 class Join
-  def initialize
+  def initialize opts = {}
     @sep_in  = ENV['stream_map_output_field_separator'] || "\t"
     @sep_out = ENV['streaming_join_output_separator'] || "\t"
     @sep_out = $1.hex.chr if @sep_out =~ /\A(?:\\u?)?(\d+)\Z/
+    @report  = opts.fetch :report, true
+    @cols_l  = ENV['streaming_join_cols_left'].to_i
+    @cols_r  = ENV['streaming_join_cols_right'].to_i
   end
 
   def report detail
+    return if not @report
     STDERR.puts "reporter:counter:join,#{detail},1"
   end
 
-  def output key
-    report 'keys'
-
-    left,right = @join
-    if not left
-      report 'null left'
-      return
-    elsif not right
-      report 'null right'
-      return
-    end
-
+  def output key, left, right
     left.each do |l|
-      report 'left and right'
-      right.each do |r|
-        o = "#{key}#{@sep_out}#{l}#{@sep_out}#{r}"
-        if block_given?
-          yield o
-        else
-          puts o
-        end
+      o = "#{key}#{@sep_out}#{l}#{@sep_out}#{right}"
+      if block_given?
+        yield o
+      else
+        puts o
       end
     end
   end
 
+  def null_left key, right
+  end
+
+  def null_right key, left
+  end
+
   def process_stream(input = STDIN, &blk)
     last_key = key = nil
-    @join = []
+    last_side = nil
+    left = []
 
     input.each do |line|
       key, side, value = line.chomp.split(@sep_in, 3)
 
-      if last_key and last_key != key
-        output(last_key, &blk)
-        @join = []
+      report 'keys' if last_key != key
+
+      side = side.to_i
+      if side == 0
+        # if we are on the left side and just processed the left side
+        # of another key, we didn't get any right side records
+        if last_key != key and last_side == 0
+          report 'null right'
+          null_right last_key, left
+        end
+
+        if last_key != key
+          left = []
+        end
+        left << value
+      else
+        # if we're in a new key and the first record is a right side
+        # record, that means we never processed a left side
+        if not last_key or last_key != key or left.empty?
+          report 'null left' if last_key != key
+          null_left key, value
+          left = []
+        else
+          report 'left and right' if last_side == 0
+          output key, left, value
+        end
       end
 
-      (@join[side.to_i] ||= []) << value
+      last_side = side
       last_key = key
     end
 
-    output(last_key, &blk) if key
+    # if the last processed record is on the left, there is
+    # not going to be a right side
+    if last_side == 0
+      report 'null right'
+      null_right(key, left)
+    end
   end
 end

data/lib/streaming_join/left_outer_join.rb

@@ -1,39 +1,17 @@
 require 'streaming_join/join'
 
 class LeftOuterJoin < Join
-  def output key
-    report 'keys'
-
-    left,right = @join
-    if not left
-      report 'null left'
-      return
-    else
-      report 'left keys'
-    end
-
-    # the number of columns on the right can be passed in as an env variable
-    # so that the full record with empty "" values can be displayed even if
-    # there is no match
-    cols_r = ENV['streaming_join_cols_right'].to_i
+  def null_right key, left
+    o = "#{key}#{@sep_out}"
 
     left.each do |l|
-      if right
-        report 'left and right'
-        right.each do |r|
-          o = "#{key}#{@sep_out}#{l}#{@sep_out}#{r}"
-          block_given? ? (yield o) : (puts o)
-        end
-      elsif cols_r > 0
-        report 'null right'
-        o = "#{key}#{@sep_out}"
-        o << "#{l}#{@sep_out}#{Array.new(cols_r).join(@sep_out)}"
-        block_given? ? (yield o) : (puts o)
+      if @cols_r > 0
+        o << "#{l}#{@sep_out}#{Array.new(@cols_r).join(@sep_out)}"
       else
-        report 'null right'
         o = "#{key}#{@sep_out}#{l}"
-        block_given? ? (yield o) : (puts o)
       end
     end
+
+    block_given? ? (yield o) : (puts o)
   end
 end

data/lib/streaming_join/right_outer_join.rb

@@ -1,39 +1,15 @@
 require 'streaming_join/join'
 
 class RightOuterJoin < Join
-  def output key
-    report 'keys'
+  def null_left key, right
+    o = "#{key}#{@sep_out}"
 
-    left,right = @join
-    if not right
-      report 'null right'
-      return
+    if @cols_l > 0
+      o << "#{Array.new(@cols_l).join(@sep_out)}#{@sep_out}#{right}"
     else
-      report 'right keys'
+      o = "#{key}#{@sep_out}#{right}"
     end
 
-    # the number of columns on the left can be passed in as an env variable
-    # so that the full record with empty "" values can be displayed even if
-    # there is no match
-    cols_l = ENV['streaming_join_cols_left'].to_i
-
-    right.each do |r|
-      if left
-        report 'left and right'
-        left.each do |l|
-          o = "#{key}#{@sep_out}#{l}#{@sep_out}#{r}"
-          block_given? ? (yield o) : (puts o)
-        end
-      elsif cols_l > 0
-        report 'null left'
-        o = "#{key}#{@sep_out}"
-        o << "#{Array.new(cols_l).join(@sep_out)}#{@sep_out}#{r}"
-        block_given? ? (yield o) : (puts o)
-      else
-        report 'null left'
-        o = "#{key}#{@sep_out}#{r}"
-        block_given? ? (yield o) : (puts o)
-      end
-    end
+    block_given? ? (yield o) : (puts o)
   end
 end

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 2
+  - 3
   - 0
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors: 
 - Frank Fejes
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-08-22 00:00:00 -05:00
+date: 2011-08-27 00:00:00 -05:00
 default_executable: 
 dependencies: []