streaming_join 0.1.0 → 0.2.0

data/CHANGELOG

@@ -1,2 +1,7 @@
-* 2011-08-19 - fsf
+* 2011-08-22 - fsfiii
+- new: add JoinMapper class, intended to be used as the mapper portion
+- new: added examples/job_full which runs an entire job with both map and
+       reduce sides using the framework
+
+* 2011-08-19 - fsfiii
 - initial import

data/lib/streaming_join/join_mapper.rb

@@ -0,0 +1,71 @@
+class JoinMapper
+  def initialize
+    # use our own input field separator variable since the stock
+    # stream variable can't handle control characters
+    @sep_in  = ENV['streaming_join_input_field_separator'] || "\t"
+    @sep_in  = $1.hex.chr if @sep_in =~ /\A(?:\\u?)?(\d+)\Z/
+    @sep_out = ENV['stream_map_output_field_separator'] || "\t"
+    @sep_out = $1.hex.chr if @sep_in =~ /\A(?:\\u?)?(\d+)\Z/
+
+    @join = []
+  end
+
+  def report detail
+    STDERR.puts "reporter:counter:join,#{detail},1"
+  end
+
+  def add_side(file_re, *columns, &filter)
+    h = {
+      file_re: file_re,
+      columns: columns,
+      filter:  filter,
+      sep:     @sep_in,
+      side:    @join.size
+    }
+    @join << h
+    h
+  end
+
+  def add_opts(file_re, opts)
+    @join.each do |j|
+      next if j[:file_re] != file_re
+      j.merge! opts
+    end
+  end
+
+  def join_side
+    input_file = ENV['map_input_file']
+    @join.each do |j|
+      return j if input_file =~ j[:file_re]
+    end
+    raise "how do I handle input file '#{input_file}'?"
+  end
+
+  def process_stream(input = STDIN)
+    last_key = key = nil
+
+    j = join_side
+    cols   = j[:columns]
+    filter = j[:filter]
+    side   = j[:side]
+    sep    = j[:sep]
+
+    input.each do |line|
+      fields = line.chomp.split(sep, -1)
+
+      c = []
+      cols.each_with_index do |col,i|
+        value = fields[col]
+        break if i == 0 and value.nil? # can't have nil key
+        c << value
+      end
+      next if c.empty?
+
+      next if filter and not filter.call(c)
+
+      o = "#{c[0]}#{@sep_out}#{side}#{@sep_out}"
+      o << c[1...c.length].join(@sep_out)
+      puts o
+    end
+  end
+end

data/lib/streaming_join.rb

@@ -4,3 +4,4 @@ require 'streaming_join/right_outer_join.rb'
 require 'streaming_join/full_outer_join.rb'
 require 'streaming_join/cross_join.rb'
 require 'streaming_join/merge_rows.rb'
+require 'streaming_join/join_mapper.rb'

metadata

@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
   prerelease: false
   segments: 
   - 0
-  - 1
+  - 2
   - 0
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors: 
 - Frank Fejes
@@ -14,11 +14,11 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-08-19 00:00:00 -05:00
+date: 2011-08-22 00:00:00 -05:00
 default_executable: 
 dependencies: []
 
-description: Classes to process various joins in Hadoop Streaming.
+description: Classes to process various joins in Hadoop Map/Reduce Streaming.
 email: frank@fejes.net
 executables: []
 
@@ -36,6 +36,7 @@ files:
 - lib/streaming_join/full_outer_join.rb
 - lib/streaming_join/cross_join.rb
 - lib/streaming_join/merge_rows.rb
+- lib/streaming_join/join_mapper.rb
 has_rdoc: true
 homepage: https://github.com/fsfiii/streaming_join
 licenses: []
@@ -67,6 +68,6 @@ rubyforge_project: streaming_join
 rubygems_version: 1.3.7
 signing_key: 
 specification_version: 3
-summary: Classes to process joins in Hadoop Streaming
+summary: Classes to process joins in Hadoop Map/Reduce Streaming.
 test_files: []