rubydoop 0.0.2

data/lib/rubydoop.rb

@@ -0,0 +1,60 @@
+def map(&proc)
+  @map = proc
+end
+
+def reduce(&proc)
+  @reduce = proc
+end
+
+def emit(key, value)
+  puts [key, value].join("\t")
+end
+
+at_exit do
+  HADOOP_HOME ||= ENV['HADOOP_HOME'] || '/usr/local/hadoop'
+  case ARGV.first
+  when 'start'
+    cmd = <<-EOC
+      hadoop fs -rmr output
+      hadoop jar #{HADOOP_HOME}/contrib/streaming/hadoop-*-streaming.jar\\
+        -inputformat org.apache.hadoop.mapred.KeyValueTextInputFormat\\
+        -output output -input input\\
+        -file #{File.expand_path __FILE__} \\
+        -file #{File.expand_path $0} \\
+        -mapper "#{File.basename $0} map" \\
+        -reducer "#{File.basename $0} reduce"
+    EOC
+    puts cmd
+    exec cmd
+  when 'map'
+    while line = STDIN.gets
+      if line =~ /^([^\t]+)\t(.+)$/
+        @map.call $1, $2
+      end
+    end
+  when 'reduce'
+    key, values = nil, []
+    while line = STDIN.gets
+      if line =~ /^([^\t]+)\t(.+)$/
+        thiskey, thisvalue = $1, $2
+        if key != thiskey && key
+          @reduce.call key, values
+          key, values = nil, []
+        end
+        key = thiskey
+        values << thisvalue
+      end
+    end
+  when 'simulate'
+    raise unless File.exists?(ARGV.last)
+    exec "cat #{ARGV.last} | #{$0} map | sort | #{$0} reduce"
+  else
+    STDERR.puts <<-EOM
+Please run "#{$0} COMMAND", where COMMAND is one of the following:
+\tstart
+\tmap
+\treduce
+EOM
+    exit -1
+  end
+end

data/lib/rubydoop/minitest.rb

@@ -0,0 +1,23 @@
+require 'open3'
+
+class Rubydoop
+  class MiniTest <  MiniTest::Unit::TestCase
+
+    def map(script, input)
+      Open3.pipeline_rw("./#{script} map", "sort") do |in_io, out_io, wt|
+        in_io.print input
+        in_io.close
+        out_io.readlines
+      end
+    end
+
+    def mapreduce(script, input)
+      Open3.pipeline_rw("./#{script} map", "sort", "./#{script} reduce") do |in_io, out_io, wt|
+        in_io.print input
+        in_io.close
+        out_io.readlines
+      end
+    end
+
+  end
+end

metadata

@@ -0,0 +1,65 @@
+--- !ruby/object:Gem::Specification 
+name: rubydoop
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 2
+  version: 0.0.2
+platform: ruby
+authors: 
+- Jacob Rothstein
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-04-21 00:00:00 -04:00
+default_executable: 
+dependencies: []
+
+description: Simple Ruby Sugar for Hadoop Streaming
+email: jbr@yakbarber.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/rubydoop/minitest.rb
+- lib/rubydoop.rb
+has_rdoc: true
+homepage: https://github.com/jbr/rubydoop
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: rubydoop
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Simple Ruby Sugar for Hadoop Streaming
+test_files: []
+