sandofsky-ralph 0.1.0

data/.autotest

@@ -0,0 +1,23 @@
+# -*- ruby -*-
+
+require 'autotest/restart'
+
+# Autotest.add_hook :initialize do |at|
+#   at.extra_files << "../some/external/dependency.rb"
+#
+#   at.libs << ":../some/external"
+#
+#   at.add_exception 'vendor'
+#
+#   at.add_mapping(/dependency.rb/) do |f, _|
+#     at.files_matching(/test_.*rb$/)
+#   end
+#
+#   %w(TestA TestB).each do |klass|
+#     at.extra_class_map[klass] = "test/test_misc.rb"
+#   end
+# end
+
+# Autotest.add_hook :run_command do |at|
+#   system "rake build"
+# end

data/History.txt

@@ -0,0 +1,6 @@
+=== 1.0.0 / 2009-08-12
+
+* 1 major enhancement
+
+  * Birthday!
+

data/Manifest.txt

@@ -0,0 +1,9 @@
+.autotest
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+bin/ralph
+lib/iterator.rb
+lib/ralph.rb
+test/test_ralph.rb

data/README.txt

@@ -0,0 +1,76 @@
+= ralph
+
+* http://github.com/sandofsky/ralph
+
+== DESCRIPTION:
+
+A pretty, simple wrapper for writing Hadoop Streaming jobs in Ruby.
+
+== FEATURES/PROBLEMS:
+
+* First version.
+
+== SYNOPSIS:
+
+Define your mappers and reducers in Ruby with the help of a domain specific language.
+
+    Ralph.job do
+
+      mapper do |row|
+        row.split(/\s/).each do |word|
+          emit word, 1
+        end
+      end
+
+      reducer do |key, values|
+        total = values.inject(0) {|t, i| t + i.to_i }
+        emit key, total
+      end
+
+    end
+
+Then run you task with:
+
+    ralph fire word_count.rb input.csv output
+
+Which would translate into:
+
+    hadoop jar $HADOOP_HOME/contrib/streaming/hadoop-*-streaming.jar \
+      -input input.csv \
+      -output output \
+      -mapper "ralph map word_count.rb" \
+      -reducer "ralph reduce word_count.rb" \
+      -file word_count.rb
+
+== REQUIREMENTS:
+
+* Make sure to export your $HADOOP_HOME as the path to hadoop.
+
+== INSTALL:
+
+* sudo gem install sandofsky-ralph --source http://gems.github.com
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2009 FIX
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,13 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'hoe'
+
+Hoe.spec 'ralph' do
+  developer('Ben Sandofsky', 'sandofsky@gmail.com')
+  extra_deps << ['fastercsv', '~> 1.4.0']
+  extra_deps << ['sandofsky-csvscan', '>= 0.1.0']
+  extra_deps << ['wycats-thor', '>= 0.11.5']
+end
+
+# vim: syntax=ruby

data/bin/ralph

@@ -0,0 +1,60 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'thor'
+require File.expand_path(File.dirname(__FILE__) + "/../lib/ralph")
+class RalphCLI < Thor
+  desc "fire <script.rb>", "run via hadooop streaming"
+  def fire(script, input, output = nil)
+    output ||= "#{input}_output"
+hadoop_command = <<-END
+#{ENV["HADOOP_HOME"]}/bin/hadoop jar #{ENV["HADOOP_HOME"]}/contrib/streaming/hadoop-*-streaming.jar \
+    -input #{input} \
+    -output #{output} \
+    -mapper "ralph map #{script}" \
+    -reducer "ralph reduce #{script}" \
+    -file #{script}
+END
+    IO.popen(hadoop_command, 'r+') do |io|
+      until io.eof?
+        puts io.readline
+      end
+    end
+  end
+  
+  desc "simulate <script.rb>", "simulates a mapreduce job through unix pipes and 'sort'"
+  def simulate(script)
+    require script
+    sorter = IO.popen("sort", 'w+')
+    threads = []
+    threads << Thread.new(sorter) do |s|
+      Ralph.run_mapper!(STDIN, s)
+      s.close_write
+    end
+    if Ralph::Job.reducer?
+      threads << Thread.new(sorter) do |s|
+        Ralph.run_reducer!(s, STDOUT)
+      end
+    else
+      threads << Thread.new(sorter) do |s|
+        until s.eof?
+          puts s.readline
+        end
+      end
+    end
+    threads.each {|t| t.join}
+  end
+  
+  desc "map <script.rb>", "Run mapper"
+  def map(script)
+    require script
+    Ralph.run_mapper!
+  end
+  
+  desc "reduce <script.rb>", "Run reducer"
+  def reduce(script)
+    require script
+    Ralph.run_reducer!
+  end
+  
+end
+RalphCLI.start

data/lib/iterator.rb

@@ -0,0 +1,67 @@
+class Ralph
+  module Iterator
+    
+    @@input = STDIN
+    def self.input=(i)
+      @@input = i
+    end
+    
+    def self.input
+      @@input
+    end
+    
+    @@last_key = nil
+    @@current_key = nil
+    @@current_value = nil
+
+    @@has_iterated = nil
+    def self.ready!
+      @@has_iterated = false
+    end
+
+    def self.current_key
+      @@current_key
+    end
+
+    def self.start_of_group?
+      @@last_key != @@current_key
+    end
+    
+    def self.next_group!
+      until start_of_group? || input.eof?
+        grab!
+      end
+    end
+
+    def self.each
+      repeat_iteration_check
+      c = @@current_key.dup
+      until c != @@current_key || input.eof?
+        yield @@current_value
+        grab!
+      end
+      @@has_iterated = true
+    end
+    
+    def self.inject(s = nil)
+      repeat_iteration_check
+      c = @@current_key.dup
+      until c != @@current_key || input.eof?
+        s = yield s, @@current_value
+        grab!
+      end
+      @@has_iterated = true
+      s
+    end
+    
+    def self.repeat_iteration_check
+      raise "You cannot call the value iterator more than once in a reduce" if @@has_iterated
+    end
+    
+    def self.grab!
+      @@last_key = @@current_key.dup unless @@current_key.nil?
+      @@current_key, @@current_value = FasterCSV.parse_line(input.readline, :col_sep => "\t")
+    end
+
+  end
+end

data/lib/ralph.rb

@@ -0,0 +1,59 @@
+require 'csvscan'
+require 'fastercsv'
+require File.dirname(__FILE__) + '/iterator'
+class Ralph
+  VERSION = '0.1.0'
+
+  @@map_input = nil
+  class Job
+    attr_accessor :output
+    def emit(key, value)
+      output << FasterCSV.generate_line((value ? [key, value] : [key]), :col_sep => "\t")
+    end
+    
+    def self.reducer?
+      instance_methods.include?('reduce')
+    end
+    
+  end
+  
+  def self.mapper(format = nil, &block)
+    Job.send(:define_method, :map, &block)
+    @@map_input = format
+  end
+  
+  def self.reducer(&block)
+    Job.send(:define_method, :reduce, &block)
+  end
+
+  def self.run_mapper!(input = STDIN, output = STDOUT)
+    j = Job.new
+    j.output = output
+    if @@map_input == :csv
+      CSVScan.scan(input) do |row|
+        j.map(row)
+      end
+    else
+      input.each_line do |line| 
+        j.map(line)
+      end
+    end
+  end
+
+  def self.run_reducer!(input = STDIN, output = STDOUT)
+    j = Job.new
+    j.output = output
+    Iterator.input = input
+    Iterator.grab! unless input.eof?
+    until input.eof?
+      Iterator.ready!
+      j.reduce(Iterator.current_key, Iterator)
+      Iterator.next_group! unless Iterator.start_of_group?
+    end
+  end
+  
+  def self.job(&block)
+    self.instance_eval(&block) if block_given?# setup
+  end
+  
+end

data/test/test_ralph.rb

@@ -0,0 +1,8 @@
+require "test/unit"
+require "ralph"
+
+class TestRalph < Test::Unit::TestCase
+  def test_sanity
+    flunk "write tests or I will kneecap you"
+  end
+end

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification 
+name: sandofsky-ralph
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Ben Sandofsky
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-08-14 00:00:00 -07:00
+default_executable: ralph
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: fastercsv
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        version: 1.4.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: sandofsky-csvscan
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.1.0
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: wycats-thor
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.11.5
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.3.3
+    version: 
+description: A pretty, simple wrapper for writing Hadoop Streaming jobs in Ruby.
+email: 
+- sandofsky@gmail.com
+executables: 
+- ralph
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- README.txt
+files: 
+- .autotest
+- History.txt
+- Manifest.txt
+- README.txt
+- Rakefile
+- bin/ralph
+- lib/iterator.rb
+- lib/ralph.rb
+- test/test_ralph.rb
+has_rdoc: false
+homepage: http://github.com/sandofsky/ralph
+licenses: 
+post_install_message: 
+rdoc_options: 
+- --main
+- README.txt
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: ralph
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: A pretty, simple wrapper for writing Hadoop Streaming jobs in Ruby.
+test_files: 
+- test/test_ralph.rb