mrtoolkit 0.1.2

data/lib/regression.rb

@@ -0,0 +1,33 @@
+
+class LinearRegression
+  attr_accessor :slope, :offset
+
+  def initialize (dx, dy=nil)
+    @size = dx.size
+    dy,dx = dx,axis() unless dy  # make 2D if given 1D
+    raise "arguments not same length!" unless @size == dy.size
+    sxx = sxy = sx = sy = 0
+    dx.zip(dy).each do |x,y|
+      sxy += x*y
+      sxx += x*x
+      sx  += x
+      sy  += y
+    end
+    @slope = ( @size * sxy - sx*sy ) / ( @size * sxx - sx * sx )
+    @offset = (sy - @slope*sx) / @size
+  end
+
+  def fit(dx = nil)
+    dx = axis unless dx
+    return dx.map{|data| predict(data) }
+  end
+
+  def predict( x )
+    y = @slope * x + @offset
+  end
+
+  def axis
+    (0...@size).to_a
+  end
+end
+

data/lib/stream_runner.rb

@@ -0,0 +1,100 @@
+
+# StreamRunner
+# This class is responsible for running stream jobs in hadoop.
+#
+# Streaming is a simplified programming model in which map and reduce
+# proceses that read from STDIN and write to STDOUT are given.  
+# StreamRunner runs *ruby programs* as the map and reduce steps.
+# 
+# Additional services provided:
+# * the number of reducers can be specified
+# * extra files to include can be given
+# * input can be one directory or an array of directories
+# * collects the output and copies it to local file in "out" directory
+# * deletes hadoop output direcory before starting job -- BE CAREFUL
+#
+# Extra files are distributed to each cluster member, and are stored in the
+# directory with the map or reduce programs.  You must include any data files 
+# your program reads or library files it requires.  You do not have to
+# include the program itself -- this is done automatically.
+#
+# HADOOP_HOME must be set.
+# It might be necessary to change HADOOP_STREAMING_VERSION if the version changes.
+
+streaming_version = ENV['HADOOP_STREAMING_VERSION']
+streaming_version ="0.20.0" unless streaming_version
+
+HADOOP_HOME=ENV['HADOOP_HOME']
+HADOOP_STREAMING="#{HADOOP_HOME}/contrib/streaming/hadoop-#{streaming_version}-streaming.jar"
+
+class StreamRunner
+  def expand_path(file)
+    return file if File.exist?(file)
+    rlib = ENV['RUBYLIB'] || File.dirname(__FILE__)
+    raise "Cannot resolve path to #{file} -- no RUBYLIB" unless rlib
+    (rlib.split(':') + [File.dirname(__FILE__)]).each do |rp|
+      trial = "#{rp}/#{file}"
+      return trial if File.exists?(trial)
+    end
+    raise "Cannot resolve path to #{file}. Is it in RUBYLIB?"
+  end
+  def expand_paths(extra)
+    extras = []
+    extra.collect { |e| expand_path(e)}
+  end
+
+  def run_hadoop_stream(input, out, mapper, reducer, reducers, extra, 
+    map_opts, reduce_opts, opts)
+    extras = ''
+    extra << mapper.split(' ')[0]
+    extra << reducer.split(' ')[0]
+    expand_paths(extra.uniq).each {|e| extras += "-file #{e} "}
+    map_opt = ''
+    map_opts.each {|n, v| map_opt += "-jobconf #{n}=#{v} "}
+    reduce_opt = ''
+    reduce_opts.each {|n, v| reduce_opt += "-jobconf #{n}=#{v} "}
+    if input.class == Array
+      input = input.collect {|i| "-input #{i}"}.join(" ")
+    else
+      input = "-input #{input}"
+    end
+
+    if reducer.nil?
+      cmd = "hadoop jar #{HADOOP_STREAMING} " +
+        "#{input} " +
+	"-output NONE " +
+	"-mapper \"ruby #{mapper}\"" +
+        "-jobconf mapred.reduce.tasks=0 " +
+	map_opt +
+        "#{extras}"
+    else
+      cmd = "hadoop jar #{HADOOP_STREAMING} " +
+        "#{input} " +
+	"-output #{out} " +
+	"-mapper \"ruby #{mapper}\" " +
+	map_opt +
+	"-reducer \"ruby #{reducer}\" " +
+        "-jobconf mapred.reduce.tasks=#{reducers} " +
+	reduce_opt +
+        "#{extras}"
+    end
+    cmd += " -verbose " if opts.has_key?(:verbose)
+    cmd += " #{opts[:hadoop_opts]}" if opts.has_key?(:hadoop_opts)
+    puts cmd if opts.has_key?(:verbose)
+    system(cmd)
+  end
+
+  def run_map_reduce(input, out, map, reduce, reducers, extra, 
+    map_opts = {}, reduce_opts = {}, opts = {})
+    system("hadoop fs -rmr #{out}")
+    system("rm -rf out/#{out}")
+    system("mkdir -p out/#{out}")
+    run_hadoop_stream(input, out, map, reduce, reducers, extra, 
+      map_opts, reduce_opts, opts)
+    (0..reducers-1).each do |i|
+      n = sprintf("%05d", i)
+      system("hadoop fs -cat #{out}/part-#{n} >out/#{out}/part-#{n}")
+    end
+  end
+end
+

data/mrtoolkit.gemspec

@@ -0,0 +1,79 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{mrtoolkit}
+  s.version = "0.1.2"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["cchayden", "vadimj", "jashmenn"]
+  s.date = %q{2010-05-17}
+  s.email = %q{nate@natemurray.com}
+  s.extra_rdoc_files = [
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "Makefile",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION.yml",
+     "examples/Rakefile",
+     "examples/Readme",
+     "examples/hour.rb",
+     "examples/import-logs",
+     "examples/import.rb",
+     "examples/ip-result.rb",
+     "examples/ip-size.rb",
+     "examples/ip-ua.rb",
+     "examples/ip.rb",
+     "examples/section.rb",
+     "examples/top-file.rb",
+     "lib/mrtoolkit.rb",
+     "lib/regression.rb",
+     "lib/stream_runner.rb",
+     "mrtoolkit.gemspec",
+     "standalone/hadoop",
+     "test/Rakefile",
+     "test/test-in/test1-in",
+     "test/test-in/test2-in",
+     "test/test-in/test3-in",
+     "test/test-in/test4-in",
+     "test/test-in/test5-in",
+     "test/test-in/test6-in",
+     "test/test-in/test7-in",
+     "test/test-in/test8-in",
+     "test/test-in/test9-in",
+     "test/utest.rb"
+  ]
+  s.homepage = %q{http://github.com/jashmenn/mrtoolkit}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.6}
+  s.summary = %q{Simplify the creation of Hadoop Map/Reduce jobs}
+  s.test_files = [
+    "test/utest.rb",
+     "examples/hour.rb",
+     "examples/import.rb",
+     "examples/ip-result.rb",
+     "examples/ip-size.rb",
+     "examples/ip-ua.rb",
+     "examples/ip.rb",
+     "examples/section.rb",
+     "examples/top-file.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end
+

data/standalone/hadoop

@@ -0,0 +1,104 @@
+#!/usr/bin/ruby
+
+# Simulate hadoop well enough to do local testing.
+
+case ARGV[0]
+# 
+# Do hadoop fs commands
+when 'fs':
+  ARGV.shift
+  case ARGV[0]
+  when '-rmr':
+    dest = ARGV[1]
+    if dest.nil?
+      STDERR.puts "missing argument"
+      exit 1
+    end
+    system("rm -rf #{dest}")
+  when '-rm':
+    dest = ARGV[1]
+    if dest.nil?
+      STDERR.puts "missing argument"
+      exit 1
+    end
+    system("rm -f #{dest}")
+  when '-mkdir':
+    dest = ARGV[1]
+    if dest.nil?
+      STDERR.puts "missing argument"
+      exit 1
+    end
+    system("mkdir -p #{dest}")
+  when '-copyFromLocal':
+  when '-put':
+    src = ARGV[1]
+    dest = ARGV[2]
+    if src.nil? || dest.nil? 
+      STDERR.puts "missing argument"
+      exit 1
+    end
+   system("cp #{src} #{dest}")
+  when '-cat':
+    src = ARGV[1]
+    if src.nil? 
+      STDERR.puts "missing argument"
+      exit 1
+    end
+    if  File.exists?(src)
+      system("cat #{src}")
+    end
+  when '-ls':
+    dest = ARGV[1]
+    if dest.nil? 
+      system("cd #{HADOOP_DIR}; ls -l '--time-style=+%Y-%m-%d %H:%M'")
+    else
+      system("cd #{HADOOP_DIR};ls -l '--time-style=+%Y-%m-%d %H:%M' #{dest}")
+    end
+  when '-lsr':
+    dest = ARGV[1]
+    if dest == '/user'
+      # special test data
+      system("cat sample-ls-data")
+    else
+      system("ls -lR '--time-style=+%Y-%m-%d %H:%M' #{dest}")
+    end
+  else
+    STDERR.puts "command not recognized #{ARGV[0]}"
+    exit 1
+  end
+#
+# Do hadoop jar commands (run map reduce)
+when 'jar':
+  ARGV.shift
+  jar = ARGV[0]
+  ARGV.shift
+  file = []
+  reducers = 1
+  while ARGV.size > 0
+    case ARGV[0]
+    when '-input':
+      indir = ARGV[1]; 2.times {ARGV.shift}
+    when '-output':
+      outdir = ARGV[1]; 2.times {ARGV.shift}
+    when '-mapper':
+      mapper = ARGV[1]; 2.times {ARGV.shift}
+    when '-reducer':
+      reducer = ARGV[1]; 2.times {ARGV.shift}
+    when '-jobconf':
+      jobconf = ARGV[1]; 2.times {ARGV.shift}
+      if jobconf =~ /mapred.reduce.tasks=([\d]*)/
+        reducers = $1.to_i
+      end
+    when '-file':
+      file << ARGV[1]; 2.times {ARGV.shift}
+    end 
+  end
+  system("hadoop fs -mkdir #{outdir}")
+  # Simulate hadoop by running mapper, then sort, then run reducer.
+  cmd = "ls #{indir}|while read line;do #{mapper} < #{indir}/$line;done|LC_ALL=C sort|#{reducer} >#{outdir}/part-00000"
+  #puts "running: #{cmd}"
+  system cmd
+else
+  STDERR.puts "*** hadoop #{ARGV.join(' ')}"
+  exit 1
+end

data/test/Rakefile

@@ -0,0 +1,21 @@
+# add mrtoolkit to $RUBYLIB
+if ENV['RUBYLIB']
+  ENV['RUBYLIB'] = ENV['RUBYLIB'].split(':').concat(['../lib']).uniq.join(':')
+else
+  ENV['RUBYLIB'] = '../lib'
+end
+# add everything in $RUBYLIB to search path
+ENV['RUBYLIB'].split(':').each {|f| $:.concat([f]) unless $:.include?(f)}
+
+######################################################################
+desc "clean up"
+task "clean" do
+  to_clean = %w{test-out}
+  to_clean.each {|f| system "rm -rf #{f}"}
+end
+######################################################################
+desc "unit tests"
+task :test do
+  system "ruby utest.rb"
+end
+ 

data/test/test-in/test1-in

@@ -0,0 +1,2 @@
+2008-10-02	11:30:00	1.2.3.5
+2008-10-01	10:30:00	1.2.3.4

data/test/test-in/test2-in

@@ -0,0 +1,4 @@
+1
+2
+10
+30

data/test/test-in/test3-in

@@ -0,0 +1,5 @@
+2008-10-31	21:01:00
+2008-10-31	21:00:00
+2008-10-31	21:10:00
+2008-10-31	21:10:00
+2008-10-31	21:10:00

data/test/test-in/test4-in

@@ -0,0 +1,6 @@
+1	1
+2	2
+3	3
+4	10
+5	1
+6	2

data/test/test-in/test5-in

@@ -0,0 +1,12 @@
+100	1
+100	1
+100	1
+101	1
+101	1
+102	1
+102	1
+102	1
+102	1
+103	1
+104	1
+104	1

data/test/test-in/test6-in

@@ -0,0 +1,3 @@
+1	1	1
+1	3	4
+10	5	3

data/test/test-in/test7-in

@@ -0,0 +1,20 @@
+1
+2
+3
+4
+5
+6
+7
+8
+9
+10
+11
+12
+13
+14
+15
+16
+17
+18
+19
+20

data/test/test-in/test8-in

@@ -0,0 +1,12 @@
+100	1000	1
+100	1001	1
+100	1000	1
+200	1000	1
+200	1001	1
+200	1000	1
+100	1000	1
+
+
+
+
+

data/test/test-in/test9-in

@@ -0,0 +1,6 @@
+1000	a	a	a
+1000	b	b	b
+1000	c	c	c
+1001	x1	y1	z1
+1001	x2	y2	z2
+1000	d1	d2	d3