jmapreduce 0.1

data/examples/wordcount.rb

@@ -0,0 +1,42 @@
+import org.fingertap.jmapreduce.JMapReduce
+
+JMapReduce.job 'Count' do
+  reduce_tasks 1
+  
+  map do |key, value|
+    value.split.each do |word|
+      emit(word, 1)
+    end
+  end
+  
+  reduce do |key, values|
+    sum = 0
+    values.each {|v| sum += v }
+    emit(key, {'sum' => sum})
+  end
+end
+
+JMapReduce.job "Histogram" do
+  setup do
+    RANGES = [0..1, 2..3, 4..5, 6..10, 11..20, 21..30, 31..40, 41..50, 51..100, 101..200, 201..300, 301..10_000, 10_001..99_999]
+  end
+  
+  map do |word, count|
+    range = RANGES.find {|range| range.include?(count['sum']) }
+    emit("#{range.first.to_s.rjust(5,'0')}-#{range.last.to_s.rjust(5,'0')}", 1)
+  end
+  
+  reduce do |range, counts|
+    total = counts.inject(0) {|sum,count| sum+count }
+    emit(range, '|'*(total/20))
+  end
+end
+
+# this job is just a pass though which takes advantage of the map/reduce shuffle to get ordered keys
+JMapReduce.job "Sort" do
+  reduce_tasks 1
+end
+
+__END__
+
+./bin/jmapreduce examples/wordcount.rb examples/alice.txt /tmp/output

data/lib/jmapreduce/runner.rb

@@ -0,0 +1,110 @@
+class Runner
+  JAVA_MAIN_CLASS = 'org.fingertap.jmapreduce.JMapReduce'
+  
+  def initialize(script, input, output, opts={})
+    @script = script
+    @input = input
+    @output = output
+    @opts = opts
+    
+    # env get / set and check
+    hadoop_home and hadoop_cmd and hadoop_classpath
+  end
+  
+  def hadoop_home
+    ENV['HADOOP_HOME']
+  end
+  
+  def hadoop_cmd
+    hadoop = `which hadoop 2>/dev/null`
+    hadoop = "#{hadoop_home}/bin/hadoop" if hadoop.empty? and (!hadoop_home.empty?)
+    raise 'cannot find hadoop command' if hadoop.empty?
+    hadoop.chomp
+  end
+  
+  def hadoop_classpath
+    ENV['HADOOP_CLASSPATH'] = ([lib_path] + dirnames + lib_jars).join(':')
+  end
+  
+  def run
+    puts cmd
+    exec cmd
+  end
+  
+  def cmd
+    "#{hadoop_cmd} jar #{main_jar_path} #{JAVA_MAIN_CLASS} #{jars_args} #{file_args} #{conf_args} #{archived_args} #{mapred_args} #{properties_args}"
+  end
+  
+  def jars_args
+    "-libjars #{lib_jars.join(',')}"
+  end
+  
+  def file_args
+    "-files #{files.join(',')}"
+  end
+  
+  def conf_args
+    args = ''
+    args += @opts[:conf] ? "-conf #{@opts[:conf]} " : ''
+    args += @opts[:namenode] ? "-fs #{@opts[:namenode]} " : ''
+    args += @opts[:jobtracker] ? "-jt #{@opts[:jobtracker]} " : ''
+    args
+  end
+  
+  def archived_args
+    return unless @opts[:dirs]
+    
+    archived_files = []
+    @opts[:dirs].split(',').each do |dir|
+      next unless File.directory?(dir)
+      tgz = "/tmp/jmapreduce-#{Process.pid}-#{Time.now.to_i}-#{rand(1000)}.tgz"
+      system("cd #{dir} && tar -czf #{tgz} *")
+      archived_files << "#{tgz}\##{File.basename(dir)}"
+    end
+    
+    "-archives #{archived_files.join(',')}"
+  end
+  
+  def mapred_args
+    "#{File.basename(@script)} #{@input} #{@output}"
+  end
+  
+  def properties_args
+    return '' if @opts[:properties].nil? && @opts[:json].nil?
+    properties = []
+    properties << @opts[:properties] if @opts[:properties]
+    properties << @opts[:json] if @opts[:json]
+    properties.join(',')
+  end
+  
+  def files
+    ret = [@script]
+    ret += @opts[:files].split(',') if @opts[:files]
+    ret
+  end
+  
+  def dirnames
+    files.map{ |f| File.dirname(f) }
+  end
+  
+  def lib_jars
+    jars = [
+      JRubyJars.core_jar_path,
+      JRubyJars.stdlib_jar_path,
+      main_jar_path,
+      File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'vendors', 'gson.jar')),
+      File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'vendors', 'javassist.jar')),
+      File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'vendors', 'msgpack.jar'))
+    ]
+    jars += @opts[:libjars].split(',') if @opts[:libjars]
+    jars
+  end
+  
+  def main_jar_path
+    File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'release', 'jmapreduce.jar'))
+  end
+  
+  def lib_path
+    File.expand_path(File.join(File.dirname(__FILE__), '..'))
+  end
+end

metadata

@@ -0,0 +1,86 @@
+--- !ruby/object:Gem::Specification 
+name: jmapreduce
+version: !ruby/object:Gem::Version 
+  hash: 9
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  version: "0.1"
+platform: ruby
+authors: 
+- Abhinay Mehta
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-09-12 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: jruby-jars
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 1
+        - 6
+        version: "1.6"
+  type: :runtime
+  version_requirements: *id001
+description: JMapReduce is JRuby Map/Reduce Framework built on top of the Hadoop Distributed computing platform.
+email: abhinay.mehta@gmail.com
+executables: 
+- jmapreduce
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- bin/jmapreduce
+- README.md
+- lib/jmapreduce/runner.rb
+- release/jmapreduce.jar
+- vendors/gson.jar
+- vendors/javassist.jar
+- vendors/msgpack.jar
+- examples/alice.txt
+- examples/wordcount.rb
+homepage: https://bitbucket.org/abhinaymehta/jmapreduce
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.7.2
+signing_key: 
+specification_version: 3
+summary: Map/Reduce Framework
+test_files: []
+