jruby-mapreduce 0.5

data/examples/wordcount.rb

@@ -0,0 +1,44 @@
+import org.fingertap.jmapreduce.JMapReduce
+
+JMapReduce.job 'Count' do
+  reduce_tasks 1
+  
+  map do |key, value|
+    value.split.each do |word|
+      emit(word, 1)
+    end
+  end
+  
+  reduce do |key, values|
+    sum = 0
+    values.each {|v| sum += v }
+    emit(key, {'sum' => sum})
+  end
+end
+
+JMapReduce.job "Histogram" do
+  setup do
+    RANGES = [0..1, 2..3, 4..5, 6..10, 11..20, 21..30, 31..40, 41..50, 51..100, 101..200, 201..300, 301..10_000, 10_001..99_999]
+  end
+  
+  map do |word, count|
+    range = RANGES.find {|range| range.include?(count['sum']) }
+    emit("#{range.first.to_s.rjust(5,'0')}-#{range.last.to_s.rjust(5,'0')}", 1)
+  end
+  
+  reduce do |range, counts|
+    total = counts.inject(0) {|sum,count| sum+count }
+    emit(range, '|'*(total/20))
+  end
+end
+
+# this job is just a pass though which takes advantage of the map/reduce shuffle to get ordered keys
+JMapReduce.job "Sort" do
+  reduce_tasks 1
+end
+
+__END__
+
+To run:
+
+./bin/jmapreduce examples/wordcount.rb examples/alice.txt /tmp/output

data/lib/jmapreduce/runner.rb

@@ -0,0 +1,111 @@
+class Runner
+  JAVA_MAIN_CLASS = 'org.fingertap.jmapreduce.JMapReduce'
+  
+  def initialize(script, input, output, opts={})
+    @script = script
+    @input = input
+    @output = output
+    @opts = opts
+    
+    # env get / set and check
+    hadoop_home and hadoop_cmd and hadoop_classpath
+  end
+  
+  def hadoop_home
+    raise 'Please set HADOOP_HOME'  unless ENV['HADOOP_HOME']
+    ENV['HADOOP_HOME']
+  end
+  
+  def hadoop_cmd
+    hadoop = `which hadoop 2>/dev/null`
+    hadoop = "#{hadoop_home}/bin/hadoop" if hadoop.empty? and (!hadoop_home.empty?)
+    raise 'Cannot find hadoop command' if hadoop.empty?
+    hadoop.chomp
+  end
+  
+  def hadoop_classpath
+    ENV['HADOOP_CLASSPATH'] = ([lib_path] + dirnames + lib_jars).join(':')
+  end
+  
+  def run
+    puts cmd
+    exec cmd
+  end
+  
+  def cmd
+    "#{hadoop_cmd} jar #{main_jar_path} #{JAVA_MAIN_CLASS} #{file_args} #{jars_args} #{conf_args} #{archived_args} #{mapred_args} \"#{properties_args}\""
+  end
+  
+  def jars_args
+    "-libjars #{lib_jars.join(',')}"
+  end
+  
+  def file_args
+    "-files #{files.join(',')}"
+  end
+  
+  def conf_args
+    args = ''
+    args += @opts[:conf] ? "-conf #{@opts[:conf]} " : ''
+    args += @opts[:namenode] ? "-fs #{@opts[:namenode]} " : ''
+    args += @opts[:jobtracker] ? "-jt #{@opts[:jobtracker]} " : ''
+    args
+  end
+  
+  def archived_args
+    return unless @opts[:dirs]
+    
+    archived_files = []
+    @opts[:dirs].split(',').each do |dir|
+      next unless File.directory?(dir)
+      tgz = "/tmp/jmapreduce-#{Process.pid}-#{Time.now.to_i}-#{rand(1000)}.tgz"
+      system("cd #{dir} && tar -czf #{tgz} *")
+      archived_files << "#{tgz}\##{File.basename(dir)}"
+    end
+    
+    "-archives #{archived_files.join(',')}"
+  end
+  
+  def mapred_args
+    "#{File.basename(@script)} #{@input} #{@output}"
+  end
+  
+  def properties_args
+    return '' if @opts[:properties].nil? && @opts[:json].nil?
+    properties = []
+    properties << @opts[:properties] if @opts[:properties]
+    properties << @opts[:json] if @opts[:json]
+    properties.join(',')
+  end
+  
+  def files
+    ret = [@script]
+    ret += @opts[:files].split(',') if @opts[:files]
+    ret
+  end
+  
+  def dirnames
+    files.map{ |f| File.dirname(f) }
+  end
+  
+  def lib_jars
+    jars = [
+      JRubyJars.core_jar_path,
+      JRubyJars.stdlib_jar_path,
+      main_jar_path,
+      File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'vendors', 'gson.jar')),
+      File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'vendors', 'javassist.jar')),
+      File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'vendors', 'msgpack.jar'))
+    ]
+    jars += @opts[:libjars].split(',') if @opts[:libjars]
+    jars
+  end
+  
+  def main_jar_path
+    File.expand_path(File.join(File.dirname(__FILE__), '..', '..', 'release', 'jmapreduce.jar'))
+  end
+  
+  def lib_path
+    File.expand_path(File.join(File.dirname(__FILE__), '..'))
+  end
+end

metadata

@@ -0,0 +1,67 @@
+--- !ruby/object:Gem::Specification
+name: jruby-mapreduce
+version: !ruby/object:Gem::Version
+  version: '0.5'
+platform: ruby
+authors:
+- Shinji Ikeda
+- Abhinay Mehta
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2014-06-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: jruby-jars
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  prerelease: false
+  type: :runtime
+description: JMapReduce is JRuby Map/Reduce Framework built on top of the Hadoop Distributed computing platform.
+email: gm.ikeda@gmail.com
+executables:
+- jmapreduce
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- bin/jmapreduce
+- examples/alice.txt
+- examples/wordcount.rb
+- lib/jmapreduce/runner.rb
+- release/jmapreduce.jar
+- vendors/gson.jar
+- vendors/javassist.jar
+- vendors/msgpack.jar
+homepage: https://github.com/shinjiikeda/jmapreduce
+licenses: []
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.2.2
+signing_key:
+specification_version: 4
+summary: Map/Reduce Framework
+test_files: []