mapleroad 0.0.1.pre

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 798e4df11da3783b2f5d91a163a58424da52c0f1
+  data.tar.gz: b467639d0217b7bd312ccf5effb6da7df8903e1d
+SHA512:
+  metadata.gz: 88ff6c7173883e7bda6a1991f2f822702742a68f4c7e89223cfad35b6b8d630303b007835a3340b7134c84d503970b6f2414883c9d6b2875ef831c44f2b892d3
+  data.tar.gz: 96987a08a197239c1d3aab27f41ac1932357f91877f7b19ab1ccb09cb5828645e9a1dc489f06aaa7b772657da437b3db9bd7e46b40c49c326e192d31ed2fbc9a

data/.gitignore

@@ -0,0 +1,11 @@
+.classpath
+.project
+.settings
+.idea
+.rspec
+.ruby-version
+.ruby-gemset
+*~
+lib/.idea
+build
+tmp

data/Gemfile

@@ -0,0 +1,9 @@
+# -*- ruby -*-
+
+source "https://rubygems.org"
+
+gem 'lock_jar', '~> 0.7.5', :platform => :jruby
+
+group :test, :development do
+  gem 'rake-compiler', '~> 0.9.1', :platform => :jruby
+end

data/Gemfile.lock

@@ -0,0 +1,18 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    lock_jar (0.7.5)
+      naether (~> 0.10.1)
+      thor (~> 0.18.1)
+    naether (0.10.1-java)
+    rake (10.3.2)
+    rake-compiler (0.9.2)
+      rake
+    thor (0.18.1)
+
+PLATFORMS
+  java
+
+DEPENDENCIES
+  lock_jar (~> 0.7.5)
+  rake-compiler (~> 0.9.1)

data/Jarfile

@@ -0,0 +1,2 @@
+jar "org.apache.hadoop:hadoop-mapreduce-client-core:2.4.0"
+jar "org.apache.hadoop:hadoop-common:2.4.0"

data/Jarfile.lock

@@ -0,0 +1,136 @@
+---
+version: 0.7.5
+groups:
+  default:
+    dependencies:
+    - aopalliance:aopalliance:jar:1.0
+    - asm:asm:jar:3.1
+    - com.google.code.findbugs:jsr305:jar:1.3.9
+    - com.google.guava:guava:jar:11.0.2
+    - com.google.inject.extensions:guice-servlet:jar:3.0
+    - com.google.inject:guice:jar:3.0
+    - com.google.protobuf:protobuf-java:jar:2.5.0
+    - com.jamesmurty.utils:java-xmlbuilder:jar:0.4
+    - com.jcraft:jsch:jar:0.1.42
+    - com.sun.jersey.contribs:jersey-guice:jar:1.9
+    - com.sun.jersey:jersey-core:jar:1.9
+    - com.sun.jersey:jersey-json:jar:1.9
+    - com.sun.jersey:jersey-server:jar:1.9
+    - com.sun.xml.bind:jaxb-impl:jar:2.2.3-1
+    - com.thoughtworks.paranamer:paranamer:jar:2.3
+    - commons-beanutils:commons-beanutils-core:jar:1.8.0
+    - commons-beanutils:commons-beanutils:jar:1.7.0
+    - commons-cli:commons-cli:jar:1.2
+    - commons-codec:commons-codec:jar:1.4
+    - commons-collections:commons-collections:jar:3.2.1
+    - commons-configuration:commons-configuration:jar:1.6
+    - commons-digester:commons-digester:jar:1.8
+    - commons-el:commons-el:jar:1.0
+    - commons-httpclient:commons-httpclient:jar:3.1
+    - commons-io:commons-io:jar:2.4
+    - commons-lang:commons-lang:jar:2.6
+    - commons-logging:commons-logging:jar:1.1.3
+    - commons-net:commons-net:jar:3.1
+    - io.netty:netty:jar:3.6.2.Final
+    - javax.activation:activation:jar:1.1
+    - javax.inject:javax.inject:jar:1
+    - javax.servlet.jsp:jsp-api:jar:2.1
+    - javax.servlet:servlet-api:jar:2.5
+    - javax.xml.bind:jaxb-api:jar:2.2.2
+    - javax.xml.stream:stax-api:jar:1.0-2
+    - jdk.tools:jdk.tools:jar:1.7
+    - log4j:log4j:jar:1.2.17
+    - net.java.dev.jets3t:jets3t:jar:0.9.0
+    - org.apache.avro:avro:jar:1.7.4
+    - org.apache.commons:commons-compress:jar:1.4.1
+    - org.apache.commons:commons-math3:jar:3.1.1
+    - org.apache.hadoop:hadoop-annotations:jar:2.4.0
+    - org.apache.hadoop:hadoop-auth:jar:2.4.0
+    - org.apache.hadoop:hadoop-common:jar:2.4.0
+    - org.apache.hadoop:hadoop-mapreduce-client-core:jar:2.4.0
+    - org.apache.hadoop:hadoop-yarn-api:jar:2.4.0
+    - org.apache.hadoop:hadoop-yarn-common:jar:2.4.0
+    - org.apache.httpcomponents:httpclient:jar:4.1.2
+    - org.apache.httpcomponents:httpcore:jar:4.1.2
+    - org.apache.zookeeper:zookeeper:jar:3.4.5
+    - org.codehaus.jackson:jackson-core-asl:jar:1.8.8
+    - org.codehaus.jackson:jackson-jaxrs:jar:1.8.3
+    - org.codehaus.jackson:jackson-mapper-asl:jar:1.8.8
+    - org.codehaus.jackson:jackson-xc:jar:1.8.3
+    - org.codehaus.jettison:jettison:jar:1.1
+    - org.mortbay.jetty:jetty-util:jar:6.1.26
+    - org.mortbay.jetty:jetty:jar:6.1.26
+    - org.slf4j:slf4j-api:jar:1.7.5
+    - org.slf4j:slf4j-log4j12:jar:1.7.5
+    - org.tukaani:xz:jar:1.0
+    - org.xerial.snappy:snappy-java:jar:1.0.4.1
+    - tomcat:jasper-compiler:jar:5.5.23
+    - tomcat:jasper-runtime:jar:5.5.23
+    - xmlenc:xmlenc:jar:0.52
+    artifacts:
+    - jar:org.apache.hadoop:hadoop-mapreduce-client-core:jar:2.4.0:
+        transitive:
+          com.google.inject.extensions:guice-servlet:jar:3.0: {}
+          org.apache.hadoop:hadoop-yarn-common:jar:2.4.0:
+            javax.xml.bind:jaxb-api:jar:2.2.2:
+              javax.xml.stream:stax-api:jar:1.0-2: {}
+              javax.activation:activation:jar:1.1: {}
+            com.google.inject:guice:jar:3.0:
+              aopalliance:aopalliance:jar:1.0: {}
+              javax.inject:javax.inject:jar:1: {}
+            com.sun.jersey.contribs:jersey-guice:jar:1.9: {}
+            org.apache.hadoop:hadoop-yarn-api:jar:2.4.0: {}
+          io.netty:netty:jar:3.6.2.Final: {}
+    - jar:org.apache.hadoop:hadoop-common:jar:2.4.0:
+        transitive:
+          org.apache.hadoop:hadoop-auth:jar:2.4.0: {}
+          com.jcraft:jsch:jar:0.1.42: {}
+          com.sun.jersey:jersey-server:jar:1.9:
+            asm:asm:jar:3.1: {}
+          org.apache.commons:commons-math3:jar:3.1.1: {}
+          org.apache.zookeeper:zookeeper:jar:3.4.5: {}
+          com.sun.jersey:jersey-core:jar:1.9: {}
+          org.slf4j:slf4j-log4j12:jar:1.7.5: {}
+          com.google.guava:guava:jar:11.0.2: {}
+          org.apache.avro:avro:jar:1.7.4:
+            org.xerial.snappy:snappy-java:jar:1.0.4.1: {}
+            com.thoughtworks.paranamer:paranamer:jar:2.3: {}
+          javax.servlet.jsp:jsp-api:jar:2.1: {}
+          org.codehaus.jackson:jackson-core-asl:jar:1.8.8: {}
+          commons-cli:commons-cli:jar:1.2: {}
+          com.google.code.findbugs:jsr305:jar:1.3.9: {}
+          org.mortbay.jetty:jetty-util:jar:6.1.26: {}
+          xmlenc:xmlenc:jar:0.52: {}
+          org.apache.hadoop:hadoop-annotations:jar:2.4.0:
+            jdk.tools:jdk.tools:jar:1.7: {}
+          commons-el:commons-el:jar:1.0: {}
+          org.apache.commons:commons-compress:jar:1.4.1:
+            org.tukaani:xz:jar:1.0: {}
+          javax.servlet:servlet-api:jar:2.5: {}
+          com.sun.jersey:jersey-json:jar:1.9:
+            org.codehaus.jackson:jackson-jaxrs:jar:1.8.3: {}
+            com.sun.xml.bind:jaxb-impl:jar:2.2.3-1: {}
+            org.codehaus.jackson:jackson-xc:jar:1.8.3: {}
+            org.codehaus.jettison:jettison:jar:1.1: {}
+          net.java.dev.jets3t:jets3t:jar:0.9.0:
+            com.jamesmurty.utils:java-xmlbuilder:jar:0.4: {}
+            org.apache.httpcomponents:httpclient:jar:4.1.2: {}
+            org.apache.httpcomponents:httpcore:jar:4.1.2: {}
+          commons-lang:commons-lang:jar:2.6: {}
+          tomcat:jasper-compiler:jar:5.5.23: {}
+          commons-collections:commons-collections:jar:3.2.1: {}
+          org.codehaus.jackson:jackson-mapper-asl:jar:1.8.8: {}
+          tomcat:jasper-runtime:jar:5.5.23: {}
+          commons-logging:commons-logging:jar:1.1.3: {}
+          com.google.protobuf:protobuf-java:jar:2.5.0: {}
+          org.mortbay.jetty:jetty:jar:6.1.26: {}
+          commons-httpclient:commons-httpclient:jar:3.1: {}
+          org.slf4j:slf4j-api:jar:1.7.5: {}
+          commons-net:commons-net:jar:3.1: {}
+          commons-io:commons-io:jar:2.4: {}
+          commons-codec:commons-codec:jar:1.4: {}
+          commons-configuration:commons-configuration:jar:1.6:
+            commons-beanutils:commons-beanutils-core:jar:1.8.0: {}
+            commons-digester:commons-digester:jar:1.8:
+              commons-beanutils:commons-beanutils:jar:1.7.0: {}
+          log4j:log4j:jar:1.2.17: {}

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 TODO: Write your name
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,65 @@
+# MapleRoad
+
+This gem is a proof of concept to write Hadoop's mapreduce code in Ruby.
+
+This gem works only on JRuby.
+Also, this gem assumes hadoop command is on the PATH and can use it just type `hadoop`.
+Prior to trying this gem, setup Hadoop's cluster(s) and check some Hadoop sample works fine.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'mapleroad'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install mapleroad
+
+## Usage
+
+1. Write mapper class. Make sure mapper class file returns an instance of mapper class when the file is evaluated. The mapper class must have map method whose arguments are key, value, and collector.
+
+2. Writer reducer class. Like mapper class, make sure to return reducer class instance. The reducer class must have reduce method whose arguments are key, value, and collector.
+
+3. Put input file(s) to the hadoop filesystem
+
+4. Run bin/mapleroad command with arguments
+
+    Example: bin/mapleroad mapper.rb reducer.rb my-dir/input my-dir/output Text:Int
+
+  - The first arg: mapper file name
+  - The second arg: reducer filename
+  - The third arg: input
+  - The fourth arg: output
+  - The fifth arg: types of output's key/value. Text:Text or Text:Int only for now
+
+## Samples
+
+This gem includes two samples, anagram and wordcount.
+
+The original anagram examples is found at [https://code.google.com/p/hadoop-map-reduce-examples/wiki/Anagram_Example](https://code.google.com/p/hadoop-map-reduce-examples/wiki/Anagram_Example).
+The data for input can be download from that wiki page.
+
+Suppose you are in the top directory of this gem,
+
+`bin/mapleroad samples/anagram/mapper.rb samples/anagram/reducer.rb anagram/input anagram/output Text:Text`
+
+
+The wordcount example is a famous Hadoop example.
+As it is explained at [http://www.cloudera.com/content/cloudera-content/cloudera-docs/HadoopTutorial/CDH4/Hadoop-Tutorial/ht_usage.html](http://www.cloudera.com/content/cloudera-content/cloudera-docs/HadoopTutorial/CDH4/Hadoop-Tutorial/ht_usage.html), create two input files.
+
+`bin/mapleroad samples/wordcount/mapper.rb samples/wordcount/reducer.rb wordcount/input wordcount/output Text:Int
+
+
+## Contributing
+
+1. Fork it ( https://github.com/yokolet/mapleroad/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,17 @@
+# -*- ruby -*-
+
+require 'rubygems'
+require 'rake/javaextensiontask'
+
+Rake::JavaExtensionTask.new('mapleroad') do |ext|
+  require 'lock_jar'
+  LockJar.lock
+  locked_jars = LockJar.load
+
+  jruby_home = ENV['MY_RUBY_HOME'] # this is available on rvm
+  jars = ["#{jruby_home}/lib/jruby.jar"] + FileList['lib/*.jar'] + locked_jars
+  ext.classpath = jars.map {|x| File.expand_path x}.join ':'
+  ext.source_version = '1.7'
+  ext.target_version = '1.7'
+  ext.name = 'mapleroad_service'
+end

data/bin/mapleroad

@@ -0,0 +1,33 @@
+#!/usr/bin/env ruby
+require 'lock_jar'
+require 'optparse'
+require 'jruby-jars'
+
+opts = OptionParser.new do |opts|
+  opts.banner = "MapleRoad: proof of concept, hadoop by Ruby"
+  opts.define_head "Usage: mapleroad <mapper filename> <reducer filename> <input> <output> <type>"
+  opts.separator ""
+  opts.separator "Example:"
+  opts.separator "  mapleroad samples/anagram/mapper.rb samples/anagram/reducer.rb anagram/input anagram/output Text:Text"
+  # TODO: options
+end
+opts.parse!
+
+MAPPER_FILENAME = File.expand_path(ARGV.shift)
+REDUCER_FILENAME = File.expand_path(ARGV.shift)
+INPUT = ARGV.shift
+OUTPUT = ARGV.shift
+TYPE = ARGV.shift
+TYPE ||= "Text:Text"
+
+LockJar.lock
+locked_jars = LockJar.load
+classpath = locked_jars.map {|x| File.expand_path(x)}.join(":")
+libjarpath = locked_jars.map {|x| File.expand_path(x)}.join(",")
+
+JAVA_CLASS = 'mapleroad.MapleRoadJob'
+command = %Q(export HADOOP_CLASSPATH=#{classpath}; hadoop jar lib/mapleroad_service.jar #{JAVA_CLASS} #{MAPPER_FILENAME} #{REDUCER_FILENAME} #{INPUT} #{OUTPUT} #{TYPE})
+puts "COMMAND: #{command}"
+
+
+exec(command)

data/ext/mapleroad/MapleRoadCollector.java

@@ -0,0 +1,32 @@
+package mapleroad;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.OutputCollector;
+
+public class MapleRoadCollector {
+    private OutputCollector<Text, Text> collector = null;
+    private Text key = null;
+    private Text value = null;
+    
+    MapleRoadCollector(Text key, Text value, OutputCollector<Text, Text> collector) {
+        this.key = key;
+        this.value = value;
+        this.collector = collector;
+    }
+
+    public void collect(Object k, Object v) throws IOException {
+        if (k instanceof String) {
+            key.set((String)k);
+        } else if (k instanceof Text) {
+            key = (Text)k;
+        }
+        if (v instanceof String) {
+            value.set((String)v);
+        } else if (v instanceof Text) {
+            value.set((Text)v);
+        }
+        collector.collect(key, value);
+    }
+}

data/ext/mapleroad/MapleRoadCollectorTextInt.java

@@ -0,0 +1,37 @@
+package mapleroad;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.jruby.RubyFixnum;
+
+public class MapleRoadCollectorTextInt {
+    private OutputCollector<Text, IntWritable> collector = null;
+    private Text key = null;
+    private IntWritable value = null;
+    
+    MapleRoadCollectorTextInt(Text key, IntWritable value, OutputCollector<Text, IntWritable> collector) {
+        this.key = key;
+        this.value = value;
+        this.collector = collector;
+    }
+
+    public void collect(Object k, Object v) throws IOException {
+        if (k instanceof String) {
+            key.set((String)k);
+        } else if (k instanceof Text) {
+            key.set((Text)k);
+        }
+        if (v instanceof RubyFixnum) {
+            Long longValue = ((RubyFixnum)v).getLongValue();
+            value.set(longValue.intValue());
+        } else if (v instanceof Long) {
+            value.set(((Long)v).intValue());
+        } else if (v instanceof Integer) {
+            value.set(((Integer)v).intValue());
+        }
+        collector.collect(key, value);
+    }
+}

data/ext/mapleroad/MapleRoadJob.java

@@ -0,0 +1,64 @@
+package mapleroad;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.FileInputFormat;
+import org.apache.hadoop.mapred.FileOutputFormat;
+import org.apache.hadoop.mapred.JobClient;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.TextInputFormat;
+import org.apache.hadoop.mapred.TextOutputFormat;
+
+public class MapleRoadJob  {
+
+    /**
+     * @throws IOException 
+     * @param args args[0]: mapper filename, 
+     *             args[1]: reducer filename,
+     *             args[2]: input,
+     *             args[3]: output,
+     *             args[4]: output key/value types
+     * @throws  
+     */
+    public static void main(String[] args) throws IOException  {
+        JobConf conf = new JobConf(MapleRoadJob.class);
+        conf.set("mapper.file.name", args[0]);
+        conf.set("reducer.file.name", args[1]);
+        conf.set("mapper.code", getCodeAsString(args[0], StandardCharsets.UTF_8));
+        System.out.println(conf.get("mapper.code"));
+        conf.set("reducer.code", getCodeAsString(args[1], StandardCharsets.UTF_8));
+        System.out.println(conf.get("reducer.code"));
+        conf.setJobName("maplpleroad_job");
+        conf.setInputFormat(TextInputFormat.class);
+        conf.setOutputFormat(TextOutputFormat.class);
+        FileInputFormat.setInputPaths(conf, new Path(args[2]));
+        FileOutputFormat.setOutputPath(conf, new Path(args[3]));
+        
+        if (args[4].equals("Text:Text")) {
+            conf.setOutputKeyClass(Text.class);
+            conf.setOutputValueClass(Text.class);
+            conf.setMapperClass(MapleRoadMapper.class);
+            conf.setReducerClass(MapleRoadReducer.class);
+        } else if (args[4].equals("Text:Int")) {
+            conf.setOutputKeyClass(Text.class);
+            conf.setOutputValueClass(IntWritable.class);
+            conf.setMapperClass(MapleRoadMapperTextInt.class);
+            conf.setReducerClass(MapleRoadReducerTextInt.class);
+        }
+
+        JobClient.runJob(conf);
+    }
+    
+    private static String getCodeAsString(String filename, Charset encoding) throws IOException {
+        byte[] encoded = Files.readAllBytes(Paths.get(filename).toAbsolutePath());
+        return new String(encoded, encoding);
+    }
+
+}

data/ext/mapleroad/MapleRoadMapper.java

@@ -0,0 +1,43 @@
+package mapleroad;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.jruby.Ruby;
+import org.jruby.RubyRuntimeAdapter;
+import org.jruby.javasupport.JavaEmbedUtils;
+import org.jruby.javasupport.JavaUtil;
+import org.jruby.runtime.Helpers;
+import org.jruby.runtime.builtin.IRubyObject;
+
+public class MapleRoadMapper extends MapReduceBase implements Mapper<LongWritable, Text, Text, Text> {
+    private String mapper_code = null;
+    private Text keyText = new Text();
+    private Text valueText = new Text();
+    
+    @Override
+    public void configure(JobConf job) {
+        mapper_code = job.get("mapper.code");
+    }
+
+    @Override
+    public void map(LongWritable key, Text value,
+            OutputCollector<Text, Text> outputCollector, Reporter arg3) throws IOException {
+        Ruby runtime = Ruby.getGlobalRuntime();
+        RubyRuntimeAdapter adapter = JavaEmbedUtils.newRuntimeAdapter();
+        IRubyObject receiver = adapter.eval(runtime, mapper_code);
+        IRubyObject rubyKey = JavaUtil.convertJavaToRuby(runtime, key);
+        IRubyObject rubyValue = JavaUtil.convertJavaToRuby(runtime, value);
+        MapleRoadCollector collector =
+                new MapleRoadCollector(keyText, valueText, outputCollector);
+        IRubyObject rubyCollector = JavaUtil.convertJavaToRuby(runtime, collector);
+        Helpers.invoke(runtime.getCurrentContext(), receiver, "map", rubyKey, rubyValue, rubyCollector);
+    }
+
+}

data/ext/mapleroad/MapleRoadMapperTextInt.java

@@ -0,0 +1,44 @@
+package mapleroad;
+
+import java.io.IOException;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.Mapper;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reporter;
+import org.jruby.Ruby;
+import org.jruby.RubyRuntimeAdapter;
+import org.jruby.javasupport.JavaEmbedUtils;
+import org.jruby.javasupport.JavaUtil;
+import org.jruby.runtime.Helpers;
+import org.jruby.runtime.builtin.IRubyObject;
+
+public class MapleRoadMapperTextInt extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {
+    private String mapper_code = null;
+    private Text keyText = new Text();
+    private IntWritable valueInt = new IntWritable();
+    
+    @Override
+    public void configure(JobConf job) {
+        mapper_code = job.get("mapper.code");
+    }
+
+    @Override
+    public void map(LongWritable key, Text value,
+            OutputCollector<Text, IntWritable> outputCollector, Reporter arg3) throws IOException {
+        Ruby runtime = Ruby.getGlobalRuntime();
+        RubyRuntimeAdapter adapter = JavaEmbedUtils.newRuntimeAdapter();
+        IRubyObject receiver = adapter.eval(runtime, mapper_code);
+        IRubyObject rubyKey = JavaUtil.convertJavaToRuby(runtime, key);
+        IRubyObject rubyValue = JavaUtil.convertJavaToRuby(runtime, value);
+        MapleRoadCollectorTextInt collector =
+                new MapleRoadCollectorTextInt(keyText, valueInt, outputCollector);
+        IRubyObject rubyCollector = JavaUtil.convertJavaToRuby(runtime, collector);
+        Helpers.invoke(runtime.getCurrentContext(), receiver, "map", rubyKey, rubyValue, rubyCollector);
+    }
+
+}

data/ext/mapleroad/MapleRoadReducer.java

@@ -0,0 +1,43 @@
+package mapleroad;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.jruby.Ruby;
+import org.jruby.RubyRuntimeAdapter;
+import org.jruby.javasupport.JavaEmbedUtils;
+import org.jruby.javasupport.JavaUtil;
+import org.jruby.runtime.Helpers;
+import org.jruby.runtime.builtin.IRubyObject;
+
+public class MapleRoadReducer extends MapReduceBase implements Reducer<Text, Text, Text, Text> {
+    private String reducer_code = null;
+    private Text keyText = new Text();
+    private Text valueText = new Text();
+    
+    @Override
+    public void configure(JobConf job) {
+        reducer_code = job.get("reducer.code");
+    }
+
+    @Override
+    public void reduce(Text key, Iterator<Text> values,
+            OutputCollector<Text, Text> outputCollector, Reporter arg3) throws IOException {;
+        Ruby runtime = Ruby.getGlobalRuntime();
+        RubyRuntimeAdapter adapter = JavaEmbedUtils.newRuntimeAdapter();
+        IRubyObject receiver = adapter.eval(runtime, reducer_code);
+        IRubyObject rubyKey = JavaUtil.convertJavaToRuby(runtime, key);
+        IRubyObject rubyValues = JavaUtil.convertJavaToRuby(runtime, values);
+        MapleRoadCollector collector =
+                new MapleRoadCollector(keyText, valueText, outputCollector);
+        IRubyObject rubyCollector = JavaUtil.convertJavaToRuby(runtime, collector);
+        Helpers.invoke(runtime.getCurrentContext(), receiver, "reduce", rubyKey, rubyValues, rubyCollector);
+    }
+
+}

data/ext/mapleroad/MapleRoadReducerTextInt.java

@@ -0,0 +1,44 @@
+package mapleroad;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.MapReduceBase;
+import org.apache.hadoop.mapred.OutputCollector;
+import org.apache.hadoop.mapred.Reducer;
+import org.apache.hadoop.mapred.Reporter;
+import org.jruby.Ruby;
+import org.jruby.RubyRuntimeAdapter;
+import org.jruby.javasupport.JavaEmbedUtils;
+import org.jruby.javasupport.JavaUtil;
+import org.jruby.runtime.Helpers;
+import org.jruby.runtime.builtin.IRubyObject;
+
+public class MapleRoadReducerTextInt extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable> {
+    private String reducer_code = null;
+    private Text keyText = new Text();
+    private IntWritable valueInt = new IntWritable();
+    
+    @Override
+    public void configure(JobConf job) {
+        reducer_code = job.get("reducer.code");
+    }
+
+    @Override
+    public void reduce(Text key, Iterator<IntWritable> values,
+            OutputCollector<Text, IntWritable> outputCollector, Reporter arg3) throws IOException {;
+        Ruby runtime = Ruby.getGlobalRuntime();
+        RubyRuntimeAdapter adapter = JavaEmbedUtils.newRuntimeAdapter();
+        IRubyObject receiver = adapter.eval(runtime, reducer_code);
+        IRubyObject rubyKey = JavaUtil.convertJavaToRuby(runtime, key);
+        IRubyObject rubyValues = JavaUtil.convertJavaToRuby(runtime, values);
+        MapleRoadCollectorTextInt collector =
+                new MapleRoadCollectorTextInt(keyText, valueInt, outputCollector);
+        IRubyObject rubyCollector = JavaUtil.convertJavaToRuby(runtime, collector);
+        Helpers.invoke(runtime.getCurrentContext(), receiver, "reduce", rubyKey, rubyValues, rubyCollector);
+    }
+
+}

data/lib/mapleroad/version.rb

@@ -0,0 +1,3 @@
+module MapleRoad
+  VERSION = '0.0.1.pre'
+end

data/mapleroad.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'mapleroad/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "mapleroad"
+  spec.version       = MapleRoad::VERSION
+  spec.authors       = ["Yoko Harada"]
+  spec.email         = ["yokolet@gmail.com"]
+  spec.summary       = %q{Proof of concept gem for hadoop's mapreduce}
+  spec.description   = %q{This gem is a proof of concept that mapper and reducer
+                          of mapreduce can be written in Ruby.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+end

data/samples/anagram/mapper.rb

@@ -0,0 +1,8 @@
+class AnagramMapper
+  def map(key, value, collector)
+    word = value.to_s
+    sorted_word = word.scan(/./).sort.join
+    collector.collect(sorted_word, word)
+  end
+end
+AnagramMapper.new

data/samples/anagram/reducer.rb

@@ -0,0 +1,11 @@
+class AnagramReducer
+  def reduce(key, values, collector)
+    output = values.inject("") do |memo, v|
+      memo = memo + v.to_s + "~"
+    end
+    if (output.split("~").length >= 2)
+      collector.collect(key.to_s, output.gsub("~", ","))
+    end
+  end
+end
+AnagramReducer.new

data/samples/wordcount/mapper.rb

@@ -0,0 +1,8 @@
+class WordCounterMapper
+  def map(key, value, collector)
+    value.to_s.split.each do |word|
+      collector.collect(word, 1)
+    end
+  end
+end
+WordCounterMapper.new

data/samples/wordcount/reducer.rb

@@ -0,0 +1,9 @@
+class WordCounterReducer
+  def reduce(key, values, collector)
+    sum = values.inject(0) do |memo, v|
+      memo = memo + v.to_string.to_i
+    end
+    collector.collect(key, sum)
+  end
+end
+WordCounterReducer.new

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification
+name: mapleroad
+version: !ruby/object:Gem::Version
+  version: 0.0.1.pre
+platform: ruby
+authors:
+- Yoko Harada
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2014-06-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+description: |-
+  This gem is a proof of concept that mapper and reducer
+                            of mapreduce can be written in Ruby.
+email:
+- yokolet@gmail.com
+executables:
+- mapleroad
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- Gemfile.lock
+- Jarfile
+- Jarfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/mapleroad
+- ext/mapleroad/MapleRoadCollector.java
+- ext/mapleroad/MapleRoadCollectorTextInt.java
+- ext/mapleroad/MapleRoadJob.java
+- ext/mapleroad/MapleRoadMapper.java
+- ext/mapleroad/MapleRoadMapperTextInt.java
+- ext/mapleroad/MapleRoadReducer.java
+- ext/mapleroad/MapleRoadReducerTextInt.java
+- lib/mapleroad/version.rb
+- lib/mapleroad_service.jar
+- mapleroad.gemspec
+- samples/anagram/mapper.rb
+- samples/anagram/reducer.rb
+- samples/wordcount/mapper.rb
+- samples/wordcount/reducer.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project:
+rubygems_version: 2.2.2
+signing_key:
+specification_version: 4
+summary: Proof of concept gem for hadoop's mapreduce
+test_files: []