jruby-on-hadoop 0.0.3 → 0.0.4

data/README.rdoc

@@ -1,6 +1,9 @@
 = JRuby on Hadoop
 
 JRuby on Hadoop is a thin wrapper for Hadoop Mapper / Reducer by JRuby.
+We recommend to use this with hadoop-rubydsl on the github / gemcutter.
+
+== Description
 
 == Install
 
@@ -10,7 +13,7 @@ Required gems are all on GemCutter.
 2. Install gems
  $ gem install jruby-on-hadoop
 
-== Description
+== Usage
 
 1. Run Hadoop cluster on your machines and set HADOOP_HOME env variable.
 2. put files into your hdfs. ex) test/inputs/file1
@@ -18,18 +21,27 @@ Required gems are all on GemCutter.
  $ joh examples/wordcount.rb test/inputs test/outputs
 You can get Hadoop job results in your hdfs test/outputs/part-*
 
-Script example. (see also examples/wordcount.rb)
+== Example 
+see also examples/wordcount.rb
 
  def setup(conf)
    # setup jobconf
  end
 
- def map(script, key, value, output, reporter)
+ def map(key, value, output, reporter)
    # mapper process
+   # (wordcount example)
+   value.split.each do |word|
+     output.collect(word, 1)
+   end
  end
  
- def reduce(script, key, values, output, reporter)
+ def reduce(key, values, output, reporter)
    # reducer process
+   # (wordcount example)
+   sum = 0
+   values.each {|v| sum += v }
+   output.collect(key, sum)
  end
 
 == Build

data/VERSION

@@ -1 +1 @@
-0.0.3
+0.0.4

data/jruby-on-hadoop.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{jruby-on-hadoop}
-  s.version = "0.0.3"
+  s.version = "0.0.4"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Koichi Fujikawa"]
-  s.date = %q{2010-01-03}
+  s.date = %q{2010-01-15}
   s.default_executable = %q{joh}
   s.description = %q{JRuby on Hadoop}
   s.email = %q{fujibee@gmail.com}
@@ -29,6 +29,7 @@ Gem::Specification.new do |s|
      "lib/jruby-on-hadoop.rb",
      "lib/jruby-on-hadoop/client.rb",
      "lib/ruby_wrapper.rb",
+     "spec/jruby-on-hadoop/client_spec.rb",
      "spec/jruby-on-hadoop_spec.rb",
      "spec/ruby_wrapper_spec.rb",
      "test/java/org/apache/hadoop/ruby/JRubyJobRunnerTest.java",
@@ -41,7 +42,8 @@ Gem::Specification.new do |s|
   s.rubygems_version = %q{1.3.5}
   s.summary = %q{JRuby on Hadoop}
   s.test_files = [
-    "spec/jruby-on-hadoop_spec.rb",
+    "spec/jruby-on-hadoop/client_spec.rb",
+     "spec/jruby-on-hadoop_spec.rb",
      "spec/ruby_wrapper_spec.rb",
      "examples/wordcount.rb"
   ]

data/lib/jruby-on-hadoop/client.rb

@@ -9,18 +9,29 @@ module JRubyOnHadoop
       parse_args
 
       # env check
-      hadoop_home = ENV['HADOOP_HOME']
-      raise 'HADOOP_HOME is not set' unless hadoop_home 
-      @hadoop_cmd = "#{hadoop_home}/bin/hadoop"
+      hadoop_home and hadoop_cmd
       ENV['HADOOP_CLASSPATH'] = "#{lib_path}:#{File.dirname(@script_path)}"
     end
 
+    def hadoop_home
+      home = ENV['HADOOP_HOME']
+      raise 'HADOOP_HOME is not set' if home.nil? or home.empty?
+      home
+    end
+
+    def hadoop_cmd
+      hadoop = `which hadoop 2>/dev/null`
+      hadoop = "#{hadoop_home}/bin/hadoop" if hadoop.nil? or hadoop.empty?
+      raise 'cannot find hadoop command' unless hadoop
+      hadoop.chomp
+    end
+
     def run
       exec cmd
     end
 
     def cmd
-      "#{@hadoop_cmd} jar #{main_jar_path} #{JAVA_MAIN_CLASS}" +
+      "#{hadoop_cmd} jar #{main_jar_path} #{JAVA_MAIN_CLASS}" +
       " -libjars #{opt_libjars} -files #{opt_files} #{mapred_args}"
     end
 

data/spec/jruby-on-hadoop/client_spec.rb

@@ -0,0 +1,79 @@
+require 'jruby-on-hadoop'
+
+describe JRubyOnHadoop::Client do
+  before do
+    @client = JRubyOnHadoop::Client.new
+  end
+
+  it 'gather necessary jar paths' do
+    version_pattern = '[\d\.]*'
+    @client.main_jar_path.should include 'hadoop-ruby.jar'
+
+    @client.opt_libjars.should match /jruby\-core\-#{version_pattern}\.jar/
+    @client.opt_libjars.should match /jruby\-stdlib\-#{version_pattern}\.jar/
+  end
+
+  it 'gather necessary ruby files' do
+    @client.opt_files.split(",").should include "mapred.rb"
+    @client.opt_files.should match /ruby_wrapper\.rb/
+  end
+
+  it 'construct command for running hadoop' do
+    path_pattern = '[\w/\-\.,]*'
+    @client.cmd.should match /hadoop jar #{path_pattern}hadoop-ruby.jar org.apache.hadoop.ruby.JRubyJobRunner -libjars #{path_pattern}.jar -files mapred.rb/
+  end
+
+  it 'can get mapred args' do
+    client = JRubyOnHadoop::Client.new(["examples/mapred.rb", "inputs", "outputs"])
+    client.mapred_args.should == "--script mapred.rb inputs outputs"
+  end
+
+  it 'can parse args' do
+    client = JRubyOnHadoop::Client.new(["examples/mapred.rb", "in", "out"])
+    client.script.should == 'mapred.rb'
+    client.inputs.should == 'in'
+    client.outputs.should == 'out'
+    client.files.should include 'examples/mapred.rb'
+  end
+
+  it 'should raise error if HADOOP_HOME env is not set' do
+    saved = ENV['HADOOP_HOME']
+    ENV['HADOOP_HOME'] = ''
+    begin
+      lambda { JRubyOnHadoop::Client.new }.should raise_error
+    ensure
+      ENV['HADOOP_HOME'] = saved
+    end
+  end
+
+  it 'can determin bin/hadoop path' do
+    @client.hadoop_cmd.should match /hadoop$/
+  end
+
+  it 'can determin bin/hadoop path if even no in PATH env var' do
+    saved = ENV['PATH']
+    begin
+      ENV['PATH'] = ''
+      ENV['HADOOP_HOME'].should_not be_empty
+      client = JRubyOnHadoop::Client.new
+      client.hadoop_cmd.should match ENV['HADOOP_HOME']
+      client.hadoop_cmd.should match /hadoop$/
+    ensure
+      ENV['PATH'] = saved
+    end
+  end
+
+  it 'should raise error if cannot determin bin/hadoop path' do
+    saved_path = ENV['PATH']
+    saved_home = ENV['HADOOP_HOME']
+    begin
+      ENV['PATH'] = ''
+      lambda { JRubyOnHadoop::Client.new }.should_not raise_error
+      ENV['HADOOP_HOME'] = ''
+      lambda { JRubyOnHadoop::Client.new }.should raise_error
+    ensure
+      ENV['PATH'] = saved_path
+      ENV['HADOOP_HOME'] = saved_home
+    end
+  end
+end

data/spec/jruby-on-hadoop_spec.rb

@@ -18,40 +18,3 @@ describe JRubyOnHadoop do
     JRubyOnHadoop.wrapper_ruby_file.should == path
   end
 end
-
-describe JRubyOnHadoop::Client do
-  before do
-    @client = JRubyOnHadoop::Client.new
-  end
-
-  it 'gather necessary jar paths' do
-    version_pattern = '[\d\.]*'
-    @client.main_jar_path.should include 'hadoop-ruby.jar'
-
-    @client.opt_libjars.should match /jruby\-core\-#{version_pattern}\.jar/
-    @client.opt_libjars.should match /jruby\-stdlib\-#{version_pattern}\.jar/
-  end
-
-  it 'gather necessary ruby files' do
-    @client.opt_files.split(",").should include "mapred.rb"
-    @client.opt_files.should match /ruby_wrapper\.rb/
-  end
-
-  it 'construct command for running hadoop' do
-    path_pattern = '[\w/\-\.,]*'
-    @client.cmd.should match /hadoop jar #{path_pattern}hadoop-ruby.jar org.apache.hadoop.ruby.JRubyJobRunner -libjars #{path_pattern}.jar -files mapred.rb/
-  end
-
-  it 'can get mapred args' do
-    client = JRubyOnHadoop::Client.new(["examples/mapred.rb", "inputs", "outputs"])
-    client.mapred_args.should == "--script mapred.rb inputs outputs"
-  end
-
-  it 'can parse args' do
-    client = JRubyOnHadoop::Client.new(["examples/mapred.rb", "in", "out"])
-    client.script.should == 'mapred.rb'
-    client.inputs.should == 'in'
-    client.outputs.should == 'out'
-    client.files.should include 'examples/mapred.rb'
-  end
-end

data/spec/ruby_wrapper_spec.rb

@@ -5,7 +5,7 @@ describe 'wrapper' do
     examples_dir = File.expand_path(File.join(File.dirname(__FILE__), '..', 'examples'))
     $: << examples_dir
 
-    @script = 'mapred.rb'
+    @script = 'wordcount.rb'
     @output, @repoter = mock('output'), mock('repoter') 
 
     @key, @value = Text.new, Text.new

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: jruby-on-hadoop
 version: !ruby/object:Gem::Version 
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors: 
 - Koichi Fujikawa
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-01-03 00:00:00 +09:00
+date: 2010-01-15 00:00:00 +09:00
 default_executable: joh
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -42,6 +42,7 @@ files:
 - lib/jruby-on-hadoop.rb
 - lib/jruby-on-hadoop/client.rb
 - lib/ruby_wrapper.rb
+- spec/jruby-on-hadoop/client_spec.rb
 - spec/jruby-on-hadoop_spec.rb
 - spec/ruby_wrapper_spec.rb
 - test/java/org/apache/hadoop/ruby/JRubyJobRunnerTest.java
@@ -76,6 +77,7 @@ signing_key:
 specification_version: 3
 summary: JRuby on Hadoop
 test_files: 
+- spec/jruby-on-hadoop/client_spec.rb
 - spec/jruby-on-hadoop_spec.rb
 - spec/ruby_wrapper_spec.rb
 - examples/wordcount.rb