wukong 1.4.2 → 1.4.5

data/lib/wukong/script/hadoop_command.rb

@@ -15,26 +15,24 @@ module Wukong
     #
     # Translate the simplified args to their hairy-assed hadoop equivalents
     #
-    HADOOP_OPTIONS_MAP = {
-      :max_node_map_tasks     => 'mapred.tasktracker.map.tasks.maximum',
-      :max_node_reduce_tasks  => 'mapred.tasktracker.reduce.tasks.maximum',
-      :map_tasks              => 'mapred.map.tasks',
-      :reduce_tasks           => 'mapred.reduce.tasks',
-      :sort_fields            => 'stream.num.map.output.key.fields',
-      :key_field_separator    => 'map.output.key.field.separator',
-      :partition_fields       => 'num.key.fields.for.partition',
-      :output_field_separator => 'stream.map.output.field.separator',
-      :map_speculative        => 'mapred.map.tasks.speculative.execution',
-      :timeout                => 'mapred.task.timeout',
-      :reuse_jvms             => 'mapred.job.reuse.jvm.num.tasks',
-      :respect_exit_status    => 'stream.non.zero.exit.is.failure',
-    }
+    Settings.define :max_node_map_tasks,     :jobconf => true, :description => 'mapred.tasktracker.map.tasks.maximum', :wukong => true
+    Settings.define :max_node_reduce_tasks,  :jobconf => true, :description => 'mapred.tasktracker.reduce.tasks.maximum', :wukong => true
+    Settings.define :map_tasks,              :jobconf => true, :description => 'mapred.map.tasks', :wukong => true
+    Settings.define :reduce_tasks,           :jobconf => true, :description => 'mapred.reduce.tasks', :wukong => true
+    Settings.define :sort_fields,            :jobconf => true, :description => 'stream.num.map.output.key.fields', :wukong => true
+    Settings.define :key_field_separator,    :jobconf => true, :description => 'map.output.key.field.separator', :wukong => true
+    Settings.define :partition_fields,       :jobconf => true, :description => 'num.key.fields.for.partition', :wukong => true
+    Settings.define :output_field_separator, :jobconf => true, :description => 'stream.map.output.field.separator', :wukong => true
+    Settings.define :map_speculative,        :jobconf => true, :description => 'mapred.map.tasks.speculative.execution', :wukong => true
+    Settings.define :timeout,                :jobconf => true, :description => 'mapred.task.timeout', :wukong => true
+    Settings.define :reuse_jvms,             :jobconf => true, :description => 'mapred.job.reuse.jvm.num.tasks', :wukong => true
+    Settings.define :respect_exit_status,    :jobconf => true, :description => 'stream.non.zero.exit.is.failure', :wukong => true
 
     # emit a -jobconf hadoop option if the simplified command line arg is present
     # if not, the resulting nil will be elided later
     def jobconf option
       if options[option]
-        "-jobconf %s=%s" % [HADOOP_OPTIONS_MAP[option], options[option]]
+        "-jobconf %s=%s" % [options.description_for(option), options[option]]
       end
     end
 
@@ -81,17 +79,21 @@ module Wukong
       end.compact
     end
 
+    # The path to the hadoop runner script
+    def hadoop_runner
+      options[:hadoop_runner] || (options[:hadoop_home]+'/bin/hadoop')
+    end
+
     #
     # Assemble the hadoop command to execute
     #
     def hadoop_command input_path, output_path
       # If this is wrong, create a config/wukong-site.rb or
-      # otherwise set Wukong::CONFIG[:hadoop_home] to the
+      # otherwise set Settings[:hadoop_home] to the
       # root of your config install.
-      hadoop_program = Wukong::CONFIG[:hadoop_home]+'/bin/hadoop'
       [
-        hadoop_program,
-        "jar #{Wukong::CONFIG[:hadoop_home]}/contrib/streaming/hadoop-*-streaming.jar",
+        hadoop_runner,
+        "jar #{Settings[:hadoop_home]}/contrib/streaming/hadoop-*-streaming.jar",
         hadoop_partition_args,
         hadoop_sort_args,
         hadoop_num_tasks_args,

data/lib/wukong/script/local_command.rb

@@ -6,8 +6,16 @@ module Wukong
     # Local execution Options
     #
 
+    # program, including arg, to sort input between mapper and reducer in local
+    # mode. You could override to for example run 'sort -n' (numeric sort).
+    def sort_command
+      'sort'
+    end
+
     def local_command input_path, output_path
-      %Q{ cat #{input_path} | #{map_command} | sort | #{reduce_command} > '#{output_path}'}
+      cmd_input_str  = (input_path  == '-') ? "" : "cat '#{input_path}' | "
+      cmd_output_str = (output_path == '-') ? "" : "> '#{output_path}'"
+      %Q{ #{cmd_input_str} #{map_command} | #{sort_command} | #{reduce_command} #{cmd_output_str} }
     end
 
   end

data/lib/wukong/streamer/base.rb

@@ -17,7 +17,7 @@ module Wukong
       # Pass each record to +#process+
       #
       def stream
-        Log.info("Streaming on:\t%s" % [Script.input_file])
+        Log.info("Streaming on:\t%s" % [Script.input_file]) unless Script.input_file.blank?
         before_stream
         $stdin.each do |line|
           record = recordize(line.chomp)

data/spec/spec.opts

@@ -0,0 +1 @@
+--color

data/spec/spec_helper.rb

@@ -0,0 +1,11 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
+require 'wukong'
+require 'spec'
+require 'spec/autorun'
+
+Spec::Runner.configure do |config|
+
+end
+

data/spec/wukong/encoding_spec.rb

@@ -0,0 +1,36 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+require 'rubygems'
+require 'wukong/encoding'
+
+describe "Wukong encoding" do
+  before do
+  end
+
+  it 'en/decodes to xml by default' do
+    Wukong.encode_str("&").should == '&'
+    Wukong.decode_str("&").should == '&'
+  end
+  it 'en/decodes to xml with :xml' do
+    Wukong.encode_str("&", :xml).should == '&'
+    Wukong.decode_str("&", :xml).should == '&'
+  end
+  it 'url en/decodes with :url' do
+    Wukong.encode_str("&", :url).should == '%26'
+    Wukong.decode_str("%26", :url).should == '&'
+  end
+  { "'" => "'", "\t" => "	", "\n" => "
", nil => '',}.each do |raw, enc|
+    it 'encodes #{raw} to #{enc}' do
+      Wukong.encode_str(raw, :xml).should == enc
+    end
+    it 'decodes #{enc} to #{raw}' do
+      Wukong.decode_str(enc, :xml).should == raw.to_s
+    end
+  end
+  ["normal_string with %punctuation should `not be molested", ""].each do |str|
+    it 'doesn\'t change #{str}' do
+      Wukong.encode_str(str, :xml).should == str
+    end
+  end
+
+
+end

data/spec/wukong/script_spec.rb

@@ -0,0 +1,80 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe "Wukong::Script" do
+  before do
+    ARGV.replace []
+    @script = Wukong::Script.new 'mapper', 'reducer'
+  end
+
+  describe 'initialize' do
+    it 'sets :reduce_tasks to 0 if reducer_klass is nil and no reduce_command or explicit setting' do
+      @script = Wukong::Script.new 'mapper', nil
+      @script.options[:reduce_tasks].should == 0
+    end
+    it 'respects :reduce_tasks if set even if reducer_klass is nil' do
+      @script = Wukong::Script.new 'mapper', nil, :reduce_tasks => 1
+      @script.options[:reduce_tasks].should == 1
+    end
+    it "doesn't set :reduce_tasks reduce_command is given" do
+      @script = Wukong::Script.new 'mapper', nil, :reduce_command => 1
+      @script.options[:reduce_tasks].should be_nil
+    end
+    it 'sets mapper_klass in initializer' do
+      @script.mapper_klass.should == 'mapper'
+    end
+    it 'sets reducer_klass in initializer' do
+      @script.reducer_klass.should == 'reducer'
+    end
+  end
+
+  describe 'child processes' do
+    it 'calls self if a mapper_klass is set' do
+      @script.should_receive(:ruby_interpreter_path).and_return('/path/to/ruby')
+      @script.should_receive(:this_script_filename).and_return('/path/to/this_script')
+      @script.map_command.should == %Q{/path/to/ruby /path/to/this_script --map }
+    end
+    it 'calls default_mapper if nil mapper_klass and no map_command is set' do
+      @script = Wukong::Script.new nil, 'reducer', :default_mapper => 'default_mapper'
+      @script.map_command.should == 'default_mapper'
+    end
+    it 'calls map_command if nil mapper_klass and map_command is set' do
+      @script = Wukong::Script.new nil, 'reducer', :map_command => 'map_command', :default_mapper => 'default_mapper'
+      @script.map_command.should == 'map_command'
+    end
+
+    it 'calls self if a reducer_klass is set' do
+      @script.should_receive(:ruby_interpreter_path).and_return('/path/to/ruby')
+      @script.should_receive(:this_script_filename).and_return('/path/to/this_script')
+      @script.reduce_command.should == %Q{/path/to/ruby /path/to/this_script --reduce }
+    end
+    it 'calls default_reducer if nil reducer_klass and no reduce_command is set' do
+      @script = Wukong::Script.new 'mapper', nil, :default_reducer => 'default_reducer'
+      @script.reduce_command.should == 'default_reducer'
+    end
+    it 'calls reduce_command if nil reducer_klass and reduce_command is set' do
+      @script = Wukong::Script.new 'mapper', nil, :reduce_command => 'reduce_command', :default_reducer => 'default_reducer'
+      @script.reduce_command.should == 'reduce_command'
+    end
+
+    it 'runs script | sort | script when in local mode' do
+      @script.should_receive(:run_mode).and_return('local')
+      @script.should_receive(:map_command).and_return('map_command')
+      @script.should_receive(:reduce_command).and_return('reduce_command')
+      @script.runner_command("/path/in", "/path/out").should == %Q{ cat '/path/in' | map_command | sort | reduce_command > '/path/out'}
+    end
+
+    it 'calls out to hadoop when in non-local mode' do
+      @script.should_receive(:run_mode).and_return('hadoop')
+      @script.should_receive(:hadoop_command).and_return('hadoop_command whee!')
+      @script.runner_command("/path/in", "/path/out").should == 'hadoop_command whee!'
+    end
+  end
+
+  describe 'runner phase'
+  it 'preserves non-internal-to-wukong params in non_wukong_params' do
+    @script.options[:foo] = 'bar'
+    @script.non_wukong_params.should == "--foo=bar"
+  end
+
+
+end

data/wukong.gemspec

@@ -1,15 +1,15 @@
 # Generated by jeweler
-# DO NOT EDIT THIS FILE
-# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
 # -*- encoding: utf-8 -*-
 
 Gem::Specification.new do |s|
   s.name = %q{wukong}
-  s.version = "1.4.2"
+  s.version = "1.4.5"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Philip (flip) Kromer"]
-  s.date = %q{2009-12-15}
+  s.date = %q{2010-01-26}
   s.description = %q{  Treat your dataset like a:
 
       * stream of lines when it’s efficient to process by lines
@@ -25,7 +25,8 @@ Gem::Specification.new do |s|
      "README.textile"
   ]
   s.files = [
-    "INSTALL.textile",
+    "CHANGELOG.textile",
+     "INSTALL.textile",
      "LICENSE.textile",
      "README.textile",
      "bin/cutc",
@@ -59,47 +60,40 @@ Gem::Specification.new do |s|
      "bin/wu-plus",
      "bin/wu-sum",
      "docpages/INSTALL.textile",
-     "docpages/INSTALL.textile",
-     "docpages/LICENSE.textile",
      "docpages/LICENSE.textile",
      "docpages/README-wulign.textile",
-     "docpages/README-wulign.textile",
      "docpages/UsingWukong-part1-get_ready.textile",
-     "docpages/UsingWukong-part1-get_ready.textile",
-     "docpages/UsingWukong-part2-ThinkingBigData.textile",
      "docpages/UsingWukong-part2-ThinkingBigData.textile",
      "docpages/UsingWukong-part3-parsing.textile",
-     "docpages/UsingWukong-part3-parsing.textile",
      "docpages/_config.yml",
      "docpages/bigdata-tips.textile",
-     "docpages/bigdata-tips.textile",
      "docpages/code/api_response_example.txt",
      "docpages/code/parser_skeleton.rb",
      "docpages/diagrams/MapReduceDiagram.graffle",
      "docpages/favicon.ico",
      "docpages/gem.css",
      "docpages/hadoop-tips.textile",
-     "docpages/hadoop-tips.textile",
-     "docpages/index.textile",
      "docpages/index.textile",
      "docpages/intro.textile",
-     "docpages/intro.textile",
-     "docpages/moreinfo.textile",
      "docpages/moreinfo.textile",
      "docpages/news.html",
      "docpages/pig/PigLatinExpressionsList.txt",
      "docpages/pig/PigLatinReferenceManual.html",
      "docpages/pig/PigLatinReferenceManual.txt",
      "docpages/tutorial.textile",
-     "docpages/tutorial.textile",
-     "docpages/usage.textile",
      "docpages/usage.textile",
      "docpages/wutils.textile",
-     "docpages/wutils.textile",
      "examples/README.txt",
      "examples/apache_log_parser.rb",
+     "examples/contrib/jeans/README.markdown",
+     "examples/contrib/jeans/data/normalized_sizes",
+     "examples/contrib/jeans/data/orders.tsv",
+     "examples/contrib/jeans/data/sizes",
+     "examples/contrib/jeans/normalize.rb",
+     "examples/contrib/jeans/sizes.rb",
      "examples/count_keys.rb",
      "examples/count_keys_at_mapper.rb",
+     "examples/foo.rb",
      "examples/graph/adjacency_list.rb",
      "examples/graph/breadth_first_search.rb",
      "examples/graph/gen_2paths.rb",
@@ -108,7 +102,6 @@ Gem::Specification.new do |s|
      "examples/package-local.rb",
      "examples/package.rb",
      "examples/pagerank/README.textile",
-     "examples/pagerank/README.textile",
      "examples/pagerank/gen_initial_pagerank_graph.pig",
      "examples/pagerank/pagerank.rb",
      "examples/pagerank/pagerank_initialize.rb",
@@ -120,7 +113,6 @@ Gem::Specification.new do |s|
      "examples/word_count.rb",
      "lib/wukong.rb",
      "lib/wukong/bad_record.rb",
-     "lib/wukong/boot.rb",
      "lib/wukong/datatypes.rb",
      "lib/wukong/datatypes/enum.rb",
      "lib/wukong/datatypes/fake_types.rb",
@@ -165,12 +157,14 @@ Gem::Specification.new do |s|
      "lib/wukong/streamer/uniq_by_last_reducer.rb",
      "lib/wukong/typed_struct.rb",
      "lib/wukong/wukong_class.rb",
-     "spec/bin/hdp-wc_spec.rb",
      "spec/data/a_atsigns_b.tsv",
      "spec/data/a_follows_b.tsv",
      "spec/data/tweet.tsv",
      "spec/data/twitter_user.tsv",
+     "spec/spec.opts",
      "spec/spec_helper.rb",
+     "spec/wukong/encoding_spec.rb",
+     "spec/wukong/script_spec.rb",
      "wukong.gemspec"
   ]
   s.homepage = %q{http://mrflip.github.com/wukong}
@@ -179,11 +173,15 @@ Gem::Specification.new do |s|
   s.rubygems_version = %q{1.3.5}
   s.summary = %q{Wukong makes Hadoop so easy a chimpanzee can use it.}
   s.test_files = [
-    "spec/bin/hdp-wc_spec.rb",
-     "spec/spec_helper.rb",
+    "spec/spec_helper.rb",
+     "spec/wukong/encoding_spec.rb",
+     "spec/wukong/script_spec.rb",
      "examples/apache_log_parser.rb",
+     "examples/contrib/jeans/normalize.rb",
+     "examples/contrib/jeans/sizes.rb",
      "examples/count_keys.rb",
      "examples/count_keys_at_mapper.rb",
+     "examples/foo.rb",
      "examples/graph/adjacency_list.rb",
      "examples/graph/breadth_first_search.rb",
      "examples/graph/gen_2paths.rb",
@@ -218,3 +216,4 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<htmlentities>, [">= 0"])
   end
 end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: wukong
 version: !ruby/object:Gem::Version 
-  version: 1.4.2
+  version: 1.4.5
 platform: ruby
 authors: 
 - Philip (flip) Kromer
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-12-15 00:00:00 -06:00
+date: 2010-01-26 00:00:00 -06:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -57,6 +57,7 @@ extra_rdoc_files:
 - LICENSE.textile
 - README.textile
 files: 
+- CHANGELOG.textile
 - INSTALL.textile
 - LICENSE.textile
 - README.textile
@@ -116,8 +117,15 @@ files:
 - docpages/wutils.textile
 - examples/README.txt
 - examples/apache_log_parser.rb
+- examples/contrib/jeans/README.markdown
+- examples/contrib/jeans/data/normalized_sizes
+- examples/contrib/jeans/data/orders.tsv
+- examples/contrib/jeans/data/sizes
+- examples/contrib/jeans/normalize.rb
+- examples/contrib/jeans/sizes.rb
 - examples/count_keys.rb
 - examples/count_keys_at_mapper.rb
+- examples/foo.rb
 - examples/graph/adjacency_list.rb
 - examples/graph/breadth_first_search.rb
 - examples/graph/gen_2paths.rb
@@ -137,7 +145,6 @@ files:
 - examples/word_count.rb
 - lib/wukong.rb
 - lib/wukong/bad_record.rb
-- lib/wukong/boot.rb
 - lib/wukong/datatypes.rb
 - lib/wukong/datatypes/enum.rb
 - lib/wukong/datatypes/fake_types.rb
@@ -182,12 +189,14 @@ files:
 - lib/wukong/streamer/uniq_by_last_reducer.rb
 - lib/wukong/typed_struct.rb
 - lib/wukong/wukong_class.rb
-- spec/bin/hdp-wc_spec.rb
 - spec/data/a_atsigns_b.tsv
 - spec/data/a_follows_b.tsv
 - spec/data/tweet.tsv
 - spec/data/twitter_user.tsv
+- spec/spec.opts
 - spec/spec_helper.rb
+- spec/wukong/encoding_spec.rb
+- spec/wukong/script_spec.rb
 - wukong.gemspec
 has_rdoc: true
 homepage: http://mrflip.github.com/wukong
@@ -218,11 +227,15 @@ signing_key:
 specification_version: 3
 summary: Wukong makes Hadoop so easy a chimpanzee can use it.
 test_files: 
-- spec/bin/hdp-wc_spec.rb
 - spec/spec_helper.rb
+- spec/wukong/encoding_spec.rb
+- spec/wukong/script_spec.rb
 - examples/apache_log_parser.rb
+- examples/contrib/jeans/normalize.rb
+- examples/contrib/jeans/sizes.rb
 - examples/count_keys.rb
 - examples/count_keys_at_mapper.rb
+- examples/foo.rb
 - examples/graph/adjacency_list.rb
 - examples/graph/breadth_first_search.rb
 - examples/graph/gen_2paths.rb