wukong-hadoop 0.0.1

data/bin/hdp-sync

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+require 'wukong'
+
+src_dir, dest_dir = ARGV[0..1]
+src_files   = Dir[src_dir + '/*']
+dest_files  = Wukong::Dfs.list_files dest_dir
+Wukong::Dfs.compare_listings(src_files, dest_files) do |comparison, src_file, dest_file|
+  case comparison
+  when :missing
+    dest_filename = "%s/%s" % [dest_dir, dest_file]
+    puts "Copying #{src_file} #{dest_filename}"
+    puts `hadoop dfs -put #{src_file} #{dest_filename}`
+  when :differ
+    src_ls = `ls -l #{src_file}`.split(/\s+/).join("\t")
+    puts "Differ: #{src_ls} \n#{dest_file}"
+  end
+end

data/bin/hdp-wc

@@ -0,0 +1,67 @@
+#!/usr/bin/env ruby
+require 'wukong'
+NEWLINE_LENGTH = $/.length # KLUDGE
+
+#
+#
+#
+# !! The +words+ count comes out higher than that of +wc+ -- don't know
+# why. (It's close: a 10GB, 1M line dataset it showed 367833839 vs. 367713271)
+#
+class WcMapper < Wukong::Streamer::LineStreamer
+  attr_accessor :lines, :fields, :words, :chars, :bytes
+
+  def before_stream
+    self.lines, self.fields, self.words, self.chars, self.bytes = [0,0,0,0,0]
+  end
+
+  def process line
+    return unless line
+    self.lines  += 1
+    self.fields += 1 + line.count("\t")
+    self.words  += 1 + line.strip.scan(/\s+/).length unless line.blank?
+    self.chars  += line.chars.to_a.length + NEWLINE_LENGTH
+    self.bytes  += line.bytesize          + NEWLINE_LENGTH
+    $stderr.puts line if (line.chars.to_a.length != line.bytesize)
+  end
+
+  def after_stream
+    emit [lines, fields, words, chars,  bytes]
+  end
+end
+
+#
+#
+class WcReducer < Wukong::Streamer::Base
+  attr_accessor :lines, :fields, :words, :chars, :bytes
+
+  def before_stream
+    self.lines, self.fields, self.words, self.chars, self.bytes = [0,0,0,0,0]
+  end
+
+  def process m_lines, m_fields, m_words, m_chars, m_bytes
+    self.lines  += m_lines.to_i
+    self.fields += m_fields.to_i
+    self.words  += m_words.to_i
+    self.chars  += m_chars.to_i
+    self.bytes  += m_bytes.to_i
+  end
+
+  def after_stream
+    emit [lines, fields, words, chars,  bytes]
+  end
+end
+
+Wukong::Script.new(WcMapper, WcReducer, :reduce_tasks => 1).run
+
+# class FooScript < Wukong::Script
+#   def map_command
+#     '/usr/bin/wc'
+#   end
+#   def reduce_command
+#     '/bin/cat'
+#   end
+# end
+# FooScript.new(nil, nil, :reduce_tasks => 1).run
+#
+#  ruby -ne 'wc_v = `echo "#{$_.chomp}" | wc`; gr_v=($_.strip.empty? ? 0 : $_.strip.scan(/\s+/).length + 1 ) ; puts [wc_v.chomp, " ", gr_v, $_.chomp].join("\t")'

data/bin/wu-hadoop

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'wukong-hadoop'
+settings = Wukong::Hadoop::Configuration
+require 'wukong/boot'
+Wukong.boot!(settings)
+
+settings[:command_prefix] = 'bundle exec' if Wukong.in_deploy_pack?
+
+if settings.rest.empty?
+  settings.dump_help
+  exit(1)
+end
+Wukong::Hadoop::Driver.run(settings, *settings.rest)

data/examples/counter.rb

@@ -0,0 +1,17 @@
+Wukong.processor(:counter, Wukong::Processor::Accumulator) do
+
+  attr_accessor :count
+  
+  def start record
+    self.count = 0
+  end
+  
+  def accumulate record
+    self.count += 1
+  end
+
+  def finalize
+    yield [key, count].join("\t")
+  end
+  
+end

data/examples/map_only.rb

@@ -0,0 +1,28 @@
+Wukong.processor(:mapper) do
+  
+  field :min_length, Integer,  :default => 1
+  field :max_length, Integer,  :default => 256
+  field :split_on,   Regexp,   :default => /\s+/
+  field :remove,     Regexp,   :default => /[^a-zA-Z0-9\']+/
+  field :fold_case,  :boolean, :default => false
+  
+  def process string
+    tokenize(string).each do |token|
+      yield token if acceptable?(token)
+    end
+  end
+
+  private
+
+  def tokenize string
+    string.split(split_on).map do |token|
+      stripped = token.gsub(remove, '')
+      fold_case ? stripped.downcase : stripped
+    end
+  end
+
+  def acceptable? token
+    (min_length..max_length).include?(token.length)
+  end
+  
+end

data/examples/processors.rb

@@ -0,0 +1,4 @@
+load File.join(File.dirname(__FILE__), 'tokenizer.rb')
+load File.join(File.dirname(__FILE__), 'counter.rb')
+
+

data/examples/sonnet_18.txt

@@ -0,0 +1,14 @@
+Shall I compare thee to a summer's day?
+Thou art more lovely and more temperate:
+Rough winds do shake the darling buds of May,
+And summer's lease hath all too short a date:
+Sometime too hot the eye of heaven shines,
+And often is his gold complexion dimm'd;
+And every fair from fair sometime declines,
+By chance or nature's changing course untrimm'd;
+But thy eternal summer shall not fade
+Nor lose possession of that fair thou owest;
+Nor shall Death brag thou wander'st in his shade,
+When in eternal lines to time thou growest:
+So long as men can breathe or eyes can see,
+So long lives this and this gives life to thee.

data/examples/tokenizer.rb

@@ -0,0 +1,28 @@
+Wukong.processor(:tokenizer) do
+  
+  field :min_length, Integer,  :default => 1
+  field :max_length, Integer,  :default => 256
+  field :split_on,   Regexp,   :default => /\s+/
+  field :remove,     Regexp,   :default => /[^a-zA-Z0-9\']+/
+  field :fold_case,  :boolean, :default => false
+  
+  def process string
+    tokenize(string).each do |token|
+      yield token if acceptable?(token)
+    end
+  end
+
+  private
+
+  def tokenize string
+    string.split(split_on).map do |token|
+      stripped = token.gsub(remove, '')
+      fold_case ? stripped.downcase : stripped
+    end
+  end
+
+  def acceptable? token
+    (min_length..max_length).include?(token.length)
+  end
+  
+end

data/examples/word_count.rb

@@ -0,0 +1,44 @@
+Wukong.processor(:mapper) do
+  
+  field :min_length, Integer,  :default => 1
+  field :max_length, Integer,  :default => 256
+  field :split_on,   Regexp,   :default => /\s+/
+  field :remove,     Regexp,   :default => /[^a-zA-Z0-9\']+/
+  field :fold_case,  :boolean, :default => false
+  
+  def process string
+    tokenize(string).each do |token|
+      yield token if acceptable?(token)
+    end
+  end
+
+  private
+
+  def tokenize string
+    string.split(split_on).map do |token|
+      stripped = token.gsub(remove, '')
+      fold_case ? stripped.downcase : stripped
+    end
+  end
+
+  def acceptable? token
+    (min_length..max_length).include?(token.length)
+  end
+end
+
+Wukong.processor(:reducer, Wukong::Processor::Accumulator) do
+
+  attr_accessor :count
+  
+  def start record
+    self.count = 0
+  end
+  
+  def accumulate record
+    self.count += 1
+  end
+
+  def finalize
+    yield [key, count].join("\t")
+  end
+end

data/features/step_definitions/wu_hadoop_steps.rb

@@ -0,0 +1,4 @@
+Given /^a wukong script "(.*?)"$/ do |wu_file|
+  Pathname(wu_file).should exist
+  write_file(wu_file, File.read(wu_file))
+end

data/features/support/env.rb

@@ -0,0 +1 @@
+require 'aruba/cucumber'

data/features/wu_hadoop.feature

@@ -0,0 +1,113 @@
+Feature: Run wu-hadoop from the command line
+  In order to execute hadoop streaming commands
+  As a user of wu-hadoop
+  I should be able run wu-hadoop with wukong processors
+
+  Scenario: Simple wu-hadoop command
+    Given a wukong script "examples/word_count.rb"
+    When  I run `bundle exec wu-hadoop examples/word_count.rb --dry_run --input=/foo --output=/bar `
+    Then  the output should contain:
+    """
+    /usr/lib/hadoop/bin/hadoop 	\
+      jar /usr/lib/hadoop/contrib/streaming/hadoop-*streaming*.jar 	\
+      -D mapred.job.name='word_count.rb---/foo---/bar' 	\
+    """
+    And the output should match:
+    """
+      -mapper       '.*ruby bundle exec wu-local .*word_count.rb --run=mapper ' 	\\
+      -reducer      '.*ruby bundle exec wu-local .*word_count.rb --run=reducer ' 	\\
+    """
+    And the output should contain:
+    """
+      -input        '/foo' 	\
+      -output       '/bar' 	\
+    """
+    And the output should match:
+    """
+      -file         '.*word_count.rb' 	\\
+      -cmdenv       'BUNDLE_GEMFILE=.*wukong-hadoop/Gemfile'
+    """
+
+  Scenario: A wu-hadoop command without an input or output
+    Given a wukong script "examples/word_count.rb"
+    When  I run `bundle exec wu-hadoop examples/word_count.rb --dry_run`
+    Then  the output should contain:
+    """
+    Missing values for: input (Comma-separated list of input paths.), output (Output directory for the hdfs.)
+    """
+
+  Scenario: Specifying an alternative gemfile
+    Given a wukong script "examples/word_count.rb"
+    When  I run `bundle exec wu-hadoop examples/word_count.rb --dry_run --input=/foo --output=/bar --gemfile=alt/Gemfile`
+    Then  the output should contain:
+    """
+      -cmdenv       'BUNDLE_GEMFILE=alt/Gemfile'
+    """
+    
+  Scenario: Skipping the reduce step
+    Given a file named "wukong_script.rb" with:
+    """
+    Wukong.processor(:mapper) do
+
+    end
+    """
+    When  I run `bundle exec wu-hadoop wukong_script.rb --dry_run --input=/foo --output=/bar`
+    Then  the output should contain:
+    """
+      -D mapred.reduce.tasks=0 	\
+    """
+
+  Scenario: A processor without a mapper
+    Given a file named "wukong_script.rb" with:
+    """
+    Wukong.processor(:reducer) do
+
+    end
+    """
+    When  I run `bundle exec wu-hadoop wukong_script.rb --dry_run --input=/foo --output=/bar`
+    Then  the output should match:
+    """
+    No :mapper definition found in .*wukong_script.rb
+    """
+
+  Scenario: Translating hadoop jobconf options
+    Given a wukong script "examples/word_count.rb"
+    When  I run `bundle exec wu-hadoop examples/word_count.rb --dry_run --input=/foo --output=/bar --max_tracker_failures=12`
+    Then  the output should match:
+    """
+      -D mapred.max.tracker.failures=12 	\\
+    """
+
+  Scenario: Passing along extra configuration options
+    Given a wukong script "examples/word_count.rb"
+    When I run `bundle exec wu-hadoop examples/word_count.rb --dry_run --input=/foo --output=/bar --foo=bar`
+    Then the output should match:
+    """
+      -mapper       '.* --foo=bar' 	\\
+      -reducer      '.* --foo=bar' 	\\
+    """
+
+  Scenario: Specifying input and output formats
+    Given a wukong script "examples/word_count.rb"
+    When I run `bundle exec wu-hadoop examples/word_count.rb --dry_run --input=/foo --output=/bar --input_format=com.foo.BarInputFormat`
+    Then the output should contain:
+    """
+      -inputformat  'com.foo.BarInputFormat' 	\
+    """
+
+  Scenario: Specifying additional java options
+    Given a wukong script "examples/word_count.rb"
+    When I run `bundle exec wu-hadoop examples/word_count.rb --dry_run --input=/foo --output=/bar --java_opts=-Dfoo.bar=baz,-Dother.opts=cool`
+    Then the output should contain:
+    """
+      -D foo.bar=baz 	\
+      -D other.opts=cool 	\
+    """
+    
+  Scenario: Failed hadoop job
+    Given a wukong script "examples/word_count.rb"
+    When I run `bundle exec wu-hadoop examples/word_count.rb --input=/foo --output=/bar`
+    Then the output should contain:
+    """
+    Streaming command failed!
+    """

data/lib/wukong-hadoop.rb

@@ -0,0 +1,21 @@
+require 'configliere'
+require 'pathname'
+require 'rbconfig'
+require 'wukong'
+
+module Wukong
+  # Wukong-Hadoop is a plugin for Wukong that lets you develop, test,
+  # and run map/reduce type workflows both locally and in the context
+  # of a Hadoop cluster.
+  #
+  # It comes with a binary program called <tt>wu-hadoop</tt> which
+  # lets you execute Ruby files containing Wukong processors as well
+  # as built-in Wukong widgets.
+  module Hadoop
+  end
+end
+
+
+require 'wukong-hadoop/configuration'
+require 'wukong-hadoop/driver'
+require 'wukong-hadoop/extensions'

data/lib/wukong-hadoop/configuration.rb

@@ -0,0 +1,133 @@
+module Wukong
+  module Hadoop
+
+    # Configure the given settings object for use with Wukong::Hadoop.
+    #
+    # @param [Configliere::Param] settings the settings to configure
+    # @return [Configliere::Param the configured settings
+    def self.configure settings
+      # Hadoop Options
+      settings.define :hadoop_home,             wukong_hadoop: true,                description: 'Path to hadoop installation. HADOOP_HOME/bin/hadoop is used to run hadoop.', env_var: 'HADOOP_HOME', default: '/usr/lib/hadoop'
+      settings.define :hadoop_runner,           wukong_hadoop: true,                description: 'Path to hadoop executable. Use this for non-standard hadoop installations.'
+
+      # Translate simplified args to their hairy hadoop equivalents
+      settings.define :io_sort_mb,              wukong_hadoop: true, jobconf: true, description: 'io.sort.mb'
+      settings.define :io_sort_record_percent,  wukong_hadoop: true, jobconf: true, description: 'io.sort.record.percent'
+      settings.define :job_name,                wukong_hadoop: true, jobconf: true, description: 'mapred.job.name'
+      settings.define :key_field_separator,     wukong_hadoop: true, jobconf: true, description: 'map.output.key.field.separator'
+      settings.define :map_speculative,         wukong_hadoop: true, jobconf: true, description: 'mapred.map.tasks.speculative.execution'
+      settings.define :map_tasks,               wukong_hadoop: true, jobconf: true, description: 'mapred.map.tasks'
+      settings.define :max_maps_per_cluster,    wukong_hadoop: true, jobconf: true, description: 'mapred.max.maps.per.cluster'
+      settings.define :max_maps_per_node,       wukong_hadoop: true, jobconf: true, description: 'mapred.max.maps.per.node'
+      settings.define :max_node_map_tasks,      wukong_hadoop: true, jobconf: true, description: 'mapred.tasktracker.map.tasks.maximum'
+      settings.define :max_node_reduce_tasks,   wukong_hadoop: true, jobconf: true, description: 'mapred.tasktracker.reduce.tasks.maximum'
+      settings.define :max_record_length,       wukong_hadoop: true, jobconf: true, description: 'mapred.linerecordreader.maxlength' 
+      settings.define :max_reduces_per_cluster, wukong_hadoop: true, jobconf: true, description: 'mapred.max.reduces.per.cluster'
+      settings.define :max_reduces_per_node,    wukong_hadoop: true, jobconf: true, description: 'mapred.max.reduces.per.node'
+      settings.define :max_tracker_failures,    wukong_hadoop: true, jobconf: true, description: 'mapred.max.tracker.failures'
+      settings.define :max_map_attempts,        wukong_hadoop: true, jobconf: true, description: 'mapred.map.max.attempts'
+      settings.define :max_reduce_attempts,     wukong_hadoop: true, jobconf: true, description: 'mapred.reduce.max.attempts'
+      settings.define :min_split_size,          wukong_hadoop: true, jobconf: true, description: 'mapred.min.split.size'
+      settings.define :output_field_separator,  wukong_hadoop: true, jobconf: true, description: 'stream.map.output.field.separator'
+      settings.define :partition_fields,        wukong_hadoop: true, jobconf: true, description: 'num.key.fields.for.partition'
+      settings.define :reduce_tasks,            wukong_hadoop: true, jobconf: true, description: 'mapred.reduce.tasks'
+      settings.define :respect_exit_status,     wukong_hadoop: true, jobconf: true, description: 'stream.non.zero.exit.is.failure'
+      settings.define :reuse_jvms,              wukong_hadoop: true, jobconf: true, description: 'mapred.job.reuse.jvm.num.tasks'
+      settings.define :sort_fields,             wukong_hadoop: true, jobconf: true, description: 'stream.num.map.output.key.fields'
+      settings.define :timeout,                 wukong_hadoop: true, jobconf: true, description: 'mapred.task.timeout'
+      settings.define :noempty,                 wukong_hadoop: true,                description: "Don't create zero-byte reduce files"
+      settings.define :split_on_xml_tag,        wukong_hadoop: true,                description: "Parse XML document by specifying the tag name: 'anything found between <tag> and </tag> will be treated as one record for map tasks'"
+      settings.define :input_format,            wukong_hadoop: true,                description: 'Fully qualified Java class name defining an alternative InputFormat.'
+      settings.define :output_format,           wukong_hadoop: true,                description: 'Fully qualified Java class name defining an alternative OutputFormat.'
+      settings.define :java_opts,               wukong_hadoop: true,                description: 'Additional java options to be passed to hadoop streaming.', :type => Array, :default => []
+
+      # Options given on the command-line
+      settings.define :mode,           description: "Run in either 'hadoop' or 'local' mode",                                        wukong_hadoop: true, :default => 'hadoop'
+      settings.define :map_command,    description: "Shell command to run as mapper, in place of a constructed wu-local command",    wukong_hadoop: true
+      settings.define :reduce_command, description: "Shell command to run as reducer, in place of a constructed wu-local command",   wukong_hadoop: true
+      settings.define :sort_command,   description: "Shell command to run as sorter (only in `local' mode)",             wukong_hadoop: true, :default => 'sort'
+      settings.define :command_prefix, description: "Prefex to insert before all Wukong commands",                       wukong_hadoop: true
+      settings.define :mapper,         description: "Name of processor to use as a mapper",                              wukong_hadoop: true
+      settings.define :reducer,        description: "Name of processor to use as a reducer",                             wukong_hadoop: true
+      settings.define :gemfile,        description: "Specify an alternative Gemfile to execute this wukong script with", wukong_hadoop: true 
+      settings.define :dry_run,        description: "Echo the command that will be run, but don't run it",               wukong_hadoop: true, :type => :boolean, :default => false
+      settings.define :rm,             description: "Recursively remove the destination directory.",                     wukong_hadoop: true, :type => :boolean, :default => false
+      settings.define :input,          description: "Comma-separated list of input paths",                               wukong_hadoop: true
+      settings.define :output,         description: "Output path.",                                                      wukong_hadoop: true
+
+      settings.use(:commandline)
+
+      def settings.usage()
+        "usage: #{File.basename($0)} PROCESSOR|FLOW [PROCESSOR|FLOW] [ --param=value | -p value | --param | -p]"
+      end
+
+      settings.description = <<EOF
+wu-hadoop is a tool to model and launch Wukong processors as
+map/reduce workflows within the Hadoop framework.
+
+Use wu-hadoop with existing processors in `local' mode to test the
+logic of your job, reading from the specified --input and printing to
+STDOUT:
+
+  $ wu-hadoop examples/word_count.rb --mode=local --input=examples/sonnet_18.txt
+  a	2
+  all	1
+  and	2
+  ...
+
+where it is assumed that your mapper is called 'mapper' and your
+reducer 'reducer'.  You can also cat in data:
+
+  $ cat examples/sonnet_18.txt | wu-hadoop examples/word_count.rb --mode=local
+
+Or pass options directly:
+
+  $ wu-hadoop examples/word_count.rb  --mode=local --input=examples/sonnet_18.txt --fold_case --min_length=3
+  all	1
+  and	5
+  art	1
+  brag	1
+  ...
+
+Or define both processors in separate files:
+
+  $ wu-hadoop examples/tokenizer.rb examples/counter.rb --mode=local --input=examples/sonnet_18.txt
+
+Or by name:
+
+  $ wu-hadoop examples/processors.rb --mode=local --input=examples/sonnet_18.txt --mapper=tokenizer --reducer=counter
+
+Or just by command:
+
+$ wu-hadoop processors.rb --mapper=tokenizer --reduce_command='uniq -c' ...
+$ wu-hadoop processors.rb --map_command='cut -f3' --reducer=counter ...
+$ wu-hadoop --map_command='cut -f3' --reduce_command='uniq -c' ...
+
+If you don't specify a --reducer explicitly, and you didn't give two
+separate arguments, and no processor named :reducer exists in the
+environment, then we assume you are launching a map-only job and
+'mapred.tasktracker.reduce.tasks.maximum' will correspondingly be set
+to 0:
+
+  $ wu-hadoop examples/tokenizer.rb --mode=local --input=examples/sonnet_18.txt
+  Shall
+  I
+  compare
+  thee
+  ...
+
+You can achieve this directly with the --reduce_tasks=0 option.
+
+Many other Hadoop options have been wrapped with similarly friendly
+names below.  These are ignored when running in `local' mode.
+
+Some options (like `--sort_command') only make sense in `local' mode.
+These are ignored in `hadoop' mode.
+EOF
+      settings
+    end
+
+    # All Hadoop configuration for Wukong lives within this object.
+    Configuration = configure(Configliere::Param.new) unless defined? Configuration
+  end
+end