wukong 3.0.0.pre → 3.0.0.pre2

data/old/wukong/helper/tokenize.rb

@@ -1,35 +0,0 @@
-require File.expand_path('stopwords', File.dirname(__FILE__))
-module Wukong
-  module Helper
-
-    module Tokenize
-      #
-      # Split a string into its constituent words.
-      #
-      # This is pretty simpleminded:
-      # * downcase the word
-      # * Split at any non-alphanumeric boundary, including '_'
-      # * However, preserve the special cases of 's, 'd or 't at the end of a
-      #   word.
-      #
-      #   tokenize("Ability is a poor man's wealth #johnwoodenquote")
-      #   # => ["ability", "is", "a", "poor", "man's", "wealth", "johnwoodenquote"]
-      #
-      def self.tokenize str
-        return [] if str.blank?
-        str = str.downcase;
-        # kill off all punctuation except [stuff]'s or [stuff]'t
-        # this includes hyphens (words are split)
-        str = str.
-          gsub(/[^a-zA-Z0-9\']+/, ' ').
-          gsub(/(\w)\'([stdm]|re|ve|ll)\b/, '\1!\2').gsub(/\'/, ' ').gsub(/!/, "'")
-        # Busticate at whitespace
-        words = str.split(/\s+/)
-        words.reject!{|w| w.length < 3 ||  Wukong::Corpus::STOPWORDS_3.include?(w) }
-        words
-      end
-
-    end
-
-  end
-end

data/old/wukong/logger.rb

@@ -1,38 +0,0 @@
-module Wukong
-  # Common logger
-  #
-  # Set your own at any time with
-  #   Wukong.logger = YourAwesomeLogger.new(...)
-  # If you have log4r installed you can use
-  #   Wukong.logger = Wukong.default_log4r_logger
-  #
-  # If Wukong.logger is too much typing for you,
-  # use the Log constant
-  #
-  # Default format:
-  #     I, [2009-07-26T19:58:46-05:00 #12332]: Up to 2000 char message
-  #
-  def self.logger
-    return @logger if defined?(@logger)
-    require 'logger'
-    @logger = Logger.new STDERR
-    @logger.instance_eval do
-      def dump *args
-        debug args.inspect
-      end
-    end
-    @logger
-  end
-
-  def self.logger= logger
-    @logger = logger
-  end
-end
-
-#
-# A convenient logger.
-#
-# define Log yourself to prevent its creation
-#
-Log         = Wukong.logger       unless defined?(Log)
-

data/old/wukong/periodic_monitor.rb

@@ -1,72 +0,0 @@
-Settings.define :log_interval, :default => 10_000, :type => Integer, :description => 'How many iterations between log statements'
-Settings.define :log_seconds,  :default => 30,     :type => Integer, :description => 'How many seconds between log statements'
-
-#
-# Periodic monitor
-#
-#
-# This is very much a work in progress
-#
-class PeriodicMonitor
-  attr_reader   :iter, :start_time, :options
-  attr_accessor :interval
-  attr_accessor :time_interval
-
-  def initialize extra_options={}
-    @options       = {}
-    @options.deep_merge!( extra_options || {} )
-    @iter          = 0
-    @start_time    = now
-    @last_report   = @start_time
-    @interval      = (options[:log_interval] || Settings[:log_interval]).to_i
-    @interval      = 1000 unless @interval >= 1
-    @time_interval = (options[:log_seconds]  || Settings[:log_seconds]).to_i
-  end
-
-  def periodically *args, &block
-    incr!
-    if ready?
-      @last_report = Time.now
-      if block
-        emit block.call(self, *args)
-      else
-        emit progress(*args)
-      end
-    end
-  end
-
-  def emit log_line
-    Log.info log_line
-  end
-
-  def incr!
-    @iter += 1
-  end
-
-  def ready?
-    (iter % @interval == 0) || (since > time_interval)
-  end
-
-  def progress *stuff
-    [
-      "%15d" % iter,
-      "%7.1f"% elapsed_time, "sec",
-      "%7.1f"% rate, "/sec",
-      now.to_flat,
-      *stuff
-    ].flatten.join("\t")
-  end
-
-  def elapsed_time
-    now - start_time
-  end
-  def since
-    now - @last_report
-  end
-  def now
-    Time.now.utc
-  end
-  def rate
-    iter.to_f / elapsed_time
-  end
-end

data/old/wukong/schema.rb

@@ -1,269 +0,0 @@
-require 'extlib/inflection'
-require 'wukong'
-
-
-#
-# Basic types: SQL conversion
-#
-class << Integer    ; def to_sql() 'INT'                              end ; end
-class << Bignum     ; def to_sql() 'BIGINT'                           end ; end
-class << String     ; def to_sql() 'VARCHAR(255) CHARACTER SET ASCII' end ; end
-class << Symbol     ; def to_sql() 'VARCHAR(255) CHARACTER SET ASCII' end ; end
-class << BigDecimal ; def to_sql() 'DECIMAL'                          end ; end if defined?(BigDecimal)
-class << EpochTime  ; def to_sql() 'INT'                              end ; end if defined?(EpochTime)
-class << FilePath   ; def to_sql() 'VARCHAR(255) CHARACTER SET ASCII' end ; end if defined?(FilePath)
-class << Flag       ; def to_sql() 'CHAR(1)      CHARACTER SET ASCII' end ; end if defined?(Flag)
-class << IPAddress  ; def to_sql() 'CHAR(15)     CHARACTER SET ASCII' end ; end if defined?(IPAddress)
-class << URI        ; def to_sql() 'VARCHAR(255) CHARACTER SET ASCII' end ; end if defined?(URI)
-class << Csv        ; def to_sql() 'TEXT'                             end ; end if defined?(Csv)
-class << Yaml       ; def to_sql() 'TEXT'                             end ; end if defined?(Yaml)
-class << Json       ; def to_sql() 'TEXT'                             end ; end if defined?(Json)
-class << Regex      ; def to_sql() 'TEXT'                             end ; end if defined?(Regex)
-class String        ; def to_sql() self             ; end ; end
-class Symbol        ; def to_sql() self.to_s.upcase ; end ; end
-
-#
-# Basic types: Pig conversion
-#
-class << Integer    ; def to_pig() 'int'           end ; end
-class << Bignum     ; def to_pig() 'long'          end ; end
-class << Float      ; def to_pig() 'float'         end ; end
-class << Symbol     ; def to_pig() 'chararray'     end ; end
-class << Date       ; def to_pig() 'long'          end ; end
-class << Time       ; def to_pig() 'long'          end ; end
-class << DateTime   ; def to_pig() 'long'          end ; end
-class << String     ; def to_pig() 'chararray'     end ; end
-class << Text       ; def to_pig() 'chararray'     end ; end if defined?(Text)
-class << Blob       ; def to_pig() 'bytearray'     end ; end if defined?(Blob)
-class << Boolean    ; def to_pig() 'bytearray'     end ; end if defined?(Boolean)
-class String        ; def to_pig() self.to_s ; end ; end
-class Symbol        ; def to_pig() self.to_s ; end ; end
-
-class << BigDecimal ; def to_pig() 'long'          end ; end if defined?(BigDecimal)
-class << EpochTime  ; def to_pig() 'integer'       end ; end if defined?(EpochTime)
-class << FilePath   ; def to_pig() 'chararray'     end ; end if defined?(FilePath)
-class << Flag       ; def to_pig() 'chararray'     end ; end if defined?(Flag)
-class << IPAddress  ; def to_pig() 'chararray'     end ; end if defined?(IPAddress)
-class << URI        ; def to_pig() 'chararray'     end ; end if defined?(URI)
-class << Csv        ; def to_pig() 'chararray'     end ; end if defined?(Csv)
-class << Yaml       ; def to_pig() 'chararray'     end ; end if defined?(Yaml)
-class << Json       ; def to_pig() 'chararray'     end ; end if defined?(Json)
-class << Regex      ; def to_pig() 'chararray'     end ; end if defined?(Regex)
-
-
-#
-# Basic types: Avro conversion
-#
-class << Integer    ; def to_avro() 'int'           end ; end
-class << Bignum     ; def to_avro() 'long'          end ; end
-class << Float      ; def to_avro() 'float'         end ; end
-class << Symbol     ; def to_avro() 'string'        end ; end
-class << Date       ; def to_avro() 'long'          end ; end
-class << Time       ; def to_avro() 'long'          end ; end
-class << DateTime   ; def to_avro() 'long'          end ; end
-class << String     ; def to_avro() 'string'        end ; end
-class << Text       ; def to_avro() 'string'        end ; end if defined?(Text)
-class << Blob       ; def to_avro() 'bytearray'     end ; end if defined?(Blob)
-class << Boolean    ; def to_avro() 'bytearray'     end ; end if defined?(Boolean)
-class String        ; def to_avro() self.to_s ;     end ; end
-class Symbol        ; def to_avro() self.to_s ;     end ; end
-
-class << BigDecimal ; def to_avro() 'long'          end ; end if defined?(BigDecimal)
-class << EpochTime  ; def to_avro() 'integer'       end ; end if defined?(EpochTime)
-class << FilePath   ; def to_avro() 'string'        end ; end if defined?(FilePath)
-class << Flag       ; def to_avro() 'string'        end ; end if defined?(Flag)
-class << IPAddress  ; def to_avro() 'string'        end ; end if defined?(IPAddress)
-class << URI        ; def to_avro() 'string'        end ; end if defined?(URI)
-class << Csv        ; def to_avro() 'string'        end ; end if defined?(Csv)
-class << Yaml       ; def to_avro() 'string'        end ; end if defined?(Yaml)
-class << Json       ; def to_avro() 'string'        end ; end if defined?(Json)
-class << Regex      ; def to_avro() 'string'        end ; end if defined?(Regex)
-
-module Wukong
-  #
-  # Export model's structure for loading and manipulating in other frameworks,
-  # such as SQL and Pig
-  #
-  # Your class should support the #resource_name and #mtypes methods
-  # An easy way to do this is by being a TypedStruct.
-  #
-  # You can use this to do silly stunts like
-  #
-  #      % ruby -rubygems -r'wukong/schema' -e 'require "/path/to/user_model.rb" ; puts User.pig_load ; '
-  #
-  # If you include the classes from Wukong::Datatypes::MoreTypes, you can draw
-  # on a richer set of type definitions
-  #
-  #     require 'wukong/datatypes/more_types'
-  #     include Wukong::Datatypes::MoreTypes
-  #     require 'wukong/schema'
-  #
-  # (if you're using Wukong to bulk-process Datamapper records, these should
-  # fall right in line as well -- make sure *not* to include
-  # Wukong::Datatypes::MoreTypes, and to require 'dm-more' before 'wukong/schema')
-  #
-  module Schema
-    module ClassMethods
-
-      #
-      # Table name for this class
-      #
-      def table_name
-        resource_name.to_s.pluralize
-      end
-
-      # ===========================================================================
-      #
-      # Pig
-      #
-
-      # Export schema as Pig
-      #
-      # Won't correctly handle complex types (struct having struct as member, eg)
-      #
-      def to_pig
-        members.zip(mtypes).map do |member, type|
-          member.to_s + ': ' + type.to_pig
-        end.join(', ')
-      end
-
-      #
-      # A pig snippet to load a tsv file containing
-      # serialized instances of this class.
-      #
-      # Assumes the first column is the resource name (you can, and probably
-      # should, follow with an immediate GENERATE to ditch that field.)
-      #
-      def pig_load filename=nil
-        filename ||= resource_name.to_s+'.tsv'
-        cmd = [
-          "%-23s" % self.to_s.gsub(/^.*\W/, ""),
-          "= LOAD '#{filename}'",
-          "AS ( rsrc:chararray,", self.to_pig, ') ;',
-        ].join(" ")
-      end
-
-      # ===========================================================================
-      #
-      # SQL
-
-      #
-      # Schema definition for use in a CREATE TABLE statement
-      #
-      def to_sql
-        sql_str = []
-        members.zip(mtypes).each do |attr, type|
-          type_str = type.respond_to?(:to_sql) ? type.to_sql : type.to_s.upcase
-          sql_str << "  %-29s\t%s" %["`#{attr}`", type_str]
-        end
-        sql_str.join(",\n")
-      end
-
-      #
-      # List off member names, to be stuffed into a SELECT or a LOAD DATA
-      #
-      def sql_members
-        members.map{|attr| "`#{attr}`" }.join(", ")
-      end
-
-      #
-      # Creates a table for the wukong class.
-      #
-      # * primary_key gives the name of one column to be set as the primary key
-      #
-      # * if drop_first is given, a "DROP TABLE IF EXISTS" statement will
-      #   precede the snippet.
-      #
-      # * table_options sets the table parameters. Useful table_options for a
-      #   read-only database in MySQL:
-      #     ENGINE=MyISAM PACK_KEYS=0
-      #
-      def sql_create_table primary_key=nil, drop_first=nil, table_options=''
-        str = []
-        str << %Q{DROP TABLE IF EXISTS `#{self.table_name}`;  } if drop_first
-        str << %Q{CREATE TABLE         `#{self.table_name}` ( }
-        str << self.to_sql
-        if primary_key then str.last << ',' ; str << %Q{  PRIMARY KEY     \t(`#{primary_key}`)} ; end
-        str << %Q{  ) #{table_options} ;}
-        str.join("\n")
-      end
-
-      #
-      # A mysql snippet to bulk load the tab-separated-values file emitted by a
-      # Wukong script.
-      #
-      # Let's say your class is ClickLog; its resource_name is "click_log"
-      # and thus its table_name is 'click_logs'. sql_load_mysql will:
-      #
-      # * disable indexing on the table
-      # * import the file, replacing any existing rows. (Replacement is governed
-      #   by primary key and unique index constraints -- see the mysql docs).
-      # * re-enable indexing on that table
-      # * show the number of
-      #
-      # The load portion will
-      #
-      # * Load into a table named click_logs
-      # * from a file named click_logs.tsv
-      # * where all rows have the string 'click_logs' in their first column
-      # * and all remaining fields in their #members order
-      # * assuming strings are wukong_encode'd and so shouldn't be escaped or enclosed.
-      #
-      # Why the "LINES STARTING BY" part? For map/reduce outputs that have many
-      # different objects jumbled together, you can just dump in the whole file,
-      # landing each object in its correct table.
-      #
-      def sql_load_mysql(filename=nil)
-        filename ||= ":resource_name.tsv"
-        filename.gsub!(/:resource_name/, self.table_name)
-        str = []
-        # disable indexing during bulk load
-        str << %Q{ALTER TABLE            `#{self.table_name}` DISABLE KEYS; }
-        # Bulk load the tab-separated-values file.
-        str << %Q{LOAD DATA LOCAL INFILE '#{filename}'}
-        str << %Q{  REPLACE INTO TABLE   `#{self.table_name}`    }
-        str << %Q{  COLUMNS                                         }
-        str << %Q{    TERMINATED BY           '\\t'                 }
-        str << %Q{    OPTIONALLY ENCLOSED BY  ''                    }
-        str << %Q{    ESCAPED BY              ''                    }
-        str << %Q{  LINES STARTING BY     '#{self.resource_name}'   }
-        str << %Q{  ( @dummy,\n }
-        str << '    '+self.sql_members
-        str << %Q{\n  ); }
-        # Re-enable indexing
-        str << %Q{ALTER TABLE `#{self.table_name}` ENABLE KEYS ; }
-        # Show it loaded correctly
-        str << %Q{SELECT NOW(), COUNT(*), '#{self.table_name}' FROM `#{self.table_name}`; }
-        str.join("\n")
-      end
-
-
-
-
-      #
-      # Avro
-      #
-      def to_avro
-        require 'json' # yikes
-        h = {}
-        h[:name]   = self.name
-        h[:type]   = "record"
-        h[:fields] =  []
-        members.zip(mtypes).each do |member, type|
-          h[:fields] << {:name => member.to_s, :type => type.to_avro}
-        end
-        h.to_json
-      end
-      
-    end
-    # standard stanza for making methods appear on the class itself on include
-    def self.included base
-      base.class_eval{ extend ClassMethods }
-    end
-  end
-end
-
-#
-# TypedStructs are class-schematizeable
-#
-Struct.class_eval do include(Wukong::Schema) ; end

data/old/wukong/script.rb

@@ -1,286 +0,0 @@
-require 'pathname'
-require 'wukong/extensions'
-require 'configliere' ; Settings.use(:commandline, :env_var, :define)
-require 'wukong'
-require 'wukong/script/hadoop_command'
-require 'wukong/script/local_command'
-require 'rbconfig' # for uncovering ruby_interpreter_path
-require 'wukong/streamer' ; include Wukong::Streamer
-module Wukong
-  # == How to run a Wukong script
-  #
-  #   your/script.rb --run path/to/input_files path/to/output_dir
-  #
-  # All of the file paths are HDFS paths ; your script path, of course, is on the local filesystem.
-  #
-  # == Command-line options
-  #
-  # If you'd like to listen for any command-line options, specify them at the
-  # command line:
-  #
-  #   your/script.rb --my_bool_opt --my_val_taking_opt=val \
-  #     --run path/to/input_files path/to/output_dir
-  #
-  # In this case the options hash for both Mapper and Reducer will contain
-  #
-  #   :my_bool_opt       => true,
-  #   :my_val_taking_opt => 'val'
-  #
-  # == Complicated input paths
-  #
-  # To use more than one file as input, you can use normal * ? [] wildcards or
-  # give a comma-separated list -- see the hadoop documentation for syntax.
-  #
-  # == Run in Elastic MapReduce Mode (--run=emr)
-  #
-  # Wukong can be used to start scripts on the amazon cloud
-  #
-  # * copies the script to s3 in two parts
-  # * invokes it using the amazon API
-  #
-  # == Run locally (--run=local)
-  #
-  # To run your script locally, use --run=local
-  #
-  #   your/script.rb --run=local path/to/input_files path/to/output_dir
-  #
-  # This will pipe the contents of path/to/input_files through first your
-  # mapper, then sort, then the reducer, storing the results in the given output
-  # directory.
-  #
-  # All paths refer to the /local/ filesystem -- hadoop is never involved and in
-  # fact doesn't even have to be installed.
-  #
-  # == How to test your scripts
-  #
-  # You can supply the --map argument in place of --run to run the mapper on its
-  # own (and similarly, --reduce to run the reducer standalone):
-  #
-  #   cat ./local/test/input.tsv | ./examples/word_count.rb --map | more
-  #
-  # or, if your test data lies on the HDFS,
-  #
-  #   hdp-cat test/input.tsv | ./examples/word_count.rb --map | more
-  #
-  #
-  class Script
-    include Wukong::HadoopCommand
-    include Wukong::LocalCommand
-    attr_reader :mapper, :reducer, :options
-    attr_reader :input_paths, :output_path
-
-    # ---------------------------------------------------------------------------
-    #
-    # Default options for Wukong
-    #   http://github.com/infochimps/wukong
-    #
-    # If you set an environment variable WUKONG_CONFIG, *or* if the file
-    # $HOME/.wukong.rb exists, that file will be +require+'d as well.
-    #
-    # Important values to set:
-    #
-    # * hadoop_home -- Path to root of hadoop install. If your hadoop runner is
-    #     /usr/local/share/hadoop/bin/hadoop
-    #   then your hadoop_home is
-    #     /usr/local/share/hadoop.
-    #   You can also set a :hadoop_runner that gives the full path to the hadoop script
-    #
-    # * default_run_mode -- Whether to run using hadoop (and
-    #   thus, requiring a working hadoop install), or to run in local mode
-    #   (script --map | sort | script --reduce)
-    #
-    Settings.define :default_run_mode,   :default => 'hadoop', :description => 'Run mode: local, hadoop, emr (elastic mapreduce)', :wukong => true, :hide_help => false
-    Settings.define :map_command,                              :description => "shell command to run as mapper, in place of this wukong script", :wukong => true
-    Settings.define :reduce_command,                           :description => "shell command to run as reducer, in place of this wukong script", :wukong => true
-    Settings.define :run,      :env_var => 'WUKONG_RUN_MODE',    :description => "run the script's workflow: Specify 'hadoop' to use hadoop streaming; 'local' to run your_script.rb --map | sort | your_script.rb --reduce; 'emr' to launch on the amazon cloud; 'map' or 'reduce' to run that phase.", :wukong => true
-    Settings.define :map,                                      :description => "run the script's map phase. Reads/writes to STDIN/STDOUT.", :wukong => true
-    Settings.define :reduce,                                   :description => "run the script's reduce phase. Reads/writes to STDIN/STDOUT. You can only choose one of --run, --map or --reduce.", :wukong => true
-    Settings.define :dry_run,                                  :description => "echo the command that will be run, but don't run it", :wukong => true
-    Settings.define :rm,                                       :description => "Recursively remove the destination directory. Only used in hadoop mode.", :wukong => true
-
-    #
-    # Instantiate the Script with the Mapper and the Reducer class (each a
-    # Wukong::Streamer) it should call back.
-    #
-    #
-    # == Identity or External program as map or reduce
-    #
-    # To use the identity reducer ('cat'), instantiate your Script class with
-    # +nil+ as the reducer class. (And similarly to use an identity mapper,
-    # supply +nil+ for the mapper class.)
-    #
-    # To use an external program as your reducer (mapper), subclass the
-    # reduce_command (map_command) method to return the full command line
-    # expression to call.
-    #
-    #   class MyMapper < Wukong::Streamer::Base
-    #     # ... awesome stuff ...
-    #   end
-    #
-    #   class MyScript < Wukong::Script
-    #     # prefix each unique line with the count of its occurrences.
-    #     def reduce_command
-    #       '/usr/bin/uniq -c'
-    #     end
-    #   end
-    #   MyScript.new(MyMapper, nil).run
-    #
-    def initialize mapper, reducer=nil, extra_options={}
-      Settings.resolve!
-      @options = Settings
-      options.merge! extra_options
-      @mapper  = (case mapper  when Class then mapper.new  when nil then nil else mapper  ; end)
-      @reducer = (case reducer when Class then reducer.new when nil then nil else reducer ; end)
-      @output_path = options.rest.pop
-      @input_paths = options.rest.reject(&:blank?)
-      if (input_paths.blank? || output_path.blank?) && (not options[:dry_run]) && (not ['map', 'reduce'].include?(run_mode))
-        raise "You need to specify a parsed input directory and a directory for output. Got #{ARGV.inspect}"
-      end
-    end
-
-    #
-    # In --run mode, use the framework (local, hadoop, emr, etc) to re-launch
-    #   the script as mapper, reducer, etc.
-    # If --map or --reduce, dispatch to the mapper or reducer.
-    #
-    def run
-      case run_mode
-      when 'map'              then mapper.stream
-      when 'reduce'           then reducer.stream
-      when 'local'            then execute_local_workflow
-      when 'cassandra'        then execute_hadoop_workflow
-      when 'hadoop', 'mapred' then execute_hadoop_workflow
-      when 'emr'
-        require 'wukong/script/emr_command'
-        execute_emr_workflow
-      else                    dump_help
-      end
-    end
-
-    # if only --run is given, assume default run mode
-    def run_mode
-      case
-      when options[:map]           then 'map'
-      when options[:reduce]        then 'reduce'
-      when ($0 =~ /-mapper\.rb$/)  then 'map'
-      when ($0 =~ /-reducer\.rb$/) then 'reduce'
-      when (options[:run] == true) then options[:default_run_mode]
-      else                         options[:run].to_s
-      end
-    end
-
-    #
-    # Shell command for map phase. By default, calls the script in --map mode
-    # In hadoop mode, this is given to the hadoop streaming command.
-    # In local mode, it's given to the system() call
-    #
-    def mapper_commandline(run_option=:local)
-      if mapper
-        case run_option
-        when :local then
-          "#{ruby_interpreter_path} #{this_script_filename} --map " + non_wukong_params
-        when :hadoop then
-          "#{ruby_interpreter_path} #{File.basename(this_script_filename)} --map " + non_wukong_params
-        end
-      else
-        options[:map_command]
-      end
-    end
-
-    #
-    # Shell command for reduce phase. By default, calls the script in --reduce mode
-    # In hadoop mode, this is given to the hadoop streaming command.
-    # In local mode, it's given to the system() call
-    #
-    def reducer_commandline(run_option=:local)
-      if reducer
-        case run_option
-        when :local then
-          "#{ruby_interpreter_path} #{this_script_filename} --reduce " + non_wukong_params
-        when :hadoop then
-          "#{ruby_interpreter_path} #{File.basename(this_script_filename)} --reduce " + non_wukong_params
-        end
-      else
-        options[:reduce_command]
-      end
-    end
-
-    def job_name
-      options[:job_name] ||
-        "#{File.basename(this_script_filename)}---#{input_paths}---#{output_path}".gsub(%r{[^\w/\.\-\+]+}, '')
-    end
-
-    # Wrapper for dangerous operations to catch errors
-    def safely action, &block
-      begin
-        block.call
-      rescue StandardError => e ; handle_error(action, e); end
-    end
-
-  protected
-
-    #
-    # Execute the runner phase:
-    # use the running framework to relaunch the script in map and in reduce mode
-    #
-    def execute_command! *args
-      command = args.flatten.reject(&:blank?).join(" \\\n    ")
-      Log.info "Running\n\n#{command}\n"
-      if options[:dry_run]
-        Log.info '== [Not running preceding command: dry run] =='
-      else
-        maybe_overwrite_output_paths! output_path
-        $stdout.puts `#{command}`
-        raise "Streaming command failed!" unless $?.success?
-      end
-    end
-
-    #
-    # In hadoop mode only, removes the destination path before launching
-    #
-    # To the panic-stricken: look in .Trash/current/path/to/accidentally_deleted_files
-    #
-    def maybe_overwrite_output_paths! output_path
-      if (options[:overwrite] || options[:rm]) && (run_mode == 'hadoop')
-        cmd = %Q{#{hadoop_runner} fs -rmr '#{output_path}'}
-        Log.info "Removing output file #{output_path}: #{cmd}"
-        puts `#{cmd}`
-      end
-    end
-
-    # Reassemble all the non-internal-to-wukong options into a command line for
-    # the map/reducer phase scripts
-    def non_wukong_params
-      options.
-        reject{|param, val| options.definition_of(param, :wukong) }.
-        map{|param,val| "--#{param}=#{val}" }.
-        join(" ")
-    end
-
-    # the full, real path to the script file
-    def this_script_filename
-      Pathname.new($0).realpath
-    end
-
-    # use the full ruby interpreter path to run slave processes
-    def ruby_interpreter_path
-      Pathname.new(File.join(
-          Config::CONFIG["bindir"],
-          Config::CONFIG["RUBY_INSTALL_NAME"]+Config::CONFIG["EXEEXT"])).realpath
-    end
-
-    #
-    # Usage
-    #
-    def dump_help
-        options.dump_help %Q{Please specify a run mode: you probably want to start with
-  #{$0} --run --local input.tsv output.tsv
-although
-  cat input.tsv | #{$0} --map > mapped.tsv
-or
-  cat mapped.tsv | sort | #{$0} --reduce > reduced.tsv
-can be useful for initial testing.}
-    end
-
-  end
-end