wukong 3.0.0.pre → 3.0.0.pre2

data/lib/away/runner/execute.rb

@@ -1,121 +0,0 @@
-# -*- coding: utf-8 -*-
-
-#
-# Taken from the [Aruba project](). Original license:
-#
-#         Copyright (c) 2010 Aslak Hellesøy, David Chelimsky
-#
-#         Permission is hereby granted, free of charge, to any person obtaining
-#         a copy of this software and associated documentation files (the
-#         "Software"), to deal in the Software without restriction, including
-#         without limitation the rights to use, copy, modify, merge, publish,
-#         distribute, sublicense, and/or sell copies of the Software, and to
-#         permit persons to whom the Software is furnished to do so, subject to
-#         the following conditions:
-#
-#         The above copyright notice and this permission notice shall be
-#         included in all copies or substantial portions of the Software.
-#
-#         THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-#         EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-#         MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-#         NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-#         LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-#         OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-#         WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-
-
-require 'childprocess'
-require 'tempfile'
-
-module Wukong
-  class Runner
-
-    class Process
-      attr_reader :keep_ansi
-
-      def initialize(cmd, exit_timeout=2.0, io_wait=2.0, keep_ansi=true)
-        @exit_timeout = exit_timeout
-        @io_wait      = io_wait
-        @keep_ansi    = true
-
-        @process           = ChildProcess.build(*cmd)
-        @process.io.stdout = raw_out_io
-        @process.io.stderr = raw_err_io
-        @process.duplex    = true
-      end
-
-      def raw_out_io
-        @raw_out_io ||= StringIO.new('', 'w')
-      end
-
-      def raw_err_io
-        @raw_err_io ||= StringIO.new('', 'w')
-      end
-
-      def run!(&block)
-        @process.start
-        yield self if block_given?
-      end
-
-      def stdin
-        wait_for_io do
-          @process.io.stdin.sync = true
-          @process.io.stdin
-        end
-      end
-
-      def output
-        stdout + stderr
-      end
-
-      def stdout
-        wait_for_io do
-          @raw_out_io.rewind
-          filter_ansi(@raw_out_io.read)
-        end
-      end
-
-      def stderr
-        wait_for_io do
-          @raw_err_io.rewind
-          filter_ansi(@raw_err_io.read)
-        end
-      end
-
-      def stop(reader)
-        return unless @process
-        unless @process.exited?
-          reader.stdout stdout
-          reader.stderr stderr
-          @process.poll_for_exit(@exit_timeout)
-        end
-        @process.exit_code
-      end
-
-      def terminate
-        if @process
-          flush
-          @process.stop
-          flush
-        end
-      end
-
-      def flush
-        stdout && stderr # flush output
-      end
-
-    private
-
-      def wait_for_io(&block)
-        sleep @io_wait if @process.alive?
-        yield
-      end
-
-      def filter_ansi(string)
-        keep_ansi ? string : string.gsub(/\e\[\d+(?>(;\d+)*)m/, '')
-      end
-
-    end
-  end
-end

data/lib/away/script.rb

@@ -1,161 +0,0 @@
-require 'wukong'
-require 'wukong/script/hadoop_command'
-require 'wukong/experimental'
-
-#
-# Runner settings
-#
-
-Settings.define :mode, :type => Symbol, :default => :mapper, :env_var => 'WUKONG_MODE',  :description => "run the script's workflow: Specify 'hadoop' to use hadoop streaming; 'local' to run your_script.rb --map | sort | your_script.rb --reduce; 'emr' to launch on the amazon cloud; 'mapper' or 'reducer' to run that phase.", :wukong => true
-Settings.define :dry_run,  :description => "echo the command that will be run, but don't run it", :wukong => true
-Settings.define :rm,       :description => "Recursively remove the destination directory. Only used in hadoop mode.", :wukong => true
-Settings.define :script_file, :type => :filename, :description => "script file to execute, or give as first arg", :wukong => true
-
-module Wukong
-  # adds ability to execute
-  extend Wukong::Mixin::FromFile
-
-  def self.from_file(filename)
-    filename  = filename.to_s
-    filename += ".rb" if filename !~ /\.rb$/
-    super(filename)
-  end
-
-  def self.run(filename=nil)
-    if filename
-      self.from_file(filename)
-    else
-      Settings.resolve!
-    end
-    if @main_run then return false ; end
-    Wukong::Script.new(Settings).run
-    @main_run = true
-  end
-
-  #
-  # sources a script file,
-  #
-  class Script
-    attr_reader :settings     # configliere hash of settings
-    attr_reader :script_file  # File to execute
-    attr_reader :input_paths
-    attr_reader :output_path
-
-    include Wukong::Script::HadoopCommand
-
-    def initialize(settings)
-      @settings = settings
-
-      @output_path = settings.rest.pop
-      @input_paths = settings.rest.reject(&:blank?)
-    end
-
-
-    # Execute the script file in the context of the Wukong module
-    def run_flow
-      Log.debug( "Running #{script_file} with settings #{settings}")
-      script_file = settings.script_file
-      mode        = settings.mode
-      Wukong.flow(mode).run
-    end
-
-    #
-    # In --run mode, use the framework (local, hadoop, emr, etc) to re-launch
-    #   the script as mapper, reducer, etc.
-    # If --map or --reduce, dispatch to the mapper or reducer.
-    #
-    def run
-      case settings.mode
-      when :local           then execute_local_workflow
-      when :hadoop, :mapred then execute_hadoop_workflow
-      else
-        run_flow
-      end
-    end
-
-    #
-    # Shell command for map phase. By default, calls the script in --map mode
-    # In hadoop mode, this is given to the hadoop streaming command.
-    # In local mode, it's given to the system() call
-    #
-    def mapper_commandline
-      "#{ruby_interpreter_path} #{this_script_filename} --mode=mapper " + non_wukong_params
-    end
-
-    #
-    # Shell command for reduce phase. By default, calls the script in --reduce mode
-    # In hadoop mode, this is given to the hadoop streaming command.
-    # In local mode, it's given to the system() call
-    #
-    def reducer_commandline
-      "#{ruby_interpreter_path} #{this_script_filename} --mode=reducer " + non_wukong_params
-    end
-
-    def job_name
-      settings[:job_name] ||
-        "#{File.basename(this_script_filename)}---#{input_paths}---#{output_path}".gsub(%r{[^\w/\.\-\+]+}, '')
-    end
-
-    # Wrapper for dangerous operations to catch errors
-    def safely action, &block
-      begin
-        block.call
-      rescue StandardError => e ; handle_error(action, e); end
-    end
-
-  protected
-
-    #
-    # Execute the runner phase:
-    # use the running framework to relaunch the script in map and in reduce mode
-    #
-    def execute_command! *args
-      command = args.flatten.reject(&:blank?).join(" \\\n    ")
-      Log.info "Running\n\n#{command}\n"
-      if settings[:dry_run]
-        Log.info '== [Not running preceding command: dry run] =='
-      else
-        maybe_overwrite_output_paths! output_path
-        $stdout.puts `#{command}`
-        raise "Streaming command failed!" unless $?.success?
-      end
-    end
-
-    #
-    # In hadoop mode only, removes the destination path before launching
-    #
-    # To the panic-stricken: look in .Trash/current/path/to/accidentally_deleted_files
-    #
-    def maybe_overwrite_output_paths! output_path
-      if (settings.rm && (settings.mode == :hadoop))
-        cmd = %Q{#{hadoop_runner} fs -rmr '#{output_path}'}
-        Log.info "Removing output file #{output_path}: #{cmd}"
-        puts `#{cmd}`
-      end
-    end
-
-    # Reassemble all the non-internal-to-wukong settings into a command line for
-    # the map/reducer phase scripts
-    def non_wukong_params
-      settings.
-        reject{|param, val| settings.definition_of(param, :wukong) }.
-        reject{|param, val| param.to_s =~ /catalog_root/ }.
-        map{|param,val| "--#{param}=#{val}" }.
-        join(" ")
-    end
-
-    # the full, real path to the script file
-    def this_script_filename
-      Pathname.new($0).realpath
-    end
-
-    # use the full ruby interpreter path to run slave processes
-    def ruby_interpreter_path
-      Pathname.new(File.join(
-          Config::CONFIG["bindir"],
-          Config::CONFIG["RUBY_INSTALL_NAME"]+Config::CONFIG["EXEEXT"])).realpath
-    end
-
-
-  end
-end

data/lib/away/script/hadoop_command.rb

@@ -1,240 +0,0 @@
-# -*- coding: utf-8 -*-
-module Wukong
-  class Script
-    module HadoopCommand
-
-      # ===========================================================================
-      #
-      # Hadoop Options
-      #
-      Settings.define :hadoop_home, :default => '/usr/lib/hadoop', :description => "Path to hadoop installation; ENV['HADOOP_HOME'] by default. HADOOP_HOME/bin/hadoop is used to run hadoop.", :env_var => 'HADOOP_HOME', :wukong => true
-      Settings.define :hadoop_runner,                            :description => "Path to hadoop script. Usually set --hadoop_home instead of this.", :wukong => true
-
-      #
-      # Translate simplified args to their hairy hadoop equivalents
-      #
-      Settings.define :io_sort_mb,             :jobconf => true, :description => 'io.sort.mb',                                             :wukong => true
-      Settings.define :io_sort_record_percent, :jobconf => true, :description => 'io.sort.record.percent',                                 :wukong => true
-      Settings.define :job_name,               :jobconf => true, :description => 'mapred.job.name',                                        :wukong => true
-      Settings.define :key_field_separator,    :jobconf => true, :description => 'map.output.key.field.separator',                         :wukong => true
-      Settings.define :map_speculative,        :jobconf => true, :description => 'mapred.map.tasks.speculative.execution',                 :wukong => true
-      Settings.define :map_tasks,              :jobconf => true, :description => 'mapred.map.tasks',                                       :wukong => true
-      Settings.define :max_maps_per_cluster,   :jobconf => true, :description => 'mapred.max.maps.per.cluster',                            :wukong => true
-      Settings.define :max_maps_per_node,      :jobconf => true, :description => 'mapred.max.maps.per.node',                               :wukong => true
-      Settings.define :max_node_map_tasks,     :jobconf => true, :description => 'mapred.tasktracker.map.tasks.maximum',                   :wukong => true
-      Settings.define :max_node_reduce_tasks,  :jobconf => true, :description => 'mapred.tasktracker.reduce.tasks.maximum',                :wukong => true
-      Settings.define :max_record_length,      :jobconf => true, :description => 'mapred.linerecordreader.maxlength',                      :wukong => true # "Safeguards against corrupted data: lines longer than this (in bytes) are treated as bad records."
-      Settings.define :max_reduces_per_cluster,:jobconf => true, :description => 'mapred.max.reduces.per.cluster',                         :wukong => true
-      Settings.define :max_reduces_per_node,   :jobconf => true, :description => 'mapred.max.reduces.per.node',                            :wukong => true
-      Settings.define :max_tracker_failures,   :jobconf => true, :description => 'mapred.max.tracker.failures',                            :wukong => true
-      Settings.define :max_map_attempts,       :jobconf => true, :description => 'mapred.map.max.attempts',                                :wukong => true
-      Settings.define :max_reduce_attempts,    :jobconf => true, :description => 'mapred.reduce.max.attempts',                             :wukong => true
-      Settings.define :min_split_size,         :jobconf => true, :description => 'mapred.min.split.size',                                  :wukong => true
-      Settings.define :output_field_separator, :jobconf => true, :description => 'stream.map.output.field.separator',                      :wukong => true
-      Settings.define :partition_fields,       :jobconf => true, :description => 'num.key.fields.for.partition',                           :wukong => true
-      Settings.define :reduce_tasks,           :jobconf => true, :description => 'mapred.reduce.tasks',                                    :wukong => true
-      Settings.define :respect_exit_status,    :jobconf => true, :description => 'stream.non.zero.exit.is.failure',                        :wukong => true
-      Settings.define :reuse_jvms,             :jobconf => true, :description => 'mapred.job.reuse.jvm.num.tasks',                         :wukong => true
-      Settings.define :sort_fields,            :jobconf => true, :description => 'stream.num.map.output.key.fields',                       :wukong => true
-      Settings.define :timeout,                :jobconf => true, :description => 'mapred.task.timeout',                                    :wukong => true
-      Settings.define :noempty,                                  :description => "don't create zero-byte reduce files (hadoop mode only)", :wukong => true
-      Settings.define :split_on_xml_tag,                         :description => "Parse XML document by specifying the tag name: 'anything found between <tag> and </tag> will be treated as one record for map tasks'", :wukong => true
-
-      # emit a -jobconf hadoop option if the simplified command line arg is present
-      # if not, the resulting nil will be elided later
-      def jobconf option
-        if settings[option]
-          # "-jobconf %s=%s" % [settings.definition_of(option, :description), settings[option]]
-          "-D %s=%s" % [settings.definition_of(option, :description), settings[option]]
-        end
-      end
-
-      #
-      # Assemble the hadoop command to execute
-      # and launch the hadoop runner to execute the script across all tasktrackers
-      #
-      # FIXME: Should add some simple logic to ensure that commands are in the
-      # right order or hadoop will complain. ie. -D settings MUST come before
-      # others
-      #
-      def execute_hadoop_workflow
-        # Input paths join by ','
-        input_paths = @input_paths.join(',')
-        #
-        # Use Settings[:hadoop_home] to set the path your config install.
-        hadoop_commandline = [
-          hadoop_runner,
-          "jar #{settings[:hadoop_home]}/contrib/streaming/hadoop-*streaming*.jar",
-          hadoop_jobconf_settings,
-          "-D mapred.job.name='#{job_name}'",
-          hadoop_other_args,
-          "-mapper  '#{mapper_commandline}'",
-          "-reducer '#{reducer_commandline}'",
-          "-input   '#{input_paths}'",
-          "-output  '#{output_path}'",
-          "-file    '#{this_script_filename}'",
-          hadoop_recycle_env,
-        ].flatten.compact.join(" \t\\\n  ")
-        Log.info "  Launching hadoop!"
-        execute_command!(hadoop_commandline)
-      end
-
-      def hadoop_jobconf_settings
-        jobconf_settings = []
-        # Fixup these settings
-        settings[:reuse_jvms] = '-1'             if (settings[:reuse_jvms] == true)
-        settings[:respect_exit_status] = 'false' if (settings[:ignore_exit_status] == true)
-        # If no reducer and no reduce_command, then skip the reduce phase
-        # FIXME: auto-detect nil reducer
-        # settings[:reduce_tasks] = 0 if (! reducer) && (! settings[:reduce_command]) && (! settings[:reduce_tasks])
-        # Fields hadoop should use to distribute records to reducers
-        unless settings[:partition_fields].blank?
-          jobconf_settings += [
-            jobconf(:partition_fields),
-            jobconf(:output_field_separator),
-          ]
-        end
-        jobconf_settings += [
-          :io_sort_mb,               :io_sort_record_percent,
-          :map_speculative,          :map_tasks,
-          :max_maps_per_cluster,     :max_maps_per_node,
-          :max_node_map_tasks,       :max_node_reduce_tasks,
-          :max_reduces_per_cluster,  :max_reduces_per_node,
-          :max_record_length,        :min_split_size,
-          :output_field_separator,   :key_field_separator,
-          :partition_fields,         :sort_fields,
-          :reduce_tasks,             :respect_exit_status,
-          :reuse_jvms,               :timeout,
-          :max_tracker_failures,     :max_map_attempts,
-          :max_reduce_attempts
-        ].map{|opt| jobconf(opt)}
-        jobconf_settings.flatten.compact
-      end
-
-      def hadoop_other_args
-        extra_str_args  = [ settings[:extra_args] ]
-        if settings.split_on_xml_tag
-          extra_str_args << %Q{-inputreader 'StreamXmlRecordReader,begin=<#{settings.split_on_xml_tag}>,end=</#{settings.split_on_xml_tag}>'}
-        end
-        extra_str_args   << ' -lazyOutput' if settings[:noempty]  # don't create reduce file if no records
-        extra_str_args   << ' -partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner' unless settings[:partition_fields].blank?
-        extra_str_args
-      end
-
-      def hadoop_recycle_env
-        %w[RUBYLIB].map do |var|
-          %Q{-cmdenv '#{var}=#{ENV[var]}'} if ENV[var]
-        end.compact
-      end
-
-      # The path to the hadoop runner script
-      def hadoop_runner
-        settings[:hadoop_runner] || (settings[:hadoop_home]+'/bin/hadoop')
-      end
-
-      module ClassMethods
-        #
-        # Via @pskomoroch via @tlipcon,
-        #
-        #  "there is a little known Hadoop Streaming trick buried in this Python
-        #   script. You will notice that the date is not actually in the raw log
-        #   data itself, but is part of the filename. It turns out that Hadoop makes
-        #   job parameters you would fetch in Java with something like
-        #   job.get("mapred.input.file") available as environment variables for
-        #   streaming jobs, with periods replaced with underscores:
-        #
-        #     filepath = os.environ["map_input_file"]
-        #     filename = os.path.split(filepath)[-1]
-        #   Thanks to Todd Lipcon for directing me to that hack.
-        #
-
-        # HDFS pathname to the input file currently being processed.
-        def input_file
-          ENV['map_input_file']
-        end
-
-        # Directory of the input file
-        def input_dir
-          ENV['mapred_input_dir']
-        end
-
-        # Offset of this chunk within the input file
-        def map_input_start_offset
-          ENV['map_input_start']
-        end
-
-        # length of the mapper's input chunk
-        def map_input_length
-          ENV['map_input_length']
-        end
-
-        def attempt_id
-          ENV['mapred_task_id']
-        end
-        def curr_task_id
-          ENV['mapred_tip_id']
-        end
-
-        def script_cmdline_urlenc
-          ENV['stream_map_streamprocessor']
-        end
-      end
-
-      # Standard ClassMethods-on-include trick
-      def self.included base
-        base.class_eval do
-          extend ClassMethods
-        end
-      end
-    end
-  end
-end
-
-        # -partitioner                          org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner \
-        # -D mapred.output.key.comparator.class=org.apache.hadoop.mapred.lib.KeyFieldBasedComparator \
-        # -D mapred.text.key.comparator.options=-k2,2nr\
-        # -D mapred.text.key.partitioner.options=-k1,2\
-        # -D mapred.text.key.partitioner.options=\"-k1,$partfields\"
-        # -D stream.num.map.output.key.fields=\"$sortfields\"
-        #
-        # -D stream.map.output.field.separator=\"'/t'\"
-        # -D    map.output.key.field.separator=. \
-        # -D       mapred.data.field.separator=. \
-        # -D map.output.key.value.fields.spec=6,5,1-3:0- \
-        # -D reduce.output.key.value.fields.spec=0-2:5- \
-
-      # "HADOOP_HOME"                             =>"/usr/lib/hadoop-0.20/bin/..",
-      # "HADOOP_IDENT_STRING"                     =>"hadoop",
-      # "HADOOP_LOGFILE"                          =>"hadoop-hadoop-tasktracker-ip-10-242-14-223.log",
-      # "HADOOP_LOG_DIR"                          =>"/usr/lib/hadoop-0.20/bin/../logs",
-      # "HOME"                                    =>"/var/run/hadoop-0.20",
-      # "JAVA_HOME"                               =>"/usr/lib/jvm/java-6-sun",
-      # "LD_LIBRARY_PATH"                         =>"/usr/lib/jvm/java-6-sun-1.6.0.10/jre/lib/i386/client:/usr/lib/jvm/java-6-sun-1.6.0.10/jre/lib/i386:/usr/lib/jvm/java-6-sun-1.6.0.10/jre/../lib/i386:/mnt/hadoop/mapred/local/taskTracker/jobcache/job_200910221152_0023/attempt_200910221152_0023_m_000000_0/work:/usr/lib/jvm/java-6-sun-1.6.0.10/jre/lib/i386/client:/usr/lib/jvm/java-6-sun-1.6.0.10/jre/lib/i386:/usr/lib/jvm/java-6-sun-1.6.0.10/jre/../lib/i386",
-      # "PATH"                                    =>"/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games",
-      # "USER"                                    =>"hadoop",
-      #
-      # "dfs_block_size"                          =>"134217728",
-      # "map_input_start"                         =>"0",
-      # "map_input_length"                        =>"125726898",
-      # "mapred_output_key_class"                 =>"org.apache.hadoop.io.Text",
-      # "mapred_output_value_class"               =>"org.apache.hadoop.io.Text",
-      # "mapred_output_format_class"              =>"org.apache.hadoop.mapred.TextOutputFormat",
-      # "mapred_output_compression_codec"         =>"org.apache.hadoop.io.compress.DefaultCodec",
-      # "mapred_output_compression_type"          =>"BLOCK",
-      # "mapred_task_partition"                   =>"0",
-      # "mapred_tasktracker_map_tasks_maximum"    =>"4",
-      # "mapred_tasktracker_reduce_tasks_maximum" =>"2",
-      # "mapred_tip_id"                           =>"task_200910221152_0023_m_000000",
-      # "mapred_task_id"                          =>"attempt_200910221152_0023_m_000000_0",
-      # "mapred_job_tracker"                      =>"ec2-174-129-141-78.compute-1.amazonaws.com:8021",
-      #
-      # "mapred_input_dir"                        =>"hdfs://ec2-174-129-141-78.compute-1.amazonaws.com/user/flip/ripd/com.tw/com.twitter.search/20090809",
-      # "map_input_file"                          =>"hdfs://ec2-174-129-141-78.compute-1.amazonaws.com/user/flip/ripd/com.tw/com.twitter.search/20090809/com.twitter.search+20090809233441-56735-womper.tsv.bz2",
-      # "mapred_working_dir"                      =>"hdfs://ec2-174-129-141-78.compute-1.amazonaws.com/user/flip",
-      # "mapred_work_output_dir"                  =>"hdfs://ec2-174-129-141-78.compute-1.amazonaws.com/user/flip/tmp/twsearch-20090809/_temporary/_attempt_200910221152_0023_m_000000_0",
-      # "mapred_output_dir"                       =>"hdfs://ec2-174-129-141-78.compute-1.amazonaws.com/user/flip/tmp/twsearch-20090809",
-      # "mapred_temp_dir"                         =>"/mnt/tmp/hadoop-hadoop/mapred/temp",
-      # "PWD"                                     =>"/mnt/hadoop/mapred/local/taskTracker/jobcache/job_200910221152_0023/attempt_200910221152_0023_m_000000_0/work",
-      # "TMPDIR"                                  =>"/mnt/hadoop/mapred/local/taskTracker/jobcache/job_200910221152_0023/attempt_200910221152_0023_m_000000_0/work/tmp",
-      # "stream_map_streamprocessor"              =>"%2Fusr%2Fbin%2Fruby1.8+%2Fmnt%2Fhome%2Fflip%2Fics%2Fwuclan%2Fexamples%2Ftwitter%2Fparse%2Fparse_twitter_search_requests.rb+--map+--rm",
-      # "user_name"                               =>"flip",