wukong 2.0.2 → 3.0.0.pre

data/lib/away/runner/execute.rb

@@ -0,0 +1,121 @@
+# -*- coding: utf-8 -*-
+
+#
+# Taken from the [Aruba project](). Original license:
+#
+#         Copyright (c) 2010 Aslak Hellesøy, David Chelimsky
+#
+#         Permission is hereby granted, free of charge, to any person obtaining
+#         a copy of this software and associated documentation files (the
+#         "Software"), to deal in the Software without restriction, including
+#         without limitation the rights to use, copy, modify, merge, publish,
+#         distribute, sublicense, and/or sell copies of the Software, and to
+#         permit persons to whom the Software is furnished to do so, subject to
+#         the following conditions:
+#
+#         The above copyright notice and this permission notice shall be
+#         included in all copies or substantial portions of the Software.
+#
+#         THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+#         EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+#         MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+#         NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+#         LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+#         OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+#         WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+
+require 'childprocess'
+require 'tempfile'
+
+module Wukong
+  class Runner
+
+    class Process
+      attr_reader :keep_ansi
+
+      def initialize(cmd, exit_timeout=2.0, io_wait=2.0, keep_ansi=true)
+        @exit_timeout = exit_timeout
+        @io_wait      = io_wait
+        @keep_ansi    = true
+
+        @process           = ChildProcess.build(*cmd)
+        @process.io.stdout = raw_out_io
+        @process.io.stderr = raw_err_io
+        @process.duplex    = true
+      end
+
+      def raw_out_io
+        @raw_out_io ||= StringIO.new('', 'w')
+      end
+
+      def raw_err_io
+        @raw_err_io ||= StringIO.new('', 'w')
+      end
+
+      def run!(&block)
+        @process.start
+        yield self if block_given?
+      end
+
+      def stdin
+        wait_for_io do
+          @process.io.stdin.sync = true
+          @process.io.stdin
+        end
+      end
+
+      def output
+        stdout + stderr
+      end
+
+      def stdout
+        wait_for_io do
+          @raw_out_io.rewind
+          filter_ansi(@raw_out_io.read)
+        end
+      end
+
+      def stderr
+        wait_for_io do
+          @raw_err_io.rewind
+          filter_ansi(@raw_err_io.read)
+        end
+      end
+
+      def stop(reader)
+        return unless @process
+        unless @process.exited?
+          reader.stdout stdout
+          reader.stderr stderr
+          @process.poll_for_exit(@exit_timeout)
+        end
+        @process.exit_code
+      end
+
+      def terminate
+        if @process
+          flush
+          @process.stop
+          flush
+        end
+      end
+
+      def flush
+        stdout && stderr # flush output
+      end
+
+    private
+
+      def wait_for_io(&block)
+        sleep @io_wait if @process.alive?
+        yield
+      end
+
+      def filter_ansi(string)
+        keep_ansi ? string : string.gsub(/\e\[\d+(?>(;\d+)*)m/, '')
+      end
+
+    end
+  end
+end

data/lib/away/script.rb

@@ -0,0 +1,161 @@
+require 'wukong'
+require 'wukong/script/hadoop_command'
+require 'wukong/experimental'
+
+#
+# Runner settings
+#
+
+Settings.define :mode, :type => Symbol, :default => :mapper, :env_var => 'WUKONG_MODE',  :description => "run the script's workflow: Specify 'hadoop' to use hadoop streaming; 'local' to run your_script.rb --map | sort | your_script.rb --reduce; 'emr' to launch on the amazon cloud; 'mapper' or 'reducer' to run that phase.", :wukong => true
+Settings.define :dry_run,  :description => "echo the command that will be run, but don't run it", :wukong => true
+Settings.define :rm,       :description => "Recursively remove the destination directory. Only used in hadoop mode.", :wukong => true
+Settings.define :script_file, :type => :filename, :description => "script file to execute, or give as first arg", :wukong => true
+
+module Wukong
+  # adds ability to execute
+  extend Wukong::Mixin::FromFile
+
+  def self.from_file(filename)
+    filename  = filename.to_s
+    filename += ".rb" if filename !~ /\.rb$/
+    super(filename)
+  end
+
+  def self.run(filename=nil)
+    if filename
+      self.from_file(filename)
+    else
+      Settings.resolve!
+    end
+    if @main_run then return false ; end
+    Wukong::Script.new(Settings).run
+    @main_run = true
+  end
+
+  #
+  # sources a script file,
+  #
+  class Script
+    attr_reader :settings     # configliere hash of settings
+    attr_reader :script_file  # File to execute
+    attr_reader :input_paths
+    attr_reader :output_path
+
+    include Wukong::Script::HadoopCommand
+
+    def initialize(settings)
+      @settings = settings
+
+      @output_path = settings.rest.pop
+      @input_paths = settings.rest.reject(&:blank?)
+    end
+
+
+    # Execute the script file in the context of the Wukong module
+    def run_flow
+      Log.debug( "Running #{script_file} with settings #{settings}")
+      script_file = settings.script_file
+      mode        = settings.mode
+      Wukong.flow(mode).run
+    end
+
+    #
+    # In --run mode, use the framework (local, hadoop, emr, etc) to re-launch
+    #   the script as mapper, reducer, etc.
+    # If --map or --reduce, dispatch to the mapper or reducer.
+    #
+    def run
+      case settings.mode
+      when :local           then execute_local_workflow
+      when :hadoop, :mapred then execute_hadoop_workflow
+      else
+        run_flow
+      end
+    end
+
+    #
+    # Shell command for map phase. By default, calls the script in --map mode
+    # In hadoop mode, this is given to the hadoop streaming command.
+    # In local mode, it's given to the system() call
+    #
+    def mapper_commandline
+      "#{ruby_interpreter_path} #{this_script_filename} --mode=mapper " + non_wukong_params
+    end
+
+    #
+    # Shell command for reduce phase. By default, calls the script in --reduce mode
+    # In hadoop mode, this is given to the hadoop streaming command.
+    # In local mode, it's given to the system() call
+    #
+    def reducer_commandline
+      "#{ruby_interpreter_path} #{this_script_filename} --mode=reducer " + non_wukong_params
+    end
+
+    def job_name
+      settings[:job_name] ||
+        "#{File.basename(this_script_filename)}---#{input_paths}---#{output_path}".gsub(%r{[^\w/\.\-\+]+}, '')
+    end
+
+    # Wrapper for dangerous operations to catch errors
+    def safely action, &block
+      begin
+        block.call
+      rescue StandardError => e ; handle_error(action, e); end
+    end
+
+  protected
+
+    #
+    # Execute the runner phase:
+    # use the running framework to relaunch the script in map and in reduce mode
+    #
+    def execute_command! *args
+      command = args.flatten.reject(&:blank?).join(" \\\n    ")
+      Log.info "Running\n\n#{command}\n"
+      if settings[:dry_run]
+        Log.info '== [Not running preceding command: dry run] =='
+      else
+        maybe_overwrite_output_paths! output_path
+        $stdout.puts `#{command}`
+        raise "Streaming command failed!" unless $?.success?
+      end
+    end
+
+    #
+    # In hadoop mode only, removes the destination path before launching
+    #
+    # To the panic-stricken: look in .Trash/current/path/to/accidentally_deleted_files
+    #
+    def maybe_overwrite_output_paths! output_path
+      if (settings.rm && (settings.mode == :hadoop))
+        cmd = %Q{#{hadoop_runner} fs -rmr '#{output_path}'}
+        Log.info "Removing output file #{output_path}: #{cmd}"
+        puts `#{cmd}`
+      end
+    end
+
+    # Reassemble all the non-internal-to-wukong settings into a command line for
+    # the map/reducer phase scripts
+    def non_wukong_params
+      settings.
+        reject{|param, val| settings.definition_of(param, :wukong) }.
+        reject{|param, val| param.to_s =~ /catalog_root/ }.
+        map{|param,val| "--#{param}=#{val}" }.
+        join(" ")
+    end
+
+    # the full, real path to the script file
+    def this_script_filename
+      Pathname.new($0).realpath
+    end
+
+    # use the full ruby interpreter path to run slave processes
+    def ruby_interpreter_path
+      Pathname.new(File.join(
+          Config::CONFIG["bindir"],
+          Config::CONFIG["RUBY_INSTALL_NAME"]+Config::CONFIG["EXEEXT"])).realpath
+    end
+
+
+  end
+end

data/lib/away/script/hadoop_command.rb

@@ -0,0 +1,240 @@
+# -*- coding: utf-8 -*-
+module Wukong
+  class Script
+    module HadoopCommand
+
+      # ===========================================================================
+      #
+      # Hadoop Options
+      #
+      Settings.define :hadoop_home, :default => '/usr/lib/hadoop', :description => "Path to hadoop installation; ENV['HADOOP_HOME'] by default. HADOOP_HOME/bin/hadoop is used to run hadoop.", :env_var => 'HADOOP_HOME', :wukong => true
+      Settings.define :hadoop_runner,                            :description => "Path to hadoop script. Usually set --hadoop_home instead of this.", :wukong => true
+
+      #
+      # Translate simplified args to their hairy hadoop equivalents
+      #
+      Settings.define :io_sort_mb,             :jobconf => true, :description => 'io.sort.mb',                                             :wukong => true
+      Settings.define :io_sort_record_percent, :jobconf => true, :description => 'io.sort.record.percent',                                 :wukong => true
+      Settings.define :job_name,               :jobconf => true, :description => 'mapred.job.name',                                        :wukong => true
+      Settings.define :key_field_separator,    :jobconf => true, :description => 'map.output.key.field.separator',                         :wukong => true
+      Settings.define :map_speculative,        :jobconf => true, :description => 'mapred.map.tasks.speculative.execution',                 :wukong => true
+      Settings.define :map_tasks,              :jobconf => true, :description => 'mapred.map.tasks',                                       :wukong => true
+      Settings.define :max_maps_per_cluster,   :jobconf => true, :description => 'mapred.max.maps.per.cluster',                            :wukong => true
+      Settings.define :max_maps_per_node,      :jobconf => true, :description => 'mapred.max.maps.per.node',                               :wukong => true
+      Settings.define :max_node_map_tasks,     :jobconf => true, :description => 'mapred.tasktracker.map.tasks.maximum',                   :wukong => true
+      Settings.define :max_node_reduce_tasks,  :jobconf => true, :description => 'mapred.tasktracker.reduce.tasks.maximum',                :wukong => true
+      Settings.define :max_record_length,      :jobconf => true, :description => 'mapred.linerecordreader.maxlength',                      :wukong => true # "Safeguards against corrupted data: lines longer than this (in bytes) are treated as bad records."
+      Settings.define :max_reduces_per_cluster,:jobconf => true, :description => 'mapred.max.reduces.per.cluster',                         :wukong => true
+      Settings.define :max_reduces_per_node,   :jobconf => true, :description => 'mapred.max.reduces.per.node',                            :wukong => true
+      Settings.define :max_tracker_failures,   :jobconf => true, :description => 'mapred.max.tracker.failures',                            :wukong => true
+      Settings.define :max_map_attempts,       :jobconf => true, :description => 'mapred.map.max.attempts',                                :wukong => true
+      Settings.define :max_reduce_attempts,    :jobconf => true, :description => 'mapred.reduce.max.attempts',                             :wukong => true
+      Settings.define :min_split_size,         :jobconf => true, :description => 'mapred.min.split.size',                                  :wukong => true
+      Settings.define :output_field_separator, :jobconf => true, :description => 'stream.map.output.field.separator',                      :wukong => true
+      Settings.define :partition_fields,       :jobconf => true, :description => 'num.key.fields.for.partition',                           :wukong => true
+      Settings.define :reduce_tasks,           :jobconf => true, :description => 'mapred.reduce.tasks',                                    :wukong => true
+      Settings.define :respect_exit_status,    :jobconf => true, :description => 'stream.non.zero.exit.is.failure',                        :wukong => true
+      Settings.define :reuse_jvms,             :jobconf => true, :description => 'mapred.job.reuse.jvm.num.tasks',                         :wukong => true
+      Settings.define :sort_fields,            :jobconf => true, :description => 'stream.num.map.output.key.fields',                       :wukong => true
+      Settings.define :timeout,                :jobconf => true, :description => 'mapred.task.timeout',                                    :wukong => true
+      Settings.define :noempty,                                  :description => "don't create zero-byte reduce files (hadoop mode only)", :wukong => true
+      Settings.define :split_on_xml_tag,                         :description => "Parse XML document by specifying the tag name: 'anything found between <tag> and </tag> will be treated as one record for map tasks'", :wukong => true
+
+      # emit a -jobconf hadoop option if the simplified command line arg is present
+      # if not, the resulting nil will be elided later
+      def jobconf option
+        if settings[option]
+          # "-jobconf %s=%s" % [settings.definition_of(option, :description), settings[option]]
+          "-D %s=%s" % [settings.definition_of(option, :description), settings[option]]
+        end
+      end
+
+      #
+      # Assemble the hadoop command to execute
+      # and launch the hadoop runner to execute the script across all tasktrackers
+      #
+      # FIXME: Should add some simple logic to ensure that commands are in the
+      # right order or hadoop will complain. ie. -D settings MUST come before
+      # others
+      #
+      def execute_hadoop_workflow
+        # Input paths join by ','
+        input_paths = @input_paths.join(',')
+        #
+        # Use Settings[:hadoop_home] to set the path your config install.
+        hadoop_commandline = [
+          hadoop_runner,
+          "jar #{settings[:hadoop_home]}/contrib/streaming/hadoop-*streaming*.jar",
+          hadoop_jobconf_settings,
+          "-D mapred.job.name='#{job_name}'",
+          hadoop_other_args,
+          "-mapper  '#{mapper_commandline}'",
+          "-reducer '#{reducer_commandline}'",
+          "-input   '#{input_paths}'",
+          "-output  '#{output_path}'",
+          "-file    '#{this_script_filename}'",
+          hadoop_recycle_env,
+        ].flatten.compact.join(" \t\\\n  ")
+        Log.info "  Launching hadoop!"
+        execute_command!(hadoop_commandline)
+      end
+
+      def hadoop_jobconf_settings
+        jobconf_settings = []
+        # Fixup these settings
+        settings[:reuse_jvms] = '-1'             if (settings[:reuse_jvms] == true)
+        settings[:respect_exit_status] = 'false' if (settings[:ignore_exit_status] == true)
+        # If no reducer and no reduce_command, then skip the reduce phase
+        # FIXME: auto-detect nil reducer
+        # settings[:reduce_tasks] = 0 if (! reducer) && (! settings[:reduce_command]) && (! settings[:reduce_tasks])
+        # Fields hadoop should use to distribute records to reducers
+        unless settings[:partition_fields].blank?
+          jobconf_settings += [
+            jobconf(:partition_fields),
+            jobconf(:output_field_separator),
+          ]
+        end
+        jobconf_settings += [
+          :io_sort_mb,               :io_sort_record_percent,
+          :map_speculative,          :map_tasks,
+          :max_maps_per_cluster,     :max_maps_per_node,
+          :max_node_map_tasks,       :max_node_reduce_tasks,
+          :max_reduces_per_cluster,  :max_reduces_per_node,
+          :max_record_length,        :min_split_size,
+          :output_field_separator,   :key_field_separator,
+          :partition_fields,         :sort_fields,
+          :reduce_tasks,             :respect_exit_status,
+          :reuse_jvms,               :timeout,
+          :max_tracker_failures,     :max_map_attempts,
+          :max_reduce_attempts
+        ].map{|opt| jobconf(opt)}
+        jobconf_settings.flatten.compact
+      end
+
+      def hadoop_other_args
+        extra_str_args  = [ settings[:extra_args] ]
+        if settings.split_on_xml_tag
+          extra_str_args << %Q{-inputreader 'StreamXmlRecordReader,begin=<#{settings.split_on_xml_tag}>,end=</#{settings.split_on_xml_tag}>'}
+        end
+        extra_str_args   << ' -lazyOutput' if settings[:noempty]  # don't create reduce file if no records
+        extra_str_args   << ' -partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner' unless settings[:partition_fields].blank?
+        extra_str_args
+      end
+
+      def hadoop_recycle_env
+        %w[RUBYLIB].map do |var|
+          %Q{-cmdenv '#{var}=#{ENV[var]}'} if ENV[var]
+        end.compact
+      end
+
+      # The path to the hadoop runner script
+      def hadoop_runner
+        settings[:hadoop_runner] || (settings[:hadoop_home]+'/bin/hadoop')
+      end
+
+      module ClassMethods
+        #
+        # Via @pskomoroch via @tlipcon,
+        #
+        #  "there is a little known Hadoop Streaming trick buried in this Python
+        #   script. You will notice that the date is not actually in the raw log
+        #   data itself, but is part of the filename. It turns out that Hadoop makes
+        #   job parameters you would fetch in Java with something like
+        #   job.get("mapred.input.file") available as environment variables for
+        #   streaming jobs, with periods replaced with underscores:
+        #
+        #     filepath = os.environ["map_input_file"]
+        #     filename = os.path.split(filepath)[-1]
+        #   Thanks to Todd Lipcon for directing me to that hack.
+        #
+
+        # HDFS pathname to the input file currently being processed.
+        def input_file
+          ENV['map_input_file']
+        end
+
+        # Directory of the input file
+        def input_dir
+          ENV['mapred_input_dir']
+        end
+
+        # Offset of this chunk within the input file
+        def map_input_start_offset
+          ENV['map_input_start']
+        end
+
+        # length of the mapper's input chunk
+        def map_input_length
+          ENV['map_input_length']
+        end
+
+        def attempt_id
+          ENV['mapred_task_id']
+        end
+        def curr_task_id
+          ENV['mapred_tip_id']
+        end
+
+        def script_cmdline_urlenc
+          ENV['stream_map_streamprocessor']
+        end
+      end
+
+      # Standard ClassMethods-on-include trick
+      def self.included base
+        base.class_eval do
+          extend ClassMethods
+        end
+      end
+    end
+  end
+end
+
+        # -partitioner                          org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner \
+        # -D mapred.output.key.comparator.class=org.apache.hadoop.mapred.lib.KeyFieldBasedComparator \
+        # -D mapred.text.key.comparator.options=-k2,2nr\
+        # -D mapred.text.key.partitioner.options=-k1,2\
+        # -D mapred.text.key.partitioner.options=\"-k1,$partfields\"
+        # -D stream.num.map.output.key.fields=\"$sortfields\"
+        #
+        # -D stream.map.output.field.separator=\"'/t'\"
+        # -D    map.output.key.field.separator=. \
+        # -D       mapred.data.field.separator=. \
+        # -D map.output.key.value.fields.spec=6,5,1-3:0- \
+        # -D reduce.output.key.value.fields.spec=0-2:5- \
+
+      # "HADOOP_HOME"                             =>"/usr/lib/hadoop-0.20/bin/..",
+      # "HADOOP_IDENT_STRING"                     =>"hadoop",
+      # "HADOOP_LOGFILE"                          =>"hadoop-hadoop-tasktracker-ip-10-242-14-223.log",
+      # "HADOOP_LOG_DIR"                          =>"/usr/lib/hadoop-0.20/bin/../logs",
+      # "HOME"                                    =>"/var/run/hadoop-0.20",
+      # "JAVA_HOME"                               =>"/usr/lib/jvm/java-6-sun",
+      # "LD_LIBRARY_PATH"                         =>"/usr/lib/jvm/java-6-sun-1.6.0.10/jre/lib/i386/client:/usr/lib/jvm/java-6-sun-1.6.0.10/jre/lib/i386:/usr/lib/jvm/java-6-sun-1.6.0.10/jre/../lib/i386:/mnt/hadoop/mapred/local/taskTracker/jobcache/job_200910221152_0023/attempt_200910221152_0023_m_000000_0/work:/usr/lib/jvm/java-6-sun-1.6.0.10/jre/lib/i386/client:/usr/lib/jvm/java-6-sun-1.6.0.10/jre/lib/i386:/usr/lib/jvm/java-6-sun-1.6.0.10/jre/../lib/i386",
+      # "PATH"                                    =>"/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games",
+      # "USER"                                    =>"hadoop",
+      #
+      # "dfs_block_size"                          =>"134217728",
+      # "map_input_start"                         =>"0",
+      # "map_input_length"                        =>"125726898",
+      # "mapred_output_key_class"                 =>"org.apache.hadoop.io.Text",
+      # "mapred_output_value_class"               =>"org.apache.hadoop.io.Text",
+      # "mapred_output_format_class"              =>"org.apache.hadoop.mapred.TextOutputFormat",
+      # "mapred_output_compression_codec"         =>"org.apache.hadoop.io.compress.DefaultCodec",
+      # "mapred_output_compression_type"          =>"BLOCK",
+      # "mapred_task_partition"                   =>"0",
+      # "mapred_tasktracker_map_tasks_maximum"    =>"4",
+      # "mapred_tasktracker_reduce_tasks_maximum" =>"2",
+      # "mapred_tip_id"                           =>"task_200910221152_0023_m_000000",
+      # "mapred_task_id"                          =>"attempt_200910221152_0023_m_000000_0",
+      # "mapred_job_tracker"                      =>"ec2-174-129-141-78.compute-1.amazonaws.com:8021",
+      #
+      # "mapred_input_dir"                        =>"hdfs://ec2-174-129-141-78.compute-1.amazonaws.com/user/flip/ripd/com.tw/com.twitter.search/20090809",
+      # "map_input_file"                          =>"hdfs://ec2-174-129-141-78.compute-1.amazonaws.com/user/flip/ripd/com.tw/com.twitter.search/20090809/com.twitter.search+20090809233441-56735-womper.tsv.bz2",
+      # "mapred_working_dir"                      =>"hdfs://ec2-174-129-141-78.compute-1.amazonaws.com/user/flip",
+      # "mapred_work_output_dir"                  =>"hdfs://ec2-174-129-141-78.compute-1.amazonaws.com/user/flip/tmp/twsearch-20090809/_temporary/_attempt_200910221152_0023_m_000000_0",
+      # "mapred_output_dir"                       =>"hdfs://ec2-174-129-141-78.compute-1.amazonaws.com/user/flip/tmp/twsearch-20090809",
+      # "mapred_temp_dir"                         =>"/mnt/tmp/hadoop-hadoop/mapred/temp",
+      # "PWD"                                     =>"/mnt/hadoop/mapred/local/taskTracker/jobcache/job_200910221152_0023/attempt_200910221152_0023_m_000000_0/work",
+      # "TMPDIR"                                  =>"/mnt/hadoop/mapred/local/taskTracker/jobcache/job_200910221152_0023/attempt_200910221152_0023_m_000000_0/work/tmp",
+      # "stream_map_streamprocessor"              =>"%2Fusr%2Fbin%2Fruby1.8+%2Fmnt%2Fhome%2Fflip%2Fics%2Fwuclan%2Fexamples%2Ftwitter%2Fparse%2Fparse_twitter_search_requests.rb+--map+--rm",
+      # "user_name"                               =>"flip",