pwrake 0.9.5 → 0.9.6

data/lib/pwrake/locality_aware_queue.rb

@@ -7,6 +7,70 @@ module Pwrake
   end
 
 
+  class LocalityConditionVariable < ConditionVariable
+    def signal(hints=nil)
+      if hints.nil?
+        super()
+      elsif Array===hints
+          thread = nil
+          @waiters_mutex.synchronize do
+            @waiters.each do |t,v|
+              if hints.include?(t[:hint])
+                thread = t
+                break
+              end
+            end
+            if thread
+              @waiters.delete(thread)
+            else
+              thread,_ = @waiters.shift
+            end
+          end
+          Log.debug "--- LCV#signal: hints=#{hints.inspect} thread_to_run=#{thread.inspect} @waiters.size=#{@waiters.size}"
+          begin
+            thread.run if thread
+          rescue ThreadError
+            retry # t was already dead?
+          end
+      else
+        raise ArgumentError,"argument must be an Array"
+      end
+      self
+    end
+
+    def broadcast(hints=nil)
+      if hints.nil?
+        super()
+      elsif Array===hints
+          threads = []
+          @waiters_mutex.synchronize do
+            hints.each do |h|
+              @waiters.each do |t,v|
+                if t[:hint] == h
+                  threads << t
+                  break
+                end
+              end
+            end
+            threads.each do |t|
+              @waiters.delete(t)
+            end
+          end
+          Log.debug "--- LCV#broadcast: hints=#{hints.inspect} threads_to_run=#{threads.inspect} @waiters.size=#{@waiters.size}"
+          threads.each do |t|
+            begin
+              t.run
+            rescue ThreadError
+            end
+          end
+      else
+        raise ArgumentError,"argument must be an Array"
+      end
+      self
+    end
+  end
+
+
   class LocalityAwareQueue < TaskQueue
 
     class Throughput
@@ -67,144 +131,90 @@ module Pwrake
     end # class Throughput
 
 
-    def initialize(hosts,opt={})
-      super(opt)
+    def init_queue(hosts)
+      @cv = LocalityConditionVariable.new
       @hosts = hosts
       @throughput = Throughput.new
       @size = 0
-      @q = nil
-      @q1 = []
-      @q2 = {}
-      @hosts.each{|h| @q2[h]=[]}
-      @q2[nil] = []
-      @enable_steal = !opt['disable_steal']
-      @time_prev = Time.now
-
-      @thread = Thread.new{thread_loop}
+      @q = {}
+      @hosts.each{|h| @q[h]=@array_class.new}
+      @q_remote = @array_class.new
+      @q_later = Array.new
+      @enable_steal = !Pwrake.application.pwrake_options['DISABLE_STEAL']
+      @steal_wait = (Pwrake.application.pwrake_options['STEAL_WAIT'] || 0).to_i
+      @steal_wait_max = (Pwrake.application.pwrake_options['STEAL_WAIT_MAX'] || 10).to_i
+      Log.info("-- @steal_wait=#{@steal_wait} @steal_wait_max=#{@steal_wait_max}")
     end
 
     attr_reader :size
 
 
-    def thread_loop
-      while !@finished
-        bulk_mvq
-        sleep # time_out
-      end
-    end
-
-
-    def time_out
-      if @size == 0
-        if @q1.size == 1
-          0.25
-        else
-          0.5
-        end
+    def enq_impl(t)
+      hints = t.suggest_location
+      if hints.nil? || hints.empty?
+        @q_later.push(t)
       else
-        if @q1.size > @size
-          0.5
-        elsif @q1.size > @size/2
-          1.0
-        else
-          3.0
-        end
-      end
-    end
-
-
-    def bulk_mvq
-      a = nil
-      @mutex.synchronize do
-        return if @q1.empty?
-
-        time_now = Time.now
-        interval = time_now-@time_prev
-        return if time_now-@time_prev < time_out
-
-        log_bulk_mvq
-        @time_prev = time_now
-
-        a = @q1
-        @q1 = []
-      end
-
-      where(a)
-
-      @mutex.synchronize do
-        while t = a.shift
-          mvq(t)
-        end
-        @cv.broadcast
-      end
-    end
-
-    def log_bulk_mvq
-      msg = @q1[0..2].map{|t| t.name}.inspect
-      msg.sub!(/]$/,",...") if @q1.size > 3
-      Log.info "-- bulk_mvq interval=%.6fs @size=#{@size} @q1.size=#{@q1.size} @q1=#{msg}"%(Time.now-@time_prev)
-    end
-
-
-    def where(task_list)
-      # implemented in child class
-    end
-
-
-    def mvq(t)
-      stored = false
-      if t.respond_to? :location
-        t.location.each do |h|
-          if q = @q2[h]
+        stored = false
+        hints.each do |h|
+          if q = @q[h]
             t.assigned.push(h)
             q.push(t)
             stored = true
           end
         end
-      end
-      if !stored
-        @q2[@hosts[rand(@hosts.size)]].push(t)
-        # @q2[nil].push(t)
+        if !stored
+          @q_remote.push(t)
+        end
       end
       @size += 1
     end
 
 
-    def enq_impl(item,hint=nil)
-      #Log.debug "--- #{self.class}#enq_impl #{item.inspect}"
-      @q1.push(item)
-    end
-
-    def enq_finish
-      @thread.run if @thread.alive?
-    end
-
     def deq_impl(host,n)
-      @thread.run if @thread.alive?
-
       if t = deq_locate(host)
         Log.info "-- deq_locate n=#{n} task=#{t.name} host=#{host}"
+        Log.debug "--- deq_impl\n#{inspect_q}"
+        return t
+      end
+
+      if !@q_remote.empty?
+        t = @q_remote.shift
+        Log.info "-- deq_remote n=#{n} task=#{t.name} host=#{host}"
+        Log.debug "--- deq_impl\n#{inspect_q}"
         return t
       end
 
-      if @enable_steal && n > 1
+      if !@q_later.empty?
+        t = @q_later.shift
+        Log.info "-- deq_later n=#{n} task=#{t.name} host=#{host}"
+        Log.debug "--- deq_impl\n#{inspect_q}"
+        return t
+      end
+
+      if @enable_steal && n > 0
         if t = deq_steal(host)
           Log.info "-- deq_steal n=#{n} task=#{t.name} host=#{host}"
+          Log.debug "--- deq_impl\n#{inspect_q}"
           return t
         end
       end
 
-      m = 0.05*(2**([n,5].min))
+      m = [@steal_wait*(2**n), @steal_wait_max].min
       @cv.wait(@mutex,m)
       nil
     end
 
 
     def deq_locate(host)
-      q = @q2[host]
+      q = @q[host]
       if q && !q.empty?
         t = q.shift
-        t.assigned.each{|x| @q2[x].delete_if{|x| t.equal? x}}
+        t.assigned.each do |h|
+          a = @q[h]
+          if i = a.index(t)
+            a.delete_at(i)
+          end
+        end
         @size -= 1
         return t
       else
@@ -216,38 +226,58 @@ module Pwrake
       # select a task based on many and close
       max_host = nil
       max_num  = 0
-      @q2.each do |h,a|
+      @q.each do |h,a|
         if !a.empty?
-          d = @throughput.interhost(host,h) * a.size
+          d = a.size
           if d > max_num
             max_host = h
             max_num  = d
           end
         end
       end
-      if max_host
-        deq_locate(max_host)
-      else
-        deq_locate(nil)
-      end
+      Log.info "-- deq_steal max_host=#{max_host} max_num=#{max_num}"
+      deq_locate(max_host)
+    end
+
+    def inspect_q
+      s = ""
+      b = proc{|h,q|
+        s += " #{h}: size=#{q.size} "
+        case q.size
+        when 0
+          s += "[]\n"
+        when 1
+          s += "[#{q[0].name}]\n"
+        else
+          s += "[#{q[0].name},..]\n"
+        end
+      }
+      @q.each(&b)
+      b.call("remote",@q_remote)
+      b.call("later",@q_later)
+      s
     end
 
     def size
-      @q1.size + @size
+      @size
     end
 
     def clear
-      @q1.clear
-      @hosts.each{|h| @q2[h].clear}
+      @q.each{|h,q| q.clear}
+      @q_remote.clear
+      @q_later.clear
+      @reserved_q.clear
     end
 
     def empty?
-      @q1.empty? && @hosts.all?{|h| @q2[h].empty?}
+      @q.all?{|h,q| q.empty?} &&
+        @q_remote.empty? &&
+        @q_later.empty? &&
+        @reserved_q.empty?
     end
 
     def finish
       super
-      @thread.run if @thread.alive?
     end
 
   end

data/lib/pwrake/logger.rb

@@ -142,12 +142,18 @@ module Pwrake
       Timer.new(prefix,*args)
     end
 
-    def output_message(message)
+    def stderr_puts(message)
       LOCK.synchronize do
         $stderr.write(message+"\n")
       end
     end
 
+    def stdout_puts(message)
+      LOCK.synchronize do
+        $stdout.write(message+"\n")
+      end
+    end
+
   end
 
 end # module Pwrake

data/lib/pwrake/master.rb

@@ -6,6 +6,7 @@ module Pwrake
 
   def current_shell=(a)
     Thread.current[:shell] = a
+    Thread.current[:hint] = a.host
   end
 
   module_function :current_shell, :current_shell=
@@ -15,16 +16,23 @@ module Pwrake
     include Pwrake::Option
 
     attr_reader :task_queue
+    attr_reader :finish_queue
     attr_reader :shell_set
+    attr_reader :filesystem
+    attr_reader :postprocess
 
     def initialize
       init_option    # Pwrake::Option
       setup_option   # Pwrake::Option
       @started = false
+      @lock = Mutex.new
+      @current_task_id = -1
     end
 
     def start
       return if @task_queue
+      timer = Timer.new("start_worker")
+      @finish_queue = Queue.new
       @task_queue = @queue_class.new(@core_list)
       @task_queue.enable_steal = !Rake.application.options.disable_steal
       @shell_set = []
@@ -32,6 +40,7 @@ module Pwrake
         @shell_set << @shell_class.new(h,@shell_opt)
       end
       start_threads
+      timer.finish
     end
 
     def finish
@@ -63,14 +72,24 @@ module Pwrake
       @task_queue.reserve(last) if last
       hint = (conn) ? conn.host : nil
       standard_exception_handling do
-        while t = @task_queue.deq(hint)
-          Log.debug "-- Master#thread_loop deq t=#{t.inspect}"
+        while true
+	  time_start = Time.now
+	  t = @task_queue.deq(hint)
+	  break if !t
+	  time_deq = Time.now - time_start
+          Log.debug "--- Master#thread_loop deq t=#{t.inspect} time=#{time_deq}sec"
           t.pw_invoke
           return if t == last
         end
       end
     end
 
+    def task_id_counter
+      @lock.synchronize do
+        @current_task_id += 1
+      end
+    end
+
     # Provide standard execption handling for the given block.
     def standard_exception_handling
       begin

data/lib/pwrake/option.rb

@@ -6,57 +6,88 @@ module Pwrake
 
     DEFAULT_CONFFILES = ["pwrake_conf.yaml","PwrakeConf.yaml"]
 
+    def format_time_pid(v)
+      START_TIME.strftime(v).sub("%$","%05d"%Process.pid)
+    end
+
+    def parse_opt(s)
+      case s
+      when /false|nil|off/i
+        false
+      when /true|on/i
+        true
+      else
+        s
+      end
+    end
+
     def option_data
       [
-       'DRYRUN',
-       'IGNORE_SYSTEM',
-       'IGNORE_DEPRECATE',
-       'LOAD_SYSTEM',
-       'NOSEARCH',
-       'RAKELIB',
-       'SHOW_PREREQS',
-       'SILENT',
-       'TRACE',
-       'TRACE_RULES',
-
-       'FILESYSTEM',
-       'SSH_OPTION',
-       'PASS_ENV',
-       'GNU_TIME',
-       'HALT_QUEUE_WHILE_SEARCH',
-       'SHOW_CONF',
-
-       ['HOSTFILE','HOSTS'],
-       ['LOGFILE','LOG',
-        proc{|v|
-          if v
-            # turn trace option on
-            Rake.application.options.trace = true
-            if v == "" || !v.kind_of?(String)
-              v = "Pwrake%Y%m%d-%H%M%S_%$.log"
+        'DRYRUN',
+        'IGNORE_SYSTEM',
+        'IGNORE_DEPRECATE',
+        'LOAD_SYSTEM',
+        'NOSEARCH',
+        'RAKELIB',
+        'SHOW_PREREQS',
+        'SILENT',
+        'TRACE',
+        'TRACE_RULES',
+
+        'FILESYSTEM',
+        'SSH_OPTION',
+        'PASS_ENV',
+        'GNU_TIME',
+        'DEBUG',
+        'PLOT_PARALLELISM',
+        'HALT_QUEUE_WHILE_SEARCH',
+        'SHOW_CONF',
+        'FAILED_TARGET', # rename(default), delete, leave
+        'QUEUE_PRIORITY', # DFS(default), FIFO,
+        'STEAL_WAIT',
+        'STEAL_WAIT_MAX',
+
+        ['HOSTFILE','HOSTS'],
+        ['LOGFILE','LOG',
+          proc{|v|
+            if v
+              # turn trace option on
+              Rake.application.options.trace = true
+              if v == "" || !v.kind_of?(String)
+                v = "Pwrake%Y%m%d-%H%M%S_%$.log"
+              end
+              format_time_pid(v)
             end
-            START_TIME.strftime(v).sub("%$",Process.pid.to_s)
-          end
-        }],
-       ['PROFILE',
-        proc{|v|
-          if v
-            if v == "" || !v.kind_of?(String)
-              v = "Pwrake%Y%m%d-%H%M%S_%$.csv"
+          }],
+        ['TASKLOG',
+          proc{|v|
+            if v
+              if v == "" || !v.kind_of?(String)
+                v = "Pwrake%Y%m%d-%H%M%S_%$.task"
+              end
+              format_time_pid(v)
             end
-            START_TIME.strftime(v).sub("%$",Process.pid.to_s)
-          end
-        }],
-       ['NUM_THREADS', proc{|v| v && v.to_i}],
-       ['DISABLE_AFFINITY', proc{|v| v || ENV['AFFINITY']=='off'}],
-       ['GFARM_BASEDIR', proc{|v| v || '/tmp'}],
-       ['GFARM_PREFIX', proc{|v| v || "pwrake_#{ENV['USER']}"}],
-       ['GFARM_SUBDIR', proc{|v| v || '/'}],
-       #['MASTER_HOSTNAME', proc{|v| v || `hostname -f`.chomp}],
-       ['WORK_DIR',proc{|v|
-          v ||= '$HOME/%CWD_RELATIVE_TO_HOME'
-          v.sub('%CWD_RELATIVE_TO_HOME',cwd_relative_to_home)
-        }]
+          }],
+        ['PROFILE',
+          proc{|v|
+            if v
+              if v == "" || !v.kind_of?(String)
+                v = "Pwrake%Y%m%d-%H%M%S_%$.csv"
+              end
+              format_time_pid(v)
+            end
+          }],
+        ['NUM_THREADS', proc{|v| v && v.to_i}],
+        ['DISABLE_AFFINITY', proc{|v| v || ENV['AFFINITY']=='off'}],
+        ['DISABLE_STEAL', proc{|v| v || ENV['STEAL']=='off'}],
+        ['GFARM_BASEDIR', proc{|v| v || '/tmp'}],
+        ['GFARM_PREFIX', proc{|v| v || "pwrake_#{ENV['USER']}"}],
+        ['GFARM_SUBDIR', proc{|v| v || '/'}],
+        #['MASTER_HOSTNAME', proc{|v| v || `hostname -f`.chomp}],
+        ['WORK_DIR',proc{|v|
+            v ||= '$HOME/%CWD_RELATIVE_TO_HOME'
+            v.sub('%CWD_RELATIVE_TO_HOME',cwd_relative_to_home)
+          }]
       ]
     end
 
@@ -68,6 +99,10 @@ module Pwrake
       init_options
       init_pass_env
       init_logger
+      Log.info "Options:"
+      @opts.each do |k,v|
+	Log.info " #{k} = #{v.inspect}"
+      end
       if @opts['SHOW_CONF']
         require "yaml"
         YAML.dump(@opts,$stdout)
@@ -81,7 +116,7 @@ module Pwrake
     attr_reader :logfile
     attr_reader :queue_class
     attr_reader :shell_class
-
+    attr_reader :task_logger
 
     def pwrake_options
       @opts
@@ -102,7 +137,7 @@ module Pwrake
       if @pwrake_conf.nil?
         @yaml = {}
       else
-        Log.debug "@pwrake_conf=#{@pwrake_conf}"
+        Log.debug "--- @pwrake_conf=#{@pwrake_conf}"
         require "yaml"
         @yaml = open(@pwrake_conf){|f| YAML.load(f) }
       end
@@ -160,11 +195,11 @@ module Pwrake
     #  command_option > ENV > pwrake_conf > DEFAULT_OPTIONS
     def search_opts(keys)
       val = Rake.application.options.send(keys[0].downcase.to_sym)
-      return val if !val.nil?
+      return parse_opt(val) if !val.nil?
       #
       keys.each do |k|
         val = ENV[k.upcase]
-        return val if !val.nil?
+        return parse_opt(val) if !val.nil?
       end
       #
       return nil if !@yaml
@@ -242,6 +277,15 @@ module Pwrake
       else
         Log.open($stdout)
       end
+
+      if @tasklog
+        @task_logger = File.open(@tasklog,'w')
+        h = %w[
+          task_id task_name start_time end_time elap_time preq preq_host
+          exec_host shell_id has_action executed file_size file_mtime file_host
+        ].join(',')+"\n"
+        @task_logger.print h
+      end
     end
 
     # ----- setup -----
@@ -266,7 +310,7 @@ module Pwrake
               begin
                 host = Socket.gethostbyname(host)[0]
               rescue
-                Log.info "FQDN not resoved : #{host}"
+                Log.info "-- FQDN not resoved : #{host}"
               end
               ncore = (ncore || 1).to_i
               group = (group || 0).to_i
@@ -290,12 +334,13 @@ module Pwrake
         @num_threads = 1 if !@num_threads
         @core_list = ['localhost'] * @num_threads
       end
+      Log.info "num_cores=#{@core_list.size}"
     end
 
 
     def set_filesystem
       if fn = @opts["PROFILE"]
-        Shell.profiler.open(fn,@opts['GNU_TIME'])
+        Shell.profiler.open(fn,@opts['GNU_TIME'],@opts['PLOT_PARALLELISM'])
       end
 
       @shell_opt = {
@@ -320,13 +365,17 @@ module Pwrake
         @shell_class = GfarmShell
         @shell_opt.merge!({
           :work_dir  => Dir.pwd,
-          :disable_steal => @opts['DISABLE_STEAL'],
           :single_mp => @opts['GFARM_SINGLE_MP'],
           :basedir   => @opts['GFARM_BASEDIR'],
           :prefix    => @opts['GFARM_PREFIX']
         })
-        @queue_class = GfarmQueue
-        # @queue_class = TaskQueue
+	if @opts['DISABLE_AFFINITY']
+	  @queue_class = TaskQueue
+	else
+	  @queue_class = LocalityAwareQueue
+	end
+        @postprocess = GfarmPostprocess.new
+        Log.debug "--- @queue_class=#{@queue_class}"
       else
         @filesystem  = 'nfs'
         @shell_class = Shell
@@ -360,6 +409,7 @@ module Pwrake
     # ----- finish -----
 
     def finish_option
+      @task_logger.close if @task_logger
       Log.close
     end