pwrake 2.0.1 → 2.1.0

data/lib/pwrake/option/host_map.rb

@@ -8,24 +8,37 @@ module Pwrake
       @weight = weight || 1.0
       @group = group || 0
       @id = id
+      @continuous_fail = 0
+      @total_fail = 0
+      @count_task = 0
     end
 
     attr_reader :name, :ncore, :weight, :group, :id, :steal_flag
     attr_accessor :idle_cores
 
     def set_ncore(n)
+      @retire = 0
+      @busy_cores = 0
       @ncore = @idle_cores = n
     end
 
-    def increase(n)
+    def idle(n)
+      @busy_cores -= n
       @idle_cores += n
+      @idle_cores -= @retire
+      @retire = 0
+      @idle_cores + @busy_cores < 1 # all retired
+    end
+
+    def busy(n)
+      @busy_cores += n
+      @idle_cores -= n
+      @idle_cores + @busy_cores < 1 # all retired
     end
 
     def decrease(n)
       @idle_cores -= n
-      if @idle_cores < 0
-        raise RuntimeError,"# of cores must be non-negative"
-      end
+      @idle_cores + @busy_cores < 1 # all retired
     end
 
     def steal_phase
@@ -34,6 +47,25 @@ module Pwrake
       @steal_flag = false
       t
     end
+
+    def retire(n)
+      @retire += n
+      Log.debug "retire n=#{n}, host=#{@name}"
+    end
+
+    def task_result(result)
+      @count_task += 1
+      case result
+      when "end"
+        @continuous_fail = 0
+      when "fail"
+        @continuous_fail += 1
+        @total_fail += 1
+      else
+        raise "unknown result: #{result}"
+      end
+      @continuous_fail
+    end
   end
 
   class HostMap < Hash

data/lib/pwrake/option/option.rb

@@ -1,3 +1,7 @@
+require "pathname"
+require "yaml"
+require "pwrake/option/host_map"
+
 module Pwrake
 
   START_TIME = Time.now
@@ -178,7 +182,7 @@ module Pwrake
         ['NUM_THREADS', proc{|v| v && v.to_i}],
         ['SHELL_START_INTERVAL', proc{|v| (v || 0.012).to_f}],
         ['HEARTBEAT', proc{|v| (v || 240).to_i}],
-        ['RETRY', proc{|v| (v || 0).to_i}],
+        ['RETRY', proc{|v| (v || 1).to_i}],
         ['DISABLE_AFFINITY', proc{|v| v || ENV['AFFINITY']=='off'}],
         ['DISABLE_STEAL', proc{|v| v || ENV['STEAL']=='off'}],
         ['GFARM_BASEDIR', proc{|v| v || '/tmp'}],
@@ -356,3 +360,5 @@ done
 
   end
 end
+
+require "pwrake/option/option_filesystem"

data/lib/pwrake/option/option_filesystem.rb

@@ -1,3 +1,6 @@
+require "pwrake/option/option_filesystem"
+require "parallel"
+
 module Pwrake
 
   class Option
@@ -8,7 +11,14 @@ module Pwrake
 
     def setup_filesystem
 
-      @worker_progs = %w[ writer log_executor executor invoker shared_directory ]
+      @worker_progs = %w[
+        parallel/processor_count
+        pwrake/worker/writer
+        pwrake/worker/log_executor
+        pwrake/worker/executor
+        pwrake/worker/invoker
+        pwrake/worker/shared_directory
+      ]
       @worker_option = {
         :base_dir  => "",
         :work_dir  => self['WORK_DIR'],
@@ -47,7 +57,7 @@ module Pwrake
           :gfarm2fs_debug_wait => self['GFARM2FS_DEBUG_WAIT'],
           :single_mp => self['GFARM_SINGLE_MP']
         })
-        @worker_progs << "gfarm_directory"
+        @worker_progs.push "pwrake/worker/gfarm_directory"
 
 	if self['DISABLE_AFFINITY']
 	  @queue_class = "TaskQueue"
@@ -61,7 +71,7 @@ module Pwrake
         #@num_noaction_threads = (n_noaction_th || 1).to_i
         @worker_option[:shared_directory] = "SharedDirectory"
       end
-      @worker_progs << "worker_main"
+      @worker_progs.push "pwrake/worker/worker_main"
       Log.debug "@queue_class=#{@queue_class}"
     end
 

data/lib/pwrake/queue/locality_aware_queue.rb

@@ -6,9 +6,13 @@ module Pwrake
       # group_map = {gid1=>[hid1,hid2,...], ...}
       @size_q = 0
       @q = {}
-      @host_map.by_id.each{|h| @q[h.id] = @array_class.new(h.ncore)}
+      @hostinfo_by_name = {}
+      @hostinfo_by_id.each do |id,h|
+        @hostinfo_by_name[h.name] = h
+        @q[id] = @array_class.new(h.ncore)
+      end
       @q_group = {}
-      group_map ||= {1=>@host_map.by_id.map{|h| h.id}}
+      group_map ||= {1=>@hostinfo_by_id.map{|id,h| id}}
       group_map.each do |gid,ary|
         q1 = {}     # same group
         q2 = @q.dup # other groups
@@ -31,7 +35,7 @@ module Pwrake
       else
         stored = false
         hints.each do |h|
-          host_info = @host_map.by_name[h]
+          host_info = @hostinfo_by_name[h]
           if host_info && q = @q[host_info.id]
             t.assigned.push(host_info.id)
             q.push(t)
@@ -88,7 +92,9 @@ module Pwrake
         t = q.shift(run_host)
         if t
           t.assigned.each do |h|
-            @q[h].delete(t)
+            if q_h = @q[h]
+              q_h.delete(t)
+            end
           end
           @size_q -= 1
         end
@@ -113,7 +119,7 @@ module Pwrake
           end
         end
         if max_num > 0
-          max_info = @host_map.by_id[max_host]
+          max_info = @hostinfo_by_id[max_host]
           Log.debug "deq_steal max_host=#{max_info.name} max_num=#{max_num}"
           t = host_info.steal_phase{|h| deq_locate(max_info,h)}
           #Log.debug "deq_steal task=#{t.inspect}"
@@ -131,7 +137,12 @@ module Pwrake
         n = 0
         @q.each do |h,q|
           if q.size > 0
-            s << _qstr(@host_map.by_id[h].name,q)
+            hinfo = @hostinfo_by_id[h]
+            if hinfo
+              s << _qstr(hinfo.name,q)
+            else
+              s << _qstr("(#{hinfo.inspect})",q)
+            end
           else
             n += 1
           end
@@ -139,6 +150,7 @@ module Pwrake
       end
       s << _qstr("local*#{n}",[]) if n > 0
       s << _qstr("remote",@q_remote)
+      s << "@size_q=#{@size_q}"
       s
     end
 
@@ -154,5 +166,28 @@ module Pwrake
         @q_remote.empty?
     end
 
+    def drop_host(host_info)
+      hid = host_info.id
+      if q_drop = @q.delete(hid)
+        n_move = 0
+        q_size = q_drop.size
+        while t = q_drop.shift
+          assigned_other = false
+          t.assigned.each do |h|
+            if h != hid && @q[h]
+              assigned_other = true
+              break
+            end
+          end
+          if !assigned_other
+            @size_q -= 1
+            @q_remote.push(t)
+            n_move += 1
+          end
+        end
+        Log.debug "LAQ#drop_host: host=#{host_info.name} q.size=#{q_size} n_move=#{n_move}"
+      end
+    end
+
   end
 end

data/lib/pwrake/queue/queue_array.rb

@@ -1,4 +1,5 @@
 require "forwardable"
+require "pwrake/task/task_rank"
 
 module Pwrake
 
@@ -65,13 +66,20 @@ module Pwrake
       super()
     end
 
-    def shift(host_info)
+    def shift(host_info=nil)
+      return super() unless host_info
+      tw_found = nil
       (size-1).downto(0) do |i|
-        if at(i).acceptable_for(host_info)
-          return delete_at(i)
+        tw = at(i)
+        if tw.acceptable_for(host_info)
+          if tw.untried_host?(host_info)
+            return delete_at(i)
+          else
+            tw_found ||= tw
+          end
         end
       end
-      nil
+      tw_found
     end
   end
 
@@ -80,13 +88,20 @@ module Pwrake
       super()
     end
 
-    def shift(host_info)
+    def shift(host_info=nil)
+      return super() unless host_info
+      tw_found = nil
       size.times do |i|
-        if at(i).acceptable_for(host_info)
-          return delete_at(i)
+        tw = at(i)
+        if tw.acceptable_for(host_info)
+          if tw.untried_host?(host_info)
+            return delete_at(i)
+          else
+            tw_found ||= tw
+          end
         end
       end
-      nil
+      tw_found
     end
   end
 
@@ -157,13 +172,18 @@ module Pwrake
     end
 
     def pop_last_rank(r,host_info)
+      tw_found = nil
       (size-1).downto(0) do |i|
         tw = at(i)
         if tw.rank == r && tw.acceptable_for(host_info)
-          return delete_at(i)
+          if tw.untried_host?(host_info)
+            return delete_at(i)
+          else
+            tw_found ||= tw
+          end
         end
       end
-      nil
+      tw_found
     end
 
     def hrf_delete(t)
@@ -197,7 +217,7 @@ module Pwrake
       hrf_push(t)
     end
 
-    def shift(host_info)
+    def shift(host_info=nil)
       return nil if empty?
       hrf_get(host_info)
     end

data/lib/pwrake/queue/task_queue.rb

@@ -1,15 +1,15 @@
+require "pwrake/queue/queue_array"
+require "pwrake/queue/no_action_queue"
+
 module Pwrake
 
   class TaskQueue
 
-    def initialize(host_map, group_map=nil)
-      @q = []
-      @empty = []
-
+    def initialize(hostinfo_by_id, group_map=nil)
       @enable_steal = true
       @q_no_action = NoActionQueue.new
 
-      @host_map = host_map
+      @hostinfo_by_id = hostinfo_by_id
 
       pri = Rake.application.pwrake_options['QUEUE_PRIORITY'] || "LIHR"
       case pri
@@ -62,22 +62,22 @@ module Pwrake
     end
 
     def deq_noaction_task(&block)
-      Log.debug "deq_task:"+(empty? ? " empty" : "\n#{inspect_q}")
+      Log.debug "deq_noaction_task:"+(empty? ? " (empty)" : "\n#{inspect_q}")
       while tw = @q_no_action.shift
         Log.debug "deq_noaction: #{tw.name}"
-        yield(tw,nil)
+        yield(tw)
       end
     end
 
     def deq_task(&block) # locality version
-      Log.debug "deq_task:"+(empty? ? " empty" : "\n#{inspect_q}")
+      Log.debug "deq_task:"+(empty? ? " (empty)" : "\n#{inspect_q}")
       queued = 0
       @n_turn.times do |turn|
         next if turn_empty?(turn)
         queued += deq_turn(turn,&block)
       end
       if queued>0
-        Log.debug "queued:#{queued}"
+        Log.debug "deq_task: queued=#{queued}"
       end
     end
 
@@ -85,9 +85,9 @@ module Pwrake
       queued = 0
       while true
         count = 0
-        @host_map.by_id.each do |host_info|
+        @hostinfo_by_id.each_value do |host_info|
           #Log.debug "TaskQueue#deq_turn host_info=#{host_info.name}"
-          if host_info.idle_cores > 0
+          if (n = host_info.idle_cores) && n > 0
           if turn_empty?(turn)
             return queued
           elsif tw = deq_impl(host_info,turn)
@@ -99,8 +99,7 @@ module Pwrake
               Log.fatal m
               raise RuntimeError,m
             else
-              host_info.decrease(n_task_cores)
-              yield(tw,host_info.id)
+              yield(tw,host_info,n_task_cores)
               count += 1
               queued += 1
             end
@@ -134,11 +133,6 @@ module Pwrake
         @q_no_input.empty?
     end
 
-    def task_end(tw, hid)
-      host_info = @host_map.by_id[hid]
-      host_info.increase(tw.n_used_cores(host_info))
-    end
-
     def _qstr(h,q)
       s = " #{h}: size=#{q.size} "
       case q.size
@@ -160,5 +154,8 @@ module Pwrake
       _qstr("no_input",@q_no_input)
     end
 
+    def drop_host(host_info)
+    end
+
   end
 end

data/lib/pwrake/report/report.rb

@@ -1,3 +1,5 @@
+require "csv"
+
 module Pwrake
 
   class Report

data/lib/pwrake/task/task_algorithm.rb

@@ -9,6 +9,7 @@ module Pwrake
     attr_reader :subsequents
     attr_reader :arguments
     attr_reader :property
+    attr_reader :unfinished_prereq
 
     def pw_search_tasks(args)
       Log.debug "#{self.class}#pw_search_tasks start, args=#{args.inspect}"
@@ -81,10 +82,12 @@ module Pwrake
     private :format_search_flags
 
     def pw_enq_subsequents
-      t = Time.now
+      #t = Time.now
       #h = application.pwrake_options['HALT_QUEUE_WHILE_SEARCH']
       #application.task_queue.synchronize(h) do
         @subsequents.each do |t|        # <<--- competition !!!
+          #u = t.unfinished_prereq.keys
+          #Log.debug "enq_subseq: self=#{self.name} subseq=#{t.name} @unfin_preq=#{u.inspect}"
           if t && t.check_prereq_finished(self.name)
             application.task_queue.enq(t.wrapper)
           end

data/lib/pwrake/task/task_manager.rb

@@ -1,3 +1,5 @@
+require "pwrake/task/task_property"
+
 module Pwrake
 
   module TaskManager

data/lib/pwrake/task/task_property.rb

@@ -39,6 +39,7 @@ module Pwrake
     end
 
     def acceptable_for(host_info)
+      return true unless host_info
       if @disable_steal && host_info.steal_flag
         #Log.debug("@disable_steal && host_info.steal_flag")
         return false

data/lib/pwrake/task/task_wrapper.rb

@@ -1,4 +1,6 @@
-require 'forwardable'
+require "forwardable"
+require "csv"
+require "pwrake/task/task_rank"
 
 module Pwrake
 
@@ -16,18 +18,19 @@ module Pwrake
       @@current_id += 1
       @location = []
       @group = []
-      @group_id
+      @group_id = nil
       @suggest_location = nil
-      @file_stat
-      @input_file_size
-      @input_file_mtime
-      @rank
-      @priority
+      @file_stat = nil
+      @input_file_size = nil
+      @input_file_mtime = nil
+      @rank = nil
+      @priority = nil
       @lock_rank = Monitor.new
       @executed = false
       @assigned = []
       @exec_host = nil
-      @nretry = @property.retry || Rake.application.pwrake_options["RETRY"] || 0
+      @tried_hosts = []
+      @n_retry = @property.retry || Rake.application.pwrake_options["RETRY"] || 1
     end
 
     def_delegators :@task, :name, :actions, :prerequisites, :subsequents
@@ -36,6 +39,7 @@ module Pwrake
     attr_reader :task, :task_id, :group, :group_id, :file_stat
     attr_reader :location
     attr_reader :assigned
+    attr_reader :tried_hosts
     attr_accessor :executed
     attr_accessor :exec_host
     attr_accessor :shell_id, :status
@@ -66,24 +70,19 @@ module Pwrake
       @time_start = Time.now
     end
 
-    def retry
-      if @nretry > 0
-        s="retry task: #{name}"
-        Log.debug(s)
-        $stderr.puts(s)
-        @nretry -= 1
-        Rake.application.task_queue.enq(self)
-        true
-      else
-        false
-      end
+    def retry?
+      @status != "end" && @n_retry > 0
+    end
+
+    def no_more_retry
+      @n_retry == 0
     end
 
     def postprocess(location)
       @executed = true if !@task.actions.empty?
-      tm_taskend = Time.now
+      #tm_taskend = Time.now
       if is_file_task?
-        t = Time.now
+        #t = Time.now
         if File.exist?(name)
           @file_stat = File::Stat.new(name)
           @location = location
@@ -92,8 +91,23 @@ module Pwrake
       #Log.debug "postprocess time=#{Time.now-tm_taskend}"
       log_task
       @shell.current_task = nil if @shell
+    end
+
+    def retry_or_subsequent
+      @tried_hosts << @exec_host
       if @status=="end"
         @task.pw_enq_subsequents
+      elsif @n_retry > 0
+        @suggest_location = []
+        s="retry task (retry_count=#{@n_retry}): #{name}"
+        Log.warn(s)
+        $stderr.puts(s)
+        @n_retry -= 1
+        Rake.application.task_queue.enq(self)
+      else
+        s="give up retry (retry_count=0): #{name}"
+        Log.error(s)
+        $stderr.puts(s)
       end
     end
 
@@ -306,5 +320,10 @@ module Pwrake
       @n_used_cores ||= @property.n_used_cores(host_info)
     end
 
+    def untried_host?(host_info)
+      return true unless host_info
+      !@tried_hosts.include?(host_info.name)
+    end
+
   end
 end