pwrake 2.2.9 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 93b60deb41a4fda0bafa94a5b240812e5c5925ab204f8b802ef63d78fbcf81b2
-  data.tar.gz: 6ac5c66ad22c8ba3316bca3aeab8b34fe0a4e9189f50a99b2056c5a8cced7277
+  metadata.gz: 99a7b5b9f4ecf97b511f58495f6b22facbd66c97d79aa0cab9f452d96ae44ea4
+  data.tar.gz: 2874a8eec50b7f508138567899da703e4c5d18687d58ad53a715ffa81b7fa402
 SHA512:
-  metadata.gz: 7b0d50fb5cb90b8c7682be343c4ef70acfb324d01b46903f39606f8e044d3315ae7cf7a6cd8e4074e5b5178b3e16aa224c827f53b1ef8ea74a27b6176f84b39b
-  data.tar.gz: 90911a12880e22276be78feae3e13b850d9614f93fc30195a7fb253e730a369bc51783778524aca2213672fca6b290c7622bdf503278f0e04dfe2eb29d46acb5
+  metadata.gz: c49bcd791f52010a2a221118bec1f59c08796c053ef43127f68142bd97d95499d23becf2c3d6ab9f351bc48384300999824fd3bd0f6fe1e94cba5de6b291664a
+  data.tar.gz: af1ecd762e914f1e891a40639ab8f59a55165132646816ccb1a5209e9c761a518cb09f0c4385086f23bb7d172e658127852780577d748baa0c83313edb72e848

data/README.md

@@ -131,10 +131,13 @@ In this case, you need the rehash of command paths:
         SSH_OPTION        SSH option
         PASS_ENV          (Array) Environment variables passed to SSH
         HEARTBEAT         default=240 - Hearbeat interval in seconds
-        RETRY             default=1 - The number of retry
+        RETRY             default=1 - The number of task retry
+        HOST_FAILURE      default=2 - The number of allowed continuous host failure (since v2.3)
         FAILED_TARGET     rename(default)|delete|leave - Treatment of failed target files
         FAILURE_TERMINATION wait(default)|kill|continue - Behavior of other tasks when a task is failed
-        QUEUE_PRIORITY          LIHR(default)|FIFO|LIFO|RANK
+        QUEUE_PRIORITY          LIFO(default)|FIFO|LIHR(LIfo&Highest-Rank-first; obsolete)
+        DISABLE_RANK_PRIORITY   false(default)|true - Disable rank-aware task scheduling (since v2.3)
+        RESERVE_NODE            false(default)|true - Reserve a node for tasks with ncore>1 (since v2.3)
         NOACTION_QUEUE_PRIORITY FIFO(default)|LIFO|RAND
         SHELL_START_INTERVAL    default=0.012 (sec)
         GRAPH_PARTITION         false(default)|true

data/lib/pwrake/branch/communicator_set.rb

@@ -91,7 +91,7 @@ class CommunicatorSet
   end
 
   def handler_set
-    @communicators.each_value.map{|comm| comm.handler}
+    @communicators.each_value.map{|comm| comm.handler}.compact
   end
 
   def kill(sig)

data/lib/pwrake/master/master.rb

@@ -148,8 +148,7 @@ module Pwrake
       if @hostinfo_by_id.empty?
         raise RuntimeError,"no worker host"
       end
-      queue_class = Pwrake.const_get(@option.queue_class)
-      @task_queue = queue_class.new(@hostinfo_by_id)
+      @task_queue = TaskQueue.new(@option.queue_class,@hostinfo_by_id)
 
       @branch_setup_thread = Thread.new do
         create_fiber(@hdl_set) do |hdl|
@@ -171,15 +170,16 @@ module Pwrake
 
     def retire(hid)
       host_info = @hostinfo_by_id[hid.to_i]
-      if host_info && host_info.decrease(1)
-        # all retired
+      return if host_info.nil?
+      host_info.retire(1)
+      if host_info.retired?
         if !@exited
           m = "retired: host #{host_info.name}"
           Log.warn(m)
           $stderr.puts(m)
           drop_host(host_info) # delete from hostinfo_by_id
           if @hostinfo_by_id.empty?
-              raise RuntimeError,"no worker host"
+            raise RuntimeError,"no worker host"
           end
         end
       end
@@ -231,7 +231,7 @@ module Pwrake
     end
 
     def setup_fiber(t)
-      n_retry = @option["RETRY"]
+      @host_fail = @option["HOST_FAILURE"]
       create_fiber(@hdl_set) do |hdl|
         while s = hdl.get_line
           Log.debug "Master:recv #{s.inspect} from branch[#{hdl.host}]"
@@ -253,9 +253,10 @@ module Pwrake
             if tw.status == "fail"
               $stderr.puts %[task "#{tw.name}" failed.]
               if host_info
-                continuous_fail = host_info.task_result(tw.status)
+                host_info.count_result(tw.status)
+                continuous_fail = host_info.continuous_fail
                 Log.debug "task=#{tw.name} continuous_fail=#{continuous_fail}"
-                if continuous_fail > n_retry && @hostinfo_by_id.size > 1
+                if continuous_fail > @host_fail && @hostinfo_by_id.size > 1
                   # retire this host
                   drop_host(host_info)
                   Log.warn("retired host:#{host_info.name} due to continuous fail")
@@ -310,11 +311,12 @@ module Pwrake
       count = 0
       # @idle_cores.decrease(..
       @task_queue.deq_task do |tw,host_info,ncore|
-        host_info.busy(ncore)
         count += 1
         @hostinfo_by_taskname[tw.name] = host_info
+        tw.set_used_cores(ncore)
         tw.preprocess
-        if tw.has_action?
+        if host_info
+          host_info.busy(ncore)
           hid = host_info.id
           s = "#{hid}:#{tw.task_id}:#{tw.name}"
           @channel_by_hostid[hid].put_line(s)
@@ -322,14 +324,13 @@ module Pwrake
           tw.exec_host_id = hid
         else
           tw.status = "end"
-          task_end(tw,host_info) # @idle_cores.increase(..
           @post_pool.enq(tw)
         end
       end
       if count == 0 && !@task_queue.empty? && @hostinfo_by_taskname.empty?
         m="No task was invoked while unexecuted tasks remain"
         Log.error m
-        Log.error "count=#{count} @hostinfo_by_taskname.empty?=#{@hostinfo_by_taskname.empty?} @task_queue.empty?={@task_queue.empty?} @task_queue=\n"+@task_queue.inspect_q
+        Log.error "count=#{count} @hostinfo_by_taskname.empty?=#{@hostinfo_by_taskname.empty?} @hostinfo_by_taskname=#{@hostinfo_by_taskname.inspect} @task_queue.empty?=#{@task_queue.empty?} @task_queue=\n"+@task_queue.inspect_q
         raise RuntimeError,m
       end
       #Log.debug "#{self.class}#send_task_to_idle_core end time=#{Time.now-tm}"
@@ -359,7 +360,9 @@ module Pwrake
     end
 
     def task_end(tw,host_info)
-      if host_info && host_info.idle(tw.n_used_cores(host_info))
+      return if host_info.nil?
+      host_info.idle(tw.n_used_cores)
+      if host_info.retired?
         # all retired
         Log.warn("retired host:#{host_info.name} because all core retired")
         drop_host(host_info)

data/lib/pwrake/option/host_map.rb

@@ -21,10 +21,15 @@ module Pwrake
       @total_fail = 0
       @count_task = 0
       @ipaddr = []
+      begin
+        @ipaddr << IPSocket.getaddress(@name)
+      rescue
+      end
     end
 
     attr_reader :name, :ncore, :weight, :group, :id, :steal_flag
     attr_reader :ipaddr
+    attr_reader :continuous_fail
     attr_accessor :idle_cores
 
     def local?
@@ -33,7 +38,6 @@ module Pwrake
     end
 
     def set_ncore(n)
-      @retire = 0
       @busy_cores = 0
       @ncore = @idle_cores = n
     end
@@ -45,19 +49,18 @@ module Pwrake
     def idle(n)
       @busy_cores -= n
       @idle_cores += n
-      @idle_cores -= @retire
-      @retire = 0
-      @idle_cores + @busy_cores < 1 # all retired
     end
 
     def busy(n)
       @busy_cores += n
       @idle_cores -= n
-      @idle_cores + @busy_cores < 1 # all retired
     end
 
-    def decrease(n)
+    def retire(n)
       @idle_cores -= n
+    end
+
+    def retired?
       @idle_cores + @busy_cores < 1 # all retired
     end
 
@@ -68,12 +71,7 @@ module Pwrake
       t
     end
 
-    def retire(n)
-      @retire += n
-      Log.debug "retire n=#{n}, host=#{@name}"
-    end
-
-    def task_result(result)
+    def count_result(result)
       @count_task += 1
       case result
       when "end"
@@ -87,20 +85,8 @@ module Pwrake
       @continuous_fail
     end
 
-    def check_cores(use_cores)
-      unless (1-@ncore..@ncore) === use_cores
-        m = "use_cores=#{use_cores} is invalid for @ncore=#{@ncore}"
-        Log.fatal m
-        raise RuntimeError,m
-      end
-      if use_cores < 1
-        use_cores += @ncore
-      end
-      use_cores
-    end
-
     def accept_core(use_cores)
-      check_cores(use_cores) <= @idle_cores
+      use_cores <= @idle_cores
     end
   end
 

data/lib/pwrake/option/option.rb

@@ -166,8 +166,10 @@ module Pwrake
         'REPORT_IMAGE',
         'FAILED_TARGET', # rename(default), delete, leave
         'FAILURE_TERMINATION', # wait, kill, continue
-        'QUEUE_PRIORITY', # RANK(default), FIFO, LIFO, DFS
+        'QUEUE_PRIORITY', # LIHR(default), FIFO, LIFO, RANK
         'NOACTION_QUEUE_PRIORITY', # FIFO(default), LIFO, RAND
+        'DISABLE_RANK_PRIORITY',
+        ['RESERVE_NODE','RESERVE_HOST'],
         'GRAPH_PARTITION',
         'PLOT_PARTITION',
 
@@ -225,6 +227,7 @@ module Pwrake
         ['SHELL_START_INTERVAL', proc{|v| (v || 0.012).to_f}],
         ['HEARTBEAT', proc{|v| v && v.to_i}],
         ['RETRY', proc{|v| (v || 1).to_i}],
+        ['HOST_FAILURE', 'HOST_FAIL', proc{|v| (v || 2).to_i}],
         ['MASTER_HOSTNAME', proc{|v| (v || Socket.gethostname).chomp}],
         ['WORK_DIR', proc{|v|
            v ||= '%CWD_RELATIVE_TO_HOME'
@@ -286,9 +289,9 @@ module Pwrake
 
     def parse_opt(s)
       case s
-      when /^(false|nil|off)$/i
+      when /^(false|nil|off|n|no)$/i
         false
-      when /^(true|on)$/i
+      when /^(true|on|y|yes)$/i
         true
       when $stdout
         "stdout"

data/lib/pwrake/option/option_default_filesystem.rb

@@ -30,7 +30,7 @@ module Pwrake
         :shared_directory => "SharedDirectory"
       }
       @filesystem  = "default"
-      @queue_class = "TaskQueue"
+      @queue_class = "NonLocalityQueue"
     end
 
     def max_postprocess_pool

data/lib/pwrake/option/option_gfarm2fs.rb

@@ -50,7 +50,7 @@ module Pwrake
         pwrake/worker/worker_main
       ]
       if self['DISABLE_AFFINITY']
-        @queue_class = "TaskQueue"
+        @queue_class = "NonLocalityQueue"
       else
         @queue_class = "LocalityAwareQueue"
       end

data/lib/pwrake/queue/locality_aware_queue.rb

@@ -1,13 +1,18 @@
 module Pwrake
 
-  class LocalityAwareQueue < TaskQueue
+  class LocalityAwareQueue
+
+    def initialize(hostinfo_by_id, array_class, median_core, group_map=nil)
+      @hostinfo_by_id = hostinfo_by_id
+      @array_class = array_class
+      @median_core = median_core
 
-    def init_queue(group_map=nil)
       # group_map = {gid1=>[hid1,hid2,...], ...}
-      @size_q = 0
+      @total_core = 0
       @q = {}
-      @hostinfo_by_id.each do |id,h|
-        @q[id] = @array_class.new(h.ncore)
+      @hostinfo_by_id.each do |id,host_info|
+        @total_core += c = host_info.ncore
+        @q[id] = @array_class.new(c)
       end
       @q_group = {}
       group_map ||= {1=>@hostinfo_by_id.map{|id,h| id}}
@@ -18,15 +23,25 @@ module Pwrake
         a = [q1,q2]
         ary.each{|hid| @q_group[hid] = a}
       end
-      @q_remote = @array_class.new(0)
+
+      @q_remote = @array_class.new(@total_core)
+      @q_all = @array_class.new(@total_core)
+
       @disable_steal = Rake.application.pwrake_options['DISABLE_STEAL']
+      Log.debug "#{self.class}: @disable_steal=#{@disable_steal.inspect}"
+      @disable_rank = Rake.application.pwrake_options['DISABLE_RANK_PRIORITY']
+      Log.debug "#{self.class}: @disable_rank=#{@disable_rank.inspect}"
+
+      @turns = @disable_steal ? [0] : [0,1]
       @last_enq_time = Time.now
-      @n_turn = @disable_steal ? 1 : 2
     end
 
+    attr_reader :turns
+
     def enq_impl(t)
       hints = t && t.suggest_location
       Log.debug "enq #{t.name} hints=#{hints.inspect}"
+      @q_all.push(t)
       if hints.nil? || hints.empty?
         @q_remote.push(t)
       else
@@ -47,9 +62,7 @@ module Pwrake
             end
           end
         end
-        if q_success
-          @size_q += 1
-        else
+        unless q_success
           @q_remote.push(t)
         end
       end
@@ -58,88 +71,71 @@ module Pwrake
 
     def turn_empty?(turn)
       case turn
-      when 0
+      when 0,2
         empty?
-      when 1
-        @size_q == 0
+      when 1,3
+        @q_all.size == @q_remote.size
       end
     end
 
+    def deq_start
+      @rank = @disable_rank ? 0 : @q_all.find_rank(@median_core)
+    end
+
     def deq_impl(host_info, turn)
-      host = host_info.name
       case turn
       when 0
-        if t = @q_no_action.shift
-          Log.debug "deq_no_action task=#{t&&t.name} host=#{host}"
-          return t
-        elsif t = deq_locate(host_info,host_info)
-          Log.debug "deq_locate task=#{t&&t.name} host=#{host}"
-          return t
-        elsif t = @q_remote.shift(host_info)
-          Log.debug "deq_remote task=#{t&&t.name}"
-          return t
-        else
-          nil
-        end
+        deq_local(host_info) ||
+        deq_remote(host_info)
       when 1
-        if t = deq_steal(host_info)
-          Log.debug "deq_steal task=#{t&&t.name} host=#{host}"
-          return t
-        else
-          nil
-        end
+        deq_steal(host_info)
       end
     end
 
-    def deq_locate(q_host,run_host)
-      q = @q[q_host.id]
+    def deq_local(run_host)
+      q = @q[run_host.id]
       if q && !q.empty?
-        t = q.shift(run_host)
+        t = q.shift(run_host,@rank)
         if t
-          t.assigned.each do |h|
-            if q_h = @q[h]
-              q_h.delete(t)
-            end
-          end
-          @size_q -= 1
+          q_delete_assigned_to(t)
+          @q_all.delete(t)
+          Log.debug "deq_local task=#{t&&t.name} host=#{run_host.name} req_rank=#{@rank}"
+          return t
         end
+      end
+      nil
+    end
+
+    def deq_remote(host_info)
+      if t = @q_remote.shift(host_info,@rank)
+        @q_all.delete(t)
+        Log.debug "deq_remote task=#{t&&t.name} host=#{host_info.name} req_rank=#{@rank}"
         return t
-      else
-        nil
       end
+      nil
     end
 
-    def deq_steal(host_info)
-      # select a task based on many and close
-      max_host = nil
-      max_num  = 0
-      @q_group[host_info.id].each do |qg|
-        qg.each do |h,a|
-          if !a.empty? # && h!=host_info.id
-            d = a.size
-            if d > max_num
-              max_host = h
-              max_num  = d
-            end
-          end
-        end
-        if max_num > 0
-          max_info = @hostinfo_by_id[max_host]
-          #Log.debug "deq_steal max_host=#{max_info.name} max_num=#{max_num}"
-          t = host_info.steal_phase{|h| deq_locate(max_info,h)}
-          #Log.debug "deq_steal task=#{t.inspect}"
-          if t
-            Log.debug "deq_steal max_host=#{max_info.name} max_num=#{max_num}"
-            return t
-          end
-        end
+    def deq_steal(run_host)
+      if t = @q_all.shift(run_host,@rank)
+        q_delete_assigned_to(t)
+        @q_remote.delete(t)
+        Log.debug "deq_steal task=#{t&&t.name} host=#{run_host.name} req_rank=#{@rank}"
+        return t
       end
       nil
     end
 
+    def q_delete_assigned_to(t)
+      t.assigned.each do |h|
+        if q_h = @q[h]
+          q_h.delete(t)
+        end
+      end
+    end
+
     def inspect_q
-      s = _qstr("noaction",@q_no_action)
-      if @size_q == 0
+      s = ""
+      if @q_all.size == @q_remote.size
         n = @q.size
       else
         n = 0
@@ -147,35 +143,33 @@ module Pwrake
           if q.size > 0
             hinfo = @hostinfo_by_id[h]
             if hinfo
-              s << _qstr(hinfo.name,q)
+              s << TaskQueue._qstr(hinfo.name,q)
             else
-              s << _qstr("(#{hinfo.inspect})",q)
+              s << TaskQueue._qstr("(#{hinfo.inspect})",q)
             end
           else
             n += 1
           end
         end
       end
-      s << _qstr("local*#{n}",[]) if n > 0
-      s << _qstr("remote",@q_remote)
-      s << _qstr("reserved",@q_reserved)
-      s << "@size_q=#{@size_q}"
+      s << TaskQueue._qstr("local*#{n}",[]) if n > 0
+      s << TaskQueue._qstr("remote",@q_remote)
+      s << TaskQueue._qstr("all",@q_all)
       s
     end
 
+    def size
+      @q_all.size
+    end
+
     def clear
-      @q_no_action.clear
-      @q_reserved.clear
       @q.each{|h,q| q.clear}
-      @size_q = 0
       @q_remote.clear
+      @q_all.clear
     end
 
     def empty?
-      @size_q == 0 &&
-        @q_no_action.empty? &&
-        @q_reserved.empty? &&
-        @q_remote.empty?
+      @q_all.empty?
     end
 
     def drop_host(host_info)
@@ -183,7 +177,7 @@ module Pwrake
       if q_drop = @q.delete(hid)
         n_move = 0
         q_size = q_drop.size
-        while t = q_drop.shift
+        while t = q_drop.shift(host_info,@rank)
           assigned_other = false
           t.assigned.each do |h|
             if h != hid && @q[h]
@@ -192,7 +186,6 @@ module Pwrake
             end
           end
           if !assigned_other
-            @size_q -= 1
             @q_remote.push(t)
             n_move += 1
           end