pwrake 2.2.2 → 2.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 700155f122d51b4ae814349492054d336daa6dad
-  data.tar.gz: 8bd23134b9c79784b95670440c0e823f139b41e5
+  metadata.gz: 1c3bfd1d1e40600085dbe880e57230260fa81b4f
+  data.tar.gz: 84a151198f3d200792e84fbe17715092b33abc15
 SHA512:
-  metadata.gz: ca62080f1e262829e1da0623c474d0415bdda728498bea11a25f8b17d2a3d60ecf8ebb701ab4eb160f8a894840e402477f2daf87335103aefa42bf89a021d1b5
-  data.tar.gz: ad190406c5a890f78e3691cac5bed219e537ac911617fe4cb1d68ce6d6cbd65c9963e0fa5bd8e328aad864e6303bef893bd5877afefcd2e85402d6864759f8ee
+  metadata.gz: 1db1ccc6ce750393100155219a6b86c012c9126b6dfe77ed7bc2f8e4d8ec7baa113741b51a43765e16513554cae8a077bd005706290b2f139d0e960a21d0d4c2
+  data.tar.gz: c2d49f2af43611bda75f92f74726bd5c7045ea2d85604b480231b5d0e5729d2d90b60f9c58b89aa12ea1e513eb8f3fa4c38a87b1beab485f266a0c759271383e

data/README.md

@@ -170,6 +170,7 @@ Properties (The leftmost item is default):
 
     ncore=integer     - The number of cores used by this task.
     exclusive=no|yes  - Exclusively execute this task in a single node.
+    reserve=no|yes    - Gives higher priority to this task if ncore>1. (reserve a host)
     allow=hostname    - Allow this host to execute this task. (accepts wild card)
     deny=hostname     - Deny this host to execute this task. (accepts wild card)
     order=deny,allow|allow,deny - The order of evaluation.
@@ -201,12 +202,12 @@ Properties (The leftmost item is default):
 
 ## Current version
 
-* Pwrake version 2.2.0
+* Pwrake version 2.2.3
 
 ## Tested Platform
 
 
-* Ruby 2.4.0
+* Ruby 2.4.1
 * Rake 12.0.0
 * CentOS 7.3
 

data/lib/pwrake/branch/communicator_set.rb

@@ -95,11 +95,15 @@ class CommunicatorSet
   end
 
   def kill(sig)
+    @selector.clear
     NBIO::Handler.kill(handler_set,sig)
+    @selector.run
   end
 
   def exit
+    @selector.clear
     NBIO::Handler.exit(handler_set)
+    @selector.run
   end
 
 end

data/lib/pwrake/master/master.rb

@@ -1,4 +1,5 @@
 require "fileutils"
+require "timeout"
 require "pwrake/logger"
 require "pwrake/nbio"
 require "pwrake/option/option"
@@ -22,6 +23,7 @@ module Pwrake
       @option = Option.new
       Log.set_logger(@option)
       TaskWrapper.init_task_logger(@option)
+      at_exit{TaskWrapper.close_task_logger}
       # moved from Option#init
       @option.put_log
       if @option['LOG_DIR'] && @option['GC_LOG_FILE']
@@ -63,31 +65,33 @@ module Pwrake
     end
 
     def signal_trap(sig)
-      case @killed
-      when 0
-        # log writing failed. can't be called from trap context
-        $stderr.puts "\nSignal trapped. (sig=#{sig} pid=#{Process.pid} ##{@killed})"
-        if Rake.application.options.debug
-          $stderr.puts "in master thread #{Thread.current}:"
-          $stderr.puts caller
-          if @thread
-            $stderr.puts "in branch thread #{@thread}:"
-            $stderr.puts @thread.backtrace.join("\n")
-          end
+      $stderr.puts "\nSignal trapped. (sig=#{sig} pid=#{Process.pid})"
+      if Rake.application.options.debug
+        $stderr.puts "in master thread #{Thread.current}:"
+        $stderr.puts caller
+        if @thread
+          $stderr.puts "in branch thread #{@thread}:"
+          $stderr.puts @thread.backtrace.join("\n")
         end
-        $stderr.puts "Exiting..."
-        @no_more_run = true
-        @failed = true
-        NBIO::Handler.kill(@hdl_set,sig)
-        # @selector.run : not required here
-      when 1
-        $stderr.puts "\nOnce more Ctrl-C (SIGINT) for exit."
-      when 2
-        @thread.kill if @thread
-      else
-        Kernel.exit(false) # must wait for nomral exit
       end
-      @killed += 1
+      kill_end(sig)
+    end
+
+    def kill_end(sig)
+      # log writing failed. can't be called from trap context
+      $stderr.puts "Exiting..."
+      @no_more_run = true
+      @failed = true
+      @selector.clear
+      NBIO::Handler.kill(@hdl_set,sig)
+      begin
+        Timeout.timeout(30) do
+          @selector.run
+          @thread.join if @thread
+        end
+      rescue
+      end
+      Kernel.exit(false)
     end
 
     def setup_branches
@@ -167,12 +171,14 @@ module Pwrake
       host_info = @hostinfo_by_id[hid.to_i]
       if host_info && host_info.decrease(1)
         # all retired
-        m = "retired: host #{host_info.name}"
-        Log.warn(m)
-        $stderr.puts(m)
-        drop_host(host_info) # delete from hostinfo_by_id
-        if @hostinfo_by_id.empty?
-          raise RuntimeError,"no worker host"
+        if !@exited
+          m = "retired: host #{host_info.name}"
+          Log.warn(m)
+          $stderr.puts(m)
+          drop_host(host_info) # delete from hostinfo_by_id
+          if @hostinfo_by_id.empty?
+              raise RuntimeError,"no worker host"
+          end
         end
       end
     end
@@ -208,7 +214,6 @@ module Pwrake
 
       setup_postprocess1
       @branch_setup_thread.join
-      @killed = 0
       [:TERM,:INT].each do |sig|
         Signal.trap(sig) do
           signal_trap(sig)
@@ -374,12 +379,18 @@ module Pwrake
     end
 
     def ending?
-      #Log.debug " @no_more_run=#{@no_more_run.inspect}"
-      #Log.debug " @task_queue.empty?=#{@task_queue.empty?}"
-      #Log.debug " @hostinfo_by_id.empty?=#{@hostinfo_by_id.empty?}"
-      #Log.debug " @hostinfo_by_taskname.keys=#{@hostinfo_by_taskname.keys.inspect}"
-      (@no_more_run || @task_queue.empty? || @hostinfo_by_id.empty?) &&
+      if @no_more_run || @task_queue.empty? || @hostinfo_by_id.empty?
+        case @hostinfo_by_taskname.size
+        when 1..2
+          Log.debug " @no_more_run=#{@no_more_run.inspect}" if @no_more_run
+          Log.debug " @task_queue.empty?=#{@task_queue.empty?}" if @task_queue.empty?
+          Log.debug " @hostinfo_by_id.empty?=#{@hostinfo_by_id.empty?}" if @hostinfo_by_id.empty?
+          Log.debug " @hostinfo_by_taskname.keys=#{@hostinfo_by_taskname.keys.inspect}"
+        end
         @hostinfo_by_taskname.empty?
+      else
+        false
+      end
     end
 
     def handle_failed_target(name)
@@ -427,16 +438,25 @@ module Pwrake
       @branch_setup_thread.join
       # continues running fibers
       Log.debug "Master#finish @selector.run begin"
-      @selector.run(60)
+      begin
+        Timeout.timeout(30){@selector.run}
+      rescue
+      end
       Log.debug "Master#finish @selector.run end"
       if !@exited
         @exited = true
         Log.debug "Master#finish Handler.exit begin"
+        @selector.clear
         NBIO::Handler.exit(@hdl_set)
-        @selector.run(60)
+        begin
+          Timeout.timeout(30) do
+            @selector.run
+            @thread.join if @thread
+          end
+        rescue
+        end
         Log.debug "Master#finish Handler.exit end"
       end
-      TaskWrapper.close_task_logger
       Log.debug "Master#finish end"
       @failed
     end

data/lib/pwrake/master/master_application.rb

@@ -31,21 +31,25 @@ module Pwrake
           top_level
           Log.debug "main: #{Time.now-t} sec"
           t = Time.now
+          @failed = @master.finish
+          Log.debug "finish: #{Time.now-t} sec"
+        rescue SystemExit => e
+          @failed = true
         rescue Exception => e
           # Exit with error message
           m = Log.bt(e)
           if @master.thread
             m += "\nIn branch thread #{@master.thread}:\n "
-            m += @master.thread.backtrace.join("\n ")
+            if @master.thread.backtrace
+              m += @master.thread.backtrace.join("\n ")
+            end
           end
           Log.fatal m
           $stderr.puts m
-          @master.signal_trap("INT")
-        ensure
-          @failed = @master.finish
-          Log.debug "finish: #{Time.now-t} sec"
-          Log.info "pwrake elapsed time: #{Time.now-START_TIME} sec"
+          @master.kill_end("INT")
+          @failed = true
         end
+        Log.info "pwrake elapsed time: #{Time.now-START_TIME} sec"
         Kernel.exit(false) if @failed
       end
     end
@@ -183,6 +187,7 @@ module Pwrake
        ],
        ['--clear-gfarm2fs',"[Pw] Clear gfarm2fs mountpoints left after failure.",
          lambda { |value|
+           require "pwrake/option/option_gfarm2fs"
            Option.new.clear_gfarm2fs
            exit
          }

data/lib/pwrake/nbio.rb

@@ -39,6 +39,11 @@ module NBIO
       @reader.empty? && @writer.empty?
     end
 
+    def clear
+      @reader.clear
+      @writer.clear
+    end
+
     def halt
       @running = false
       @writer.each_value{|w| w.halt}
@@ -101,12 +106,22 @@ module NBIO
 
     def init_heartbeat
       t = Time.now
+      @hb_check_time = t
       @hb_time = {}
       @reader.each_key{|io| @hb_time[io] = t}
     end
 
     def check_heartbeat(ios,timeout)
       t = Time.now
+      if t - @hb_check_time < 3
+        if ios
+          ios.each do |io|
+            @hb_time[io] = t
+          end
+        end
+        return
+      end
+      @hb_check_time = t
       rds = @reader.dup
       if ios
         ios.each do |io|
@@ -119,7 +134,8 @@ module NBIO
           tdif = t - @hb_time[io]
           if tdif > timeout
             m = "Heartbeat Timeout: no response during #{tdif}s "+
-              "> timeout #{timeout}s from host=#{get_host(io)}"
+              "> timeout #{timeout}s from host=#{get_host(io)} " +
+              "@hb_time[io=#{io.inspect}]=#{@hb_time[io].strftime('%FT%T.%6N')}"
             hdl.error(TimeoutError.new(m))
           end
         end
@@ -138,6 +154,7 @@ module NBIO
       @pool = []
     end
     attr_reader :io
+    attr_accessor :host
 
     # call from Selector
     def call
@@ -237,7 +254,7 @@ module NBIO
       @chunk_size = 8192
     end
     attr_reader :io, :waiter
-    attr_accessor :check_timeout
+    attr_accessor :check_timeout, :host
 
     # call from Selector#run
     def call
@@ -455,6 +472,8 @@ module NBIO
       @writer = writer
       @reader = reader
       @host = hostname
+      @reader.host = @host
+      @writer.host = @host
       @exited = false
     end
     attr_reader :reader, :writer, :host

data/lib/pwrake/option/host_map.rb

@@ -87,25 +87,20 @@ module Pwrake
       @continuous_fail
     end
 
-    def accept_core(task_name, use_cores)
-      if @reserved_task
-        if @reserved_task == task_name
-          if use_cores <= @idle_cores
-            Log.info "use reserved: #{@name} for #{task_name} (#{use_cores} cores)"
-            @reserved_task = nil
-            return :ok
-          end
-        end
-      else
-        if use_cores <= @idle_cores
-          return :ok
-        elsif use_cores > 1
-          @reserved_task = task_name
-          Log.info "reserve host: #{@name} for #{task_name} (#{use_cores} cores)"
-          return :reserve
-        end
+    def check_cores(use_cores)
+      unless (1-@ncore..@ncore) === use_cores
+        m = "use_cores=#{use_cores} is invalid for @ncore=#{@ncore}"
+        Log.fatal m
+        raise RuntimeError,m
       end
-      :busy
+      if use_cores < 1
+        use_cores += @ncore
+      end
+      use_cores
+    end
+
+    def accept_core(use_cores)
+      check_cores(use_cores) <= @idle_cores
     end
   end
 

data/lib/pwrake/queue/locality_aware_queue.rb

@@ -37,7 +37,12 @@ module Pwrake
         if !kv.empty?
           kv.each_key do |id|
             t.assigned.push(id)
-            @q[id].push(t)
+            q = @q[id]
+            if q
+              q.push(t)
+            else
+              Log.warn("@q[id]=#{q.inspect} : @q.keys=#{@q.keys.inspect} id=#{id.inspect}")
+            end
           end
           @size_q += 1
         else
@@ -116,10 +121,13 @@ module Pwrake
         end
         if max_num > 0
           max_info = @hostinfo_by_id[max_host]
-          Log.debug "deq_steal max_host=#{max_info.name} max_num=#{max_num}"
+          #Log.debug "deq_steal max_host=#{max_info.name} max_num=#{max_num}"
           t = host_info.steal_phase{|h| deq_locate(max_info,h)}
           #Log.debug "deq_steal task=#{t.inspect}"
-          return t if t
+          if t
+            Log.debug "deq_steal max_host=#{max_info.name} max_num=#{max_num}"
+            return t
+          end
         end
       end
       nil
@@ -152,13 +160,16 @@ module Pwrake
 
     def clear
       @q_no_action.clear
+      @q_reserved.clear
       @q.each{|h,q| q.clear}
+      @size_q = 0
       @q_remote.clear
     end
 
     def empty?
       @size_q == 0 &&
         @q_no_action.empty? &&
+        @q_reserved.empty? &&
         @q_remote.empty?
     end
 

data/lib/pwrake/queue/task_queue.rb

@@ -8,6 +8,7 @@ module Pwrake
     def initialize(hostinfo_by_id, group_map=nil)
       @enable_steal = true
       @q_no_action = NoActionQueue.new
+      @q_reserved = Hash.new
 
       @hostinfo_by_id = hostinfo_by_id
 
@@ -84,22 +85,29 @@ module Pwrake
         count = 0
         @hostinfo_by_id.each_value do |host_info|
           #Log.debug "TaskQueue#deq_turn host_info=#{host_info.name}"
-          if (n = host_info.idle_cores) && n > 0
-            if turn_empty?(turn)
-              return queued
+          if turn_empty?(turn)
+            return queued
+          elsif (n_idle = host_info.idle_cores) && n_idle > 0
+            if tw = @q_reserved[host_info]
+              n_use = tw.n_used_cores(host_info)
+              if n_idle < n_use
+                next
+              end
+              @q_reserved.delete(host_info)
+              Log.debug "deq_reserve: #{tw.name} n_use_cores=#{n_use}"
             elsif tw = deq_impl(host_info,turn)
-              n_task_cores = tw.n_used_cores(host_info)
-              Log.debug "deq: #{tw.name} n_task_cores=#{n_task_cores}"
-              if host_info.idle_cores < n_task_cores
-                m = "task.n_used_cores=#{n_task_cores} must be "+
-                  "<= host_info.idle_cores=#{host_info.idle_cores}"
-                Log.fatal m
-                raise RuntimeError,m
-              else
-                yield(tw,host_info,n_task_cores)
-                count += 1
-                queued += 1
+              n_use = tw.n_used_cores(host_info)
+              if n_idle < n_use
+                @q_reserved[host_info] = tw
+                Log.debug "reserve host: #{host_info.name} for #{tw.name} (#{n_use} cores)"
+                next
               end
+              Log.debug "deq: #{tw.name} n_use_cores=#{n_use}"
+            end
+            if tw
+              yield(tw,host_info,n_use)
+              count += 1
+              queued += 1
             end
           end
         end
@@ -120,12 +128,14 @@ module Pwrake
 
     def clear
       @q_no_action.clear
+      @q_reserved.clear
       @q_input.clear
       @q_no_input.clear
     end
 
     def empty?
       @q_no_action.empty? &&
+        @q_reserved.empty? &&
         @q_input.empty? &&
         @q_no_input.empty?
     end

data/lib/pwrake/report/report.rb

@@ -59,11 +59,14 @@ EOL
 
       h = {}
       @elap_sum = 0
+      @elap_core_sum = 0
       @sh_table.each do |row|
         if host = row['host']
           h[host] = true
         end
-        @elap_sum += row['elap_time'].to_f
+        t = row['elap_time'].to_f
+        @elap_sum += t
+        @elap_core_sum += t * row['ncore'].to_f
       end
       @hosts = h.keys.sort
       @start_time = Time.parse(@sh_table[0]["start_time"])
@@ -207,18 +210,18 @@ EOL
       html = HTML_HEAD + "<body><h1>Pwrake Statistics</h1>\n"
       html << "<h2>Workflow</h2>\n"
       html << "<table>\n"
-      html << "<tr><th>log file</th><td>#{@base}</td><tr>\n"
+      html << "<tr><th>log directory</th><td>#{@base}</td><tr>\n"
       html << "<tr><th>ncore</th><td>#{@ncore}</td><tr>\n"
       html << "<tr><th>elapsed time</th><td>%.3f sec</td><tr>\n"%[@elap]
-      html << "<tr><th>cumulative process time</th><td>%.3f sec</td><tr>\n"%[@elap_sum]
-      html << "<tr><th>occupancy</th><td>%.3f %%</td><tr>\n"%[@elap_sum/@elap/@ncore*100]
+      html << "<tr><th>accumulated process time</th><td>%.3f sec</td><tr>\n"%[@elap_sum]
+      html << "<tr><th>occupancy</th><td>%.3f %%</td><tr>\n"%[@elap_core_sum/@elap/@ncore*100]
       html << "<tr><th>start time</th><td>#{@start_time}</td><tr>\n"
       html << "<tr><th>end time</th><td>#{@end_time}</td><tr>\n"
-      html << "</table>\n"
+      html << "</table><br/>\n"
       html << "<table>\n"
-      html << "<tr><th>hosts</th><tr>\n"
-      @hosts.each do |h|
-        html << "<tr><td>#{h}</td><tr>\n"
+      html << "<tr><th colspan=5>#{@hosts.size} hosts &times; #{@ncore.fdiv @hosts.size} cores/host</th><tr>\n"
+      @hosts.each_slice(5) do |a|
+        html << "<tr>"+a.map{|h|"<td>#{h}</td>"}.join("")+"<tr>\n"
       end
       html << "</table>\n"
       html << "<h2>Parallelism</h2>\n"

data/lib/pwrake/task/task_property.rb

@@ -3,7 +3,7 @@ module Pwrake
   class TaskProperty
 
     attr_reader :ncore, :exclusive, :allow, :deny, :order_allow_deny,
-      :retry, :disable_steal
+      :retry, :disable_steal, :reserve
     attr_accessor :subflow
 
     def parse_description(description)
@@ -18,6 +18,11 @@ module Pwrake
           @exclusive = true
         end
       end
+      if /\breserve[=:]\s*(\S+)/i =~ description
+        if /^(y|t)/i =~ $1
+          @reserve = true
+        end
+      end
       if /\ballow[=:]\s*(\S+)/i =~ description
         @allow = $1
       end
@@ -37,25 +42,24 @@ module Pwrake
           @disable_steal = true
         end
       end
+      @use_cores = nil
     end
 
     def merge(prop)
       @ncore = prop.ncore if prop.ncore
       @exclusive = prop.exclusive if prop.exclusive
+      @reserve = prop.reserve if prop.reserve
       @allow = prop.allow if prop.allow
       @deny = prop.deny if prop.deny
       @order_allow_deny = prop.order_allow_deny if prop.order_allow_deny
       @retry = prop.retry if prop.retry
       @disable_steal = prop.disable_steal if prop.disable_steal
       @subflow = prop.subflow if prop.subflow
+      @use_cores = nil
     end
 
-    def use_cores(host_info)
-      nc = (@exclusive) ? 0 : (@ncore || 1)
-      if nc < 1
-        nc += host_info.ncore
-      end
-      return nc
+    def use_cores
+      @use_cores ||= (@exclusive) ? 0 : (@ncore || 1)
     end
 
     def accept_host(host_info)
@@ -84,31 +88,17 @@ module Pwrake
     end
 
     def n_used_cores(host_info=nil)
-      nc_node = host_info && host_info.ncore
-      if @ncore.nil?
+      n = use_cores
+      if n == 1
         return 1
-      elsif @ncore > 0
-        if nc_node && @ncore > nc_node
-          m = "ncore=#{@ncore} must be <= nc_node=#{nc_node}"
-          Log.fatal m
-          raise RuntimeError,m
-        end
-        return @ncore
-      else
-        if nc_node.nil?
-          m = "host_info.ncore is not set"
-          Log.fatal m
-          raise RuntimeError,m
-        end
-        n = @ncore + nc_node
-        if n > 0
-          return n
-        else
-          m = "ncore+nc_node=#{n} must be > 0"
-          Log.fatal m
-          raise RuntimeError,m
-        end
+      elsif host_info
+        return host_info.check_cores(n)
+      elsif n < 1
+        m = "invalid for use_cores=#{n}"
+        Log.fatal m
+        raise RuntimeError,m
       end
+      return n
     end
 
   end

data/lib/pwrake/task/task_wrapper.rb

@@ -343,20 +343,16 @@ module Pwrake
     end
 
     def acceptable_for(host_info)
+      unless host_info
+        return true
+      end
       unless @property.accept_host(host_info)
         return false
       end
-      use_cores = @property.use_cores(host_info)
-      if @reserved_host.nil? || @reserved_host == host_info
-        case host_info.accept_core(name,use_cores)
-        when :ok
-          @reserved_host = nil
-          return true
-        when :reserve
-          @reserved_host = host_info
-        end
+      if @property.reserve
+        return true
       end
-      false
+      host_info.accept_core(@property.use_cores)
     end
 
     def tried_host?(host_info)

data/lib/pwrake/version.rb

@@ -1,3 +1,3 @@
 module Pwrake
-  VERSION = "2.2.2"
+  VERSION = "2.2.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pwrake
 version: !ruby/object:Gem::Version
-  version: 2.2.2
+  version: 2.2.3
 platform: ruby
 authors:
 - Masahiro TANAKA
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-07-14 00:00:00.000000000 Z
+date: 2017-07-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel