pwrake 2.0.1 → 2.1.0

data/lib/pwrake/branch/shell_profiler.rb

@@ -1,3 +1,5 @@
+require "csv"
+
 module Pwrake
 
   class ShellProfiler

data/lib/pwrake/gfarm/gfarm_postprocess.rb

@@ -2,11 +2,11 @@ module Pwrake
 
   class GfarmPostprocess
 
-    def initialize(runner)
+    def initialize(selector)
       @io = IO.popen('gfwhere-pipe','r+')
       @io.sync = true
-      @hdl = Handler.new(runner,@io,@io)
-      @chan = Channel.new(@hdl)
+      @reader = NBIO::Reader.new(selector,@io)
+      @writer = NBIO::Writer.new(selector,@io)
     end
 
     def run(task_wrap)
@@ -15,13 +15,12 @@ module Pwrake
       end
       filename = task_wrap.name
       begin
-        @hdl.iow.puts(filename)
-        @hdl.iow.flush
+        @writer.put_line(filename)
       rescue Errno::EPIPE
         Log.warn "GfarmPostprocess#run: Errno::EPIPE for #{filename}"
         return []
       end
-      s = @chan.get_line
+      s = @reader.get_line
       if s.nil?
         raise "gfwhere: unexpected end"
       end
@@ -29,7 +28,7 @@ module Pwrake
       if s != filename
         raise "gfwhere: file=#{filename}, result=#{s}"
       end
-      while s = @chan.get_line
+      while s = @reader.get_line
         s.chomp!
         case s
         when /^gfarm:\/\//
@@ -47,6 +46,8 @@ module Pwrake
     end
 
     def close
+      @writer.halt
+      @reader.halt
       @io.close
     end
   end

data/lib/pwrake/logger.rb

@@ -1,3 +1,5 @@
+require "logger"
+
 module Pwrake
 
   module Log
@@ -8,5 +10,8 @@ module Pwrake
       Rake.application.logger.send(meth_id,*args)
     end
 
+    def bt(e)
+      "#{e.class}: #{e.message}\n "+(e.backtrace||[]).join("\n ")
+    end
   end
 end

data/lib/pwrake/master/master.rb

@@ -1,15 +1,24 @@
+require "fileutils"
+require "pwrake/logger"
+require "pwrake/nbio"
+require "pwrake/option/option"
+require "pwrake/task/task_wrapper"
+require "pwrake/queue/task_queue"
+require "pwrake/master/fiber_pool"
+
 module Pwrake
 
   class Master
 
     def initialize
-      @runner = Runner.new
-      @hostid_by_taskname = {}
+      @selector = NBIO::Selector.new
+      @hostinfo_by_taskname = {}
       @option = Option.new
-      @hdl_set = HandlerSet.new
+      @hdl_set = []
       @channel_by_hostid = {}
       @channels = []
-      @hosts = {}
+      @hostinfo_by_id = {}
+      @n_retry = @option["RETRY"]
       init_logger
     end
 
@@ -43,35 +52,31 @@ module Pwrake
     end
 
     def setup_branch_handler(sub_host)
+      ior,w0 = IO.pipe
+      r2,iow = IO.pipe
       if sub_host == "localhost" && /^(n|f)/i !~ ENV['T']
-        hdl = Handler.new(@runner) do |w0,w1,r2|
-          @thread = Thread.new(r2,w0,@option) do |r,w,o|
-            Rake.application.run_branch_in_thread(r,w,o)
-          end
+        @thread = Thread.new(r2,w0,@option) do |r,w,o|
+          Rake.application.run_branch_in_thread(r,w,o)
         end
       else
-        hdl = Handler.new(@runner) do |w0,w1,r2|
-          dir = File.absolute_path(File.dirname($PROGRAM_NAME))
-          #args = Shellwords.shelljoin(@args)
-          cmd = "ssh -x -T -q #{sub_host} '" +
-            "cd \"#{Dir.pwd}\";"+
-            "PATH=#{dir}:${PATH} exec pwrake_branch'"
-          Log.debug("BranchCommunicator cmd=#{cmd}")
-          #$stderr.puts "BranchCommunicator cmd=#{cmd}"
-          spawn(cmd,:pgroup=>true,:out=>w0,:err=>w1,:in=>r2)
-          w0.close
-          w1.close
-          r2.close
-        end
-        Marshal.dump(@option,hdl.iow)
-        hdl.iow.flush
-        s = hdl.ior.gets
+        dir = File.absolute_path(File.dirname($PROGRAM_NAME))
+        cmd = "ssh -x -T -q #{sub_host} '" +
+          "cd \"#{Dir.pwd}\";"+
+          "PATH=#{dir}:${PATH} exec pwrake_branch'"
+        Log.debug("BranchCommunicator cmd=#{cmd}")
+        spawn(cmd,:pgroup=>true,:out=>w0,:in=>r2)
+        w0.close
+        r2.close
+        Marshal.dump(@option,iow)
+        iow.flush
+        s = ior.gets
         if !s or s.chomp != "pwrake_branch start"
           raise RuntimeError,"pwrake_branch start failed: receive #{s.inspect}"
         end
       end
-      hdl.host = sub_host
-      return hdl
+      rd = NBIO::Reader.new(@selector,ior)
+      wt = NBIO::Writer.new(@selector,iow)
+      return NBIO::Handler.new(rd,wt,sub_host)
     end
 
     def signal_trap(sig)
@@ -89,9 +94,12 @@ module Pwrake
         $stderr.puts "Exiting..."
         @no_more_run = true
         @failed = true
-        @hdl_set.kill(sig)
+        NBIO::Handler.kill(@hdl_set,sig)
+        # @selector.run : not required here
       when 1
         $stderr.puts "\nOnce more Ctrl-C (SIGINT) for exit."
+      when 2
+        @thread.kill if @thread
       else
         Kernel.exit(false) # must wait for nomral exit
       end
@@ -100,30 +108,28 @@ module Pwrake
 
     def setup_branches
       sum_ncore = 0
-
       @option.host_map.each do |sub_host, wk_hosts|
         @hdl_set << hdl = setup_branch_handler(sub_host)
-        @channels << chan = Channel.new(hdl)
-        chan.puts "host_list_begin"
+        Fiber.new do
+        hdl.put_line "host_list_begin"
         wk_hosts.each do |host_info|
           name = host_info.name
           ncore = host_info.ncore
           host_id = host_info.id
           Log.debug "connecting #{name} ncore=#{ncore} id=#{host_id}"
-          chan.puts "host:#{host_id} #{name} #{ncore}"
-          @channel_by_hostid[host_id] = chan
-          @hosts[host_id] = name
+          hdl.put_line "host:#{host_id} #{name} #{ncore}"
+          @channel_by_hostid[host_id] = hdl
+          @hostinfo_by_id[host_id] = host_info
         end
-        chan.puts "host_list_end"
-
-        while s = chan.gets
+        hdl.put_line "host_list_end"
+        while s = hdl.get_line
           case s
           when /^ncore:done$/
             break
           when /^ncore:(\d+):(\d+)$/
             id, ncore = $1.to_i, $2.to_i
             Log.debug "worker_id=#{id} ncore=#{ncore}"
-            @option.host_map.by_id[id].set_ncore(ncore)
+            @hostinfo_by_id[id].set_ncore(ncore)
             sum_ncore += ncore
           when /^exited$/
             raise RuntimeError,"Unexpected branch exit"
@@ -132,23 +138,39 @@ module Pwrake
             raise RuntimeError,"invalid return: #{msg}"
           end
         end
+        end.resume
       end
+      @selector.run
 
       Log.info "num_cores=#{sum_ncore}"
-      @hosts.each do |id,host|
-        Log.info "#{host} id=#{id} ncore=#{
-          @option.host_map.by_id[id].idle_cores}"
+      @hostinfo_by_id.each do |id,host|
+        if ncore = @hostinfo_by_id[id].idle_cores
+          Log.info "#{host} id=#{id} ncore=#{ncore}"
+        else
+          @hostinfo_by_id.delete(id)
+        end
+      end
+      if @hostinfo_by_id.empty?
+        raise RuntimeError,"no worker host"
       end
       queue_class = Pwrake.const_get(@option.queue_class)
-      @task_queue = queue_class.new(@option.host_map)
+      @task_queue = queue_class.new(@hostinfo_by_id)
 
       @branch_setup_thread = Thread.new do
-        @channels.each do |chan|
-          s = chan.gets
-          if /^branch_setup:done$/ !~ s
-            raise RuntimeError,"branch_setup failed"
+        #@channels.each do |chan|
+        create_fiber(@hdl_set) do |hdl|
+          while s = hdl.get_line
+            case s
+            when /^retire:(\d+)$/
+              retire($1.to_i)
+            when /^branch_setup:done$/
+              break
+            else
+              raise RuntimeError,"branch_setup failed: s=#{s.inspect}"
+            end
           end
         end
+        @selector.run
         @killed = 0
         [:TERM,:INT].each do |sig|
           Signal.trap(sig) do
@@ -159,6 +181,20 @@ module Pwrake
 
     end
 
+    def retire(hid)
+      host_info = @hostinfo_by_id[hid.to_i]
+      if host_info && host_info.decrease(1)
+        # all retired
+        m = "retired: host #{host_info.name}"
+        Log.warn(m)
+        $stderr.puts(m)
+        drop_host(host_info) # delete from hostinfo_by_id
+        if @hostinfo_by_id.empty?
+          raise RuntimeError,"no worker host"
+        end
+      end
+    end
+
     def create_fiber(channels,&blk)
       channels.each do |chan|
         fb = Fiber.new(&blk)
@@ -172,12 +208,12 @@ module Pwrake
 
       if @option['GRAPH_PARTITION']
         setup_postprocess0
-        @task_queue.deq_noaction_task do |tw,hid|
+        @task_queue.deq_noaction_task do |tw|
           tw.preprocess
           tw.status = "end"
           @post_pool.enq(tw)
         end
-        @runner.run
+        @selector.run
         @post_pool.finish
         Log.debug "@post_pool.finish"
 
@@ -189,43 +225,59 @@ module Pwrake
       @branch_setup_thread.join
       send_task_to_idle_core
       #
-      create_fiber(@channels) do |chan|
-        while s = chan.get_line
-          Log.debug "Master:recv #{s.inspect} from branch[#{chan.handler.host}]"
+      create_fiber(@hdl_set) do |hdl|
+        while s = hdl.get_line
+          Log.debug "Master:recv #{s.inspect} from branch[#{hdl.host}]"
           case s
           when /^task(\w+):(\d*):(.*)$/o
             status, shell_id, task_name = $1, $2.to_i, $3
             tw = Rake.application[task_name].wrapper
             tw.shell_id = shell_id
             tw.status = status
-            hid = @hostid_by_taskname[task_name]
-            @task_queue.task_end(tw,hid) # @idle_cores.increase(..
+            host_info = @hostinfo_by_taskname[task_name]
+            if host_info.nil?
+              m = "unknown hostid: task_name=#{task_name} s=#{s.inspect}"+
+                " @hostinfo_by_taskname.keys=#{@hostinfo_by_taskname.keys.inspect}"
+              Log.error(m)
+              $stderr.puts(m)
+            end
+            task_end(tw,host_info) # @idle_cores.increase(..
             # check failure
             if tw.status == "fail"
               $stderr.puts %[task "#{tw.name}" failed.]
-              if !tw.retry
-                if !@failed
-                  @failed = true
-                  case @option['FAILURE_TERMINATION']
-                  when 'kill'
-                    @hdl_set.kill("INT")
-                    @no_more_run = true
-                    $stderr.puts "... Kill running tasks."
-                  when 'continue'
-                    $stderr.puts "... Continue runable tasks."
-                  else # 'wait'
-                    @no_more_run = true
-                    $stderr.puts "... Wait for running tasks."
-                  end
+              if host_info
+                continuous_fail = host_info.task_result(tw.status)
+                Log.debug "task=#{tw.name} continuous_fail=#{continuous_fail}"
+                if continuous_fail > @n_retry && @hostinfo_by_id.size > 1
+                  # retire this host
+                  drop_host(host_info)
+                  Log.warn("retired host:#{host_info.name} due to continuous fail")
                 end
-                if tw.has_output_file? && File.exist?(tw.name)
-                  handle_failed_target(tw.name)
+              end
+              if tw.no_more_retry && !@failed
+                @failed = true
+                case @option['FAILURE_TERMINATION']
+                when 'kill'
+                  NBIO::Handler.kill(@hdl_set,"INT")
+                  @selector.run
+                  @no_more_run = true
+                  $stderr.puts "... Kill running tasks."
+                when 'continue'
+                  $stderr.puts "... Continue runable tasks."
+                else # 'wait'
+                  @no_more_run = true
+                  $stderr.puts "... Wait for running tasks."
                 end
               end
+              if tw.has_output_file? && File.exist?(tw.name)
+                handle_failed_target(tw.name)
+              end
             end
             # postprocess
             @post_pool.enq(tw) # must be after @no_more_run = true
             break if @finished
+          when /^retire:(\d+)$/
+            retire($1.to_i)
           when /^exited$/o
             @exited = true
             Log.debug "receive #{s.chomp} from branch"
@@ -237,7 +289,11 @@ module Pwrake
         end
         Log.debug "Master#invoke: fiber end"
       end
-      @runner.run
+      if !ending?
+        Log.debug "@selector.run begin"
+        @selector.run
+        Log.debug "@selector.run end"
+      end
       @post_pool.finish
       Log.debug "Master#invoke: end of task=#{t.name}"
     end
@@ -246,23 +302,26 @@ module Pwrake
       #Log.debug "#{self.class}#send_task_to_idle_core start"
       count = 0
       # @idle_cores.decrease(..
-      @task_queue.deq_task do |tw,hid|
+      @task_queue.deq_task do |tw,host_info,ncore|
+        host_info.busy(ncore)
         count += 1
-        @hostid_by_taskname[tw.name] = hid
+        @hostinfo_by_taskname[tw.name] = host_info
         tw.preprocess
         if tw.has_action?
+          hid = host_info.id
           s = "#{hid}:#{tw.task_id}:#{tw.name}"
           @channel_by_hostid[hid].put_line(s)
-          tw.exec_host = @hosts[hid]
+          tw.exec_host = host_info.name
         else
           tw.status = "end"
-          @task_queue.task_end(tw,hid) # @idle_cores.increase(..
+          task_end(tw,host_info) # @idle_cores.increase(..
           @post_pool.enq(tw)
         end
       end
-      if count == 0 && !@task_queue.empty? && @hostid_by_taskname.empty?
+      if count == 0 && !@task_queue.empty? && @hostinfo_by_taskname.empty?
         m="No task was invoked while unexecuted tasks remain"
         Log.error m
+        Log.error "@task_queue="+@task_queue.inspect
         raise RuntimeError,m
       end
       #Log.debug "#{self.class}#send_task_to_idle_core end time=#{Time.now-tm}"
@@ -272,7 +331,7 @@ module Pwrake
       i = 0
       n = @option.max_postprocess_pool
       @post_pool = FiberPool.new(n) do |pool|
-        postproc = @option.postprocess(@runner)
+        postproc = @option.postprocess(@selector)
         i += 1
         Log.debug "New postprocess fiber ##{i}"
         Fiber.new do
@@ -282,7 +341,8 @@ module Pwrake
             loc = postproc.run(tw)
             tw.postprocess(loc)
             pool.count_down
-            @hostid_by_taskname.delete(tw.name)
+            @hostinfo_by_taskname.delete(tw.name)
+            tw.retry_or_subsequent unless @exited
             break if yield(pool,j)
           end
           postproc.close
@@ -291,21 +351,25 @@ module Pwrake
       end
     end
 
+    def task_end(tw,host_info)
+      if host_info && host_info.idle(tw.n_used_cores(host_info))
+        # all retired
+        Log.warn("retired host:#{host_info.name} because all core retired")
+        drop_host(host_info)
+      end
+    end
+
     def setup_postprocess0
       setup_postprocess{false}
     end
 
     def setup_postprocess1
       setup_postprocess do |pool,j|
-        #Log.debug "@no_more_run=#{@no_more_run.inspect}"
-        #Log.debug "@task_queue.empty?=#{@task_queue.empty?}"
-        #Log.debug "@hostid_by_taskname=#{@hostid_by_taskname.inspect}"
-        #Log.debug "pool.empty?=#{pool.empty?}"
-        if (@no_more_run || @task_queue.empty?) &&
-            @hostid_by_taskname.empty?
-          Log.debug "postproc##{j} closing @channels=#{@channels.inspect}"
+        #Log.debug " pool.empty?=#{pool.empty?}"
+        if ending?
+          Log.debug "postproc##{j} closing"
           @finished = true
-          @channels.each{|ch| ch.finish} # exit
+          @selector.halt
           true
         elsif !@no_more_run
           send_task_to_idle_core
@@ -314,19 +378,28 @@ module Pwrake
       end
     end
 
+    def ending?
+      #Log.debug " @no_more_run=#{@no_more_run.inspect}"
+      #Log.debug " @task_queue.empty?=#{@task_queue.empty?}"
+      #Log.debug " @hostinfo_by_id.empty?=#{@hostinfo_by_id.empty?}"
+      #Log.debug " @hostinfo_by_taskname.keys=#{@hostinfo_by_taskname.keys.inspect}"
+      (@no_more_run || @task_queue.empty? || @hostinfo_by_id.empty?) &&
+        @hostinfo_by_taskname.empty?
+    end
+
     def handle_failed_target(name)
       case @option['FAILED_TARGET']
         #
       when /rename/i, NilClass
         dst = name+"._fail_"
         ::FileUtils.mv(name,dst)
-        msg = "Rename failed target file '#{name}' to '#{dst}'"
+        msg = "Rename output file '#{name}' to '#{dst}'"
         $stderr.puts(msg)
         Log.warn(msg)
         #
       when /delete/i
         ::FileUtils.rm(name)
-        msg = "Delete failed target file '#{name}'"
+        msg = "Delete output file '#{name}'"
         $stderr.puts(msg)
         Log.warn(msg)
         #
@@ -334,10 +407,40 @@ module Pwrake
       end
     end
 
+    def drop_host(host_info)
+      Log.debug "drop_host: #{host_info.name}"
+      hid = host_info.id
+      if @hostinfo_by_id[hid]
+        s = "drop:#{hid}"
+        @channel_by_hostid[hid].put_line(s)
+        @task_queue.drop_host(host_info)
+        @hostinfo_by_id.delete(hid)
+        if @hostinfo_by_id.empty?
+          if @finished
+            Log.debug "drop_host: @finished and @hostinfo_by_id.empty?"
+          else
+            Log.error "drop_host: All workers retired."
+            $stderr.puts "All workers retired."
+            @failed = true
+          end
+        end
+      end
+    end
+
     def finish
       Log.debug "Master#finish begin"
       @branch_setup_thread.join
-      @hdl_set.exit unless @exited
+      # continues running fibers
+      Log.debug "Master#finish @selector.run begin"
+      @selector.run(60)
+      Log.debug "Master#finish @selector.run end"
+      if !@exited
+        @exited = true
+        Log.debug "Master#finish Hander.exit begin"
+        NBIO::Handler.exit(@hdl_set)
+        @selector.run(60)
+        Log.debug "Master#finish Hander.exit end"
+      end
       TaskWrapper.close_task_logger
       Log.debug "Master#finish end"
       @failed