pwrake 2.0.1 → 2.1.0

data/lib/pwrake/branch/branch_application.rb

@@ -1,3 +1,6 @@
+require "pwrake/logger"
+require "pwrake/branch/branch"
+
 module Pwrake
 
   # The TaskManager module is a mixin for managing tasks.
@@ -36,6 +39,7 @@ module Pwrake
     def run_branch_in_thread(r,w,opts)
       #standard_exception_handling do
         @branch = Branch.new(opts,r,w)
+        @branch.init_logger
         begin
           @branch.run
         rescue => e

data/lib/pwrake/branch/communicator.rb

@@ -0,0 +1,173 @@
+module Pwrake
+
+class CommChannel
+
+  def initialize(host,id,queue,writer,ios=[])
+    @host = host
+    @id = id
+    @queue = queue
+    @writer = writer
+    @ios = ios
+  end
+
+  attr_reader :host, :id
+
+  def put_line(s)
+    if $cause_fault
+      $cause_fault = nil
+      Log.warn("closing writer io caller=\n#{caller.join("\n")}")
+      @ios.each{|io| io.close}
+    end
+    @writer.put_line(s,@id)
+  end
+
+  def get_line
+    @queue.deq
+  end
+
+  def halt
+    @queue.halt
+    @writer.halt
+  end
+end
+
+class Communicator
+
+  class ConnectError < IOError; end
+
+  attr_reader :id, :host, :ncore, :channel
+  attr_reader :reader, :writer, :handler
+  attr_reader :shells
+
+  def initialize(set,id,host,ncore,selector,option)
+    @set = set
+    @id = id
+    @host = host
+    @ncore = @ncore_given = ncore
+    @selector = selector
+    @option = option
+    @shells = {}
+  end
+
+  def inspect
+    "#<#{self.class} @id=#{@id},@host=#{@host},@ncore=#{@ncore}>"
+  end
+
+  def new_channel
+    i,q = @reader.new_queue
+    CommChannel.new(@host,i,q,@writer,[@ior,@iow,@ioe])
+  end
+
+  def connect(worker_code)
+    rb_cmd = "ruby -e 'eval ARGF.read(#{worker_code.size})'"
+    if ['localhost','localhost.localdomain','127.0.0.1'].include? @host
+    #if /^localhost/ =~ @host
+      cmd = rb_cmd
+    else
+      cmd = "ssh -x -T #{@option[:ssh_option]} #{@host} \"#{rb_cmd}\""
+    end
+    #
+    @ior,w0 = IO.pipe
+    @ioe,w1 = IO.pipe
+    r2,@iow = IO.pipe
+    @pid = Kernel.spawn(cmd,:pgroup=>true,:out=>w0,:err=>w1,:in=>r2)
+    w0.close
+    w1.close
+    r2.close
+    sel = @set.selector
+    @reader = NBIO::MultiReader.new(sel,@ior)
+    @rd_err = NBIO::Reader.new(sel,@ioe)
+    @writer = NBIO::Writer.new(sel,@iow)
+    @handler = NBIO::Handler.new(@reader,@writer,@host)
+    #
+    @writer.write(worker_code)
+    @writer.write(Marshal.dump(@ncore))
+    @writer.write(Marshal.dump(@option))
+    # read ncore
+    while s = @reader.get_line
+      if /^ncore:(.*)$/ =~ s
+        a = $1
+        Log.debug "ncore=#{a} @#{@host}"
+        if /^(\d+)$/ =~ a
+          @ncore = $1.to_i
+          return false
+        else
+          raise ConnectError, "invalid for ncore: #{a.inspect}"
+        end
+      else
+        return false if !common_line(s)
+      end
+    end
+    raise ConnectError, "fail to connect #{cmd.inspect}"
+  rescue => e
+    dropout(e)
+  end
+
+  def common_line(s)
+    x = "Communicator#common_line(id=#{@id},host=#{@host})"
+    case s
+    when /^heartbeat$/
+      Log.debug "#{x}: #{s.inspect}"
+      @selector.heartbeat(@reader.io)
+    when /^exited$/
+      Log.debug "#{x}: #{s.inspect}"
+      return false
+    when /^log:(.*)$/
+      Log.info "#{x}: log>#{$1}"
+    when String
+      Log.warn "#{x}: out>#{s.inspect}"
+    when Exception
+      Log.warn "#{x}: err>#{s.class}: #{s.message}"
+      dropout(s)
+      return false
+    else
+      raise ConnectError, "#{x}: invalid for read: #{s.inspect}"
+    end
+    true
+  end
+
+  def finish_shells
+    @shells.keys.each{|sh| sh.finish_task_q}
+  end
+
+  def dropout(exc=nil)
+    # Error output
+    err_out = []
+    begin
+      finish_shells
+      @handler.exit
+      while s = @rd_err.get_line
+        err_out << s
+      end
+    rescue => e
+      m = Log.bt(e)
+      #$stderr.puts m
+      Log.error(m)
+    end
+    # Error output
+    if !err_out.empty?
+      $stderr.puts err_out.join("\n")
+      Log.error((["process error output:"]+err_out).join("\n "))
+    end
+    # Exception
+    if exc
+      m = Log.bt(exc)
+      #$stderr.puts m
+      Log.error m
+    end
+  ensure
+    @set.delete(self)
+  end
+
+  def finish
+    @iow.close
+    while s=@ior.gets
+      puts "out=#{s.chomp}"
+    end
+    while s=@ioe.gets
+      puts "err=#{s.chomp}"
+    end
+  end
+
+end
+end

data/lib/pwrake/branch/communicator_set.rb

@@ -0,0 +1,100 @@
+require "forwardable"
+require "pwrake/branch/communicator"
+
+module Pwrake
+class CommunicatorSet
+
+  extend Forwardable
+
+  def initialize(master_rd,selector,option)
+    @master_rd = master_rd
+    @selector = selector
+    @option = option
+    @communicators = {}
+    @initial_communicators = []
+    if hb = @option[:heartbeat]
+      @heartbeat_timeout = hb + 15
+    end
+  end
+
+  attr_reader :selector
+
+  def_delegators :@communicators, :each, :each_value, :values, :size
+
+  def create_communicators
+    Fiber.new do
+      s = @master_rd.get_line
+      if s.chomp != "host_list_begin"
+        raise "Branch#setup_worker: recv=#{s.chomp} expected=host_list_begin"
+      end
+
+      while s = @master_rd.get_line
+        s.chomp!
+        break if s == "host_list_end"
+        if /^host:(\d+) (\S+) ([+-]?\d+)?$/ =~ s
+          id, host, ncore = $1,$2,$3
+          ncore &&= ncore.to_i
+          @communicators[id] = Communicator.new(self,id,host,ncore,@selector,@option)
+        else
+          raise "Branch#setup_worker: recv=#{s.chomp} expected=host:id hostname ncore"
+        end
+      end
+    end.resume
+    @selector.run(@heartbeat_timeout)
+    @initial_communicators = @communicators.dup
+  end
+
+  def add(comm)
+    @communicators[comm.id] = comm
+  end
+
+  def delete(comm)
+    @communicators.delete(comm.id)
+    @error_host << comm.host
+  end
+
+  def drop(id)
+    comm = @communicators[id]
+    Log.debug "drop:id=#{id} comm=#{comm.inspect} @communicators.keys=#{@communicators.keys}"
+    comm.dropout if comm
+  end
+
+  def drop_all
+    Log.debug "drop_all"
+    @communicators.keys.each do |id|
+      @communicators[id].dropout
+    end
+  end
+
+  def finish_shells
+    Log.debug "finish_shells"
+    @communicators.keys.each do |id|
+      @communicators[id].finish_shells
+    end
+  end
+
+  def run(message)
+    @error_host = []
+    n1 = @communicators.size
+    @selector.run(@heartbeat_timeout)
+    n2 = @communicators.size
+    if n1 != n2
+      Log.info "# of communicators: #{n1}->#{n2} during #{message.inspect}"
+      Log.info "retired hosts=[#{@error_host.join(',')}]"
+    end
+  end
+
+  def handler_set
+    @communicators.each_value.map{|comm| comm.handler}
+  end
+
+  def kill(sig)
+    NBIO::Handler.kill(handler_set,sig)
+  end
+
+  def exit
+    NBIO::Handler.exit(handler_set)
+  end
+
+end
+end

data/lib/pwrake/branch/fiber_queue.rb

@@ -2,6 +2,9 @@ require 'fiber'
 
 module Pwrake
 
+  class FiberQueueError < StandardError
+  end
+
   class FiberQueue
 
     def initialize
@@ -11,6 +14,9 @@ module Pwrake
     end
 
     def enq(x)
+      if @finished
+        raise FiberQueueError,"cannot enq to already finished queue"
+      end
       @q.push(x)
       f = @waiter.shift
       f.resume if f
@@ -25,6 +31,10 @@ module Pwrake
       return @q.shift
     end
 
+    def deq_nonblock
+      @q.shift
+    end
+
     def finish
       @finished = true
       while f = @waiter.shift

data/lib/pwrake/branch/shell.rb

@@ -1,3 +1,5 @@
+require 'pwrake/branch/shell_profiler'
+
 module Pwrake
 
   class DummyMutex
@@ -20,15 +22,16 @@ module Pwrake
       BY_FIBER[Fiber.current]
     end
 
-    def initialize(chan,task_q,opt={})
+    def initialize(chan,comm,task_q,opt={})
       @chan = chan
-      @host = chan.handler.host
+      @id   = chan.id
+      @host = chan.host
+      @comm = comm
       @task_q = task_q
       @lock = DummyMutex.new
-      @id = chan.id
-      #
       @option = opt
       @work_dir = @option[:work_dir] || Dir.pwd
+      @comm.shells[self] = true
     end
 
     attr_reader :id, :host, :status, :profile
@@ -36,11 +39,12 @@ module Pwrake
     def open
       if @opened
         Log.warn "already opened: host=#{@host} id=#{@id}"
+        return
       end
+      @opened = true
       _puts("open")
       if (s = _gets) == "open"
         OPEN_LIST[__id__] = self
-        @opened = true
         true
       else
         Log.error("Shell#open failed: recieve #{s.inspect}")
@@ -48,19 +52,24 @@ module Pwrake
       end
     end
 
-    def close
+    def exit
       if !@opened
-        Log.warn "already closed: host=#{@host} id=#{@id}"
+        Log.debug "already exited: host=#{@host} id=#{@id}"
+        return
       end
+      @opened = false
       _puts("exit")
       if (s = _gets) == "exit"
         OPEN_LIST.delete(__id__)
-        @opened = false
+        Log.debug("Shell#exit: recieve #{s.inspect}")
         true
       else
-        Log.warn("Shell#close failed: recieve #{s.inspect}")
+        Log.debug("Shell#exit: recieve #{s.inspect}")
         false
       end
+    rescue IOError,Errno::EPIPE => e
+      Log.debug("Shell#exit: #{Log.bt(e)}")
+      false
     end
 
     def set_current_task(task_id,task_name)
@@ -106,20 +115,17 @@ module Pwrake
 
     def _gets
       s = @chan.get_line
-      #Log.debug "Shell#_gets(host=#{@host},id=#{@id}): #{s.inspect}"
-      if s.nil?
-        begin
-          raise
-        rescue => e
-          Log.debug e
-        end
+      Log.debug "Shell#_gets(host=#{@host},id=#{@id}): #{s.inspect}"
+      case s
+      when Exception
+        @chan.halt
+        Log.error Log.bt(s)
       end
       s
     end
 
     def _system(cmd)
       @cmd = cmd
-      #raise "@chan is closed" if @chan.closed?
       @lock.synchronize do
         _puts(cmd)
         status = io_read_loop{}
@@ -129,11 +135,10 @@ module Pwrake
 
     def _backquote(cmd)
       @cmd = cmd
-      #raise "@chan is closed" if @chan.closed?
       a = []
       @lock.synchronize do
         _puts(cmd)
-        status = io_read_loop{|x| a << x}
+        @status = io_read_loop{|x| a << x}
       end
       a.join("\n")
     end
@@ -141,9 +146,9 @@ module Pwrake
     def _execute(cmd,quote=nil,&block)
       @cmd = cmd
       if !@opened
-        raise "closed"
+        raise "non opened"
       end
-      status = nil
+      @status = nil
       start_time = Time.now
       begin
         _puts(cmd)
@@ -184,6 +189,18 @@ module Pwrake
             end
             return status
           end
+        when "exit"
+          msg = "Shell#io_read_loop: exit"
+          $stderr.puts(msg)
+          Log.error(msg)
+          @chan.halt
+          return "exit"
+        when IOError
+          @chan.halt
+          return "ioerror"
+        when NBIO::TimeoutError
+          @chan.halt
+          return "timeout"
         end
         msg = "Shell#io_read_loop: Invalid result: #{s.inspect}"
         $stderr.puts(msg)
@@ -193,7 +210,8 @@ module Pwrake
 
     public
 
-    def create_fiber(hdl)
+    def create_fiber(master_w)
+      @master_w = master_w
       if !@opened
         Log.warn "not opened: host=#{@host} id=#{@id}"
       end
@@ -218,13 +236,39 @@ module Pwrake
               Rake.application.display_error_message(e)
               Log.error e
               result = "taskfail:#{@id}:#{task.name}"
+            ensure
+              master_w.put_line result
             end
-            hdl.put_line result
           end
-        ensure
           Log.debug "shell id=#{@id} fiber end"
+          master_w.put_line "retire:#{@comm.id}"
+          @comm.shells.delete(self)
+          exit
+          if @comm.shells.empty?
+            @comm.dropout
+          end
+          @chan.halt
+        rescue => e
+          m = Log.bt(e)
+          #$stderr.puts m
+          Log.error(m)
+        end
+      end
+    end
+
+    def finish_task_q
+      @task_q.finish
+      #Log.debug "finish_task_q: @task_q=#{@task_q.inspect}"
+      while task_str = @task_q.deq_nonblock
+        if /^(\d+):(.*)$/ =~ task_str
+          task_id, task_name = $1.to_i, $2
+        else
+          raise RuntimeError, "invalid task_str: #{task_str}"
         end
+        @master_w.put_line "taskfail:#{@id}:#{task_name}"
+        Log.warn "unexecuted task: #{result}"
       end
+      @chan.halt
     end
 
   end