pwrake 2.1.3 → 2.2.0

data/lib/pwrake/master/master.rb

@@ -13,43 +13,24 @@ module Pwrake
     def initialize
       @selector = NBIO::Selector.new
       @hostinfo_by_taskname = {}
-      @option = Option.new
       @hdl_set = []
       @channel_by_hostid = {}
       @channels = []
       @hostinfo_by_id = {}
-      @n_retry = @option["RETRY"]
-      init_logger
+      # init
+      @option = Option.new
+      Log.set_logger(@option)
+      TaskWrapper.init_task_logger(@option)
+      # moved from Option#init
+      @option.put_log
+      if @option['LOG_DIR'] && @option['GC_LOG_FILE']
+        GC::Profiler.enable
+      end
     end
 
     attr_reader :task_queue
     attr_reader :option
-    attr_reader :logger
-
-    def init_logger
-      if logdir = @option['LOG_DIR']
-        ::FileUtils.mkdir_p(logdir)
-        logfile = File.join(logdir,@option['LOG_FILE'])
-        @logger = Logger.new(logfile)
-      else
-        if @option['DEBUG']
-          @logger = Logger.new($stderr)
-        else
-          @logger = Logger.new(File::NULL)
-        end
-      end
-
-      if @option['DEBUG']
-        @logger.level = Logger::DEBUG
-      else
-        @logger.level = Logger::INFO
-      end
-    end
-
-    def init(hosts=nil)
-      @option.init
-      TaskWrapper.init_task_logger(@option)
-    end
+    attr_reader :thread
 
     def setup_branch_handler(sub_host)
       ior,w0 = IO.pipe
@@ -83,13 +64,14 @@ module Pwrake
       case @killed
       when 0
         # log writing failed. can't be called from trap context
+        $stderr.puts "\nSignal trapped. (sig=#{sig} pid=#{Process.pid} ##{@killed})"
         if Rake.application.options.debug
-          $stderr.puts "\nSignal trapped. (sig=#{sig} pid=#{Process.pid}"+
-            " thread=#{Thread.current} ##{@killed})"
+          $stderr.puts "in master thread #{Thread.current}:"
           $stderr.puts caller
-        else
-          $stderr.puts "\nSignal trapped. (sig=#{sig} pid=#{Process.pid}"+
-            " ##{@killed})"
+          if @thread
+            $stderr.puts "in branch thread #{@thread}:"
+            $stderr.puts @thread.backtrace.join("\n")
+          end
         end
         $stderr.puts "Exiting..."
         @no_more_run = true
@@ -111,33 +93,37 @@ module Pwrake
       @option.host_map.each do |sub_host, wk_hosts|
         @hdl_set << hdl = setup_branch_handler(sub_host)
         Fiber.new do
-        hdl.put_line "host_list_begin"
-        wk_hosts.each do |host_info|
-          name = host_info.name
-          ncore = host_info.ncore
-          host_id = host_info.id
-          Log.debug "connecting #{name} ncore=#{ncore} id=#{host_id}"
-          hdl.put_line "host:#{host_id} #{name} #{ncore}"
-          @channel_by_hostid[host_id] = hdl
-          @hostinfo_by_id[host_id] = host_info
-        end
-        hdl.put_line "host_list_end"
-        while s = hdl.get_line
-          case s
-          when /^ncore:done$/
-            break
-          when /^ncore:(\d+):(\d+)$/
-            id, ncore = $1.to_i, $2.to_i
-            Log.debug "worker_id=#{id} ncore=#{ncore}"
-            @hostinfo_by_id[id].set_ncore(ncore)
-            sum_ncore += ncore
-          when /^exited$/
-            raise RuntimeError,"Unexpected branch exit"
-          else
-            msg = "#{hdl.host}:#{s.inspect}"
-            raise RuntimeError,"invalid return: #{msg}"
+          hdl.put_line "host_list_begin"
+          wk_hosts.each do |host_info|
+            name = host_info.name
+            ncore = host_info.ncore
+            host_id = host_info.id
+            Log.debug "connecting #{name} ncore=#{ncore} id=#{host_id}"
+            hdl.put_line "host:#{host_id} #{name} #{ncore}"
+            @channel_by_hostid[host_id] = hdl
+            @hostinfo_by_id[host_id] = host_info
+          end
+          hdl.put_line "host_list_end"
+          while s = hdl.get_line
+            case s
+            when /^ncore:done$/
+              break
+            when /^ncore:(\d+):(\d+)$/
+              id, ncore = $1.to_i, $2.to_i
+              Log.debug "worker_id=#{id} ncore=#{ncore}"
+              @hostinfo_by_id[id].set_ncore(ncore)
+              sum_ncore += ncore
+            when /^ip:(\d+):(\S+)$/
+              id, ipa = $1.to_i, $2
+              Log.debug "worker_id=#{id} ip=#{ipa}"
+              @hostinfo_by_id[id].set_ip(ipa)
+            when /^exited$/
+              raise RuntimeError,"Unexpected branch exit"
+            else
+              msg = "#{hdl.host}:#{s.inspect}"
+              raise RuntimeError,"invalid return: #{msg}"
+            end
           end
-        end
         end.resume
       end
       @selector.run
@@ -146,7 +132,7 @@ module Pwrake
       @option.total_cores = sum_ncore
       @hostinfo_by_id.each do |id,host|
         if ncore = @hostinfo_by_id[id].idle_cores
-          Log.info "#{host} id=#{id} ncore=#{ncore}"
+          Log.info "#{host.name} id=#{id} ncore=#{ncore}"
         else
           @hostinfo_by_id.delete(id)
         end
@@ -226,6 +212,7 @@ module Pwrake
       @branch_setup_thread.join
       send_task_to_idle_core
       #
+      n_retry = @option["RETRY"]
       create_fiber(@hdl_set) do |hdl|
         while s = hdl.get_line
           Log.debug "Master:recv #{s.inspect} from branch[#{hdl.host}]"
@@ -249,7 +236,7 @@ module Pwrake
               if host_info
                 continuous_fail = host_info.task_result(tw.status)
                 Log.debug "task=#{tw.name} continuous_fail=#{continuous_fail}"
-                if continuous_fail > @n_retry && @hostinfo_by_id.size > 1
+                if continuous_fail > n_retry && @hostinfo_by_id.size > 1
                   # retire this host
                   drop_host(host_info)
                   Log.warn("retired host:#{host_info.name} due to continuous fail")
@@ -313,6 +300,7 @@ module Pwrake
           s = "#{hid}:#{tw.task_id}:#{tw.name}"
           @channel_by_hostid[hid].put_line(s)
           tw.exec_host = host_info.name
+          tw.exec_host_id = hid
         else
           tw.status = "end"
           task_end(tw,host_info) # @idle_cores.increase(..
@@ -339,8 +327,7 @@ module Pwrake
           j = i
           while tw = pool.deq()
             Log.debug "postproc##{j} deq=#{tw.name}"
-            loc = postproc.run(tw)
-            tw.postprocess(loc)
+            tw.postprocess(postproc)
             pool.count_down
             @hostinfo_by_taskname.delete(tw.name)
             tw.retry_or_subsequent unless @exited

data/lib/pwrake/master/master_application.rb

@@ -9,14 +9,6 @@ module Pwrake
       @role.option
     end
 
-    def logger
-      @role.logger
-    end
-
-    def task_logger
-      @role.task_logger
-    end
-
     def task_queue
       @role.task_queue
     end
@@ -27,7 +19,6 @@ module Pwrake
         init("pwrake")  # <- parse options here
         @role = @master = Master.new
         t = Time.now
-        @master.init
         @master.setup_branches
         load_rakefile
         begin
@@ -39,6 +30,10 @@ module Pwrake
         rescue Exception => e
           # Exit with error message
           m = Log.bt(e)
+          if @master.thread
+            m += "\nIn branch thread #{@master.thread}:\n "
+            m += @master.thread.backtrace.join("\n ")
+          end
           Log.fatal m
           $stderr.puts m
           @master.signal_trap("INT")

data/lib/pwrake/mpi/branch.rb

@@ -0,0 +1,76 @@
+begin
+  require 'rake'
+rescue LoadError
+  require 'rubygems'
+  require 'rake'
+end
+
+require "pwrake/version"
+require "pwrake/master/master_application"
+require "pwrake/task/task_manager"
+require "pwrake/task/task_algorithm"
+require "pwrake/branch/branch_application"
+
+class Rake::Application
+  include Pwrake::BranchApplication
+  prepend Pwrake::MasterApplication
+  prepend Pwrake::TaskManager
+end
+
+class Rake::Task
+  include Pwrake::TaskAlgorithm
+end
+
+module Pwrake
+
+  class CommunicatorSet
+
+    def init_hosts
+      comm_size = MPipe::Comm.size
+      @ipaddr_to_rank = {}
+      @rank_to_ipaddr = Array.new(comm_size)
+      # read ip addresses
+      (1..comm_size-1).each do |rank|
+        io  = MPipe.new(rank)
+        sz, = io.read(4).unpack("V")
+        s   = io.read(sz)
+        v   = s.split('|')
+        v.each{|a| @ipaddr_to_rank[a] = rank}
+        @rank_to_ipaddr[rank] = v
+      end
+      Log.debug "@ipaddr_to_rank="+@ipaddr_to_rank.inspect
+    end
+
+    attr_reader :ipaddr_to_rank
+    attr_reader :rank_to_ipaddr
+  end
+
+  class Communicator
+
+    def setup_pipe(worker_code)
+      ipa = IPSocket.getaddress(@host)
+      if %w[127.0.0.1 ::1].include?(ipa)
+        ipa = IPSocket.getaddress(Socket.gethostname)
+      end
+      @rank = @set.ipaddr_to_rank[ipa]
+      if @rank.nil?
+        raise RuntimeError,"no rank for #{@host}"
+      end
+      mp = MPipe.new(@rank)
+      @ior = mp
+      @ioe,w1 = IO.pipe
+      @iow = mp
+      @pid = nil
+      w1.close
+      @ipaddr = @set.rank_to_ipaddr[@rank]
+    end
+  end
+
+end
+
+Pwrake::Branch.io_class = MPipe
+
+# does NOT exit when writing to broken pipe
+Signal.trap(:PIPE, "IGNORE")
+
+Rake.application.run

data/lib/pwrake/mpi/worker.rb

@@ -0,0 +1,42 @@
+require "parallel/processor_count.rb"
+require "pwrake/nbio"
+require "pwrake/branch/fiber_queue"
+require "pwrake/worker/writer"
+require "pwrake/worker/log_executor"
+require "pwrake/worker/executor"
+require "pwrake/worker/invoker"
+require "pwrake/worker/shared_directory"
+require "pwrake/worker/gfarm_directory"
+
+require "thread"
+require "fileutils"
+require "timeout"
+require "socket"
+
+module Pwrake
+
+  class Invoker
+
+    def get_io
+      # get IP addresses
+      v = Socket.getifaddrs
+      v = v.select{|a| a.addr.ip? && (a.flags & Socket::IFF_MULTICAST != 0)}
+      v = v.map{|a| a.addr.ip_address}
+      s = v.join('|')
+      # write IP addresses
+      iow = MPipe.new(0)
+      iow.write([s.size].pack("V"))
+      iow.write(s)
+      iow.flush
+      # returns IO, $stdin, $stdout
+      [MPipe, MPipe.new(0), MPipe.new(0)]
+    end
+
+    def send_ipaddr
+      # do nothing
+    end
+
+  end
+end
+
+require "pwrake/worker/worker_main"

data/lib/pwrake/nbio.rb

@@ -1,5 +1,7 @@
 require "fiber"
 
+$debug=false
+
 module Pwrake
 module NBIO
 
@@ -8,10 +10,11 @@ module NBIO
 
   class Selector
 
-    def initialize
+    def initialize(io_class=IO)
       @reader = {}
       @writer = {}
       @running = false
+      @io_class = io_class
     end
 
     attr_reader :reader, :writer
@@ -52,9 +55,12 @@ module NBIO
       @running = true
       init_heartbeat if timeout
       while @running && !empty?
-        if $debug
-          Log.debug "Selector#run: "+caller[0..1].join(", ")+
-            " @reader.size=#{@reader.size} @writer.size=#{@writer.size}"
+        if $debug && defined? Log
+          rd_insp = @reader.map{|k,v|
+            "%s=>%s,%s" % [k.inspect,v.class.inspect,v.waiter.inspect]
+          }.join(",")
+          Log.debug "Selector#run:\n "+caller[0..1].join("\n ")+
+            "\n @reader={#{rd_insp}}\n @writer.size=#{@writer.size}"
           $stderr.puts "Selector#run: "+caller[0]
         end
         run_select(timeout)
@@ -67,7 +73,7 @@ module NBIO
     private
     def run_select(timeout)
       to = (timeout) ? timeout*0.75 : nil
-      r, w, = IO.select(@reader.keys,@writer.keys,[],to)
+      r, w, = @io_class.select(@reader.keys,@writer.keys,[],to)
       check_heartbeat(r,timeout) if timeout
       r.each{|io| @reader[io].call} if r
       w.each{|io| @writer[io].call} if w
@@ -76,7 +82,7 @@ module NBIO
       @reader.keys.each do |io|
         if io.closed?
           m = "#{em} io=#{io}"
-          Log.error(m)
+          Log.error(m) if defined? Log
           $stderr.puts m
           hdl = @reader.delete(io)
           hdl.error(e)
@@ -85,13 +91,12 @@ module NBIO
       @writer.keys.each do |io|
         if io.closed?
           m = "#{em} io=#{io}"
-          Log.error(m)
+          Log.error(m) if defined? Log
           $stderr.puts m
           hdl = @writer.delete(io)
           hdl.error(e)
         end
       end
-      #raise e
     end
 
     def init_heartbeat
@@ -169,6 +174,12 @@ module NBIO
       flush unless buffered
     end
 
+    alias print :write
+
+    def puts(s)
+      write(s+"\n")
+    end
+
     def flush
       until @pool.empty?
         len = _write(@pool[0])
@@ -221,10 +232,11 @@ module NBIO
       @io = io
       @waiter = []
       @buf = ''
+      @eof = false
       @sep = "\n"
       @chunk_size = 8192
     end
-    attr_reader :io
+    attr_reader :io, :waiter
     attr_accessor :check_timeout
 
     # call from Selector#run
@@ -240,19 +252,26 @@ module NBIO
       @buf.slice!(0, index+@sep.bytesize)
     rescue EOFError => e
       if @buf.empty?
-        #return nil
         raise e
       else
         buf = @buf; @buf = ''
         return buf
       end
-    #rescue IO::WaitReadable
     end
 
     # call from Reader#_read and FiberReaderQueue#deq
     def select_io
-      @selector.add_reader(self) if @waiter.empty?
+      if @waiter.empty?
+        @selector.add_reader(self)
+      else
+        if @selector.reader[@io] != self
+          raise RuntimeError, "access from multiple Fiber"
+        end
+      end
       @waiter.push(Fiber.current)
+      if $debug && defined? Log
+        Log.debug("Reader#select_io: #{Fiber.current.inspect}\n "+caller.join("\n "))
+      end
       Fiber.yield
     ensure
       @waiter.delete(Fiber.current)
@@ -271,11 +290,16 @@ module NBIO
       @halting = false
     end
 
+    def eof?
+      @eof && @buf.empty?
+    end
+
     # from Bartender
 
     def _read(sz)
       @io.read_nonblock(sz)
     rescue EOFError
+      @eof = true
       nil
     rescue IO::WaitReadable
       return nil if @halting
@@ -313,6 +337,7 @@ module NBIO
     end
 
     alias get_line :readln
+    alias gets :readln
 
   end
 
@@ -368,7 +393,6 @@ module NBIO
     rescue EOFError
       halt
     rescue IO::WaitReadable
-      #p IO::WaitReadable
     end
 
     def error(e)
@@ -378,6 +402,7 @@ module NBIO
     end
 
     def halt
+      Log.debug("Handler.halt") if defined? Log
       @queue.each{|q| q.halt}
       @default_queue.halt
     end
@@ -430,6 +455,7 @@ module NBIO
       @writer = writer
       @reader = reader
       @host = hostname
+      @exited = false
     end
     attr_reader :reader, :writer, :host
 
@@ -442,8 +468,7 @@ module NBIO
     end
 
     def put_kill(sig="INT")
-      #@writer.put_line("kill:#{sig}")
-      @writer.io.puts("kill:#{sig}")
+      @writer.io.write("kill:#{sig}\n")
       @writer.io.flush
     end
 
@@ -452,24 +477,31 @@ module NBIO
     end
 
     def exit
-      exit_msg = "exited"
       iow = @writer.io
-      Log.debug "Handler#exit iow=#{iow.inspect}"
-      return if iow.closed?
+      Log.debug "Handler#exit iow=#{iow.inspect}" if defined? Log
+      if @exited
+        Log.debug "Handler<##{object_id}#exit multiple called" if defined? Log
+        bt = caller.join("\n ")
+        Log.debug "Handler#exit bt=\n #{bt}" if defined? Log
+        return
+      end
+      @exited = true
+      exit_msg = "exited"
+      #return if iow.closed?
       @writer.put_line "exit"
-      while line = @reader.get_line
-        # here might receive "retire:0" from branch...
-        line.chomp!
-        Log.debug "Handler#exit: #{line} host=#{@host}"
-        return if line == exit_msg
+      Log.debug "Handler#exit: end: @writer.put_line \"exit\"" if defined? Log
+      #
+      if @reader.class == Reader # MultiReader not work
+        while line = @reader.get_line
+          # here might receive "retire:0" from branch...
+          line.chomp!
+          Log.debug "Handler#exit: #{line} host=#{@host}" if defined? Log
+          return if line == exit_msg
+        end
       end
     rescue Errno::EPIPE => e
-      if Rake.application.options.debug
-        #$stderr.puts "Errno::EPIPE in #{self.class}#exit iow=#{iow.inspect}"
-        #$stderr.puts e.backtrace.join("\n")
-      end
       Log.error "Errno::EPIPE in #{self.class}.exit iow=#{iow.inspect}\n"+
-        e.backtrace.join("\n")
+        e.backtrace.join("\n") if defined? Log
     end
 
     def halt
@@ -496,38 +528,4 @@ module NBIO
   end
 
 end
-
-#------------------------------------------------------------------
-
-if __FILE__ == $0
-  iosel = NBIO::Selector.new
-
-  io = 5.times.map do
-    IO.pipe
-  end
-
-  io.each do |ior,iow|
-    rd = NBIO::MultiReader.new(iosel,ior,1)
-    Fiber.new do
-      while s = rd.get_line(0)
-        puts s
-      end
-      puts "fiber end"
-    end.resume
-  end
-
-  io.each do |ior,iow|
-    wr = NBIO::Writer.new(iosel,iow)
-    Fiber.new do
-      2000.times do |i|
-        wr.put_line("test str#{i}"+"-"*80,0)
-        #iow.puts "0:test str#{i}"+"-"*80
-      end
-      #iow.print "hage"
-      iow.close
-    end.resume
-  end
-
-  iosel.run
-end
 end