pwrake 2.1.3 → 2.2.0

data/lib/pwrake/queue/locality_aware_queue.rb

@@ -6,9 +6,7 @@ module Pwrake
       # group_map = {gid1=>[hid1,hid2,...], ...}
       @size_q = 0
       @q = {}
-      @hostinfo_by_name = {}
       @hostinfo_by_id.each do |id,h|
-        @hostinfo_by_name[h.name] = h
         @q[id] = @array_class.new(h.ncore)
       end
       @q_group = {}
@@ -26,23 +24,21 @@ module Pwrake
       @n_turn = @disable_steal ? 1 : 2
     end
 
-
     def enq_impl(t)
       hints = t && t.suggest_location
       Log.debug "enq #{t.name} hints=#{hints.inspect}"
       if hints.nil? || hints.empty?
         @q_remote.push(t)
       else
-        stored = false
+        kv = {}
         hints.each do |h|
-          host_info = @hostinfo_by_name[h]
-          if host_info && q = @q[host_info.id]
-            t.assigned.push(host_info.id)
-            q.push(t)
-            stored = true
-          end
+          HostMap.ipmatch_for_name(h).each{|id| kv[id] = true}
         end
-        if stored
+        if !kv.empty?
+          kv.each_key do |id|
+            t.assigned.push(id)
+            @q[id].push(t)
+          end
           @size_q += 1
         else
           @q_remote.push(t)

data/lib/pwrake/report/task_stat.rb

@@ -39,12 +39,10 @@ module Pwrake
 
     def task_locality
       file_size = {}
-      file_host = {}
       h = {}
       @task_table.each do |row|
         name            = row['task_name']
         file_size[name] = row['file_size'].to_i
-        file_host[name] = (row['file_host']||'').split('|')
         exec_host = row['exec_host'] || ""
         h[exec_host] = true
       end
@@ -54,15 +52,16 @@ module Pwrake
         if row['executed']=='1'
           name      = row['task_name']
           exec_host = row['exec_host']
-          loc = file_host[name].include?(exec_host)
+          loc = (row['write_loc'] == "L")
           count(exec_host, loc, :out_num, 1)
           count(exec_host, loc, :out_size, file_size[name])
 
           preq_files = (row['preq']||'').split('|')
-          preq_files.each do |preq|
+          preq_loc = row['preq_loc']||''
+          preq_files.each_with_index do |preq,i|
             sz = file_size[preq]
             if sz && sz > 0
-              loc = file_host[preq].include?(exec_host)
+              loc = (preq_loc[i] == "L")
               count(exec_host, loc, :in_num, 1)
               count(exec_host, loc, :in_size, sz)
             end

data/lib/pwrake/task/task_wrapper.rb

@@ -25,10 +25,10 @@ module Pwrake
       @input_file_mtime = nil
       @rank = nil
       @priority = nil
-      @lock_rank = Monitor.new
       @executed = false
       @assigned = []
       @exec_host = nil
+      @exec_host_id = nil
       @tried_hosts = []
       @n_retry = @property.retry || Rake.application.pwrake_options["RETRY"] || 1
     end
@@ -41,7 +41,7 @@ module Pwrake
     attr_reader :assigned
     attr_reader :tried_hosts
     attr_accessor :executed
-    attr_accessor :exec_host
+    attr_accessor :exec_host, :exec_host_id
     attr_accessor :shell_id, :status
 
     def self.format_time(t)
@@ -53,8 +53,8 @@ module Pwrake
         fn = File.join(dir,option['TASK_CSV_FILE'])
         @@task_logger = CSV.open(fn,'w')
         @@task_logger.puts %w[
-          task_id task_name start_time end_time elap_time preq preq_host
-          exec_host shell_id has_action executed file_size file_mtime file_host
+          task_id task_name start_time end_time elap_time preq preq_host preq_loc
+          exec_host shell_id has_action executed file_size file_mtime file_host write_loc
         ]
       end
     end
@@ -78,14 +78,14 @@ module Pwrake
       @n_retry == 0
     end
 
-    def postprocess(location)
+    def postprocess(postproc)
       @executed = true if !@task.actions.empty?
       #tm_taskend = Time.now
       if is_file_task?
         #t = Time.now
         if File.exist?(name)
           @file_stat = File::Stat.new(name)
-          @location = location
+          @location = postproc.run(self)
         end
       end
       #Log.debug "postprocess time=#{Time.now-tm_taskend}"
@@ -114,11 +114,11 @@ module Pwrake
     def log_task
       @time_end = Time.now
       #
-      loc = suggest_location()
+      sug_host = suggest_location()
       shell = Pwrake::Shell.current
       #
-      if loc && !loc.empty? && shell && !actions.empty?
-        Rake.application.count( loc, shell.host )
+      if sug_host && !sug_host.empty? && shell && !actions.empty?
+        Rake.application.count( sug_host, shell.host )
       end
       return if !@@task_logger
       #
@@ -127,20 +127,34 @@ module Pwrake
         RANK_STAT.add_sample(rank,elap)
       end
       #
+      # locality check
+      loc_na = true
+      preq_loc = prerequisites.map do |preq|
+        locs = Rake.application[preq].wrapper.location
+        if loc = file_locality(locs)
+          loc_na = false
+          loc
+        else
+          "n"
+        end
+      end.join("")
+      preq_loc = nil if loc_na
+      write_loc = file_locality(@location)
+      #
       if @file_stat
-        fstat = [@file_stat.size, @file_stat.mtime, self.location.join('|')]
+        fstat = [@file_stat.size, @file_stat.mtime, self.location.join('|'), write_loc]
       else
-        fstat = [nil]*3
+        fstat = [nil]*4
       end
       #
-      # task_id task_name start_time end_time elap_time preq preq_host
-      # exec_host shell_id has_action executed file_size file_mtime file_host
+      # task_id task_name start_time end_time elap_time preq preq_host preq_loc
+      # exec_host shell_id has_action executed file_size file_mtime file_host write_loc
       #
       row = [ @task_id, name, @time_start, @time_end, elap,
-              prerequisites, loc, @exec_host, @shell_id,
+              prerequisites, sug_host, preq_loc, @exec_host, @shell_id,
               (actions.empty?) ? 0 : 1,
               (@executed) ? 1 : 0,
-              *fstat ]
+            ] + fstat
       row.map!{|x|
         if x.kind_of?(Time)
           TaskWrapper.format_time(x)
@@ -166,6 +180,17 @@ module Pwrake
       end
     end
 
+    def file_locality(nodes)
+      if nodes.empty? || !@exec_host_id
+        nil  # not available
+      elsif nodes.any?{|node|
+          HostMap.ipmatch_for_name(node).include?(@exec_host_id)}
+        "L"  # Local
+      else
+        "R"  # Remote
+      end
+    end
+
     def is_file_task?
       @task.kind_of?(Rake::FileTask)
     end
@@ -223,28 +248,26 @@ module Pwrake
     end
 
     def rank
-      #@lock_rank.synchronize do
-        if @rank.nil?
-          if subsequents.nil? || subsequents.empty?
-            @rank = 0
-          else
-            max_rank = 0
-            subsequents.each do |subsq|
-              r = subsq.wrapper.rank
-              if max_rank < r
-                max_rank = r
-              end
-            end
-            if has_output_file?
-              step = 1
-            else
-              step = 0
+      if @rank.nil?
+        if subsequents.nil? || subsequents.empty?
+          @rank = 0
+        else
+          max_rank = 0
+          subsequents.each do |subsq|
+            r = subsq.wrapper.rank
+            if max_rank < r
+              max_rank = r
             end
-            @rank = max_rank + step
           end
-          Log.debug "Task[#{name}] rank=#{@rank.inspect}"
+          if has_output_file?
+            step = 1
+          else
+            step = 0
+          end
+          @rank = max_rank + step
         end
-      #end
+        Log.debug "Task[#{name}] rank=#{@rank.inspect}"
+      end
       @rank
     end
 

data/lib/pwrake/version.rb

@@ -1,3 +1,3 @@
 module Pwrake
-  VERSION = "2.1.3"
+  VERSION = "2.2.0"
 end

data/lib/pwrake/worker/executor.rb

@@ -7,17 +7,37 @@ module Pwrake
       @id = id
       @out = Writer.instance
       @log = LogExecutor.instance
-      @queue = []
+      @queue = FiberQueue.new
       @rd_list = []
       @dir = dir_class.new
       @dir.open
       @dir.open_messages.each{|m| @log.info(m)}
       @out.puts "#{@id}:open"
+
+      r,w = IO.pipe
+      @command_pipe_r = NBIO::Reader.new(@selector,r)
+      @command_pipe_w = NBIO::Writer.new(@selector,w)
+      @start_process_fiber = Fiber.new do
+        while line = @queue.deq
+          cmd = line
+          while /\\$/ =~ line  # line continues
+            line = @queue.deq
+            break if !line
+            cmd += line
+          end
+          break if @stopped
+          cmd.chomp!
+          if !cmd.empty?
+            start_process(cmd)
+          end
+          Fiber.yield
+        end
+      end
     end
 
     def stop
       @stopped = true
-      @queue.clear
+      @queue.finish
     end
 
     def close
@@ -43,13 +63,12 @@ module Pwrake
 
     def execute(cmd)
       return if @stopped
-      @queue.push(cmd)
-      start_process
+      @queue.enq(cmd)
+      @start_process_fiber.resume
     end
 
-    def start_process
+    def start_process(command)
       return if @thread      # running
-      command = @queue.shift
       return if !command     # empty queue
       @spawn_in, @sh_in = IO.pipe
       @sh_out, @spawn_out = IO.pipe
@@ -71,20 +90,19 @@ module Pwrake
         @spawn_err.close
       end
 
-      @rd_out = Reader.new(@sh_out,"o")
-      @rd_err = Reader.new(@sh_err,"e")
+      @rd_out = NBIO::Reader.new(@selector,@sh_out)
+      @rd_err = NBIO::Reader.new(@selector,@sh_err)
       @rd_list = [@rd_out,@rd_err]
 
-      @selector.add_reader(@sh_out){callback(@rd_out)}
-      @selector.add_reader(@sh_err){callback(@rd_err)}
+      Fiber.new{callback(@rd_err,"e")}.resume
+      Fiber.new{callback(@rd_out,"o")}.resume
     end
 
-    def callback(rd)
+    def callback(rd,mode)
       while s = rd.gets
-        @out.puts "#{@id}:#{rd.mode}:#{s.chomp}"
+        @out.puts "#{@id}:#{mode}:#{s.chomp}"
       end
       if rd.eof?
-        @selector.delete_reader(rd.io)
         @rd_list.delete(rd)
         if @rd_list.empty?  # process_end
           @thread = @pid = nil
@@ -93,7 +111,7 @@ module Pwrake
           @sh_in.close
           @sh_out.close
           @sh_err.close
-          start_process     # next process
+          @start_process_fiber.resume # next process
         end
       end
     rescue => exc

data/lib/pwrake/worker/invoker.rb

@@ -1,3 +1,5 @@
+require "socket"
+
 module Pwrake
 
   class Invoker
@@ -12,37 +14,65 @@ module Pwrake
       end
     end
 
-    def initialize(dir_class, ncore, option)
-      @dir_class = dir_class
-      @option = option
-      @selector = Selector.new
+    def get_io
+      [IO, $stdin, $stdout]
+    end
+
+    def setup_connection
+      ioc, ior, iow = get_io()
+      # read @ncore and @option
+      @ncore,len = ior.read(8).unpack("V2")
+      @option = Marshal.load(ior.read(len))
+      # set pipe to branch-master
+      @selector = NBIO::Selector.new(ioc)
+      @rd = NBIO::Reader.new(@selector,ior)
+      @out = Writer.instance
+      @out.out = iow
+    end
+
+    def initialize
+      setup_connection
+      @dir_class = Pwrake.const_get(@option[:shared_directory])
+      @dir_class.init(@option)
       @ex_list = {}
-      @out = Writer.instance # firstly replace $stderr
       @log = LogExecutor.instance
       @log.init(@option)
       @log.open(@dir_class)
       @out.add_logger(@log)
-      if ncore.kind_of?(Integer)
-        if ncore > 0
-          @ncore = ncore
-        else
-          @ncore = processor_count() + ncore
+      send_ipaddr
+      send_ncore
+      # does NOT exit when writing to broken pipe
+      Signal.trap("PIPE", "SIG_IGN")
+    end
+
+    def send_ipaddr
+      # get IP addresses
+      v = Socket.getifaddrs.
+        select{|a| a.addr.ip? && (a.flags & Socket::IFF_MULTICAST != 0)}
+      # write IP addresses
+      v.each do |a|
+        @out.puts "ip:#{a.addr.ip_address}"
+      end
+    end
+
+    def send_ncore
+      if @ncore.kind_of?(Integer)
+        if @ncore <= 0
+          @ncore += processor_count()
         end
         if @ncore <= 0
-          m = "Out of range: ncore=#{ncore.inspect}"
+          m = "Out of range: ncore=#{@ncore.inspect}"
           @out.puts "ncore:"+m
           raise ArgumentError,m
         end
-      elsif ncore.nil?
+      elsif @ncore.nil?
         @ncore = processor_count()
       else
-        m = "Invalid argument: ncore=#{ncore.inspect}"
+        m = "Invalid argument: ncore=#{@ncore.inspect}"
         @out.puts "ncore:"+m
         raise ArgumentError,m
       end
       @out.puts "ncore:#{@ncore}"
-      # does NOT exit when writing to broken pipe
-      Signal.trap("PIPE", "SIG_IGN")
     end
 
     def get_line(io)
@@ -57,10 +87,10 @@ module Pwrake
 
     def run
       setup_option
-      setup_loop
-      @rd = Reader.new($stdin)
-      @selector.add_reader($stdin){command_callback(@rd)}
-      @selector.loop
+      Fiber.new{setup_loop}.resume
+      @selector.run
+      Fiber.new{command_callback}.resume
+      @selector.run
     rescue => exc
       @log.error(([exc.to_s]+exc.backtrace).join("\n"))
     ensure
@@ -70,15 +100,14 @@ module Pwrake
     def setup_option
       @log.info @option.inspect
       @out.heartbeat = @option[:heartbeat]
-      @shell_cmd = @option[:shell_command]
-      @shell_rc = @option[:shell_rc] || []
       (@option[:pass_env]||{}).each do |k,v|
         ENV[k] = v
       end
     end
 
     def setup_loop
-      while line = get_line($stdin)
+      loop do
+        line = get_line(@rd)
         case line
         when /^(\d+):open$/o
           $1.split.each do |id|
@@ -92,12 +121,16 @@ module Pwrake
           end
         end
       end
-      raise RuntimeError,"incomplete setup_loop"
     end
 
-    def command_callback(rd)
-      while line = get_line(rd) # rd returns nil if line is incomplete
+    def command_callback
+      while line = get_line(@rd)
         case line
+        when /^(\d+):exit$/o
+          id = $1
+          ex = @ex_list.delete(id)
+          ex.close
+          ex.join
         when /^(\d+):(.*)$/o
           id,cmd = $1,$2
           @ex_list[id].execute(cmd.chomp)
@@ -105,7 +138,7 @@ module Pwrake
           break if common_line(line)
         end
       end
-      if rd.eof?
+      if @rd.eof?
         # connection lost
         raise RuntimeError,"lost connection to master"
       end
@@ -114,7 +147,6 @@ module Pwrake
     def common_line(line)
       case line
       when /^exit$/o
-        @selector.delete_reader($stdin)
         return true
         #
       when /^kill:(.*)$/o
@@ -129,7 +161,7 @@ module Pwrake
         return false
         #
       else
-        msg = "invalid line: #{line}"
+        msg = "invalid line: #{line.inspect}"
         @log.fatal msg
         raise RuntimeError,msg
       end
@@ -142,12 +174,7 @@ module Pwrake
       @ex_list.each_value{|ex| ex.close}
       @ex_list.each_value{|ex| ex.join}
       @log.info "worker:end:#{@ex_list.keys.inspect}"
-      begin
-        Timeout.timeout(20){@log.close}
-      rescue => e
-        $stdout.puts e
-        $stdout.puts e.backtrace.join("\n")
-      end
+      Timeout.timeout(20){@log.close}
     ensure
       @out.puts "exited"
     end