pwrake 0.9.9.2 → 2.0.0

data/lib/pwrake/queue/task_queue.rb

@@ -0,0 +1,164 @@
+module Pwrake
+
+  class TaskQueue
+
+    def initialize(host_map, group_map=nil)
+      @q = []
+      @empty = []
+
+      @enable_steal = true
+      @q_no_action = NoActionQueue.new
+
+      @host_map = host_map
+
+      pri = Rake.application.pwrake_options['QUEUE_PRIORITY'] || "LIHR"
+      case pri
+      when /prio/i
+        @array_class = PriorityQueueArray
+      when /fifo/i
+        @array_class = FifoQueueArray # Array # Fifo
+      when /lifo/i
+        @array_class = LifoQueueArray
+      when /lihr/i
+        @array_class = LifoHrfQueueArray
+      when /prhr/i
+        @array_class = PriorityHrfQueueArray
+      when /rank/i
+        @array_class = RankQueueArray
+      else
+        raise RuntimeError,"unknown option for QUEUE_PRIORITY: "+pri
+      end
+      Log.debug "@array_class=#{@array_class.inspect}"
+      init_queue(group_map)
+    end
+
+    def init_queue(group_map=nil)
+      @q_input = @array_class.new(0)
+      @q_no_input = FifoQueueArray.new
+      @n_turn = 1
+    end
+
+    attr_accessor :enable_steal
+
+    # enq
+    def enq(tw)
+      if tw.nil? || tw.actions.empty?
+        @q_no_action.push(tw)
+      else
+        enq_body(tw)
+      end
+    end
+
+    def enq_body(tw)
+      enq_impl(tw)
+    end
+
+    def enq_impl(tw)
+      if tw.has_input_file?
+        @q_input.push(tw)
+      else
+        @q_no_input.push(tw)
+      end
+    end
+
+    def deq_noaction_task(&block)
+      Log.debug "deq_task:"+(empty? ? " empty" : "\n#{inspect_q}")
+      while tw = @q_no_action.shift
+        Log.debug "deq_noaction: #{tw.name}"
+        yield(tw,nil)
+      end
+    end
+
+    def deq_task(&block) # locality version
+      Log.debug "deq_task:"+(empty? ? " empty" : "\n#{inspect_q}")
+      queued = 0
+      @n_turn.times do |turn|
+        next if turn_empty?(turn)
+        queued += deq_turn(turn,&block)
+      end
+      if queued>0
+        Log.debug "queued:#{queued}"
+      end
+    end
+
+    def deq_turn(turn,&block)
+      queued = 0
+      while true
+        count = 0
+        @host_map.by_id.each do |host_info|
+          #Log.debug "TaskQueue#deq_turn host_info=#{host_info.name}"
+          if host_info.idle_cores > 0
+          if turn_empty?(turn)
+            return queued
+          elsif tw = deq_impl(host_info,turn)
+            n_task_cores = tw.n_used_cores(host_info)
+            Log.debug "deq: #{tw.name} n_task_cores=#{n_task_cores}"
+            if host_info.idle_cores < n_task_cores
+              m = "task.n_used_cores=#{n_task_cores} must be "+
+                "<= host_info.idle_cores=#{host_info.idle_cores}"
+              Log.fatal m
+              raise RuntimeError,m
+            else
+              host_info.decrease(n_task_cores)
+              yield(tw,host_info.id)
+              count += 1
+              queued += 1
+            end
+          end
+          end
+        end
+        break if count == 0
+      end
+      queued
+    end
+
+    def turn_empty?(turn)
+      empty?
+    end
+
+    def deq_impl(host_info=nil, turn=nil)
+      @q_no_action.shift ||
+        @q_input.shift(host_info) ||
+        @q_no_input.shift(host_info)
+    end
+
+    def clear
+      @q_no_action.clear
+      @q_input.clear
+      @q_no_input.clear
+    end
+
+    def empty?
+      @q_no_action.empty? &&
+        @q_input.empty? &&
+        @q_no_input.empty?
+    end
+
+    def task_end(tw, hid)
+      host_info = @host_map.by_id[hid]
+      host_info.increase(tw.n_used_cores(host_info))
+    end
+
+    def _qstr(h,q)
+      s = " #{h}: size=#{q.size} "
+      case q.size
+      when 0
+        s << "[]\n"
+      when 1
+        s << "[#{q.first.name}]\n"
+      when 2
+        s << "[#{q.first.name}, #{q.last.name}]\n"
+      else
+        s << "[#{q.first.name},..,#{q.last.name}]\n"
+      end
+      s
+    end
+
+    def inspect_q
+      _qstr("noaction",@q_no_action) +
+      _qstr("input",   @q_input) +
+      _qstr("no_input",@q_no_input)
+    end
+
+  end
+end

data/lib/pwrake/report.rb

@@ -4,3 +4,4 @@ require 'pwrake/report/parallelism.rb'
 require 'pwrake/report/report.rb'
 require 'pwrake/report/report_multi.rb'
 require 'pwrake/report/stat.rb'
+require 'pwrake/report/task_stat.rb'

data/lib/pwrake/report/parallelism.rb

@@ -125,7 +125,7 @@ plot '#{fpara}' w l axis x1y1 title 'parallelism'
 
       density = exec_density(a)
 
-      fimg = base+'.png'
+      fimg = base+'/parallelism.png'
 
       n = a.size
       i = 0
@@ -152,6 +152,7 @@ plot '#{fpara}' w l axis x1y1 title 'parallelism'
 
       t_end = (a.last)[0]
 
+      if system("which gnuplot >/dev/null 2>&1")
       IO.popen("gnuplot","r+") do |f|
         f.print "
 set terminal png
@@ -178,6 +179,7 @@ plot '-' w l axis x1y1 title 'parallelism', '-' w l axis x1y2 title 'exec/sec'
           f.puts "#{t} #{d}"
         end
       end
+      end
 
       #puts "Parallelism plot: #{fimg}"
       fimg
@@ -288,8 +290,9 @@ plot '-' w l axis x1y1 title 'parallelism', '-' w l axis x1y2 title 'exec/sec'
         end
       end
 
-      fimg = base+'_para_cmd.png'
+      fimg = base+'/para_cmd.png'
 
+      if system("which gnuplot >/dev/null 2>&1")
       IO.popen("gnuplot","r+") do |f|
         #begin f = $stdout
         f.print "
@@ -308,6 +311,7 @@ set ylabel 'parallelism'
           f.puts "e"
         end
       end
+      end
 
       #puts "Parallelism plot: #{fimg}"
       fimg
@@ -335,7 +339,7 @@ set ylabel 'parallelism'
     end
 
     def plot_parallelism_by_host(csvtable,base)
-      fpng = base+"_para_host.png"
+      fpng = base+"/para_host.png"
       data = read_time_by_host_from_csv(csvtable)
       return fpng if data.size == 0
 
@@ -346,6 +350,7 @@ set ylabel 'parallelism'
         grid << a
       end
 
+      if system("which gnuplot >/dev/null 2>&1")
       IO.popen("gnuplot","r+") do |f|
         f.puts "
 set terminal png
@@ -379,6 +384,7 @@ set format y ''
         end
         f.printf "e\n"
       end
+      end
       fpng
     end
 

data/lib/pwrake/report/report.rb

@@ -30,28 +30,25 @@ EOL
     @@id = 0
     @@id_fmt = nil
 
-    def initialize(base,pattern)
-      @base = base
+    def initialize(option,pattern)
+      @dir = option['REPORT_DIR']
+      if !File.directory?(@dir)
+        raise ArgumentError,"Could not find log directory: #{@dir}"
+      end
       @pattern = pattern
 
       @@id = @@id.succ
       @id = @@id
+      @base = @dir
 
-      @csv_file = base+'.csv'
-      @task_file = base+'.task'
-      @html_file = base+'.html'
-
-      open(@base+".log","r").each do |s|
-        if /num_cores=(\d+)/ =~ s
-          @ncore = $1.to_i
-          break
-        end
-      end
+      @cmd_file = File.join(@dir,option['COMMAND_CSV_FILE'])
+      @task_file = File.join(@dir,option['TASK_CSV_FILE'])
+      @html_file = File.join(@dir,'report.html')
 
       begin
-        @sh_table = CSV.read(@csv_file,:headers=>true)
+        @sh_table = CSV.read(@cmd_file,:headers=>true,:skip_lines=>/\A#/)
       rescue
-        $stderr.puts "error in reading "+@csv_file
+        $stderr.puts "error in reading "+@cmd_file
         $stderr.puts $!, $@
         exit
       end
@@ -72,14 +69,27 @@ EOL
       make_cmd_stat
 
       @stat = TaskStat.new(@task_file,@sh_table)
+      @ncore = @stat.ncore
     end
 
     attr_reader :base, :ncore, :elap
-    attr_reader :csv_file, :html_file
+    attr_reader :cmd_file, :html_file
     attr_reader :cmd_elap, :cmd_stat
     attr_reader :sh_table, :task_table
     attr_reader :id
 
+    def find_single_file(pattern)
+      g = Dir.glob(File.join(@dir,pattern))
+      case g.size
+      when 0
+        raise ArgumentError, "Could not find any file with '#{pattern}' in #{@dir}"
+      when 1
+      else
+        raise ArgumentError, "Found multiple files '#{pattern}' in #{@dir}"
+      end
+      g[0]
+    end
+
     def id_str
       if @@id_fmt.nil?
         id_len = Math.log10(@@id).floor + 1
@@ -151,13 +161,18 @@ EOL
 
     def tr_count(x,y)
       sum = x+y
-      xp = x*100.0/sum
-      yp = y*100.0/sum
+      if sum==0
+        xp = "--%"
+        yp = "--%"
+      else
+        xp = "%.2f%%"%(x*100.0/sum)
+        yp = "%.2f%%"%(y*100.0/sum)
+      end
       td = "<td align='right' valign='top'>"
-      m  = td + '%s<br/>''%.2f%%</td>' % [format_comma(x),xp]
-      m << td + '%s<br/>''%.2f%%</td>' % [format_comma(y),yp]
-      m << td + "%s</td>" % format_comma(sum)
-      m
+      return \
+        td + '%s<br/>%s</td>' % [format_comma(x),xp] +
+        td + '%s<br/>%s</td>' % [format_comma(y),yp] +
+        td + "%s</td>" % format_comma(sum)
     end
 
     def report_html
@@ -227,15 +242,17 @@ EOL
       html << "<th>local</th><th>remote</th><th>total</th>"
       html << "</tr>\n"
       @stat.exec_hosts.each do |h|
-        html << "<tr><td>#{h}</td>"
-        html << "<td align='right'>%.3f</td>" % @stat[h,nil,:elap]
-        html << "<td></td>"
-        html << tr_count(@stat[h,true,:in_num],@stat[h,false,:in_num])
-        html << tr_count(@stat[h,true,:in_size],@stat[h,false,:in_size])
-        html << "<td></td>"
-        html << tr_count(@stat[h,true,:out_num],@stat[h,false,:out_num])
-        html << tr_count(@stat[h,true,:out_size],@stat[h,false,:out_size])
-        html << "</tr>\n"
+        if h.to_s!="" || @stat[h,nil,:elap]!=0
+          html << "<tr><td>#{h}</td>"
+          html << "<td align='right'>%.3f</td>" % @stat[h,nil,:elap]
+          html << "<td></td>"
+          html << tr_count(@stat[h,true,:in_num],@stat[h,false,:in_num])
+          html << tr_count(@stat[h,true,:in_size],@stat[h,false,:in_size])
+          html << "<td></td>"
+          html << tr_count(@stat[h,true,:out_num],@stat[h,false,:out_num])
+          html << tr_count(@stat[h,true,:out_size],@stat[h,false,:out_size])
+          html << "</tr>\n"
+        end
       end
       html << "<tr><td>total</td>"
       html << "<td align='right'>%.3f</td>" % @stat.total(nil,:elap)
@@ -266,7 +283,8 @@ EOL
           command_list << cmd
         end
       end
-      hist_image = @base+"_hist.png"
+      hist_image = @base+"/hist.png"
+      if system("which gnuplot >/dev/null 2>&1")
       IO.popen("gnuplot","r+") do |f|
         f.puts "
 set terminal png # size 480,360
@@ -293,80 +311,9 @@ set title 'histogram of elapsed time'"
             f.puts "e"
           end
         end
-        hist_image
-      end
-    end
-  end
-
-  class TaskStat
-
-    def initialize(task_file, sh_table)
-      begin
-        @task_table = CSV.read(task_file,:headers=>true)
-      rescue
-        $stderr.puts "error in reading "+task_file
-        $stderr.puts $!, $@
-        exit
       end
-      @count = Hash.new(0)
-      task_locality
-      stat_sh_table(sh_table)
-    end
-
-    attr_reader :exec_hosts
-
-    def count(exec_host, loc, key, val)
-      @count[[exec_host,loc,key]] += val
-      @count[[loc,key]] += val
-    end
-
-    def total(loc,key)
-      @count[[loc,key]]
-    end
-
-    def [](exec_host,loc,key)
-      @count[[exec_host,loc,key]]
-    end
-
-    def task_locality
-      file_size = {}
-      file_host = {}
-      h = {}
-      @task_table.each do |row|
-        name            = row['task_name']
-        file_size[name] = row['file_size'].to_i
-        file_host[name] = (row['file_host']||'').split('|')
-        h[row['exec_host']] = true
-      end
-      @exec_hosts = h.keys.sort
-
-      @task_table.each do |row|
-        if row['executed']=='1'
-          name      = row['task_name']
-          exec_host = row['exec_host']
-          loc = file_host[name].include?(exec_host)
-          count(exec_host, loc, :out_num, 1)
-          count(exec_host, loc, :out_size, file_size[name])
-
-          preq_files = (row['preq']||'').split('|')
-          preq_files.each do |preq|
-            sz = file_size[preq]
-            if sz && sz > 0
-              loc = file_host[preq].include?(exec_host)
-              count(exec_host, loc, :in_num, 1)
-              count(exec_host, loc, :in_size, sz)
-            end
-          end
-        end
-      end
-    end
-
-    def stat_sh_table(sh_table)
-      sh_table.each do |row|
-        if (h = row['host']) && (t = row['elap_time'])
-          count(h, nil, :elap, t.to_f)
-        end
       end
+      hist_image
     end
 
   end

data/lib/pwrake/report/task_stat.rb

@@ -0,0 +1,83 @@
+module Pwrake
+
+  class TaskStat
+
+    def initialize(task_file, sh_table)
+      begin
+        @task_table = CSV.read(task_file,:headers=>true,:skip_lines=>/\A#/)
+      rescue
+        $stderr.puts "error in reading "+task_file
+        $stderr.puts $!, $@
+        exit
+      end
+      shell_id = {}
+      @task_table.each do |row|
+        if id=row['shell_id']
+          shell_id[id.to_i] = true
+        end
+      end
+      @ncore = shell_id.size
+      @count = Hash.new(0)
+      task_locality
+      stat_sh_table(sh_table)
+    end
+
+    attr_reader :exec_hosts, :ncore
+
+    def count(exec_host, loc, key, val)
+      @count[[exec_host,loc,key]] += val
+      @count[[loc,key]] += val
+    end
+
+    def total(loc,key)
+      @count[[loc,key]]
+    end
+
+    def [](exec_host,loc,key)
+      @count[[exec_host,loc,key]]
+    end
+
+    def task_locality
+      file_size = {}
+      file_host = {}
+      h = {}
+      @task_table.each do |row|
+        name            = row['task_name']
+        file_size[name] = row['file_size'].to_i
+        file_host[name] = (row['file_host']||'').split('|')
+        exec_host = row['exec_host'] || ""
+        h[exec_host] = true
+      end
+      @exec_hosts = h.keys.sort
+
+      @task_table.each do |row|
+        if row['executed']=='1'
+          name      = row['task_name']
+          exec_host = row['exec_host']
+          loc = file_host[name].include?(exec_host)
+          count(exec_host, loc, :out_num, 1)
+          count(exec_host, loc, :out_size, file_size[name])
+
+          preq_files = (row['preq']||'').split('|')
+          preq_files.each do |preq|
+            sz = file_size[preq]
+            if sz && sz > 0
+              loc = file_host[preq].include?(exec_host)
+              count(exec_host, loc, :in_num, 1)
+              count(exec_host, loc, :in_size, sz)
+            end
+          end
+        end
+      end
+    end
+
+    def stat_sh_table(sh_table)
+      sh_table.each do |row|
+        if (h = row['host']) && (t = row['elap_time'])
+          count(h, nil, :elap, t.to_f)
+        end
+      end
+    end
+
+  end
+end