RubyGems - pwrake - Versions diffs - 0.9.5 → 0.9.6 - Mend

pwrake 0.9.5 → 0.9.6

Files changed (25) hide show

checksums.yaml +7 -0
data/README.md +83 -4
data/lib/pwrake/application.rb +34 -0
data/lib/pwrake/file_utils.rb +43 -16
data/lib/pwrake/gfarm_feature.rb +79 -40
data/lib/pwrake/locality_aware_queue.rb +141 -111
data/lib/pwrake/logger.rb +7 -1
data/lib/pwrake/master.rb +21 -2
data/lib/pwrake/option.rb +106 -56
data/lib/pwrake/profiler.rb +44 -12
data/lib/pwrake/report/parallelism.rb +262 -0
data/lib/pwrake/report/report.rb +355 -0
data/lib/pwrake/report/report_multi.rb +196 -0
data/lib/pwrake/report/stat.rb +115 -0
data/lib/pwrake/report.rb +6 -0
data/lib/pwrake/shell.rb +30 -10
data/lib/pwrake/task_algorithm.rb +174 -50
data/lib/pwrake/task_queue.rb +130 -50
data/lib/pwrake/version.rb +1 -1
data/spec/004/Rakefile +1 -1
data/spec/009/pwrake_conf.yaml +1 -0
data/spec/011/Rakefile +15 -0
data/spec/helper.rb +7 -3
data/spec/pwrake_spec.rb +12 -0
metadata +15 -9

data/lib/pwrake/report/report.rb ADDED Viewed

@@ -0,0 +1,355 @@
+module Pwrake
+  class Report
+    HTML_HEAD = <<EOL
+<html><head><style>
+<!--
+h2 {
+  background-color:#eee;
+}
+h1 {
+  background-color:#0ff;
+}
+table {
+ margin:0px;
+ border-style:solid;
+ border-width:1px;
+}
+td {
+  margin:0px;
+  border-style:solid;
+  border-width:1px;
+}
+-->
+</style>
+</head>
+<body>
+EOL
+    @@id = 0
+    @@id_fmt = nil
+    def initialize(base,pattern)
+      @base = base
+      @pattern = pattern
+      @@id = @@id.succ
+      @id = @@id
+      @csv_file = base+'.csv'
+      @task_file = base+'.task'
+      @html_file = base+'.html'
+      open(@base+".log","r").each do |s|
+        if /num_cores=(\d+)/ =~ s
+          @ncore = $1.to_i
+          break
+        end
+      end
+      @sh_table = CSV.read(@csv_file,:headers=>true)
+      h = {}
+      @sh_table.each do |row|
+        if host = row['host']
+          h[host] = true
+        end
+      end
+      @hosts = h.keys.sort
+      @start_time = Time.parse(@sh_table[0]["start_time"])
+      @end_time = Time.parse(@sh_table[-1]["start_time"])
+      @elap = @end_time - @start_time
+      read_elap_each_cmd
+      make_cmd_stat
+    end
+    attr_reader :base, :ncore, :elap
+    attr_reader :csv_file, :html_file
+    attr_reader :cmd_elap, :cmd_stat
+    attr_reader :sh_table, :task_table
+    attr_reader :id
+    def id_str
+      if @@id_fmt.nil?
+        id_len = Math.log10(@@id).floor + 1
+        @@id_fmt = "#%0#{id_len}d"
+      end
+      @@id_fmt % @id
+    end
+    def read_elap_each_cmd
+      @cmd_elap = {}
+      @sh_table.each do |row|
+        command = row['command']
+        elap = row['elap_time']
+        if command && elap
+          elap = elap.to_f
+          found = nil
+          @pattern.each do |cmd,regex|
+            if regex =~ command
+              if a = @cmd_elap[cmd]
+                a << elap
+              else
+                @cmd_elap[cmd] = [elap]
+              end
+              found = true
+            end
+          end
+          if !found
+            if cmd = get_command( command )
+              if a = @cmd_elap[cmd]
+                a << elap
+              else
+                @cmd_elap[cmd] = [elap]
+              end
+            end
+          end
+        end
+      end
+      @cmd_elap
+    end
+    def get_command(s)
+      if /\(([^()]+)\)/ =~ s
+        s = $1
+      end
+      a = s.split(/;/)
+      a.each do |x|
+        if /^\s*(\S+)/ =~ x
+          k = $1
+          next if k=='cd'
+          return k
+        end
+      end
+      nil
+    end
+    def make_cmd_stat
+      @cmd_stat = {}
+      @cmd_elap.each do |cmd,elap|
+        @cmd_stat[cmd] = s = Stat.new(elap)
+        if elap.size > 1
+          s.make_logx_histogram(1.0/8)
+        end
+      end
+    end
+    def format_comma(x)
+      x.to_s.gsub(/(?<=\d)(?=(?:\d\d\d)+(?!\d))/, ',')
+    end
+    def tr_count(x,y)
+      sum = x+y
+      td = "<td align='right' valign='top'>"
+      m  = td + "%s<br/>%.2f%%</td>" % [format_comma(x),x*100.0/sum]
+      m << td + "%s<br/>%.2f%%</td>" % [format_comma(y),y*100.0/sum]
+      m << td + "%s</td>" % format_comma(sum)
+      m
+    end
+    def report_html
+      html = HTML_HEAD + "<body><h1>Pwrake Statistics</h1>\n"
+      html << "<h2>Workflow</h2>\n"
+      html << "<table>\n"
+      html << "<tr><th>log file</th><td>#{@base}</td><tr>\n"
+      html << "<tr><th>ncore</th><td>#{@ncore}</td><tr>\n"
+      html << "<tr><th>elapsed time(sec)</th><td>#{@elap}</td><tr>\n"
+      html << "<tr><th>start time</th><td>#{@start_time}</td><tr>\n"
+      html << "<tr><th>end time</th><td>#{@end_time}</td><tr>\n"
+      html << "</table>\n"
+      html << "<table>\n"
+      html << "<tr><th>hosts</th><tr>\n"
+      @hosts.each do |h|
+        html << "<tr><td>#{h}</td><tr>\n"
+      end
+      html << "</table>\n"
+      html << "<h2>Parallelism</h2>\n"
+      fimg = Parallelism.plot_parallelism2(@csv_file)
+      html << "<img src='#{fimg}' align='top'/></br>\n"
+      html << "<h2>Parallelism by host</h2>\n"
+      fimg2 = Parallelism.plot_parallelizm_by_host(@sh_table,@base)
+      html << "<img src='#{fimg2}' align='top'/></br>\n"
+      html << "<h2>Command statistics</h2>\n"
+      html << "<table>\n"
+      html << "<tr><th>command</th>"
+      html << Stat.html_th
+      html << "</tr>\n"
+      @cmd_stat.each do |cmd,stat|
+        html << "<tr><td>#{cmd}</td>"
+        html << stat.html_td
+        html << "</tr>\n"
+      end
+      html << "<table>\n"
+      html << "<img src='#{histogram_plot}' align='top'/></br>\n"
+      task_locality
+      html << "<h2>Locality statistics</h2>\n"
+      html << "<table>\n"
+      html << "<tr><th></th><th rowspan=3>gross elapsed time (sec)</th><th></th>"
+      html << "<th colspan=6>read</th>"
+      html << "<th></th>"
+      html << "<th colspan=6>write</th>"
+      html << "</tr>\n"
+      html << "<tr><th></th><th></th>"
+      html << "<th colspan=3>count</th><th colspan=3>file size (bytes)</th>"
+      html << "<th></th>"
+      html << "<th colspan=3>count</th><th colspan=3>file size (bytes)</th>"
+      html << "</tr>\n"
+      html << "<tr><th>host</th><th></th>"
+      html << "<th>local</th><th>remote</th><th>total</th>"
+      html << "<th>local</th><th>remote</th><th>total</th>"
+      html << "<th></th>"
+      html << "<th>local</th><th>remote</th><th>total</th>"
+      html << "<th>local</th><th>remote</th><th>total</th>"
+      html << "</tr>\n"
+      n_same_input     = 0
+      size_same_input  = 0
+      n_diff_input     = 0
+      size_diff_input  = 0
+      n_same_output    = 0
+      size_same_output = 0
+      n_diff_output    = 0
+      size_diff_output = 0
+      elap_host = 0
+      @exec_hosts.each do |h|
+        html << "<tr><td>#{h}</td>"
+        html << "<td align='right'>%.3f</td>" % @elap_host[h]
+        html << "<td></td>"
+        html << tr_count(@n_same_input[h],@n_diff_input[h])
+        html << tr_count(@size_same_input[h],@size_diff_input[h])
+        html << "<td></td>"
+        html << tr_count(@n_same_output[h],@n_diff_output[h])
+        html << tr_count(@size_same_output[h],@size_diff_output[h])
+        html << "</tr>\n"
+        n_same_input     += @n_same_input[h]
+        size_same_input  += @size_same_input[h]
+        n_diff_input     += @n_diff_input[h]
+        size_diff_input  += @size_diff_input[h]
+        n_same_output    += @n_same_output[h]
+        size_same_output += @size_same_output[h]
+        n_diff_output    += @n_diff_output[h]
+        size_diff_output += @size_diff_output[h]
+        elap_host += @elap_host[h]
+      end
+      html << "<tr><td>total</td>"
+      html << "<td align='right'>%.3f</td>" % elap_host
+      html << "<td></td>"
+      html << tr_count(n_same_input,n_diff_input)
+      html << tr_count(size_same_input,size_diff_input)
+      html << "<td></td>"
+      html << tr_count(n_same_output,n_diff_output)
+      html << tr_count(size_same_output,size_diff_output)
+      html << "</tr>\n"
+      html << "<table>\n"
+      html << "</body></html>\n"
+      File.open(@html_file,"w") do |f|
+        f.puts html
+      end
+      puts "generate "+@html_file
+    end
+    def task_locality
+      @task_table = CSV.read(@task_file,:headers=>true)
+      file_size = {}
+      file_host = {}
+      @task_table.each do |row|
+        name = row['task_name']
+        file_size[name] = row['file_size'].to_i
+        file_host[name] = (row['file_host']||'').split('|')
+      end
+      @n_same_output   = Hash.new(0)
+      @size_same_output= Hash.new(0)
+      @n_diff_output   = Hash.new(0)
+      @size_diff_output= Hash.new(0)
+      @n_same_input    = Hash.new(0)
+      @size_same_input = Hash.new(0)
+      @n_diff_input    = Hash.new(0)
+      @size_diff_input = Hash.new(0)
+      h = {}
+      @task_table.each do |row|
+        if row['executed']=='1'
+          name = row['task_name']
+          exec_host = row['exec_host']
+          h[exec_host] = true
+          if file_host[name].include?(exec_host)
+            @n_same_output[exec_host] += 1
+            @size_same_output[exec_host] += file_size[name]
+          else
+            @n_diff_output[exec_host] += 1
+            @size_diff_output[exec_host] += file_size[name]
+          end
+          preq_files = (row['preq']||'').split('|')
+          preq_files.each do |preq|
+            if (sz=file_size[preq]) && sz > 0
+              if file_host[preq].include?(exec_host)
+                @n_same_input[exec_host] += 1
+                @size_same_input[exec_host] += sz
+              else
+                @n_diff_input[exec_host] += 1
+                @size_diff_input[exec_host] += sz
+              end
+            end
+          end
+        end
+      end
+      @exec_hosts = h.keys.sort
+      @elap_host = Hash.new(0)
+      @sh_table.each do |row|
+        if (h = row['host']) && (t = row['elap_time'])
+          @elap_host[h] += t.to_f
+        end
+      end
+    end
+    def histogram_plot
+      command_list = []
+      @cmd_stat.each do |cmd,stat|
+        if stat.n > 2
+          command_list << cmd
+        end
+      end
+      hist_image = @base+"_hist.png"
+      IO.popen("gnuplot","r+") do |f|
+        f.puts "
+set terminal png # size 480,360
+set output '#{hist_image}'
+set ylabel 'histogram'
+set xlabel 'Execution time (sec)'
+set logscale x
+set title 'histogram of elapsed time'"
+        a = []
+        command_list.each_with_index do |n,i|
+          a << "'-' w histeps ls #{i+1} title ''"
+          a << "'-' w lines ls #{i+1} title '#{n}'"
+        end
+        f.puts "plot "+ a.join(',')
+        command_list.each do |cmd|
+          stat = @cmd_stat[cmd]
+          2.times do
+            stat.hist_each do |x1,x2,y|
+              x = Math.sqrt(x1*x2)
+              f.printf "%f %d\n", x, y
+            end
+            f.puts "e"
+          end
+        end
+        hist_image
+      end
+    end
+  end
+end

data/lib/pwrake/report/report_multi.rb ADDED Viewed

@@ -0,0 +1,196 @@
+module Pwrake
+  class ReportMulti
+    def initialize(list,pattern)
+      @reports = list.map do |base|
+        r = Report.new(base,pattern)
+        puts r.base+" elap=#{r.elap}"
+        r
+      end
+      @pattern = pattern
+      @elap_png = 'elap.png'
+    end
+    def report(stat_html)
+      if true
+        @reports.each do |r|
+          r.report_html
+        end
+        plot_elap
+      end
+      html = Report::HTML_HEAD + "<body><h1>Pwrake Statistics</h1>\n"
+      html << "<h2>Log files</h2>\n"
+      html << "<table>\n"
+      html << "<tr><th>log file</th><th>id</th><th>ncore</th><th>elapsed time(sec)</th><tr>\n"
+      @reports.each do |r|
+        html << "<tr><td><a href='#{r.html_file}'>#{r.base}</a></td>"
+        html << "<td>#{r.id_str}</td><td>#{r.ncore}</td><td>#{r.elap}</td><tr>\n"
+      end
+      html << "</table>\n"
+      html << "<h2>Elapsed time</h2>\n"
+      html << "<img src='#{@elap_png}'  align='top'/></br>\n"
+      html << "<h2>Histogram of Execution time</h2>\n"
+      html << report_histogram()
+      html << "</body></html>\n"
+      File.open(stat_html,"w") do |f|
+        f.puts html
+      end
+    end
+    def plot_elap
+      a = @reports.map{|r| r.ncore * r.elap}.min
+      elaps = @reports.map{|r| r.elap}
+      logmin = Math.log10(elaps.min)
+      logmax = Math.log10(elaps.max)
+      mid = (logmin+logmax)/2
+      wid = (logmax-logmin).ceil*0.5
+      ymin = 10**(mid-wid)
+      ymax = 10**(mid+wid)
+      IO.popen("gnuplot","r+") do |f|
+        f.puts "
+set terminal png size 640,480
+set output '#{@elap_png}'
+set xlabel 'ncore'
+set ylabel 'time (sec)'
+set yrange [#{ymin}:#{ymax}]
+set logscale xy
+plot #{a}/x,'-' w lp lw 2 ps 2 title 'elapsed time'
+"
+        @reports.sort_by{|r| r.ncore}.each do |r|
+          f.puts "#{r.ncore} #{r.elap}"
+        end
+        f.puts "e"
+      end
+      puts "Ncore-time plot: "+@elap_png
+    end
+    def report_histogram
+      @images = {}
+      @cmd_rep = {}
+      @reports.each do |r|
+        r.cmd_stat.each do |cmd,stat|
+          if stat.n > 2
+            @cmd_rep[cmd] ||= {}
+            @cmd_rep[cmd][r.id_str] = r # stat
+          end
+        end
+      end
+      @cmd_rep.each_key do |cmd|
+        @images[cmd] = 'hist_'+cmd.gsub(/[\/.]/,'_')+'.png'
+      end
+      histogram_plot
+      histogram_html
+    end
+    def histogram_html
+      html = ""
+      @cmd_rep.each do |cmd,cmd_rep|
+        html << "<p>Statistics of Elapsed time of #{cmd}</p>\n<table>\n"
+        html << "<th>id</th><th>ncore</th>"+Stat.html_th
+        cmd_rep.each do |id,r|
+          s = r.cmd_stat[cmd]
+          html << "<tr><td>#{id}</td><td>#{r.ncore}</td>" + s.html_td + "</tr>\n"
+        end
+        html << "</table>\n"
+        html << "<img src='./#{@images[cmd]}'/>\n"
+      end
+      html
+    end
+    def histogram_plot
+      @cmd_rep.each do |cmd,cmd_rep|
+        IO.popen("gnuplot","r+") do |f|
+          f.puts "
+set terminal png # size 480,360
+set output '#{@images[cmd]}'
+set ylabel 'histogram'
+set xlabel 'Execution time (sec)'
+set logscale x
+set title '#{cmd}'"
+          a = []
+          ncores = cmd_rep.keys
+          ncores.each_with_index{|n,i|
+            a << "'-' w histeps ls #{i+1} title ''"
+            a << "'-' w lines ls #{i+1} title '#{n}'"
+          }
+          f.puts "plot "+ a.join(',')
+          cmd_rep.each do |ncore,r|
+            s = r.cmd_stat[cmd]
+            2.times do
+              s.hist_each do |x1,x2,y|
+                x = Math.sqrt(x1*x2)
+                f.printf "%f %d\n", x, y
+              end
+              f.puts "e"
+            end
+          end
+        end
+        puts "Histogram plot: #{@images[cmd]}"
+      end
+    end
+    def histogram_plot2
+      @cmd_rep.each do |cmd,cmd_rep|
+        IO.popen("gnuplot","r+") do |f|
+          f.puts "
+set terminal png # size 480,360
+set output '#{@images[cmd]}'
+set nohidden3d
+set palette rgb 33,13,10
+set pm3d
+set ticslevel 0
+unset colorbox
+set yrange [#{cmd_rep.size}:0]
+set logscale x
+set title '#{cmd}'"
+          a = []
+          ncores = cmd_rep.keys.sort
+          ncores.each_with_index{|n,i|
+            a << "'-' w lines ls #{i+1} title '#{n} cores'"
+          }
+          f.puts "splot "+ a.join(',')
+          ncores.each_with_index do |ncore,i|
+            s = cmd_rep[ncore]
+            y = i
+            s.hist_each do |x1,x2,z|
+              f.printf "%g %g 0\n", x1,y
+              f.printf "%g %g 0\n", x2,y
+              f.printf "%g %g 0\n", x2,y
+            end
+            f.puts ""
+            s.hist_each do |x1,x2,z|
+              f.printf "%g %g %g\n", x1,y,z
+              f.printf "%g %g %g\n", x2,y,z
+              f.printf "%g %g 0\n", x2,y,z
+            end
+            f.puts ""
+            y = i+1
+            s.hist_each do |x1,x2,z|
+              f.printf "%g %g %g\n", x1,y,z
+              f.printf "%g %g %g\n", x2,y,z
+              f.printf "%g %g 0\n", x2,y,z
+            end
+            f.puts ""
+            s.hist_each do |x1,x2,z|
+              f.printf "%g %g 0\n", x1,y
+              f.printf "%g %g 0\n", x2,y
+              f.printf "%g %g 0\n", x2,y
+            end
+            f.puts "e"
+            i = i+1
+          end
+        end
+        puts "Histogram plot: #{@images[cmd]}"
+      end
+    end
+  end
+end

data/lib/pwrake/report/stat.rb ADDED Viewed

@@ -0,0 +1,115 @@
+module Pwrake
+  class Stat
+    def initialize(data)
+      @data = data
+      @n = data.size
+      if @n>0
+        @min = data.min
+        @max = data.max
+        @sum = data.inject(0){|s,x| s+x}
+        @mean = @sum/@n
+        @median = _median
+        @mean_absolute_deviation = data.inject(0){|s,x| (x-@mean).abs} / @n
+        if @n>1
+          @variance = data.inject(0){|s,x| y=x-@mean; y**2} / (@n-1)
+          @sdev = Math.sqrt(@variance)
+          @skew = data.inject(0){|s,x| y=(x-@mean)/@sdev; y**3} / @n
+          @kurt = data.inject(0){|s,x| y=(x-@mean)/@sdev; y**4} / @n - 3
+        end
+      end
+    end
+    attr_reader :n
+    attr_reader :min, :max, :sum, :mean, :median
+    attr_reader :mean_absolute_deviation
+    attr_reader :variance, :sdev, :skew, :kurt
+    attr_reader :hist, :hist_min, :hist_max, :bin
+    def make_logx_histogram(bin)
+      @bin = bin # 1.0/10
+      @i_max = (Math.log10(@max)/@bin).floor
+      @i_min = (Math.log10(@min)/@bin).floor
+      @hist_min = 10**(@i_min * @bin)
+      @hist_max = 10**((@i_max+1) * @bin)
+      @hist = Array.new(@i_max-@i_min+1,0)
+      @data.each do |x|
+        i = (Math.log10(x)/@bin-@i_min).floor
+        raise "invalid index i=#{i}" if i<0 || i>@i_max-@i_min
+        @hist[i] += 1
+      end
+    end
+    def hist_each
+      if @hist
+        n = @hist.size
+        n.times do |i|
+          x1 = 10**(@bin*(i+@i_min))
+          x2 = 10**(@bin*(i+1+@i_min))
+          y  = @hist[i]
+          yield x1,x2,y
+        end
+      end
+    end
+    def _median
+      if @n==1
+        @data[0]
+      elsif @n==2
+        @mean
+      else
+        case @n%2
+        when 1
+          i = (@n-1)/2
+          @data.sort[i]
+        else
+          i = @n/2
+          s = @data.sort
+          (s[i]+s[i+1])/2
+        end
+      end
+    end
+    def report
+      case @n
+      when 0
+        "no data"
+      when 1
+        "n=1 mean=%g"%@mean
+      else
+        "n=%i mean=%g median=%g min=%g max=%g sdev=%g skew=%g kurt=%g" %
+          [@n, @mean, @median, @min, @max, @sdev||0, @skew||0, @kurt||0]
+        "n=%i mean=%g median=%g min=%g max=%g sdev=%g" %
+          [@n, @mean, @median, @min, @max, @sdev||0]
+      end
+    end
+    def stat_array
+      [@n, @mean, @median, @min, @max, @sdev||0]
+    end
+    def html_td
+      "<td>%i</td><td>%g</td><td>%g</td><td>%g</td><td>%g</td><td>%g</td>" %
+        [@n, @mean, @median, @min, @max, @sdev||0]
+    end
+    def self.html_th
+      "<th>%s</th><th>%s</th><th>%s</th><th>%s</th><th>%s</th><th>%s</th>" %
+      %w[n mean median min max sdev]
+    end
+    def report2
+      case @n
+      when 0
+        "  no data"
+      when 1
+        "  n=1\n  mean=%g"%@mean
+      else
+        "  n=%i\n  mean=%g\n  median=%g\n  min=%g\n  max=%g\n  sdev=%g\n  skew=%g\n  kurt=%g" %
+          [@n, @mean, @median, @min, @max, @sdev||0, @skew||0, @kurt||0]
+      end
+    end
+  end
+end

data/lib/pwrake/report.rb ADDED Viewed

@@ -0,0 +1,6 @@
+require 'time'
+require 'csv'
+require 'pwrake/report/parallelism.rb'
+require 'pwrake/report/report.rb'
+require 'pwrake/report/report_multi.rb'
+require 'pwrake/report/stat.rb'