RubyGems - pwrake - Versions diffs - 0.9.5 → 0.9.6 - Mend

pwrake 0.9.5 → 0.9.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (25) hide show

checksums.yaml +7 -0
data/README.md +83 -4
data/lib/pwrake/application.rb +34 -0
data/lib/pwrake/file_utils.rb +43 -16
data/lib/pwrake/gfarm_feature.rb +79 -40
data/lib/pwrake/locality_aware_queue.rb +141 -111
data/lib/pwrake/logger.rb +7 -1
data/lib/pwrake/master.rb +21 -2
data/lib/pwrake/option.rb +106 -56
data/lib/pwrake/profiler.rb +44 -12
data/lib/pwrake/report/parallelism.rb +262 -0
data/lib/pwrake/report/report.rb +355 -0
data/lib/pwrake/report/report_multi.rb +196 -0
data/lib/pwrake/report/stat.rb +115 -0
data/lib/pwrake/report.rb +6 -0
data/lib/pwrake/shell.rb +30 -10
data/lib/pwrake/task_algorithm.rb +174 -50
data/lib/pwrake/task_queue.rb +130 -50
data/lib/pwrake/version.rb +1 -1
data/spec/004/Rakefile +1 -1
data/spec/009/pwrake_conf.yaml +1 -0
data/spec/011/Rakefile +15 -0
data/spec/helper.rb +7 -3
data/spec/pwrake_spec.rb +12 -0
metadata +15 -9

data/lib/pwrake/report/report.rb ADDED Viewed

@@ -0,0 +1,355 @@
+module Pwrake
+  class Report
+    HTML_HEAD = <<EOL
+<html><head><style>
+<!--
+h2 {
+  background-color:#eee;
+}
+h1 {
+  background-color:#0ff;
+}
+table {
+ margin:0px;
+ border-style:solid;
+ border-width:1px;
+}
+td {
+  margin:0px;
+  border-style:solid;
+  border-width:1px;
+}
+-->
+</style>
+</head>
+<body>
+EOL
+    @@id = 0
+    @@id_fmt = nil
+    def initialize(base,pattern)
+      @base = base
+      @pattern = pattern
+      @@id = @@id.succ
+      @id = @@id
+      @csv_file = base+'.csv'
+      @task_file = base+'.task'
+      @html_file = base+'.html'
+      open(@base+".log","r").each do |s|
+        if /num_cores=(\d+)/ =~ s
+          @ncore = $1.to_i
+          break
+        end
+      end
+      @sh_table = CSV.read(@csv_file,:headers=>true)
+      h = {}
+      @sh_table.each do |row|
+        if host = row['host']
+          h[host] = true
+        end
+      end
+      @hosts = h.keys.sort
+      @start_time = Time.parse(@sh_table[0]["start_time"])
+      @end_time = Time.parse(@sh_table[-1]["start_time"])
+      @elap = @end_time - @start_time
+      read_elap_each_cmd
+      make_cmd_stat
+    end
+    attr_reader :base, :ncore, :elap
+    attr_reader :csv_file, :html_file
+    attr_reader :cmd_elap, :cmd_stat
+    attr_reader :sh_table, :task_table
+    attr_reader :id
+    def id_str
+      if @@id_fmt.nil?
+        id_len = Math.log10(@@id).floor + 1
+        @@id_fmt = "#%0#{id_len}d"
+      end
+      @@id_fmt % @id
+    end
+    def read_elap_each_cmd
+      @cmd_elap = {}
+      @sh_table.each do |row|
+        command = row['command']
+        elap = row['elap_time']
+        if command && elap
+          elap = elap.to_f
+          found = nil
+          @pattern.each do |cmd,regex|
+            if regex =~ command
+              if a = @cmd_elap[cmd]
+                a << elap
+              else
+                @cmd_elap[cmd] = [elap]
+              end
+              found = true
+            end
+          end
+          if !found
+            if cmd = get_command( command )
+              if a = @cmd_elap[cmd]
+                a << elap
+              else
+                @cmd_elap[cmd] = [elap]
+              end
+            end
+          end
+        end
+      end
+      @cmd_elap
+    end
+    def get_command(s)
+      if /\(([^()]+)\)/ =~ s
+        s = $1
+      end
+      a = s.split(/;/)
+      a.each do |x|
+        if /^\s*(\S+)/ =~ x
+          k = $1
+          next if k=='cd'
+          return k
+        end
+      end
+      nil
+    end
+    def make_cmd_stat
+      @cmd_stat = {}
+      @cmd_elap.each do |cmd,elap|
+        @cmd_stat[cmd] = s = Stat.new(elap)
+        if elap.size > 1
+          s.make_logx_histogram(1.0/8)
+        end
+      end
+    end
+    def format_comma(x)
+      x.to_s.gsub(/(?<=\d)(?=(?:\d\d\d)+(?!\d))/, ',')
+    end
+    def tr_count(x,y)
+      sum = x+y
+      td = "<td align='right' valign='top'>"
+      m  = td + "%s<br/>%.2f%%</td>" % [format_comma(x),x*100.0/sum]
+      m << td + "%s<br/>%.2f%%</td>" % [format_comma(y),y*100.0/sum]
+      m << td + "%s</td>" % format_comma(sum)
+      m
+    end
+    def report_html
+      html = HTML_HEAD + "<body><h1>Pwrake Statistics</h1>\n"
+      html << "<h2>Workflow</h2>\n"
+      html << "<table>\n"
+      html << "<tr><th>log file</th><td>#{@base}</td><tr>\n"
+      html << "<tr><th>ncore</th><td>#{@ncore}</td><tr>\n"
+      html << "<tr><th>elapsed time(sec)</th><td>#{@elap}</td><tr>\n"
+      html << "<tr><th>start time</th><td>#{@start_time}</td><tr>\n"
+      html << "<tr><th>end time</th><td>#{@end_time}</td><tr>\n"
+      html << "</table>\n"
+      html << "<table>\n"
+      html << "<tr><th>hosts</th><tr>\n"
+      @hosts.each do |h|
+        html << "<tr><td>#{h}</td><tr>\n"
+      end
+      html << "</table>\n"
+      html << "<h2>Parallelism</h2>\n"
+      fimg = Parallelism.plot_parallelism2(@csv_file)
+      html << "<img src='#{fimg}' align='top'/></br>\n"
+      html << "<h2>Parallelism by host</h2>\n"
+      fimg2 = Parallelism.plot_parallelizm_by_host(@sh_table,@base)
+      html << "<img src='#{fimg2}' align='top'/></br>\n"
+      html << "<h2>Command statistics</h2>\n"
+      html << "<table>\n"
+      html << "<tr><th>command</th>"
+      html << Stat.html_th
+      html << "</tr>\n"
+      @cmd_stat.each do |cmd,stat|
+        html << "<tr><td>#{cmd}</td>"
+        html << stat.html_td
+        html << "</tr>\n"
+      end
+      html << "<table>\n"
+      html << "<img src='#{histogram_plot}' align='top'/></br>\n"
+      task_locality
+      html << "<h2>Locality statistics</h2>\n"
+      html << "<table>\n"
+      html << "<tr><th></th><th rowspan=3>gross elapsed time (sec)</th><th></th>"
+      html << "<th colspan=6>read</th>"
+      html << "<th></th>"
+      html << "<th colspan=6>write</th>"
+      html << "</tr>\n"
+      html << "<tr><th></th><th></th>"
+      html << "<th colspan=3>count</th><th colspan=3>file size (bytes)</th>"
+      html << "<th></th>"
+      html << "<th colspan=3>count</th><th colspan=3>file size (bytes)</th>"
+      html << "</tr>\n"
+      html << "<tr><th>host</th><th></th>"
+      html << "<th>local</th><th>remote</th><th>total</th>"
+      html << "<th>local</th><th>remote</th><th>total</th>"
+      html << "<th></th>"
+      html << "<th>local</th><th>remote</th><th>total</th>"
+      html << "<th>local</th><th>remote</th><th>total</th>"
+      html << "</tr>\n"
+      n_same_input     = 0
+      size_same_input  = 0
+      n_diff_input     = 0
+      size_diff_input  = 0
+      n_same_output    = 0
+      size_same_output = 0
+      n_diff_output    = 0
+      size_diff_output = 0
+      elap_host = 0
+      @exec_hosts.each do |h|
+        html << "<tr><td>#{h}</td>"
+        html << "<td align='right'>%.3f</td>" % @elap_host[h]
+        html << "<td></td>"
+        html << tr_count(@n_same_input[h],@n_diff_input[h])
+        html << tr_count(@size_same_input[h],@size_diff_input[h])
+        html << "<td></td>"
+        html << tr_count(@n_same_output[h],@n_diff_output[h])
+        html << tr_count(@size_same_output[h],@size_diff_output[h])
+        html << "</tr>\n"
+        n_same_input     += @n_same_input[h]
+        size_same_input  += @size_same_input[h]
+        n_diff_input     += @n_diff_input[h]
+        size_diff_input  += @size_diff_input[h]
+        n_same_output    += @n_same_output[h]
+        size_same_output += @size_same_output[h]
+        n_diff_output    += @n_diff_output[h]
+        size_diff_output += @size_diff_output[h]
+        elap_host += @elap_host[h]
+      end
+      html << "<tr><td>total</td>"
+      html << "<td align='right'>%.3f</td>" % elap_host
+      html << "<td></td>"
+      html << tr_count(n_same_input,n_diff_input)
+      html << tr_count(size_same_input,size_diff_input)
+      html << "<td></td>"
+      html << tr_count(n_same_output,n_diff_output)
+      html << tr_count(size_same_output,size_diff_output)
+      html << "</tr>\n"
+      html << "<table>\n"
+      html << "</body></html>\n"
+      File.open(@html_file,"w") do |f|
+        f.puts html
+      end
+      puts "generate "+@html_file
+    end
+    def task_locality
+      @task_table = CSV.read(@task_file,:headers=>true)
+      file_size = {}
+      file_host = {}
+      @task_table.each do |row|
+        name = row['task_name']
+        file_size[name] = row['file_size'].to_i
+        file_host[name] = (row['file_host']||'').split('|')
+      end
+      @n_same_output   = Hash.new(0)
+      @size_same_output= Hash.new(0)
+      @n_diff_output   = Hash.new(0)
+      @size_diff_output= Hash.new(0)
+      @n_same_input    = Hash.new(0)
+      @size_same_input = Hash.new(0)
+      @n_diff_input    = Hash.new(0)
+      @size_diff_input = Hash.new(0)
+      h = {}
+      @task_table.each do |row|
+        if row['executed']=='1'
+          name = row['task_name']
+          exec_host = row['exec_host']
+          h[exec_host] = true
+          if file_host[name].include?(exec_host)
+            @n_same_output[exec_host] += 1
+            @size_same_output[exec_host] += file_size[name]
+          else
+            @n_diff_output[exec_host] += 1
+            @size_diff_output[exec_host] += file_size[name]
+          end
+          preq_files = (row['preq']||'').split('|')
+          preq_files.each do |preq|
+            if (sz=file_size[preq]) && sz > 0
+              if file_host[preq].include?(exec_host)
+                @n_same_input[exec_host] += 1
+                @size_same_input[exec_host] += sz
+              else
+                @n_diff_input[exec_host] += 1
+                @size_diff_input[exec_host] += sz
+              end
+            end
+          end
+        end
+      end
+      @exec_hosts = h.keys.sort
+      @elap_host = Hash.new(0)
+      @sh_table.each do |row|
+        if (h = row['host']) && (t = row['elap_time'])
+          @elap_host[h] += t.to_f
+        end
+      end
+    end
+    def histogram_plot
+      command_list = []
+      @cmd_stat.each do |cmd,stat|
+        if stat.n > 2
+          command_list << cmd
+        end
+      end
+      hist_image = @base+"_hist.png"
+      IO.popen("gnuplot","r+") do |f|
+        f.puts "
+set terminal png # size 480,360
+set output '#{hist_image}'
+set ylabel 'histogram'
+set xlabel 'Execution time (sec)'
+set logscale x
+set title 'histogram of elapsed time'"
+        a = []
+        command_list.each_with_index do |n,i|
+          a << "'-' w histeps ls #{i+1} title ''"
+          a << "'-' w lines ls #{i+1} title '#{n}'"
+        end
+        f.puts "plot "+ a.join(',')
+        command_list.each do |cmd|
+          stat = @cmd_stat[cmd]
+          2.times do
+            stat.hist_each do |x1,x2,y|
+              x = Math.sqrt(x1*x2)
+              f.printf "%f %d\n", x, y
+            end
+            f.puts "e"
+          end
+        end
+        hist_image
+      end
+    end
+  end
+end

data/lib/pwrake/report/report_multi.rb ADDED Viewed

@@ -0,0 +1,196 @@
+module Pwrake
+  class ReportMulti
+    def initialize(list,pattern)
+      @reports = list.map do |base|
+        r = Report.new(base,pattern)
+        puts r.base+" elap=#{r.elap}"
+        r
+      end
+      @pattern = pattern
+      @elap_png = 'elap.png'
+    end
+    def report(stat_html)
+      if true
+        @reports.each do |r|
+          r.report_html
+        end
+        plot_elap
+      end
+      html = Report::HTML_HEAD + "<body><h1>Pwrake Statistics</h1>\n"
+      html << "<h2>Log files</h2>\n"
+      html << "<table>\n"
+      html << "<tr><th>log file</th><th>id</th><th>ncore</th><th>elapsed time(sec)</th><tr>\n"
+      @reports.each do |r|
+        html << "<tr><td><a href='#{r.html_file}'>#{r.base}</a></td>"
+        html << "<td>#{r.id_str}</td><td>#{r.ncore}</td><td>#{r.elap}</td><tr>\n"
+      end
+      html << "</table>\n"
+      html << "<h2>Elapsed time</h2>\n"
+      html << "<img src='#{@elap_png}'  align='top'/></br>\n"
+      html << "<h2>Histogram of Execution time</h2>\n"
+      html << report_histogram()
+      html << "</body></html>\n"
+      File.open(stat_html,"w") do |f|
+        f.puts html
+      end
+    end
+    def plot_elap
+      a = @reports.map{|r| r.ncore * r.elap}.min
+      elaps = @reports.map{|r| r.elap}
+      logmin = Math.log10(elaps.min)
+      logmax = Math.log10(elaps.max)
+      mid = (logmin+logmax)/2
+      wid = (logmax-logmin).ceil*0.5
+      ymin = 10**(mid-wid)
+      ymax = 10**(mid+wid)
+      IO.popen("gnuplot","r+") do |f|
+        f.puts "
+set terminal png size 640,480
+set output '#{@elap_png}'
+set xlabel 'ncore'
+set ylabel 'time (sec)'
+set yrange [#{ymin}:#{ymax}]
+set logscale xy
+plot #{a}/x,'-' w lp lw 2 ps 2 title 'elapsed time'
+"
+        @reports.sort_by{|r| r.ncore}.each do |r|
+          f.puts "#{r.ncore} #{r.elap}"
+        end
+        f.puts "e"
+      end
+      puts "Ncore-time plot: "+@elap_png
+    end
+    def report_histogram
+      @images = {}
+      @cmd_rep = {}
+      @reports.each do |r|
+        r.cmd_stat.each do |cmd,stat|
+          if stat.n > 2
+            @cmd_rep[cmd] ||= {}
+            @cmd_rep[cmd][r.id_str] = r # stat
+          end
+        end
+      end
+      @cmd_rep.each_key do |cmd|
+        @images[cmd] = 'hist_'+cmd.gsub(/[\/.]/,'_')+'.png'
+      end
+      histogram_plot
+      histogram_html
+    end
+    def histogram_html
+      html = ""
+      @cmd_rep.each do |cmd,cmd_rep|
+        html << "<p>Statistics of Elapsed time of #{cmd}</p>\n<table>\n"
+        html << "<th>id</th><th>ncore</th>"+Stat.html_th
+        cmd_rep.each do |id,r|
+          s = r.cmd_stat[cmd]
+          html << "<tr><td>#{id}</td><td>#{r.ncore}</td>" + s.html_td + "</tr>\n"
+        end
+        html << "</table>\n"
+        html << "<img src='./#{@images[cmd]}'/>\n"
+      end
+      html
+    end
+    def histogram_plot
+      @cmd_rep.each do |cmd,cmd_rep|
+        IO.popen("gnuplot","r+") do |f|
+          f.puts "
+set terminal png # size 480,360
+set output '#{@images[cmd]}'
+set ylabel 'histogram'
+set xlabel 'Execution time (sec)'
+set logscale x
+set title '#{cmd}'"
+          a = []
+          ncores = cmd_rep.keys
+          ncores.each_with_index{|n,i|
+            a << "'-' w histeps ls #{i+1} title ''"
+            a << "'-' w lines ls #{i+1} title '#{n}'"
+          }
+          f.puts "plot "+ a.join(',')
+          cmd_rep.each do |ncore,r|
+            s = r.cmd_stat[cmd]
+            2.times do
+              s.hist_each do |x1,x2,y|
+                x = Math.sqrt(x1*x2)
+                f.printf "%f %d\n", x, y
+              end
+              f.puts "e"
+            end
+          end
+        end
+        puts "Histogram plot: #{@images[cmd]}"
+      end
+    end
+    def histogram_plot2
+      @cmd_rep.each do |cmd,cmd_rep|
+        IO.popen("gnuplot","r+") do |f|
+          f.puts "
+set terminal png # size 480,360
+set output '#{@images[cmd]}'
+set nohidden3d
+set palette rgb 33,13,10
+set pm3d
+set ticslevel 0
+unset colorbox
+set yrange [#{cmd_rep.size}:0]
+set logscale x
+set title '#{cmd}'"
+          a = []
+          ncores = cmd_rep.keys.sort
+          ncores.each_with_index{|n,i|
+            a << "'-' w lines ls #{i+1} title '#{n} cores'"
+          }
+          f.puts "splot "+ a.join(',')
+          ncores.each_with_index do |ncore,i|
+            s = cmd_rep[ncore]
+            y = i
+            s.hist_each do |x1,x2,z|
+              f.printf "%g %g 0\n", x1,y
+              f.printf "%g %g 0\n", x2,y
+              f.printf "%g %g 0\n", x2,y
+            end
+            f.puts ""
+            s.hist_each do |x1,x2,z|
+              f.printf "%g %g %g\n", x1,y,z
+              f.printf "%g %g %g\n", x2,y,z
+              f.printf "%g %g 0\n", x2,y,z
+            end
+            f.puts ""
+            y = i+1
+            s.hist_each do |x1,x2,z|
+              f.printf "%g %g %g\n", x1,y,z
+              f.printf "%g %g %g\n", x2,y,z
+              f.printf "%g %g 0\n", x2,y,z
+            end
+            f.puts ""
+            s.hist_each do |x1,x2,z|
+              f.printf "%g %g 0\n", x1,y
+              f.printf "%g %g 0\n", x2,y
+              f.printf "%g %g 0\n", x2,y
+            end
+            f.puts "e"
+            i = i+1
+          end
+        end
+        puts "Histogram plot: #{@images[cmd]}"
+      end
+    end
+  end
+end

data/lib/pwrake/report/stat.rb ADDED Viewed

@@ -0,0 +1,115 @@
+module Pwrake
+  class Stat
+    def initialize(data)
+      @data = data
+      @n = data.size
+      if @n>0
+        @min = data.min
+        @max = data.max
+        @sum = data.inject(0){|s,x| s+x}
+        @mean = @sum/@n
+        @median = _median
+        @mean_absolute_deviation = data.inject(0){|s,x| (x-@mean).abs} / @n
+        if @n>1
+          @variance = data.inject(0){|s,x| y=x-@mean; y**2} / (@n-1)
+          @sdev = Math.sqrt(@variance)
+          @skew = data.inject(0){|s,x| y=(x-@mean)/@sdev; y**3} / @n
+          @kurt = data.inject(0){|s,x| y=(x-@mean)/@sdev; y**4} / @n - 3
+        end
+      end
+    end
+    attr_reader :n
+    attr_reader :min, :max, :sum, :mean, :median
+    attr_reader :mean_absolute_deviation
+    attr_reader :variance, :sdev, :skew, :kurt
+    attr_reader :hist, :hist_min, :hist_max, :bin
+    def make_logx_histogram(bin)
+      @bin = bin # 1.0/10
+      @i_max = (Math.log10(@max)/@bin).floor
+      @i_min = (Math.log10(@min)/@bin).floor
+      @hist_min = 10**(@i_min * @bin)
+      @hist_max = 10**((@i_max+1) * @bin)
+      @hist = Array.new(@i_max-@i_min+1,0)
+      @data.each do |x|
+        i = (Math.log10(x)/@bin-@i_min).floor
+        raise "invalid index i=#{i}" if i<0 || i>@i_max-@i_min
+        @hist[i] += 1
+      end
+    end
+    def hist_each
+      if @hist
+        n = @hist.size
+        n.times do |i|
+          x1 = 10**(@bin*(i+@i_min))
+          x2 = 10**(@bin*(i+1+@i_min))
+          y  = @hist[i]
+          yield x1,x2,y
+        end
+      end
+    end
+    def _median
+      if @n==1
+        @data[0]
+      elsif @n==2
+        @mean
+      else
+        case @n%2
+        when 1
+          i = (@n-1)/2
+          @data.sort[i]
+        else
+          i = @n/2
+          s = @data.sort
+          (s[i]+s[i+1])/2
+        end
+      end
+    end
+    def report
+      case @n
+      when 0
+        "no data"
+      when 1
+        "n=1 mean=%g"%@mean
+      else
+        "n=%i mean=%g median=%g min=%g max=%g sdev=%g skew=%g kurt=%g" %
+          [@n, @mean, @median, @min, @max, @sdev||0, @skew||0, @kurt||0]
+        "n=%i mean=%g median=%g min=%g max=%g sdev=%g" %
+          [@n, @mean, @median, @min, @max, @sdev||0]
+      end
+    end
+    def stat_array
+      [@n, @mean, @median, @min, @max, @sdev||0]
+    end
+    def html_td
+      "<td>%i</td><td>%g</td><td>%g</td><td>%g</td><td>%g</td><td>%g</td>" %
+        [@n, @mean, @median, @min, @max, @sdev||0]
+    end
+    def self.html_th
+      "<th>%s</th><th>%s</th><th>%s</th><th>%s</th><th>%s</th><th>%s</th>" %
+      %w[n mean median min max sdev]
+    end
+    def report2
+      case @n
+      when 0
+        "  no data"
+      when 1
+        "  n=1\n  mean=%g"%@mean
+      else
+        "  n=%i\n  mean=%g\n  median=%g\n  min=%g\n  max=%g\n  sdev=%g\n  skew=%g\n  kurt=%g" %
+          [@n, @mean, @median, @min, @max, @sdev||0, @skew||0, @kurt||0]
+      end
+    end
+  end
+end

data/lib/pwrake/report.rb ADDED Viewed

@@ -0,0 +1,6 @@
+require 'time'
+require 'csv'
+require 'pwrake/report/parallelism.rb'
+require 'pwrake/report/report.rb'
+require 'pwrake/report/report_multi.rb'
+require 'pwrake/report/stat.rb'