pwrake 0.9.7 → 0.9.8

data/lib/pwrake/report/parallelism.rb

@@ -24,6 +24,27 @@ module Pwrake
       a.sort{|x,y| x[0]<=>y[0]}
     end
 
+    def count_start_end_from_csv_table(csvtable)
+      a = []
+      start_time = nil
+
+      csvtable.each do |row|
+        if row['command'] == 'pwrake_profile_start'
+          start_time = Time.parse(row['start_time'])
+        elsif row['command'] == 'pwrake_profile_end'
+          t = Time.parse(row['start_time']) - start_time
+          a << [t,0]
+        elsif start_time
+          t = Time.parse(row['start_time']) - start_time
+          a << [t,+1]
+          t = Time.parse(row['end_time']) - start_time
+          a << [t,-1]
+        end
+      end
+
+      a.sort{|x,y| x[0]<=>y[0]}
+    end
+
     def exec_density(a)
       reso = 0.1
       delta = 1/reso
@@ -98,51 +119,41 @@ plot '#{fpara}' w l axis x1y1 title 'parallelism'
     end
 
 
-    def plot_parallelism2(file)
-      a = count_start_end_from_csv(file)
+    def plot_parallelism2(csvtable, base)
+      a = count_start_end_from_csv_table(csvtable)
       return if a.size < 4
 
       density = exec_density(a)
 
-      base = file.sub(/\.csv$/,"")
-      fpara = base+"_para.dat"
-      fdens = base+'_dens.dat'
       fimg = base+'.png'
 
-      File.open(fdens,"w") do |f|
-        density.each do |t,d|
-          f.puts "#{t} #{d}"
-        end
-      end
-
       n = a.size
       i = 0
       y = 0
       y_max = 0
 
-      File.open(fpara,"w") do |f|
-        begin
-          t = 0
-          y_pre = 0
-          n.times do |i|
-            if a[i][0]-t > 0.001
-              f.printf "%.3f %d\n", t, y_pre
-              t = a[i][0]
-              f.printf "%.3f %d\n", t, y
-            end
-            y += a[i][1]
-            y_pre = y
-            y_max = y if y > y_max
+      para = []
+      begin
+        t = 0
+        y_pre = 0
+        n.times do |i|
+          if a[i][0]-t > 0.001
+            para.push "%.3f %d" % [t, y_pre]
+            t = a[i][0]
+            para.push "%.3f %d" % [t, y]
           end
-        rescue
-          p a[i]
+          y += a[i][1]
+          y_pre = y
+          y_max = y if y > y_max
         end
+      rescue
+        p a[i]
       end
 
       t_end = (a.last)[0]
 
       IO.popen("gnuplot","r+") do |f|
-        f.puts "
+        f.print "
 set terminal png
 set output '#{fimg}'
 #set rmargin 10
@@ -156,8 +167,16 @@ set y2label 'exec/sec'
 set arrow 1 from #{t_end},#{y_max*0.5} to #{t_end},0 linecolor rgb 'blue'
 set label 1 \"#{t_end}\\nsec\" at first #{t_end},#{y_max*0.5} right front textcolor rgb 'blue'
 
-plot '#{fpara}' w l axis x1y1 title 'parallelism', '#{fdens}' w l axis x1y2 title 'exec/sec'
+plot '-' w l axis x1y1 title 'parallelism', '-' w l axis x1y2 title 'exec/sec'
 "
+        para.each do |x|
+          f.puts x
+        end
+        f.puts "e"
+
+        density.each do |t,d|
+          f.puts "#{t} #{d}"
+        end
       end
 
       #puts "Parallelism plot: #{fimg}"
@@ -188,6 +207,112 @@ plot '#{fpara}' w l axis x1y1 title 'parallelism', '#{fdens}' w l axis x1y2 titl
       a
     end
 
+
+    def get_command_key(s, pattern)
+      pattern.each do |cmd,regex|
+        if regex =~ s
+          return cmd
+        end
+      end
+      if /\(([^()]+)\)/ =~ s
+        s = $1
+      end
+      a = s.split(/;/)
+      a.each do |x|
+        if /^\s*(\S+)/ =~ x
+          k = $1
+          next if k=='cd'
+          return k
+        end
+      end
+      nil
+    end
+
+    def count_start_end_by_pattern(csvtable, pattern)
+      h = Hash.new
+      start_time = nil
+
+      csvtable.each do |row|
+        command = row['command']
+        if command == 'pwrake_profile_start'
+          start_time = Time.parse(row['start_time'])
+        elsif command == 'pwrake_profile_end'
+          t = Time.parse(row['start_time']) - start_time
+          h.values.each do |a|
+            a << [t,0]
+          end
+        elsif start_time
+          cmd = get_command_key(command, pattern)
+          h[cmd] ||= []
+          t = Time.parse(row['start_time']) - start_time
+          h[cmd] << [t,+1]
+          t = Time.parse(row['end_time']) - start_time
+          h[cmd] << [t,-1]
+        end
+      end
+
+      h.each do |k,a|
+        a.sort!{|x,y| x[0]<=>y[0]}
+      end
+      h
+    end
+
+    def plot_parallelism_by_pattern(csvtable, base, pattern)
+      y_max = 0
+      t_end = 0
+      para = {}
+      t_pat = count_start_end_by_pattern(csvtable, pattern)
+      t_pat.each do |cmd,a|
+        n = a.size
+        i = 0
+        y = 0
+        para[cmd] = dat = []
+        begin
+          t = 0
+          y_pre = 0
+          n.times do |i|
+            if a[i][0]-t > 0.001
+              dat.push "%.3f %d" % [t, y_pre]
+              t = a[i][0]
+              dat.push "%.3f %d" % [t, y]
+            end
+            y += a[i][1]
+            y_pre = y
+            y_max = y if y > y_max
+          end
+        rescue
+          p a[i]
+        end
+        if (a.last)[0] > t_end
+          t_end = (a.last)[0]
+        end
+      end
+
+      fimg = base+'_para_cmd.png'
+
+      IO.popen("gnuplot","r+") do |f|
+        #begin f = $stdout
+        f.print "
+set terminal png
+set output '#{fimg}'
+set title '#{base}'
+set xlabel 'time (sec)'
+set ylabel 'parallelism'
+"
+        f.print "plot "
+        f.puts para.map{|cmd,re| "'-' w l title '#{cmd}'"}.join(",")
+        para.each do |cmd,dat|
+          dat.each do |x|
+            f.puts x
+          end
+          f.puts "e"
+        end
+      end
+
+      #puts "Parallelism plot: #{fimg}"
+      fimg
+    end
+
     def timeline_to_grid(a)
       resolution = Rational(1,10)
 
@@ -209,7 +334,7 @@ plot '#{fpara}' w l axis x1y1 title 'parallelism', '#{fdens}' w l axis x1y2 titl
       return grid
     end
 
-    def plot_parallelizm_by_host(csvtable,base)
+    def plot_parallelism_by_host(csvtable,base)
       fpng = base+"_para_host.png"
       data = read_time_by_host_from_csv(csvtable)
       return fpng if data.size == 0

data/lib/pwrake/report/report.rb

@@ -50,10 +50,12 @@ EOL
 
       @sh_table = CSV.read(@csv_file,:headers=>true)
       h = {}
+      @elap_sum = 0
       @sh_table.each do |row|
         if host = row['host']
           h[host] = true
         end
+        @elap_sum += row['elap_time'].to_f
       end
       @hosts = h.keys.sort
       @start_time = Time.parse(@sh_table[0]["start_time"])
@@ -153,7 +155,9 @@ EOL
       html << "<table>\n"
       html << "<tr><th>log file</th><td>#{@base}</td><tr>\n"
       html << "<tr><th>ncore</th><td>#{@ncore}</td><tr>\n"
-      html << "<tr><th>elapsed time(sec)</th><td>#{@elap}</td><tr>\n"
+      html << "<tr><th>elapsed time</th><td>%.3f sec</td><tr>\n"%[@elap]
+      html << "<tr><th>cumulative process time</th><td>%.3f sec</td><tr>\n"%[@elap_sum]
+      html << "<tr><th>occupancy</th><td>%.3f %%</td><tr>\n"%[@elap_sum/@elap/@ncore*100]
       html << "<tr><th>start time</th><td>#{@start_time}</td><tr>\n"
       html << "<tr><th>end time</th><td>#{@end_time}</td><tr>\n"
       html << "</table>\n"
@@ -164,12 +168,16 @@ EOL
       end
       html << "</table>\n"
       html << "<h2>Parallelism</h2>\n"
-      fimg = Parallelism.plot_parallelism2(@csv_file)
-      html << "<img src='#{fimg}' align='top'/></br>\n"
+      fimg = Parallelism.plot_parallelism2(@sh_table,@base)
+      html << "<img src='./#{File.basename(fimg)}' align='top'/></br>\n"
+
+      html << "<h2>Parallelism by command</h2>\n"
+      fimg3 = Parallelism.plot_parallelism_by_pattern(@sh_table,@base,@pattern)
+      html << "<img src='./#{File.basename(fimg3)}' align='top'/></br>\n"
 
       html << "<h2>Parallelism by host</h2>\n"
-      fimg2 = Parallelism.plot_parallelizm_by_host(@sh_table,@base)
-      html << "<img src='#{fimg2}' align='top'/></br>\n"
+      fimg2 = Parallelism.plot_parallelism_by_host(@sh_table,@base)
+      html << "<img src='./#{File.basename(fimg2)}' align='top'/></br>\n"
 
       html << "<h2>Command statistics</h2>\n"
       html << "<table>\n"
@@ -182,7 +190,7 @@ EOL
         html << "</tr>\n"
       end
       html << "<table>\n"
-      html << "<img src='#{histogram_plot}' align='top'/></br>\n"
+      html << "<img src='./#{File.basename(histogram_plot)}' align='top'/></br>\n"
 
       task_locality
       html << "<h2>Locality statistics</h2>\n"

data/lib/pwrake/report/report_multi.rb

@@ -29,7 +29,7 @@ module Pwrake
       end
       html << "</table>\n"
       html << "<h2>Elapsed time</h2>\n"
-      html << "<img src='#{@elap_png}'  align='top'/></br>\n"
+      html << "<img src='./#{File.basename(@elap_png)}'  align='top'/></br>\n"
 
       html << "<h2>Histogram of Execution time</h2>\n"
       html << report_histogram()
@@ -97,7 +97,7 @@ plot #{a}/x,'-' w lp lw 2 ps 2 title 'elapsed time'
           html << "<tr><td>#{id}</td><td>#{r.ncore}</td>" + s.html_td + "</tr>\n"
         end
         html << "</table>\n"
-        html << "<img src='./#{@images[cmd]}'/>\n"
+        html << "<img src='./#{File.basename(@images[cmd])}'/>\n"
       end
       html
     end

data/lib/pwrake/shell.rb

@@ -19,9 +19,6 @@ module Pwrake
     @@current_id = 0
     @@profiler = Profiler.new
 
-#    @@global_lock = Mutex.new
-#    @@last_exec_time = Time.now
-
     def self.profiler
       @@profiler
     end
@@ -35,7 +32,6 @@ module Pwrake
       @work_dir = @option[:work_dir] || Dir.pwd
       @pass_env = @option[:pass_env]
       @ssh_opt = @option[:ssh_opt]
-#      @ssh_min_exec_interval = Pwrake.application.pwrake_options['SHELL_MIN_EXEC_INTERVAL'].to_f
       @terminator = ""
       TLEN.times{ @terminator << CHARS[rand(CHARS.length)] }
     end
@@ -152,19 +148,6 @@ module Pwrake
       raise "@io is closed" if @io.closed?
       status = nil
 
-#      t0 = Time.now
-#      @@global_lock.synchronize do
-#        t = Time.now - @@last_exec_time
-#        if t < @ssh_min_exec_interval
-#          sleep @ssh_min_exec_interval-t
-#        end
-#        @@last_exec_time = Time.now
-#      end
-#      t = Time.now-t0
-#      #if t>0.001
-#        Log.info "-- locktime %.6f"%t
-#      #end
-
       start_time = Time.now
       begin
         @io.puts @@profiler.command(cmd,@terminator)
@@ -187,4 +170,17 @@ module Pwrake
     end
   end
 
+
+  class NoActionShell < Shell
+    def initialize()
+      @host = '(noaction)'
+      @@current_id += 1
+      @id = @@current_id
+    end
+    def start
+    end
+    def finish
+    end
+  end
+
 end # module Pwrake

data/lib/pwrake/task_algorithm.rb

@@ -3,6 +3,50 @@ module Pwrake
   InvocationChain = Rake::InvocationChain
   TaskArguments = Rake::TaskArguments
 
+  class RankStat
+    def initialize
+      @lock = Mutex.new
+      @stat = []
+    end
+
+    def add_sample(rank,elap)
+      @lock.synchronize do
+        stat = @stat[rank]
+        if stat.nil?
+          @stat[rank] = stat = [0,0.0]
+        end
+        stat[0] += 1
+        stat[1] += elap
+        Log.debug "--- add_sample rank=#{rank} stat=#{stat.inspect} weight=#{stat[0]/stat[1]}"
+      end
+    end
+
+    def rank_weight
+      @lock.synchronize do
+        sum = 0.0
+        count = 0
+        weight = @stat.map do |stat|
+          if stat
+            w = stat[0]/stat[1]
+            sum += w
+            count += 1
+            w
+          else
+            nil
+          end
+        end
+        if count == 0
+          avg = 1.0
+        else
+          avg = sum/count
+        end
+        [weight, avg]
+      end
+    end
+  end
+
+  RANK_STAT = RankStat.new
+
   module TaskAlgorithm
 
     def location
@@ -63,7 +107,7 @@ module Pwrake
       timer = Timer.new("search_task")
       h = application.pwrake_options['HALT_QUEUE_WHILE_SEARCH']
       application.task_queue.synchronize(h) do
-	search_with_call_chain(self, task_args, InvocationChain::EMPTY)
+	search_with_call_chain(nil, task_args, InvocationChain::EMPTY)
       end
       timer.finish
     end
@@ -73,24 +117,28 @@ module Pwrake
       if shell = Pwrake.current_shell
         shell.current_task = self
       end
-
       @lock.synchronize do
         return if @already_invoked
         @already_invoked = true
       end
       pw_execute(@arg_data) if needed?
+      Log.debug("pw_execute: #{Time.now-time_start} sec, name=#{name}")
       if kind_of?(Rake::FileTask)
-        application.postprocess(self) #        <---------
+        t = Time.now
+        application.postprocess(self)
+        Log.debug("postprocess: #{Time.now-t} sec, name=#{name}")
         if File.exist?(name)
+          t = Time.now
           @file_stat = File::Stat.new(name)
+          Log.debug("File::Stat: #{Time.now-t} sec, name=#{name}")
         end
       end
       log_task(time_start)
       t = Time.now
       application.finish_queue.enq(self)
       shell.current_task = nil if shell
-      pw_enq_subsequents              #        <---------
-      Log.debug "--- pw_invoke (#{name}) postprocess time=#{Time.now-t} sec"
+      pw_enq_subsequents
+      Log.debug "--- pw_invoke (#{name}) enq_subseq time=#{Time.now-t} sec"
     end
 
     def log_task(time_start)
@@ -104,10 +152,12 @@ module Pwrake
       end
       return if !application.task_logger
 
-      row = [ @task_id, name,
-        time_start, time_end, time_end-time_start,
-        @prerequisites.join('|')
-      ]
+      elap = time_end - time_start
+      if !@actions.empty? && kind_of?(Rake::FileTask)
+        RANK_STAT.add_sample(rank,elap)
+      end
+
+      row = [ @task_id, name, time_start, time_end, elap, @prerequisites.join('|') ]
 
       if loc
         row << loc.join('|')
@@ -190,9 +240,12 @@ module Pwrake
     def pw_enq_subsequents
       @lock.synchronize do
         t = Time.now
-        @subsequents.each do |t|        # <<--- competition !!!
-          if t && t.check_prereq_finished(self.name)
-            application.task_queue.enq(t)
+        h = application.pwrake_options['HALT_QUEUE_WHILE_SEARCH']
+        application.task_queue.synchronize(h) do
+          @subsequents.each do |t|        # <<--- competition !!!
+            if t && t.check_prereq_finished(self.name)
+              application.task_queue.enq(t)
+            end
           end
         end
         @already_finished = true        # <<--- competition !!!
@@ -217,11 +270,12 @@ module Pwrake
 
         return true if @already_finished # <<--- competition !!!
         @subsequents ||= []
-        @subsequents << subseq           # <<--- competition !!!
+        @subsequents << subseq if subseq # <<--- competition !!!
 
         if ! @already_searched
           @already_searched = true
           @arg_data = task_args
+          @lock_rank = Monitor.new
           if @prerequisites.empty?
             @unfinished_prereq = {}
           else
@@ -262,16 +316,56 @@ module Pwrake
     end
     private :format_search_flags
 
+    def rank
+      @lock_rank.synchronize do
+        if @rank.nil?
+          if @subsequents.nil? || @subsequents.empty?
+            @rank = 0
+          else
+            max_rank = 0
+            @subsequents.each do |subsq|
+              r = subsq.rank
+              if max_rank < r
+                max_rank = r
+              end
+            end
+            if @actions.empty? || !kind_of?(Rake::FileTask)
+              step = 0
+            else
+              step = 1
+            end
+            @rank = max_rank + step
+          end
+          Log.debug "--- Task[#{name}] rank=#{@rank.inspect}"
+        end
+      end
+      @rank
+    end
+
     def file_size
       @file_stat ? @file_stat.size : 0
     end
 
-    def prior?
+    def file_mtime
+      @file_stat ? @file_stat.mtime : Time.at(0)
+    end
+
+    def input_file_size
+      unless @input_file_size
+        @input_file_size = 0
+        @prerequisites.each do |preq|
+          @input_file_size += application[preq].file_size
+        end
+      end
+      @input_file_size
+    end
+
+    def has_input_file?
       kind_of?(Rake::FileTask) && !@prerequisites.empty?
     end
 
     def suggest_location
-      if prior? && @suggest_location.nil?
+      if has_input_file? && @suggest_location.nil?
         @suggest_location = []
         loc_fsz = Hash.new(0)
         @prerequisites.each do |preq|
@@ -297,6 +391,33 @@ module Pwrake
       @suggest_location
     end
 
+    def input_file_mtime
+      if has_input_file? && @input_file_mtime.nil?
+        hash = Hash.new
+        max_sz = 0
+        @prerequisites.each do |preq|
+          t = application[preq]
+          sz = t.file_size
+          if sz > 0
+            hash[t] = sz
+            if sz > max_sz
+              max_sz = sz
+            end
+          end
+        end
+        half_max_sz = max_sz / 2
+        hash.each do |t,sz|
+          if sz > half_max_sz
+            time = t.file_mtime
+            if @input_file_mtime.nil? || @input_file_mtime < time
+              @input_file_mtime = time
+            end
+          end
+        end
+      end
+      @input_file_mtime
+    end
+
     def suggest_location2
       if kind_of?(Rake::FileTask) && preq_name = @prerequisites[0]
         application[preq_name].location