pwrake 2.2.7 → 2.2.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 20590e6007af5bc0c567fe8266a82f40aaeec0bbb853bdd87e180f0f1c53bc31
-  data.tar.gz: 4ec4760f2428d854019675a2f51e49f791102f7eab6b3141e258cfbc25a8842a
+  metadata.gz: 4053be2a3de706030e8ee37157453416b379f22c7751db94d2122b607eec6d3a
+  data.tar.gz: c76ad99c55d4ee592522337d66327845d72c239df9cfd8bed294e7d7acdc3e83
 SHA512:
-  metadata.gz: 391db5799b9091fbbc0a242f78729fe265087581b6b88dbaf9bf01f188e8d4412c749683075c38dfcea4803589289205f85c42320913727fe51628eb7fdaa2f8
-  data.tar.gz: fc7a41165bdce7fa685e3d922d5b24f6146524deaa52a31049c459d8fa39430b3948b4d02943ec143014181f2f916e3ab6d06f9596b61c05d397243a02770fe4
+  metadata.gz: bc954608b0bf14af8782f27273cc9a802951a31a04c2507ef42255a0855af738c9ab71dc9cc5f6afdde2d8c097b27cc01d606d2cf667b083e270a138e9e5ac4d
+  data.tar.gz: 192a8cbc976188d637f74b08cec65054235e156e82ecaa1354a75a2dbdadd91e1c8b14b0d21733f15c0d85d601c15db366f0c91c95d0b4919e25a23b886cdb87

data/bin/gfwhere-pipe

@@ -1,5 +1,6 @@
 #! /usr/bin/env ruby
 
+require 'pathname'
 require 'fiddle'
 
 module Gfarm
@@ -153,10 +154,7 @@ module Gfarm
     end
 
     def initialize(path)
-      while File.symlink?(path)
-        link = File.readlink(path)
-        path = File.expand_path(link, File.dirname(path))
-      end
+      path = Pathname.new(path).realpath.to_s
       @realpath = LibGfarm.realpath_by_gfarm2fs(path)
       ptr_buf = [0].pack('J')
       e = LibGfarm.gfs_replica_info_by_name(@realpath,@@flags,ptr_buf)
@@ -180,7 +178,8 @@ module Gfarm
 
 end
 
-if $0 == __FILE__
+
+def gfwhere_pipe
 
   [:PIPE,:TERM,:INT].each do |sig|
     Signal.trap(sig, "EXIT")
@@ -204,3 +203,5 @@ if $0 == __FILE__
   end
 
 end
+
+gfwhere_pipe

data/bin/pwrake

@@ -14,6 +14,7 @@ require "pwrake/version"
 require "pwrake/master/master_application"
 require "pwrake/task/task_manager"
 require "pwrake/task/task_algorithm"
+require "pwrake/task/file_task_algorithm"
 require "pwrake/branch/branch_application"
 
 class Rake::Application
@@ -25,6 +26,9 @@ class Rake::Task
   include Pwrake::TaskAlgorithm
   prepend Pwrake::TaskInvoke
 end
+class Rake::FileTask
+  prepend Pwrake::FileTaskAlgorithm
+end
 
 # does NOT exit when writing to broken pipe
 Signal.trap(:PIPE, "IGNORE")

data/lib/pwrake/branch/file_utils.rb

@@ -17,10 +17,9 @@ module Pwrake
       else
         options[:verbose] ||= RakeFileUtils.verbose_flag
       end
-      options[:noop]    ||= RakeFileUtils.nowrite_flag
+      options[:noop] ||= RakeFileUtils.nowrite_flag
       Rake.rake_check_options options, :noop, :verbose
       Rake.rake_output_message cmd.join(" ") if options[:verbose]
-      #Pwrake::Log.stderr_puts cmd.join(" ") if options[:verbose]
       unless options[:noop]
         res,status = Pwrake::FileUtils.pwrake_system(*cmd)
         block.call(res, status)
@@ -41,10 +40,9 @@ module Pwrake
       else
         options[:verbose] ||= RakeFileUtils.verbose_flag
       end
-      options[:noop]    ||= RakeFileUtils.nowrite_flag
+      options[:noop] ||= RakeFileUtils.nowrite_flag
       Rake.rake_check_options options, :noop, :verbose
       Rake.rake_output_message cmd.join(" ") if options[:verbose]
-      #Pwrake::Log.stderr_puts cmd.join(" ") if options[:verbose]
       unless options[:noop]
         res,status = Pwrake::FileUtils.pwrake_backquote(*cmd)
         block.call(res, status)
@@ -53,9 +51,6 @@ module Pwrake
     end
 
     def pwrake_system(*cmd)
-      cmd_log = cmd.join(" ").inspect
-      #tm = Pwrake::Timer.new("sh",cmd_log)
-
       conn = Pwrake::Shell.current
       if conn.kind_of?(Pwrake::Shell)
         res    = conn.system(*cmd)
@@ -65,16 +60,11 @@ module Pwrake
         status = $?
         status = Rake::PseudoStatus.new(1) if !res && status.nil?
       end
-
-      #tm.finish("status=%s cmd=%s"%[status.exitstatus,cmd_log])
       [res,status]
     end
 
     # Pwrake version of backquote command
     def pwrake_backquote(cmd)
-      cmd_log = cmd.inspect
-      #tm = Pwrake::Timer.new("bq",cmd_log)
-
       conn = Pwrake::Shell.current
       if conn.kind_of?(Pwrake::Shell)
         res    = conn.backquote(*cmd)
@@ -84,8 +74,6 @@ module Pwrake
         status = $?
         status = Rake::PseudoStatus.new(1) if status.nil?
       end
-
-      #tm.finish("status=%s cmd=%s"%[status.exitstatus,cmd_log])
       [res,status]
     end
 

data/lib/pwrake/gfarm/gfarm_postprocess.rb

@@ -1,5 +1,7 @@
 module Pwrake
 
+  class GfwhereError < StandardError; end
+
   class GfarmPostprocess
 
     def initialize(selector)
@@ -22,11 +24,11 @@ module Pwrake
       end
       s = @reader.get_line
       if s.nil?
-        raise "gfwhere: unexpected end"
+        raise GfwhereError,"lost connection to gfwhere-pipe"
       end
       s.chomp!
       if s != filename
-        raise "gfwhere: file=#{filename}, result=#{s}"
+        raise GfwhereError,"path mismatch: send=#{filename}, return=#{s}"
       end
       while s = @reader.get_line
         s.chomp!

data/lib/pwrake/master/master.rb

@@ -223,7 +223,11 @@ module Pwrake
         end
       end
       send_task_to_idle_core
-      setup_fiber(t)
+      if ending?
+        @post_pool.finish # need?
+      else
+        setup_fiber(t)
+      end
     end
 
     def setup_fiber(t)

data/lib/pwrake/master/master_application.rb

@@ -179,8 +179,10 @@ module Pwrake
         "[Pw] Show Pwrake configuration options",
         lambda {|value| options.show_conf = true }
        ],
-       ['--report LOGDIR',"[Pw] Generate `report.html' (Report of workflow statistics) in LOGDIR and exit.",
-        lambda {|value| options.report_dir = value }
+       ['--report [LOGDIR]',"[Pw] Generate `report.html' (Report of workflow statistics) in LOGDIR and exit.",
+        lambda {|value|
+          options.report_dir = value || Dir.pwd
+        }
        ],
        ['--report-image IMAGE_TYPE',"[Pw] Gnuplot output format (png,jpg,svg etc.) in report.html.",
         lambda {|value| options.report_image = value }

data/lib/pwrake/option/option.rb

@@ -172,7 +172,7 @@ module Pwrake
         'PLOT_PARTITION',
 
         ['HOSTFILE','HOSTS'],
-        ['LOG_DIR','LOG',
+        ['LOG_DIR',
           proc{|v|
             if v
               if v == "" || !v.kind_of?(String)
@@ -223,7 +223,7 @@ module Pwrake
          }],
         ['NUM_THREADS', proc{|v| v && v.to_i}],
         ['SHELL_START_INTERVAL', proc{|v| (v || 0.012).to_f}],
-        ['HEARTBEAT', proc{|v| (v || 240).to_i}],
+        ['HEARTBEAT', proc{|v| v && v.to_i}],
         ['RETRY', proc{|v| (v || 1).to_i}],
         ['MASTER_HOSTNAME', proc{|v| (v || Socket.gethostname).chomp}],
         ['WORK_DIR', proc{|v|

data/lib/pwrake/report/parallelism.rb

@@ -59,59 +59,63 @@ module Pwrake
       d
     end
 
-    def plot_parallelism(file,fmt)
-      a = count_start_end_from_csv(file)
+
+    def plot_parallelism(csvtable, base, fmt)
+      a = count_start_end_from_csv_table(csvtable)
       return if a.size < 4
 
       #density = exec_density(a)
 
-      base = file.sub(/\.csv$/,"")
-      fpara = base+"_para.dat"
+      fimg = base+'/parallelism.'+fmt
 
       n = a.size
+      i = 0
       y = 0
       y_max = 0
 
-      File.open(fpara,"w") do |f|
-        i = 0
+      para = []
+      begin
         t = 0
         y_pre = 0
-        begin
-          while i < n
-            if a[i][0]-t > 0.001
-              f.printf "%.3f %d\n", t, y_pre
-              t = a[i][0]
-              f.printf "%.3f %d\n", t, y
-            end
-            y += a[i][1]
-            y_pre = y
-            y_max = y if y > y_max
-            i += 1
+        while i < n
+          if a[i][0]-t > 0.001
+            para.push "%.3f %d" % [t, y_pre]
+            t = a[i][0]
+            para.push "%.3f %d" % [t, y]
           end
-        rescue
-          p a[i]
+          y += a[i][1]
+          y_pre = y
+          y_max = y if y > y_max
+          i += 1
         end
+      rescue
+        p a[i]
       end
 
       t_end = (a.last)[0]
 
-      IO.popen("gnuplot","r+") do |f|
-        f.puts "
+      if system("which gnuplot >/dev/null 2>&1")
+        IO.popen("gnuplot","r+") do |f|
+          f.print "
 set terminal #{fmt}
-set output '#{base}.#{fmt}'
-#set rmargin 10
-set title '#{base}'
+set output '#{fimg}'
 set xlabel 'time (sec)'
-set ylabel 'parallelism'
+set ylabel '# of cores'
+set ytics nomirror
 
 set arrow 1 from #{t_end},#{y_max*0.5} to #{t_end},0 linecolor rgb 'blue'
-set label 1 at first #{t_end},#{y_max*0.5} right \"#{t_end}\\nsec\" textcolor rgb 'blue'
+set label 1 \"#{t_end}\\nsec\" at first #{t_end},#{y_max*0.5} right front textcolor rgb 'blue'
 
-plot '#{fpara}' w l axis x1y1 title 'parallelism'
+plot '-' w l notitle
 "
+          para.each do |x|
+            f.puts x
+          end
+        end
       end
 
-      #puts "Parallelism plot: #{base}.#{fmt}"
+      #puts "Parallelism plot: #{fimg}"
+      fimg
     end
 
 
@@ -154,10 +158,8 @@ plot '#{fpara}' w l axis x1y1 title 'parallelism'
         f.print "
 set terminal #{fmt}
 set output '#{fimg}'
-#set rmargin 10
-set title '#{base}'
 set xlabel 'time (sec)'
-set ylabel 'parallelism'
+set ylabel '# of cores'
 set y2tics
 set ytics nomirror
 set y2label 'exec/sec'
@@ -291,9 +293,8 @@ plot '-' w l axis x1y1 title 'parallelism', '-' w l axis x1y2 title 'exec/sec'
         f.print "
 set terminal #{fmt}
 set output '#{fimg}'
-set title '#{base}'
 set xlabel 'time (sec)'
-set ylabel 'parallelism'
+set ylabel '# of cores'
 "
         f.print "plot "
         f.puts para.map{|cmd,re| "'-' w l title #{cmd.inspect}"}.join(",")

data/lib/pwrake/report/report.rb

@@ -227,7 +227,7 @@ EOL
       end
       html << "</table>\n"
       html << "<h2>Parallelism</h2>\n"
-      fimg = Parallelism.plot_parallelism2(@sh_table,@base,@img_fmt)
+      fimg = Parallelism.plot_parallelism(@sh_table,@base,@img_fmt)
       html << "<img src='./#{File.basename(fimg)}' align='top'/></br>\n"
 
       html << "<h2>Parallelism by command</h2>\n"
@@ -238,11 +238,9 @@ EOL
       fimg2 = Parallelism.plot_parallelism_by_host(@sh_table,@base,@img_fmt)
       html << "<img src='./#{File.basename(fimg2)}' align='top'/></br>\n"
 
-      html << "<h2>Command statistics</h2>\n"
+      html << "<h2>Command time statistics</h2>\n"
       html << "<table>\n"
-      html << "<tr><th>command</th>"
       html << Stat.html_th
-      html << "</tr>\n"
       @cmd_stat.each do |cmd,stat|
         html << "<tr><td>#{cmd}</td>"
         html << stat.html_td

data/lib/pwrake/report/stat.rb

@@ -10,9 +10,9 @@ module Pwrake
         @max = data.max
         @sum = data.inject(0){|s,x| s+x}
         @mean = @sum/@n
-        @median = _median
+        @median = calc_median
         @mean_absolute_deviation = data.inject(0){|s,x| (x-@mean).abs} / @n
-        if @n>1
+        if false and @n>1
           @variance = data.inject(0){|s,x| y=x-@mean; y**2} / (@n-1)
           @sdev = Math.sqrt(@variance)
           @skew = data.inject(0){|s,x| y=(x-@mean)/@sdev; y**3} / @n
@@ -55,7 +55,7 @@ module Pwrake
       end
     end
 
-    def _median
+    def calc_median
       if @n==1
         @data[0]
       elsif @n==2
@@ -73,53 +73,34 @@ module Pwrake
       end
     end
 
-    def report
-      case @n
-      when 0
-        "no data"
-      when 1
-        "n=1 mean=%g"%@mean
-      else
-        "n=%i mean=%g median=%g min=%g max=%g sdev=%g skew=%g kurt=%g" %
-          [@n, @mean, @median, @min, @max, @sdev||0, @skew||0, @kurt||0]
-        "n=%i mean=%g median=%g min=%g max=%g sdev=%g" %
-          [@n, @mean, @median, @min, @max, @sdev||0]
-      end
-    end
-
-    def stat_array
-      [@n, @mean, @median, @min, @max, @sdev||0]
-    end
-
     def fmt(x)
       case x
       when Numeric
-        "%g"%x
+        a = x.abs
+        if a == 0
+          "0"
+        elsif a < 1
+          "%.3g" % x
+        else
+          "%.3f" % x
+        end
       else
         x.to_s
       end
     end
 
     def html_td
-      "<td>%i</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td><td>%s</td>" %
-        [@n, fmt(@mean), fmt(@median), fmt(@min), fmt(@max), fmt(@sdev)]
+      '<td align="right">%i</td><td align="right">%s</td><td align="right">%s</td><td align="right">%s</td><td align="right">%s</td><td align="right">%s</td>' %
+        [@n, fmt(@sum), fmt(@mean), fmt(@median), fmt(@min), fmt(@max)]
     end
 
     def self.html_th
-      "<th>%s</th><th>%s</th><th>%s</th><th>%s</th><th>%s</th><th>%s</th>" %
-      %w[n mean median min max sdev]
-    end
-
-    def report2
-      case @n
-      when 0
-        "  no data"
-      when 1
-        "  n=1\n  mean=%g"%@mean
-      else
-        "  n=%i\n  mean=%g\n  median=%g\n  min=%g\n  max=%g\n  sdev=%g\n  skew=%g\n  kurt=%g" %
-          [@n, @mean, @median, @min, @max, @sdev||0, @skew||0, @kurt||0]
-      end
+      a = %w[command count sum mean median min max]
+      "<tr>" + "<th></th>"*2 +
+        "<th colspan=#{a.size-2}>time (seconds)</th>" +
+        "</tr>\n<tr>" +
+        "<th>%s</th>" * a.size % a +
+        "</tr>\n"
     end
 
   end

data/lib/pwrake/task/file_task_algorithm.rb

@@ -0,0 +1,16 @@
+module Pwrake
+
+  module FileTaskAlgorithm
+
+    # Cache time stamp to reduce load on file system.
+    def timestamp
+      @file_mtime ||
+        if File.exist?(name)
+          @file_mtime = File.mtime(name.to_s)
+        else
+          Rake::LATE
+        end
+    end
+
+  end
+end

data/lib/pwrake/task/task_algorithm.rb

@@ -33,8 +33,8 @@ module Pwrake
       new_chain = InvocationChain.append(self, invocation_chain)
       @lock.synchronize do
         if application.options.trace
-          #Log.debug "** Search #{name} #{format_search_flags}"
-          application.trace "** Search #{name} #{format_search_flags}"
+          #Log.debug "** Search #{name}#{format_search_flags}"
+          application.trace "** Search #{name}#{format_search_flags}"
         end
 
         return true if @already_finished # <<--- competition !!!
@@ -80,7 +80,7 @@ module Pwrake
       flags << "finished" if @already_finished
       flags << "first_time" unless @already_searched
       flags << "not_needed" unless needed?
-      flags.empty? ? "" : "(" + flags.join(", ") + ")"
+      flags.empty? ? "" : " (" + flags.join(", ") + ")"
     end
     private :format_search_flags
 

data/lib/pwrake/version.rb

@@ -1,3 +1,3 @@
 module Pwrake
-  VERSION = "2.2.7"
+  VERSION = "2.2.8"
 end

data/lib/pwrake/worker/writer.rb

@@ -9,21 +9,24 @@ module Pwrake
       @out = $stderr
       @mutex = Mutex.new
       @cond_hb = true
-      @heartbeat = 120
+      @heartbeat = nil
       @thread = Thread.new{ heartbeat_loop }
     end
 
     attr_accessor :out
 
     def heartbeat=(t)
-      @heartbeat = t.to_i
+      if t
+        t = t.to_i
+        t = 15 if t < 15
+      end
+      @heartbeat = t
       @thread.run
     end
 
     def heartbeat_loop
-      sleep
       loop do
-        sleep(@heartbeat)
+        @heartbeat ? sleep(@heartbeat) : sleep
         if @cond_hb
           _puts "heartbeat"
         end
@@ -36,9 +39,9 @@ module Pwrake
     end
 
     def puts(s)
+      _puts(s)
       @cond_hb = false
       @thread.run
-      _puts(s)
     end
 
     def flush

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pwrake
 version: !ruby/object:Gem::Version
-  version: 2.2.7
+  version: 2.2.8
 platform: ruby
 authors:
 - Masahiro TANAKA
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-11-30 00:00:00.000000000 Z
+date: 2019-01-21 00:00:00.000000000 Z
 dependencies: []
 description: Parallel and distributed Rake for workflow execution on multicores, clusters,
   clouds using SSH. It has locality-aware scheduling designed for Gfarm file system.
@@ -70,6 +70,7 @@ files:
 - lib/pwrake/report/report_multi.rb
 - lib/pwrake/report/stat.rb
 - lib/pwrake/report/task_stat.rb
+- lib/pwrake/task/file_task_algorithm.rb
 - lib/pwrake/task/task_algorithm.rb
 - lib/pwrake/task/task_manager.rb
 - lib/pwrake/task/task_property.rb
@@ -121,8 +122,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: Parallel and distributed Rake for workflow execution on multicores, clusters,