pwrake 2.2.0 → 2.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f781cdfe39747649da0fafb2ebb37e579d294483
-  data.tar.gz: 5985be277367367065e9fef64188fc43ec8a2de4
+  metadata.gz: 165b50b8450d65b495aefffe22cc3a8e02ae3828
+  data.tar.gz: 210a2fcb2b42cb1b70696e84595e20dd71332c83
 SHA512:
-  metadata.gz: 847964dd1146902e2f6f5988a26188184845e1b7879ed43a311560266e90d33d3c5febe7bb6727cc2003075b540046028a6cbaea6c28451888866d1e43aa1cfe
-  data.tar.gz: 76af1e8e961e9ff22e1c6ca0626a416eb73f2f43a9ce83c0c4b72a311841fd319b6a99c1fbf9594cbf5a4efafe8dc00833dae772550b705837f58e40f48ad69a
+  metadata.gz: ce57a49375d39fc42a4a785bb9e703fe4f90af8f5ec2db63c4a1d5fb7beb3c3d3f44b37742a1cbb8c9f40621cbc3d66a8c458cb17a6d530d9ad43374bda829e9
+  data.tar.gz: fae5ad1e68b9b7f87ab12428e9826a50ed8809120c2f4a5a45215518ff7e590f82051d6e4fb7e63914222b81c6dbea762c8ef7cfaf8e88a4823f6e176d325df0

data/bin/pwrake

@@ -15,6 +15,7 @@ require "pwrake/master/master_application"
 require "pwrake/task/task_manager"
 require "pwrake/task/task_algorithm"
 require "pwrake/branch/branch_application"
+
 class Rake::Application
   include Pwrake::BranchApplication
   prepend Pwrake::MasterApplication
@@ -22,6 +23,7 @@ class Rake::Application
 end
 class Rake::Task
   include Pwrake::TaskAlgorithm
+  prepend Pwrake::TaskInvoke
 end
 
 # does NOT exit when writing to broken pipe

data/lib/pwrake/branch/communicator_set.rb

@@ -14,7 +14,7 @@ class CommunicatorSet
     @error_host = []
     @initial_communicators = []
     if hb = @option[:heartbeat]
-      @heartbeat_timeout = hb + 15
+      @heartbeat_timeout = hb + 30
     end
     init_hosts
   end

data/lib/pwrake/branch/shell.rb

@@ -72,11 +72,6 @@ module Pwrake
       false
     end
 
-    def set_current_task(task_id,task_name)
-      @task_id = task_id
-      @task_name = task_name
-    end
-
     def backquote(*command)
       command = command.join(' ')
       @lock.synchronize do
@@ -156,7 +151,7 @@ module Pwrake
       ensure
         end_time = Time.now
         @status = @@profiler.profile(@task_id, @task_name, cmd,
-                                     start_time, end_time, host, @status)
+                     start_time, end_time, host, @ncore, @status)
       end
     end
 
@@ -232,6 +227,7 @@ module Pwrake
             @task_id = task_id
             @task_name = task_name
             task = Rake.application[task_name]
+            @ncore = task.wrapper.n_used_cores
             begin
               task.execute(task.arguments) if task.needed?
               result = "taskend:#{@id}:#{task.name}"

data/lib/pwrake/branch/shell_profiler.rb

@@ -6,7 +6,7 @@ module Pwrake
 
     HEADER_FOR_PROFILE =
       %w[exec_id task_id task_name command
-         start_time end_time elap_time host status]
+         start_time end_time elap_time host ncore status]
 
     HEADER_FOR_GNU_TIME =
       %w[realtime systime usrtime maxrss averss memsz
@@ -85,7 +85,8 @@ module Pwrake
       t.strftime("%F %T.%L")
     end
 
-    def profile(task_id, task_name, cmd, start_time, end_time, host=nil, status=nil)
+    def profile(task_id, task_name, cmd, start_time, end_time,
+                host=nil, ncore=nil, status=nil)
       id = @lock.synchronize do
         id = @id
         @id += 1
@@ -96,7 +97,7 @@ module Pwrake
                format_time(start_time),
                format_time(end_time),
                "%.3f" % (end_time-start_time),
-               host, status ]
+               host, ncore, status ]
       end
       case status
       when ""

data/lib/pwrake/master/master.rb

@@ -144,7 +144,6 @@ module Pwrake
       @task_queue = queue_class.new(@hostinfo_by_id)
 
       @branch_setup_thread = Thread.new do
-        #@channels.each do |chan|
         create_fiber(@hdl_set) do |hdl|
           while s = hdl.get_line
             case s
@@ -158,12 +157,6 @@ module Pwrake
           end
         end
         @selector.run
-        @killed = 0
-        [:TERM,:INT].each do |sig|
-          Signal.trap(sig) do
-            signal_trap(sig)
-          end
-        end
       end
 
     end
@@ -190,8 +183,11 @@ module Pwrake
     end
 
     def invoke(t, args)
+      Log.debug "Master#invoke start: #{t.class}[#{t.name}]"
       @failed = false
       t.pw_search_tasks(args)
+      return if @running
+      @running = true
 
       if @option['GRAPH_PARTITION']
         setup_postprocess0
@@ -210,8 +206,17 @@ module Pwrake
 
       setup_postprocess1
       @branch_setup_thread.join
+      @killed = 0
+      [:TERM,:INT].each do |sig|
+        Signal.trap(sig) do
+          signal_trap(sig)
+        end
+      end
       send_task_to_idle_core
-      #
+      setup_fiber(t)
+    end
+
+    def setup_fiber(t)
       n_retry = @option["RETRY"]
       create_fiber(@hdl_set) do |hdl|
         while s = hdl.get_line

data/lib/pwrake/master/master_application.rb

@@ -52,6 +52,10 @@ module Pwrake
       @master.invoke(t,args)
     end
 
+    def invoke(t,*args)
+      @master.invoke(t,args)
+    end
+
     def standard_rake_options
       opts = super
       opts.each_with_index do |a,i|

data/lib/pwrake/mpi/branch.rb

@@ -19,6 +19,7 @@ end
 
 class Rake::Task
   include Pwrake::TaskAlgorithm
+  prepend Pwrake::TaskInvoke
 end
 
 module Pwrake

data/lib/pwrake/queue/queue_array.rb

@@ -68,18 +68,18 @@ module Pwrake
 
     def shift(host_info=nil)
       return super() unless host_info
-      tw_found = nil
+      i_found = nil
       (size-1).downto(0) do |i|
         tw = at(i)
         if tw.acceptable_for(host_info)
           if tw.untried_host?(host_info)
             return delete_at(i)
           else
-            tw_found ||= tw
+            i_found ||= i
           end
         end
       end
-      tw_found
+      (i_found) ? delete_at(i_found) : nil
     end
   end
 
@@ -90,18 +90,18 @@ module Pwrake
 
     def shift(host_info=nil)
       return super() unless host_info
-      tw_found = nil
+      i_found = nil
       size.times do |i|
         tw = at(i)
         if tw.acceptable_for(host_info)
           if tw.untried_host?(host_info)
             return delete_at(i)
           else
-            tw_found ||= tw
+            i_found ||= i
           end
         end
       end
-      tw_found
+      (i_found) ? delete_at(i_found) : nil
     end
   end
 
@@ -172,18 +172,18 @@ module Pwrake
     end
 
     def pop_last_rank(r,host_info)
-      tw_found = nil
+      i_found = nil
       (size-1).downto(0) do |i|
         tw = at(i)
         if tw.rank == r && tw.acceptable_for(host_info)
           if tw.untried_host?(host_info)
             return delete_at(i)
           else
-            tw_found ||= tw
+            i_found ||= i
           end
         end
       end
-      tw_found
+      (i_found) ? delete_at(i_found) : nil
     end
 
     def hrf_delete(t)

data/lib/pwrake/queue/task_queue.rb

@@ -88,23 +88,23 @@ module Pwrake
         @hostinfo_by_id.each_value do |host_info|
           #Log.debug "TaskQueue#deq_turn host_info=#{host_info.name}"
           if (n = host_info.idle_cores) && n > 0
-          if turn_empty?(turn)
-            return queued
-          elsif tw = deq_impl(host_info,turn)
-            n_task_cores = tw.n_used_cores(host_info)
-            Log.debug "deq: #{tw.name} n_task_cores=#{n_task_cores}"
-            if host_info.idle_cores < n_task_cores
-              m = "task.n_used_cores=#{n_task_cores} must be "+
-                "<= host_info.idle_cores=#{host_info.idle_cores}"
-              Log.fatal m
-              raise RuntimeError,m
-            else
-              yield(tw,host_info,n_task_cores)
-              count += 1
-              queued += 1
+            if turn_empty?(turn)
+              return queued
+            elsif tw = deq_impl(host_info,turn)
+              n_task_cores = tw.n_used_cores(host_info)
+              Log.debug "deq: #{tw.name} n_task_cores=#{n_task_cores}"
+              if host_info.idle_cores < n_task_cores
+                m = "task.n_used_cores=#{n_task_cores} must be "+
+                  "<= host_info.idle_cores=#{host_info.idle_cores}"
+                Log.fatal m
+                raise RuntimeError,m
+              else
+                yield(tw,host_info,n_task_cores)
+                count += 1
+                queued += 1
+              end
             end
           end
-          end
         end
         break if count == 0
       end

data/lib/pwrake/report/parallelism.rb

@@ -3,22 +3,28 @@ module Pwrake
   module Parallelism
     module_function
 
+    def push_time_event(a, row, start_time)
+      command = row['command']
+      if command == 'pwrake_profile_start'
+        start_time[0] = Time.parse(row['start_time'])
+      elsif command == 'pwrake_profile_end'
+        t = Time.parse(row['start_time']) - start_time[0]
+        a << [t,0]
+      elsif start_time[0]
+        n = begin Integer(row['ncore']) rescue 1 end
+        t = Time.parse(row['start_time']) - start_time[0]
+        a << [t,+n]
+        t = Time.parse(row['end_time']) - start_time[0]
+        a << [t,-n]
+      end
+    end
+
     def count_start_end_from_csv(file)
       a = []
-      start_time = nil
+      start_time = []
 
       CSV.foreach(file,:headers=>true) do |row|
-        if row['command'] == 'pwrake_profile_start'
-          start_time = Time.parse(row['start_time'])
-        elsif row['command'] == 'pwrake_profile_end'
-          t = Time.parse(row['start_time']) - start_time
-          a << [t,0]
-        elsif start_time
-          t = Time.parse(row['start_time']) - start_time
-          a << [t,+1]
-          t = Time.parse(row['end_time']) - start_time
-          a << [t,-1]
-        end
+        push_time_event(a, row, start_time)
       end
 
       a.sort{|x,y| x[0]<=>y[0]}
@@ -26,20 +32,10 @@ module Pwrake
 
     def count_start_end_from_csv_table(csvtable)
       a = []
-      start_time = nil
+      start_time = []
 
       csvtable.each do |row|
-        if row['command'] == 'pwrake_profile_start'
-          start_time = Time.parse(row['start_time'])
-        elsif row['command'] == 'pwrake_profile_end'
-          t = Time.parse(row['start_time']) - start_time
-          a << [t,0]
-        elsif start_time
-          t = Time.parse(row['start_time']) - start_time
-          a << [t,+1]
-          t = Time.parse(row['end_time']) - start_time
-          a << [t,-1]
-        end
+        push_time_event(a, row, start_time)
       end
 
       a.sort{|x,y| x[0]<=>y[0]}
@@ -48,16 +44,15 @@ module Pwrake
     def exec_density(a)
       reso = 0.1
       delta = 1/reso
-      t_end = (a.last)[0]
-      n = (t_end/reso).to_i + 1
-      i = 0
-      t_next = reso
+      t_end = a.last[0]
+      n = (t_end/reso).round + 1
       d = (n+1).times.map{|i| [reso*i,0]}
+      i = 0
       a.each do |x|
         while d[i+1][0] <= x[0]
           i += 1
         end
-        if x[1] == 1
+        if x[1] > 0
           d[i][1] += delta
         end
       end
@@ -68,21 +63,21 @@ module Pwrake
       a = count_start_end_from_csv(file)
       return if a.size < 4
 
-      density = exec_density(a)
+      #density = exec_density(a)
 
       base = file.sub(/\.csv$/,"")
       fpara = base+"_para.dat"
 
       n = a.size
-      i = 0
       y = 0
       y_max = 0
 
       File.open(fpara,"w") do |f|
+        i = 0
+        t = 0
+        y_pre = 0
         begin
-          t = 0
-          y_pre = 0
-          n.times do |i|
+          while i < n
             if a[i][0]-t > 0.001
               f.printf "%.3f %d\n", t, y_pre
               t = a[i][0]
@@ -91,6 +86,7 @@ module Pwrake
             y += a[i][1]
             y_pre = y
             y_max = y if y > y_max
+            i += 1
           end
         rescue
           p a[i]
@@ -136,7 +132,7 @@ plot '#{fpara}' w l axis x1y1 title 'parallelism'
       begin
         t = 0
         y_pre = 0
-        n.times do |i|
+        while i < n
           if a[i][0]-t > 0.001
             para.push "%.3f %d" % [t, y_pre]
             t = a[i][0]
@@ -145,6 +141,7 @@ plot '#{fpara}' w l axis x1y1 title 'parallelism'
           y += a[i][1]
           y_pre = y
           y_max = y if y > y_max
+          i += 1
         end
       rescue
         p a[i]
@@ -185,72 +182,65 @@ plot '-' w l axis x1y1 title 'parallelism', '-' w l axis x1y2 title 'exec/sec'
       fimg
     end
 
+
+    def push_time_by_key(h, row, key, start_time)
+      command = row['command']
+      if command == 'pwrake_profile_start'
+        start_time[0] = Time.parse(row['start_time'])
+      elsif command == 'pwrake_profile_end'
+        t = Time.parse(row['start_time']) - start_time[0]
+        h.each_value do |v|
+          v << [t,0]
+        end
+      elsif start_time[0]
+        a = (h[key] ||= [])
+        n = begin Integer(row['ncore']) rescue 1 end
+        t = Time.parse(row['start_time']) - start_time[0]
+        a << [t,+n]
+        t = Time.parse(row['end_time']) - start_time[0]
+        a << [t,-n]
+      end
+    end
+
     def read_time_by_host_from_csv(csvtable)
       a = {}
-      start_time = nil
+      start_time = []
 
       csvtable.each do |row|
-        host = row['host']
-        if row['command'] == 'pwrake_profile_start'
-          start_time = Time.parse(row['start_time'])
-        elsif row['command'] == 'pwrake_profile_end'
-          t = Time.parse(row['start_time']) - start_time
-          a.each do |h,v|
-            v << [t,0]
-          end
-        elsif start_time
-          a[host] ||= []
-          t = Time.parse(row['start_time']) - start_time
-          a[host] << [t,+1]
-          t = Time.parse(row['end_time']) - start_time
-          a[host] << [t,-1]
-        end
+        push_time_by_key(a, row, row['host'], start_time)
       end
       a
     end
 
 
-    def get_command_key(s, pattern)
+    def get_command_key(s, pattern=[])
       pattern.each do |cmd,regex|
         if regex =~ s
           return cmd
         end
       end
-      if /\(([^()]+)\)/ =~ s
-        s = $1
-      end
-      a = s.split(/;/)
-      a.each do |x|
-        if /^\s*(\S+)/ =~ x
-          k = $1
-          next if k=='cd'
-          return k
+      case s
+      when /^\s*\((.*)$/
+        get_command_key($1)
+      when /^\s*([\w.,~^\/=+-]+)(.*)$/
+        cmd, rest = $1, $2
+        if cmd == "cd" && /[^;]*;(.*)$/ =~ rest
+          return get_command_key($1)
+        else
+          cmd
         end
+      else
+        s[0..15]
       end
-      nil
     end
 
     def count_start_end_by_pattern(csvtable, pattern)
       h = Hash.new
-      start_time = nil
+      start_time = []
 
       csvtable.each do |row|
-        command = row['command']
-        if command == 'pwrake_profile_start'
-          start_time = Time.parse(row['start_time'])
-        elsif command == 'pwrake_profile_end'
-          t = Time.parse(row['start_time']) - start_time
-          h.values.each do |a|
-            a << [t,0]
-          end
-        elsif start_time
-          cmd = get_command_key(command, pattern)
-          h[cmd] ||= []
-          t = Time.parse(row['start_time']) - start_time
-          h[cmd] << [t,+1]
-          t = Time.parse(row['end_time']) - start_time
-          h[cmd] << [t,-1]
-        end
+        cmd = get_command_key(row['command'], pattern)
+        push_time_by_key(h, row, cmd, start_time)
       end
 
       h.each do |k,a|
@@ -272,7 +262,7 @@ plot '-' w l axis x1y1 title 'parallelism', '-' w l axis x1y2 title 'exec/sec'
         begin
           t = 0
           y_pre = 0
-          n.times do |i|
+          while i < n
             if a[i][0]-t > 0.001
               dat.push "%.3f %d" % [t, y_pre]
               t = a[i][0]
@@ -281,6 +271,7 @@ plot '-' w l axis x1y1 title 'parallelism', '-' w l axis x1y2 title 'exec/sec'
             y += a[i][1]
             y_pre = y
             y_max = y if y > y_max
+            i += 1
           end
         rescue
           p a[i]
@@ -303,7 +294,7 @@ set xlabel 'time (sec)'
 set ylabel 'parallelism'
 "
         f.print "plot "
-        f.puts para.map{|cmd,re| "'-' w l title '#{cmd}'"}.join(",")
+        f.puts para.map{|cmd,re| "'-' w l title #{cmd.inspect}"}.join(",")
         para.each do |cmd,dat|
           dat.each do |x|
             f.puts x
@@ -321,9 +312,8 @@ set ylabel 'parallelism'
       resolution = Rational(1,10)
 
       a = a.sort{|x,y| x[0]<=>y[0]}
-      t_end = (a.last)[0]
-
-      ngrid = (t_end/resolution).floor
+      #t_end = (a.last)[0]
+      #ngrid = (t_end/resolution).floor
       grid = [[0,0]]
 
       j = 0

data/lib/pwrake/report/report.rb

@@ -135,18 +135,42 @@ EOL
     end
 
     def get_command(s)
-      if /\(([^()]+)\)/ =~ s
-        s = $1
-      end
-      a = s.split(/;/)
-      a.each do |x|
-        if /^\s*(\S+)/ =~ x
-          k = $1
-          next if k=='cd'
-          return k
+      case s
+      when /^\s*\((.*)$/
+        get_command($1)
+      when /^\s*([\w.,~^\/=+-]+)(.*)$/
+        cmd, rest = $1, $2
+        case cmd
+        when "cd"
+          if /[^;]*;(.*)$/ =~ rest
+            return get_command($1)
+          end
+        when /^ruby[\d.]*$/
+          case rest
+          when /([\w.,~^\/=+-]+\.rb)\b/
+            return "#{cmd} #{$1}"
+          when /\s+-e\s+("[^"]*"|'[^']*'|\S+)/
+            return "#{cmd} -e #{$1}"
+          end
+        when /^python[\d.]*$/
+          case rest
+          when /([\w.,~^\/=+-]+\.py)\b/
+            return "#{cmd} #{$1}"
+          when /\s+-c\s+("[^"]*"|'[^']*'|\S+)/
+            return "python -c #{$1}"
+          end
+        when /^perl[\d.]*$/
+          case rest
+          when /([\w.,~^\/=+-]+\.pl)\b/
+            return "#{cmd} #{$1}"
+          when /\s+-e\s+("[^"]*"|'[^']*'|\S+)/
+            return "#{cmd} -e #{$1}"
+          end
         end
+        cmd
+      else
+        s[0..15]
       end
-      nil
     end
 
     def make_cmd_stat
@@ -299,9 +323,9 @@ set logscale x
 set title 'histogram of elapsed time'"
         a = []
 
-        command_list.each_with_index do |n,i|
+        command_list.each_with_index do |cmd,i|
           a << "'-' w histeps ls #{i+1} title ''"
-          a << "'-' w lines ls #{i+1} title '#{n}'"
+          a << "'-' w lines ls #{i+1} title #{cmd.inspect}"
         end
         f.puts "plot "+ a.join(',')
 

data/lib/pwrake/task/task_algorithm.rb

@@ -5,22 +5,25 @@ module Pwrake
 
   module TaskAlgorithm
 
-    attr_reader :wrapper
     attr_reader :subsequents
     attr_reader :arguments
     attr_reader :property
     attr_reader :unfinished_prereq
 
+    def wrapper
+      if @wrapper.nil?
+        raise "TaskWrapper is not defined for #{self.class}[#{name}]"
+      end
+      @wrapper
+    end
+
     def pw_search_tasks(args)
-      Log.debug "#{self.class}#pw_search_tasks start, args=#{args.inspect}"
+      Log.debug "#{self.class}#pw_search_tasks start, task=#{name} args=#{args.inspect}"
       tm = Time.now
       task_args = TaskArguments.new(arg_names, args)
-      #timer = Timer.new("search_task")
-      #h = application.pwrake_options['HALT_QUEUE_WHILE_SEARCH']
-      #application.task_queue.synchronize(h) do
-	search_with_call_chain(nil, task_args, InvocationChain::EMPTY)
-      #end
-      #timer.finish
+      # not synchronize owing to fiber
+      search_with_call_chain(nil, task_args, InvocationChain::EMPTY)
+      #
       Log.debug "#{self.class}#pw_search_tasks end #{Time.now-tm}"
     end
 
@@ -30,7 +33,7 @@ module Pwrake
       new_chain = InvocationChain.append(self, invocation_chain)
       @lock.synchronize do
         if application.options.trace
-          #Log.info "** Search #{name} #{format_search_flags}"
+          #Log.debug "** Search #{name} #{format_search_flags}"
           application.trace "** Search #{name} #{format_search_flags}"
         end
 
@@ -48,7 +51,7 @@ module Pwrake
             search_prerequisites(task_args, new_chain)
           end
           #check_and_enq
-          if @unfinished_prereq.empty?
+          if !@already_finished && @unfinished_prereq.empty?
             application.task_queue.enq(@wrapper)
           end
         end
@@ -82,23 +85,18 @@ module Pwrake
     private :format_search_flags
 
     def pw_enq_subsequents
-      #t = Time.now
-      #h = application.pwrake_options['HALT_QUEUE_WHILE_SEARCH']
-      #application.task_queue.synchronize(h) do
-        @subsequents.each do |t|        # <<--- competition !!!
-          #u = t.unfinished_prereq.keys
-          #Log.debug "enq_subseq: self=#{self.name} subseq=#{t.name} @unfin_preq=#{u.inspect}"
-          if t && t.check_prereq_finished(self.name)
-            application.task_queue.enq(t.wrapper)
-          end
+      # not synchronize owing to fiber
+      @subsequents.each do |t|        # <<--- competition !!!
+        if t && t.check_prereq_finished(self.name)
+          application.task_queue.enq(t.wrapper)
         end
-      #end
+      end
       @already_finished = true        # <<--- competition !!!
     end
 
     def check_prereq_finished(preq_name=nil)
       @unfinished_prereq.delete(preq_name)
-      @unfinished_prereq.empty?
+      !@already_finished && @unfinished_prereq.empty?
     end
 
     def pw_set_property(property)
@@ -106,5 +104,18 @@ module Pwrake
       self
     end
 
+  end # TaskAlgorithm
+
+
+  module TaskInvoke
+
+    def invoke(*args)
+      Rake.application.invoke(self,*args)
+    end
+
+    def reenable
+      @already_invoked = false
+      @already_searched = false
+    end
   end
 end

data/lib/pwrake/task/task_wrapper.rb

@@ -53,8 +53,9 @@ module Pwrake
         fn = File.join(dir,option['TASK_CSV_FILE'])
         @@task_logger = CSV.open(fn,'w')
         @@task_logger.puts %w[
-          task_id task_name start_time end_time elap_time preq preq_host preq_loc
-          exec_host shell_id has_action executed file_size file_mtime file_host write_loc
+          task_id task_name start_time end_time elap_time preq
+          preq_host preq_loc exec_host shell_id has_action executed ncore
+          file_size file_mtime file_host write_loc
         ]
       end
     end
@@ -64,9 +65,6 @@ module Pwrake
     end
 
     def preprocess
-      if @shell = Pwrake::Shell.current
-        @shell.current_task = self
-      end
       @time_start = Time.now
     end
 
@@ -90,7 +88,6 @@ module Pwrake
       end
       #Log.debug "postprocess time=#{Time.now-tm_taskend}"
       log_task
-      @shell.current_task = nil if @shell
     end
 
     def retry_or_subsequent
@@ -142,18 +139,21 @@ module Pwrake
       write_loc = file_locality(@location)
       #
       if @file_stat
-        fstat = [@file_stat.size, @file_stat.mtime, self.location.join('|'), write_loc]
+        fstat = [ @file_stat.size, @file_stat.mtime,
+                  self.location.join('|'), write_loc ]
       else
         fstat = [nil]*4
       end
       #
-      # task_id task_name start_time end_time elap_time preq preq_host preq_loc
-      # exec_host shell_id has_action executed file_size file_mtime file_host write_loc
+      # task_id task_name start_time end_time elap_time preq preq_host
+      # preq_loc exec_host shell_id has_action executed ncore
+      # file_size file_mtime file_host write_loc
       #
       row = [ @task_id, name, @time_start, @time_end, elap,
               prerequisites, sug_host, preq_loc, @exec_host, @shell_id,
               (actions.empty?) ? 0 : 1,
               (@executed) ? 1 : 0,
+              @n_used_cores,
             ] + fstat
       row.map!{|x|
         if x.kind_of?(Time)

data/lib/pwrake/version.rb

@@ -1,3 +1,3 @@
 module Pwrake
-  VERSION = "2.2.0"
+  VERSION = "2.2.1"
 end

data/lib/pwrake/worker/executor.rb

@@ -2,6 +2,43 @@ module Pwrake
 
   class Executor
 
+    ENV = {
+"OMPI_APP_CTX_NUM_PROCS" => nil,
+"OMPI_COMM_WORLD_LOCAL_RANK" => nil,
+"OMPI_COMM_WORLD_LOCAL_SIZE" => nil,
+"OMPI_COMM_WORLD_NODE_RANK" => nil,
+"OMPI_COMM_WORLD_RANK" => nil,
+"OMPI_COMM_WORLD_SIZE" => nil,
+"OMPI_FILE_LOCATION" => nil,
+"OMPI_FIRST_RANKS" => nil,
+"OMPI_MCA_db" => nil,
+"OMPI_MCA_ess" => nil,
+"OMPI_MCA_ess_base_jobid" => nil,
+"OMPI_MCA_ess_base_vpid" => nil,
+"OMPI_MCA_grpcomm" => nil,
+"OMPI_MCA_initial_wdir" => nil,
+"OMPI_MCA_mpi_yield_when_idle" => nil,
+"OMPI_MCA_orte_app_num" => nil,
+"OMPI_MCA_orte_bound_at_launch" => nil,
+"OMPI_MCA_orte_ess_jobid" => nil,
+"OMPI_MCA_orte_ess_node_rank" => nil,
+"OMPI_MCA_orte_ess_num_procs" => nil,
+"OMPI_MCA_orte_ess_vpid" => nil,
+"OMPI_MCA_orte_hnp_uri" => nil,
+"OMPI_MCA_orte_local_daemon_uri" => nil,
+"OMPI_MCA_orte_num_nodes" => nil,
+"OMPI_MCA_orte_num_restarts" => nil,
+"OMPI_MCA_orte_tmpdir_base" => nil,
+"OMPI_MCA_plm" => nil,
+"OMPI_MCA_pubsub" => nil,
+"OMPI_MCA_shmem_RUNTIME_QUERY_hint" => nil,
+"OMPI_NUM_APP_CTX" => nil,
+"OMPI_UNIVERSE_SIZE" => nil,
+"PMI_RANK" => nil,
+"PMI_FD" => nil,
+"PMI_SIZE" => nil,
+}
+
     def initialize(selector,dir_class,id)
       @selector = selector
       @id = id
@@ -74,7 +111,7 @@ module Pwrake
       @sh_out, @spawn_out = IO.pipe
       @sh_err, @spawn_err = IO.pipe
 
-      @pid = Kernel.spawn(command,
+      @pid = Kernel.spawn(ENV, command,
                           :in=>@spawn_in,
                           :out=>@spawn_out,
                           :err=>@spawn_err,

data/lib/pwrake/worker/invoker.rb

@@ -86,15 +86,18 @@ module Pwrake
     end
 
     def run
+      run_setup
+      run_command
+    ensure
+      close_all
+    end
+
+    def run_setup
       setup_option
       Fiber.new{setup_loop}.resume
       @selector.run
-      Fiber.new{command_callback}.resume
-      @selector.run
     rescue => exc
       @log.error(([exc.to_s]+exc.backtrace).join("\n"))
-    ensure
-      close_all
     end
 
     def setup_option
@@ -123,6 +126,14 @@ module Pwrake
       end
     end
 
+    def run_command
+      Fiber.new{command_callback}.resume
+      @selector.run
+    rescue => exc
+      @log.error(([exc.to_s]+exc.backtrace).join("\n"))
+      kill_all("TERM")
+    end
+
     def command_callback
       while line = get_line(@rd)
         case line
@@ -152,8 +163,7 @@ module Pwrake
       when /^kill:(.*)$/o
         sig = $1
         sig = sig.to_i if /^\d+$/o =~ sig
-        @log.warn "killing worker, signal=#{sig}"
-        @ex_list.each{|id,ex| ex.kill(sig)}
+        kill_all(sig)
         return false
         #
       when /^p$/o
@@ -167,6 +177,11 @@ module Pwrake
       end
     end
 
+    def kill_all(sig)
+      @log.warn "killing worker, signal=#{sig}"
+      @ex_list.each{|id,ex| ex.kill(sig)}
+    end
+
     def close_all
       @log.info "close_all"
       @heartbeat_thread.kill if @heartbeat_thread

data/lib/pwrake/worker/writer.rb

@@ -8,9 +8,8 @@ module Pwrake
     def initialize
       @out = $stderr
       @mutex = Mutex.new
-      @mutex_hb = Mutex.new
       @cond_hb = true
-      @heartbeat = nil
+      @heartbeat = 120
       @thread = Thread.new{ heartbeat_loop }
     end
 
@@ -22,14 +21,13 @@ module Pwrake
     end
 
     def heartbeat_loop
+      sleep
       loop do
-        @heartbeat ? sleep(@heartbeat) : sleep
-        @mutex_hb.synchronize do
-          if @cond_hb
-            _puts "heartbeat"
-          end
-          @cond_hb = true
+        sleep(@heartbeat)
+        if @cond_hb
+          _puts "heartbeat"
         end
+        @cond_hb = true
       end
     end
 
@@ -38,25 +36,11 @@ module Pwrake
     end
 
     def puts(s)
-      @mutex_hb.synchronize do
-        @cond_hb = false
-        @thread.run
-      end
+      @cond_hb = false
+      @thread.run
       _puts(s)
     end
 
-    def _puts(s)
-      begin
-        @mutex.synchronize do
-          @out.print s+"\n"
-        end
-        @out.flush
-      rescue Errno::EPIPE => e
-        @log.info "<#{e.inspect}" if @log
-      end
-      @log.info "<#{s}" if @log
-    end
-
     def flush
       begin
         @out.flush
@@ -68,5 +52,17 @@ module Pwrake
       puts(s) if $DEBUG
     end
 
+    private
+    def _puts(s)
+      begin
+        @mutex.synchronize do
+          @out.print s+"\n"
+        end
+        @out.flush
+      rescue Errno::EPIPE => e
+        @log.info "<#{e.inspect}" if @log
+      end
+      @log.info "<#{s}" if @log
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pwrake
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 2.2.1
 platform: ruby
 authors:
 - Masahiro TANAKA
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-22 00:00:00.000000000 Z
+date: 2017-04-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel
@@ -136,7 +136,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.8
+rubygems_version: 2.6.11
 signing_key: 
 specification_version: 4
 summary: Parallel Workflow engine based on Rake