pwrake 2.2.4 → 2.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fab2af42b144ed4aa2389827ba675f7da65bc7b9
-  data.tar.gz: 81af34ffcac4a101b7b46161b6880812d1b6f0d1
+  metadata.gz: 169f1dc9dd2e2b5dbc2b07d4b4ac11abb74cbc60
+  data.tar.gz: e1efd61651870235421f6a10f531daea0dc6ba83
 SHA512:
-  metadata.gz: 7090c590af93e6b1d2ad56b4235888ddd6edf22de3c107f5f97df805893563e8d41b2b3090352139b13fcb2fdb4d425fb16bcd32320c90dcb40e813e2a88ceb4
-  data.tar.gz: 6f51a807cad92475432495a5eeeaf95b5e892730229fd889283dc2c15c7ec83b2f14a9ca8f7f2a59fa987c1cfad2ec059f5fda6240187531ba0297b002312ba8
+  metadata.gz: dd4a51d8ae3c2c919a553668b4a0009aea6c4c0bb5b80c1f5ad849a8d569f5e600fe296abb3e19ee5f6b1f90e060b17cbe432ac41c74c587aeac3289bc8b7dd9
+  data.tar.gz: 39de4c6360f0d02414dcd6f23fc820f12fef64de7a7b7017c81527df33448b406f5e6d9e9f0a5710d9768ddbff9bf15fb92cbf7800663f2fc2d80a2ee430bc69

data/lib/pwrake/branch/branch.rb

@@ -178,10 +178,12 @@ module Pwrake
             taskq = @task_q.delete(id)
             Log.debug "drop @task_q[#{id}]=#{taskq.inspect}"
             @cs.drop(id)
+            break if @cs.empty?
             #
           when /^kill:(.*)$/o
             sig = $1
             kill(sig)
+            break
             #
           else
             Log.debug "Branch: invalid line from master: #{s}"

data/lib/pwrake/branch/communicator_set.rb

@@ -25,7 +25,7 @@ class CommunicatorSet
 
   attr_reader :selector
 
-  def_delegators :@communicators, :each, :each_value, :values, :size
+  def_delegators :@communicators, :each, :each_value, :values, :size, :empty?
 
   def create_communicators
     Fiber.new do

data/lib/pwrake/branch/shell.rb

@@ -151,11 +151,13 @@ module Pwrake
       ensure
         end_time = Time.now
         @status = @@profiler.profile(@task_id, @task_name, cmd,
-                     start_time, end_time, host, @ncore, @status)
+                     start_time, end_time, host, @ncore, @status,
+                     @gnu_time_status)
       end
     end
 
     def io_read_loop
+      @gnu_time_status = nil
       while s = _gets
         case s
         when /^(\w+):(.*)$/
@@ -167,6 +169,9 @@ module Pwrake
           when "e"
             $stderr.print x[1]+"\n"
             next
+          when "t"
+            @gnu_time_status = x[1].split(',')
+            next
           when "z"
             # see Executor#status_to_str
             status = x[1]

data/lib/pwrake/branch/shell_profiler.rb

@@ -86,18 +86,22 @@ module Pwrake
     end
 
     def profile(task_id, task_name, cmd, start_time, end_time,
-                host=nil, ncore=nil, status=nil)
+                host=nil, ncore=nil, status=nil, gnu_time_status=nil)
       id = @lock.synchronize do
         id = @id
         @id += 1
         id
       end
       if @io
-        _puts [ id, task_id, task_name, cmd,
-               format_time(start_time),
-               format_time(end_time),
-               "%.3f" % (end_time-start_time),
-               host, ncore, status ]
+        prof = [ id, task_id, task_name, cmd,
+                format_time(start_time),
+                format_time(end_time),
+                "%.3f" % (end_time-start_time),
+                host, ncore, status ]
+        if @gnu_time && gnu_time_status
+          prof.concat(gnu_time_status)
+        end
+        _puts prof
       end
       case status
       when ""

data/lib/pwrake/master/master.rb

@@ -67,12 +67,12 @@ module Pwrake
     def signal_trap(sig)
       $stderr.puts "\nSignal trapped. (sig=#{sig} pid=#{Process.pid})"
       if Rake.application.options.debug
-        $stderr.puts "in master thread #{Thread.current}:"
-        $stderr.puts caller
+        $stderr.print "in master thread #{Thread.current}:\n "
+        $stderr.puts caller.join("\n ")
         if @thread
-          $stderr.puts "in branch thread #{@thread}:"
+          $stderr.print "in branch thread #{@thread}:\n "
           if bt = @thread.backtrace
-            $stderr.puts bt.join("\n")
+            $stderr.puts bt.join("\n ")
           end
         end
       end

data/lib/pwrake/nbio.rb

@@ -80,8 +80,8 @@ module NBIO
       to = (timeout) ? timeout*0.75 : nil
       r, w, = @io_class.select(@reader.keys,@writer.keys,[],to)
       check_heartbeat(r,timeout) if timeout
-      r.each{|io| @reader[io].call} if r
-      w.each{|io| @writer[io].call} if w
+      r.each{|io| x = @reader[io]; x.call if x} if r
+      w.each{|io| x = @writer[io]; x.call if x} if w
     rescue IOError => e
       em = "#{e.class.name}: #{e.message}"
       @reader.keys.each do |io|

data/lib/pwrake/option/option_default_filesystem.rb

@@ -24,6 +24,7 @@ module Pwrake
         :base_dir  => "",
         :work_dir  => self['WORK_DIR'],
         :log_dir   => self['LOG_DIR'],
+        :gnu_time  => self['GNU_TIME'],
         :pass_env  => self['PASS_ENV'],
         :ssh_option => self['SSH_OPTION'],
         :heartbeat => self['HEARTBEAT'],

data/lib/pwrake/option/option_gfarm2fs.rb

@@ -26,6 +26,7 @@ module Pwrake
       @worker_option = {
         :log_dir   => self['LOG_DIR'],
         :pass_env  => self['PASS_ENV'],
+        :gnu_time  => self['GNU_TIME'],
         :ssh_option => self['SSH_OPTION'],
         :heartbeat => self['HEARTBEAT'],
         #

data/lib/pwrake/version.rb

@@ -1,3 +1,3 @@
 module Pwrake
-  VERSION = "2.2.4"
+  VERSION = "2.2.5"
 end

data/lib/pwrake/worker/executor.rb

@@ -39,9 +39,10 @@ module Pwrake
 "PMI_SIZE" => nil,
 }
 
-    def initialize(selector,dir_class,id)
+    def initialize(selector,dir_class,id,option)
       @selector = selector
       @id = id
+      @option = option
       @out = Writer.instance
       @log = LogExecutor.instance
       @queue = FiberQueue.new
@@ -110,22 +111,39 @@ module Pwrake
       @spawn_in, @sh_in = IO.pipe
       @sh_out, @spawn_out = IO.pipe
       @sh_err, @spawn_err = IO.pipe
-
-      @pid = Kernel.spawn(ENV, command,
-                          :in=>@spawn_in,
-                          :out=>@spawn_out,
-                          :err=>@spawn_err,
-                          :chdir=>@dir.current,
-                          :pgroup=>true
-                         )
-      @log.info "pid=#{@pid} started. command=#{command.inspect}"
-
-      @thread = Thread.new do
-        @pid2,@status = Process.waitpid2(@pid)
-        @spawn_in.close
-        @spawn_out.close
-        @spawn_err.close
+      if @option[:gnu_time]
+        @sh_gtm, @spawn_gtm = IO.pipe
+        @pid = Kernel.spawn(ENV, wrap_gnu_time(command),
+                            in:@spawn_in,
+                            out:@spawn_out,
+                            err:@spawn_err,
+                            3=>@spawn_gtm,
+                            chdir:@dir.current,
+                            pgroup:true,
+                           )
+        @thread = Thread.new do
+          @pid2,@status = Process.waitpid2(@pid)
+          @spawn_in.close
+          @spawn_out.close
+          @spawn_err.close
+          @spawn_gtm.close
+        end
+      else
+        @pid = Kernel.spawn(ENV, command,
+                            in:@spawn_in,
+                            out:@spawn_out,
+                            err:@spawn_err,
+                            chdir:@dir.current,
+                            pgroup:true
+                           )
+        @thread = Thread.new do
+          @pid2,@status = Process.waitpid2(@pid)
+          @spawn_in.close
+          @spawn_out.close
+          @spawn_err.close
+        end
       end
+      @log.info "pid=#{@pid} started. command=#{command.inspect}"
 
       @rd_out = NBIO::Reader.new(@selector,@sh_out)
       @rd_err = NBIO::Reader.new(@selector,@sh_err)
@@ -135,20 +153,38 @@ module Pwrake
       Fiber.new{callback(@rd_out,"o")}.resume
     end
 
+    def wrap_gnu_time(cmd)
+      if /\[|\]|<|>|\(|\)|\&|\||\\|\$|;|`|'|"|\n/ =~ cmd
+        cmd = cmd.gsub(/'/,"'\"'\"'")
+        cmd = (ENV['SHELL']||"sh")+" -c '#{cmd}'"
+      end
+      f = "%e,%S,%U,%M,%t,%K,%D,%p,%X,%Z,%F,%R,%W,%c,%w,%I,%O,%r,%s,%k"
+      "/usr/bin/time -o /dev/fd/3 -f '#{f}' #{cmd}"
+    end
+
     def callback(rd,mode)
       while s = rd.gets
-        @out.puts "#{@id}:#{mode}:#{s.chomp}"
+        s.chomp!
+        @out.puts "#{@id}:#{mode}:#{s}"
       end
       if rd.eof?
         @rd_list.delete(rd)
         if @rd_list.empty?  # process_end
           @thread = @pid = nil
           @log.info inspect_status
+          if @option[:gnu_time]
+            if gt = @sh_gtm.gets
+              @out.puts "#{@id}:t:#{gt.chomp}"
+            end
+            @sh_gtm.close
+          end
           @out.puts "#{@id}:z:#{exit_status}"
           @sh_in.close
           @sh_out.close
           @sh_err.close
-          @start_process_fiber.resume # next process
+          if @start_process_fiber.alive?
+            @start_process_fiber.resume # next process
+          end
         end
       end
     rescue => exc

data/lib/pwrake/worker/invoker.rb

@@ -114,7 +114,7 @@ module Pwrake
         case line
         when /^(\d+):open$/o
           $1.split.each do |id|
-            @ex_list[id] = Executor.new(@selector,@dir_class,id)
+            @ex_list[id] = Executor.new(@selector,@dir_class,id,@option)
           end
         when "setup_end"
           return
@@ -164,7 +164,7 @@ module Pwrake
         sig = $1
         sig = sig.to_i if /^\d+$/o =~ sig
         kill_all(sig)
-        return false
+        return true
         #
       when /^p$/o
         $stderr.puts "@ex_list = #{@ex_list.inspect}"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pwrake
 version: !ruby/object:Gem::Version
-  version: 2.2.4
+  version: 2.2.5
 platform: ruby
 authors:
 - Masahiro TANAKA
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-08-22 00:00:00.000000000 Z
+date: 2017-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel
@@ -136,7 +136,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.6.13
 signing_key: 
 specification_version: 4
 summary: Parallel Workflow engine based on Rake