pwrake 2.2.1 → 2.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 165b50b8450d65b495aefffe22cc3a8e02ae3828
-  data.tar.gz: 210a2fcb2b42cb1b70696e84595e20dd71332c83
+  metadata.gz: 700155f122d51b4ae814349492054d336daa6dad
+  data.tar.gz: 8bd23134b9c79784b95670440c0e823f139b41e5
 SHA512:
-  metadata.gz: ce57a49375d39fc42a4a785bb9e703fe4f90af8f5ec2db63c4a1d5fb7beb3c3d3f44b37742a1cbb8c9f40621cbc3d66a8c458cb17a6d530d9ad43374bda829e9
-  data.tar.gz: fae5ad1e68b9b7f87ab12428e9826a50ed8809120c2f4a5a45215518ff7e590f82051d6e4fb7e63914222b81c6dbea762c8ef7cfaf8e88a4823f6e176d325df0
+  metadata.gz: ca62080f1e262829e1da0623c474d0415bdda728498bea11a25f8b17d2a3d60ecf8ebb701ab4eb160f8a894840e402477f2daf87335103aefa42bf89a021d1b5
+  data.tar.gz: ad190406c5a890f78e3691cac5bed219e537ac911617fe4cb1d68ce6d6cbd65c9963e0fa5bd8e328aad864e6303bef893bd5877afefcd2e85402d6864759f8ee

data/lib/pwrake/branch/communicator.rb

@@ -114,7 +114,7 @@ class Communicator
         return false if !common_line(s)
       end
     end
-    raise ConnectError, "fail to connect #{cmd.inspect}"
+    raise ConnectError, "lost connection to #{@host} during setup"
   rescue => e
     dropout(e)
   end

data/lib/pwrake/branch/shell.rb

@@ -229,6 +229,7 @@ module Pwrake
             task = Rake.application[task_name]
             @ncore = task.wrapper.n_used_cores
             begin
+              Rake.application.current_flow[Fiber.current] = task.property.subflow
               task.execute(task.arguments) if task.needed?
               result = "taskend:#{@id}:#{task.name}"
             rescue Exception=>e
@@ -237,6 +238,7 @@ module Pwrake
               result = "taskfail:#{@id}:#{task.name}"
               break if @exited
             ensure
+              Rake.application.current_flow[Fiber.current] = nil
               master_w.put_line result
             end
           end

data/lib/pwrake/master/master.rb

@@ -17,6 +17,7 @@ module Pwrake
       @channel_by_hostid = {}
       @channels = []
       @hostinfo_by_id = {}
+      @current_flow = {}
       # init
       @option = Option.new
       Log.set_logger(@option)
@@ -31,6 +32,7 @@ module Pwrake
     attr_reader :task_queue
     attr_reader :option
     attr_reader :thread
+    attr_reader :current_flow # current_flow[Fiber.current] = task.property.subflow
 
     def setup_branch_handler(sub_host)
       ior,w0 = IO.pipe
@@ -331,7 +333,7 @@ module Pwrake
         Fiber.new do
           j = i
           while tw = pool.deq()
-            Log.debug "postproc##{j} deq=#{tw.name}"
+            #Log.debug "postproc##{j} deq=#{tw.name}"
             tw.postprocess(postproc)
             pool.count_down
             @hostinfo_by_taskname.delete(tw.name)
@@ -339,7 +341,7 @@ module Pwrake
             break if yield(pool,j)
           end
           postproc.close
-          Log.debug "postproc##{j} end"
+          #Log.debug "postproc##{j} end"
         end
       end
     end

data/lib/pwrake/master/master_application.rb

@@ -13,6 +13,10 @@ module Pwrake
       @role.task_queue
     end
 
+    def current_flow
+      @role.current_flow
+    end
+
     # Run the Pwrake application.
     def run
       standard_exception_handling do
@@ -46,10 +50,33 @@ module Pwrake
       end
     end
 
-    def invoke_task(task_string)
-      name, args = parse_task_string(task_string)
-      t = self[name]
-      @master.invoke(t,args)
+    # Find the rakefile and then load it and any pending imports.
+    def load_rakefile
+      if subdirs = pwrake_options['SUBDIR']
+        subdirs.each do |x|
+          begin
+            Rake.application.current_flow[Fiber.current] = x
+            super
+          ensure
+            Rake.application.current_flow[Fiber.current] = nil
+          end
+        end
+      else
+        super
+      end
+    end
+
+    def find_rakefile_location # :nodoc:
+      if subdir = Rake.application.current_flow[Fiber.current]
+        Dir.chdir(subdir)
+        if fn = have_rakefile
+          [File.join(subdir,fn), Rake.original_dir]
+        end
+      else
+        super
+      end
+    ensure
+      Dir.chdir(Rake.original_dir)
     end
 
     def invoke(t,*args)
@@ -138,6 +165,12 @@ module Pwrake
         "[Pw] Pwrake configuation file in YAML",
         lambda {|value| options.pwrake_conf = value}
        ],
+       ['--subdir "dir1 dir2 ..."',
+        "[Pw] Join workflows in subdirectories",
+        lambda { |value|
+          options.subdir = value.split(/[\s,;:]+/)
+        }
+       ],
        ['--show-conf','--show-config',
         "[Pw] Show Pwrake configuration options",
         lambda {|value| options.show_conf = true }

data/lib/pwrake/option/host_map.rb

@@ -86,6 +86,27 @@ module Pwrake
       end
       @continuous_fail
     end
+
+    def accept_core(task_name, use_cores)
+      if @reserved_task
+        if @reserved_task == task_name
+          if use_cores <= @idle_cores
+            Log.info "use reserved: #{@name} for #{task_name} (#{use_cores} cores)"
+            @reserved_task = nil
+            return :ok
+          end
+        end
+      else
+        if use_cores <= @idle_cores
+          return :ok
+        elsif use_cores > 1
+          @reserved_task = task_name
+          Log.info "reserve host: #{@name} for #{task_name} (#{use_cores} cores)"
+          return :reserve
+        end
+      end
+      :busy
+    end
   end
 
   class HostMap < Hash

data/lib/pwrake/option/option.rb

@@ -1,5 +1,6 @@
 require "pathname"
 require "yaml"
+require "socket"
 require "parallel"
 require "pwrake/option/host_map"
 
@@ -144,6 +145,19 @@ module Pwrake
         'DEBUG',
         'PLOT_PARALLELISM',
         'SHOW_CONF',
+        ['SUBDIR','SUBDIRS',
+          proc{|v|
+            if Array===v
+              v.each do |d|
+                if !File.directory?(d)
+                  raise "directory #{d.inspect} does not exist"
+                end
+              end
+            elsif !v.nil?
+              raise "invalid argument for SUBDIR: #{v.inspect}"
+            end
+          }
+        ],
         ['REPORT_DIR','REPORT'],
         'REPORT_IMAGE',
         'FAILED_TARGET', # rename(default), delete, leave
@@ -207,7 +221,7 @@ module Pwrake
         ['SHELL_START_INTERVAL', proc{|v| (v || 0.012).to_f}],
         ['HEARTBEAT', proc{|v| (v || 240).to_i}],
         ['RETRY', proc{|v| (v || 1).to_i}],
-        ['MASTER_HOSTNAME', proc{|v| (v || begin;`hostname -f`;rescue;end || '').chomp}],
+        ['MASTER_HOSTNAME', proc{|v| (v || Socket.gethostname).chomp}],
         ['WORK_DIR', proc{|v|
            v ||= '%CWD_RELATIVE_TO_HOME'
            v.sub('%CWD_RELATIVE_TO_HOME',cwd_relative_if_under_home)

data/lib/pwrake/queue/queue_array.rb

@@ -68,18 +68,22 @@ module Pwrake
 
     def shift(host_info=nil)
       return super() unless host_info
-      i_found = nil
+      i_tried = []
       (size-1).downto(0) do |i|
+        tw = at(i)
+        if tw.tried_host?(host_info)
+          i_tried << i
+        elsif tw.acceptable_for(host_info)
+          return delete_at(i)
+        end
+      end
+      i_tried.each do |i|
         tw = at(i)
         if tw.acceptable_for(host_info)
-          if tw.untried_host?(host_info)
-            return delete_at(i)
-          else
-            i_found ||= i
-          end
+          return delete_at(i)
         end
       end
-      (i_found) ? delete_at(i_found) : nil
+      nil
     end
   end
 
@@ -90,18 +94,22 @@ module Pwrake
 
     def shift(host_info=nil)
       return super() unless host_info
-      i_found = nil
+      i_tried = []
       size.times do |i|
+        tw = at(i)
+        if tw.tried_host?(host_info)
+          i_tried << i
+        elsif tw.acceptable_for(host_info)
+          return delete_at(i)
+        end
+      end
+      i_tried.each do |i|
         tw = at(i)
         if tw.acceptable_for(host_info)
-          if tw.untried_host?(host_info)
-            return delete_at(i)
-          else
-            i_found ||= i
-          end
+          return delete_at(i)
         end
       end
-      (i_found) ? delete_at(i_found) : nil
+      nil
     end
   end
 
@@ -172,18 +180,24 @@ module Pwrake
     end
 
     def pop_last_rank(r,host_info)
-      i_found = nil
+      i_tried = []
       (size-1).downto(0) do |i|
         tw = at(i)
-        if tw.rank == r && tw.acceptable_for(host_info)
-          if tw.untried_host?(host_info)
+        if tw.rank == r
+          if tw.tried_host?(host_info)
+            i_tried << i
+          elsif tw.acceptable_for(host_info)
             return delete_at(i)
-          else
-            i_found ||= i
           end
         end
       end
-      (i_found) ? delete_at(i_found) : nil
+      i_tried.each do |i|
+        tw = at(i)
+        if tw.acceptable_for(host_info)
+          return delete_at(i)
+        end
+      end
+      nil
     end
 
     def hrf_delete(t)

data/lib/pwrake/queue/task_queue.rb

@@ -70,15 +70,12 @@ module Pwrake
     end
 
     def deq_task(&block) # locality version
-      Log.debug "deq_task:"+(empty? ? " (empty)" : "\n#{inspect_q}")
+      Log.debug "deq_task from:"+(empty? ? " (empty)" : "\n#{inspect_q}")
       queued = 0
       @n_turn.times do |turn|
         next if turn_empty?(turn)
         queued += deq_turn(turn,&block)
       end
-      if queued>0
-        Log.debug "deq_task: queued=#{queued}"
-      end
     end
 
     def deq_turn(turn,&block)

data/lib/pwrake/task/task_algorithm.rb

@@ -100,7 +100,11 @@ module Pwrake
     end
 
     def pw_set_property(property)
-      @property = property
+      if @property
+        @property.merge(property)
+      else
+        @property = property
+      end
       self
     end
 

data/lib/pwrake/task/task_manager.rb

@@ -21,7 +21,17 @@ module Pwrake
         prop = @last_property
         @last_property = TaskProperty.new
       end
-      super.pw_set_property(prop)
+      if !@disable_subflow
+        @subflow_prefix = Rake.application.current_flow[Fiber.current]
+      end
+      task = super
+      task.pw_set_property(prop)
+      if !@disable_subflow
+        task.property.subflow = @subflow_prefix
+      end
+      task
+    ensure
+      @subflow_prefix = nil
     end
 
     def create_rule(*args, &block) # :nodoc:
@@ -30,5 +40,44 @@ module Pwrake
       super
     end
 
+    def resolve_args(args)
+      task_name, arg_names, deps = super(args)
+      if @subflow_prefix
+        # fix prerequisites
+        deps.map! do |task_name|
+          add_prefix(task_name)
+        end
+      end
+      [task_name, arg_names, deps]
+    end
+
+    def intern(task_class, task_name)
+      if @subflow_prefix
+        base_name = task_name
+        # fix task name
+        task_name = add_prefix(task_name)
+        if base_name != task_name && task_class == Rake::Task
+          sbfl = @subflow_prefix
+          begin
+            @subflow_prefix = nil
+            @disable_subflow = true
+            Rake::Task.define_task(base_name => task_name)
+          ensure
+            @subflow_prefix = sbfl
+            @disable_subflow = false
+          end
+        end
+      end
+      super(task_class, task_name)
+    end
+
+    def add_prefix(task_name)
+      task_name = task_name.to_s
+      if /^\.?\.?\// =~ task_name
+        task_name
+      else
+        File.join(@subflow_prefix, task_name)
+      end
+    end
   end
 end

data/lib/pwrake/task/task_property.rb

@@ -4,6 +4,7 @@ module Pwrake
 
     attr_reader :ncore, :exclusive, :allow, :deny, :order_allow_deny,
       :retry, :disable_steal
+    attr_accessor :subflow
 
     def parse_description(description)
       if /\bn_?cores?[=:]\s*([+-]?\d+)/i =~ description
@@ -38,19 +39,31 @@ module Pwrake
       end
     end
 
-    def acceptable_for(host_info)
+    def merge(prop)
+      @ncore = prop.ncore if prop.ncore
+      @exclusive = prop.exclusive if prop.exclusive
+      @allow = prop.allow if prop.allow
+      @deny = prop.deny if prop.deny
+      @order_allow_deny = prop.order_allow_deny if prop.order_allow_deny
+      @retry = prop.retry if prop.retry
+      @disable_steal = prop.disable_steal if prop.disable_steal
+      @subflow = prop.subflow if prop.subflow
+    end
+
+    def use_cores(host_info)
+      nc = (@exclusive) ? 0 : (@ncore || 1)
+      if nc < 1
+        nc += host_info.ncore
+      end
+      return nc
+    end
+
+    def accept_host(host_info)
       return true unless host_info
       if @disable_steal && host_info.steal_flag
         #Log.debug("@disable_steal && host_info.steal_flag")
         return false
       end
-      ncore = (@exclusive) ? 0 : (@ncore || 1)
-      if ncore > 0
-        return false if ncore > host_info.idle_cores
-      else
-        n = host_info.ncore + ncore
-        return false if n < 1 || n > host_info.idle_cores
-      end
       hn = host_info.name
       if @allow
         if @deny

data/lib/pwrake/task/task_wrapper.rb

@@ -29,12 +29,11 @@ module Pwrake
       @assigned = []
       @exec_host = nil
       @exec_host_id = nil
-      @tried_hosts = []
+      @tried_hosts = {}
       @n_retry = @property.retry || Rake.application.pwrake_options["RETRY"] || 1
     end
 
     def_delegators :@task, :name, :actions, :prerequisites, :subsequents
-    def_delegators :@property, :acceptable_for
 
     attr_reader :task, :task_id, :group, :group_id, :file_stat
     attr_reader :location
@@ -91,7 +90,7 @@ module Pwrake
     end
 
     def retry_or_subsequent
-      @tried_hosts << @exec_host
+      @tried_hosts[@exec_host] = true
       if @status=="end"
         @task.pw_enq_subsequents
       elsif @n_retry > 0
@@ -102,7 +101,7 @@ module Pwrake
         @n_retry -= 1
         Rake.application.task_queue.enq(self)
       else
-        s="give up retry (retry_count=0): #{name}"
+        s="give up retry (retry_count=#{@n_retry}): #{name}"
         Log.error(s)
         $stderr.puts(s)
       end
@@ -171,7 +170,7 @@ module Pwrake
       @@task_logger << row
       #
       clsname = @task.class.to_s.sub(/^(Rake|Pwrake)::/o,"")
-      msg = '%s:"%s" %s: id=%d elap=%.6f exec_host=%s' %
+      msg = '%s[%s] status=%s id=%d elap=%.6f exec_host=%s' %
         [clsname,name,@status,@task_id,elap,@exec_host]
       if @status=="end"
         Log.info msg
@@ -266,7 +265,7 @@ module Pwrake
           end
           @rank = max_rank + step
         end
-        Log.debug "Task[#{name}] rank=#{@rank.inspect}"
+        #Log.debug "Task[#{name}] rank=#{@rank.inspect}"
       end
       @rank
     end
@@ -343,9 +342,25 @@ module Pwrake
       @n_used_cores ||= @property.n_used_cores(host_info)
     end
 
-    def untried_host?(host_info)
-      return true unless host_info
-      !@tried_hosts.include?(host_info.name)
+    def acceptable_for(host_info)
+      unless @property.accept_host(host_info)
+        return false
+      end
+      use_cores = @property.use_cores(host_info)
+      if @reserved_host.nil? || @reserved_host == host_info
+        case host_info.accept_core(name,use_cores)
+        when :ok
+          @reserved_host = nil
+          return true
+        when :reserve
+          @reserved_host = host_info
+        end
+      end
+      false
+    end
+
+    def tried_host?(host_info)
+      host_info && @tried_hosts[host_info.name]
     end
 
   end

data/lib/pwrake/version.rb

@@ -1,3 +1,3 @@
 module Pwrake
-  VERSION = "2.2.1"
+  VERSION = "2.2.2"
 end

data/lib/pwrake/worker/gfarm_directory.rb

@@ -1,3 +1,5 @@
+require "socket"
+
 module Pwrake
 
   class GfarmDirectory < SharedDirectory
@@ -8,7 +10,7 @@ module Pwrake
     @@gfarm2fs_debug = nil
     @@gfarm2fs_debug_wait = 1
     @@current_id = 0
-    @@hostname = `hostname`.chomp
+    @@hostname = Socket.gethostname
 
     def self.init(opts)
       @@prefix   = opts[:base_dir]
@@ -57,7 +59,7 @@ module Pwrake
       path = @log.path
       begin
         if @@gfarm2fs_debug && path
-          f = path+("gfarm2fs-"+`hostname`.chomp+"-"+@suffix)
+          f = path+("gfarm2fs-"+@@hostname+"-"+@suffix)
           spawn_cmd "gfarm2fs #{@@gfarm2fs_option} -d #{@gfarm_mountpoint} > #{f} 2>&1 & sleep #{@@gfarm2fs_debug_wait}"
         else
           spawn_cmd "gfarm2fs #{@@gfarm2fs_option} #{@gfarm_mountpoint}"

data/lib/pwrake/worker/log_executor.rb

@@ -1,6 +1,7 @@
 require "singleton"
 require "forwardable"
 require "logger"
+require "socket"
 
 module Pwrake
 
@@ -46,7 +47,7 @@ module Pwrake
         @dir = dir_class.new
         @dir.open
         @path = @dir.log_path
-        fn = "worker-#{`hostname`.chomp}-#{Process.pid}.log"
+        fn = "worker-#{Socket.gethostname}-#{Process.pid}.log"
         @logfile = (@path + fn).to_s
         ::FileUtils.mkdir_p(@path.to_s)
         @logger = @logger_file = ::Logger.new(@logfile)

data/lib/pwrake/worker/worker_main.rb

@@ -1,6 +1,7 @@
 require "thread"
 require "fileutils"
 require "timeout"
+require "socket"
 
 begin
   Pwrake::Invoker.new.run
@@ -8,7 +9,7 @@ rescue => exc
   log = Pwrake::LogExecutor.instance
   log.error exc
   log.error exc.backtrace.join("\n")
-  open("pwrake_worker_err-#{ENV['USER']}-#{Process.pid}","w") do |f|
+  open("pwrake_worker_err-#{Socket.gethostname}-#{Process.pid}","w") do |f|
     f.puts exc
     f.puts exc.backtrace.join("\n")
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pwrake
 version: !ruby/object:Gem::Version
-  version: 2.2.1
+  version: 2.2.2
 platform: ruby
 authors:
 - Masahiro TANAKA
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-30 00:00:00.000000000 Z
+date: 2017-07-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel