pwrake 0.9.7 → 0.9.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8936d0357e38b7b50bee817594555714412fb763
-  data.tar.gz: 1aca5697a842735520300b7c8c81355c08b31240
+  metadata.gz: d50d7fd5c00627f0eea3d59e26632c4e9e4c0f6d
+  data.tar.gz: 3d9d6638a9dde0ce9feebc1ba210d9ce1d94b310
 SHA512:
-  metadata.gz: c4204a54c3920365eaaf4b430a0217b7e63cbe19a4878594dbd7c6381d21e8fceb0e8f3a4a31e238ec8b2dbc4b1d2ca95b914199e1e0c973565540db1c740654
-  data.tar.gz: f2c433d66cb1ecd7eb5340394e1b93d13530238b055d481d5b28c16f48f61402d69c1350a02b5389eaab1af39670db4e108346e70e33afdcaf4236909224df73
+  metadata.gz: c7517d29d640a06db61bc8a656615d43706564c49596d594830cadbbe0134b79bfe71c542e182de003baff492b58008bcb90e60b97aab587de1460f21cd99ce6
+  data.tar.gz: 99f101b28c359ea3157caf688205da6734bb95569dd1eea6fed5cc6787e6b1f9a7bd01a35ab1e7dd694e2782c0969480f4abea1d18bc68d963bc85658d9432c6

data/lib/pwrake/application.rb

@@ -64,6 +64,7 @@ module Pwrake
       standard_exception_handling do
         init("pwrake")
         @master = Master.new
+        @master.init
         load_rakefile
         begin
           top_level

data/lib/pwrake/cache_aware_queue.rb

@@ -0,0 +1,184 @@
+module Pwrake
+
+  class MTimePriorityArray < Array
+
+    def order_by(item)
+      item.input_file_mtime
+    end
+
+    def push(t)
+      order_value = order_by(t)
+      if empty? || order_by(last) <= order_value
+        super(t)
+      elsif order_by(first) > order_value
+        unshift(t)
+      else
+        lower = 0
+        upper = size-1
+        while lower+1 < upper
+          mid = ((lower + upper) / 2).to_i
+          if order_by(self[mid]) <= order_value
+            lower = mid
+          else
+            upper = mid
+          end
+        end
+        insert(upper,t)
+      end
+    end
+
+    def index(t)
+      if size < 40
+        return super(t)
+      end
+      order_value = t.order_by
+      if order_by(last) < order_value || order_by(first) > order_value
+        nil
+      else
+        lower = 0
+        upper = size-1
+        while lower+1 < upper
+          mid = ((lower + upper) / 2).to_i
+          if order_by(self[mid]) < order_value
+            lower = mid
+          else
+            upper = mid
+          end
+        end
+        mid = upper
+        if order_by(self[mid]) == order_value
+          Log.debug "--- TQA#index=#{mid}, order_value=#{order_value}"
+          mid
+        end
+      end
+    end
+  end
+
+
+  # rank-Even Last In First Out
+  class CacheAwareQueue
+    def initialize(n)
+      @q = []
+      @i = 0
+      @size = 0
+      @n = (n>0) ? n : 1
+    end
+
+    def push(t)
+      r = t.rank
+      a = @q[r]
+      if a.nil?
+        @q[r] = a = MTimePriorityArray.new
+      end
+      @size += 1
+      a.push(t)
+    end
+
+    def size
+      @size
+    end
+
+    def empty?
+      @size == 0
+    end
+
+    def shift
+      if empty?
+        return nil
+      elsif @size < @n*2
+        return shift_high_rank
+      else
+        return shift_weighted
+      end
+    end
+
+    def shift_high_rank
+      (@q.size-1).downto(0) do |i|
+        a = @q[i]
+        next if a.nil? || a.empty?
+        @size -= 1
+        return pop_last_max(a)
+      end
+      nil
+    end
+
+    def shift_weighted
+      weight, weight_avg = RANK_STAT.rank_weight
+      wsum = 0.0
+      q = []
+      @q.each_with_index do |a,i|
+        next if a.nil? || a.empty?
+        w = weight[i]
+        w = weight_avg if w.nil?
+        # w *= a.size
+        wsum += w
+        q << [a,wsum]
+      end
+      #
+      x = rand() * wsum
+      Log.debug "--- shift_weighted x=#{x} wsum=#{wsum} weight=#{weight.inspect}"
+      @size -= 1
+      q.each do |a,w|
+        if w > x
+          return a.pop
+        end
+      end
+      raise "ELIFO: wsum=#{wsum} x=#{x}"
+    end
+
+    def pop_last_max(a)
+      if a.size < 2
+        return a.pop
+      end
+      y_max = nil
+      i_max = nil
+      n = [@n, a.size].min
+      (-n..-1).each do |i|
+        y = a[i].input_file_size
+        if y_max.nil? || y > y_max
+          y_max = y
+          i_max = i
+        end
+      end
+      a.delete_at(i_max)
+    end
+
+    def first
+      return nil if empty?
+      @q.size.times do |i|
+        a = @q[i]
+        unless a.nil? || a.empty?
+          return a.first
+        end
+      end
+    end
+
+    def last
+      return nil if empty?
+      @q.size.times do |i|
+        a = @q[-i-1]
+        unless a.nil? || a.empty?
+          return a.last
+        end
+      end
+    end
+
+    def delete(t)
+      n = 0
+      @q.each do |a|
+        if a
+          a.delete(t)
+          n += a.size
+        end
+      end
+      @size = n
+    end
+
+    def clear
+      @q.clear
+      @i = 0
+      @size = 0
+    end
+  end
+
+end

data/lib/pwrake/gfarm_feature.rb

@@ -1,3 +1,5 @@
+require 'pwrake/gfwhere_pool'
+
 module Pwrake
 
   module GfarmPath
@@ -207,43 +209,13 @@ module Pwrake
   class GfarmPostprocess
 
     def initialize
-      @lock = Mutex.new
-      @io = IO.popen('gfwhere-pipe','r+')
-      @io.sync = true
-    end
-
-    def gfwhere(file)
-      return [] if file==''
-      @lock.synchronize do
-        @io.puts(file)
-        @io.flush
-        s = @io.gets
-        if s.nil?
-          raise "gfwhere: unexpected end"
-        end
-        s.chomp!
-        if s != file
-          raise "gfwhere: file=#{file}, result=#{s}"
-        end
-        while s = @io.gets
-          s.chomp!
-          case s
-          when ""
-            next
-          when /^gfarm:\/\//
-            next
-          when /^Error:/
-            return []
-          else
-            return s.split(/\s+/)
-          end
-        end
-      end
+      max = Pwrake.application.pwrake_options['MAX_GFWHERE_WORKER']
+      @gfwhere_pool = WorkerPool.new(GfwhereWorker,max)
     end
 
     def postprocess(t)
       if t.kind_of? Rake::FileTask
-        t.location = gfwhere(t.name)
+        t.location = @gfwhere_pool.work(t.name)
       end
     end
 

data/lib/pwrake/gfwhere_pool.rb

@@ -0,0 +1,112 @@
+module Pwrake
+
+  class WorkerPool
+
+    def initialize(wk_class, max)
+      @worker_class = wk_class
+      @max = max
+      @pool = []
+      @cond_pool = ConditionVariable.new
+      @mutex = Mutex.new
+    end
+
+    def find_worker
+      @mutex.synchronize do
+        while true
+          @pool.each do |w|
+            return w if w.acquire
+          end
+          if @pool.size < @max
+            Log.debug "--- #{@worker_class}:new_worker #{@pool.size+1}"
+            w = @worker_class.new(@cond_pool)
+            @pool << w
+            return w if w.acquire
+          end
+          # wait for end of work in @pool
+          print "wait\n"
+          @cond_pool.wait(@mutex)
+        end
+      end
+    end
+
+    def work(*args)
+      w = find_worker
+      w.run(*args)
+    end
+  end
+
+
+  class Worker
+
+    def initialize(cond_pool)
+      @cond_pool = cond_pool
+      @mutex = Mutex.new
+      @aquired = false
+    end
+
+    def acquire
+      return false if @mutex.locked?
+      if @aquired
+        return false
+      else
+        @aquired = true
+        return true
+      end
+    end
+
+    def run(*args)
+      @mutex.synchronize do
+        raise "no aquired" unless @aquired
+        r = work(*args)
+        @aquired = false
+        @cond_pool.signal
+        return r
+      end
+    end
+
+    def work(*args)
+      # inplement in subclass
+      return nil
+    end
+  end
+
+
+  class GfwhereWorker < Worker
+
+    def initialize(cond_pool)
+      super(cond_pool)
+      @io = IO.popen('gfwhere-pipe','r+')
+      @io.sync = true
+    end
+
+    def work(file)
+      return [] if file==''
+      t = Time.now
+      @io.puts(file)
+      @io.flush
+      s = @io.gets
+      if s.nil?
+        raise "gfwhere: unexpected end"
+      end
+      s.chomp!
+      if s != file
+        raise "gfwhere: file=#{file}, result=#{s}"
+      end
+      while s = @io.gets
+        s.chomp!
+        case s
+        when ""
+          next
+        when /^gfarm:\/\//
+          next
+        when /^Error:/
+          return []
+        else
+          Log.debug "gfwhere:path %.6f sec, file=%s" % [Time.now-t,file]
+          return s.split(/\s+/)
+        end
+      end
+    end
+  end
+
+end

data/lib/pwrake/locality_aware_queue.rb

@@ -131,14 +131,16 @@ module Pwrake
     end # class Throughput
 
 
-    def init_queue(hosts)
+    def init_queue(core_list)
+      @host_count = Hash.new{0}
+      core_list.each{|h| @host_count[h] += 1}
+      @hosts = @host_count.keys
       @cv = LocalityConditionVariable.new
-      @hosts = hosts
       @throughput = Throughput.new
       @size = 0
       @q = {}
-      @hosts.each{|h| @q[h]=@array_class.new}
-      @q_remote = @array_class.new
+      @host_count.each{|h,n| @q[h] = @array_class.new(n)}
+      @q_remote = @array_class.new(0)
       @q_later = Array.new
       @enable_steal = !Pwrake.application.pwrake_options['DISABLE_STEAL']
       @steal_wait = (Pwrake.application.pwrake_options['STEAL_WAIT'] || 0).to_i
@@ -213,10 +215,7 @@ module Pwrake
       if q && !q.empty?
         t = q.shift
         t.assigned.each do |h|
-          a = @q[h]
-          if i = a.index(t)
-            a.delete_at(i)
-          end
+          @q[h].delete(t)
         end
         @size -= 1
         return t
@@ -250,11 +249,14 @@ module Pwrake
         when 0
           s += "[]\n"
         when 1
-          s += "[#{q[0].name}]\n"
+          s += "[#{q.first.name}]\n"
+        when 2
+          s += "[#{q.first.name}, #{q.last.name}]\n"
         else
-          s += "[#{q[0].name},..]\n"
+          s += "[#{q.first.name},.. #{q.last.name}]\n"
         end
       }
+      b.call("noaction",@q_noaction)
       @q.each(&b)
       b.call("remote",@q_remote)
       b.call("later",@q_later)
@@ -266,17 +268,17 @@ module Pwrake
     end
 
     def clear
+      @q_noaction.clear
       @q.each{|h,q| q.clear}
       @q_remote.clear
       @q_later.clear
-      @reserved_q.clear
     end
 
     def empty?
       @q.all?{|h,q| q.empty?} &&
+        @q_noaction.empty? &&
         @q_remote.empty? &&
-        @q_later.empty? &&
-        @reserved_q.empty?
+        @q_later.empty?
     end
 
     def finish

data/lib/pwrake/master.rb

@@ -22,19 +22,22 @@ module Pwrake
     attr_reader :postprocess
 
     def initialize
-      init_option    # Pwrake::Option
-      setup_option   # Pwrake::Option
       @started = false
       @lock = Mutex.new
       @current_task_id = -1
     end
 
+    def init
+      init_option    # Pwrake::Option
+      setup_option   # Pwrake::Option
+    end
+
     def start
       return if @task_queue
       timer = Timer.new("start_worker")
       @finish_queue = Queue.new
       @task_queue = @queue_class.new(@core_list)
-      @shell_set = []
+      @shell_set = (1..@num_noaction_threads).map{ NoActionShell.new }
       @core_list.each_with_index do |h,i|
         @shell_set << @shell_class.new(h,@shell_opt)
       end
@@ -53,6 +56,7 @@ module Pwrake
     def start_threads
       Thread.abort_on_exception = true
       @threads = []
+      @exit_task = {}
       t_intvl = Pwrake.application.pwrake_options['THREAD_CREATE_INTERVAL']
       @shell_set.each do |c|
         tc0 = Time.now
@@ -75,7 +79,9 @@ module Pwrake
     end
 
     def thread_loop(conn,last=nil)
-      @task_queue.reserve(last) if last
+      if last
+        @exit_task[last] = Thread.current
+      end
       hint = (conn) ? conn.host : nil
       standard_exception_handling do
         while true
@@ -85,7 +91,9 @@ module Pwrake
 	  time_deq = Time.now - time_start
           Log.debug "--- Master#thread_loop deq t=#{t.inspect} time=#{time_deq}sec"
           t.pw_invoke
-          return if t == last
+          if th = @exit_task.delete(t)
+            @task_queue.thread_end(th)
+          end
         end
       end
     end

data/lib/pwrake/option.rb

@@ -12,9 +12,9 @@ module Pwrake
 
     def parse_opt(s)
       case s
-      when /false|nil|off/i
+      when /^(false|nil|off)$/i
         false
-      when /true|on/i
+      when /^(true|on)$/i
         true
       else
         s
@@ -41,10 +41,11 @@ module Pwrake
         'DEBUG',
         'PLOT_PARALLELISM',
         'HALT_QUEUE_WHILE_SEARCH',
-        'THREAD_CREATE_INTERVAL',
         'SHOW_CONF',
         'FAILED_TARGET', # rename(default), delete, leave
-        'QUEUE_PRIORITY', # DFS(default), FIFO,
+        'QUEUE_PRIORITY', # RANK(default), FIFO, LIFO, DFS
+        'NOACTION_QUEUE_PRIORITY', # FIFO(default), LIFO, RAND
+        'NUM_NOACTION_THREADS', # default=4 when gfarm, else 1
         'STEAL_WAIT',
         'STEAL_WAIT_MAX',
 
@@ -88,16 +89,18 @@ module Pwrake
             end
          }],
         ['NUM_THREADS', proc{|v| v && v.to_i}],
+        ['THREAD_CREATE_INTERVAL', proc{|v| (v || 0.010).to_f}],
         ['DISABLE_AFFINITY', proc{|v| v || ENV['AFFINITY']=='off'}],
         ['DISABLE_STEAL', proc{|v| v || ENV['STEAL']=='off'}],
         ['GFARM_BASEDIR', proc{|v| v || '/tmp'}],
         ['GFARM_PREFIX', proc{|v| v || "pwrake_#{ENV['USER']}"}],
         ['GFARM_SUBDIR', proc{|v| v || '/'}],
-        #['MASTER_HOSTNAME', proc{|v| v || `hostname -f`.chomp}],
+        ['MAX_GFWHERE_WORKER', proc{|v| (v || 8).to_i}],
+        ['MASTER_HOSTNAME', proc{|v| (v || begin;`hostname -f`;rescue;end || '').chomp}],
         ['WORK_DIR',proc{|v|
             v ||= '$HOME/%CWD_RELATIVE_TO_HOME'
             v.sub('%CWD_RELATIVE_TO_HOME',cwd_relative_to_home)
-          }]
+          }],
       ]
     end
 
@@ -155,9 +158,7 @@ module Pwrake
         @yaml = open(@pwrake_conf){|f| YAML.load(f) }
       end
 
-      @opts = {'PWRAKE_CONF' => @pwrake_conf,
-        'THREAD_CREATE_INTERVAL' => 0.006,
-      }
+      @opts = { 'PWRAKE_CONF' => @pwrake_conf, }
 
       option_data.each do |a|
         prc = nil
@@ -376,6 +377,8 @@ module Pwrake
         end
       end
 
+      n_noaction_th = @opts['NUM_NOACTION_THREADS']
+
       case @filesystem
       when 'gfarm'
         require "pwrake/locality_aware_queue"
@@ -394,12 +397,14 @@ module Pwrake
 	else
 	  @queue_class = LocalityAwareQueue
 	end
+        @num_noaction_threads = (n_noaction_th || [8,@num_threads].max).to_i
         @postprocess = GfarmPostprocess.new
         Log.debug "--- @queue_class=#{@queue_class}"
       else
         @filesystem  = 'nfs'
         @shell_class = Shell
         @queue_class = TaskQueue
+        @num_noaction_threads = (n_noaction_th || 1).to_i
       end
     end