pwrake 0.9.7 → 0.9.8

data/lib/pwrake/task_queue.rb

@@ -1,3 +1,5 @@
+require "pwrake/cache_aware_queue.rb"
+
 module Pwrake
 
   class TaskConditionVariable < ConditionVariable
@@ -6,7 +8,12 @@ module Pwrake
     end
   end
 
+
   class PriorityQueueArray < Array
+    def initialize(n)
+      super()
+    end
+
     def push(t)
       task_id = t.task_id
       if empty? || last.task_id <= task_id
@@ -55,80 +62,324 @@ module Pwrake
     end
   end # PriorityQueueArray
 
+
   class LifoQueueArray < Array
+    def initialize(n)
+      super()
+    end
+
     def shift
       pop
     end
   end
 
+
+  class FifoQueueArray < Array
+    def initialize(n)
+      super()
+    end
+  end
+
+
+  # Rank-Even Last In First Out
+  class RankQueueArray
+    def initialize(n)
+      @q = []
+      @size = 0
+      @n = (n>0) ? n : 1
+    end
+
+    def push(t)
+      r = t.rank
+      a = @q[r]
+      if a.nil?
+        @q[r] = a = []
+      end
+      @size += 1
+      a.push(t)
+    end
+
+    def size
+      @size
+    end
+
+    def empty?
+      @size == 0
+    end
+
+    def shift
+      if empty?
+        return nil
+      end
+      (@q.size-1).downto(0) do |i|
+        a = @q[i]
+        next if a.nil? || a.empty?
+        @size -= 1
+        if a.size <= @n
+          return pop_last_max(a)
+        else
+          return shift_weighted
+        end
+      end
+      raise "ELIFO: @q=#{@q.inspect}"
+    end
+
+    def shift_weighted
+      weight, weight_avg = RANK_STAT.rank_weight
+      wsum = 0.0
+      q = []
+      @q.each_with_index do |a,i|
+        next if a.nil? || a.empty?
+        w = weight[i]
+        w = weight_avg if w.nil?
+        # w *= a.size
+        wsum += w
+        q << [a,wsum]
+      end
+      #
+      x = rand() * wsum
+      Log.debug "--- shift_weighted x=#{x} wsum=#{wsum} weight=#{weight.inspect}"
+      q.each do |a,w|
+        if w > x
+          return a.pop
+        end
+      end
+      raise "ELIFO: wsum=#{wsum} x=#{x}"
+    end
+
+    def pop_last_max(a)
+      if a.size < 2
+        return a.pop
+      end
+      y_max = nil
+      i_max = nil
+      n = [@n, a.size].min
+      (-n..-1).each do |i|
+        y = a[i].input_file_size
+        if y_max.nil? || y > y_max
+          y_max = y
+          i_max = i
+        end
+      end
+      a.delete_at(i_max)
+    end
+
+    def first
+      return nil if empty?
+      @q.size.times do |i|
+        a = @q[i]
+        unless a.nil? || a.empty?
+          return a.first
+        end
+      end
+    end
+
+    def last
+      return nil if empty?
+      @q.size.times do |i|
+        a = @q[-i-1]
+        unless a.nil? || a.empty?
+          return a.last
+        end
+      end
+    end
+
+    def delete(t)
+      n = 0
+      @q.each do |a|
+        if a
+          a.delete(t)
+          n += a.size
+        end
+      end
+      @size = n
+    end
+
+    def clear
+      @q.clear
+      @size = 0
+    end
+  end
+
+
+  class NoActionQueue
+    def initialize
+      @que = []
+      @num_waiting = 0
+      @mutex = Mutex.new
+      @cond = ConditionVariable.new
+      @halt = false
+      @th_end = {}
+      prio = Pwrake.application.pwrake_options['NOACTION_QUEUE_PRIORITY'] || 'fifo'
+      case prio
+      when /fifo/i
+        @prio = 0
+        Log.debug "--- NOACTION_QUEUE_PRIORITY=FIFO"
+      when /lifo/i
+        @prio = 1
+        Log.debug "--- NOACTION_QUEUE_PRIORITY=LIFO"
+      when /rand/i
+        @prio = 2
+        Log.debug "--- NOACTION_QUEUE_PRIORITY=RAND"
+      else
+        raise RuntimeError,"unknown option for NOACTION_QUEUE_PRIORITY: "+prio
+      end
+    end
+
+    def push(obj)
+      if @halt
+        @que.push obj
+      else
+        @mutex.synchronize do
+          @que.push obj
+          @cond.signal
+        end
+      end
+    end
+    alias << push
+    alias enq push
+
+    def pop
+      @mutex.synchronize do
+        t = Time.now
+        while true
+          if @th_end.delete(Thread.current)
+            return false
+          elsif @halt
+            @cond.wait @mutex
+          elsif @que.empty?
+            if @finished
+              @cond.signal
+              return false
+            end
+            @cond.wait @mutex
+          else
+            case @prio
+            when 0
+              x = @que.shift
+            when 1
+              x = @que.pop
+            when 2
+              x = @que.delete_at(rand(@que.size))
+            end
+            Log.debug "--- NATQ#deq %.6f sec #{x.inspect}"%[Time.now-t]
+            return x
+          end
+        end
+      end
+    end
+
+    alias shift pop
+    alias deq pop
+
+    def halt
+      @mutex.lock
+      @halt = true
+    end
+
+    def resume
+      @halt = false
+      @mutex.unlock
+      @cond.broadcast
+    end
+
+    def empty?
+      @que.empty?
+    end
+
+    def clear
+      @que.clear
+    end
+
+    def length
+      @que.length
+    end
+    alias size length
+
+    def first
+      @que.first
+    end
+
+    def last
+      @que.last
+    end
+
+    def finish
+      @finished = true
+      @cond.broadcast
+    end
+
+    def thread_end(th)
+      @th_end[th] = true
+      @cond.broadcast
+    end
+
+    def stop
+      clear
+      finish
+    end
+  end
+
+
   class TaskQueue
 
-    def initialize(*args)
+    def initialize(core_list)
       @finished = false
       @halt = false
       @mutex = Mutex.new
-      @th_end = []
+      @th_end = {}
       @enable_steal = true
-      @reservation = {}
-      @reserved_q = {}
-      case Pwrake.application.pwrake_options['QUEUE_PRIORITY']||"DFS"
+      @q_noaction = NoActionQueue.new
+      pri = Pwrake.application.pwrake_options['QUEUE_PRIORITY'] || "RANK"
+      case pri
       when /dfs/i
         @array_class = PriorityQueueArray
       when /fifo/i
-        @array_class = Array # Fifo
+        @array_class = FifoQueueArray # Array # Fifo
       when /lifo/i
         @array_class = LifoQueueArray
+      when /rank/i
+        @array_class = RankQueueArray
       else
-        raise RuntimeError,"unknown option for QUEUE_PRIORITY"
+        raise RuntimeError,"unknown option for QUEUE_PRIORITY: "+pri
       end
       Log.debug "--- TQ#initialize @array_class=#{@array_class.inspect}"
-      init_queue(*args)
+      init_queue(core_list)
     end
 
-    def init_queue(*args)
+    def init_queue(core_list)
       @cv = TaskConditionVariable.new
-      @q_prior = @array_class.new
+      @q_input = @array_class.new(core_list.size)
       @q_later = Array.new
     end
 
     attr_reader :mutex
     attr_accessor :enable_steal
 
-    def reserve(item)
-      @reservation[item] = Thread.current
-    end
-
     def halt
-      @mutex.synchronize do
-        @halt = true
-      end
+      @q_noaction.halt
+      @mutex.lock
+      @halt = true
     end
 
     def resume
-      @mutex.synchronize do
-        @halt = false
-        @cv.broadcast
-      end
+      @halt = false
+      @q_noaction.resume
+      @mutex.unlock
+      @cv.broadcast
     end
 
     def synchronize(condition)
       ret = nil
       if condition
-        @mutex.lock
-	@halt = true
-	begin
+        halt
+        begin
           ret = yield
-	  @cv.broadcast
-	ensure
-	  @halt = false
-	  @mutex.unlock
+        ensure
+          resume
         end
       else
-	ret = yield
-      end
-      @reserved_q.keys.each do |th|
-        Log.debug "--- run #{th}";
-        th.run
+        ret = yield
       end
       ret
     end
@@ -137,46 +388,41 @@ module Pwrake
     def enq(item)
       Log.debug "--- TQ#enq #{item.name}"
       t0 = Time.now
-      if @halt
-	enq_body(item)
+      if item.actions.empty?
+        @q_noaction.enq(item)
+      elsif @halt
+        enq_body(item)
       else
         @mutex.synchronize do
-	  enq_body(item)
-	  @cv.signal(item.suggest_location)
+          enq_body(item)
+          @cv.signal(item.suggest_location)
         end
       end
-      @reserved_q.keys.each{|th|
-        Log.debug "--- run #{th}";
-        th.run
-      }
       Log.debug "--- TQ#enq #{item.name} enq_time=#{Time.now-t0}"
     end
 
     def enq_body(item)
-      if th = @reservation[item]
-	@reserved_q[th] = item
-      else
-	enq_impl(item)
-      end
+      enq_impl(item)
     end
 
-    def enq_impl(item)
-      if item.prior?
-        @q_prior.push(item)
+    def enq_impl(t)
+      if t.has_input_file?
+        @q_input.push(t)
       else
-        @q_later.push(item)
+        @q_later.push(t)
       end
     end
 
-
     # deq
     def deq(hint=nil)
+      if hint == '(noaction)'
+        return @q_noaction.deq
+      end
       n = 0
       loop do
         @mutex.synchronize do
           t0 = Time.now
-          if @th_end.first == Thread.current
-            @th_end.shift
+          if @th_end.delete(Thread.current)
             return false
 
           elsif @halt
@@ -184,17 +430,11 @@ module Pwrake
             @cv.wait(@mutex)
             n = 0
 
-          elsif item = @reserved_q.delete(Thread.current)
-            Log.debug "--- deq from reserved_q=#{item.inspect}"
-            return item
-
           elsif empty? # no item in queue
-            #Log.debug "--- empty=true in #{self.class}#deq @finished=#{@finished.inspect}"
             if @finished
-	      @cv.signal
+              @cv.signal
               return false
             end
-            #Log.debug "--- waiting in #{self.class}#deq @finished=#{@finished.inspect}"
             @cv.wait(@mutex)
             n = 0
 
@@ -204,36 +444,38 @@ module Pwrake
               Log.debug "--- TQ#deq #{t_inspect} deq_time=#{Time.now-t0}"
               return t
             end
-            #@cv.signal([hint])
             n += 1
           end
         end
-        #Thread.pass
       end
     end
 
     def deq_impl(hint,n)
       Log.debug "--- TQ#deq_impl #{@q.inspect}"
-      @q_prior.shift || @q_later.shift
+      @q_input.shift || @q_later.shift
     end
 
     def clear
-      @q_prior.clear
+      @q_noaction.clear
+      @q_input.clear
       @q_later.clear
-      @reserved_q.clear
     end
 
     def empty?
-      @q_prior.empty? && @q_later.empty? && @reserved_q.empty?
+      @q_noaction.empty? &&
+        @q_input.empty? &&
+        @q_later.empty?
     end
 
     def finish
       Log.debug "--- TQ#finish"
+      @q_noaction.finish
       @finished = true
       @cv.broadcast
     end
 
     def stop
+      @q_noaction.stop
       @mutex.synchronize do
         clear
         finish
@@ -241,8 +483,9 @@ module Pwrake
     end
 
     def thread_end(th)
-      @th_end.push(th)
+      @th_end[th] = true
       @cv.broadcast
+      @q_noaction.thread_end(th)
     end
 
     def after_check(tasks)

data/lib/pwrake/version.rb

@@ -1,3 +1,3 @@
 module Pwrake
-  VERSION = "0.9.7"
+  VERSION = "0.9.8"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pwrake
 version: !ruby/object:Gem::Version
-  version: 0.9.7
+  version: 0.9.8
 platform: ruby
 authors:
 - Masahiro TANAKA
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-06 00:00:00.000000000 Z
+date: 2013-10-17 00:00:00.000000000 Z
 dependencies: []
 description: Parallel workflow extension for Rake
 email:
@@ -26,9 +26,11 @@ files:
 - bin/pwrake
 - lib/pwrake.rb
 - lib/pwrake/application.rb
+- lib/pwrake/cache_aware_queue.rb
 - lib/pwrake/counter.rb
 - lib/pwrake/file_utils.rb
 - lib/pwrake/gfarm_feature.rb
+- lib/pwrake/gfwhere_pool.rb
 - lib/pwrake/graphviz.rb
 - lib/pwrake/locality_aware_queue.rb
 - lib/pwrake/logger.rb