pwrake 0.9.9 → 0.9.9.2

data/lib/pwrake/option.rb

@@ -48,6 +48,8 @@ module Pwrake
         'NUM_NOACTION_THREADS', # default=4 when gfarm, else 1
         'STEAL_WAIT',
         'STEAL_WAIT_MAX',
+        'GRAPH_PARTITION',
+        'PLOT_PARTITION',
 
         ['HOSTFILE','HOSTS'],
         ['LOGFILE','LOG',
@@ -108,7 +110,6 @@ module Pwrake
     # ----- init -----
 
     def init_option
-      @host_group = []
       init_options
       init_pass_env
       init_logger
@@ -127,7 +128,7 @@ module Pwrake
       @counter = Counter.new
     end
 
-    attr_reader :core_list
+    attr_reader :host_list
     attr_reader :counter
     attr_reader :logfile
     attr_reader :queue_class
@@ -312,53 +313,10 @@ module Pwrake
       if @hostfile && @num_threads
         raise "Cannot set `hostfile' and `num_threads' simultaneously"
       end
-      if @hostfile
-        require "socket"
-        tmplist = []
-        File.open(@hostfile) do |f|
-          re = /\[\[([\w\d]+)-([\w\d]+)\]\]/o
-          while l = f.gets
-            l = $1 if /^([^#]*)#/ =~ l
-            host, ncore, group = l.split
-            if host
-              if re =~ host
-                hosts = ($1..$2).map{|i| host.sub(re,i)}
-              else
-                hosts = [host]
-              end
-              hosts.each do |host|
-                begin
-                  host = Socket.gethostbyname(host)[0]
-                rescue
-                  Log.info "-- FQDN not resoved : #{host}"
-                end
-                ncore = (ncore || 1).to_i
-                group = (group || 0).to_i
-                tmplist << ([host] * ncore.to_i)
-                @host_group[group] ||= []
-                @host_group[group] << host
-              end
-            end
-          end
-        end
-        #
-        @core_list = []
-        begin # alternative order
-          sz = 0
-          tmplist.each do |a|
-            @core_list << a.shift if !a.empty?
-            sz += a.size
-          end
-        end while sz>0
-        @num_threads = @core_list.size
-      else
-        @num_threads = 1 if !@num_threads
-        @core_list = ['localhost'] * @num_threads
-      end
-      Log.info "num_cores=#{@core_list.size}"
+      @host_list = HostList.new(@hostfile || @num_threads)
+      Log.info "num_cores=#{@host_list.size}"
     end
 
-
     def set_filesystem
       if fn = @opts["PROFILE"]
         Shell.profiler.open(fn,@opts['GNU_TIME'],@opts['PLOT_PARALLELISM'])
@@ -397,7 +355,7 @@ module Pwrake
 	else
 	  @queue_class = LocalityAwareQueue
 	end
-        @num_noaction_threads = (n_noaction_th || [8,@num_threads].max).to_i
+        @num_noaction_threads = (n_noaction_th || [8,@host_list.num_threads].max).to_i
         @postprocess = GfarmPostprocess.new
         Log.debug "--- @queue_class=#{@queue_class}"
       else

data/lib/pwrake/report/report.rb

@@ -48,7 +48,14 @@ EOL
         end
       end
 
-      @sh_table = CSV.read(@csv_file,:headers=>true)
+      begin
+        @sh_table = CSV.read(@csv_file,:headers=>true)
+      rescue
+        $stderr.puts "error in reading "+@csv_file
+        $stderr.puts $!, $@
+        exit
+      end
+
       h = {}
       @elap_sum = 0
       @sh_table.each do |row|
@@ -294,7 +301,13 @@ set title 'histogram of elapsed time'"
   class TaskStat
 
     def initialize(task_file, sh_table)
-      @task_table = CSV.read(task_file,:headers=>true)
+      begin
+        @task_table = CSV.read(task_file,:headers=>true)
+      rescue
+        $stderr.puts "error in reading "+task_file
+        $stderr.puts $!, $@
+        exit
+      end
       @count = Hash.new(0)
       task_locality
       stat_sh_table(sh_table)

data/lib/pwrake/task_algorithm.rb

@@ -55,6 +55,26 @@ module Pwrake
 
     def location=(a)
       @location = a
+      @group = []
+      @location.each do |host|
+        @group |= [Pwrake.application.host_list.host2group[host]]
+      end
+    end
+
+    def group
+      @group ||= []
+    end
+
+    def group_id
+      @group_id
+    end
+
+    def group_id=(i)
+      @group_id = i
+    end
+
+    def suggest_location=(a)
+      @suggest_location = a
     end
 
     def task_id
@@ -64,6 +84,11 @@ module Pwrake
     def invoke_modify(*args)
       return if @already_invoked
 
+      if Pwrake.application.pwrake_options['GRAPH_PARTITION']
+        require 'pwrake/mcgp'
+        MCGP.graph_partition
+      end
+
       application.start_worker
 
       if false
@@ -141,6 +166,10 @@ module Pwrake
       Log.debug "--- pw_invoke (#{name}) enq_subseq time=#{Time.now-t} sec"
     end
 
+    def get_file_stat
+      @file_stat ||= File::Stat.new(name)
+    end
+
     def log_task(time_start)
       time_end = Time.now
 
@@ -391,6 +420,29 @@ module Pwrake
       @suggest_location
     end
 
+    def priority
+      if has_input_file? && @priority.nil?
+        sum_tm = 0
+        sum_sz = 0
+        @prerequisites.each do |preq|
+          pq = application[preq]
+          sz = pq.file_size
+          if sz > 0
+            tm = pq.file_mtime - START_TIME
+            sum_tm += tm * sz
+            sum_sz += sz
+          end
+        end
+        if sum_sz > 0
+          @priority = sum_tm / sum_sz
+        else
+          @priority = 0
+        end
+        Log.debug "--- task_name=#{name} priority=#{@priority} sum_file_size=#{sum_sz}"
+      end
+      @priority || 0
+    end
+
     def input_file_mtime
       if has_input_file? && @input_file_mtime.nil?
         hash = Hash.new
@@ -418,25 +470,6 @@ module Pwrake
       @input_file_mtime
     end
 
-    def suggest_location2
-      if kind_of?(Rake::FileTask) && preq_name = @prerequisites[0]
-        application[preq_name].location
-      end
-    end
-
-    def log_host(exec_host)
-      # exec_host = Pwrake.current_shell.host
-      if loc = suggest_location()
-        Pwrake.application.count( loc, exec_host )
-        if loc.include? exec_host
-          compare = "=="
-        else
-          compare = "!="
-        end
-        Log.info "-- access to #{@prerequisites[0]}: file_host=#{loc.inspect} #{compare} exec_host=#{exec_host}"
-      end
-    end
-
   end
 
 end # module Pwrake

data/lib/pwrake/task_queue.rb

@@ -12,18 +12,22 @@ module Pwrake
       super()
     end
 
+    def shift
+      pop
+    end
+
     def push(t)
-      task_id = t.task_id
-      if empty? || last.task_id <= task_id
+      priority = t.priority
+      if empty? || last.priority <= priority
         super(t)
-      elsif first.task_id > task_id
+      elsif first.priority > priority
         unshift(t)
       else
         lower = 0
         upper = size-1
         while lower+1 < upper
           mid = ((lower + upper) / 2).to_i
-          if self[mid].task_id <= task_id
+          if self[mid].priority <= priority
             lower = mid
           else
             upper = mid
@@ -37,23 +41,23 @@ module Pwrake
       if size < 40
         return super(t)
       end
-      task_id = t.task_id
-      if last.task_id < task_id || first.task_id > task_id
+      priority = t.priority
+      if last.priority < priority || first.priority > priority
         nil
       else
         lower = 0
         upper = size-1
         while lower+1 < upper
           mid = ((lower + upper) / 2).to_i
-          if self[mid].task_id < task_id
+          if self[mid].priority < priority
             lower = mid
           else
             upper = mid
           end
         end
         mid = upper
-        if self[mid].task_id == task_id
-          Log.debug "--- TQA#index=#{mid}, task_id=#{task_id}"
+        if self[mid].priority == priority
+          Log.debug "--- TQA#index=#{mid}, priority=#{priority}"
           mid
         end
       end
@@ -78,39 +82,79 @@ module Pwrake
     end
   end
 
+  class RankCounter
 
-  # LIFO + HRF
-  class LifoHrfQueueArray < Array
-    def initialize(n)
-      @n = (n>0) ? n : 1
-      @count = []
+    def initialize
+      @ntask = []
+      @nproc = 0
+      @mutex = Mutex.new
     end
 
-    def push(t)
-      super(t)
-      r = t.rank
-      @count[r] = (@count[r]||0) + 1
+    def add_nproc(n)
+      @mutex.synchronize do
+        @nproc += n
+      end
     end
 
-    def shift
-      if empty?
-        return nil
+    def incr(r)
+      @mutex.synchronize do
+        @ntask[r] = (@ntask[r]||0) + 1
       end
+    end
+
+    def get_task
+      @mutex.synchronize do
+        (@ntask.size-1).downto(0) do |r|
+          c = @ntask[r]
+          if c && c>0
+            t = yield(c, @nproc, r)
+            #t = (c<=@n) ? pop_last_rank(r) : pop
+            if t
+              @ntask[t.rank] -= 1
+              Log.debug "--- RankCount rank=#{r} nproc=#{@nproc} count=#{c} t.rank=#{t.rank} t.name=#{t.name}"
+            end
+            return t
+          end
+        end
+      end
+      nil
+    end
+  end
+
+  # HRF mixin module
+  module HrfQueue
+
+    def hrf_init(n=nil)
+      @nproc = n || 0
+      @count = []
+    end
+
+    def hrf_push(t)
+      if t
+        r = t.rank
+        @count[r] = (@count[r]||0) + 1
+      end
+    end
+
+    def hrf_get
       (@count.size-1).downto(0) do |r|
         c = @count[r]
         if c && c>0
-          t = (c<=@n) ? pop_last_rank(r) : pop
-          @count[t.rank] -= 1
+          t = (c <= @nproc) ? pop_last_rank(r) : pop_super
+          if t
+            @count[t.rank] -= 1
+          end
           return t
         end
       end
+      raise "no element"
       nil
     end
 
     def pop_last_rank(r)
       (size-1).downto(0) do |i|
         t = at(i)
-        if t.rank == r
+        if t && t.rank == r
           delete_at(i)
           return t
         end
@@ -119,9 +163,58 @@ module Pwrake
     end
   end
 
+  # LIFO + HRF
+  class LifoHrfQueueArray < Array
+    include HrfQueue
+
+    def initialize(n)
+      super()
+      hrf_init(n)
+    end
+
+    def push(t)
+      super(t)
+      hrf_push(t)
+    end
+
+    def shift
+      return nil if empty?
+      hrf_get
+    end
+
+    def pop_super
+      pop
+    end
+  end
+
+  # Priority + HRF
+  class PriorityHrfQueueArray < PriorityQueueArray
+    include HrfQueue
+
+    def initialize(n)
+      super(n)
+      hrf_init(n)
+    end
+
+    def push(t)
+      super(t)
+      hrf_push(t)
+    end
+
+    def shift
+      return nil if empty?
+      hrf_get
+    end
+
+    def pop_super
+      pop
+    end
+  end
+
 
   # Rank-Even Last In First Out
   class RankQueueArray
+
     def initialize(n)
       @q = []
       @size = 0
@@ -129,7 +222,7 @@ module Pwrake
     end
 
     def push(t)
-      r = t.rank
+      r = t ? t.rank : 0
       a = @q[r]
       if a.nil?
         @q[r] = a = []
@@ -362,7 +455,7 @@ module Pwrake
 
   class TaskQueue
 
-    def initialize(core_list)
+    def initialize(host_list)
       @finished = false
       @halt = false
       @mutex = Mutex.new
@@ -371,7 +464,7 @@ module Pwrake
       @q_noaction = NoActionQueue.new
       pri = Pwrake.application.pwrake_options['QUEUE_PRIORITY'] || "RANK"
       case pri
-      when /dfs/i
+      when /prio/i
         @array_class = PriorityQueueArray
       when /fifo/i
         @array_class = FifoQueueArray # Array # Fifo
@@ -379,18 +472,20 @@ module Pwrake
         @array_class = LifoQueueArray
       when /lihr/i
         @array_class = LifoHrfQueueArray
+      when /prhr/i
+        @array_class = PriorityHrfQueueArray
       when /rank/i
         @array_class = RankQueueArray
       else
         raise RuntimeError,"unknown option for QUEUE_PRIORITY: "+pri
       end
       Log.debug "--- TQ#initialize @array_class=#{@array_class.inspect}"
-      init_queue(core_list)
+      init_queue(host_list)
     end
 
-    def init_queue(core_list)
+    def init_queue(host_list)
       @cv = TaskConditionVariable.new
-      @q_input = @array_class.new(core_list.size)
+      @q_input = @array_class.new(host_list.size)
       @q_later = Array.new
     end
 
@@ -476,6 +571,7 @@ module Pwrake
               @cv.signal
               return false
             end
+            #@cv.signal
             @cv.wait(@mutex)
             n = 0