pwrake 0.9.9.2 → 2.0.0

data/lib/pwrake/option/option_filesystem.rb

@@ -0,0 +1,112 @@
+module Pwrake
+
+  class Option
+
+    attr_reader :worker_option
+    attr_reader :worker_progs
+    attr_reader :queue_class
+
+    def setup_filesystem
+
+      @worker_progs = %w[ writer log_executor executor invoker shared_directory ]
+      @worker_option = {
+        :base_dir  => "",
+        :work_dir  => self['WORK_DIR'],
+        :log_dir   => self['LOG_DIR'],
+        :pass_env  => self['PASS_ENV'],
+        :ssh_option => self['SSH_OPTION'],
+        :shell_command => self['SHELL_COMMAND'],
+        :shell_rc  => self['SHELL_RC'],
+        :heartbeat => self['HEARTBEAT']
+      }
+
+      if @filesystem.nil?
+        case mount_type
+        when /gfarm2fs/
+          self['FILESYSTEM'] = @filesystem = 'gfarm'
+        end
+      end
+
+      #n_noaction_th = self['NUM_NOACTION_THREADS']
+
+      case @filesystem
+      when 'gfarm'
+        require "pwrake/queue/locality_aware_queue"
+        require "pwrake/gfarm/gfarm_path"
+        GfarmPath.subdir = self['GFARM_SUBDIR']
+        @filesystem  = 'gfarm'
+        base = self['GFARM_BASEDIR']
+        prefix = self['GFARM_PREFIX']
+        mntpnt = "#{base}/#{prefix}"
+        @worker_option.merge!({
+          :shared_directory => "GfarmDirectory",
+          :base_dir => mntpnt,
+          :work_dir => GfarmPath.pwd.to_s,
+          :gfarm2fs_option => self['GFARM2FS_OPTION'],
+          :gfarm2fs_debug => self['GFARM2FS_DEBUG'],
+          :gfarm2fs_debug_wait => self['GFARM2FS_DEBUG_WAIT'],
+          :single_mp => self['GFARM_SINGLE_MP']
+        })
+        @worker_progs << "gfarm_directory"
+
+	if self['DISABLE_AFFINITY']
+	  @queue_class = "TaskQueue"
+	else
+	  @queue_class = "LocalityAwareQueue"
+	end
+        #@num_noaction_threads = (n_noaction_th || [8,@host_map.num_threads].max).to_i
+      else
+        @filesystem  = 'nfs'
+        @queue_class = "TaskQueue"
+        #@num_noaction_threads = (n_noaction_th || 1).to_i
+        @worker_option[:shared_directory] = "SharedDirectory"
+      end
+      @worker_progs << "worker_main"
+      Log.debug "@queue_class=#{@queue_class}"
+    end
+
+    def max_postprocess_pool
+      case @filesystem
+      when 'gfarm'
+        self['MAX_GFWHERE_WORKER']
+      else
+        1
+      end
+    end
+
+    def postprocess(runner)
+      case @filesystem
+      when 'gfarm'
+        require "pwrake/gfarm/gfarm_postprocess"
+        GfarmPostprocess.new(runner)
+      else
+        require "pwrake/master/postprocess"
+        Postprocess.new(runner)
+      end
+    end
+
+    def mount_type(d=nil)
+      mtab = '/etc/mtab'
+      if File.exist?(mtab)
+        d ||= mountpoint_of_cwd
+        open(mtab,'r') do |f|
+          f.each_line do |l|
+            if /#{d} (?:type )?(\S+)/o =~ l
+              return $1
+            end
+          end
+        end
+      end
+      nil
+    end
+
+    def mountpoint_of_cwd
+      d = Pathname.pwd
+      while !d.mountpoint?
+        d = d.parent
+      end
+      d
+    end
+
+  end
+end

data/lib/pwrake/queue/locality_aware_queue.rb

@@ -0,0 +1,158 @@
+module Pwrake
+
+  class LocalityAwareQueue < TaskQueue
+
+    def init_queue(group_map=nil)
+      # group_map = {gid1=>[hid1,hid2,...], ...}
+      @size_q = 0
+      @q = {}
+      @host_map.by_id.each{|h| @q[h.id] = @array_class.new(h.ncore)}
+      @q_group = {}
+      group_map ||= {1=>@host_map.by_id.map{|h| h.id}}
+      group_map.each do |gid,ary|
+        q1 = {}     # same group
+        q2 = @q.dup # other groups
+        ary.each{|hid| q1[hid] = q2.delete(hid)}
+        a = [q1,q2]
+        ary.each{|hid| @q_group[hid] = a}
+      end
+      @q_remote = @array_class.new(0)
+      @disable_steal = Rake.application.pwrake_options['DISABLE_STEAL']
+      @last_enq_time = Time.now
+      @n_turn = @disable_steal ? 1 : 2
+    end
+
+
+    def enq_impl(t)
+      hints = t && t.suggest_location
+      Log.debug "enq #{t.name} hints=#{hints.inspect}"
+      if hints.nil? || hints.empty?
+        @q_remote.push(t)
+      else
+        stored = false
+        hints.each do |h|
+          host_info = @host_map.by_name[h]
+          if host_info && q = @q[host_info.id]
+            t.assigned.push(host_info.id)
+            q.push(t)
+            stored = true
+          end
+        end
+        if stored
+          @size_q += 1
+        else
+          @q_remote.push(t)
+        end
+      end
+      @last_enq_time = Time.now
+    end
+
+    def turn_empty?(turn)
+      case turn
+      when 0
+        @q_no_action.empty? && @size_q == 0 && @q_remote.empty?
+      when 1
+        @size_q == 0
+      end
+    end
+
+    def deq_impl(host_info, turn)
+      host = host_info.name
+      case turn
+      when 0
+        if t = @q_no_action.shift
+          Log.debug "deq_no_action task=#{t&&t.name} host=#{host}"
+          return t
+        elsif t = deq_locate(host_info,host_info)
+          Log.debug "deq_locate task=#{t&&t.name} host=#{host}"
+          return t
+        elsif t = @q_remote.shift(host_info)
+          Log.debug "deq_remote task=#{t&&t.name}"
+          return t
+        else
+          nil
+        end
+      when 1
+        if t = deq_steal(host_info)
+          Log.debug "deq_steal task=#{t&&t.name} host=#{host}"
+          return t
+        else
+          nil
+        end
+      end
+    end
+
+    def deq_locate(q_host,run_host)
+      q = @q[q_host.id]
+      if q && !q.empty?
+        t = q.shift(run_host)
+        if t
+          t.assigned.each do |h|
+            @q[h].delete(t)
+          end
+          @size_q -= 1
+        end
+        return t
+      else
+        nil
+      end
+    end
+
+    def deq_steal(host_info)
+      # select a task based on many and close
+      max_host = nil
+      max_num  = 0
+      @q_group[host_info.id].each do |qg|
+        qg.each do |h,a|
+          if !a.empty? # && h!=host_info.id
+            d = a.size
+            if d > max_num
+              max_host = h
+              max_num  = d
+            end
+          end
+        end
+        if max_num > 0
+          max_info = @host_map.by_id[max_host]
+          Log.debug "deq_steal max_host=#{max_info.name} max_num=#{max_num}"
+          t = host_info.steal_phase{|h| deq_locate(max_info,h)}
+          #Log.debug "deq_steal task=#{t.inspect}"
+          return t if t
+        end
+      end
+      nil
+    end
+
+    def inspect_q
+      s = _qstr("noaction",@q_no_action)
+      if @size_q == 0
+        n = @q.size
+      else
+        n = 0
+        @q.each do |h,q|
+          if q.size > 0
+            s << _qstr(@host_map.by_id[h].name,q)
+          else
+            n += 1
+          end
+        end
+      end
+      s << _qstr("local*#{n}",[]) if n > 0
+      s << _qstr("remote",@q_remote)
+      s
+    end
+
+    def clear
+      @q_no_action.clear
+      @q.each{|h,q| q.clear}
+      @q_remote.clear
+    end
+
+    def empty?
+      @size_q == 0 &&
+        @q_no_action.empty? &&
+        @q_remote.empty?
+    end
+
+  end
+end

data/lib/pwrake/queue/no_action_queue.rb

@@ -0,0 +1,67 @@
+module Pwrake
+
+  class NoActionQueue
+
+    def initialize
+      @que = []
+      prio = Rake.application.pwrake_options['NOACTION_QUEUE_PRIORITY'] || 'fifo'
+      case prio
+      when /fifo/i
+        @prio = 0
+        Log.debug "NOACTION_QUEUE_PRIORITY=FIFO"
+      when /lifo/i
+        @prio = 1
+        Log.debug "NOACTION_QUEUE_PRIORITY=LIFO"
+      when /rand/i
+        @prio = 2
+        Log.debug "NOACTION_QUEUE_PRIORITY=RAND"
+      else
+        raise RuntimeError,"unknown option for NOACTION_QUEUE_PRIORITY: "+prio
+      end
+    end
+
+    def push(obj)
+      @que.push obj
+    end
+
+    alias << push
+    alias enq push
+
+    def pop
+      case @prio
+      when 0
+        x = @que.shift
+      when 1
+        x = @que.pop
+      when 2
+        x = @que.delete_at(rand(@que.size))
+      end
+      return x
+    end
+
+    alias shift pop
+    alias deq pop
+
+    def empty?
+      @que.empty?
+    end
+
+    def clear
+      @que.clear
+    end
+
+    def length
+      @que.length
+    end
+    alias size length
+
+    def first
+      @que.first
+    end
+
+    def last
+      @que.last
+    end
+
+  end
+end

data/lib/pwrake/queue/queue_array.rb

@@ -0,0 +1,366 @@
+require "forwardable"
+
+module Pwrake
+
+  class PriorityQueueArray < Array
+    def initialize(n)
+      super()
+    end
+
+    def shift
+      pop
+    end
+
+    def push(t)
+      priority = t.priority
+      if empty? || last.priority <= priority
+        super(t)
+      elsif first.priority > priority
+        unshift(t)
+      else
+        lower = 0
+        upper = size-1
+        while lower+1 < upper
+          mid = ((lower + upper) / 2).to_i
+          if self[mid].priority <= priority
+            lower = mid
+          else
+            upper = mid
+          end
+        end
+        insert(upper,t)
+      end
+    end
+
+    def index(t)
+      if size < 40
+        return super(t)
+      end
+      priority = t.priority
+      if last.priority < priority || first.priority > priority
+        nil
+      else
+        lower = 0
+        upper = size-1
+        while lower+1 < upper
+          mid = ((lower + upper) / 2).to_i
+          if self[mid].priority < priority
+            lower = mid
+          else
+            upper = mid
+          end
+        end
+        mid = upper
+        if self[mid].priority == priority
+          Log.debug "TQA#index=#{mid}, priority=#{priority}"
+          mid
+        end
+      end
+    end
+  end # PriorityQueueArray
+
+
+  class LifoQueueArray < Array
+    def initialize(n_cores=nil)
+      super()
+    end
+
+    def shift(host_info)
+      (size-1).downto(0) do |i|
+        if at(i).acceptable_for(host_info)
+          return delete_at(i)
+        #else
+        #  Log.debug "task[#{at(i).name}] is rejected"
+        end
+      end
+      nil
+    end
+  end
+
+  class FifoQueueArray < Array
+    def initialize(n_cores=nil)
+      super()
+    end
+
+    def shift(host_info)
+      size.times do |i|
+        if at(i).acceptable_for(host_info)
+          return delete_at(i)
+        end
+      end
+      nil
+    end
+  end
+
+  class RankCounter
+
+    def initialize
+      @ntask = []
+      @nproc = 0
+      @mutex = Mutex.new
+    end
+
+    def add_nproc(n)
+      @mutex.synchronize do
+        @nproc += n
+      end
+    end
+
+    def incr(r)
+      @mutex.synchronize do
+        @ntask[r] = (@ntask[r]||0) + 1
+      end
+    end
+
+    def get_task
+      @mutex.synchronize do
+        (@ntask.size-1).downto(0) do |r|
+          c = @ntask[r]
+          if c && c>0
+            t = yield(c, @nproc, r)
+            #t = (c<=@n) ? pop_last_rank(r) : pop
+            if t
+              @ntask[t.rank] -= 1
+              Log.debug "RankCount rank=#{r} nproc=#{@nproc} count=#{c} t.rank=#{t.rank} t.name=#{t.name}"
+            end
+            return t
+          end
+        end
+      end
+      nil
+    end
+  end
+
+  # HRF mixin module
+  module HrfQueue
+
+    def hrf_init(n_cores=nil)
+      @nproc = n_cores || 0
+      @count = []
+    end
+
+    def hrf_push(t)
+      r = t.rank
+      c = @count[r]
+      @count[r] = (c) ? c+1 : 1
+    end
+
+    def hrf_get(host_info)
+      (@count.size-1).downto(0) do |r|
+        c = @count[r]
+        if c && c>0
+          t = (c <= @nproc) ? pop_last_rank(r,host_info) : pop_super(host_info)
+          hrf_delete(t) if t
+          return t
+        end
+      end
+      raise "no element"
+      nil
+    end
+
+    def pop_last_rank(r,host_info)
+      (size-1).downto(0) do |i|
+        tw = at(i)
+        if tw.rank == r && tw.acceptable_for(host_info)
+          return delete_at(i)
+        end
+      end
+      nil
+    end
+
+    def hrf_delete(t)
+      @count[t.rank] -= 1
+    end
+
+    def check(t=nil)
+      sum = 0
+      @count.each{|x| sum+=x if x}
+      if size != sum
+        #$stderr.puts self.inspect
+        #$stderr.puts t.inspect if t
+        raise "sise != @count.sum"
+      end
+    end
+  end
+
+  # LIFO + HRF
+  class LifoHrfQueueArray
+    include HrfQueue
+    extend Forwardable
+    def_delegators :@a, :empty?, :size, :first, :last, :at, :delete_at
+
+    def initialize(n_cores)
+      @a = LifoQueueArray.new
+      hrf_init(n_cores)
+    end
+
+    def push(t)
+      @a.push(t)
+      hrf_push(t)
+    end
+
+    def shift(host_info)
+      return nil if empty?
+      hrf_get(host_info)
+    end
+
+    def delete(t)
+      if x=@a.delete(t)
+        hrf_delete(t)
+      end
+      x
+    end
+
+    def pop_super(host_info)
+      @a.shift(host_info)
+    end
+  end
+
+
+  # Priority + HRF
+  class PriorityHrfQueueArray < PriorityQueueArray
+    include HrfQueue
+
+    def initialize(n)
+      super(n)
+      hrf_init(n)
+    end
+
+    def push(t)
+      super(t)
+      hrf_push(t)
+    end
+
+    def shift
+      return nil if empty?
+      hrf_get
+    end
+
+    def pop_super
+      pop
+    end
+  end
+
+
+  # Rank-Even Last In First Out
+  class RankQueueArray
+
+    def initialize(n)
+      @q = []
+      @size = 0
+      @n = (n>0) ? n : 1
+    end
+
+    def push(t)
+      r = t ? t.rank : 0
+      a = @q[r]
+      if a.nil?
+        @q[r] = a = []
+      end
+      @size += 1
+      a.push(t)
+    end
+
+    def size
+      @size
+    end
+
+    def empty?
+      @size == 0
+    end
+
+    def shift
+      if empty?
+        return nil
+      end
+      (@q.size-1).downto(0) do |i|
+        a = @q[i]
+        next if a.nil? || a.empty?
+        @size -= 1
+        if a.size <= @n
+          return pop_last_max(a)
+        else
+          return shift_weighted
+        end
+      end
+      raise "ELIFO: @q=#{@q.inspect}"
+    end
+
+    def shift_weighted
+      weight, weight_avg = RANK_STAT.rank_weight
+      wsum = 0.0
+      q = []
+      @q.each_with_index do |a,i|
+        next if a.nil? || a.empty?
+        w = weight[i]
+        w = weight_avg if w.nil?
+        # w *= a.size
+        wsum += w
+        q << [a,wsum]
+      end
+      #
+      x = rand() * wsum
+      Log.debug "shift_weighted x=#{x} wsum=#{wsum} weight=#{weight.inspect}"
+      q.each do |a,w|
+        if w > x
+          return a.pop
+        end
+      end
+      raise "ELIFO: wsum=#{wsum} x=#{x}"
+    end
+
+    def pop_last_max(a)
+      if a.size < 2
+        return a.pop
+      end
+      y_max = nil
+      i_max = nil
+      n = [@n, a.size].min
+      (-n..-1).each do |i|
+        y = a[i].input_file_size
+        if y_max.nil? || y > y_max
+          y_max = y
+          i_max = i
+        end
+      end
+      a.delete_at(i_max)
+    end
+
+    def first
+      return nil if empty?
+      @q.size.times do |i|
+        a = @q[i]
+        unless a.nil? || a.empty?
+          return a.first
+        end
+      end
+    end
+
+    def last
+      return nil if empty?
+      @q.size.times do |i|
+        a = @q[-i-1]
+        unless a.nil? || a.empty?
+          return a.last
+        end
+      end
+    end
+
+    def delete(t)
+      n = 0
+      @q.each do |a|
+        if a
+          a.delete(t)
+          n += a.size
+        end
+      end
+      @size = n
+    end
+
+    def clear
+      @q.clear
+      @size = 0
+    end
+  end
+
+end