pwrake 0.9.9.2 → 2.0.0

data/lib/pwrake/file_utils.rb

@@ -1,98 +0,0 @@
-module Pwrake
-module FileUtils
-  module_function
-
-  def sh(*cmd, &block)
-    options = (Hash === cmd.last) ? cmd.pop : {}
-    unless block_given?
-      show_command = cmd.join(" ")
-      show_command = show_command[0,42] + "..."
-      block = lambda { |ok, status|
-        ok or fail "Command failed with status (#{status.exitstatus}): [#{show_command}]"
-      }
-    end
-    if RakeFileUtils.verbose_flag == :default
-      options[:verbose] = true
-    else
-      options[:verbose] ||= RakeFileUtils.verbose_flag
-    end
-    options[:noop]    ||= RakeFileUtils.nowrite_flag
-    Rake.rake_check_options options, :noop, :verbose
-    Pwrake::Log.stderr_puts cmd.join(" ") if options[:verbose]
-    unless options[:noop]
-      res,status = Pwrake::FileUtils.pwrake_system(*cmd)
-      block.call(res, status)
-    end
-  end
-
-  def bq(*cmd, &block)
-    options = (Hash === cmd.last) ? cmd.pop : {}
-    unless block_given?
-      show_command = cmd.join(" ")
-      show_command = show_command[0,42] + "..."
-      block = lambda { |ok, status|
-        ok or fail "Command failed with status (#{status.exitstatus}): [#{show_command}]"
-      }
-    end
-    if RakeFileUtils.verbose_flag == :default
-      options[:verbose] = true
-    else
-      options[:verbose] ||= RakeFileUtils.verbose_flag
-    end
-    options[:noop]    ||= RakeFileUtils.nowrite_flag
-    Rake.rake_check_options options, :noop, :verbose
-    Pwrake::Log.stderr_puts cmd.join(" ") if options[:verbose]
-    unless options[:noop]
-      res,status = Pwrake::FileUtils.pwrake_backquote(*cmd)
-      block.call(res, status)
-    end
-    res
-  end
-
-  def pwrake_system(*cmd)
-    cmd_log = cmd.join(" ").inspect
-    tm = Pwrake::Timer.new("sh",cmd_log)
-
-    conn = Pwrake.current_shell
-    if conn.kind_of?(Pwrake::Shell)
-      res    = conn.system(*cmd)
-      status = Rake::PseudoStatus.new(conn.status)
-    else
-      res    = system(*cmd)
-      status = $?
-      status = Rake::PseudoStatus.new(1) if !res && status.nil?
-    end
-
-    tm.finish("status=%s cmd=%s"%[status.exitstatus,cmd_log])
-    [res,status]
-  end
-
-  # Pwrake version of backquote command
-  def pwrake_backquote(cmd)
-    cmd_log = cmd.inspect
-    tm = Pwrake::Timer.new("bq",cmd_log)
-
-    conn = Pwrake.current_shell
-    if conn.kind_of?(Pwrake::Shell)
-      res    = conn.backquote(*cmd)
-      status = Rake::PseudoStatus.new(conn.status)
-    else
-      res    = `#{cmd}`
-      status = $?
-      status = Rake::PseudoStatus.new(1) if status.nil?
-    end
-
-    tm.finish("status=%s cmd=%s"%[status.exitstatus,cmd_log])
-    [res,status]
-  end
-
-end # module Pwrake::FileUtils
-end
-
-module Rake
-  module DSL
-    include Pwrake::FileUtils
-    private(*Pwrake::FileUtils.instance_methods(false))
-  end
-end
-self.extend Rake::DSL

data/lib/pwrake/gfwhere_pool.rb

@@ -1,109 +0,0 @@
-module Pwrake
-
-  class WorkerPool
-
-    def initialize(wk_class, max)
-      @worker_class = wk_class
-      @max = max
-      @pool = []
-      @cond_pool = ConditionVariable.new
-      @mutex = Mutex.new
-    end
-
-    def find_worker
-      @mutex.synchronize do
-        while true
-          @pool.each do |w|
-            return w if w.acquire
-          end
-          if @pool.size < @max
-            Log.debug "--- #{@worker_class}:new_worker #{@pool.size+1}"
-            w = @worker_class.new(@cond_pool)
-            @pool << w
-            return w if w.acquire
-          end
-          # wait for end of work in @pool
-          @cond_pool.wait(@mutex)
-        end
-      end
-    end
-
-    def work(*args)
-      w = find_worker
-      w.run(*args)
-    end
-  end
-
-
-  class Worker
-
-    def initialize(cond_pool)
-      @cond_pool = cond_pool
-      @mutex = Mutex.new
-      @aquired = false
-    end
-
-    def acquire
-      return false if @mutex.locked?
-      if @aquired
-        return false
-      else
-        @aquired = true
-        return true
-      end
-    end
-
-    def run(*args)
-      @mutex.synchronize do
-        raise "no aquired" unless @aquired
-        r = work(*args)
-        @aquired = false
-        @cond_pool.signal
-        return r
-      end
-    end
-
-    def work(*args)
-      # inplement in subclass
-      return nil
-    end
-  end
-
-
-  class GfwhereWorker < Worker
-
-    def initialize(cond_pool)
-      super(cond_pool)
-      @io = IO.popen('gfwhere-pipe','r+')
-      @io.sync = true
-    end
-
-    def work(file)
-      return [] if file==''
-      t = Time.now
-      @io.puts(file)
-      @io.flush
-      s = @io.gets
-      if s.nil?
-        raise "gfwhere: unexpected end"
-      end
-      s.chomp!
-      if s != file
-        raise "gfwhere: file=#{file}, result=#{s}"
-      end
-      while s = @io.gets
-        s.chomp!
-        case s
-        when /^gfarm:\/\//
-          next
-        when /^Error:/
-          return []
-        else
-          Log.debug "gfwhere:path %.6f sec, file=%s" % [Time.now-t,file]
-          return s.split(/\s+/)
-        end
-      end
-    end
-  end
-
-end

data/lib/pwrake/host_list.rb

@@ -1,88 +0,0 @@
-module Pwrake
-
-  class HostInfo
-    def initalize(name,group=0,weight=1)
-      @name = name
-      @group = group
-      @weight = weight
-    end
-    attr_reader :name, :group, :weight
-  end
-
-  class HostList
-    attr_reader :group_hosts
-    attr_reader :group_core_weight
-    attr_reader :group_weight_sum
-    attr_reader :host2group
-    attr_reader :num_threads
-    attr_reader :core_list
-    attr_reader :host_count
-
-    def initialize(file=nil)
-      @file = file
-      @group_hosts = []
-      @group_core_weight = []
-      @group_weight_sum = []
-      @host2group = {}
-      require "socket"
-      if @file
-        read_host(@file)
-        @num_threads = @core_list.size
-      else
-        @num_threads = 1 if !@num_threads
-        @core_list = ['localhost'] * @num_threads
-      end
-    end
-
-    def size
-      @num_threads
-    end
-
-    def read_host(file)
-      tmplist = []
-      File.open(file) do |f|
-        re = /\[\[([\w\d]+)-([\w\d]+)\]\]/o
-        while l = f.gets
-          l = $1 if /^([^#]*)#/ =~ l
-          host, ncore, weight, group = l.split
-          if host
-            if re =~ host
-              hosts = ($1..$2).map{|i| host.sub(re,i)}
-            else
-              hosts = [host]
-            end
-            hosts.each do |host|
-              begin
-                host = Socket.gethostbyname(host)[0]
-              rescue
-                Log.info "-- FQDN not resoved : #{host}"
-              end
-              ncore = (ncore || 1).to_i
-              weight = (weight || 1).to_f
-              w = ncore * weight
-              group = (group || 0).to_i
-              tmplist << ([host] * ncore.to_i)
-              (@group_hosts[group] ||= []) << host
-              (@group_core_weight[group] ||= []) << w
-              @group_weight_sum[group] = (@group_weight_sum[group]||0) + w
-              @host2group[host] = group
-            end
-          end
-        end
-      end
-
-      @core_list = []
-      begin # alternative order
-        sz = 0
-        tmplist.each do |a|
-          @core_list << a.shift if !a.empty?
-          sz += a.size
-        end
-      end while sz>0
-
-      @host_count = Hash.new{0}
-      core_list.each{|h| @host_count[h] += 1}
-    end
-
-  end
-end

data/lib/pwrake/locality_aware_queue.rb

@@ -1,413 +0,0 @@
-module Pwrake
-
-  module TaskAlgorithm
-    def assigned
-      @assigned ||= []
-    end
-  end
-
-  # copy from ruby's thread.rb
-  class ConditionVariable
-    #
-    # Creates a new ConditionVariable
-    #
-    def initialize
-      @waiters = {}
-      @waiters_mutex = Mutex.new
-    end
-
-    #
-    # Releases the lock held in +mutex+ and waits; reacquires the lock on wakeup.
-    #
-    # If +timeout+ is given, this method returns after +timeout+ seconds passed,
-    # even if no other thread doesn't signal.
-    #
-    def wait(mutex, timeout=nil)
-      Thread.handle_interrupt(StandardError => :never) do
-        begin
-          Thread.handle_interrupt(StandardError => :on_blocking) do
-            @waiters_mutex.synchronize do
-              @waiters[Thread.current] = true
-            end
-            mutex.sleep timeout
-          end
-        ensure
-          @waiters_mutex.synchronize do
-            @waiters.delete(Thread.current)
-          end
-        end
-      end
-      self
-    end
-
-    #
-    # Wakes up the first thread in line waiting for this lock.
-    #
-    def signal
-      Thread.handle_interrupt(StandardError => :on_blocking) do
-        begin
-          t, _ = @waiters_mutex.synchronize { @waiters.shift }
-          t.run if t
-        rescue ThreadError
-          retry # t was already dead?
-        end
-      end
-      self
-    end
-
-    #
-    # Wakes up all threads waiting for this lock.
-    #
-    def broadcast
-      Thread.handle_interrupt(StandardError => :on_blocking) do
-        threads = nil
-        @waiters_mutex.synchronize do
-          threads = @waiters.keys
-          @waiters.clear
-        end
-        for t in threads
-          begin
-            t.run
-          rescue ThreadError
-          end
-        end
-      end
-      self
-    end
-  end
-
-  class LocalityConditionVariable < ConditionVariable
-
-    def signal(hints=nil)
-      if hints.nil?
-        super()
-      elsif Array===hints
-          thread = nil
-          @waiters_mutex.synchronize do
-            @waiters.each do |t,v|
-              if hints.include?(t[:hint])
-                thread = t
-                break
-              end
-            end
-            if thread
-              @waiters.delete(thread)
-            else
-              thread,_ = @waiters.shift
-            end
-          end
-          Log.debug "--- LCV#signal: hints=#{hints.inspect} thread_to_run=#{thread.inspect} @waiters.size=#{@waiters.size}"
-          begin
-            thread.run if thread
-          rescue ThreadError
-            retry # t was already dead?
-          end
-      else
-        raise ArgumentError,"argument must be an Array"
-      end
-      self
-    end
-
-
-    def signal_with_hints(hints)
-      if !Array===hints
-        raise ArgumentError,"argument must be an Array"
-      end
-      thread =
-        @waiters_mutex.synchronize do
-        th = nil
-        @waiters.each do |t,v|
-          Log.debug "--- LCV#signal_with_hints: t[:hint]=#{t[:hint]}"
-          if hints.include?(t[:hint])
-            th = t
-            break
-          end
-        end
-        Log.debug "--- LCV#signal_with_hints: hints=#{hints.inspect} thread_to_run=#{th.inspect} @waiters.size=#{@waiters.size}"
-        if th
-          @waiters.delete(th)
-        end
-        th
-      end
-      begin
-        thread.run if thread
-      rescue ThreadError
-        retry # t was already dead?
-      end
-      thread
-    end
-
-
-    def broadcast(hints=nil)
-      if hints.nil?
-        super()
-      elsif Array===hints
-          threads = []
-          @waiters_mutex.synchronize do
-            hints.each do |h|
-              @waiters.each do |t,v|
-                if t[:hint] == h
-                  threads << t
-                  break
-                end
-              end
-            end
-            threads.each do |t|
-              @waiters.delete(t)
-            end
-          end
-          Log.debug "--- LCV#broadcast: hints=#{hints.inspect} threads_to_run=#{threads.inspect} @waiters.size=#{@waiters.size}"
-          threads.each do |t|
-            begin
-              t.run
-            rescue ThreadError
-            end
-          end
-      else
-        raise ArgumentError,"argument must be an Array"
-      end
-      self
-    end
-  end
-
-
-  class LocalityAwareQueue < TaskQueue
-
-    class Throughput
-
-      def initialize(list=nil)
-        @interdomain_list = {}
-        @interhost_list = {}
-        if list
-          values = []
-          list.each do |x,y,v|
-            hash_x = (@interdomain_list[x] ||= {})
-            hash_x[y] = n = v.to_f
-            values << n
-          end
-          @min_value = values.min
-        else
-          @min_value = 1
-        end
-      end
-
-      def interdomain(x,y)
-        hash_x = (@interdomain_list[x] ||= {})
-        if v = hash_x[y]
-          return v
-        elsif v = (@interdomain_list[y] || {})[x]
-          hash_x[y] = v
-        else
-          if x == y
-            hash_x[y] = 1
-          else
-            hash_x[y] = 0.1
-          end
-        end
-        hash_x[y]
-      end
-
-      def interhost(x,y)
-        return @min_value if !x
-        hash_x = (@interhost_list[x] ||= {})
-        if v = hash_x[y]
-          return v
-        elsif v = (@interhost_list[y] || {})[x]
-          hash_x[y] = v
-        else
-          x_short, x_domain = parse_hostname(x)
-          y_short, y_domain = parse_hostname(y)
-          v = interdomain(x_domain,y_domain)
-          hash_x[y] = v
-        end
-        hash_x[y]
-      end
-
-      def parse_hostname(host)
-        /^([^.]*)\.?(.*)$/ =~ host
-        [$1,$2]
-      end
-
-    end # class Throughput
-
-
-    def init_queue(host_list)
-      @host_list = host_list
-      @cv = LocalityConditionVariable.new
-      @size = 0
-      @q = {}
-      @host_list.host_count.each{|h,n| @q[h] = @array_class.new(n)}
-      @q_group = {}
-      @host_list.group_hosts.each do |g|
-        other = @host_list.host_count.dup
-        q1 = {}
-        g.each{|h| q1[h] = @q[h]; other.delete(h)}
-        q2 = {}
-        other.each{|h,v| q2[h] = @q[h]}
-        a = [q1,q2]
-        g.each{|h| @q_group[h] = a}
-      end
-      @q_remote = @array_class.new(0)
-      @q_later = Array.new
-      @enable_steal = !Pwrake.application.pwrake_options['DISABLE_STEAL']
-      @steal_wait = (Pwrake.application.pwrake_options['STEAL_WAIT'] || 0).to_i
-      @steal_wait_max = (Pwrake.application.pwrake_options['STEAL_WAIT_MAX'] || 10).to_i
-      @steal_wait_after_enq = (Pwrake.application.pwrake_options['STEAL_WAIT_AFTER_ENQ'] || 0.1).to_f
-      @last_enq_time = Time.now
-      Log.info("-- @enable_steal=#{@enable_steal.inspect} @steal_wait=#{@steal_wait} @steal_wait_max=#{@steal_wait_max} @steal_wait_after_enq={@steal_wait_after_enq}")
-    end
-
-    attr_reader :size
-
-
-    def enq_impl(t)
-      hints = t && t.suggest_location
-      if hints.nil? || hints.empty?
-        @q_later.push(t)
-      else
-        stored = false
-        hints.each do |h|
-          if q = @q[h]
-            t.assigned.push(h)
-            q.push(t)
-            stored = true
-          end
-        end
-        if !stored
-          @q_remote.push(t)
-        end
-      end
-      @last_enq_time = Time.now
-      @size += 1
-    end
-
-
-    def deq_impl(host,n)
-      if t = deq_locate(host)
-        Log.info "-- deq_locate n=#{n} task=#{t&&t.name} host=#{host}"
-        Log.debug "--- deq_impl\n#{inspect_q}"
-        return t
-      end
-
-      #hints = []
-      #@q.each do |h,q|
-      #  hints << h if !q.empty?
-      #end
-      #if (!hints.empty?) && @cv.signal_with_hints(hints)
-      #  return nil
-      #end
-
-      if !@q_remote.empty?
-        t = @q_remote.shift
-        Log.info "-- deq_remote n=#{n} task=#{t&&t.name} host=#{host}"
-        Log.debug "--- deq_impl\n#{inspect_q}"
-        return t
-      end
-
-      if !@q_later.empty?
-        t = @q_later.shift
-        Log.info "-- deq_later n=#{n} task=#{t&&t.name} host=#{host}"
-        Log.debug "--- deq_impl\n#{inspect_q}"
-        return t
-      end
-
-      if @enable_steal && n > 0 && Time.now-@last_enq_time > @steal_wait_after_enq
-        if t = deq_steal(host)
-          Log.info "-- deq_steal n=#{n} task=#{t&&t.name} host=#{host}"
-          Log.debug "--- deq_impl\n#{inspect_q}"
-          return t
-        end
-      end
-
-      #m = [@steal_wait*(2**n), @steal_wait_max].min
-      #@cv.wait(@mutex,m)
-      @cv.wait(@mutex)
-      nil
-    end
-
-
-    def deq_locate(host)
-      q = @q[host]
-      if q && !q.empty?
-        t = q.shift
-        if t
-          t.assigned.each do |h|
-            @q[h].delete(t)
-          end
-        end
-        @size -= 1
-        return t
-      else
-        nil
-      end
-    end
-
-    def deq_steal(host)
-      # select a task based on many and close
-      max_host = nil
-      max_num  = 0
-      @q_group[host].each do |qg|
-        qg.each do |h,a|
-          if !a.empty?
-            d = a.size
-            if d > max_num
-              max_host = h
-              max_num  = d
-            end
-          end
-        end
-        if max_num > 0
-          Log.info "-- deq_steal max_host=#{max_host} max_num=#{max_num}"
-          t = deq_locate(max_host)
-          return t if t
-        end
-      end
-      nil
-    end
-
-    def inspect_q
-      s = ""
-      b = proc{|h,q|
-        s += " #{h}: size=#{q.size} "
-        case q.size
-        when 0
-          s += "[]\n"
-        when 1
-          s += "[#{q.first.name}]\n"
-        when 2
-          s += "[#{q.first.name}, #{q.last.name}]\n"
-        else
-          s += "[#{q.first.name},.. #{q.last.name}]\n"
-        end
-      }
-      b.call("noaction",@q_noaction)
-      @q.each(&b)
-      b.call("remote",@q_remote)
-      b.call("later",@q_later)
-      s
-    end
-
-    def size
-      @size
-    end
-
-    def clear
-      @q_noaction.clear
-      @q.each{|h,q| q.clear}
-      @q_remote.clear
-      @q_later.clear
-    end
-
-    def empty?
-      @q.all?{|h,q| q.empty?} &&
-        @q_noaction.empty? &&
-        @q_remote.empty? &&
-        @q_later.empty?
-    end
-
-    def finish
-      super
-    end
-
-  end
-end