pwrake 0.9.3

data/lib/pwrake/graphviz.rb

@@ -0,0 +1,113 @@
+module Pwrake
+
+  class Graphviz
+
+    def initialize
+      @nodes = []
+      @edges = []
+      # @node_id = {}
+      @filenode_id = {}
+      @tasknode_id = {}
+      @node_name = {}
+      @count = 0
+      @traced = {}
+    end
+
+    attr_reader :filenode_id, :tasknode_id, :node_name
+
+    def trace( name = :default, target = nil )
+      traced_cond = @traced[name]
+
+      task = Rake.application[name]
+
+      #if task.kind_of?(Rake::FileTask)
+      if task.kind_of?(Rake::Task)
+        push_filenode( name )
+        if !task.actions.empty? and !traced_cond
+          push_tasknode( name )
+          push_taskedge( name )
+        end
+        push_fileedge( name, target )
+        target = name
+      end
+      @traced[name] = true
+
+      if !traced_cond
+        task.prerequisites.each_with_index do |prereq,i|
+          trace( prereq, target )
+        end
+      end
+    end
+
+    def trim( name )
+      name = name.to_s
+      name = File.basename(name)
+      name.sub(/H\d+/,'').sub(/object\d+/,"")
+    end
+
+    def push_filenode( name )
+      if @filenode_id[name].nil?
+        tag = "T#{@count}"
+        @count += 1
+        @filenode_id[name] = tag
+        @node_name[tag] = name
+        @nodes.push "#{tag} [label=\"#{trim(name)}\", shape=box];"
+      end
+    end
+
+    def push_tasknode( name )
+      if @tasknode_id[name].nil?
+        tag = "T#{@count}"
+        @count += 1
+        @tasknode_id[name] = tag
+        @node_name[tag] = name
+        label = Rake.application[name].comment
+        @nodes.push "#{tag} [label=\"#{label}\", shape=ellipse];"
+      end
+    end
+
+    def push_fileedge( name, target )
+      if target
+        if n2 = @tasknode_id[target]
+          n1 = @filenode_id[name]
+        elsif n1 = @tasknode_id[name]
+          n2 = @filenode_id[target]
+        else
+          n1 = @filenode_id[name]
+          n2 = @filenode_id[target]
+        end
+        @edges.push "#{n1} -> #{n2};"
+      end
+    end
+
+    def push_taskedge( name )
+      if n1 = @tasknode_id[name]
+        n2 = @filenode_id[name]
+        @edges.push "#{n1} -> #{n2};"
+      end
+    end
+
+    def write(file)
+      open(file, "w") do |w|
+        #w.puts "digraph sample {\ngraph [size=\"12,100\",ranksep=1.5,nodesep=0.2];"
+        w.puts "digraph sample {"
+        w.puts "graph [size=\"70,70\", rankdir=LR];"
+        @nodes.each do |x|
+          w.puts x
+        end
+        @edges.each do |x|
+          w.puts x
+        end
+        w.puts "}"
+      end
+    end
+  end
+end
+
+task "graphviz", :file do |t,a|
+  file = a[:file] || 'pwrake.dot'
+  g = Pwrake::Graphviz.new
+  g.trace
+  g.write(file)
+  $stderr.puts "Wrote task graph to `#{file}'"
+end

data/lib/pwrake/locality_aware_queue.rb

@@ -0,0 +1,254 @@
+module Pwrake
+
+  module TaskAlgorithm
+    def assigned
+      @assigned ||= []
+    end
+  end
+
+
+  class LocalityAwareQueue < TaskQueue
+
+    class Throughput
+
+      def initialize(list=nil)
+        @interdomain_list = {}
+        @interhost_list = {}
+        if list
+          values = []
+          list.each do |x,y,v|
+            hash_x = (@interdomain_list[x] ||= {})
+            hash_x[y] = n = v.to_f
+            values << n
+          end
+          @min_value = values.min
+        else
+          @min_value = 1
+        end
+      end
+
+      def interdomain(x,y)
+        hash_x = (@interdomain_list[x] ||= {})
+        if v = hash_x[y]
+          return v
+        elsif v = (@interdomain_list[y] || {})[x]
+          hash_x[y] = v
+        else
+          if x == y
+            hash_x[y] = 1
+          else
+            hash_x[y] = 0.1
+          end
+        end
+        hash_x[y]
+      end
+
+      def interhost(x,y)
+        return @min_value if !x
+        hash_x = (@interhost_list[x] ||= {})
+        if v = hash_x[y]
+          return v
+        elsif v = (@interhost_list[y] || {})[x]
+          hash_x[y] = v
+        else
+          x_short, x_domain = parse_hostname(x)
+          y_short, y_domain = parse_hostname(y)
+          v = interdomain(x_domain,y_domain)
+          hash_x[y] = v
+        end
+        hash_x[y]
+      end
+
+      def parse_hostname(host)
+        /^([^.]*)\.?(.*)$/ =~ host
+        [$1,$2]
+      end
+
+    end # class Throughput
+
+
+    def initialize(hosts,opt={})
+      super(opt)
+      @hosts = hosts
+      @throughput = Throughput.new
+      @size = 0
+      @q = nil
+      @q1 = []
+      @q2 = {}
+      @hosts.each{|h| @q2[h]=[]}
+      @q2[nil] = []
+      @enable_steal = !opt['disable_steal']
+      @time_prev = Time.now
+
+      @thread = Thread.new{thread_loop}
+    end
+
+    attr_reader :size
+
+
+    def thread_loop
+      while !@finished
+        bulk_mvq
+        sleep # time_out
+      end
+    end
+
+
+    def time_out
+      if @size == 0
+        if @q1.size == 1
+          0.25
+        else
+          0.5
+        end
+      else
+        if @q1.size > @size
+          0.5
+        elsif @q1.size > @size/2
+          1.0
+        else
+          3.0
+        end
+      end
+    end
+
+
+    def bulk_mvq
+      a = nil
+      @mutex.synchronize do
+        return if @q1.empty?
+
+        time_now = Time.now
+        interval = time_now-@time_prev
+        return if time_now-@time_prev < time_out
+
+        log_bulk_mvq
+        @time_prev = time_now
+
+        a = @q1
+        @q1 = []
+      end
+
+      where(a)
+
+      @mutex.synchronize do
+        while t = a.shift
+          mvq(t)
+        end
+        @cv.broadcast
+      end
+    end
+
+    def log_bulk_mvq
+      msg = @q1[0..2].map{|t| t.name}.inspect
+      msg.sub!(/]$/,",...") if @q1.size > 3
+      Log.info "-- bulk_mvq interval=%.6fs @size=#{@size} @q1.size=#{@q1.size} @q1=#{msg}"%(Time.now-@time_prev)
+    end
+
+
+    def where(task_list)
+      # implemented in child class
+    end
+
+
+    def mvq(t)
+      stored = false
+      if t.respond_to? :location
+        t.location.each do |h|
+          if q = @q2[h]
+            t.assigned.push(h)
+            q.push(t)
+            stored = true
+          end
+        end
+      end
+      if !stored
+        @q2[@hosts[rand(@hosts.size)]].push(t)
+        # @q2[nil].push(t)
+      end
+      @size += 1
+    end
+
+
+    def enq_impl(item,hint=nil)
+      #Log.debug "--- #{self.class}#enq_impl #{item.inspect}"
+      @q1.push(item)
+    end
+
+    def enq_finish
+      @thread.run if @thread.alive?
+    end
+
+    def deq_impl(host,n)
+      @thread.run if @thread.alive?
+
+      if t = deq_locate(host)
+        Log.info "-- deq_locate n=#{n} task=#{t.name} host=#{host}"
+        return t
+      end
+
+      if @enable_steal && n > 1
+        if t = deq_steal(host)
+          Log.info "-- deq_steal n=#{n} task=#{t.name} host=#{host}"
+          return t
+        end
+      end
+
+      m = 0.05*(2**([n,5].min))
+      @cv.wait(@mutex,m)
+      nil
+    end
+
+
+    def deq_locate(host)
+      q = @q2[host]
+      if q && !q.empty?
+        t = q.shift
+        t.assigned.each{|x| @q2[x].delete_if{|x| t.equal? x}}
+        @size -= 1
+        return t
+      else
+        nil
+      end
+    end
+
+    def deq_steal(host)
+      # select a task based on many and close
+      max_host = nil
+      max_num  = 0
+      @q2.each do |h,a|
+        if !a.empty?
+          d = @throughput.interhost(host,h) * a.size
+          if d > max_num
+            max_host = h
+            max_num  = d
+          end
+        end
+      end
+      if max_host
+        deq_locate(max_host)
+      else
+        deq_locate(nil)
+      end
+    end
+
+    def size
+      @q1.size + @size
+    end
+
+    def clear
+      @q1.clear
+      @hosts.each{|h| @q2[h].clear}
+    end
+
+    def empty?
+      @q1.empty? && @hosts.all?{|h| @q2[h].empty?}
+    end
+
+    def finish
+      super
+      @thread.run if @thread.alive?
+    end
+
+  end
+end

data/lib/pwrake/logger.rb

@@ -0,0 +1,153 @@
+module Pwrake
+
+  LOCK = Mutex.new
+
+  class Logger
+
+    attr_accessor :level
+
+    module Severity
+      # Low-level information, mostly for developers
+      DEBUG = 0
+      INFO = 1
+      WARN = 2
+      ERROR = 3
+      FATAL = 4
+      UNKNOWN = 5
+    end
+    include Severity
+
+    def initialize
+      @level = WARN
+      @out = nil
+      @filename = nil
+      @lock = Mutex.new
+    end
+
+    def open(file)
+      close if @out
+      case file
+      when IO
+        @out = file
+        @filename = nil
+      when String
+        @out = File.open(file,"w")
+        @filename = file
+      else
+        raise "file arg must be IO or String"
+      end
+      @start_time = Time.now
+      info "LogStart=" + fmt_time(@start_time)
+      info "logfile=#{@filename}" if @filename
+    end
+
+    def finish(str, start_time)
+      if @out
+        finish_time = Time.now
+        t1 = Log.fmt_time(start_time)
+        t2 = Log.fmt_time(finish_time)
+        elap = finish_time - start_time
+        info "#{str} : start=#{t1} end=#{t2} elap=#{elap}"
+      end
+    end
+
+    def add(severity, message)
+      if !severity || severity >= @level
+        if @out
+          @lock.synchronize do
+            @out.write(message+"\n")
+          end
+        else
+          LOCK.synchronize do
+            $stderr.write(message+"\n")
+          end
+        end
+      end
+      true
+    end
+    alias log add
+
+    def info(msg)
+      add(INFO, msg)
+    end
+
+    def debug(msg)
+      add(DEBUG, msg)
+    end
+
+    def warn(msg)
+      add(WARN, msg)
+    end
+
+    def fmt_time(t)
+      t.strftime("%Y-%m-%dT%H:%M:%S.%%06d") % t.usec
+    end
+
+    def timer(prefix,*args)
+      Timer.new(prefix,*args)
+    end
+
+    def close
+      finish "LogEnd", @start_time
+      @lock.synchronize do
+        @out.close if @filename
+        @out=nil
+      end
+      @filename=nil
+    end
+
+  end # class Logger
+
+
+  LOGGER = Logger.new
+
+  module Log
+    include Logger::Severity
+
+    module_function
+
+    def open(file)
+      LOGGER.open(file)
+    end
+
+    def close
+      LOGGER.close
+    end
+
+    def info(s)
+      LOGGER.info(s)
+    end
+
+    def debug(s)
+      LOGGER.debug(s)
+    end
+
+    def warn(s)
+      LOGGER.warn(s)
+    end
+
+    def level
+      LOGGER.level
+    end
+
+    def level=(x)
+      LOGGER.level = x
+    end
+
+    def fmt_time(t)
+      t.strftime("%Y-%m-%dT%H:%M:%S.%%06d") % t.usec
+    end
+
+    def timer(prefix,*args)
+      Timer.new(prefix,*args)
+    end
+
+    def output_message(message)
+      LOCK.synchronize do
+        $stderr.write(message+"\n")
+      end
+    end
+
+  end
+
+end # module Pwrake

data/lib/pwrake/master.rb

@@ -0,0 +1,109 @@
+module Pwrake
+
+  def current_shell
+    Thread.current[:shell]
+  end
+
+  def current_shell=(a)
+    Thread.current[:shell] = a
+  end
+
+  module_function :current_shell, :current_shell=
+
+
+  class Master
+    include Pwrake::Option
+
+    attr_reader :task_queue
+    attr_reader :shell_set
+
+    def initialize
+    end
+
+    def init
+      init_option   # Pwrake::Option
+    end
+
+    def setup
+      setup_option   # Pwrake::Option
+    end
+
+    def start
+      @counter = Counter.new
+      @task_queue = @queue_class.new(@core_list)
+      @task_queue.enable_steal = !Rake.application.options.disable_steal
+      @shell_set = []
+      @core_list.each_with_index do |h,i|
+        @shell_set << @shell_class.new(h,@shell_opt)
+      end
+      start_threads
+    end
+
+    def finish
+      Log.debug "-- Master#finish called"
+      @task_queue.finish if @task_queue
+      @threads.each{|t| t.join }
+      @counter.print
+      finish_option   # Pwrake::Option
+    end
+
+    def start_threads
+      Thread.abort_on_exception = true
+      @threads = []
+      @shell_set.each do |c|
+        @threads << Thread.new(c) do |conn|
+          Pwrake.current_shell = conn
+          conn.start
+          begin
+            thread_loop(conn)
+          ensure
+            Log.info "-- worker[#{conn.id}] ensure : closing #{conn.host}"
+            conn.finish
+          end
+        end
+      end
+    end
+
+    def thread_loop(conn,last=nil)
+      @task_queue.reserve(last) if last
+      hint = (conn) ? conn.host : nil
+      standard_exception_handling do
+        while t = @task_queue.deq(hint)
+          Log.debug "-- Master#thread_loop deq t=#{t.inspect}"
+          t.pw_invoke
+          return if t == last
+        end
+      end
+    end
+
+    # Provide standard execption handling for the given block.
+    def standard_exception_handling
+      begin
+        yield
+      rescue SystemExit => ex
+        # Exit silently with current status
+        @task_queue.stop
+        raise
+      rescue OptionParser::InvalidOption => ex
+        # Exit silently
+        @task_queue.stop
+        exit(false)
+      rescue Exception => ex
+        # Exit with error message
+        name = "pwrake"
+        $stderr.puts "#{name} aborted!"
+        $stderr.puts ex.message
+        if Rake.application.options.trace
+          $stderr.puts ex.backtrace.join("\n")
+        else
+          $stderr.puts ex.backtrace.find {|str| str =~ /#{@rakefile}/ } || ""
+          $stderr.puts "(See full trace by running task with --trace)"
+        end
+        @task_queue.stop
+        exit(false)
+      end
+    end
+
+  end
+
+end # module Pwrake