pwrake 0.9.8 → 0.9.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d50d7fd5c00627f0eea3d59e26632c4e9e4c0f6d
-  data.tar.gz: 3d9d6638a9dde0ce9feebc1ba210d9ce1d94b310
+  metadata.gz: cdc080259b5fb3f1b2c487375c56d2c319c4ad4b
+  data.tar.gz: 8f28b6ff5df2e0160abf7c259808d85632c5db27
 SHA512:
-  metadata.gz: c7517d29d640a06db61bc8a656615d43706564c49596d594830cadbbe0134b79bfe71c542e182de003baff492b58008bcb90e60b97aab587de1460f21cd99ce6
-  data.tar.gz: 99f101b28c359ea3157caf688205da6734bb95569dd1eea6fed5cc6787e6b1f9a7bd01a35ab1e7dd694e2782c0969480f4abea1d18bc68d963bc85658d9432c6
+  metadata.gz: ba5588c3bf612d56321bba8e36d331f2a4404b2815d3765b6b0baf7a483462498e852f31780147c38631c3c11b46ba9583849faeaec00d3792bbff29f9e332fb
+  data.tar.gz: b6dd586b0982cfedd38ea166c9f7621fce13f90af61ad2f0ae334c659b9bd70be014d63c7ed9719c259ae5ca613940ab338fd627c11218fb5f2be3ba5510d08e

data/lib/pwrake/locality_aware_queue.rb

@@ -6,8 +6,78 @@ module Pwrake
     end
   end
 
+  # copy from ruby's thread.rb
+  class ConditionVariable
+    #
+    # Creates a new ConditionVariable
+    #
+    def initialize
+      @waiters = {}
+      @waiters_mutex = Mutex.new
+    end
+
+    #
+    # Releases the lock held in +mutex+ and waits; reacquires the lock on wakeup.
+    #
+    # If +timeout+ is given, this method returns after +timeout+ seconds passed,
+    # even if no other thread doesn't signal.
+    #
+    def wait(mutex, timeout=nil)
+      Thread.handle_interrupt(StandardError => :never) do
+        begin
+          Thread.handle_interrupt(StandardError => :on_blocking) do
+            @waiters_mutex.synchronize do
+              @waiters[Thread.current] = true
+            end
+            mutex.sleep timeout
+          end
+        ensure
+          @waiters_mutex.synchronize do
+            @waiters.delete(Thread.current)
+          end
+        end
+      end
+      self
+    end
+
+    #
+    # Wakes up the first thread in line waiting for this lock.
+    #
+    def signal
+      Thread.handle_interrupt(StandardError => :on_blocking) do
+        begin
+          t, _ = @waiters_mutex.synchronize { @waiters.shift }
+          t.run if t
+        rescue ThreadError
+          retry # t was already dead?
+        end
+      end
+      self
+    end
+
+    #
+    # Wakes up all threads waiting for this lock.
+    #
+    def broadcast
+      Thread.handle_interrupt(StandardError => :on_blocking) do
+        threads = nil
+        @waiters_mutex.synchronize do
+          threads = @waiters.keys
+          @waiters.clear
+        end
+        for t in threads
+          begin
+            t.run
+          rescue ThreadError
+          end
+        end
+      end
+      self
+    end
+  end
 
   class LocalityConditionVariable < ConditionVariable
+
     def signal(hints=nil)
       if hints.nil?
         super()

data/lib/pwrake/report/report.rb

@@ -63,6 +63,8 @@ EOL
       @elap = @end_time - @start_time
       read_elap_each_cmd
       make_cmd_stat
+
+      @stat = TaskStat.new(@task_file,@sh_table)
     end
 
     attr_reader :base, :ncore, :elap
@@ -142,9 +144,11 @@ EOL
 
     def tr_count(x,y)
       sum = x+y
+      xp = x*100.0/sum
+      yp = y*100.0/sum
       td = "<td align='right' valign='top'>"
-      m  = td + "%s<br/>%.2f%%</td>" % [format_comma(x),x*100.0/sum]
-      m << td + "%s<br/>%.2f%%</td>" % [format_comma(y),y*100.0/sum]
+      m  = td + '%s<br/>''%.2f%%</td>' % [format_comma(x),xp]
+      m << td + '%s<br/>''%.2f%%</td>' % [format_comma(y),yp]
       m << td + "%s</td>" % format_comma(sum)
       m
     end
@@ -192,7 +196,6 @@ EOL
       html << "<table>\n"
       html << "<img src='./#{File.basename(histogram_plot)}' align='top'/></br>\n"
 
-      task_locality
       html << "<h2>Locality statistics</h2>\n"
       html << "<table>\n"
 
@@ -216,44 +219,25 @@ EOL
       html << "<th>local</th><th>remote</th><th>total</th>"
       html << "<th>local</th><th>remote</th><th>total</th>"
       html << "</tr>\n"
-      n_same_input     = 0
-      size_same_input  = 0
-      n_diff_input     = 0
-      size_diff_input  = 0
-      n_same_output    = 0
-      size_same_output = 0
-      n_diff_output    = 0
-      size_diff_output = 0
-      elap_host = 0
-      @exec_hosts.each do |h|
+      @stat.exec_hosts.each do |h|
         html << "<tr><td>#{h}</td>"
-        html << "<td align='right'>%.3f</td>" % @elap_host[h]
+        html << "<td align='right'>%.3f</td>" % @stat[h,nil,:elap]
         html << "<td></td>"
-        html << tr_count(@n_same_input[h],@n_diff_input[h])
-        html << tr_count(@size_same_input[h],@size_diff_input[h])
+        html << tr_count(@stat[h,true,:in_num],@stat[h,false,:in_num])
+        html << tr_count(@stat[h,true,:in_size],@stat[h,false,:in_size])
         html << "<td></td>"
-        html << tr_count(@n_same_output[h],@n_diff_output[h])
-        html << tr_count(@size_same_output[h],@size_diff_output[h])
-
+        html << tr_count(@stat[h,true,:out_num],@stat[h,false,:out_num])
+        html << tr_count(@stat[h,true,:out_size],@stat[h,false,:out_size])
         html << "</tr>\n"
-        n_same_input     += @n_same_input[h]
-        size_same_input  += @size_same_input[h]
-        n_diff_input     += @n_diff_input[h]
-        size_diff_input  += @size_diff_input[h]
-        n_same_output    += @n_same_output[h]
-        size_same_output += @size_same_output[h]
-        n_diff_output    += @n_diff_output[h]
-        size_diff_output += @size_diff_output[h]
-        elap_host += @elap_host[h]
       end
       html << "<tr><td>total</td>"
-      html << "<td align='right'>%.3f</td>" % elap_host
+      html << "<td align='right'>%.3f</td>" % @stat.total(nil,:elap)
       html << "<td></td>"
-      html << tr_count(n_same_input,n_diff_input)
-      html << tr_count(size_same_input,size_diff_input)
+      html << tr_count(@stat.total(true,:in_num),@stat.total(false,:in_num))
+      html << tr_count(@stat.total(true,:in_size),@stat.total(false,:in_size))
       html << "<td></td>"
-      html << tr_count(n_same_output,n_diff_output)
-      html << tr_count(size_same_output,size_diff_output)
+      html << tr_count(@stat.total(true,:out_num),@stat.total(false,:out_num))
+      html << tr_count(@stat.total(true,:out_size),@stat.total(false,:out_size))
 
       html << "</tr>\n"
       html << "<table>\n"
@@ -262,64 +246,12 @@ EOL
       File.open(@html_file,"w") do |f|
         f.puts html
       end
-      puts "generate "+@html_file
-    end
-
-    def task_locality
-      @task_table = CSV.read(@task_file,:headers=>true)
-      file_size = {}
-      file_host = {}
-      @task_table.each do |row|
-        name = row['task_name']
-        file_size[name] = row['file_size'].to_i
-        file_host[name] = (row['file_host']||'').split('|')
-      end
+      #puts "generate "+@html_file
 
-      @n_same_output   = Hash.new(0)
-      @size_same_output= Hash.new(0)
-      @n_diff_output   = Hash.new(0)
-      @size_diff_output= Hash.new(0)
-      @n_same_input    = Hash.new(0)
-      @size_same_input = Hash.new(0)
-      @n_diff_input    = Hash.new(0)
-      @size_diff_input = Hash.new(0)
-      h = {}
-      @task_table.each do |row|
-        if row['executed']=='1'
-          name = row['task_name']
-          exec_host = row['exec_host']
-          h[exec_host] = true
-          if file_host[name].include?(exec_host)
-            @n_same_output[exec_host] += 1
-            @size_same_output[exec_host] += file_size[name]
-          else
-            @n_diff_output[exec_host] += 1
-            @size_diff_output[exec_host] += file_size[name]
-          end
-          preq_files = (row['preq']||'').split('|')
-          preq_files.each do |preq|
-            if (sz=file_size[preq]) && sz > 0
-              if file_host[preq].include?(exec_host)
-                @n_same_input[exec_host] += 1
-                @size_same_input[exec_host] += sz
-              else
-                @n_diff_input[exec_host] += 1
-                @size_diff_input[exec_host] += sz
-              end
-            end
-          end
-        end
-      end
-      @exec_hosts = h.keys.sort
-
-      @elap_host = Hash.new(0)
-      @sh_table.each do |row|
-        if (h = row['host']) && (t = row['elap_time'])
-          @elap_host[h] += t.to_f
-        end
-      end
+      printf "%s,%d,%d,%d,%d\n",@html_file, @stat.total(true,:in_num),@stat.total(false,:in_num),@stat.total(true,:in_size),@stat.total(false,:in_size)
     end
 
+
     def histogram_plot
       command_list = []
       @cmd_stat.each do |cmd,stat|
@@ -357,7 +289,72 @@ set title 'histogram of elapsed time'"
         hist_image
       end
     end
+  end
+
+  class TaskStat
+
+    def initialize(task_file, sh_table)
+      @task_table = CSV.read(task_file,:headers=>true)
+      @count = Hash.new(0)
+      task_locality
+      stat_sh_table(sh_table)
+    end
+
+    attr_reader :exec_hosts
+
+    def count(exec_host, loc, key, val)
+      @count[[exec_host,loc,key]] += val
+      @count[[loc,key]] += val
+    end
+
+    def total(loc,key)
+      @count[[loc,key]]
+    end
+
+    def [](exec_host,loc,key)
+      @count[[exec_host,loc,key]]
+    end
+
+    def task_locality
+      file_size = {}
+      file_host = {}
+      h = {}
+      @task_table.each do |row|
+        name            = row['task_name']
+        file_size[name] = row['file_size'].to_i
+        file_host[name] = (row['file_host']||'').split('|')
+        h[row['exec_host']] = true
+      end
+      @exec_hosts = h.keys.sort
+
+      @task_table.each do |row|
+        if row['executed']=='1'
+          name      = row['task_name']
+          exec_host = row['exec_host']
+          loc = file_host[name].include?(exec_host)
+          count(exec_host, loc, :out_num, 1)
+          count(exec_host, loc, :out_size, file_size[name])
+
+          preq_files = (row['preq']||'').split('|')
+          preq_files.each do |preq|
+            sz = file_size[preq]
+            if sz && sz > 0
+              loc = file_host[preq].include?(exec_host)
+              count(exec_host, loc, :in_num, 1)
+              count(exec_host, loc, :in_size, sz)
+            end
+          end
+        end
+      end
+    end
 
+    def stat_sh_table(sh_table)
+      sh_table.each do |row|
+        if (h = row['host']) && (t = row['elap_time'])
+          count(h, nil, :elap, t.to_f)
+        end
+      end
+    end
 
   end
 end

data/lib/pwrake/task_queue.rb

@@ -1,5 +1,3 @@
-require "pwrake/cache_aware_queue.rb"
-
 module Pwrake
 
   class TaskConditionVariable < ConditionVariable
@@ -81,6 +79,47 @@ module Pwrake
   end
 
 
+  # LIFO + HRF
+  class LifoHrfQueueArray < Array
+    def initialize(n)
+      @n = (n>0) ? n : 1
+      @count = []
+    end
+
+    def push(t)
+      super(t)
+      r = t.rank
+      @count[r] = (@count[r]||0) + 1
+    end
+
+    def shift
+      if empty?
+        return nil
+      end
+      (@count.size-1).downto(0) do |r|
+        c = @count[r]
+        if c && c>0
+          t = (c<=@n) ? pop_last_rank(r) : pop
+          @count[t.rank] -= 1
+          return t
+        end
+      end
+      nil
+    end
+
+    def pop_last_rank(r)
+      (size-1).downto(0) do |i|
+        t = at(i)
+        if t.rank == r
+          delete_at(i)
+          return t
+        end
+      end
+      nil
+    end
+  end
+
+
   # Rank-Even Last In First Out
   class RankQueueArray
     def initialize(n)
@@ -338,6 +377,8 @@ module Pwrake
         @array_class = FifoQueueArray # Array # Fifo
       when /lifo/i
         @array_class = LifoQueueArray
+      when /lihr/i
+        @array_class = LifoHrfQueueArray
       when /rank/i
         @array_class = RankQueueArray
       else

data/lib/pwrake/version.rb

@@ -1,3 +1,3 @@
 module Pwrake
-  VERSION = "0.9.8"
+  VERSION = "0.9.9"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pwrake
 version: !ruby/object:Gem::Version
-  version: 0.9.8
+  version: 0.9.9
 platform: ruby
 authors:
 - Masahiro TANAKA
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-17 00:00:00.000000000 Z
+date: 2014-02-12 00:00:00.000000000 Z
 dependencies: []
 description: Parallel workflow extension for Rake
 email:
@@ -18,7 +18,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -26,7 +26,6 @@ files:
 - bin/pwrake
 - lib/pwrake.rb
 - lib/pwrake/application.rb
-- lib/pwrake/cache_aware_queue.rb
 - lib/pwrake/counter.rb
 - lib/pwrake/file_utils.rb
 - lib/pwrake/gfarm_feature.rb
@@ -77,17 +76,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.0
 signing_key: 
 specification_version: 4
 summary: Adding Parallel and Distributed feature to Rake
@@ -109,4 +108,3 @@ test_files:
 - spec/helper.rb
 - spec/hosts
 - spec/pwrake_spec.rb
-has_rdoc: 

data/lib/pwrake/cache_aware_queue.rb

@@ -1,184 +0,0 @@
-module Pwrake
-
-  class MTimePriorityArray < Array
-
-    def order_by(item)
-      item.input_file_mtime
-    end
-
-    def push(t)
-      order_value = order_by(t)
-      if empty? || order_by(last) <= order_value
-        super(t)
-      elsif order_by(first) > order_value
-        unshift(t)
-      else
-        lower = 0
-        upper = size-1
-        while lower+1 < upper
-          mid = ((lower + upper) / 2).to_i
-          if order_by(self[mid]) <= order_value
-            lower = mid
-          else
-            upper = mid
-          end
-        end
-        insert(upper,t)
-      end
-    end
-
-    def index(t)
-      if size < 40
-        return super(t)
-      end
-      order_value = t.order_by
-      if order_by(last) < order_value || order_by(first) > order_value
-        nil
-      else
-        lower = 0
-        upper = size-1
-        while lower+1 < upper
-          mid = ((lower + upper) / 2).to_i
-          if order_by(self[mid]) < order_value
-            lower = mid
-          else
-            upper = mid
-          end
-        end
-        mid = upper
-        if order_by(self[mid]) == order_value
-          Log.debug "--- TQA#index=#{mid}, order_value=#{order_value}"
-          mid
-        end
-      end
-    end
-  end
-
-
-  # rank-Even Last In First Out
-  class CacheAwareQueue
-    def initialize(n)
-      @q = []
-      @i = 0
-      @size = 0
-      @n = (n>0) ? n : 1
-    end
-
-    def push(t)
-      r = t.rank
-      a = @q[r]
-      if a.nil?
-        @q[r] = a = MTimePriorityArray.new
-      end
-      @size += 1
-      a.push(t)
-    end
-
-    def size
-      @size
-    end
-
-    def empty?
-      @size == 0
-    end
-
-    def shift
-      if empty?
-        return nil
-      elsif @size < @n*2
-        return shift_high_rank
-      else
-        return shift_weighted
-      end
-    end
-
-    def shift_high_rank
-      (@q.size-1).downto(0) do |i|
-        a = @q[i]
-        next if a.nil? || a.empty?
-        @size -= 1
-        return pop_last_max(a)
-      end
-      nil
-    end
-
-    def shift_weighted
-      weight, weight_avg = RANK_STAT.rank_weight
-      wsum = 0.0
-      q = []
-      @q.each_with_index do |a,i|
-        next if a.nil? || a.empty?
-        w = weight[i]
-        w = weight_avg if w.nil?
-        # w *= a.size
-        wsum += w
-        q << [a,wsum]
-      end
-      #
-      x = rand() * wsum
-      Log.debug "--- shift_weighted x=#{x} wsum=#{wsum} weight=#{weight.inspect}"
-      @size -= 1
-      q.each do |a,w|
-        if w > x
-          return a.pop
-        end
-      end
-      raise "ELIFO: wsum=#{wsum} x=#{x}"
-    end
-
-    def pop_last_max(a)
-      if a.size < 2
-        return a.pop
-      end
-      y_max = nil
-      i_max = nil
-      n = [@n, a.size].min
-      (-n..-1).each do |i|
-        y = a[i].input_file_size
-        if y_max.nil? || y > y_max
-          y_max = y
-          i_max = i
-        end
-      end
-      a.delete_at(i_max)
-    end
-
-    def first
-      return nil if empty?
-      @q.size.times do |i|
-        a = @q[i]
-        unless a.nil? || a.empty?
-          return a.first
-        end
-      end
-    end
-
-    def last
-      return nil if empty?
-      @q.size.times do |i|
-        a = @q[-i-1]
-        unless a.nil? || a.empty?
-          return a.last
-        end
-      end
-    end
-
-    def delete(t)
-      n = 0
-      @q.each do |a|
-        if a
-          a.delete(t)
-          n += a.size
-        end
-      end
-      @size = n
-    end
-
-    def clear
-      @q.clear
-      @i = 0
-      @size = 0
-    end
-  end
-
-end