pwrake 0.9.9 → 0.9.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cdc080259b5fb3f1b2c487375c56d2c319c4ad4b
-  data.tar.gz: 8f28b6ff5df2e0160abf7c259808d85632c5db27
+  metadata.gz: b5f302ea2a00a59015b371e07ddf10f646f06eaa
+  data.tar.gz: 57d72fefb63e1a2b8270adf3b479aaba0add62f4
 SHA512:
-  metadata.gz: ba5588c3bf612d56321bba8e36d331f2a4404b2815d3765b6b0baf7a483462498e852f31780147c38631c3c11b46ba9583849faeaec00d3792bbff29f9e332fb
-  data.tar.gz: b6dd586b0982cfedd38ea166c9f7621fce13f90af61ad2f0ae334c659b9bd70be014d63c7ed9719c259ae5ca613940ab338fd627c11218fb5f2be3ba5510d08e
+  metadata.gz: 4990bd96c5a5bc7e268d26efbe07e0cb297b6a629f63f350b3d595e8b27c0831569fdece5a4070991ca7c7017b9914e934c5ed6a0ca444cc5ca3ef937721d040
+  data.tar.gz: 5b89d6aad26a1f802b1e4626aba67815f9914d3549dbb154864eb927558a58e8a16afcdf06025ffafe3724ae99666b8e4ef4db2bf62a3de57cf7f0678d50d7bb

data/README.md

@@ -3,7 +3,7 @@
 Parallel workflow extension for Rake
 * Author: Masahiro Tanaka
 
-([日本語README](https://github.com/masa16/pwrake/wiki/Pwrake.ja)),
+([README in Japanese](https://github.com/masa16/pwrake/wiki/Pwrake.ja)),
 ([GitHub Repository](https://github.com/masa16/pwrake))
 
 ## Features
@@ -99,7 +99,12 @@ Or, gem install:
         GNU_TIME          If true, obtains PROFILEs using GNU time
         PLOT_PARALLELISM  If true, plot parallelism using GNUPLOT
         FAILED_TARGET     ( rename(default) | delete | leave ) failed files
-        QUEUE_PRIORITY    ( DFS(default) | FIFO )
+        QUEUE_PRIORITY          RANK(default), FIFO, LIFO, LIHR
+        NOACTION_QUEUE_PRIORITY FIFO(default), LIFO, RAND
+        NUM_NOACTION_THREADS    default=4 when gfarm, else 1
+        THREAD_CREATE_INTERVAL  default=0.01 (sec)
+        HALT_QUEUE_WHILE_SEARCH true|false
+        GRAPH_PARTITION         true|false
 
   for Gfarm system:
 
@@ -111,22 +116,29 @@ Or, gem install:
         GFARM_BASEDIR     default="/tmp"
         GFARM_PREFIX      default="pwrake_$USER"
         GFARM_SUBDIR      default='/'
+        MAX_GFWHERE_WORKER  default=8
 
 ## Note for Gfarm
 
-* `gfwhere-pipe` command is required for file-affinity scheduling.
+* `gfwhere-pipe` script (included in Pwrake) is used for file-affinity scheduling.
+  This script requires Ruby/FFI (https://github.com/ffi/ffi). Install FFI by
 
-        wget https://gist.github.com/masa16/5787473/raw/6df5deeb80a4cea6b9d1d1ce01f390f65d650717/gfwhere-pipe.patch
-        cd gfarm-2.5.8.1
-        patch -p1 < ../gfwhere-pipe.patch
-        ./configure --prefix=...
-        make
-        make install
+        gem install ffi
+
+## For Graph Partitioning
+
+* Compile and Install METIS 5.1.0 (http://www.cs.umn.edu/~metis/). This requires CMake.
+
+* Install RbMetis (https://github.com/masa16/rbmetis) by
+
+        gem install rbmetis -- \
+         --with-metis-include=/usr/local/include \
+         --with-metis-lib=/usr/local/lib
 
 ## Tested Platform
 
-* Ruby 2.0.0
-* Rake 0.9.6
+* Ruby 2.1.4
+* Rake 10.1.0
 * CentOS 6.4
 
 ## Acknowledgment

data/bin/gfwhere-pipe

@@ -0,0 +1,159 @@
+#! /usr/bin/env ruby
+
+require 'ffi'
+require 'singleton'
+
+module Gfarm
+  class GfarmError < StandardError
+  end
+  GFARM_ERR_NO_ERROR = 0
+
+  module FFI
+    def find_executable(name)
+      path = "/usr/local/bin:/usr/ucb:/usr/bin:/bin"
+      begin
+        $stderr = open(File::NULL,"w")
+        path = ENV['PATH']
+      ensure
+        $stderr = STDERR
+      end
+      path = path.split(File::PATH_SEPARATOR)
+      path.each do |dir|
+        file = File.join(dir, name)
+        begin
+          stat = File.stat(file)
+        rescue SystemCallError
+      else
+          return file if stat.file? and stat.executable?
+        end
+      end
+      nil
+    end
+    module_function :find_executable
+
+    LIBGFARM_PATH = nil
+    LIBGFARM_FILE = "libgfarm.so"
+    if LIBGFARM_PATH
+      path = File.join(LIBGFARM_PATH,LIBGFARM_FILE)
+    elsif x = find_executable('gfwhere')
+      d = File.dirname(x)
+      %w[lib64 lib].each do |l|
+        f = File.join(d,"..",l,LIBGFARM_FILE)
+        if File.file?(f)
+          path = f
+          break
+        end
+      end
+    end
+    extend ::FFI::Library
+    ffi_lib path
+    attach_function :gfarm_initialize, [:pointer, :pointer], :int
+    attach_function :gfarm_terminate, [], :int
+    attach_function :gfarm_realpath_by_gfarm2fs, [:string, :pointer], :int
+    attach_function :gfarm_error_string, [:int], :string
+    attach_function :gfs_replica_info_by_name, [:string, :int, :pointer], :int
+    attach_function :gfs_replica_info_number, [:pointer], :int
+    attach_function :gfs_replica_info_free, [:pointer], :void
+    attach_function :gfs_replica_info_nth_host, [:pointer, :int], :string
+  end
+
+
+  def connection(*args)
+    Connection.set_args(args)
+    Connection.instance
+  end
+  module_function :connection
+
+
+  class Connection
+    include Singleton
+
+    def self.callback
+      proc{ FFI.gfarm_terminate }
+    end
+
+    def self.set_args(args)
+      @@args = args
+    end
+
+    def initialize
+      args = @@args || []
+      argc = ::FFI::MemoryPointer.new(:int, 1)
+      argc.write_int(args.size)
+      ary = args.map do |s|
+        str = ::FFI::MemoryPointer.new(:string, s.size)
+        str.write_string(s)
+        str
+      end
+      ptr = ::FFI::MemoryPointer.new(:pointer, args.size)
+      ptr.write_array_of_pointer(ary)
+      argv = ::FFI::MemoryPointer.new(:pointer, 1)
+      argv.write_pointer(ptr)
+      e = FFI.gfarm_initialize(argc, argv)
+      if e != GFARM_ERR_NO_ERROR
+        raise GfarmError, FFI.gfarm_error_string(e)
+      end
+      ObjectSpace.define_finalizer(self, self.class.callback)
+    end
+
+    def realpath_by_gfarm2fs(path)
+      ptr = ::FFI::MemoryPointer.new(:pointer, 1)
+      e = FFI.gfarm_realpath_by_gfarm2fs(path, ptr)
+      if e != GFARM_ERR_NO_ERROR
+        raise GfarmError, FFI.gfarm_error_string(e)
+      end
+      ptr.read_pointer().read_string()
+    end
+
+    def replica_info_by_name(name)
+      ReplicaInfo.new(self,name)
+    end
+  end
+
+
+  class ReplicaInfo < ::FFI::AutoPointer
+
+    def self.release(ptr)
+      FFI.gfs_replica_info_free(ptr)
+    end
+
+    def initialize(gfarm, path)
+      @gfarm = gfarm
+      @realpath = @gfarm.realpath_by_gfarm2fs(path)
+      flag = 0
+      ptr = ::FFI::MemoryPointer.new(:pointer, 1)
+      e = FFI.gfs_replica_info_by_name(@realpath, flag, ptr)
+      if e != GFARM_ERR_NO_ERROR
+        raise GfarmError, @realpath+" "+FFI.gfarm_error_string(e)
+      end
+      @ri = ptr.read_pointer()
+      super @ri
+    end
+    attr_reader :realpath
+
+    def number
+      FFI.gfs_replica_info_number(@ri)
+    end
+
+    def nth_host(i)
+      FFI.gfs_replica_info_nth_host(@ri,i)
+    end
+  end
+
+end
+
+gfarm = Gfarm.connection
+
+while path=$stdin.gets
+  path.chomp!
+  $stdout.print path+"\n"
+  $stdout.flush
+  begin
+    ri = gfarm.replica_info_by_name(path)
+    hosts = ri.number.times.map{|i| ri.nth_host(i) }
+    $stdout.print ri.realpath+":\n"+hosts.join(" ")+"\n"
+  rescue
+    $stdout.print "Error: "+path+"\n"
+  end
+  $stdout.flush
+end

data/lib/pwrake.rb

@@ -11,6 +11,7 @@ require "pwrake/profiler"
 require "pwrake/shell"
 require "pwrake/task_algorithm"
 require "pwrake/task_queue"
+require "pwrake/host_list"
 require "pwrake/option"
 require "pwrake/master"
 require "pwrake/application"

data/lib/pwrake/application.rb

@@ -47,8 +47,8 @@ module Pwrake
       @master.start
     end
 
-    def core_list
-      @master.core_list
+    def host_list
+      @master.host_list
     end
 
     def task_logger
@@ -67,7 +67,11 @@ module Pwrake
         @master.init
         load_rakefile
         begin
-          top_level
+          begin
+            top_level
+          rescue
+            puts $!.message
+          end
         ensure
           @master.finish
         end
@@ -189,6 +193,12 @@ module Pwrake
       @master.counter.count( host_list, host )
     end
 
+=begin
+    def host_weigts
+      @master.counter.host_weights
+    end
+=end
+
     # from Michael Grosser's parallel
     # https://github.com/grosser/parallel
     def processor_count

data/lib/pwrake/gfarm_feature.rb

@@ -214,7 +214,7 @@ module Pwrake
     end
 
     def postprocess(t)
-      if t.kind_of? Rake::FileTask
+      if t.kind_of?(Rake::FileTask) && t.location.empty?
         t.location = @gfwhere_pool.work(t.name)
       end
     end

data/lib/pwrake/gfwhere_pool.rb

@@ -23,7 +23,6 @@ module Pwrake
             return w if w.acquire
           end
           # wait for end of work in @pool
-          print "wait\n"
           @cond_pool.wait(@mutex)
         end
       end
@@ -95,8 +94,6 @@ module Pwrake
       while s = @io.gets
         s.chomp!
         case s
-        when ""
-          next
         when /^gfarm:\/\//
           next
         when /^Error:/

data/lib/pwrake/host_list.rb

@@ -0,0 +1,88 @@
+module Pwrake
+
+  class HostInfo
+    def initalize(name,group=0,weight=1)
+      @name = name
+      @group = group
+      @weight = weight
+    end
+    attr_reader :name, :group, :weight
+  end
+
+  class HostList
+    attr_reader :group_hosts
+    attr_reader :group_core_weight
+    attr_reader :group_weight_sum
+    attr_reader :host2group
+    attr_reader :num_threads
+    attr_reader :core_list
+    attr_reader :host_count
+
+    def initialize(file=nil)
+      @file = file
+      @group_hosts = []
+      @group_core_weight = []
+      @group_weight_sum = []
+      @host2group = {}
+      require "socket"
+      if @file
+        read_host(@file)
+        @num_threads = @core_list.size
+      else
+        @num_threads = 1 if !@num_threads
+        @core_list = ['localhost'] * @num_threads
+      end
+    end
+
+    def size
+      @num_threads
+    end
+
+    def read_host(file)
+      tmplist = []
+      File.open(file) do |f|
+        re = /\[\[([\w\d]+)-([\w\d]+)\]\]/o
+        while l = f.gets
+          l = $1 if /^([^#]*)#/ =~ l
+          host, ncore, weight, group = l.split
+          if host
+            if re =~ host
+              hosts = ($1..$2).map{|i| host.sub(re,i)}
+            else
+              hosts = [host]
+            end
+            hosts.each do |host|
+              begin
+                host = Socket.gethostbyname(host)[0]
+              rescue
+                Log.info "-- FQDN not resoved : #{host}"
+              end
+              ncore = (ncore || 1).to_i
+              weight = (weight || 1).to_f
+              w = ncore * weight
+              group = (group || 0).to_i
+              tmplist << ([host] * ncore.to_i)
+              (@group_hosts[group] ||= []) << host
+              (@group_core_weight[group] ||= []) << w
+              @group_weight_sum[group] = (@group_weight_sum[group]||0) + w
+              @host2group[host] = group
+            end
+          end
+        end
+      end
+
+      @core_list = []
+      begin # alternative order
+        sz = 0
+        tmplist.each do |a|
+          @core_list << a.shift if !a.empty?
+          sz += a.size
+        end
+      end while sz>0
+
+      @host_count = Hash.new{0}
+      core_list.each{|h| @host_count[h] += 1}
+    end
+
+  end
+end

data/lib/pwrake/locality_aware_queue.rb

@@ -108,6 +108,36 @@ module Pwrake
       self
     end
 
+
+    def signal_with_hints(hints)
+      if !Array===hints
+        raise ArgumentError,"argument must be an Array"
+      end
+      thread =
+        @waiters_mutex.synchronize do
+        th = nil
+        @waiters.each do |t,v|
+          Log.debug "--- LCV#signal_with_hints: t[:hint]=#{t[:hint]}"
+          if hints.include?(t[:hint])
+            th = t
+            break
+          end
+        end
+        Log.debug "--- LCV#signal_with_hints: hints=#{hints.inspect} thread_to_run=#{th.inspect} @waiters.size=#{@waiters.size}"
+        if th
+          @waiters.delete(th)
+        end
+        th
+      end
+      begin
+        thread.run if thread
+      rescue ThreadError
+        retry # t was already dead?
+      end
+      thread
+    end
+
+
     def broadcast(hints=nil)
       if hints.nil?
         super()
@@ -201,15 +231,22 @@ module Pwrake
     end # class Throughput
 
 
-    def init_queue(core_list)
-      @host_count = Hash.new{0}
-      core_list.each{|h| @host_count[h] += 1}
-      @hosts = @host_count.keys
+    def init_queue(host_list)
+      @host_list = host_list
       @cv = LocalityConditionVariable.new
-      @throughput = Throughput.new
       @size = 0
       @q = {}
-      @host_count.each{|h,n| @q[h] = @array_class.new(n)}
+      @host_list.host_count.each{|h,n| @q[h] = @array_class.new(n)}
+      @q_group = {}
+      @host_list.group_hosts.each do |g|
+        other = @host_list.host_count.dup
+        q1 = {}
+        g.each{|h| q1[h] = @q[h]; other.delete(h)}
+        q2 = {}
+        other.each{|h,v| q2[h] = @q[h]}
+        a = [q1,q2]
+        g.each{|h| @q_group[h] = a}
+      end
       @q_remote = @array_class.new(0)
       @q_later = Array.new
       @enable_steal = !Pwrake.application.pwrake_options['DISABLE_STEAL']
@@ -224,7 +261,7 @@ module Pwrake
 
 
     def enq_impl(t)
-      hints = t.suggest_location
+      hints = t && t.suggest_location
       if hints.nil? || hints.empty?
         @q_later.push(t)
       else
@@ -247,35 +284,44 @@ module Pwrake
 
     def deq_impl(host,n)
       if t = deq_locate(host)
-        Log.info "-- deq_locate n=#{n} task=#{t.name} host=#{host}"
+        Log.info "-- deq_locate n=#{n} task=#{t&&t.name} host=#{host}"
         Log.debug "--- deq_impl\n#{inspect_q}"
         return t
       end
 
+      #hints = []
+      #@q.each do |h,q|
+      #  hints << h if !q.empty?
+      #end
+      #if (!hints.empty?) && @cv.signal_with_hints(hints)
+      #  return nil
+      #end
+
       if !@q_remote.empty?
         t = @q_remote.shift
-        Log.info "-- deq_remote n=#{n} task=#{t.name} host=#{host}"
+        Log.info "-- deq_remote n=#{n} task=#{t&&t.name} host=#{host}"
         Log.debug "--- deq_impl\n#{inspect_q}"
         return t
       end
 
       if !@q_later.empty?
         t = @q_later.shift
-        Log.info "-- deq_later n=#{n} task=#{t.name} host=#{host}"
+        Log.info "-- deq_later n=#{n} task=#{t&&t.name} host=#{host}"
         Log.debug "--- deq_impl\n#{inspect_q}"
         return t
       end
 
       if @enable_steal && n > 0 && Time.now-@last_enq_time > @steal_wait_after_enq
         if t = deq_steal(host)
-          Log.info "-- deq_steal n=#{n} task=#{t.name} host=#{host}"
+          Log.info "-- deq_steal n=#{n} task=#{t&&t.name} host=#{host}"
           Log.debug "--- deq_impl\n#{inspect_q}"
           return t
         end
       end
 
-      m = [@steal_wait*(2**n), @steal_wait_max].min
-      @cv.wait(@mutex,m)
+      #m = [@steal_wait*(2**n), @steal_wait_max].min
+      #@cv.wait(@mutex,m)
+      @cv.wait(@mutex)
       nil
     end
 
@@ -284,8 +330,10 @@ module Pwrake
       q = @q[host]
       if q && !q.empty?
         t = q.shift
-        t.assigned.each do |h|
-          @q[h].delete(t)
+        if t
+          t.assigned.each do |h|
+            @q[h].delete(t)
+          end
         end
         @size -= 1
         return t
@@ -298,17 +346,23 @@ module Pwrake
       # select a task based on many and close
       max_host = nil
       max_num  = 0
-      @q.each do |h,a|
-        if !a.empty?
-          d = a.size
-          if d > max_num
-            max_host = h
-            max_num  = d
+      @q_group[host].each do |qg|
+        qg.each do |h,a|
+          if !a.empty?
+            d = a.size
+            if d > max_num
+              max_host = h
+              max_num  = d
+            end
           end
         end
+        if max_num > 0
+          Log.info "-- deq_steal max_host=#{max_host} max_num=#{max_num}"
+          t = deq_locate(max_host)
+          return t if t
+        end
       end
-      Log.info "-- deq_steal max_host=#{max_host} max_num=#{max_num}"
-      deq_locate(max_host)
+      nil
     end
 
     def inspect_q