pwrake 2.1.3 → 2.2.0

data/lib/pwrake/option/host_map.rb

@@ -1,7 +1,16 @@
+require "socket"
+
 module Pwrake
 
   class HostInfo
 
+    @@local_ip = nil
+
+    def self.local_ip
+      @@local_ip ||=
+        Socket.getifaddrs.select{|a| a.addr.ip?}.map{|a| a.addr.ip_address}
+    end
+
     def initialize(name,id,ncore,weight,group=nil)
       @name = name
       @ncore = ncore
@@ -11,17 +20,28 @@ module Pwrake
       @continuous_fail = 0
       @total_fail = 0
       @count_task = 0
+      @ipaddr = []
     end
 
     attr_reader :name, :ncore, :weight, :group, :id, :steal_flag
+    attr_reader :ipaddr
     attr_accessor :idle_cores
 
+    def local?
+      ipa = IPSocket.getaddress(@name)
+      HostInfo.local_ip.include?(ipa)
+    end
+
     def set_ncore(n)
       @retire = 0
       @busy_cores = 0
       @ncore = @idle_cores = n
     end
 
+    def set_ip(ipa)
+      @ipaddr.push(ipa)
+    end
+
     def idle(n)
       @busy_cores -= n
       @idle_cores += n
@@ -70,11 +90,17 @@ module Pwrake
 
   class HostMap < Hash
 
+    def self.ipmatch_for_name(name)
+      @@hostmap.ipmatch_for_name(name)
+    end
+
     def initialize(arg=nil)
       @host_map = {}
       @by_id = []
       @by_name = {}
-      require "socket"
+      @is_local = false
+      @ipmatch_for_name = {}
+      @@hostmap = self
       case arg
       when /\.yaml$/
         read_yaml(arg)
@@ -86,12 +112,21 @@ module Pwrake
         parse_hosts(["localhost 1"])
       else
         raise ArgumentError, "arg=#{arg.inspect}"
-        #@num_threads = 1 if !@num_threads
-        #@core_list = ['localhost'] * @num_threads
+      end
+
+      # local check
+      if @by_id.size == 1
+        if @by_id[0].local?
+          @is_local = true
+        end
       end
     end
     attr_reader :by_id, :by_name
 
+    def local?
+      @is_local
+    end
+
     def host_count
       @by_id.size
     end
@@ -120,6 +155,18 @@ module Pwrake
       a
     end
 
+    def ipmatch_for_name(node)
+      unless a = @ipmatch_for_name[node]
+        @ipmatch_for_name[node] = a = []
+        ip = IPSocket.getaddress(node)
+        @by_id.each_with_index do |h,id|
+          a << id if h.ipaddr.include?(ip)
+        end
+        Log.debug "node:#{node} hosts:#{a.map{|id|@by_id[id].name}.inspect}"
+      end
+      a
+    end
+
     private
 
     def read_host(file)
@@ -137,7 +184,6 @@ module Pwrake
     end
 
     def parse_hosts(hosts)
-      #p hosts
       if hosts.kind_of? Array
         hosts = {"localhost"=>hosts}
       end
@@ -166,11 +212,6 @@ module Pwrake
           hosts = [host]
         end
         hosts.each do |host|
-          begin
-            host = Socket.gethostbyname(host)[0]
-          rescue
-            Log.warn "FQDN not resoved : #{host}"
-          end
           ncore  &&= ncore.to_i
           weitht &&= weight.to_i
           #weight = (weight || 1).to_f

data/lib/pwrake/option/option.rb

@@ -1,5 +1,6 @@
 require "pathname"
 require "yaml"
+require "parallel"
 require "pwrake/option/host_map"
 
 module Pwrake
@@ -10,6 +11,7 @@ module Pwrake
 
     def initialize
       load_pwrake_conf
+      init_filesystem
       init_options
       init_pass_env
       if self['SHOW_CONF']
@@ -21,24 +23,11 @@ module Pwrake
         Report.new(self,[]).report_html
         exit
       end
-    end
-
-    def init
-      Log.info "Options:"
-      self.each do |k,v|
-	Log.info " #{k} = #{v.inspect}"
-      end
-      #@counter = Counter.new
       setup_hosts
-      setup_filesystem # require 'option_filesystem.rb'
-      #
-      if self['LOG_DIR'] && self['GC_LOG_FILE']
-        GC::Profiler.enable
-      end
+      set_filesystem_option
     end
 
     attr_reader :counter
-    attr_reader :logger
     attr_accessor :total_cores
 
     DEFAULT_CONFFILES = ["pwrake_conf.yaml","PwrakeConf.yaml"]
@@ -59,12 +48,52 @@ module Pwrake
       if pwrake_conf.nil?
         @yaml = {}
       else
-        #Log.debug "load pwrake_conf=#{pwrake_conf}"
         require "yaml"
         @yaml = open(pwrake_conf){|f| YAML.load(f) }
       end
     end
 
+    # ----------------------------------------------------------
+
+    def init_filesystem
+      @filesystem = Rake.application.options.filesystem
+      @filesystem ||= mount_type.sub(/fuse\./,"")
+      begin
+        require "pwrake/option/option_#{@filesystem}"
+      rescue LoadError
+        require "pwrake/option/option_default_filesystem"
+      end
+    end
+    attr_reader :worker_progs
+    attr_reader :worker_option
+    attr_reader :queue_class
+
+    def mount_type(d=nil)
+      mtab = '/etc/mtab'
+      if File.exist?(mtab)
+        d ||= mountpoint_of_cwd
+        open(mtab,'r') do |f|
+          f.each_line do |l|
+            a = l.split
+            if a[1] == d
+              return a[2]
+            end
+          end
+        end
+      end
+      nil
+    end
+
+    def mountpoint_of_cwd
+      d = Pathname.pwd
+      while !d.mountpoint?
+        d = d.parent
+      end
+      d.to_s
+    end
+
+    # ----------------------------------------------------------
+
     def init_options
       option_data.each do |a|
         prc = nil
@@ -109,14 +138,8 @@ module Pwrake
         'TRACE_OUTPUT',
         'TRACE_RULES',
 
-        'FILESYSTEM',
         'SSH_OPTION',
         'PASS_ENV',
-        ['SHELL_COMMAND', proc{|v| v||ENV['SHELL']}],
-        ['SHELL_RC','SHELLRC'],
-        'GFARM2FS_OPTION',
-        'GFARM2FS_DEBUG',
-        ['GFARM2FS_DEBUG_WAIT', proc{|v| v ? v.to_i : 1}],
         'GNU_TIME',
         'DEBUG',
         'PLOT_PARALLELISM',
@@ -127,7 +150,6 @@ module Pwrake
         'FAILURE_TERMINATION', # wait, kill, continue
         'QUEUE_PRIORITY', # RANK(default), FIFO, LIFO, DFS
         'NOACTION_QUEUE_PRIORITY', # FIFO(default), LIFO, RAND
-        #'NUM_NOACTION_THREADS', # default=4 when gfarm, else 1
         'GRAPH_PARTITION',
         'PLOT_PARTITION',
 
@@ -185,18 +207,12 @@ module Pwrake
         ['SHELL_START_INTERVAL', proc{|v| (v || 0.012).to_f}],
         ['HEARTBEAT', proc{|v| (v || 240).to_i}],
         ['RETRY', proc{|v| (v || 1).to_i}],
-        ['DISABLE_AFFINITY', proc{|v| v || ENV['AFFINITY']=='off'}],
-        ['DISABLE_STEAL', proc{|v| v || ENV['STEAL']=='off'}],
-        ['GFARM_BASEDIR', proc{|v| v || '/tmp'}],
-        ['GFARM_PREFIX', proc{|v| v || "pwrake_#{ENV['USER']}"}],
-        ['GFARM_SUBDIR', proc{|v| v || '/'}],
-        ['MAX_GFWHERE_WORKER', proc{|v| (v || 8).to_i}],
         ['MASTER_HOSTNAME', proc{|v| (v || begin;`hostname -f`;rescue;end || '').chomp}],
         ['WORK_DIR', proc{|v|
            v ||= '%CWD_RELATIVE_TO_HOME'
-           v.sub('%CWD_RELATIVE_TO_HOME',cwd_relative_to_home)
+           v.sub('%CWD_RELATIVE_TO_HOME',cwd_relative_if_under_home)
          }],
-      ]
+      ].concat(option_data_filesystem)
     end
 
     def format_time_pid(v)
@@ -281,7 +297,7 @@ module Pwrake
       return pwd.relative_path_from(home).to_s
     end
 
-    # ------------------------------------------------------------------------
+    # ----------------------------------------------------------
 
     def init_pass_env
       if envs = self['PASS_ENV']
@@ -314,6 +330,7 @@ module Pwrake
       end
     end
 
+    # ----------------------------------------------------------
 
     def setup_hosts
       if @hostfile && @num_threads
@@ -323,44 +340,16 @@ module Pwrake
     end
     attr_reader :host_map
 
+    # ----------------------------------------------------------
 
-    def clear_gfarm2fs
-      setup_hosts
-      d = File.join(self['GFARM_BASEDIR'],self['GFARM_PREFIX'])
-      rcmd = "
-for i in #{d}*; do
-  if [ -d \"$i\" ]; then
-    case \"$i\" in
-      *_000) ;;
-      *) fusermount -u $i; rmdir $i ;;
-    esac
-  fi
-done
-sleep 1
-for i in #{d}*_000; do
-  if [ -d \"$i\" ]; then
-    fusermount -u $i; rmdir $i
-  fi
-done
-"
-      threads = []
-      @host_map.each do |k,hosts|
-        hosts.each do |info|
-          threads << Thread.new do
-            system "ssh #{info.name} '#{rcmd}'"
-          end
-        end
+    def put_log
+      Log.info "Options:"
+      self.each do |k,v|
+        Log.info " #{k} = #{v.inspect}"
       end
-      threads.each{|t| t.join}
-    end
-
-    # ----- finish -----
-
-    def finish_option
-      Log.close
+      Log.debug "@queue_class=#{@queue_class}"
+      Log.debug "@filesystem=#{@filesystem}"
     end
 
   end
 end
-
-require "pwrake/option/option_filesystem"

data/lib/pwrake/option/option_default_filesystem.rb

@@ -0,0 +1,48 @@
+require "pwrake/master/postprocess"
+
+module Pwrake
+
+  module DefaultFileSystemOption
+
+    def option_data_filesystem
+      []
+    end
+
+    def set_filesystem_option
+      @worker_progs = %w[
+        parallel/processor_count.rb
+        pwrake/nbio
+        pwrake/branch/fiber_queue
+        pwrake/worker/writer
+        pwrake/worker/log_executor
+        pwrake/worker/executor
+        pwrake/worker/invoker
+        pwrake/worker/shared_directory
+        pwrake/worker/worker_main
+      ]
+      @worker_option = {
+        :base_dir  => "",
+        :work_dir  => self['WORK_DIR'],
+        :log_dir   => self['LOG_DIR'],
+        :pass_env  => self['PASS_ENV'],
+        :ssh_option => self['SSH_OPTION'],
+        :heartbeat => self['HEARTBEAT'],
+        :shared_directory => "SharedDirectory"
+      }
+      @filesystem  = "default"
+      @queue_class = "TaskQueue"
+    end
+
+    def max_postprocess_pool
+      1
+    end
+
+    def postprocess(runner)
+      Postprocess.new(runner)
+    end
+  end
+
+  class Option
+    include DefaultFileSystemOption
+  end
+end

data/lib/pwrake/option/option_gfarm.rb

@@ -0,0 +1 @@
+lib/pwrake/option/option_gfarm2fs.rb

data/lib/pwrake/option/option_gfarm2fs.rb

@@ -0,0 +1,101 @@
+require "pwrake/queue/locality_aware_queue"
+require "pwrake/gfarm/gfarm_path"
+require "pwrake/gfarm/gfarm_postprocess"
+
+module Pwrake
+
+  module GfarmFileSystemOption
+
+    def option_data_filesystem
+      [
+        'GFARM2FS_OPTION',
+        'GFARM2FS_DEBUG',
+        ['GFARM2FS_DEBUG_WAIT', proc{|v| v ? v.to_i : 1}],
+        ['DISABLE_AFFINITY', proc{|v| v || ENV['AFFINITY']=='off'}],
+        ['DISABLE_STEAL', proc{|v| v || ENV['STEAL']=='off'}],
+        ['GFARM_BASEDIR', proc{|v| v || '/tmp'}],
+        ['GFARM_PREFIX', proc{|v| v || "pwrake_#{ENV['USER']}"}],
+        ['GFARM_SUBDIR', proc{|v| v || '/'}],
+        ['MAX_GFWHERE_WORKER', proc{|v| (v || 8).to_i}],
+      ]
+    end
+
+    def set_filesystem_option
+      @filesystem  = 'gfarm'
+      GfarmPath.subdir = self['GFARM_SUBDIR']
+      @worker_option = {
+        :log_dir   => self['LOG_DIR'],
+        :pass_env  => self['PASS_ENV'],
+        :ssh_option => self['SSH_OPTION'],
+        :heartbeat => self['HEARTBEAT'],
+        #
+        :shared_directory => "GfarmDirectory",
+        :base_dir => self['GFARM_BASEDIR']+"/"+self['GFARM_PREFIX'],
+        :work_dir => GfarmPath.pwd.to_s,
+        :gfarm2fs_option => self['GFARM2FS_OPTION'],
+        :gfarm2fs_debug => self['GFARM2FS_DEBUG'],
+        :gfarm2fs_debug_wait => self['GFARM2FS_DEBUG_WAIT'],
+        :single_mp => self['GFARM_SINGLE_MP']
+      }
+      @worker_progs = %w[
+        parallel/processor_count.rb
+        pwrake/nbio
+        pwrake/branch/fiber_queue
+        pwrake/worker/writer
+        pwrake/worker/log_executor
+        pwrake/worker/executor
+        pwrake/worker/invoker
+        pwrake/worker/shared_directory
+        pwrake/worker/gfarm_directory
+        pwrake/worker/worker_main
+      ]
+      if self['DISABLE_AFFINITY']
+        @queue_class = "TaskQueue"
+      else
+        @queue_class = "LocalityAwareQueue"
+      end
+    end
+
+    def max_postprocess_pool
+      self['MAX_GFWHERE_WORKER']
+    end
+
+    def postprocess(runner)
+      GfarmPostprocess.new(runner)
+    end
+
+    def clear_gfarm2fs
+      setup_hosts
+      d = File.join(self['GFARM_BASEDIR'],self['GFARM_PREFIX'])
+      rcmd = "
+for i in #{d}*; do
+  if [ -d \"$i\" ]; then
+    case \"$i\" in
+      *_000) ;;
+      *) fusermount -u $i; rmdir $i ;;
+    esac
+  fi
+done
+sleep 1
+for i in #{d}*_000; do
+  if [ -d \"$i\" ]; then
+    fusermount -u $i; rmdir $i
+  fi
+done
+"
+      threads = []
+      @host_map.each do |k,hosts|
+        hosts.each do |info|
+          threads << Thread.new do
+            system "ssh #{info.name} '#{rcmd}'"
+          end
+        end
+      end
+      threads.each{|t| t.join}
+    end
+  end
+
+  class Option
+    include GfarmFileSystemOption
+  end
+end