pwrake 0.9.9.2 → 2.0.0

data/lib/pwrake/master/postprocess.rb

@@ -0,0 +1,16 @@
+module Pwrake
+
+  class Postprocess
+
+    def initialize(runner)
+    end
+
+    def run(filename)
+      []
+    end
+
+    def close
+    end
+
+  end
+end

data/lib/pwrake/{mcgp.rb → misc/mcgp.rb}

@@ -5,25 +5,25 @@ module Pwrake
 
   module MCGP
 
-    def graph_partition(target=nil)
+    def graph_partition(host_map, target=nil)
       t1 = Time.now
-      wgts = Pwrake.application.host_list.group_weight_sum
+      wgts = host_map.group_weight_sum
       if wgts.size > 1
         list = wgts.size.times.to_a
-        g = GraphTracer.new([list],[wgts]){|t| t.group}
+        g = GraphTracerGroup.new([list],[wgts])
         trace(g,target)
-        g.part_graph_group
+        g.part_graph
         #g.write_dot('dag1.dot')
         #return
       end
 
-      #$debug2=true
+      #$debug=true
 
-      list = Pwrake.application.host_list.group_hosts
-      wgts = Pwrake.application.host_list.group_core_weight
-      g = GraphTracer.new(list,wgts){|t| t.location}
+      list = host_map.group_hosts
+      wgts = host_map.group_core_weight
+      g = GraphTracerNode.new(list,wgts)
       trace(g,target)
-      g.part_graph_node
+      g.part_graph
       t2 = Time.now
       Pwrake::Log.info "Time for TOTAL Graph Partitioning: #{t2-t1} sec"
       #g.write_dot('dag2.dot')
@@ -47,14 +47,13 @@ module Pwrake
 
   class GraphTracer
 
-    def initialize(loc_list, weight_list, &block)
+    def initialize(loc_list, weight_list)
       if loc_list.size != weight_list.size
         raise ArgumentError, "array size of args mismatch"
       end
       @loc_list = loc_list
       @weight_list = weight_list
       @n_part = @loc_list.size
-      @location_finder = block
       @traced = {}
       @vertex_depth = {}
       # @grviz = Grviz.new
@@ -63,26 +62,27 @@ module Pwrake
       end
     end
 
-    def trace( name = "default", target = nil )
+    def trace(name="default", target=nil)
 
       task = Rake.application[name]
-      group_id = task.group_id || 0
+      tw = task.wrapper
+      group_id = tw.group_id || 0
       group = @group_list[group_id]
-      loc_list = @loc_list[group_id]
+      #loc_list = @loc_list[group_id]
       depth = 0
 
       if task.class == Rake::FileTask
-        tgid = target ? (Rake.application[target].group_id||0) : nil
+        tgid = (target) ? (Rake.application[target].wrapper.group_id||0) : nil
 
         if File.file?(name)
           if tgid == group_id
-            locs = @location_finder.call(task)
-            if locs.empty?
-              Pwrake.application.postprocess(task)
-              locs = @location_finder.call(task)
-            end
-            task.get_file_stat
-            fsz = task.file_size
+            locs = get_location(tw)
+            #if locs.empty?
+            #  Pwrake.application.postprocess(task)
+            #  locs = get_location(tw)
+            #end
+            #tw.get_file_stat
+            fsz = tw.file_size
             if fsz > 100000
               #puts "g=#{group_id}, task=#{name}, target=#{target}, fsz=#{fsz}, locs="+locs.join("|")
               group.push_loc_edge( locs, name, target, fsz/10000 )
@@ -119,23 +119,39 @@ module Pwrake
       return @vertex_depth[name]
     end
 
-    def part_graph_group
+    def write_dot(file)
+      @grviz.write(file)
+    end
+  end
+
+
+  class GraphTracerGroup < GraphTracer
+
+    def get_location(tw)
+      tw.group
+    end
+
+    def part_graph
       @group_list.each do |g|
         g.part_graph
         g.set_group
       end
     end
+  end
+
+
+  class GraphTracerNode < GraphTracer
 
-    def part_graph_node
+    def get_location(tw)
+      tw.location
+    end
+
+    def part_graph
       @group_list.each do |g|
         g.part_graph
         g.set_node
       end
     end
-
-    def write_dot(file)
-      @grviz.write(file)
-    end
   end
 
 
@@ -296,7 +312,9 @@ module Pwrake
       if false
         puts "@vertex_id2name.size=#{@vertex_id2name.size}"
         if $debug2
-          @vertex_id2name.each_with_index{|x,i| puts "#{i} #{x} #{@edges[i].inspect}"}
+          @vertex_id2name.each_with_index{|x,i|
+            puts "#{i} #{x} #{@edges[i].inspect}"
+          }
         end
         puts "@edges.size=#{@edges.size}"
         puts "ncon=#{c}"
@@ -328,22 +346,25 @@ module Pwrake
         s += "]"
         Log.info s
         options = RbMetis.default_options
-        RbMetis::OPTION_NITER
         options[RbMetis::OPTION_NCUTS] = 30
         options[RbMetis::OPTION_NSEPS] = 30
         options[RbMetis::OPTION_NITER] = 10
-        @part = RbMetis.part_graph_recursive(@xadj, @adjcny, @n_part, ncon:c, vwgt:@vwgt, adjwgt:@adjwgt, tpwgts:tpw, ubvec:ubv, options:options)
+        @part = RbMetis.part_graph_recursive(
+                  @xadj, @adjcny, @n_part,
+                  ncon:c, vwgt:@vwgt, adjwgt:@adjwgt,
+                  tpwgts:tpw, ubvec:ubv, options:options)
       else
         puts "tpw=#{tpw.inspect}"
-        @part = Metis.mc_part_graph_recursive2(c, @xadj,@adjcny, @vwgt,nil, @tpwgts)
+        @part = Metis.mc_part_graph_recursive2(
+                  c,@xadj,@adjcny,@vwgt,nil,@tpwgts)
       end
       t2 = Time.now
       Pwrake::Log.info "Time for Graph Partitioning: #{t2-t1} sec"
       count_partition
-      #puts "Time for Graph Partitioning: #{t2-t1} sec"
-      #if $debug
-      #p @part
-      #end
+      if $debug
+        puts "Time for Graph Partitioning: #{t2-t1} sec"
+        p @part
+      end
     end
 
     def count_partition
@@ -411,15 +432,15 @@ module Pwrake
       @vertex_id2name.each_with_index do |name,idx|
         if idx >= @n_part
           i_part = @part[idx]
-          task = Rake.application[name]
-          task.group_id = loc_list[i_part]
+          tw = Rake.application[name].wrapper
+          tw.group_id = loc_list[i_part]
           #puts "task=#{task.inspect}, i_part=#{i_part}, host=#{host}"
         end
       end
       @loc_files.each do |gid,files|
         files.each do |f|
-          task = Rake.application[f]
-          task.group_id = gid
+          tw = Rake.application[f].wrapper
+          tw.group_id = gid
           # puts "gid=#{gid}, task=#{f}"
         end
       end
@@ -431,9 +452,9 @@ module Pwrake
       @vertex_id2name.each_with_index do |name,idx|
         if idx >= @n_part
           i_part = @part[idx]
-          task = Rake.application[name]
+          tw = Rake.application[name].wrapper
           host = loc_list[i_part]
-          task.suggest_location = [host]
+          tw.suggest_location = [host]
           #puts "task=#{task.inspect}, i_part=#{i_part}, host=#{host}"
         end
       end

data/lib/pwrake/option/host_map.rb

@@ -0,0 +1,158 @@
+module Pwrake
+
+  class HostInfo
+
+    def initialize(name,id,ncore,weight,group=nil)
+      @name = name
+      @ncore = ncore
+      @weight = weight || 1.0
+      @group = group || 0
+      @id = id
+    end
+
+    attr_reader :name, :ncore, :weight, :group, :id, :steal_flag
+    attr_accessor :idle_cores
+
+    def set_ncore(n)
+      @ncore = @idle_cores = n
+    end
+
+    def increase(n)
+      @idle_cores += n
+    end
+
+    def decrease(n)
+      @idle_cores -= n
+      if @idle_cores < 0
+        raise RuntimeError,"# of cores must be non-negative"
+      end
+    end
+
+    def steal_phase
+      @steal_flag = true
+      t = yield(self)
+      @steal_flag = false
+      t
+    end
+  end
+
+  class HostMap < Hash
+
+    def initialize(arg=nil)
+      @host_map = {}
+      @by_id = []
+      @by_name = {}
+      require "socket"
+      case arg
+      when /\.yaml$/
+        read_yaml(arg)
+      when String
+        read_host(arg)
+      when Integer
+        parse_hosts(["localhost #{arg}"])
+      when NilClass
+        parse_hosts(["localhost 1"])
+      else
+        raise ArgumentError, "arg=#{arg.inspect}"
+        #@num_threads = 1 if !@num_threads
+        #@core_list = ['localhost'] * @num_threads
+      end
+    end
+    attr_reader :by_id, :by_name
+
+    def host_count
+      @by_id.size
+    end
+
+    def group_hosts
+      a = []
+      self.each do |sub,list|
+        list.each{|h| (a[h.group] ||= []) << h.name}
+      end
+      a
+    end
+
+    def group_core_weight
+      a = []
+      self.each do |sub,list|
+        list.each{|h| (a[h.group] ||= []) << h.weight}
+      end
+      a
+    end
+
+    def group_weight_sum
+      a = []
+      self.each do |sub,list|
+        list.each{|h| a[h.group] = (a[h.group]||0) + h.weight}
+      end
+      a
+    end
+
+    private
+
+    def read_host(file)
+      ary = []
+      File.open(file) do |f|
+        while l = f.gets
+          ary << l
+        end
+      end
+      parse_hosts(ary)
+    end
+
+    def read_yaml(file)
+      parse_hosts(YAML.load(open(file))[0])
+    end
+
+    def parse_hosts(hosts)
+      #p hosts
+      if hosts.kind_of? Array
+        hosts = {"localhost"=>hosts}
+      end
+      hosts.each do |branch, list|
+        self[branch] = parse_list(list)
+      end
+    end
+
+    def parse_list(line_list)
+      info_list = []
+      line_list.each do |line|
+        parse_line(info_list,line)
+      end
+      info_list
+    end
+
+    def parse_line(info_list,line)
+      line = $1 if /^([^#]*)#/ =~ line
+      host, ncore, weight, group = line.split
+      if host
+        if /\[\[([\w\d]+)-([\w\d]+)\]\]/o =~ host
+          hosts = ($1..$2).map{|i| host.sub(re,i)}
+        else
+          hosts = [host]
+        end
+        hosts.each do |host|
+          begin
+            host = Socket.gethostbyname(host)[0]
+          rescue
+            Log.warn "FQDN not resoved : #{host}"
+          end
+          ncore  &&= ncore.to_i
+          weitht &&= weight.to_i
+          #weight = (weight || 1).to_f
+          group  &&= group.to_i
+          if host_info = @by_name[host]
+            raise RuntimeError,"duplicated hostname: #{host}"
+          else
+            id = @by_id.size
+            host_info = HostInfo.new(host,id,ncore,weight,group)
+            @by_name[host] = host_info
+            info_list << host_info
+            @by_id << host_info
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/pwrake/option/option.rb

@@ -0,0 +1,357 @@
+module Pwrake
+
+  START_TIME = Time.now
+
+  class Option < Hash
+
+    def initialize
+      load_pwrake_conf
+      init_options
+      init_pass_env
+      if self['SHOW_CONF']
+        require "yaml"
+        YAML.dump(self,$stdout)
+        exit
+      elsif self['REPORT_DIR']
+        require 'pwrake/report'
+        Report.new(self,[]).report_html
+        exit
+      end
+    end
+
+    def init
+      Log.info "Options:"
+      self.each do |k,v|
+	Log.info " #{k} = #{v.inspect}"
+      end
+      #@counter = Counter.new
+      setup_hosts
+      setup_filesystem # require 'option_filesystem.rb'
+      #
+      if self['LOG_DIR'] && self['GC_LOG_FILE']
+        GC::Profiler.enable
+      end
+    end
+
+    attr_reader :counter
+    attr_reader :logger
+
+    DEFAULT_CONFFILES = ["pwrake_conf.yaml","PwrakeConf.yaml"]
+
+    # ----- init -----
+
+    def load_pwrake_conf
+      # Read pwrake_conf
+      pwrake_conf = Rake.application.options.pwrake_conf
+      if pwrake_conf
+        if !File.exist?(pwrake_conf)
+          raise "Configuration file not found: #{pwrake_conf}"
+        end
+      else
+        pwrake_conf = DEFAULT_CONFFILES.find{|fn| File.exist?(fn)}
+      end
+      self['PWRAKE_CONF'] = pwrake_conf
+      if pwrake_conf.nil?
+        @yaml = {}
+      else
+        #Log.debug "load pwrake_conf=#{pwrake_conf}"
+        require "yaml"
+        @yaml = open(pwrake_conf){|f| YAML.load(f) }
+      end
+    end
+
+    def init_options
+      option_data.each do |a|
+        prc = nil
+        keys = []
+        case a
+        when String
+          keys << a
+        when Array
+          a.each do |x|
+            case x
+            when String
+              keys << x
+            when Proc
+              prc = x
+            end
+          end
+        end
+        key = keys[0]
+        val = search_opts(keys)
+        val = prc.call(val) if prc
+        self[key] = val if !val.nil?
+        instance_variable_set("@"+key.downcase, val)
+      end
+
+      feedback_options
+
+      Rake.verbose(false) if Rake.application.options.silent
+    end
+
+    def option_data
+      [
+        'DRYRUN',
+        'IGNORE_SYSTEM',
+        'IGNORE_DEPRECATE',
+        'LOAD_SYSTEM',
+        'NOSEARCH',
+        'RAKELIB',
+        'SHOW_PREREQS',
+        'SILENT',
+        'TRACE',
+        'BACKTRACE',
+        'TRACE_OUTPUT',
+        'TRACE_RULES',
+
+        'FILESYSTEM',
+        'SSH_OPTION',
+        'PASS_ENV',
+        ['SHELL_COMMAND', proc{|v| v||ENV['SHELL']}],
+        ['SHELL_RC','SHELLRC'],
+        'GFARM2FS_OPTION',
+        'GFARM2FS_DEBUG',
+        ['GFARM2FS_DEBUG_WAIT', proc{|v| v ? v.to_i : 1}],
+        'GNU_TIME',
+        'DEBUG',
+        'PLOT_PARALLELISM',
+        'SHOW_CONF',
+        ['REPORT_DIR','REPORT'],
+        'FAILED_TARGET', # rename(default), delete, leave
+        'FAILURE_TERMINATION', # wait, kill, continue
+        'QUEUE_PRIORITY', # RANK(default), FIFO, LIFO, DFS
+        'NOACTION_QUEUE_PRIORITY', # FIFO(default), LIFO, RAND
+        #'NUM_NOACTION_THREADS', # default=4 when gfarm, else 1
+        'GRAPH_PARTITION',
+        'PLOT_PARTITION',
+
+        ['HOSTFILE','HOSTS'],
+        ['LOG_DIR','LOG',
+          proc{|v|
+            if v
+              if v == "" || !v.kind_of?(String)
+                v = "Pwrake%Y%m%d-%H%M%S"
+              end
+              d = v = format_time_pid(v)
+              i = 1
+              while File.exist?(d)
+                d = "#{v}.#{i}"
+                i += 1
+              end
+              d
+            end
+          }],
+        ['LOG_FILE',
+          proc{|v|
+            if v.kind_of?(String) && v != ""
+              v
+            else
+              "pwrake.log"
+            end
+          }],
+        ['TASK_CSV_FILE',
+          proc{|v|
+            if v.kind_of?(String) && v != ""
+              v
+            else
+              "task.csv"
+            end
+          }],
+        ['COMMAND_CSV_FILE',
+          proc{|v|
+            if v.kind_of?(String) && v != ""
+              v
+            else
+              "command.csv"
+            end
+          }],
+        ['GC_LOG_FILE',
+         proc{|v|
+           if v
+             if v.kind_of?(String) && v != ""
+               v
+             else
+               "gc.log"
+             end
+           end
+         }],
+        ['NUM_THREADS', proc{|v| v && v.to_i}],
+        ['SHELL_START_INTERVAL', proc{|v| (v || 0.012).to_f}],
+        ['HEARTBEAT', proc{|v| (v || 240).to_i}],
+        ['DISABLE_AFFINITY', proc{|v| v || ENV['AFFINITY']=='off'}],
+        ['DISABLE_STEAL', proc{|v| v || ENV['STEAL']=='off'}],
+        ['GFARM_BASEDIR', proc{|v| v || '/tmp'}],
+        ['GFARM_PREFIX', proc{|v| v || "pwrake_#{ENV['USER']}"}],
+        ['GFARM_SUBDIR', proc{|v| v || '/'}],
+        ['MAX_GFWHERE_WORKER', proc{|v| (v || 8).to_i}],
+        ['MASTER_HOSTNAME', proc{|v| (v || begin;`hostname -f`;rescue;end || '').chomp}],
+        ['WORK_DIR', proc{|v|
+           v ||= '%CWD_RELATIVE_TO_HOME'
+           v.sub('%CWD_RELATIVE_TO_HOME',cwd_relative_to_home)
+         }],
+      ]
+    end
+
+    def format_time_pid(v)
+      START_TIME.strftime(v).sub("%$","%05d"%Process.pid)
+    end
+
+    def feedback_options
+      opts = Rake.application.options
+      ['DRYRUN',
+       'IGNORE_SYSTEM',
+       'IGNORE_DEPRECATE',
+       'LOAD_SYSTEM',
+       'NOSEARCH',
+       'RAKELIB',
+       'SHOW_PREREQS',
+       'SILENT',
+       'TRACE',
+       'BACKTRACE',
+       'TRACE_OUTPUT',
+       'TRACE_RULES'
+      ].each do |k|
+        if v=self[k]
+          m = (k.downcase+"=").to_sym
+          opts.send(m,v)
+        end
+      end
+      case opts.trace_output
+      when 'stdout'
+        opts.trace_output = $stdout
+      when 'stderr', nil
+        opts.trace_output = $stderr
+      end
+    end
+
+    # Priority of Option:
+    #  command_option > ENV > pwrake_conf > DEFAULT_OPTIONS
+    def search_opts(keys)
+      val = Rake.application.options.send(keys[0].downcase.to_sym)
+      return parse_opt(val) if !val.nil?
+      #
+      keys.each do |k|
+        val = ENV[k.upcase]
+        return parse_opt(val) if !val.nil?
+      end
+      #
+      return nil if !@yaml
+      keys.each do |k|
+        val = @yaml[k.upcase]
+        return val if !val.nil?
+      end
+      nil
+    end
+
+    def parse_opt(s)
+      case s
+      when /^(false|nil|off)$/i
+        false
+      when /^(true|on)$/i
+        true
+      when $stdout
+        "stdout"
+      when $stderr
+        "stderr"
+      else
+        s
+      end
+    end
+
+    def cwd_relative_to_home
+      Pathname.pwd.relative_path_from(Pathname.new(ENV['HOME'])).to_s
+    end
+
+    def cwd_relative_if_under_home
+      home = Pathname.new(ENV['HOME']).realpath
+      path = pwd = Pathname.pwd.realpath
+      while path != home
+        if path.root?
+          return pwd.to_s
+        end
+        path = path.parent
+      end
+      return pwd.relative_path_from(home).to_s
+    end
+
+    # ------------------------------------------------------------------------
+
+    def init_pass_env
+      if envs = self['PASS_ENV']
+        pass_env = {}
+
+        case envs
+        when Array
+          envs.each do |k|
+            k = k.to_s
+            if v = ENV[k]
+              pass_env[k] = v
+            end
+          end
+        when Hash
+          envs.each do |k,v|
+            k = k.to_s
+            if v = ENV[k] || v
+              pass_env[k] = v
+            end
+          end
+        else
+          raise "invalid option for PASS_ENV in pwrake_conf.yaml"
+        end
+
+        if pass_env.empty?
+          self.delete('PASS_ENV')
+        else
+          self['PASS_ENV'] = pass_env
+        end
+      end
+    end
+
+
+    def setup_hosts
+      if @hostfile && @num_threads
+        raise "Cannot set `hostfile' and `num_threads' simultaneously"
+      end
+      @host_map = HostMap.new(@hostfile || @num_threads)
+    end
+    attr_reader :host_map
+
+
+    def clear_gfarm2fs
+      setup_hosts
+      d = File.join(self['GFARM_BASEDIR'],self['GFARM_PREFIX'])
+      rcmd = "
+for i in #{d}*; do
+  if [ -d \"$i\" ]; then
+    case \"$i\" in
+      *_000) ;;
+      *) fusermount -u $i; rmdir $i ;;
+    esac
+  fi
+done
+sleep 1
+for i in #{d}*_000; do
+  if [ -d \"$i\" ]; then
+    fusermount -u $i; rmdir $i
+  fi
+done
+"
+      threads = []
+      @host_map.each do |k,hosts|
+        hosts.each do |info|
+          threads << Thread.new do
+            system "ssh #{info.name} '#{rcmd}'"
+          end
+        end
+      end
+      threads.each{|t| t.join}
+    end
+
+    # ----- finish -----
+
+    def finish_option
+      Log.close
+    end
+
+  end
+end