pwrake 0.9.5 → 0.9.6

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 6779de017cf5e05367aecc0f5fef0f12bfd4c54d
+  data.tar.gz: f9aa0b823cf6d147ae595c86643859ebb283992c
+SHA512:
+  metadata.gz: df46550084799df1303f17a4dafbe5ee7ed2e2589a165bb2c340f37d4972b37a195da8d3f03922a79cca1c21f9ddb794a4042fa085437820d4093a24cfccd8d3
+  data.tar.gz: 992425a4aef9202594598b66c18f42dbd3d7d06a50f77315fa4252f72e0e6c9e483965f28ad1c166c3475956758d2e17bbbd2700d7837423682e862a0f73e008

data/README.md

@@ -3,6 +3,9 @@
 Parallel workflow extension for Rake
 * Author: Masahiro Tanaka
 
+([日本語README](https://github.com/masa16/pwrake/wiki/Pwrake.ja)),
+([GitHub Repository](https://github.com/masa16/pwrake))
+
 ## Features
 
 * Parallelize all tasks; no need to modify Rakefile, no need to use `multitask`.
@@ -47,11 +50,87 @@ Or, gem install:
 
         $ pwrake --hostfile=hosts
 
+## Options
+
+### Command line option
+
+        -F, --hostfile FILE              [Pw] Read hostnames from FILE
+        -j, --jobs [N]                   [Pw] Number of threads at localhost (default: # of processors)
+        -L, --logfile [FILE]             [Pw] Write log to FILE
+            --ssh-opt, --ssh-option OPTION
+                                         [Pw] Option passed to SSH
+            --filesystem FILESYSTEM      [Pw] Specify FILESYSTEM (nfs|gfarm)
+            --gfarm                      [Pw] FILESYSTEM=gfarm
+        -A, --disable-affinity           [Pw] Turn OFF affinity (AFFINITY=off)
+        -S, --disable-steal              [Pw] Turn OFF task steal
+        -d, --debug                      [Pw] Output Debug messages
+            --pwrake-conf [FILE]         [Pw] Pwrake configuation file in YAML
+            --show-conf, --show-config   [Pw] Show Pwrake configuration options
+            --report LOG                 [Pw] Report profile HTML from LOG and exit.
+
+### pwrake_conf.yaml
+
+* If `pwrake_conf.yaml` exists at current directory, Pwrake reads options from it.
+* Example (in YAML form):
+
+        HOSTFILE : hosts
+        LOGFILE : true
+        TASKLOG : true
+        PROFILE : true
+        GNU_TIME : true
+        PLOT_PARALLELISM : true
+        DISABLE_AFFINITY: true
+        DISABLE_STEAL: true
+        FAILED_TARGET : delete
+        PASS_ENV :
+         - ENV1
+         - ENV2
+
+* Option list:
+
+        HOSTFILE, HOSTS   default=false
+        LOGFILE, LOG      default=none, string=filename, true="Pwrake%Y%m%d-%H%M%S_%$.log"
+        TASKLOG           default=none, string=filename, true="Pwrake%Y%m%d-%H%M%S_%$.task"
+        PROFILE           default=none, string=filename, true="Pwrake%Y%m%d-%H%M%S_%$.csv"
+        WORK_DIR          default=$PWD
+        FILESYSTEM        default=nil (autodetect)
+        SSH_OPTION        (String) SSH option
+        PASS_ENV          (Array) Environment variables passed to SSH
+        GNU_TIME          If true, obtains PROFILEs using GNU time
+        PLOT_PARALLELISM  If true, plot parallelism using GNUPLOT
+        FAILED_TARGET     ( rename(default) | delete | leave ) failed files
+        QUEUE_PRIORITY    ( DFS(default) | FIFO )
+
+  for Gfarm system:
+
+        DISABLE_AFFINITY  default=false
+        DISABLE_STEAL     default=false
+        STEAL_WAIT        default=0 (sec)
+        STEAL_WAIT_MAX    default=10 (sec)
+         : Wait min(STEAL_WAIT*2**n, STEAL_WAIT_MAX) sec for task steal.
+        GFARM_BASEDIR     default="/tmp"
+        GFARM_PREFIX      default="pwrake_$USER"
+        GFARM_SUBDIR      default='/'
+
+## Note for Gfarm
+
+* `gfwhere-pipe` command is required for file-affinity scheduling.
+
+        wget https://gist.github.com/masa16/5787473/raw/6df5deeb80a4cea6b9d1d1ce01f390f65d650717/gfwhere-pipe.patch
+        cd gfarm-2.5.8.1
+        patch -p1 < ../gfwhere-pipe.patch
+        ./configure --prefix=...
+        make
+        make install
+
 ## Tested Platform
 
-* Ruby 1.9.3, 2.0.0-preview1
-* Rake 0.9.2.2
-* Fedora 16 / Debian 5.0.7
+* Ruby 2.0.0
+* Rake 0.9.6
+* CentOS 6.4
 
-## Workflow demo
+## Acknowledgment
 
+This work is supported by
+* JST CREST, research area: "Development of System Software Technologies for Post-Peta Scale High Performance Computing," and
+* MEXT Promotion of Research for Next Generation IT Infrastructure "Resources Linkage for e-Science (RENKEI)."

data/lib/pwrake/application.rb

@@ -17,10 +17,24 @@ module Pwrake
 
   class Application < ::Rake::Application
 
+    def filesystem
+      @master.filesystem
+    end
+
     def task_queue
       @master.task_queue
     end
 
+    def finish_queue
+      @master.finish_queue
+    end
+
+    def postprocess(t)
+      if @master.postprocess
+        @master.postprocess.postprocess(t)
+      end
+    end
+
     def thread_loop(*args)
       @master.thread_loop(*args)
     end
@@ -33,6 +47,18 @@ module Pwrake
       @master.start
     end
 
+    def core_list
+      @master.core_list
+    end
+
+    def task_logger
+      @master.task_logger
+    end
+
+    def task_id_counter
+      @master.task_id_counter
+    end
+
     # Run the Pwrake application.
     def run
       standard_exception_handling do
@@ -144,7 +170,15 @@ module Pwrake
        ['--show-conf','--show-config',
         "[Pw] Show Pwrake configuration options",
         lambda {|value| options.show_conf = true }
+       ],
+       ['--report LOG', "[Pw] Report profile HTML from LOG and exit.",
+         lambda { |value|
+           require 'pwrake/report'
+           Report.new(File.basename(value.sub(/\.[^.]+$/,"")),[]).report_html
+           exit
+         }
        ]
+
       ])
       opts
     end

data/lib/pwrake/file_utils.rb

@@ -1,6 +1,6 @@
+module Pwrake
 module FileUtils
-
-  alias sh_orig :sh
+  module_function
 
   def sh(*cmd, &block)
     options = (Hash === cmd.last) ? cmd.pop : {}
@@ -17,14 +17,38 @@ module FileUtils
       options[:verbose] ||= RakeFileUtils.verbose_flag
     end
     options[:noop]    ||= RakeFileUtils.nowrite_flag
-    rake_check_options options, :noop, :verbose
-    Pwrake::Log.output_message cmd.join(" ") if options[:verbose]
+    Rake.rake_check_options options, :noop, :verbose
+    Pwrake::Log.stderr_puts cmd.join(" ") if options[:verbose]
     unless options[:noop]
-      res,status = pwrake_system(*cmd)
+      res,status = Pwrake::FileUtils.pwrake_system(*cmd)
       block.call(res, status)
     end
   end
 
+  def bq(*cmd, &block)
+    options = (Hash === cmd.last) ? cmd.pop : {}
+    unless block_given?
+      show_command = cmd.join(" ")
+      show_command = show_command[0,42] + "..."
+      block = lambda { |ok, status|
+        ok or fail "Command failed with status (#{status.exitstatus}): [#{show_command}]"
+      }
+    end
+    if RakeFileUtils.verbose_flag == :default
+      options[:verbose] = true
+    else
+      options[:verbose] ||= RakeFileUtils.verbose_flag
+    end
+    options[:noop]    ||= RakeFileUtils.nowrite_flag
+    Rake.rake_check_options options, :noop, :verbose
+    Pwrake::Log.stderr_puts cmd.join(" ") if options[:verbose]
+    unless options[:noop]
+      res,status = Pwrake::FileUtils.pwrake_backquote(*cmd)
+      block.call(res, status)
+    end
+    res
+  end
+
   def pwrake_system(*cmd)
     cmd_log = cmd.join(" ").inspect
     tm = Pwrake::Timer.new("sh",cmd_log)
@@ -42,8 +66,6 @@ module FileUtils
     tm.finish("status=%s cmd=%s"%[status.exitstatus,cmd_log])
     [res,status]
   end
-  private :pwrake_system
-
 
   # Pwrake version of backquote command
   def pwrake_backquote(cmd)
@@ -53,18 +75,23 @@ module FileUtils
     conn = Pwrake.current_shell
     if conn.kind_of?(Pwrake::Shell)
       res    = conn.backquote(*cmd)
-      status = conn.status
+      status = Rake::PseudoStatus.new(conn.status)
     else
       res    = `#{cmd}`
-      if !res && status.nil?
-        status = 1
-      else
-        status = $?.exitstatus
-      end
+      status = $?
+      status = Rake::PseudoStatus.new(1) if status.nil?
     end
 
-    tm.finish("status=%s cmd=%s"%[status,cmd_log])
-    res
+    tm.finish("status=%s cmd=%s"%[status.exitstatus,cmd_log])
+    [res,status]
   end
 
-end # module FileUtils
+end # module Pwrake::FileUtils
+end
+
+module Rake
+  module DSL
+    include Pwrake::FileUtils
+    private(*Pwrake::FileUtils.instance_methods(false))
+  end
+end

data/lib/pwrake/gfarm_feature.rb

@@ -87,7 +87,20 @@ module Pwrake
       x.to_s
     end
 
+    def gfpath(file='.')
+      begin
+	IO.popen("gfstat '#{file}'") do |f|
+	  if /File: "([^"]+)"/ =~ f.gets #"
+	    return $1
+	  end
+	end
+      rescue
+      end
+      nil
+    end
+
     def gfwhere(list)
+      system "sync"
       result = {}
       count = 0
       cmd = "gfwhere"
@@ -95,7 +108,7 @@ module Pwrake
         if count==1
           result[cmd[8..-1]] = x.split
         else
-          x.scan(/^([^\n]+):\n([^\n]*)$/m) do |file,hosts|
+          x.scan(/^(?:gfarm:\/\/[^\/]+)?([^\n]+):\n([^\n]*)$/m) do |file,hosts|
             h = hosts.split
             result[file] = h if !h.empty?
           end
@@ -106,7 +119,7 @@ module Pwrake
         if a
           path = local_to_fs(a)
           if cmd.size + path.size + 1 > 20480 # 131000
-            x = `#{cmd}`
+            x = `#{cmd} 2> /dev/null`
             parse_proc.call(x)
             cmd = "gfwhere"
             count = 0
@@ -117,7 +130,7 @@ module Pwrake
         end
       end
       if count > 0
-        x = `#{cmd}`
+        x = `#{cmd} 2> /dev/null`
         parse_proc.call(x)
       end
       result
@@ -153,16 +166,22 @@ module Pwrake
       open(system_cmd)
       cd
       if not _system "test -d #{@remote_mountpoint}"
-        _system "mkdir -p #{@remote_mountpoint}"
-        subdir = GfarmPath.subdir
-        if ["/","",nil].include?(subdir)
-          _system "gfarm2fs #{@remote_mountpoint}"
-        else
-          _system "gfarm2fs -o modules=subdir,subdir=#{subdir} #{@remote_mountpoint}"
+        _system "mkdir -p #{@remote_mountpoint}" or die
+      else
+        lines = _backquote("sync; mount")
+        if /#{@remote_mountpoint} (?:type )?(\S+)/om =~ lines
+          _system "sync; fusermount -u #{@remote_mountpoint}"
+          _system "sync"
         end
       end
+      subdir = GfarmPath.subdir
+      if ["/","",nil].include?(subdir)
+        _system "gfarm2fs #{@remote_mountpoint}"
+      else
+        _system "gfarm2fs -o modules=subdir,subdir=#{subdir} #{@remote_mountpoint}"
+      end
       path = ENV['PATH'].gsub( /#{GfarmPath.mountpoint}/, @remote_mountpoint )
-      _system "export PATH=#{path}"
+      _system "export PATH=#{path}" or die
       cd_work_dir
     end
 
@@ -185,45 +204,65 @@ module Pwrake
   end
 
 
-  class GfarmQueue < LocalityAwareQueue
+  class GfarmPostprocess
 
-    def abr_msg(a)
-      m = a[0..5].map{|x| x}.inspect
-      m.sub!(/]$/,",...") if a.size > 6
-      "size=#{a.size} #{m}"
+    def initialize
+      @lock = Mutex.new
+      @io = IO.popen('gfwhere-pipe','r+')
+      @io.sync = true
+    end
+
+    def gfwhere(file)
+      return [] if file==''
+      @lock.synchronize do
+        @io.puts(file)
+        @io.flush
+        s = @io.gets
+        if s.nil?
+          raise "gfwhere: unexpected end"
+        end
+        s.chomp!
+        if s != file
+          raise "gfwhere: file=#{file}, result=#{s}"
+        end
+        while s = @io.gets
+          s.chomp!
+          case s
+          when ""
+            next
+          when /^gfarm:\/\//
+            next
+          when /^Error:/
+            return []
+          else
+            return s.split(/\s+/)
+          end
+        end
+      end
     end
 
-    def where(tasks)
-      if Pwrake.application.options.dryrun ||
-          Pwrake.application.options.disable_affinity
-        return tasks
+    def postprocess(t)
+      if t.kind_of? Rake::FileTask
+        t.location = gfwhere(t.name)
       end
+    end
 
-      start_time = Time.now
-      #Log.debug "--- GfarmQueue#where #{tasks.inspect}"
-      #if Pwrake.manager.gfarm and Pwrake.manager.affinity
-      gfwhere_result = {}
-      filenames = []
+    def postprocess_bulk(tasks)
+      list = []
       tasks.each do |t|
-        if t.kind_of?(Rake::FileTask) and
-            name = t.prerequisites[0] and
-            !filenames.include?(name)
-          filenames << name
-        end
+       list << t.name if t.kind_of? Rake::FileTask
       end
-
-      if !filenames.empty?
-        gfwhere_result = GfarmPath.gfwhere(filenames)
-        tasks.each do |t|
-          if t.kind_of? Rake::FileTask and prereq_name = t.prerequisites[0]
-            t.location = gfwhere_result[GfarmPath.local_to_fs(prereq_name)]
-          end
-        end
+      if !list.empty?
+       Log.info "-- after_check: size=#{list.size} #{list.inspect}"
+       gfwhere_result = GfarmPath.gfwhere(list)
+       tasks.each do |t|
+         if t.kind_of? Rake::FileTask
+           t.location = gfwhere_result[GfarmPath.local_to_fs(t.name)]
+         end
+       end
+       #puts "'#{self.name}' exist? => #{File.exist?(self.name)} loc => #{loc}"
       end
-      Log.info "-- GfarmQueue#where %.6fs %s" % [Time.now-start_time,abr_msg(filenames)]
-      tasks
     end
 
   end
-
 end