pwrake 2.0.1 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d3d87023233f03e030697675ddd3869f47f1784a
-  data.tar.gz: 984fc85baa604dfbd0092e833f193e74c56b5a71
+  metadata.gz: 91439940b04a74b462713ec03d5f5bd4f282965f
+  data.tar.gz: 06a86ba801618144354c31957a73430a3cb00725
 SHA512:
-  metadata.gz: de7d46e56c7487ad87b70b47c5f6d60b0d8231c16b0f86f40aa3437f4618c949986cb14293b05af853a72573f9f110be6ce665905027673b3ad7c2e38a3e911e
-  data.tar.gz: 767e0e3e4a88dfc64224c4358023bde7761084265783386f981b78d60ecd3c573438929a0e426443be1f21b70ba4db5988c5bd9f198586004d3ac4ca8efcdde4
+  metadata.gz: 8b9ce53904fa158730c7b0a58e68b70fc18e6df5bd73b2114b04064376bce3cfd02de3b36ac7996a077a969e0790dd6ce5c9006344307950fef9fe42e2255512
+  data.tar.gz: a8e8ba98341678f4a9a4744236a31a7b6769287fc6dc5554a1408d960d7b915c1397e88f8d957e0b39f9331d73df2a6dc50696bd4972c048257c543d214e9cc4

data/README.md

@@ -18,20 +18,31 @@ Parallel Workflow extension for Rake, runs on multicores, clusters, clouds.
   * Pwrake automatically connects to remote hosts using SSH. You do not need to start a daemon.
   * Remote host names and the number of cores to use are provided in a hostfile.
 * [Gfarm file system](http://sourceforge.net/projects/gfarm/) utilizes storage of compute nodes. It provides the high-performance parallel I/O.
-  * Parallel I/O access to local stroage of compute nodes enables scalable increase in the I/O performance.
+  * Parallel I/O access to local storage of compute nodes enables scalable increase in the I/O performance.
   * Gfarm schedules a compute node to store an output file, to local storage.
   * Pwrake schedules a compute node to execute a task, to a node where input files are stored.
   * Other supports for Gfarm: Automatic mount of the Gfarm file system, etc.
 
 ## Installation
 
-Download source tgz/zip and expand, cd to subdirectory and install:
+Install with RubyGems:
+
+    $ gem install pwrake
+
+Or download source tgz/zip and expand, cd to subdirectory and install:
 
     $ ruby setup.rb
 
-Or, gem install:
+In the latter case, you need install [Parallel](https://github.com/grosser/parallel) manually. It is required by Pwrake for processor count.
+
+If you use rbenv, your system may fail to find pwrake command after installation:
+
+    -bash: pwrake: command not found
+
+In this case, you need the rehash of command paths:
+
+    $ rbenv rehash
 
-    $ gem install pwrake
 
 ## Usage
 
@@ -73,7 +84,7 @@ Or, gem install:
     -A, --disable-affinity           [Pw] Turn OFF affinity (AFFINITY=off)
     -S, --disable-steal              [Pw] Turn OFF task steal
     -d, --debug                      [Pw] Output Debug messages
-        --pwrake-conf [FILE]         [Pw] Pwrake configuation file in YAML
+        --pwrake-conf [FILE]         [Pw] Pwrake configuration file in YAML
         --show-conf, --show-config   [Pw] Show Pwrake configuration options
         --report LOGDIR              [Pw] Report workflow statistics from LOGDIR to HTML and exit.
         --clear-gfarm2fs             [Pw] Clear gfarm2fs mountpoints left after failure.
@@ -106,8 +117,8 @@ Or, gem install:
         SHELL_COMMAND     default=$SHELL
         SHELL_RC          Run-Command when shell starts
         PASS_ENV          (Array) Environment variables passed to SSH
-        HEARTBEAT         defulat=240 - Hearbeat interval in seconds
-        RETRY             default=0 - The number of default task retry
+        HEARTBEAT         default=240 - Hearbeat interval in seconds
+        RETRY             default=1 - The number of retry
         FAILED_TARGET     rename(default)|delete|leave - Treatment of failed target files
         FAILURE_TERMINATION wait(default)|kill|continue - Behavior of other tasks when a task is failed
         QUEUE_PRIORITY          LIHR(default)|FIFO|LIFO|RANK
@@ -186,6 +197,8 @@ Properties (The leftmost item is default):
 
 ## Acknowledgment
 
-This work is supported by
-* JST CREST, research area: "Development of System Software Technologies for Post-Peta Scale High Performance Computing," and
+This work is supported by:
+* JST CREST, research themes:
+  * ["Statistical Computational Cosmology with Big Astronomical Imaging Data,"](http://www.jst.go.jp/kisoken/crest/en/project/44/14532369.html)
+  * ["System Software for Post Petascale Data Intensive Science,"](http://postpeta.jst.go.jp/en/researchers/tatebe22.html)
 * MEXT Promotion of Research for Next Generation IT Infrastructure "Resources Linkage for e-Science (RENKEI)."

data/bin/gfwhere-pipe

@@ -65,13 +65,6 @@ module Gfarm
   end
 
 
-  def connection(*args)
-    Connection.set_args(args)
-    Connection.instance
-  end
-  module_function :connection
-
-
   class Connection
     include Singleton
 
@@ -124,10 +117,14 @@ module Gfarm
       FFI.gfs_replica_info_free(ptr)
     end
 
+    def self.set_opts(opts)
+      @@opts = opts
+    end
+
     def initialize(gfarm, path)
       @gfarm = gfarm
       @realpath = @gfarm.realpath_by_gfarm2fs(path)
-      flag = 0
+      flag = @@opts.flags
       ptr = ::FFI::MemoryPointer.new(:pointer, 1)
       e = FFI.gfs_replica_info_by_name(@realpath, flag, ptr)
       if e != GFARM_ERR_NO_ERROR
@@ -147,13 +144,40 @@ module Gfarm
     end
   end
 
+  class Options
+    INCLUDING_DEAD_HOST       = 1
+    INCLUDING_INCOMPLETE_COPY = 2
+    INCLUDING_DEAD_COPY       = 4
+
+    def initialize(argv)
+      @args = []
+      @flags = 0
+      argv.each do |x|
+        case x
+        when "-i"
+          @including_incomplete_copy = true
+          @flags |= INCLUDING_INCOMPLETE_COPY
+        else
+          @args << x
+        end
+      end
+    end
+
+    attr_reader :args
+    attr_reader :flags
+    attr_reader :including_incomplete_copy
+  end
+
 end
 
 [:PIPE,:TERM,:INT].each do |sig|
   Signal.trap(sig, "EXIT")
 end
 
-gfarm = Gfarm.connection
+opts = Gfarm::Options.new(ARGV)
+Gfarm::ReplicaInfo.set_opts(opts)
+Gfarm::Connection.set_args(opts.args)
+gfarm = Gfarm::Connection.instance
 
 while path=$stdin.gets
   path.chomp!

data/bin/pwrake

@@ -10,8 +10,11 @@ end
 libpath = File.absolute_path(File.dirname(__FILE__))+"/../lib"
 $LOAD_PATH.unshift libpath
 
-require "pwrake/master"
-require "pwrake/branch"
+require "pwrake/version"
+require "pwrake/master/master_application"
+require "pwrake/task/task_manager"
+require "pwrake/task/task_algorithm"
+require "pwrake/branch/branch_application"
 class Rake::Application
   include Pwrake::BranchApplication
   prepend Pwrake::MasterApplication

data/bin/pwrake_branch

@@ -10,13 +10,15 @@ end
 libpath = File.absolute_path(File.dirname(__FILE__))+"/../lib"
 $LOAD_PATH.unshift libpath
 
-require "pwrake/branch"
+require "pwrake/branch/branch_application"
 class Rake::Application
   include Pwrake::BranchApplication
 end
 
-#require "tracer"
-#Tracer.on
+require "pwrake/task/task_algorithm"
+class Rake::Task
+  include Pwrake::TaskAlgorithm
+end
 
 # does NOT exit when writing to broken pipe
 Signal.trap(:PIPE, "IGNORE")

data/lib/pwrake/branch/branch.rb

@@ -1,19 +1,24 @@
+require "pwrake/nbio"
+require "pwrake/branch/communicator_set"
+require "pwrake/branch/fiber_queue"
+require "pwrake/branch/shell"
+require "pwrake/branch/file_utils"
+require "pwrake/option/option"
+
 module Pwrake
 
   class Branch
 
     def initialize(opts,r,w)
-      #Thread.abort_on_exception = true
+      Thread.abort_on_exception = true
       @option = opts
       @task_q = {}  # worker_id => FiberQueue.new
       @shells = []
       @ior = r
       @iow = w
-      @runner = Runner.new(@option['HEARTBEAT'])
-      @master_hdl = Handler.new(@runner,@ior,@iow)
-      @master_chan = Channel.new(@master_hdl)
-      @wk_comm = {}
-      @wk_hdl_set = HandlerSet.new
+      @selector = NBIO::Selector.new
+      @master_rd = NBIO::Reader.new(@selector,@ior)
+      @master_wt = NBIO::Writer.new(@selector,@iow)
       @shell_start_interval = @option['SHELL_START_INTERVAL']
     end
 
@@ -24,8 +29,8 @@ module Pwrake
       setup_shell
       setup_fiber
       setup_master_channel
-      @runner.run
-      Log.debug "Brandh#run end"
+      @cs.run("task execution")
+      Log.debug "Branch#run end"
     end
 
     attr_reader :logger
@@ -49,83 +54,73 @@ module Pwrake
       else
         @logger.level = Logger::WARN
       end
-    end
-
-    def setup_worker
-      s = @ior.gets
-      if s.chomp != "host_list_begin"
-        raise "Branch#setup_worker: recv=#{s.chomp} expected=host_list_begin"
-      end
 
       if dir = @option['LOG_DIR']
         fn = File.join(dir,@option["COMMAND_CSV_FILE"])
         Shell.profiler.open(fn,@option['GNU_TIME'],@option['PLOT_PARALLELISM'])
       end
+    end
 
-      worker_code = WorkerCommunicator.read_worker_progs(@option)
-
-      while s = @ior.gets
-        s.chomp!
-        break if s == "host_list_end"
-        if /^host:(\d+) (\S+) ([+-]?\d+)?$/ =~ s
-          id, host, ncore = $1,$2,$3
-          ncore &&= ncore.to_i
-          comm = WorkerCommunicator.new(id,host,ncore,@runner,@option)
-          comm.setup_connection(worker_code)
-          @wk_comm[id] = comm
-          @wk_hdl_set << comm.handler
-          @task_q[id] = FiberQueue.new
-        else
-          raise "Branch#setup_worker: recv=#{s.chomp} expected=host:id hostname ncore"
-        end
+    def setup_worker
+      @cs = CommunicatorSet.new(@master_rd,@selector,@option.worker_option)
+      @cs.create_communicators
+      worker_code = read_worker_progs(@option.worker_progs)
+      @cs.each_value do |comm|
+        Fiber.new do
+          comm.connect(worker_code)
+        end.resume
       end
+      @cs.run("connect to workers")
+      #
+      Fiber.new do
+        @cs.each_value do |comm|
+          # set WorkerChannel#ncore at Master
+          @master_wt.put_line "ncore:#{comm.id}:#{comm.ncore}"
+        end
+        @master_wt.put_line "ncore:done"
+      end.resume
+      @selector.run
+    end
 
-      errors = []
-
-      @wk_comm.values.each do |comm|
-        Fiber.new do
-          while true
-            if s = comm.channel.get_line
-              break unless comm.ncore_proc(s)
-            else
-              errors << comm
+    def read_worker_progs(worker_progs)
+      code = ""
+      modpath = {}
+      worker_progs.each do |f|
+        m = f.split(/\//).first
+        if !modpath[m]
+          $LOAD_PATH.each do |x|
+            if File.directory?(File.join(x,m))
+              modpath[m] = x
               break
             end
           end
-          Log.debug "Branch#setup_worker: fiber end of ncore_proc"
-        end.resume
-      end
-
-      @runner.run
-
-      if !errors.empty?
-        errors.each{|comm| @wk_hdl_set.delete(comm.handler)}
-        hosts = errors.map{|comm| comm.host}.join(",")
-        raise RuntimeError,"Failed to connect to workers: #{hosts}"
-      end
-
-      # ncore
-      @wk_comm.each_value do |comm|
-        # set WorkerChannel#ncore at Master
-        @master_hdl.put_line "ncore:#{comm.id}:#{comm.ncore}"
+          if !modpath[m]
+            raise RuntimeError,"load path for module #{m} not found"
+          end
+        end
+        path = File.join(modpath[m],f)
+        path += '.rb' if /\.rb$/ !~ path
+        if !File.exist?(path)
+          raise RuntimeError,"program file #{path} not found"
+        end
+        code << IO.read(path) + "\n"
       end
-      @master_hdl.put_line "ncore:done"
+      code
     end
 
     def setup_shell
       @shells = []
-      errors = []
-      shell_id = 0
-      @wk_comm.each_value do |comm|
+      @cs.each_value do |comm|
+        @task_q[comm.id] = task_q = FiberQueue.new
         comm.ncore.times do
-          chan = Channel.new(comm.handler,shell_id)
-          shell_id += 1
-          shell = Shell.new(chan,@task_q[comm.id],@option.worker_option)
-          @shells << shell
+          chan = comm.new_channel
+          shell = Shell.new(chan,comm,task_q,@option.worker_option)
           # wait for remote shell open
           Fiber.new do
-            if !shell.open
-              errors << comm.host
+            if shell.open
+              @shells << shell
+            else
+              @master_wt.put_line "retire:#{comm.id}"
             end
             Log.debug "Branch#setup_shells: end of fiber to open shell"
           end.resume
@@ -133,24 +128,20 @@ module Pwrake
         end
       end
 
-      @runner.run
-
-      if !errors.empty?
-        raise RuntimeError,"Failed to start workers: #{errors.inspect}"
-      end
+      @cs.run("setup shells")
     end
 
     def setup_fiber
       # start fibers
       @shells.each do |shell|
-        shell.create_fiber(@master_hdl).resume
+        shell.create_fiber(@master_wt).resume
       end
       Log.debug "all fiber started"
 
-      @wk_comm.each_value do |comm|
+      @cs.each_value do |comm|
         #comm.start_default_fiber
         Fiber.new do
-          while s = comm.channel.get_line
+          while s = comm.reader.get_line
             break unless comm.common_line(s)
           end
           Log.debug "Branch#setup_fiber: end of fiber for default channel"
@@ -158,34 +149,52 @@ module Pwrake
       end
 
       # setup end
-      @wk_comm.values.each do |comm|
-        comm.handler.put_line "setup_end"
+      @cs.each_value do |comm|
+        comm.writer.put_line "setup_end"
       end
 
-      @master_hdl.put_line "branch_setup:done"
+      @master_wt.put_line "branch_setup:done"
       Log.debug "Branch#setup_fiber: setup end"
     end
 
     def setup_master_channel
       Fiber.new do
-        while s = @master_chan.get_line
+        while s = @master_rd.get_line
           # receive command from main pwrake
           Log.debug "Branch:recv #{s.inspect} from master"
           case s
             #
           when /^(\d+):(.+)$/o
             id, tname = $1,$2
-            @task_q[id].enq(tname)
+            begin
+              task_name = tname.sub(/^\d+:/,"")
+              @task_q[id].enq(tname)
+            rescue => e
+              Log.error Log.bt(e)
+              ret="taskfail:#{id}:#{task_name}"
+              Log.debug "fail to enq task_q[#{id}], ret=#{ret}"
+              @master_wt.put_line(ret)
+            end
             #
           when /^exit$/
-            @task_q.each_value{|q| q.finish}
-            @shells.each{|shell| shell.close}
-            @runner.finish
+            #@task_q.each_value{|q| q.finish}
+            #@cs.drop_all
+            @cs.finish_shells
+
+            #@shells.each{|shell| shell.exit} # just for comfirm
+            #@selector.halt # should halt after exited
             break
             #
+          when /^drop:(.*)$/o
+            id = $1
+            taskq = @task_q.delete(id)
+            Log.debug "drop @task_q[#{id}]=#{taskq.inspect}"
+            @cs.drop(id)
+            #
           when /^kill:(.*)$/o
             sig = $1
             kill(sig)
+            #
           else
             Log.debug "Branch: invalid line from master: #{s}"
           end
@@ -196,16 +205,16 @@ module Pwrake
 
     def kill(sig="INT")
       Log.warn "Branch#kill #{sig}"
-      @wk_hdl_set.kill(sig)
+      @cs.kill(sig)
     end
 
     def finish
       return if @finished
       @finished = true
-      Log.debug "Branch#finish: begin"
-      @wk_hdl_set.exit
+      #Log.debug "Branch#finish: begin"
+      @cs.exit
       Log.debug "Branch#finish: worker exited"
-      @master_hdl.put_line "exited"
+      @master_wt.put_line "exited"
       Log.debug "Branch#finish: sent 'exited' to master"
     end