pwrake 2.1.3 → 2.2.0

data/lib/pwrake/worker/worker_main.rb

@@ -2,14 +2,14 @@ require "thread"
 require "fileutils"
 require "timeout"
 
-ncore = Marshal.load($stdin)
-opts = Marshal.load($stdin)
 begin
-  dc = Pwrake.const_get(opts[:shared_directory])
-  dc.init(opts)
-  Pwrake::Invoker.new(dc, ncore, opts).run
+  Pwrake::Invoker.new.run
 rescue => exc
   log = Pwrake::LogExecutor.instance
   log.error exc
   log.error exc.backtrace.join("\n")
+  open("pwrake_worker_err-#{ENV['USER']}-#{Process.pid}","w") do |f|
+    f.puts exc
+    f.puts exc.backtrace.join("\n")
+  end
 end

data/lib/pwrake/worker/writer.rb

@@ -1,5 +1,4 @@
 require "singleton"
-require "timeout"
 
 module Pwrake
 
@@ -7,36 +6,43 @@ module Pwrake
     include Singleton
 
     def initialize
-      @out = $stdout
+      @out = $stderr
       @mutex = Mutex.new
-      @queue = Queue.new
-      @heartbeat = 0
-      @thread = Thread.new do
-        loop do
-          begin
-            Timeout.timeout(@heartbeat) do
-              if s = @queue.deq
-                _puts s
-              end
-            end
-          rescue Timeout::Error
+      @mutex_hb = Mutex.new
+      @cond_hb = true
+      @heartbeat = nil
+      @thread = Thread.new{ heartbeat_loop }
+    end
+
+    attr_accessor :out
+
+    def heartbeat=(t)
+      @heartbeat = t.to_i
+      @thread.run
+    end
+
+    def heartbeat_loop
+      loop do
+        @heartbeat ? sleep(@heartbeat) : sleep
+        @mutex_hb.synchronize do
+          if @cond_hb
             _puts "heartbeat"
           end
+          @cond_hb = true
         end
       end
     end
 
-    def heartbeat=(heartbeat)
-      @heartbeat = heartbeat
-      @queue.enq(nil)
-    end
-
     def add_logger(log)
       @log = log
     end
 
     def puts(s)
-      @queue.enq(s)
+      @mutex_hb.synchronize do
+        @cond_hb = false
+        @thread.run
+      end
+      _puts(s)
     end
 
     def _puts(s)
@@ -45,7 +51,8 @@ module Pwrake
           @out.print s+"\n"
         end
         @out.flush
-      rescue Errno::EPIPE
+      rescue Errno::EPIPE => e
+        @log.info "<#{e.inspect}" if @log
       end
       @log.info "<#{s}" if @log
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pwrake
 version: !ruby/object:Gem::Version
-  version: 2.1.3
+  version: 2.2.0
 platform: ruby
 authors:
 - Masahiro TANAKA
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-24 00:00:00.000000000 Z
+date: 2017-02-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: parallel
@@ -31,6 +31,8 @@ email:
 executables:
 - gfwhere-pipe
 - pwrake
+- pwrake-mpi
+- pwrake-mpi-run
 - pwrake_branch
 extensions: []
 extra_rdoc_files: []
@@ -43,6 +45,8 @@ files:
 - Rakefile
 - bin/gfwhere-pipe
 - bin/pwrake
+- bin/pwrake-mpi
+- bin/pwrake-mpi-run
 - bin/pwrake_branch
 - lib/pwrake/branch/branch.rb
 - lib/pwrake/branch/branch_application.rb
@@ -62,10 +66,14 @@ files:
 - lib/pwrake/master/postprocess.rb
 - lib/pwrake/misc/graphviz.rb
 - lib/pwrake/misc/mcgp.rb
+- lib/pwrake/mpi/branch.rb
+- lib/pwrake/mpi/worker.rb
 - lib/pwrake/nbio.rb
 - lib/pwrake/option/host_map.rb
 - lib/pwrake/option/option.rb
-- lib/pwrake/option/option_filesystem.rb
+- lib/pwrake/option/option_default_filesystem.rb
+- lib/pwrake/option/option_gfarm.rb
+- lib/pwrake/option/option_gfarm2fs.rb
 - lib/pwrake/queue/locality_aware_queue.rb
 - lib/pwrake/queue/no_action_queue.rb
 - lib/pwrake/queue/queue_array.rb
@@ -85,9 +93,7 @@ files:
 - lib/pwrake/worker/executor.rb
 - lib/pwrake/worker/gfarm_directory.rb
 - lib/pwrake/worker/invoker.rb
-- lib/pwrake/worker/load.rb
 - lib/pwrake/worker/log_executor.rb
-- lib/pwrake/worker/reader.rb
 - lib/pwrake/worker/shared_directory.rb
 - lib/pwrake/worker/worker_main.rb
 - lib/pwrake/worker/writer.rb

data/lib/pwrake/option/option_filesystem.rb

@@ -1,123 +0,0 @@
-require "pwrake/option/option_filesystem"
-require "parallel"
-
-module Pwrake
-
-  class Option
-
-    attr_reader :worker_option
-    attr_reader :worker_progs
-    attr_reader :queue_class
-
-    def setup_filesystem
-
-      @worker_progs = %w[
-        parallel/processor_count.rb
-        pwrake/worker/reader
-        pwrake/worker/writer
-        pwrake/worker/log_executor
-        pwrake/worker/executor
-        pwrake/worker/invoker
-        pwrake/worker/shared_directory
-      ]
-      @worker_option = {
-        :base_dir  => "",
-        :work_dir  => self['WORK_DIR'],
-        :log_dir   => self['LOG_DIR'],
-        :pass_env  => self['PASS_ENV'],
-        :ssh_option => self['SSH_OPTION'],
-        :shell_command => self['SHELL_COMMAND'],
-        :shell_rc  => self['SHELL_RC'],
-        :heartbeat => self['HEARTBEAT']
-      }
-
-      if @filesystem.nil?
-        case mount_type
-        when /gfarm2fs/
-          self['FILESYSTEM'] = @filesystem = 'gfarm'
-        end
-      end
-
-      #n_noaction_th = self['NUM_NOACTION_THREADS']
-
-      case @filesystem
-      when 'gfarm'
-        require "pwrake/queue/locality_aware_queue"
-        require "pwrake/gfarm/gfarm_path"
-        GfarmPath.subdir = self['GFARM_SUBDIR']
-        @filesystem  = 'gfarm'
-        base = self['GFARM_BASEDIR']
-        prefix = self['GFARM_PREFIX']
-        mntpnt = "#{base}/#{prefix}"
-        @worker_option.merge!({
-          :shared_directory => "GfarmDirectory",
-          :base_dir => mntpnt,
-          :work_dir => GfarmPath.pwd.to_s,
-          :gfarm2fs_option => self['GFARM2FS_OPTION'],
-          :gfarm2fs_debug => self['GFARM2FS_DEBUG'],
-          :gfarm2fs_debug_wait => self['GFARM2FS_DEBUG_WAIT'],
-          :single_mp => self['GFARM_SINGLE_MP']
-        })
-        @worker_progs.push "pwrake/worker/gfarm_directory"
-
-	if self['DISABLE_AFFINITY']
-	  @queue_class = "TaskQueue"
-	else
-	  @queue_class = "LocalityAwareQueue"
-	end
-        #@num_noaction_threads = (n_noaction_th || [8,@host_map.num_threads].max).to_i
-      else
-        @filesystem  = 'nfs'
-        @queue_class = "TaskQueue"
-        #@num_noaction_threads = (n_noaction_th || 1).to_i
-        @worker_option[:shared_directory] = "SharedDirectory"
-      end
-      @worker_progs.push "pwrake/worker/worker_main"
-      Log.debug "@queue_class=#{@queue_class}"
-    end
-
-    def max_postprocess_pool
-      case @filesystem
-      when 'gfarm'
-        self['MAX_GFWHERE_WORKER']
-      else
-        1
-      end
-    end
-
-    def postprocess(runner)
-      case @filesystem
-      when 'gfarm'
-        require "pwrake/gfarm/gfarm_postprocess"
-        GfarmPostprocess.new(runner)
-      else
-        require "pwrake/master/postprocess"
-        Postprocess.new(runner)
-      end
-    end
-
-    def mount_type(d=nil)
-      mtab = '/etc/mtab'
-      if File.exist?(mtab)
-        d ||= mountpoint_of_cwd
-        open(mtab,'r') do |f|
-          f.each_line do |l|
-            if /#{d} (?:type )?(\S+)/o =~ l
-              return $1
-            end
-          end
-        end
-      end
-      nil
-    end
-
-    def mountpoint_of_cwd
-      d = Pathname.pwd
-      while !d.mountpoint?
-        d = d.parent
-      end
-      d
-    end
-
-  end
-end

data/lib/pwrake/worker/load.rb

@@ -1,14 +0,0 @@
-require "thread"
-require "pathname"
-require "fileutils"
-require "singleton"
-require "forwardable"
-require "logger"
-require "timeout"
-
-require "pwrake/worker/writer"
-require "pwrake/worker/log_executor"
-require "pwrake/worker/executor"
-require "pwrake/worker/invoker"
-
-require "pwrake/worker/shared_directory"

data/lib/pwrake/worker/reader.rb

@@ -1,73 +0,0 @@
-module Pwrake
-
-  class Reader
-
-    def initialize(io,mode="")
-      @io = io
-      @buf = ''
-      @eof = false
-      @mode = mode
-    end
-
-    attr_reader :io, :mode
-
-    def eof?
-      @eof && @buf.empty?
-    end
-
-    def gets
-      read_until("\n")
-    end
-
-    def read_until(sep="\r\n", chunk_size=8192)
-      until i = @buf.index(sep)
-        if s = _read(chunk_size)
-          @buf += s
-        else
-          if !@buf.empty? && @eof
-            buf = @buf; @buf = ''
-            return buf
-          else
-            return nil
-          end
-        end
-      end
-      @buf.slice!(0, i+sep.bytesize)
-    end
-
-    def _read(sz)
-      @io.read_nonblock(sz)
-    rescue EOFError
-      @eof = true
-      nil
-    rescue IO::WaitReadable
-      nil
-    end
-
-  end
-
-
-  class Selector
-
-    def initialize
-      @readers = {}
-    end
-
-    def add_reader(io,&callback)
-      @readers[io] = callback
-    end
-
-    def delete_reader(io)
-      @readers.delete(io)
-    end
-
-    def loop
-      while !@readers.empty?
-        r, = IO.select(@readers.keys,nil,nil)
-        r.each{|io| @readers[io].call} if r
-      end
-    end
-
-  end
-
-end