pwrake 2.1.3 → 2.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -2,14 +2,14 @@ require "thread"
2
2
  require "fileutils"
3
3
  require "timeout"
4
4
 
5
- ncore = Marshal.load($stdin)
6
- opts = Marshal.load($stdin)
7
5
  begin
8
- dc = Pwrake.const_get(opts[:shared_directory])
9
- dc.init(opts)
10
- Pwrake::Invoker.new(dc, ncore, opts).run
6
+ Pwrake::Invoker.new.run
11
7
  rescue => exc
12
8
  log = Pwrake::LogExecutor.instance
13
9
  log.error exc
14
10
  log.error exc.backtrace.join("\n")
11
+ open("pwrake_worker_err-#{ENV['USER']}-#{Process.pid}","w") do |f|
12
+ f.puts exc
13
+ f.puts exc.backtrace.join("\n")
14
+ end
15
15
  end
@@ -1,5 +1,4 @@
1
1
  require "singleton"
2
- require "timeout"
3
2
 
4
3
  module Pwrake
5
4
 
@@ -7,36 +6,43 @@ module Pwrake
7
6
  include Singleton
8
7
 
9
8
  def initialize
10
- @out = $stdout
9
+ @out = $stderr
11
10
  @mutex = Mutex.new
12
- @queue = Queue.new
13
- @heartbeat = 0
14
- @thread = Thread.new do
15
- loop do
16
- begin
17
- Timeout.timeout(@heartbeat) do
18
- if s = @queue.deq
19
- _puts s
20
- end
21
- end
22
- rescue Timeout::Error
11
+ @mutex_hb = Mutex.new
12
+ @cond_hb = true
13
+ @heartbeat = nil
14
+ @thread = Thread.new{ heartbeat_loop }
15
+ end
16
+
17
+ attr_accessor :out
18
+
19
+ def heartbeat=(t)
20
+ @heartbeat = t.to_i
21
+ @thread.run
22
+ end
23
+
24
+ def heartbeat_loop
25
+ loop do
26
+ @heartbeat ? sleep(@heartbeat) : sleep
27
+ @mutex_hb.synchronize do
28
+ if @cond_hb
23
29
  _puts "heartbeat"
24
30
  end
31
+ @cond_hb = true
25
32
  end
26
33
  end
27
34
  end
28
35
 
29
- def heartbeat=(heartbeat)
30
- @heartbeat = heartbeat
31
- @queue.enq(nil)
32
- end
33
-
34
36
  def add_logger(log)
35
37
  @log = log
36
38
  end
37
39
 
38
40
  def puts(s)
39
- @queue.enq(s)
41
+ @mutex_hb.synchronize do
42
+ @cond_hb = false
43
+ @thread.run
44
+ end
45
+ _puts(s)
40
46
  end
41
47
 
42
48
  def _puts(s)
@@ -45,7 +51,8 @@ module Pwrake
45
51
  @out.print s+"\n"
46
52
  end
47
53
  @out.flush
48
- rescue Errno::EPIPE
54
+ rescue Errno::EPIPE => e
55
+ @log.info "<#{e.inspect}" if @log
49
56
  end
50
57
  @log.info "<#{s}" if @log
51
58
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pwrake
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.1.3
4
+ version: 2.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Masahiro TANAKA
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-12-24 00:00:00.000000000 Z
11
+ date: 2017-02-22 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: parallel
@@ -31,6 +31,8 @@ email:
31
31
  executables:
32
32
  - gfwhere-pipe
33
33
  - pwrake
34
+ - pwrake-mpi
35
+ - pwrake-mpi-run
34
36
  - pwrake_branch
35
37
  extensions: []
36
38
  extra_rdoc_files: []
@@ -43,6 +45,8 @@ files:
43
45
  - Rakefile
44
46
  - bin/gfwhere-pipe
45
47
  - bin/pwrake
48
+ - bin/pwrake-mpi
49
+ - bin/pwrake-mpi-run
46
50
  - bin/pwrake_branch
47
51
  - lib/pwrake/branch/branch.rb
48
52
  - lib/pwrake/branch/branch_application.rb
@@ -62,10 +66,14 @@ files:
62
66
  - lib/pwrake/master/postprocess.rb
63
67
  - lib/pwrake/misc/graphviz.rb
64
68
  - lib/pwrake/misc/mcgp.rb
69
+ - lib/pwrake/mpi/branch.rb
70
+ - lib/pwrake/mpi/worker.rb
65
71
  - lib/pwrake/nbio.rb
66
72
  - lib/pwrake/option/host_map.rb
67
73
  - lib/pwrake/option/option.rb
68
- - lib/pwrake/option/option_filesystem.rb
74
+ - lib/pwrake/option/option_default_filesystem.rb
75
+ - lib/pwrake/option/option_gfarm.rb
76
+ - lib/pwrake/option/option_gfarm2fs.rb
69
77
  - lib/pwrake/queue/locality_aware_queue.rb
70
78
  - lib/pwrake/queue/no_action_queue.rb
71
79
  - lib/pwrake/queue/queue_array.rb
@@ -85,9 +93,7 @@ files:
85
93
  - lib/pwrake/worker/executor.rb
86
94
  - lib/pwrake/worker/gfarm_directory.rb
87
95
  - lib/pwrake/worker/invoker.rb
88
- - lib/pwrake/worker/load.rb
89
96
  - lib/pwrake/worker/log_executor.rb
90
- - lib/pwrake/worker/reader.rb
91
97
  - lib/pwrake/worker/shared_directory.rb
92
98
  - lib/pwrake/worker/worker_main.rb
93
99
  - lib/pwrake/worker/writer.rb
@@ -1,123 +0,0 @@
1
- require "pwrake/option/option_filesystem"
2
- require "parallel"
3
-
4
- module Pwrake
5
-
6
- class Option
7
-
8
- attr_reader :worker_option
9
- attr_reader :worker_progs
10
- attr_reader :queue_class
11
-
12
- def setup_filesystem
13
-
14
- @worker_progs = %w[
15
- parallel/processor_count.rb
16
- pwrake/worker/reader
17
- pwrake/worker/writer
18
- pwrake/worker/log_executor
19
- pwrake/worker/executor
20
- pwrake/worker/invoker
21
- pwrake/worker/shared_directory
22
- ]
23
- @worker_option = {
24
- :base_dir => "",
25
- :work_dir => self['WORK_DIR'],
26
- :log_dir => self['LOG_DIR'],
27
- :pass_env => self['PASS_ENV'],
28
- :ssh_option => self['SSH_OPTION'],
29
- :shell_command => self['SHELL_COMMAND'],
30
- :shell_rc => self['SHELL_RC'],
31
- :heartbeat => self['HEARTBEAT']
32
- }
33
-
34
- if @filesystem.nil?
35
- case mount_type
36
- when /gfarm2fs/
37
- self['FILESYSTEM'] = @filesystem = 'gfarm'
38
- end
39
- end
40
-
41
- #n_noaction_th = self['NUM_NOACTION_THREADS']
42
-
43
- case @filesystem
44
- when 'gfarm'
45
- require "pwrake/queue/locality_aware_queue"
46
- require "pwrake/gfarm/gfarm_path"
47
- GfarmPath.subdir = self['GFARM_SUBDIR']
48
- @filesystem = 'gfarm'
49
- base = self['GFARM_BASEDIR']
50
- prefix = self['GFARM_PREFIX']
51
- mntpnt = "#{base}/#{prefix}"
52
- @worker_option.merge!({
53
- :shared_directory => "GfarmDirectory",
54
- :base_dir => mntpnt,
55
- :work_dir => GfarmPath.pwd.to_s,
56
- :gfarm2fs_option => self['GFARM2FS_OPTION'],
57
- :gfarm2fs_debug => self['GFARM2FS_DEBUG'],
58
- :gfarm2fs_debug_wait => self['GFARM2FS_DEBUG_WAIT'],
59
- :single_mp => self['GFARM_SINGLE_MP']
60
- })
61
- @worker_progs.push "pwrake/worker/gfarm_directory"
62
-
63
- if self['DISABLE_AFFINITY']
64
- @queue_class = "TaskQueue"
65
- else
66
- @queue_class = "LocalityAwareQueue"
67
- end
68
- #@num_noaction_threads = (n_noaction_th || [8,@host_map.num_threads].max).to_i
69
- else
70
- @filesystem = 'nfs'
71
- @queue_class = "TaskQueue"
72
- #@num_noaction_threads = (n_noaction_th || 1).to_i
73
- @worker_option[:shared_directory] = "SharedDirectory"
74
- end
75
- @worker_progs.push "pwrake/worker/worker_main"
76
- Log.debug "@queue_class=#{@queue_class}"
77
- end
78
-
79
- def max_postprocess_pool
80
- case @filesystem
81
- when 'gfarm'
82
- self['MAX_GFWHERE_WORKER']
83
- else
84
- 1
85
- end
86
- end
87
-
88
- def postprocess(runner)
89
- case @filesystem
90
- when 'gfarm'
91
- require "pwrake/gfarm/gfarm_postprocess"
92
- GfarmPostprocess.new(runner)
93
- else
94
- require "pwrake/master/postprocess"
95
- Postprocess.new(runner)
96
- end
97
- end
98
-
99
- def mount_type(d=nil)
100
- mtab = '/etc/mtab'
101
- if File.exist?(mtab)
102
- d ||= mountpoint_of_cwd
103
- open(mtab,'r') do |f|
104
- f.each_line do |l|
105
- if /#{d} (?:type )?(\S+)/o =~ l
106
- return $1
107
- end
108
- end
109
- end
110
- end
111
- nil
112
- end
113
-
114
- def mountpoint_of_cwd
115
- d = Pathname.pwd
116
- while !d.mountpoint?
117
- d = d.parent
118
- end
119
- d
120
- end
121
-
122
- end
123
- end
@@ -1,14 +0,0 @@
1
- require "thread"
2
- require "pathname"
3
- require "fileutils"
4
- require "singleton"
5
- require "forwardable"
6
- require "logger"
7
- require "timeout"
8
-
9
- require "pwrake/worker/writer"
10
- require "pwrake/worker/log_executor"
11
- require "pwrake/worker/executor"
12
- require "pwrake/worker/invoker"
13
-
14
- require "pwrake/worker/shared_directory"
@@ -1,73 +0,0 @@
1
- module Pwrake
2
-
3
- class Reader
4
-
5
- def initialize(io,mode="")
6
- @io = io
7
- @buf = ''
8
- @eof = false
9
- @mode = mode
10
- end
11
-
12
- attr_reader :io, :mode
13
-
14
- def eof?
15
- @eof && @buf.empty?
16
- end
17
-
18
- def gets
19
- read_until("\n")
20
- end
21
-
22
- def read_until(sep="\r\n", chunk_size=8192)
23
- until i = @buf.index(sep)
24
- if s = _read(chunk_size)
25
- @buf += s
26
- else
27
- if !@buf.empty? && @eof
28
- buf = @buf; @buf = ''
29
- return buf
30
- else
31
- return nil
32
- end
33
- end
34
- end
35
- @buf.slice!(0, i+sep.bytesize)
36
- end
37
-
38
- def _read(sz)
39
- @io.read_nonblock(sz)
40
- rescue EOFError
41
- @eof = true
42
- nil
43
- rescue IO::WaitReadable
44
- nil
45
- end
46
-
47
- end
48
-
49
-
50
- class Selector
51
-
52
- def initialize
53
- @readers = {}
54
- end
55
-
56
- def add_reader(io,&callback)
57
- @readers[io] = callback
58
- end
59
-
60
- def delete_reader(io)
61
- @readers.delete(io)
62
- end
63
-
64
- def loop
65
- while !@readers.empty?
66
- r, = IO.select(@readers.keys,nil,nil)
67
- r.each{|io| @readers[io].call} if r
68
- end
69
- end
70
-
71
- end
72
-
73
- end