pwrake 2.3.0 → 2.3.1

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 99a7b5b9f4ecf97b511f58495f6b22facbd66c97d79aa0cab9f452d96ae44ea4
4
- data.tar.gz: 2874a8eec50b7f508138567899da703e4c5d18687d58ad53a715ffa81b7fa402
3
+ metadata.gz: e666a4e98f080f07f41475896df0a2d115805b8af57e3541b8d898c60a8122c9
4
+ data.tar.gz: a320f42590130bab37518996ed11496d8b107ba3162a8458c896aa6daa3b322d
5
5
  SHA512:
6
- metadata.gz: c49bcd791f52010a2a221118bec1f59c08796c053ef43127f68142bd97d95499d23becf2c3d6ab9f351bc48384300999824fd3bd0f6fe1e94cba5de6b291664a
7
- data.tar.gz: af1ecd762e914f1e891a40639ab8f59a55165132646816ccb1a5209e9c761a518cb09f0c4385086f23bb7d172e658127852780577d748baa0c83313edb72e848
6
+ metadata.gz: cd4a9b2641c220d2a1265ccdf76ed87f2594ce0d498f521a24d67a6428d5c757129c31408f9fb9058b4e184aa2a09b01096edb17e2b42c5d5f634bd5b61125dc
7
+ data.tar.gz: 92f066ae2246bdd86097795432124d129a21e60e27d96515fe16059a05e14c0c7d4a1eefff85e574860626f600efb6c8698e1969824bcffdd5429c0905cc75cf
data/README.md CHANGED
@@ -93,8 +93,8 @@ In this case, you need the rehash of command paths:
93
93
  -L, --log, --log-dir [DIRECTORY] [Pw] Write log to DIRECTORY
94
94
  --ssh-opt, --ssh-option OPTION
95
95
  [Pw] Option passed to SSH
96
- --filesystem FILESYSTEM [Pw] Specify FILESYSTEM (nfs|gfarm)
97
- --gfarm [Pw] FILESYSTEM=gfarm
96
+ --filesystem FILESYSTEM [Pw] Specify FILESYSTEM (nfs|gfarm2fs)
97
+ --gfarm [Pw] (obsolete; Start pwrake on Gfarm FS)
98
98
  -A, --disable-affinity [Pw] Turn OFF affinity (AFFINITY=off)
99
99
  -S, --disable-steal [Pw] Turn OFF task steal
100
100
  -d, --debug [Pw] Output Debug messages
@@ -151,6 +151,7 @@ In this case, you need the rehash of command paths:
151
151
  GFARM_PREFIX default="pwrake_$USER"
152
152
  GFARM_SUBDIR default='/'
153
153
  MAX_GFWHERE_WORKER default=8
154
+ GFARM2FS_COMMAND default='gfarm2fs'
154
155
  GFARM2FS_OPTION default=""
155
156
  GFARM2FS_DEBUG default=false
156
157
  GFARM2FS_DEBUG_WAIT default=1
@@ -177,14 +178,14 @@ end
177
178
 
178
179
  Properties (The leftmost item is default):
179
180
 
180
- ncore=integer - The number of cores used by this task.
181
- exclusive=no|yes - Exclusively execute this task in a single node.
182
- reserve=no|yes - Gives higher priority to this task if ncore>1. (reserve a host)
183
- allow=hostname - Allow this host to execute this task. (accepts wild card)
184
- deny=hostname - Deny this host to execute this task. (accepts wild card)
185
- order=deny,allow|allow,deny - The order of evaluation.
186
- steal=yes|no - Allow task stealing for this task.
187
- retry=integer - The number of retry for this task.
181
+ ncore=integer|rational - The number of cores used by this task.
182
+ exclusive=no|yes - Exclusively execute this task in a single node.
183
+ reserve=no|yes - Gives higher priority to this task if ncore>1. (reserve a host)
184
+ allow=hostname - Allow this host to execute this task. (accepts wild card)
185
+ deny=hostname - Deny this host to execute this task. (accepts wild card)
186
+ order=deny,allow|a llow,deny - The order of evaluation.
187
+ steal=yes|no - Allow task stealing for this task.
188
+ retry=integer - The number of retry for this task.
188
189
 
189
190
  ## Note for Gfarm
190
191
 
@@ -42,6 +42,7 @@ module Pwrake
42
42
  setup_shell
43
43
  setup_fiber
44
44
  setup_master_channel
45
+ setup_search_thread
45
46
  @cs.run("task execution")
46
47
  Log.debug "Branch#run end"
47
48
  end
@@ -193,6 +194,29 @@ module Pwrake
193
194
  end.resume
194
195
  end
195
196
 
197
+ def setup_search_thread
198
+ @search_que = Queue.new
199
+ Thread.new do
200
+ while a = @search_que.deq
201
+ t,args,w = *a
202
+ t.pw_search_tasks(args)
203
+ w.puts(t.name)
204
+ end
205
+ end
206
+ end
207
+
208
+ def invoke(t,args)
209
+ Log.debug "Branch#invoke start: #{t.class}[#{t.name}]"
210
+ r,w = IO.pipe
211
+ rd = NBIO::Reader.new(@selector,r)
212
+ @search_que.enq([t,args,w])
213
+ task_name = rd.get_line.chomp
214
+ if t.name != task_name
215
+ raise "t.name=#{t.name} != task_name=#{task_name}"
216
+ end
217
+ Log.debug "Branch#invoke end: #{t.class}[#{t.name}]"
218
+ end
219
+
196
220
  def kill(sig="INT")
197
221
  Log.warn "Branch#kill #{sig}"
198
222
  @cs.kill(sig)
@@ -90,7 +90,6 @@ class Communicator
90
90
 
91
91
  sel = @set.selector
92
92
  @reader = NBIO::MultiReader.new(sel,@ior)
93
- @rd_err = NBIO::Reader.new(sel,@ioe)
94
93
  @writer = NBIO::Writer.new(sel,@iow)
95
94
  @handler = NBIO::Handler.new(@reader,@writer,@host)
96
95
 
@@ -146,48 +145,39 @@ class Communicator
146
145
  end
147
146
 
148
147
  def dropout(exc=nil)
149
- # Error output
150
- err_out = []
148
+ # Finish worker
151
149
  begin
152
150
  finish_shells
153
151
  if @handler
154
152
  @handler.exit
155
153
  @handler = nil
156
154
  end
157
- if @rd_err
158
- while s = @rd_err.get_line
159
- err_out << s
160
- end
161
- end
162
155
  rescue => e
163
156
  m = Log.bt(e)
164
- #$stderr.puts m
157
+ $stderr.puts(m)
165
158
  Log.error(m)
166
159
  end
167
- # Error output
168
- if !err_out.empty?
169
- $stderr.puts err_out.join("\n")
170
- Log.error((["process error output:"]+err_out).join("\n "))
160
+ # Error output from worker
161
+ if @ioe
162
+ err_out = ["standard error from worker:"]
163
+ while s = @ioe.gets
164
+ err_out << s.chomp
165
+ end
166
+ if err_out.size > 1
167
+ m = err_out.join("\n ")
168
+ $stderr.puts(m)
169
+ Log.error(m)
170
+ end
171
171
  end
172
- # Exception
172
+ # Exception message
173
173
  if exc
174
174
  m = Log.bt(exc)
175
- #$stderr.puts m
176
- Log.error m
175
+ $stderr.puts(m)
176
+ Log.error(m)
177
177
  end
178
178
  ensure
179
179
  @set.delete(self)
180
180
  end
181
181
 
182
- def finish
183
- @iow.close
184
- while s=@ior.gets
185
- puts "out=#{s.chomp}"
186
- end
187
- while s=@ioe.gets
188
- puts "err=#{s.chomp}"
189
- end
190
- end
191
-
192
182
  end
193
183
  end
@@ -43,7 +43,7 @@ module Pwrake
43
43
  break
44
44
  end
45
45
  end
46
- Log.debug "Gfarm file=#{filename} nodes=#{a.join("|")}"
46
+ #Log.debug "Gfarm file=#{filename} nodes=#{a.join("|")}"
47
47
  a
48
48
  end
49
49
 
@@ -226,11 +226,11 @@ module Pwrake
226
226
  if ending?
227
227
  @post_pool.finish # need?
228
228
  else
229
- setup_fiber(t)
229
+ setup_fiber
230
230
  end
231
231
  end
232
232
 
233
- def setup_fiber(t)
233
+ def setup_fiber
234
234
  @host_fail = @option["HOST_FAILURE"]
235
235
  create_fiber(@hdl_set) do |hdl|
236
236
  while s = hdl.get_line
@@ -295,15 +295,17 @@ module Pwrake
295
295
  $stderr.puts(s)
296
296
  end
297
297
  end
298
- Log.debug "Master#invoke: fiber end"
298
+ Log.debug "Master#setup_fiber: end of fiber"
299
299
  end
300
+
300
301
  if !ending?
301
302
  Log.debug "@selector.run begin"
302
303
  @selector.run
303
304
  Log.debug "@selector.run end"
305
+ else
306
+ Log.debug "@selector.run skipped"
304
307
  end
305
308
  @post_pool.finish
306
- Log.debug "Master#invoke: end of task=#{t.name}"
307
309
  end
308
310
 
309
311
  def send_task_to_idle_core
@@ -361,7 +363,7 @@ module Pwrake
361
363
 
362
364
  def task_end(tw,host_info)
363
365
  return if host_info.nil?
364
- host_info.idle(tw.n_used_cores)
366
+ host_info.idle(tw.n_used_cores||1)
365
367
  if host_info.retired?
366
368
  # all retired
367
369
  Log.warn("retired host:#{host_info.name} because all core retired")
@@ -396,8 +398,9 @@ module Pwrake
396
398
  Log.debug " @task_queue.empty?=#{@task_queue.empty?}" if @task_queue.empty?
397
399
  Log.debug " @hostinfo_by_id.empty?=#{@hostinfo_by_id.empty?}" if @hostinfo_by_id.empty?
398
400
  Log.debug " @hostinfo_by_taskname.keys=#{@hostinfo_by_taskname.keys.inspect}"
401
+ Log.debug " @post_pool.empty?=#{@post_pool.empty?}" if @post_pool.empty?
399
402
  end
400
- @hostinfo_by_taskname.empty?
403
+ @hostinfo_by_taskname.empty? && @post_pool.empty?
401
404
  else
402
405
  false
403
406
  end
@@ -26,13 +26,13 @@ module Pwrake
26
26
  @master.setup_branches
27
27
  load_rakefile
28
28
  begin
29
- Log.debug "init: #{Pwrake.clock-t} sec"
29
+ Log.debug "init: %.6f sec" % (Pwrake.clock-t)
30
30
  t = Pwrake.clock
31
31
  top_level
32
- Log.debug "main: #{Pwrake.clock-t} sec"
32
+ Log.debug "main: %.6f sec" % (Pwrake.clock-t)
33
33
  t = Pwrake.clock
34
34
  @failed = @master.finish
35
- Log.debug "finish: #{Pwrake.clock-t} sec"
35
+ Log.debug "finish: %.6f sec" % (Pwrake.clock-t)
36
36
  rescue SystemExit => e
37
37
  @failed = true
38
38
  rescue Exception => e
@@ -84,7 +84,11 @@ module Pwrake
84
84
  end
85
85
 
86
86
  def invoke(t,*args)
87
- @master.invoke(t,args)
87
+ if Thread.current == @master.thread
88
+ @branch.invoke(t,args)
89
+ else
90
+ @master.invoke(t,args)
91
+ end
88
92
  end
89
93
 
90
94
  def standard_rake_options
@@ -139,14 +143,15 @@ module Pwrake
139
143
  options.ssh_option = value
140
144
  }
141
145
  ],
142
- ['--filesystem FILESYSTEM', "[Pw] Specify FILESYSTEM (nfs|gfarm)",
146
+ ['--filesystem FILESYSTEM', "[Pw] Specify FILESYSTEM (nfs|gfarm2fs)",
143
147
  lambda { |value|
144
148
  options.filesystem = value
145
149
  }
146
150
  ],
147
- ['--gfarm', "[Pw] FILESYSTEM=gfarm",
151
+ ['--gfarm', "[Pw] FILESYSTEM=gfarm2fs",
148
152
  lambda { |value|
149
- options.filesystem = "gfarm"
153
+ warn 'The --gfarm option may not work correctly. Execute pwrake under gfarm2fs.'
154
+ options.filesystem = "gfarm2fs"
150
155
  }
151
156
  ],
152
157
  ['-A', '--disable-affinity', "[Pw] Turn OFF affinity (AFFINITY=off)",
@@ -506,8 +506,17 @@ module NBIO
506
506
  end
507
507
  @exited = true
508
508
  exit_msg = "exited"
509
- #return if iow.closed?
510
- @writer.put_line "exit"
509
+ begin
510
+ @writer.put_line "exit"
511
+ rescue Errno::EPIPE => e
512
+ m = "#{e} in #{self.class}.exit iow=#{iow.inspect}"
513
+ if defined? Log
514
+ Log.warn m
515
+ else
516
+ $stderr.puts m
517
+ end
518
+ return
519
+ end
511
520
  Log.debug "Handler#exit: end: @writer.put_line \"exit\"" if defined? Log
512
521
  #
513
522
  if @reader.class == Reader # MultiReader not work
@@ -518,9 +527,6 @@ module NBIO
518
527
  return if line == exit_msg
519
528
  end
520
529
  end
521
- rescue Errno::EPIPE => e
522
- Log.error "Errno::EPIPE in #{self.class}.exit iow=#{iow.inspect}\n"+
523
- e.backtrace.join("\n") if defined? Log
524
530
  end
525
531
 
526
532
  def halt
@@ -13,7 +13,7 @@ module Pwrake
13
13
 
14
14
  def initialize(name,id,ncore,weight,group=nil)
15
15
  @name = name
16
- @ncore = ncore
16
+ @ncore = ncore || 1
17
17
  @weight = weight || 1.0
18
18
  @group = group || 0
19
19
  @id = id
@@ -32,6 +32,17 @@ module Pwrake
32
32
  attr_reader :continuous_fail
33
33
  attr_accessor :idle_cores
34
34
 
35
+ def add_line(ncore=nil,weight=nil,group=nil)
36
+ ncore ||= 1
37
+ weight ||= 1.0
38
+ group ||= 0
39
+ if @group != group
40
+ raise "different group=#{group} for host=#{@name}"
41
+ end
42
+ @weight = (@weight*@ncore + weight*ncore)/(@ncore+ncore)
43
+ @ncore += ncore
44
+ end
45
+
35
46
  def local?
36
47
  ipa = IPSocket.getaddress(@name)
37
48
  HostInfo.local_ip.include?(ipa)
@@ -231,7 +242,7 @@ module Pwrake
231
242
  #weight = (weight || 1).to_f
232
243
  group &&= group.to_i
233
244
  if host_info = @by_name[host]
234
- raise RuntimeError,"duplicated hostname: #{host}"
245
+ host_info.add_line(ncore,weight,group)
235
246
  else
236
247
  id = @by_id.size
237
248
  host_info = HostInfo.new(host,id,ncore,weight,group)
@@ -61,11 +61,11 @@ module Pwrake
61
61
  # ----------------------------------------------------------
62
62
 
63
63
  def init_filesystem
64
- @filesystem = Rake.application.options.filesystem
65
- @filesystem ||= mount_type.sub(/fuse\./,"")
66
- begin
67
- require "pwrake/option/option_#{@filesystem}"
68
- rescue LoadError
64
+ @filesystem = Rake.application.options.filesystem || mount_type
65
+ case @filesystem
66
+ when 'gfarm2fs'
67
+ require "pwrake/option/option_gfarm2fs"
68
+ else
69
69
  require "pwrake/option/option_default_filesystem"
70
70
  end
71
71
  end
@@ -73,15 +73,15 @@ module Pwrake
73
73
  attr_reader :worker_option
74
74
  attr_reader :queue_class
75
75
 
76
- def mount_type(d=nil)
76
+ def mount_type(dir=nil)
77
77
  mtab = '/etc/mtab'
78
78
  if File.exist?(mtab)
79
- d ||= mountpoint_of_cwd
79
+ dir ||= mountpoint_of_cwd
80
80
  open(mtab,'r') do |f|
81
81
  f.each_line do |l|
82
82
  a = l.split
83
- if a[1] == d
84
- return a[2]
83
+ if a[1] == dir
84
+ return a[2].sub(/^fuse\./,'')
85
85
  end
86
86
  end
87
87
  end
@@ -354,6 +354,14 @@ module Pwrake
354
354
  # ----------------------------------------------------------
355
355
 
356
356
  def setup_hosts
357
+ if f = ENV['PBS_NODEFILE']
358
+ if @hostfile
359
+ Log.info "HOSTFILE=#{@hostfile} overrides PBS_NODEFILE=#{f}"
360
+ else
361
+ Log.info "use PBS_NODEFILE=#{f}"
362
+ @hostfile = f
363
+ end
364
+ end
357
365
  if @hostfile && @num_threads
358
366
  raise "Cannot set `hostfile' and `num_threads' simultaneously"
359
367
  end
@@ -364,6 +372,7 @@ module Pwrake
364
372
  # ----------------------------------------------------------
365
373
 
366
374
  def put_log
375
+ Log.info "Pwrake::VERSION=#{Pwrake::VERSION}"
367
376
  Log.info "Options:"
368
377
  self.each do |k,v|
369
378
  Log.info " #{k} = #{v.inspect}"
@@ -29,7 +29,6 @@ module Pwrake
29
29
  :heartbeat => self['HEARTBEAT'],
30
30
  :shared_directory => "SharedDirectory"
31
31
  }
32
- @filesystem = "default"
33
32
  @queue_class = "NonLocalityQueue"
34
33
  end
35
34
 
@@ -8,6 +8,7 @@ module Pwrake
8
8
 
9
9
  def option_data_filesystem
10
10
  [
11
+ 'GFARM2FS_COMMAND',
11
12
  'GFARM2FS_OPTION',
12
13
  'GFARM2FS_DEBUG',
13
14
  ['GFARM2FS_DEBUG_WAIT', proc{|v| v ? v.to_i : 1}],
@@ -21,7 +22,6 @@ module Pwrake
21
22
  end
22
23
 
23
24
  def set_filesystem_option
24
- @filesystem = 'gfarm'
25
25
  GfarmPath.subdir = self['GFARM_SUBDIR']
26
26
  @worker_option = {
27
27
  :log_dir => self['LOG_DIR'],
@@ -33,6 +33,7 @@ module Pwrake
33
33
  :shared_directory => "GfarmDirectory",
34
34
  :base_dir => self['GFARM_BASEDIR']+"/"+self['GFARM_PREFIX'],
35
35
  :work_dir => GfarmPath.pwd.to_s,
36
+ :gfarm2fs_command => self['GFARM2FS_COMMAND'],
36
37
  :gfarm2fs_option => self['GFARM2FS_OPTION'],
37
38
  :gfarm2fs_debug => self['GFARM2FS_DEBUG'],
38
39
  :gfarm2fs_debug_wait => self['GFARM2FS_DEBUG_WAIT'],
@@ -30,13 +30,12 @@ module Pwrake
30
30
  def pop
31
31
  case @prio
32
32
  when 0
33
- x = @que.shift
33
+ @que.shift
34
34
  when 1
35
- x = @que.pop
35
+ @que.pop
36
36
  when 2
37
- x = @que.delete_at(rand(@que.size))
37
+ @que.delete_at(rand(@que.size))
38
38
  end
39
- return x
40
39
  end
41
40
 
42
41
  alias shift pop
@@ -9,8 +9,11 @@ module Pwrake
9
9
  def initialize(queue_class, hostinfo_by_id, group_map=nil)
10
10
  @queue_class = Pwrake.const_get(queue_class)
11
11
  @hostinfo_by_id = hostinfo_by_id
12
+ @lock = Monitor.new
12
13
  @q_no_action = NoActionQueue.new
13
14
  @q_reserved = Hash.new
15
+ @nenq = 0
16
+ @ndeq = 0
14
17
  def @q_reserved.first
15
18
  super.last
16
19
  end
@@ -40,27 +43,39 @@ module Pwrake
40
43
  end
41
44
 
42
45
  def enq(tw)
46
+ @lock.synchronize do
43
47
  if tw.nil? || tw.actions.empty?
44
48
  @q_no_action.push(tw)
45
49
  else
46
50
  @q.enq_impl(tw)
47
51
  end
52
+ @nenq += 1
53
+ end
48
54
  end
49
55
 
50
56
  def deq_task(&block)
51
- Log.debug "deq_task from:"+(empty? ? " (empty)" : "\n#{inspect_q}")
57
+ @lock.synchronize do
58
+ if @nenq > 0
59
+ Log.debug "deq_task nenq=#{@nenq}:"+(empty? ? " (empty)" : "\n"+inspect_q)
60
+ @nenq = 0
61
+ end
52
62
  deq_noaction_task(&block)
53
63
  deq_reserve(&block)
54
64
  @q.deq_start
55
65
  unless @q.empty?
56
66
  @q.turns.each{|turn| deq_turn(turn,&block) }
57
67
  end
68
+ if @ndeq > 0
69
+ Log.debug "deq_task ndeq=#{@ndeq}:"+(empty? ? " (empty)" : "\n"+inspect_q)
70
+ @ndeq = 0
71
+ end
72
+ end
58
73
  end
59
74
 
60
75
  def deq_noaction_task(&block)
61
76
  while tw = @q_no_action.shift
62
- Log.debug "deq_noaction: #{tw.name}"
63
77
  yield(tw)
78
+ @ndeq += 1
64
79
  end
65
80
  end
66
81
 
@@ -72,6 +87,7 @@ module Pwrake
72
87
  @q_reserved.delete(host_info)
73
88
  Log.debug "deq_reserve: #{tw.name} n_use_cores=#{n_core}"
74
89
  yield(tw,host_info,n_core)
90
+ @ndeq += 1
75
91
  end
76
92
  end
77
93
  end
@@ -89,6 +105,7 @@ module Pwrake
89
105
  Log.debug "deq: #{tw.name} n_use_cores=#{n_core}"
90
106
  yield(tw,host_info,n_core)
91
107
  count += 1
108
+ @ndeq += 1
92
109
  else
93
110
  @q_reserved[host_info] = tw
94
111
  Log.debug "reserve host: #{host_info.name} for #{tw.name} (#{n_core} cores)"
@@ -93,6 +93,7 @@ module Pwrake
93
93
 
94
94
  t_end = (a.last)[0]
95
95
 
96
+ begin
96
97
  if system("which gnuplot >/dev/null 2>&1")
97
98
  IO.popen("gnuplot","r+") do |f|
98
99
  f.print "
@@ -112,6 +113,10 @@ plot '-' w l notitle
112
113
  end
113
114
  end
114
115
  end
116
+ rescue => exc
117
+ $stderr.puts exc
118
+ $stderr.puts exc.backtrace.join("\n")
119
+ end
115
120
 
116
121
  #puts "Parallelism plot: #{fimg}"
117
122
  fimg
@@ -151,6 +156,7 @@ plot '-' w l notitle
151
156
 
152
157
  t_end = (a.last)[0]
153
158
 
159
+ begin
154
160
  if system("which gnuplot >/dev/null 2>&1")
155
161
  IO.popen("gnuplot","r+") do |f|
156
162
  f.print "
@@ -177,6 +183,10 @@ plot '-' w l axis x1y1 title 'parallelism', '-' w l axis x1y2 title 'exec/sec'
177
183
  end
178
184
  end
179
185
  end
186
+ rescue => exc
187
+ $stderr.puts exc
188
+ $stderr.puts exc.backtrace.join("\n")
189
+ end
180
190
 
181
191
  #puts "Parallelism plot: #{fimg}"
182
192
  fimg
@@ -341,6 +351,7 @@ set ylabel '# of cores'
341
351
  grid << a
342
352
  end
343
353
 
354
+ begin
344
355
  if system("which gnuplot >/dev/null 2>&1")
345
356
  IO.popen("gnuplot","r+") do |f|
346
357
  f.puts "
@@ -376,6 +387,10 @@ set format y ''
376
387
  f.printf "e\n"
377
388
  end
378
389
  end
390
+ rescue => exc
391
+ $stderr.puts exc
392
+ $stderr.puts exc.backtrace.join("\n")
393
+ end
379
394
  fimg
380
395
  end
381
396
 
@@ -2,15 +2,60 @@ module Pwrake
2
2
 
3
3
  module FileTaskAlgorithm
4
4
 
5
+ def needed?
6
+ !_exist?(name) || out_of_date?(timestamp) || @application.options.build_all
7
+ end
8
+
5
9
  # Cache time stamp to reduce load on file system.
6
10
  def timestamp
7
11
  @file_mtime ||
8
- if File.exist?(name)
9
- @file_mtime = File.mtime(name.to_s)
12
+ if _exist?(name)
13
+ @file_mtime = _mtime(name.to_s)
10
14
  else
11
15
  Rake::LATE
12
16
  end
13
17
  end
14
18
 
19
+ private
20
+
21
+ @@t_mtime = 0
22
+ @@n_mtime = 0
23
+ @@l_mtime = 100
24
+ @@c_mtime = 0
25
+ @@t_exist = 0
26
+ @@n_exist = 0
27
+ @@l_exist = 100
28
+ @@c_exist = 0
29
+
30
+ def _mtime(name)
31
+ t = Pwrake.clock
32
+ m = File.mtime(name.to_s)
33
+ @@t_mtime = @@t_mtime + (Pwrake.clock-t)
34
+ @@n_mtime += 1
35
+ if @@n_mtime >= 100
36
+ Log.debug('mtime: mean=%.9f s (%d times)'%[@@t_mtime/@@n_mtime,@@n_mtime])
37
+ @@t_mtime = 0
38
+ @@n_mtime = 0
39
+ @@c_mtime += 1
40
+ @@l_mtime = 1000 if @@c_mtime == 10
41
+ end
42
+ m
43
+ end
44
+
45
+ def _exist?(name)
46
+ t = Pwrake.clock
47
+ e = File.exist?(name.to_s)
48
+ @@t_exist = @@t_exist + (Pwrake.clock-t)
49
+ @@n_exist += 1
50
+ if @@n_exist >= @@l_exist
51
+ Log.debug('exist: mean=%.9f s (%d times)'%[@@t_exist/@@n_exist,@@n_exist])
52
+ @@t_exist = 0
53
+ @@n_exist = 0
54
+ @@c_exist += 1
55
+ @@l_exist = 1000 if @@c_exist == 10
56
+ end
57
+ e
58
+ end
59
+
15
60
  end
16
61
  end
@@ -12,20 +12,24 @@ module Pwrake
12
12
 
13
13
  def wrapper
14
14
  if @wrapper.nil?
15
- raise "TaskWrapper is not defined for #{self.class}[#{name}]"
15
+ Log.debug "TaskWrapper is not defined for #{self.class}[#{name}]"
16
+ @wrapper = TaskWrapper.new(self)
16
17
  end
17
18
  @wrapper
18
19
  end
19
20
 
20
21
  def pw_search_tasks(args)
21
- Log.debug "#{self.class}#pw_search_tasks start, task=#{name} args=#{args.inspect}"
22
+ Log.debug "#{self.class}[#{name}]#pw_search_tasks start, args=#{args.inspect}"
23
+ if application.options.trace
24
+ application.trace "** Search #{name}#{format_search_flags}"
25
+ end
22
26
  cl = Pwrake.clock
23
27
  TaskWrapper.clear_rank
24
28
  task_args = TaskArguments.new(arg_names, args)
25
29
  # not synchronize owing to fiber
26
30
  search_with_call_chain(nil, task_args, InvocationChain::EMPTY)
27
31
  #
28
- Log.debug "#{self.class}#pw_search_tasks end #{Pwrake.clock-cl}"
32
+ Log.debug "#{self.class}[#{name}]#pw_search_tasks end t=%.6f" % (Pwrake.clock-cl)
29
33
  end
30
34
 
31
35
  # Same as search, but explicitly pass a call chain to detect
@@ -33,11 +37,6 @@ module Pwrake
33
37
  def search_with_call_chain(subseq, task_args, invocation_chain) # :nodoc:
34
38
  new_chain = InvocationChain.append(self, invocation_chain)
35
39
  @lock.synchronize do
36
- if application.options.trace
37
- #Log.debug "** Search #{name}#{format_search_flags}"
38
- application.trace "** Search #{name}#{format_search_flags}"
39
- end
40
-
41
40
  return true if @already_finished # <<--- competition !!!
42
41
  @subsequents ||= []
43
42
  @subsequents << subseq if subseq # <<--- competition !!!
@@ -89,14 +89,10 @@ module Pwrake
89
89
  end
90
90
 
91
91
  case @ncore
92
- when Rational
93
- if @ncore > 0 && @ncore <= 1
94
- return [(@ncore*ppn).to_i, 1].min
95
- end
96
- when 1-ppn..ppn
97
- return (@ncore>0) ? @ncore : @ncore+ppn
98
92
  when nil
99
93
  return 1
94
+ when 1-ppn..ppn
95
+ return (@ncore>0) ? @ncore : @ncore+ppn
100
96
  end
101
97
 
102
98
  m = "ncore=#{@ncore} is out of range of cores per node: #{ppn}"
@@ -10,6 +10,7 @@ module Pwrake
10
10
  @@current_id = 1
11
11
  @@task_logger = nil
12
12
  @@instances = []
13
+ LOCK = Monitor.new
13
14
 
14
15
  def initialize(task,task_args=nil)
15
16
  @task = task
@@ -68,8 +69,10 @@ module Pwrake
68
69
  end
69
70
 
70
71
  def self.clear_rank
71
- Log.debug "#{self}.clear_rank"
72
- @@instances.each{|w| w.clear_rank}
72
+ LOCK.synchronize do
73
+ Log.debug "#{self}.clear_rank"
74
+ @@instances.each{|w| w.clear_rank}
75
+ end
73
76
  end
74
77
 
75
78
  def preprocess
@@ -87,15 +90,12 @@ module Pwrake
87
90
 
88
91
  def postprocess(postproc)
89
92
  @executed = true if !@task.actions.empty?
90
- #tm_taskend = Pwrake.clock
91
93
  if is_file_task?
92
- #t = Pwrake.clock
93
94
  if File.exist?(name)
94
95
  @file_stat = File::Stat.new(name)
95
96
  @location = postproc.run(self)
96
97
  end
97
98
  end
98
- #Log.debug "postprocess time=#{Pwrake.clock-tm_taskend}"
99
99
  log_task
100
100
  end
101
101
 
@@ -129,7 +129,12 @@ module Pwrake
129
129
  end
130
130
  return if !@@task_logger
131
131
  #
132
- elap = @clock_end - @clock_start
132
+ if @clock_start
133
+ elap = @clock_end - @clock_start
134
+ else
135
+ Loag.debug "@clock_start is not defined for #{@task.class}[#{name}]"
136
+ elap = 0
137
+ end
133
138
  if has_output_file?
134
139
  RANK_STAT.add_sample(rank,elap)
135
140
  end
@@ -231,6 +236,7 @@ module Pwrake
231
236
 
232
237
  def suggest_location
233
238
  if has_input_file? && @suggest_location.nil?
239
+ cl = Pwrake.clock
234
240
  @suggest_location = []
235
241
  loc_fsz = Hash.new(0)
236
242
  prerequisites.each do |preq|
@@ -246,18 +252,19 @@ module Pwrake
246
252
  #Log.debug "input=#{prerequisites.join('|')}"
247
253
  if !loc_fsz.empty?
248
254
  half_max_fsz = loc_fsz.values.max / 2
249
- Log.debug "loc_fsz=#{loc_fsz.inspect} half_max_fsz=#{half_max_fsz}"
250
255
  loc_fsz.each do |h,sz|
251
256
  if sz > half_max_fsz
252
257
  @suggest_location << h
253
258
  end
254
259
  end
260
+ Log.debug "locate:%.6f #{name} loc_fsz=#{loc_fsz.inspect} half_max_fsz=#{half_max_fsz} suggest=#{@suggest_location.inspect}"%(Pwrake.clock-cl)
255
261
  end
256
262
  end
257
263
  @suggest_location
258
264
  end
259
265
 
260
266
  def rank
267
+ LOCK.synchronize do
261
268
  if @rank.nil?
262
269
  if subsequents.nil? || subsequents.empty?
263
270
  @rank = 0
@@ -265,8 +272,12 @@ module Pwrake
265
272
  max_rank = 0
266
273
  subsequents.each do |subsq|
267
274
  r = subsq.wrapper.rank
268
- if max_rank < r
269
- max_rank = r
275
+ if r
276
+ if max_rank < r
277
+ max_rank = r
278
+ end
279
+ else
280
+ Log.warn "subsq.wrapper.rank=#{rank.inspect}"
270
281
  end
271
282
  end
272
283
  if has_output_file?
@@ -276,9 +287,9 @@ module Pwrake
276
287
  end
277
288
  @rank = max_rank + step
278
289
  end
279
- Log.debug "Task[#{name}] rank=#{@rank.inspect}"
280
290
  end
281
291
  @rank
292
+ end
282
293
  end
283
294
 
284
295
  def clear_rank
@@ -348,7 +359,7 @@ module Pwrake
348
359
  else
349
360
  @priority = 0
350
361
  end
351
- Log.debug "task_name=#{name} priority=#{@priority} sum_file_size=#{sum_sz}"
362
+ Log.debug "task=#{name} priority=#{@priority} sum_file_size=#{sum_sz}"
352
363
  end
353
364
  @priority || 0
354
365
  end
@@ -1,3 +1,3 @@
1
1
  module Pwrake
2
- VERSION = "2.3.0"
2
+ VERSION = "2.3.1"
3
3
  end
@@ -16,6 +16,7 @@ module Pwrake
16
16
  @@prefix = opts[:base_dir]
17
17
  @@work_dir = opts[:work_dir]
18
18
  @@log_dir = opts[:log_dir]
19
+ @@gfarm2fs_command = opts[:gfarm2fs_command] || 'gfarm2fs'
19
20
  @@gfarm2fs_option = opts[:gfarm2fs_option]
20
21
  @@gfarm2fs_debug = opts[:gfarm2fs_debug]
21
22
  @@gfarm2fs_debug_wait = opts[:gfarm2fs_debug_wait]
@@ -56,13 +57,14 @@ module Pwrake
56
57
 
57
58
  def open
58
59
  FileUtils.mkdir_p @gfarm_mountpoint
60
+ @log.info "mkdir -p #{@@hostname}:#{@gfarm_mountpoint}"
59
61
  path = @log.path
60
62
  begin
61
63
  if @@gfarm2fs_debug && path
62
64
  f = path+("gfarm2fs-"+@@hostname+"-"+@suffix)
63
- spawn_cmd "gfarm2fs #{@@gfarm2fs_option} -d #{@gfarm_mountpoint} > #{f} 2>&1 & sleep #{@@gfarm2fs_debug_wait}"
65
+ spawn_cmd "#{@@gfarm2fs_command} #{@@gfarm2fs_option} -d #{@gfarm_mountpoint} > #{f} 2>&1 & sleep #{@@gfarm2fs_debug_wait}"
64
66
  else
65
- spawn_cmd "gfarm2fs #{@@gfarm2fs_option} #{@gfarm_mountpoint}"
67
+ spawn_cmd "#{@@gfarm2fs_command} #{@@gfarm2fs_option} #{@gfarm_mountpoint}"
66
68
  end
67
69
  rescue => exc
68
70
  sleep 1
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pwrake
3
3
  version: !ruby/object:Gem::Version
4
- version: 2.3.0
4
+ version: 2.3.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Masahiro TANAKA
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-07-30 00:00:00.000000000 Z
11
+ date: 2019-11-26 00:00:00.000000000 Z
12
12
  dependencies: []
13
13
  description: Parallel and distributed Rake for workflow execution on multicores, clusters,
14
14
  clouds using SSH. It has locality-aware scheduling designed for Gfarm file system.
@@ -58,7 +58,6 @@ files:
58
58
  - lib/pwrake/option/host_map.rb
59
59
  - lib/pwrake/option/option.rb
60
60
  - lib/pwrake/option/option_default_filesystem.rb
61
- - lib/pwrake/option/option_gfarm.rb
62
61
  - lib/pwrake/option/option_gfarm2fs.rb
63
62
  - lib/pwrake/queue/locality_aware_queue.rb
64
63
  - lib/pwrake/queue/no_action_queue.rb
@@ -1 +0,0 @@
1
- lib/pwrake/option/option_gfarm2fs.rb