pwrake 0.9.9.2 → 2.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +2 -0
  3. data/CHANGES_V2.md +90 -0
  4. data/{LICENSE.txt → MIT-LICENSE} +2 -3
  5. data/README +12 -0
  6. data/README.md +75 -52
  7. data/bin/gfwhere-pipe +23 -12
  8. data/bin/pwrake +22 -29
  9. data/bin/pwrake_branch +24 -0
  10. data/lib/pwrake/branch.rb +22 -0
  11. data/lib/pwrake/branch/branch.rb +213 -0
  12. data/lib/pwrake/branch/branch_application.rb +53 -0
  13. data/lib/pwrake/branch/fiber_queue.rb +36 -0
  14. data/lib/pwrake/branch/file_utils.rb +101 -0
  15. data/lib/pwrake/branch/shell.rb +231 -0
  16. data/lib/pwrake/{profiler.rb → branch/shell_profiler.rb} +28 -27
  17. data/lib/pwrake/branch/worker_communicator.rb +104 -0
  18. data/lib/pwrake/{gfarm_feature.rb → gfarm/gfarm_path.rb} +2 -100
  19. data/lib/pwrake/gfarm/gfarm_postprocess.rb +53 -0
  20. data/lib/pwrake/iomux/channel.rb +70 -0
  21. data/lib/pwrake/iomux/handler.rb +124 -0
  22. data/lib/pwrake/iomux/handler_set.rb +35 -0
  23. data/lib/pwrake/iomux/runner.rb +62 -0
  24. data/lib/pwrake/logger.rb +3 -150
  25. data/lib/pwrake/master.rb +30 -137
  26. data/lib/pwrake/master/fiber_pool.rb +69 -0
  27. data/lib/pwrake/master/idle_cores.rb +30 -0
  28. data/lib/pwrake/master/master.rb +345 -0
  29. data/lib/pwrake/master/master_application.rb +150 -0
  30. data/lib/pwrake/master/postprocess.rb +16 -0
  31. data/lib/pwrake/{graphviz.rb → misc/graphviz.rb} +0 -0
  32. data/lib/pwrake/{mcgp.rb → misc/mcgp.rb} +63 -42
  33. data/lib/pwrake/option/host_map.rb +158 -0
  34. data/lib/pwrake/option/option.rb +357 -0
  35. data/lib/pwrake/option/option_filesystem.rb +112 -0
  36. data/lib/pwrake/queue/locality_aware_queue.rb +158 -0
  37. data/lib/pwrake/queue/no_action_queue.rb +67 -0
  38. data/lib/pwrake/queue/queue_array.rb +366 -0
  39. data/lib/pwrake/queue/task_queue.rb +164 -0
  40. data/lib/pwrake/report.rb +1 -0
  41. data/lib/pwrake/report/parallelism.rb +9 -3
  42. data/lib/pwrake/report/report.rb +50 -103
  43. data/lib/pwrake/report/task_stat.rb +83 -0
  44. data/lib/pwrake/task/task_algorithm.rb +107 -0
  45. data/lib/pwrake/task/task_manager.rb +32 -0
  46. data/lib/pwrake/task/task_property.rb +98 -0
  47. data/lib/pwrake/task/task_rank.rb +48 -0
  48. data/lib/pwrake/task/task_wrapper.rb +296 -0
  49. data/lib/pwrake/version.rb +1 -1
  50. data/lib/pwrake/worker/executor.rb +169 -0
  51. data/lib/pwrake/worker/gfarm_directory.rb +90 -0
  52. data/lib/pwrake/worker/invoker.rb +199 -0
  53. data/lib/pwrake/worker/load.rb +14 -0
  54. data/lib/pwrake/worker/log_executor.rb +73 -0
  55. data/lib/pwrake/worker/shared_directory.rb +74 -0
  56. data/lib/pwrake/worker/worker_main.rb +14 -0
  57. data/lib/pwrake/worker/writer.rb +59 -0
  58. data/setup.rb +1212 -1502
  59. data/spec/003/Rakefile +2 -2
  60. data/spec/008/Rakefile +2 -1
  61. data/spec/009/Rakefile +1 -1
  62. data/spec/009/pwrake_conf.yaml +1 -3
  63. data/spec/hosts +0 -2
  64. data/spec/pwrake_spec.rb +9 -8
  65. metadata +50 -21
  66. data/lib/pwrake.rb +0 -19
  67. data/lib/pwrake/application.rb +0 -232
  68. data/lib/pwrake/counter.rb +0 -54
  69. data/lib/pwrake/file_utils.rb +0 -98
  70. data/lib/pwrake/gfwhere_pool.rb +0 -109
  71. data/lib/pwrake/host_list.rb +0 -88
  72. data/lib/pwrake/locality_aware_queue.rb +0 -413
  73. data/lib/pwrake/option.rb +0 -400
  74. data/lib/pwrake/rake_modify.rb +0 -14
  75. data/lib/pwrake/shell.rb +0 -186
  76. data/lib/pwrake/task_algorithm.rb +0 -475
  77. data/lib/pwrake/task_queue.rb +0 -633
  78. data/lib/pwrake/timer.rb +0 -22
@@ -0,0 +1,16 @@
1
+ module Pwrake
2
+
3
+ class Postprocess
4
+
5
+ def initialize(runner)
6
+ end
7
+
8
+ def run(filename)
9
+ []
10
+ end
11
+
12
+ def close
13
+ end
14
+
15
+ end
16
+ end
@@ -5,25 +5,25 @@ module Pwrake
5
5
 
6
6
  module MCGP
7
7
 
8
- def graph_partition(target=nil)
8
+ def graph_partition(host_map, target=nil)
9
9
  t1 = Time.now
10
- wgts = Pwrake.application.host_list.group_weight_sum
10
+ wgts = host_map.group_weight_sum
11
11
  if wgts.size > 1
12
12
  list = wgts.size.times.to_a
13
- g = GraphTracer.new([list],[wgts]){|t| t.group}
13
+ g = GraphTracerGroup.new([list],[wgts])
14
14
  trace(g,target)
15
- g.part_graph_group
15
+ g.part_graph
16
16
  #g.write_dot('dag1.dot')
17
17
  #return
18
18
  end
19
19
 
20
- #$debug2=true
20
+ #$debug=true
21
21
 
22
- list = Pwrake.application.host_list.group_hosts
23
- wgts = Pwrake.application.host_list.group_core_weight
24
- g = GraphTracer.new(list,wgts){|t| t.location}
22
+ list = host_map.group_hosts
23
+ wgts = host_map.group_core_weight
24
+ g = GraphTracerNode.new(list,wgts)
25
25
  trace(g,target)
26
- g.part_graph_node
26
+ g.part_graph
27
27
  t2 = Time.now
28
28
  Pwrake::Log.info "Time for TOTAL Graph Partitioning: #{t2-t1} sec"
29
29
  #g.write_dot('dag2.dot')
@@ -47,14 +47,13 @@ module Pwrake
47
47
 
48
48
  class GraphTracer
49
49
 
50
- def initialize(loc_list, weight_list, &block)
50
+ def initialize(loc_list, weight_list)
51
51
  if loc_list.size != weight_list.size
52
52
  raise ArgumentError, "array size of args mismatch"
53
53
  end
54
54
  @loc_list = loc_list
55
55
  @weight_list = weight_list
56
56
  @n_part = @loc_list.size
57
- @location_finder = block
58
57
  @traced = {}
59
58
  @vertex_depth = {}
60
59
  # @grviz = Grviz.new
@@ -63,26 +62,27 @@ module Pwrake
63
62
  end
64
63
  end
65
64
 
66
- def trace( name = "default", target = nil )
65
+ def trace(name="default", target=nil)
67
66
 
68
67
  task = Rake.application[name]
69
- group_id = task.group_id || 0
68
+ tw = task.wrapper
69
+ group_id = tw.group_id || 0
70
70
  group = @group_list[group_id]
71
- loc_list = @loc_list[group_id]
71
+ #loc_list = @loc_list[group_id]
72
72
  depth = 0
73
73
 
74
74
  if task.class == Rake::FileTask
75
- tgid = target ? (Rake.application[target].group_id||0) : nil
75
+ tgid = (target) ? (Rake.application[target].wrapper.group_id||0) : nil
76
76
 
77
77
  if File.file?(name)
78
78
  if tgid == group_id
79
- locs = @location_finder.call(task)
80
- if locs.empty?
81
- Pwrake.application.postprocess(task)
82
- locs = @location_finder.call(task)
83
- end
84
- task.get_file_stat
85
- fsz = task.file_size
79
+ locs = get_location(tw)
80
+ #if locs.empty?
81
+ # Pwrake.application.postprocess(task)
82
+ # locs = get_location(tw)
83
+ #end
84
+ #tw.get_file_stat
85
+ fsz = tw.file_size
86
86
  if fsz > 100000
87
87
  #puts "g=#{group_id}, task=#{name}, target=#{target}, fsz=#{fsz}, locs="+locs.join("|")
88
88
  group.push_loc_edge( locs, name, target, fsz/10000 )
@@ -119,23 +119,39 @@ module Pwrake
119
119
  return @vertex_depth[name]
120
120
  end
121
121
 
122
- def part_graph_group
122
+ def write_dot(file)
123
+ @grviz.write(file)
124
+ end
125
+ end
126
+
127
+
128
+ class GraphTracerGroup < GraphTracer
129
+
130
+ def get_location(tw)
131
+ tw.group
132
+ end
133
+
134
+ def part_graph
123
135
  @group_list.each do |g|
124
136
  g.part_graph
125
137
  g.set_group
126
138
  end
127
139
  end
140
+ end
141
+
142
+
143
+ class GraphTracerNode < GraphTracer
128
144
 
129
- def part_graph_node
145
+ def get_location(tw)
146
+ tw.location
147
+ end
148
+
149
+ def part_graph
130
150
  @group_list.each do |g|
131
151
  g.part_graph
132
152
  g.set_node
133
153
  end
134
154
  end
135
-
136
- def write_dot(file)
137
- @grviz.write(file)
138
- end
139
155
  end
140
156
 
141
157
 
@@ -296,7 +312,9 @@ module Pwrake
296
312
  if false
297
313
  puts "@vertex_id2name.size=#{@vertex_id2name.size}"
298
314
  if $debug2
299
- @vertex_id2name.each_with_index{|x,i| puts "#{i} #{x} #{@edges[i].inspect}"}
315
+ @vertex_id2name.each_with_index{|x,i|
316
+ puts "#{i} #{x} #{@edges[i].inspect}"
317
+ }
300
318
  end
301
319
  puts "@edges.size=#{@edges.size}"
302
320
  puts "ncon=#{c}"
@@ -328,22 +346,25 @@ module Pwrake
328
346
  s += "]"
329
347
  Log.info s
330
348
  options = RbMetis.default_options
331
- RbMetis::OPTION_NITER
332
349
  options[RbMetis::OPTION_NCUTS] = 30
333
350
  options[RbMetis::OPTION_NSEPS] = 30
334
351
  options[RbMetis::OPTION_NITER] = 10
335
- @part = RbMetis.part_graph_recursive(@xadj, @adjcny, @n_part, ncon:c, vwgt:@vwgt, adjwgt:@adjwgt, tpwgts:tpw, ubvec:ubv, options:options)
352
+ @part = RbMetis.part_graph_recursive(
353
+ @xadj, @adjcny, @n_part,
354
+ ncon:c, vwgt:@vwgt, adjwgt:@adjwgt,
355
+ tpwgts:tpw, ubvec:ubv, options:options)
336
356
  else
337
357
  puts "tpw=#{tpw.inspect}"
338
- @part = Metis.mc_part_graph_recursive2(c, @xadj,@adjcny, @vwgt,nil, @tpwgts)
358
+ @part = Metis.mc_part_graph_recursive2(
359
+ c,@xadj,@adjcny,@vwgt,nil,@tpwgts)
339
360
  end
340
361
  t2 = Time.now
341
362
  Pwrake::Log.info "Time for Graph Partitioning: #{t2-t1} sec"
342
363
  count_partition
343
- #puts "Time for Graph Partitioning: #{t2-t1} sec"
344
- #if $debug
345
- #p @part
346
- #end
364
+ if $debug
365
+ puts "Time for Graph Partitioning: #{t2-t1} sec"
366
+ p @part
367
+ end
347
368
  end
348
369
 
349
370
  def count_partition
@@ -411,15 +432,15 @@ module Pwrake
411
432
  @vertex_id2name.each_with_index do |name,idx|
412
433
  if idx >= @n_part
413
434
  i_part = @part[idx]
414
- task = Rake.application[name]
415
- task.group_id = loc_list[i_part]
435
+ tw = Rake.application[name].wrapper
436
+ tw.group_id = loc_list[i_part]
416
437
  #puts "task=#{task.inspect}, i_part=#{i_part}, host=#{host}"
417
438
  end
418
439
  end
419
440
  @loc_files.each do |gid,files|
420
441
  files.each do |f|
421
- task = Rake.application[f]
422
- task.group_id = gid
442
+ tw = Rake.application[f].wrapper
443
+ tw.group_id = gid
423
444
  # puts "gid=#{gid}, task=#{f}"
424
445
  end
425
446
  end
@@ -431,9 +452,9 @@ module Pwrake
431
452
  @vertex_id2name.each_with_index do |name,idx|
432
453
  if idx >= @n_part
433
454
  i_part = @part[idx]
434
- task = Rake.application[name]
455
+ tw = Rake.application[name].wrapper
435
456
  host = loc_list[i_part]
436
- task.suggest_location = [host]
457
+ tw.suggest_location = [host]
437
458
  #puts "task=#{task.inspect}, i_part=#{i_part}, host=#{host}"
438
459
  end
439
460
  end
@@ -0,0 +1,158 @@
1
+ module Pwrake
2
+
3
+ class HostInfo
4
+
5
+ def initialize(name,id,ncore,weight,group=nil)
6
+ @name = name
7
+ @ncore = ncore
8
+ @weight = weight || 1.0
9
+ @group = group || 0
10
+ @id = id
11
+ end
12
+
13
+ attr_reader :name, :ncore, :weight, :group, :id, :steal_flag
14
+ attr_accessor :idle_cores
15
+
16
+ def set_ncore(n)
17
+ @ncore = @idle_cores = n
18
+ end
19
+
20
+ def increase(n)
21
+ @idle_cores += n
22
+ end
23
+
24
+ def decrease(n)
25
+ @idle_cores -= n
26
+ if @idle_cores < 0
27
+ raise RuntimeError,"# of cores must be non-negative"
28
+ end
29
+ end
30
+
31
+ def steal_phase
32
+ @steal_flag = true
33
+ t = yield(self)
34
+ @steal_flag = false
35
+ t
36
+ end
37
+ end
38
+
39
+ class HostMap < Hash
40
+
41
+ def initialize(arg=nil)
42
+ @host_map = {}
43
+ @by_id = []
44
+ @by_name = {}
45
+ require "socket"
46
+ case arg
47
+ when /\.yaml$/
48
+ read_yaml(arg)
49
+ when String
50
+ read_host(arg)
51
+ when Integer
52
+ parse_hosts(["localhost #{arg}"])
53
+ when NilClass
54
+ parse_hosts(["localhost 1"])
55
+ else
56
+ raise ArgumentError, "arg=#{arg.inspect}"
57
+ #@num_threads = 1 if !@num_threads
58
+ #@core_list = ['localhost'] * @num_threads
59
+ end
60
+ end
61
+ attr_reader :by_id, :by_name
62
+
63
+ def host_count
64
+ @by_id.size
65
+ end
66
+
67
+ def group_hosts
68
+ a = []
69
+ self.each do |sub,list|
70
+ list.each{|h| (a[h.group] ||= []) << h.name}
71
+ end
72
+ a
73
+ end
74
+
75
+ def group_core_weight
76
+ a = []
77
+ self.each do |sub,list|
78
+ list.each{|h| (a[h.group] ||= []) << h.weight}
79
+ end
80
+ a
81
+ end
82
+
83
+ def group_weight_sum
84
+ a = []
85
+ self.each do |sub,list|
86
+ list.each{|h| a[h.group] = (a[h.group]||0) + h.weight}
87
+ end
88
+ a
89
+ end
90
+
91
+ private
92
+
93
+ def read_host(file)
94
+ ary = []
95
+ File.open(file) do |f|
96
+ while l = f.gets
97
+ ary << l
98
+ end
99
+ end
100
+ parse_hosts(ary)
101
+ end
102
+
103
+ def read_yaml(file)
104
+ parse_hosts(YAML.load(open(file))[0])
105
+ end
106
+
107
+ def parse_hosts(hosts)
108
+ #p hosts
109
+ if hosts.kind_of? Array
110
+ hosts = {"localhost"=>hosts}
111
+ end
112
+ hosts.each do |branch, list|
113
+ self[branch] = parse_list(list)
114
+ end
115
+ end
116
+
117
+ def parse_list(line_list)
118
+ info_list = []
119
+ line_list.each do |line|
120
+ parse_line(info_list,line)
121
+ end
122
+ info_list
123
+ end
124
+
125
+ def parse_line(info_list,line)
126
+ line = $1 if /^([^#]*)#/ =~ line
127
+ host, ncore, weight, group = line.split
128
+ if host
129
+ if /\[\[([\w\d]+)-([\w\d]+)\]\]/o =~ host
130
+ hosts = ($1..$2).map{|i| host.sub(re,i)}
131
+ else
132
+ hosts = [host]
133
+ end
134
+ hosts.each do |host|
135
+ begin
136
+ host = Socket.gethostbyname(host)[0]
137
+ rescue
138
+ Log.warn "FQDN not resoved : #{host}"
139
+ end
140
+ ncore &&= ncore.to_i
141
+ weitht &&= weight.to_i
142
+ #weight = (weight || 1).to_f
143
+ group &&= group.to_i
144
+ if host_info = @by_name[host]
145
+ raise RuntimeError,"duplicated hostname: #{host}"
146
+ else
147
+ id = @by_id.size
148
+ host_info = HostInfo.new(host,id,ncore,weight,group)
149
+ @by_name[host] = host_info
150
+ info_list << host_info
151
+ @by_id << host_info
152
+ end
153
+ end
154
+ end
155
+ end
156
+
157
+ end
158
+ end
@@ -0,0 +1,357 @@
1
+ module Pwrake
2
+
3
+ START_TIME = Time.now
4
+
5
+ class Option < Hash
6
+
7
+ def initialize
8
+ load_pwrake_conf
9
+ init_options
10
+ init_pass_env
11
+ if self['SHOW_CONF']
12
+ require "yaml"
13
+ YAML.dump(self,$stdout)
14
+ exit
15
+ elsif self['REPORT_DIR']
16
+ require 'pwrake/report'
17
+ Report.new(self,[]).report_html
18
+ exit
19
+ end
20
+ end
21
+
22
+ def init
23
+ Log.info "Options:"
24
+ self.each do |k,v|
25
+ Log.info " #{k} = #{v.inspect}"
26
+ end
27
+ #@counter = Counter.new
28
+ setup_hosts
29
+ setup_filesystem # require 'option_filesystem.rb'
30
+ #
31
+ if self['LOG_DIR'] && self['GC_LOG_FILE']
32
+ GC::Profiler.enable
33
+ end
34
+ end
35
+
36
+ attr_reader :counter
37
+ attr_reader :logger
38
+
39
+ DEFAULT_CONFFILES = ["pwrake_conf.yaml","PwrakeConf.yaml"]
40
+
41
+ # ----- init -----
42
+
43
+ def load_pwrake_conf
44
+ # Read pwrake_conf
45
+ pwrake_conf = Rake.application.options.pwrake_conf
46
+ if pwrake_conf
47
+ if !File.exist?(pwrake_conf)
48
+ raise "Configuration file not found: #{pwrake_conf}"
49
+ end
50
+ else
51
+ pwrake_conf = DEFAULT_CONFFILES.find{|fn| File.exist?(fn)}
52
+ end
53
+ self['PWRAKE_CONF'] = pwrake_conf
54
+ if pwrake_conf.nil?
55
+ @yaml = {}
56
+ else
57
+ #Log.debug "load pwrake_conf=#{pwrake_conf}"
58
+ require "yaml"
59
+ @yaml = open(pwrake_conf){|f| YAML.load(f) }
60
+ end
61
+ end
62
+
63
+ def init_options
64
+ option_data.each do |a|
65
+ prc = nil
66
+ keys = []
67
+ case a
68
+ when String
69
+ keys << a
70
+ when Array
71
+ a.each do |x|
72
+ case x
73
+ when String
74
+ keys << x
75
+ when Proc
76
+ prc = x
77
+ end
78
+ end
79
+ end
80
+ key = keys[0]
81
+ val = search_opts(keys)
82
+ val = prc.call(val) if prc
83
+ self[key] = val if !val.nil?
84
+ instance_variable_set("@"+key.downcase, val)
85
+ end
86
+
87
+ feedback_options
88
+
89
+ Rake.verbose(false) if Rake.application.options.silent
90
+ end
91
+
92
+ def option_data
93
+ [
94
+ 'DRYRUN',
95
+ 'IGNORE_SYSTEM',
96
+ 'IGNORE_DEPRECATE',
97
+ 'LOAD_SYSTEM',
98
+ 'NOSEARCH',
99
+ 'RAKELIB',
100
+ 'SHOW_PREREQS',
101
+ 'SILENT',
102
+ 'TRACE',
103
+ 'BACKTRACE',
104
+ 'TRACE_OUTPUT',
105
+ 'TRACE_RULES',
106
+
107
+ 'FILESYSTEM',
108
+ 'SSH_OPTION',
109
+ 'PASS_ENV',
110
+ ['SHELL_COMMAND', proc{|v| v||ENV['SHELL']}],
111
+ ['SHELL_RC','SHELLRC'],
112
+ 'GFARM2FS_OPTION',
113
+ 'GFARM2FS_DEBUG',
114
+ ['GFARM2FS_DEBUG_WAIT', proc{|v| v ? v.to_i : 1}],
115
+ 'GNU_TIME',
116
+ 'DEBUG',
117
+ 'PLOT_PARALLELISM',
118
+ 'SHOW_CONF',
119
+ ['REPORT_DIR','REPORT'],
120
+ 'FAILED_TARGET', # rename(default), delete, leave
121
+ 'FAILURE_TERMINATION', # wait, kill, continue
122
+ 'QUEUE_PRIORITY', # RANK(default), FIFO, LIFO, DFS
123
+ 'NOACTION_QUEUE_PRIORITY', # FIFO(default), LIFO, RAND
124
+ #'NUM_NOACTION_THREADS', # default=4 when gfarm, else 1
125
+ 'GRAPH_PARTITION',
126
+ 'PLOT_PARTITION',
127
+
128
+ ['HOSTFILE','HOSTS'],
129
+ ['LOG_DIR','LOG',
130
+ proc{|v|
131
+ if v
132
+ if v == "" || !v.kind_of?(String)
133
+ v = "Pwrake%Y%m%d-%H%M%S"
134
+ end
135
+ d = v = format_time_pid(v)
136
+ i = 1
137
+ while File.exist?(d)
138
+ d = "#{v}.#{i}"
139
+ i += 1
140
+ end
141
+ d
142
+ end
143
+ }],
144
+ ['LOG_FILE',
145
+ proc{|v|
146
+ if v.kind_of?(String) && v != ""
147
+ v
148
+ else
149
+ "pwrake.log"
150
+ end
151
+ }],
152
+ ['TASK_CSV_FILE',
153
+ proc{|v|
154
+ if v.kind_of?(String) && v != ""
155
+ v
156
+ else
157
+ "task.csv"
158
+ end
159
+ }],
160
+ ['COMMAND_CSV_FILE',
161
+ proc{|v|
162
+ if v.kind_of?(String) && v != ""
163
+ v
164
+ else
165
+ "command.csv"
166
+ end
167
+ }],
168
+ ['GC_LOG_FILE',
169
+ proc{|v|
170
+ if v
171
+ if v.kind_of?(String) && v != ""
172
+ v
173
+ else
174
+ "gc.log"
175
+ end
176
+ end
177
+ }],
178
+ ['NUM_THREADS', proc{|v| v && v.to_i}],
179
+ ['SHELL_START_INTERVAL', proc{|v| (v || 0.012).to_f}],
180
+ ['HEARTBEAT', proc{|v| (v || 240).to_i}],
181
+ ['DISABLE_AFFINITY', proc{|v| v || ENV['AFFINITY']=='off'}],
182
+ ['DISABLE_STEAL', proc{|v| v || ENV['STEAL']=='off'}],
183
+ ['GFARM_BASEDIR', proc{|v| v || '/tmp'}],
184
+ ['GFARM_PREFIX', proc{|v| v || "pwrake_#{ENV['USER']}"}],
185
+ ['GFARM_SUBDIR', proc{|v| v || '/'}],
186
+ ['MAX_GFWHERE_WORKER', proc{|v| (v || 8).to_i}],
187
+ ['MASTER_HOSTNAME', proc{|v| (v || begin;`hostname -f`;rescue;end || '').chomp}],
188
+ ['WORK_DIR', proc{|v|
189
+ v ||= '%CWD_RELATIVE_TO_HOME'
190
+ v.sub('%CWD_RELATIVE_TO_HOME',cwd_relative_to_home)
191
+ }],
192
+ ]
193
+ end
194
+
195
+ def format_time_pid(v)
196
+ START_TIME.strftime(v).sub("%$","%05d"%Process.pid)
197
+ end
198
+
199
+ def feedback_options
200
+ opts = Rake.application.options
201
+ ['DRYRUN',
202
+ 'IGNORE_SYSTEM',
203
+ 'IGNORE_DEPRECATE',
204
+ 'LOAD_SYSTEM',
205
+ 'NOSEARCH',
206
+ 'RAKELIB',
207
+ 'SHOW_PREREQS',
208
+ 'SILENT',
209
+ 'TRACE',
210
+ 'BACKTRACE',
211
+ 'TRACE_OUTPUT',
212
+ 'TRACE_RULES'
213
+ ].each do |k|
214
+ if v=self[k]
215
+ m = (k.downcase+"=").to_sym
216
+ opts.send(m,v)
217
+ end
218
+ end
219
+ case opts.trace_output
220
+ when 'stdout'
221
+ opts.trace_output = $stdout
222
+ when 'stderr', nil
223
+ opts.trace_output = $stderr
224
+ end
225
+ end
226
+
227
+ # Priority of Option:
228
+ # command_option > ENV > pwrake_conf > DEFAULT_OPTIONS
229
+ def search_opts(keys)
230
+ val = Rake.application.options.send(keys[0].downcase.to_sym)
231
+ return parse_opt(val) if !val.nil?
232
+ #
233
+ keys.each do |k|
234
+ val = ENV[k.upcase]
235
+ return parse_opt(val) if !val.nil?
236
+ end
237
+ #
238
+ return nil if !@yaml
239
+ keys.each do |k|
240
+ val = @yaml[k.upcase]
241
+ return val if !val.nil?
242
+ end
243
+ nil
244
+ end
245
+
246
+ def parse_opt(s)
247
+ case s
248
+ when /^(false|nil|off)$/i
249
+ false
250
+ when /^(true|on)$/i
251
+ true
252
+ when $stdout
253
+ "stdout"
254
+ when $stderr
255
+ "stderr"
256
+ else
257
+ s
258
+ end
259
+ end
260
+
261
+ def cwd_relative_to_home
262
+ Pathname.pwd.relative_path_from(Pathname.new(ENV['HOME'])).to_s
263
+ end
264
+
265
+ def cwd_relative_if_under_home
266
+ home = Pathname.new(ENV['HOME']).realpath
267
+ path = pwd = Pathname.pwd.realpath
268
+ while path != home
269
+ if path.root?
270
+ return pwd.to_s
271
+ end
272
+ path = path.parent
273
+ end
274
+ return pwd.relative_path_from(home).to_s
275
+ end
276
+
277
+ # ------------------------------------------------------------------------
278
+
279
+ def init_pass_env
280
+ if envs = self['PASS_ENV']
281
+ pass_env = {}
282
+
283
+ case envs
284
+ when Array
285
+ envs.each do |k|
286
+ k = k.to_s
287
+ if v = ENV[k]
288
+ pass_env[k] = v
289
+ end
290
+ end
291
+ when Hash
292
+ envs.each do |k,v|
293
+ k = k.to_s
294
+ if v = ENV[k] || v
295
+ pass_env[k] = v
296
+ end
297
+ end
298
+ else
299
+ raise "invalid option for PASS_ENV in pwrake_conf.yaml"
300
+ end
301
+
302
+ if pass_env.empty?
303
+ self.delete('PASS_ENV')
304
+ else
305
+ self['PASS_ENV'] = pass_env
306
+ end
307
+ end
308
+ end
309
+
310
+
311
+ def setup_hosts
312
+ if @hostfile && @num_threads
313
+ raise "Cannot set `hostfile' and `num_threads' simultaneously"
314
+ end
315
+ @host_map = HostMap.new(@hostfile || @num_threads)
316
+ end
317
+ attr_reader :host_map
318
+
319
+
320
+ def clear_gfarm2fs
321
+ setup_hosts
322
+ d = File.join(self['GFARM_BASEDIR'],self['GFARM_PREFIX'])
323
+ rcmd = "
324
+ for i in #{d}*; do
325
+ if [ -d \"$i\" ]; then
326
+ case \"$i\" in
327
+ *_000) ;;
328
+ *) fusermount -u $i; rmdir $i ;;
329
+ esac
330
+ fi
331
+ done
332
+ sleep 1
333
+ for i in #{d}*_000; do
334
+ if [ -d \"$i\" ]; then
335
+ fusermount -u $i; rmdir $i
336
+ fi
337
+ done
338
+ "
339
+ threads = []
340
+ @host_map.each do |k,hosts|
341
+ hosts.each do |info|
342
+ threads << Thread.new do
343
+ system "ssh #{info.name} '#{rcmd}'"
344
+ end
345
+ end
346
+ end
347
+ threads.each{|t| t.join}
348
+ end
349
+
350
+ # ----- finish -----
351
+
352
+ def finish_option
353
+ Log.close
354
+ end
355
+
356
+ end
357
+ end