pwrake 0.9.9.2 → 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/CHANGES_V2.md +90 -0
- data/{LICENSE.txt → MIT-LICENSE} +2 -3
- data/README +12 -0
- data/README.md +75 -52
- data/bin/gfwhere-pipe +23 -12
- data/bin/pwrake +22 -29
- data/bin/pwrake_branch +24 -0
- data/lib/pwrake/branch.rb +22 -0
- data/lib/pwrake/branch/branch.rb +213 -0
- data/lib/pwrake/branch/branch_application.rb +53 -0
- data/lib/pwrake/branch/fiber_queue.rb +36 -0
- data/lib/pwrake/branch/file_utils.rb +101 -0
- data/lib/pwrake/branch/shell.rb +231 -0
- data/lib/pwrake/{profiler.rb → branch/shell_profiler.rb} +28 -27
- data/lib/pwrake/branch/worker_communicator.rb +104 -0
- data/lib/pwrake/{gfarm_feature.rb → gfarm/gfarm_path.rb} +2 -100
- data/lib/pwrake/gfarm/gfarm_postprocess.rb +53 -0
- data/lib/pwrake/iomux/channel.rb +70 -0
- data/lib/pwrake/iomux/handler.rb +124 -0
- data/lib/pwrake/iomux/handler_set.rb +35 -0
- data/lib/pwrake/iomux/runner.rb +62 -0
- data/lib/pwrake/logger.rb +3 -150
- data/lib/pwrake/master.rb +30 -137
- data/lib/pwrake/master/fiber_pool.rb +69 -0
- data/lib/pwrake/master/idle_cores.rb +30 -0
- data/lib/pwrake/master/master.rb +345 -0
- data/lib/pwrake/master/master_application.rb +150 -0
- data/lib/pwrake/master/postprocess.rb +16 -0
- data/lib/pwrake/{graphviz.rb → misc/graphviz.rb} +0 -0
- data/lib/pwrake/{mcgp.rb → misc/mcgp.rb} +63 -42
- data/lib/pwrake/option/host_map.rb +158 -0
- data/lib/pwrake/option/option.rb +357 -0
- data/lib/pwrake/option/option_filesystem.rb +112 -0
- data/lib/pwrake/queue/locality_aware_queue.rb +158 -0
- data/lib/pwrake/queue/no_action_queue.rb +67 -0
- data/lib/pwrake/queue/queue_array.rb +366 -0
- data/lib/pwrake/queue/task_queue.rb +164 -0
- data/lib/pwrake/report.rb +1 -0
- data/lib/pwrake/report/parallelism.rb +9 -3
- data/lib/pwrake/report/report.rb +50 -103
- data/lib/pwrake/report/task_stat.rb +83 -0
- data/lib/pwrake/task/task_algorithm.rb +107 -0
- data/lib/pwrake/task/task_manager.rb +32 -0
- data/lib/pwrake/task/task_property.rb +98 -0
- data/lib/pwrake/task/task_rank.rb +48 -0
- data/lib/pwrake/task/task_wrapper.rb +296 -0
- data/lib/pwrake/version.rb +1 -1
- data/lib/pwrake/worker/executor.rb +169 -0
- data/lib/pwrake/worker/gfarm_directory.rb +90 -0
- data/lib/pwrake/worker/invoker.rb +199 -0
- data/lib/pwrake/worker/load.rb +14 -0
- data/lib/pwrake/worker/log_executor.rb +73 -0
- data/lib/pwrake/worker/shared_directory.rb +74 -0
- data/lib/pwrake/worker/worker_main.rb +14 -0
- data/lib/pwrake/worker/writer.rb +59 -0
- data/setup.rb +1212 -1502
- data/spec/003/Rakefile +2 -2
- data/spec/008/Rakefile +2 -1
- data/spec/009/Rakefile +1 -1
- data/spec/009/pwrake_conf.yaml +1 -3
- data/spec/hosts +0 -2
- data/spec/pwrake_spec.rb +9 -8
- metadata +50 -21
- data/lib/pwrake.rb +0 -19
- data/lib/pwrake/application.rb +0 -232
- data/lib/pwrake/counter.rb +0 -54
- data/lib/pwrake/file_utils.rb +0 -98
- data/lib/pwrake/gfwhere_pool.rb +0 -109
- data/lib/pwrake/host_list.rb +0 -88
- data/lib/pwrake/locality_aware_queue.rb +0 -413
- data/lib/pwrake/option.rb +0 -400
- data/lib/pwrake/rake_modify.rb +0 -14
- data/lib/pwrake/shell.rb +0 -186
- data/lib/pwrake/task_algorithm.rb +0 -475
- data/lib/pwrake/task_queue.rb +0 -633
- data/lib/pwrake/timer.rb +0 -22
data/lib/pwrake/file_utils.rb
DELETED
@@ -1,98 +0,0 @@
|
|
1
|
-
module Pwrake
|
2
|
-
module FileUtils
|
3
|
-
module_function
|
4
|
-
|
5
|
-
def sh(*cmd, &block)
|
6
|
-
options = (Hash === cmd.last) ? cmd.pop : {}
|
7
|
-
unless block_given?
|
8
|
-
show_command = cmd.join(" ")
|
9
|
-
show_command = show_command[0,42] + "..."
|
10
|
-
block = lambda { |ok, status|
|
11
|
-
ok or fail "Command failed with status (#{status.exitstatus}): [#{show_command}]"
|
12
|
-
}
|
13
|
-
end
|
14
|
-
if RakeFileUtils.verbose_flag == :default
|
15
|
-
options[:verbose] = true
|
16
|
-
else
|
17
|
-
options[:verbose] ||= RakeFileUtils.verbose_flag
|
18
|
-
end
|
19
|
-
options[:noop] ||= RakeFileUtils.nowrite_flag
|
20
|
-
Rake.rake_check_options options, :noop, :verbose
|
21
|
-
Pwrake::Log.stderr_puts cmd.join(" ") if options[:verbose]
|
22
|
-
unless options[:noop]
|
23
|
-
res,status = Pwrake::FileUtils.pwrake_system(*cmd)
|
24
|
-
block.call(res, status)
|
25
|
-
end
|
26
|
-
end
|
27
|
-
|
28
|
-
def bq(*cmd, &block)
|
29
|
-
options = (Hash === cmd.last) ? cmd.pop : {}
|
30
|
-
unless block_given?
|
31
|
-
show_command = cmd.join(" ")
|
32
|
-
show_command = show_command[0,42] + "..."
|
33
|
-
block = lambda { |ok, status|
|
34
|
-
ok or fail "Command failed with status (#{status.exitstatus}): [#{show_command}]"
|
35
|
-
}
|
36
|
-
end
|
37
|
-
if RakeFileUtils.verbose_flag == :default
|
38
|
-
options[:verbose] = true
|
39
|
-
else
|
40
|
-
options[:verbose] ||= RakeFileUtils.verbose_flag
|
41
|
-
end
|
42
|
-
options[:noop] ||= RakeFileUtils.nowrite_flag
|
43
|
-
Rake.rake_check_options options, :noop, :verbose
|
44
|
-
Pwrake::Log.stderr_puts cmd.join(" ") if options[:verbose]
|
45
|
-
unless options[:noop]
|
46
|
-
res,status = Pwrake::FileUtils.pwrake_backquote(*cmd)
|
47
|
-
block.call(res, status)
|
48
|
-
end
|
49
|
-
res
|
50
|
-
end
|
51
|
-
|
52
|
-
def pwrake_system(*cmd)
|
53
|
-
cmd_log = cmd.join(" ").inspect
|
54
|
-
tm = Pwrake::Timer.new("sh",cmd_log)
|
55
|
-
|
56
|
-
conn = Pwrake.current_shell
|
57
|
-
if conn.kind_of?(Pwrake::Shell)
|
58
|
-
res = conn.system(*cmd)
|
59
|
-
status = Rake::PseudoStatus.new(conn.status)
|
60
|
-
else
|
61
|
-
res = system(*cmd)
|
62
|
-
status = $?
|
63
|
-
status = Rake::PseudoStatus.new(1) if !res && status.nil?
|
64
|
-
end
|
65
|
-
|
66
|
-
tm.finish("status=%s cmd=%s"%[status.exitstatus,cmd_log])
|
67
|
-
[res,status]
|
68
|
-
end
|
69
|
-
|
70
|
-
# Pwrake version of backquote command
|
71
|
-
def pwrake_backquote(cmd)
|
72
|
-
cmd_log = cmd.inspect
|
73
|
-
tm = Pwrake::Timer.new("bq",cmd_log)
|
74
|
-
|
75
|
-
conn = Pwrake.current_shell
|
76
|
-
if conn.kind_of?(Pwrake::Shell)
|
77
|
-
res = conn.backquote(*cmd)
|
78
|
-
status = Rake::PseudoStatus.new(conn.status)
|
79
|
-
else
|
80
|
-
res = `#{cmd}`
|
81
|
-
status = $?
|
82
|
-
status = Rake::PseudoStatus.new(1) if status.nil?
|
83
|
-
end
|
84
|
-
|
85
|
-
tm.finish("status=%s cmd=%s"%[status.exitstatus,cmd_log])
|
86
|
-
[res,status]
|
87
|
-
end
|
88
|
-
|
89
|
-
end # module Pwrake::FileUtils
|
90
|
-
end
|
91
|
-
|
92
|
-
module Rake
|
93
|
-
module DSL
|
94
|
-
include Pwrake::FileUtils
|
95
|
-
private(*Pwrake::FileUtils.instance_methods(false))
|
96
|
-
end
|
97
|
-
end
|
98
|
-
self.extend Rake::DSL
|
data/lib/pwrake/gfwhere_pool.rb
DELETED
@@ -1,109 +0,0 @@
|
|
1
|
-
module Pwrake
|
2
|
-
|
3
|
-
class WorkerPool
|
4
|
-
|
5
|
-
def initialize(wk_class, max)
|
6
|
-
@worker_class = wk_class
|
7
|
-
@max = max
|
8
|
-
@pool = []
|
9
|
-
@cond_pool = ConditionVariable.new
|
10
|
-
@mutex = Mutex.new
|
11
|
-
end
|
12
|
-
|
13
|
-
def find_worker
|
14
|
-
@mutex.synchronize do
|
15
|
-
while true
|
16
|
-
@pool.each do |w|
|
17
|
-
return w if w.acquire
|
18
|
-
end
|
19
|
-
if @pool.size < @max
|
20
|
-
Log.debug "--- #{@worker_class}:new_worker #{@pool.size+1}"
|
21
|
-
w = @worker_class.new(@cond_pool)
|
22
|
-
@pool << w
|
23
|
-
return w if w.acquire
|
24
|
-
end
|
25
|
-
# wait for end of work in @pool
|
26
|
-
@cond_pool.wait(@mutex)
|
27
|
-
end
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
def work(*args)
|
32
|
-
w = find_worker
|
33
|
-
w.run(*args)
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
|
38
|
-
class Worker
|
39
|
-
|
40
|
-
def initialize(cond_pool)
|
41
|
-
@cond_pool = cond_pool
|
42
|
-
@mutex = Mutex.new
|
43
|
-
@aquired = false
|
44
|
-
end
|
45
|
-
|
46
|
-
def acquire
|
47
|
-
return false if @mutex.locked?
|
48
|
-
if @aquired
|
49
|
-
return false
|
50
|
-
else
|
51
|
-
@aquired = true
|
52
|
-
return true
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
def run(*args)
|
57
|
-
@mutex.synchronize do
|
58
|
-
raise "no aquired" unless @aquired
|
59
|
-
r = work(*args)
|
60
|
-
@aquired = false
|
61
|
-
@cond_pool.signal
|
62
|
-
return r
|
63
|
-
end
|
64
|
-
end
|
65
|
-
|
66
|
-
def work(*args)
|
67
|
-
# inplement in subclass
|
68
|
-
return nil
|
69
|
-
end
|
70
|
-
end
|
71
|
-
|
72
|
-
|
73
|
-
class GfwhereWorker < Worker
|
74
|
-
|
75
|
-
def initialize(cond_pool)
|
76
|
-
super(cond_pool)
|
77
|
-
@io = IO.popen('gfwhere-pipe','r+')
|
78
|
-
@io.sync = true
|
79
|
-
end
|
80
|
-
|
81
|
-
def work(file)
|
82
|
-
return [] if file==''
|
83
|
-
t = Time.now
|
84
|
-
@io.puts(file)
|
85
|
-
@io.flush
|
86
|
-
s = @io.gets
|
87
|
-
if s.nil?
|
88
|
-
raise "gfwhere: unexpected end"
|
89
|
-
end
|
90
|
-
s.chomp!
|
91
|
-
if s != file
|
92
|
-
raise "gfwhere: file=#{file}, result=#{s}"
|
93
|
-
end
|
94
|
-
while s = @io.gets
|
95
|
-
s.chomp!
|
96
|
-
case s
|
97
|
-
when /^gfarm:\/\//
|
98
|
-
next
|
99
|
-
when /^Error:/
|
100
|
-
return []
|
101
|
-
else
|
102
|
-
Log.debug "gfwhere:path %.6f sec, file=%s" % [Time.now-t,file]
|
103
|
-
return s.split(/\s+/)
|
104
|
-
end
|
105
|
-
end
|
106
|
-
end
|
107
|
-
end
|
108
|
-
|
109
|
-
end
|
data/lib/pwrake/host_list.rb
DELETED
@@ -1,88 +0,0 @@
|
|
1
|
-
module Pwrake
|
2
|
-
|
3
|
-
class HostInfo
|
4
|
-
def initalize(name,group=0,weight=1)
|
5
|
-
@name = name
|
6
|
-
@group = group
|
7
|
-
@weight = weight
|
8
|
-
end
|
9
|
-
attr_reader :name, :group, :weight
|
10
|
-
end
|
11
|
-
|
12
|
-
class HostList
|
13
|
-
attr_reader :group_hosts
|
14
|
-
attr_reader :group_core_weight
|
15
|
-
attr_reader :group_weight_sum
|
16
|
-
attr_reader :host2group
|
17
|
-
attr_reader :num_threads
|
18
|
-
attr_reader :core_list
|
19
|
-
attr_reader :host_count
|
20
|
-
|
21
|
-
def initialize(file=nil)
|
22
|
-
@file = file
|
23
|
-
@group_hosts = []
|
24
|
-
@group_core_weight = []
|
25
|
-
@group_weight_sum = []
|
26
|
-
@host2group = {}
|
27
|
-
require "socket"
|
28
|
-
if @file
|
29
|
-
read_host(@file)
|
30
|
-
@num_threads = @core_list.size
|
31
|
-
else
|
32
|
-
@num_threads = 1 if !@num_threads
|
33
|
-
@core_list = ['localhost'] * @num_threads
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
def size
|
38
|
-
@num_threads
|
39
|
-
end
|
40
|
-
|
41
|
-
def read_host(file)
|
42
|
-
tmplist = []
|
43
|
-
File.open(file) do |f|
|
44
|
-
re = /\[\[([\w\d]+)-([\w\d]+)\]\]/o
|
45
|
-
while l = f.gets
|
46
|
-
l = $1 if /^([^#]*)#/ =~ l
|
47
|
-
host, ncore, weight, group = l.split
|
48
|
-
if host
|
49
|
-
if re =~ host
|
50
|
-
hosts = ($1..$2).map{|i| host.sub(re,i)}
|
51
|
-
else
|
52
|
-
hosts = [host]
|
53
|
-
end
|
54
|
-
hosts.each do |host|
|
55
|
-
begin
|
56
|
-
host = Socket.gethostbyname(host)[0]
|
57
|
-
rescue
|
58
|
-
Log.info "-- FQDN not resoved : #{host}"
|
59
|
-
end
|
60
|
-
ncore = (ncore || 1).to_i
|
61
|
-
weight = (weight || 1).to_f
|
62
|
-
w = ncore * weight
|
63
|
-
group = (group || 0).to_i
|
64
|
-
tmplist << ([host] * ncore.to_i)
|
65
|
-
(@group_hosts[group] ||= []) << host
|
66
|
-
(@group_core_weight[group] ||= []) << w
|
67
|
-
@group_weight_sum[group] = (@group_weight_sum[group]||0) + w
|
68
|
-
@host2group[host] = group
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
|
-
|
74
|
-
@core_list = []
|
75
|
-
begin # alternative order
|
76
|
-
sz = 0
|
77
|
-
tmplist.each do |a|
|
78
|
-
@core_list << a.shift if !a.empty?
|
79
|
-
sz += a.size
|
80
|
-
end
|
81
|
-
end while sz>0
|
82
|
-
|
83
|
-
@host_count = Hash.new{0}
|
84
|
-
core_list.each{|h| @host_count[h] += 1}
|
85
|
-
end
|
86
|
-
|
87
|
-
end
|
88
|
-
end
|
@@ -1,413 +0,0 @@
|
|
1
|
-
module Pwrake
|
2
|
-
|
3
|
-
module TaskAlgorithm
|
4
|
-
def assigned
|
5
|
-
@assigned ||= []
|
6
|
-
end
|
7
|
-
end
|
8
|
-
|
9
|
-
# copy from ruby's thread.rb
|
10
|
-
class ConditionVariable
|
11
|
-
#
|
12
|
-
# Creates a new ConditionVariable
|
13
|
-
#
|
14
|
-
def initialize
|
15
|
-
@waiters = {}
|
16
|
-
@waiters_mutex = Mutex.new
|
17
|
-
end
|
18
|
-
|
19
|
-
#
|
20
|
-
# Releases the lock held in +mutex+ and waits; reacquires the lock on wakeup.
|
21
|
-
#
|
22
|
-
# If +timeout+ is given, this method returns after +timeout+ seconds passed,
|
23
|
-
# even if no other thread doesn't signal.
|
24
|
-
#
|
25
|
-
def wait(mutex, timeout=nil)
|
26
|
-
Thread.handle_interrupt(StandardError => :never) do
|
27
|
-
begin
|
28
|
-
Thread.handle_interrupt(StandardError => :on_blocking) do
|
29
|
-
@waiters_mutex.synchronize do
|
30
|
-
@waiters[Thread.current] = true
|
31
|
-
end
|
32
|
-
mutex.sleep timeout
|
33
|
-
end
|
34
|
-
ensure
|
35
|
-
@waiters_mutex.synchronize do
|
36
|
-
@waiters.delete(Thread.current)
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
40
|
-
self
|
41
|
-
end
|
42
|
-
|
43
|
-
#
|
44
|
-
# Wakes up the first thread in line waiting for this lock.
|
45
|
-
#
|
46
|
-
def signal
|
47
|
-
Thread.handle_interrupt(StandardError => :on_blocking) do
|
48
|
-
begin
|
49
|
-
t, _ = @waiters_mutex.synchronize { @waiters.shift }
|
50
|
-
t.run if t
|
51
|
-
rescue ThreadError
|
52
|
-
retry # t was already dead?
|
53
|
-
end
|
54
|
-
end
|
55
|
-
self
|
56
|
-
end
|
57
|
-
|
58
|
-
#
|
59
|
-
# Wakes up all threads waiting for this lock.
|
60
|
-
#
|
61
|
-
def broadcast
|
62
|
-
Thread.handle_interrupt(StandardError => :on_blocking) do
|
63
|
-
threads = nil
|
64
|
-
@waiters_mutex.synchronize do
|
65
|
-
threads = @waiters.keys
|
66
|
-
@waiters.clear
|
67
|
-
end
|
68
|
-
for t in threads
|
69
|
-
begin
|
70
|
-
t.run
|
71
|
-
rescue ThreadError
|
72
|
-
end
|
73
|
-
end
|
74
|
-
end
|
75
|
-
self
|
76
|
-
end
|
77
|
-
end
|
78
|
-
|
79
|
-
class LocalityConditionVariable < ConditionVariable
|
80
|
-
|
81
|
-
def signal(hints=nil)
|
82
|
-
if hints.nil?
|
83
|
-
super()
|
84
|
-
elsif Array===hints
|
85
|
-
thread = nil
|
86
|
-
@waiters_mutex.synchronize do
|
87
|
-
@waiters.each do |t,v|
|
88
|
-
if hints.include?(t[:hint])
|
89
|
-
thread = t
|
90
|
-
break
|
91
|
-
end
|
92
|
-
end
|
93
|
-
if thread
|
94
|
-
@waiters.delete(thread)
|
95
|
-
else
|
96
|
-
thread,_ = @waiters.shift
|
97
|
-
end
|
98
|
-
end
|
99
|
-
Log.debug "--- LCV#signal: hints=#{hints.inspect} thread_to_run=#{thread.inspect} @waiters.size=#{@waiters.size}"
|
100
|
-
begin
|
101
|
-
thread.run if thread
|
102
|
-
rescue ThreadError
|
103
|
-
retry # t was already dead?
|
104
|
-
end
|
105
|
-
else
|
106
|
-
raise ArgumentError,"argument must be an Array"
|
107
|
-
end
|
108
|
-
self
|
109
|
-
end
|
110
|
-
|
111
|
-
|
112
|
-
def signal_with_hints(hints)
|
113
|
-
if !Array===hints
|
114
|
-
raise ArgumentError,"argument must be an Array"
|
115
|
-
end
|
116
|
-
thread =
|
117
|
-
@waiters_mutex.synchronize do
|
118
|
-
th = nil
|
119
|
-
@waiters.each do |t,v|
|
120
|
-
Log.debug "--- LCV#signal_with_hints: t[:hint]=#{t[:hint]}"
|
121
|
-
if hints.include?(t[:hint])
|
122
|
-
th = t
|
123
|
-
break
|
124
|
-
end
|
125
|
-
end
|
126
|
-
Log.debug "--- LCV#signal_with_hints: hints=#{hints.inspect} thread_to_run=#{th.inspect} @waiters.size=#{@waiters.size}"
|
127
|
-
if th
|
128
|
-
@waiters.delete(th)
|
129
|
-
end
|
130
|
-
th
|
131
|
-
end
|
132
|
-
begin
|
133
|
-
thread.run if thread
|
134
|
-
rescue ThreadError
|
135
|
-
retry # t was already dead?
|
136
|
-
end
|
137
|
-
thread
|
138
|
-
end
|
139
|
-
|
140
|
-
|
141
|
-
def broadcast(hints=nil)
|
142
|
-
if hints.nil?
|
143
|
-
super()
|
144
|
-
elsif Array===hints
|
145
|
-
threads = []
|
146
|
-
@waiters_mutex.synchronize do
|
147
|
-
hints.each do |h|
|
148
|
-
@waiters.each do |t,v|
|
149
|
-
if t[:hint] == h
|
150
|
-
threads << t
|
151
|
-
break
|
152
|
-
end
|
153
|
-
end
|
154
|
-
end
|
155
|
-
threads.each do |t|
|
156
|
-
@waiters.delete(t)
|
157
|
-
end
|
158
|
-
end
|
159
|
-
Log.debug "--- LCV#broadcast: hints=#{hints.inspect} threads_to_run=#{threads.inspect} @waiters.size=#{@waiters.size}"
|
160
|
-
threads.each do |t|
|
161
|
-
begin
|
162
|
-
t.run
|
163
|
-
rescue ThreadError
|
164
|
-
end
|
165
|
-
end
|
166
|
-
else
|
167
|
-
raise ArgumentError,"argument must be an Array"
|
168
|
-
end
|
169
|
-
self
|
170
|
-
end
|
171
|
-
end
|
172
|
-
|
173
|
-
|
174
|
-
class LocalityAwareQueue < TaskQueue
|
175
|
-
|
176
|
-
class Throughput
|
177
|
-
|
178
|
-
def initialize(list=nil)
|
179
|
-
@interdomain_list = {}
|
180
|
-
@interhost_list = {}
|
181
|
-
if list
|
182
|
-
values = []
|
183
|
-
list.each do |x,y,v|
|
184
|
-
hash_x = (@interdomain_list[x] ||= {})
|
185
|
-
hash_x[y] = n = v.to_f
|
186
|
-
values << n
|
187
|
-
end
|
188
|
-
@min_value = values.min
|
189
|
-
else
|
190
|
-
@min_value = 1
|
191
|
-
end
|
192
|
-
end
|
193
|
-
|
194
|
-
def interdomain(x,y)
|
195
|
-
hash_x = (@interdomain_list[x] ||= {})
|
196
|
-
if v = hash_x[y]
|
197
|
-
return v
|
198
|
-
elsif v = (@interdomain_list[y] || {})[x]
|
199
|
-
hash_x[y] = v
|
200
|
-
else
|
201
|
-
if x == y
|
202
|
-
hash_x[y] = 1
|
203
|
-
else
|
204
|
-
hash_x[y] = 0.1
|
205
|
-
end
|
206
|
-
end
|
207
|
-
hash_x[y]
|
208
|
-
end
|
209
|
-
|
210
|
-
def interhost(x,y)
|
211
|
-
return @min_value if !x
|
212
|
-
hash_x = (@interhost_list[x] ||= {})
|
213
|
-
if v = hash_x[y]
|
214
|
-
return v
|
215
|
-
elsif v = (@interhost_list[y] || {})[x]
|
216
|
-
hash_x[y] = v
|
217
|
-
else
|
218
|
-
x_short, x_domain = parse_hostname(x)
|
219
|
-
y_short, y_domain = parse_hostname(y)
|
220
|
-
v = interdomain(x_domain,y_domain)
|
221
|
-
hash_x[y] = v
|
222
|
-
end
|
223
|
-
hash_x[y]
|
224
|
-
end
|
225
|
-
|
226
|
-
def parse_hostname(host)
|
227
|
-
/^([^.]*)\.?(.*)$/ =~ host
|
228
|
-
[$1,$2]
|
229
|
-
end
|
230
|
-
|
231
|
-
end # class Throughput
|
232
|
-
|
233
|
-
|
234
|
-
def init_queue(host_list)
|
235
|
-
@host_list = host_list
|
236
|
-
@cv = LocalityConditionVariable.new
|
237
|
-
@size = 0
|
238
|
-
@q = {}
|
239
|
-
@host_list.host_count.each{|h,n| @q[h] = @array_class.new(n)}
|
240
|
-
@q_group = {}
|
241
|
-
@host_list.group_hosts.each do |g|
|
242
|
-
other = @host_list.host_count.dup
|
243
|
-
q1 = {}
|
244
|
-
g.each{|h| q1[h] = @q[h]; other.delete(h)}
|
245
|
-
q2 = {}
|
246
|
-
other.each{|h,v| q2[h] = @q[h]}
|
247
|
-
a = [q1,q2]
|
248
|
-
g.each{|h| @q_group[h] = a}
|
249
|
-
end
|
250
|
-
@q_remote = @array_class.new(0)
|
251
|
-
@q_later = Array.new
|
252
|
-
@enable_steal = !Pwrake.application.pwrake_options['DISABLE_STEAL']
|
253
|
-
@steal_wait = (Pwrake.application.pwrake_options['STEAL_WAIT'] || 0).to_i
|
254
|
-
@steal_wait_max = (Pwrake.application.pwrake_options['STEAL_WAIT_MAX'] || 10).to_i
|
255
|
-
@steal_wait_after_enq = (Pwrake.application.pwrake_options['STEAL_WAIT_AFTER_ENQ'] || 0.1).to_f
|
256
|
-
@last_enq_time = Time.now
|
257
|
-
Log.info("-- @enable_steal=#{@enable_steal.inspect} @steal_wait=#{@steal_wait} @steal_wait_max=#{@steal_wait_max} @steal_wait_after_enq={@steal_wait_after_enq}")
|
258
|
-
end
|
259
|
-
|
260
|
-
attr_reader :size
|
261
|
-
|
262
|
-
|
263
|
-
def enq_impl(t)
|
264
|
-
hints = t && t.suggest_location
|
265
|
-
if hints.nil? || hints.empty?
|
266
|
-
@q_later.push(t)
|
267
|
-
else
|
268
|
-
stored = false
|
269
|
-
hints.each do |h|
|
270
|
-
if q = @q[h]
|
271
|
-
t.assigned.push(h)
|
272
|
-
q.push(t)
|
273
|
-
stored = true
|
274
|
-
end
|
275
|
-
end
|
276
|
-
if !stored
|
277
|
-
@q_remote.push(t)
|
278
|
-
end
|
279
|
-
end
|
280
|
-
@last_enq_time = Time.now
|
281
|
-
@size += 1
|
282
|
-
end
|
283
|
-
|
284
|
-
|
285
|
-
def deq_impl(host,n)
|
286
|
-
if t = deq_locate(host)
|
287
|
-
Log.info "-- deq_locate n=#{n} task=#{t&&t.name} host=#{host}"
|
288
|
-
Log.debug "--- deq_impl\n#{inspect_q}"
|
289
|
-
return t
|
290
|
-
end
|
291
|
-
|
292
|
-
#hints = []
|
293
|
-
#@q.each do |h,q|
|
294
|
-
# hints << h if !q.empty?
|
295
|
-
#end
|
296
|
-
#if (!hints.empty?) && @cv.signal_with_hints(hints)
|
297
|
-
# return nil
|
298
|
-
#end
|
299
|
-
|
300
|
-
if !@q_remote.empty?
|
301
|
-
t = @q_remote.shift
|
302
|
-
Log.info "-- deq_remote n=#{n} task=#{t&&t.name} host=#{host}"
|
303
|
-
Log.debug "--- deq_impl\n#{inspect_q}"
|
304
|
-
return t
|
305
|
-
end
|
306
|
-
|
307
|
-
if !@q_later.empty?
|
308
|
-
t = @q_later.shift
|
309
|
-
Log.info "-- deq_later n=#{n} task=#{t&&t.name} host=#{host}"
|
310
|
-
Log.debug "--- deq_impl\n#{inspect_q}"
|
311
|
-
return t
|
312
|
-
end
|
313
|
-
|
314
|
-
if @enable_steal && n > 0 && Time.now-@last_enq_time > @steal_wait_after_enq
|
315
|
-
if t = deq_steal(host)
|
316
|
-
Log.info "-- deq_steal n=#{n} task=#{t&&t.name} host=#{host}"
|
317
|
-
Log.debug "--- deq_impl\n#{inspect_q}"
|
318
|
-
return t
|
319
|
-
end
|
320
|
-
end
|
321
|
-
|
322
|
-
#m = [@steal_wait*(2**n), @steal_wait_max].min
|
323
|
-
#@cv.wait(@mutex,m)
|
324
|
-
@cv.wait(@mutex)
|
325
|
-
nil
|
326
|
-
end
|
327
|
-
|
328
|
-
|
329
|
-
def deq_locate(host)
|
330
|
-
q = @q[host]
|
331
|
-
if q && !q.empty?
|
332
|
-
t = q.shift
|
333
|
-
if t
|
334
|
-
t.assigned.each do |h|
|
335
|
-
@q[h].delete(t)
|
336
|
-
end
|
337
|
-
end
|
338
|
-
@size -= 1
|
339
|
-
return t
|
340
|
-
else
|
341
|
-
nil
|
342
|
-
end
|
343
|
-
end
|
344
|
-
|
345
|
-
def deq_steal(host)
|
346
|
-
# select a task based on many and close
|
347
|
-
max_host = nil
|
348
|
-
max_num = 0
|
349
|
-
@q_group[host].each do |qg|
|
350
|
-
qg.each do |h,a|
|
351
|
-
if !a.empty?
|
352
|
-
d = a.size
|
353
|
-
if d > max_num
|
354
|
-
max_host = h
|
355
|
-
max_num = d
|
356
|
-
end
|
357
|
-
end
|
358
|
-
end
|
359
|
-
if max_num > 0
|
360
|
-
Log.info "-- deq_steal max_host=#{max_host} max_num=#{max_num}"
|
361
|
-
t = deq_locate(max_host)
|
362
|
-
return t if t
|
363
|
-
end
|
364
|
-
end
|
365
|
-
nil
|
366
|
-
end
|
367
|
-
|
368
|
-
def inspect_q
|
369
|
-
s = ""
|
370
|
-
b = proc{|h,q|
|
371
|
-
s += " #{h}: size=#{q.size} "
|
372
|
-
case q.size
|
373
|
-
when 0
|
374
|
-
s += "[]\n"
|
375
|
-
when 1
|
376
|
-
s += "[#{q.first.name}]\n"
|
377
|
-
when 2
|
378
|
-
s += "[#{q.first.name}, #{q.last.name}]\n"
|
379
|
-
else
|
380
|
-
s += "[#{q.first.name},.. #{q.last.name}]\n"
|
381
|
-
end
|
382
|
-
}
|
383
|
-
b.call("noaction",@q_noaction)
|
384
|
-
@q.each(&b)
|
385
|
-
b.call("remote",@q_remote)
|
386
|
-
b.call("later",@q_later)
|
387
|
-
s
|
388
|
-
end
|
389
|
-
|
390
|
-
def size
|
391
|
-
@size
|
392
|
-
end
|
393
|
-
|
394
|
-
def clear
|
395
|
-
@q_noaction.clear
|
396
|
-
@q.each{|h,q| q.clear}
|
397
|
-
@q_remote.clear
|
398
|
-
@q_later.clear
|
399
|
-
end
|
400
|
-
|
401
|
-
def empty?
|
402
|
-
@q.all?{|h,q| q.empty?} &&
|
403
|
-
@q_noaction.empty? &&
|
404
|
-
@q_remote.empty? &&
|
405
|
-
@q_later.empty?
|
406
|
-
end
|
407
|
-
|
408
|
-
def finish
|
409
|
-
super
|
410
|
-
end
|
411
|
-
|
412
|
-
end
|
413
|
-
end
|