drake 0.8.2.1.0.5 → 0.8.2.1.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGES.drake +1 -1
- data/Rakefile +1 -0
- data/Rakefile.drake +2 -2
- data/lib/rake.rb +1 -1
- data/lib/rake/comp_tree/algorithm.rb +210 -0
- data/lib/rake/comp_tree/bucket_ipc.rb +151 -0
- data/lib/rake/comp_tree/driver.rb +267 -0
- data/lib/rake/comp_tree/error.rb +27 -0
- data/lib/rake/comp_tree/node.rb +165 -0
- data/lib/rake/comp_tree/quix/builtin/kernel/tap.rb +9 -0
- data/lib/rake/comp_tree/quix/diagnostic.rb +44 -0
- data/lib/rake/comp_tree/quix/kernel.rb +61 -0
- data/lib/rake/comp_tree/retriable_fork.rb +43 -0
- data/lib/rake/comp_tree/task_node.rb +22 -0
- metadata +11 -1
data/CHANGES.drake
CHANGED
data/Rakefile
CHANGED
data/Rakefile.drake
CHANGED
@@ -40,7 +40,7 @@ task :pull_contrib => [ :init_contrib, :run_pull_contrib ]
|
|
40
40
|
######################################################################
|
41
41
|
# drake_release
|
42
42
|
|
43
|
-
task :drake_prerelease => :clean do
|
43
|
+
task :drake_prerelease => [:clean, :gemspec] do
|
44
44
|
rm_rf("html")
|
45
45
|
rm_rf("pkg")
|
46
46
|
unless `git status` =~ %r!nothing to commit \(working directory clean\)!
|
@@ -77,7 +77,7 @@ task :drake_finish_release do
|
|
77
77
|
}
|
78
78
|
|
79
79
|
git("tag", "drake-" + SPEC.version.to_s)
|
80
|
-
git(
|
80
|
+
git(*%w(push --tags origin master))
|
81
81
|
end
|
82
82
|
|
83
83
|
task :drake_release =>
|
data/lib/rake.rb
CHANGED
@@ -0,0 +1,210 @@
|
|
1
|
+
|
2
|
+
require 'rake/comp_tree/quix/diagnostic'
|
3
|
+
require 'rake/comp_tree/retriable_fork'
|
4
|
+
|
5
|
+
module Rake::CompTree
|
6
|
+
module Algorithm
|
7
|
+
include Quix::Diagnostic
|
8
|
+
|
9
|
+
def compute_multithreaded(root, num_threads, use_fork, buckets)
|
10
|
+
trace "Computing #{root.name} with #{num_threads} threads"
|
11
|
+
result = nil
|
12
|
+
mutex = Mutex.new
|
13
|
+
node_finished_condition = ConditionVariable.new
|
14
|
+
thread_wake_condition = ConditionVariable.new
|
15
|
+
threads = []
|
16
|
+
|
17
|
+
# workaround: jruby gives "run" status for waiting on
|
18
|
+
# condition variable
|
19
|
+
num_threads_ready = 0
|
20
|
+
|
21
|
+
num_threads.times { |thread_index|
|
22
|
+
threads << Thread.new {
|
23
|
+
#
|
24
|
+
# wait for main thread
|
25
|
+
#
|
26
|
+
mutex.synchronize {
|
27
|
+
trace "Thread #{thread_index} waiting to start"
|
28
|
+
num_threads_ready += 1
|
29
|
+
thread_wake_condition.wait(mutex)
|
30
|
+
}
|
31
|
+
|
32
|
+
while true
|
33
|
+
trace "Thread #{thread_index} node search"
|
34
|
+
|
35
|
+
#
|
36
|
+
# Done! Thread will exit.
|
37
|
+
#
|
38
|
+
break if mutex.synchronize {
|
39
|
+
result
|
40
|
+
}
|
41
|
+
|
42
|
+
#
|
43
|
+
# Lock the tree and find a node. The node we
|
44
|
+
# obtain, if any, is already locked.
|
45
|
+
#
|
46
|
+
node = mutex.synchronize {
|
47
|
+
find_node(root)
|
48
|
+
}
|
49
|
+
|
50
|
+
if node
|
51
|
+
trace "Thread #{thread_index} found node #{node.name}"
|
52
|
+
|
53
|
+
node_result =
|
54
|
+
compute_node(
|
55
|
+
node,
|
56
|
+
use_fork,
|
57
|
+
buckets ? buckets[thread_index] : nil)
|
58
|
+
|
59
|
+
mutex.synchronize {
|
60
|
+
node.result = node_result
|
61
|
+
}
|
62
|
+
|
63
|
+
#
|
64
|
+
# remove locks for this node (shared lock and own lock)
|
65
|
+
#
|
66
|
+
mutex.synchronize {
|
67
|
+
node.unlock
|
68
|
+
if node == root
|
69
|
+
#
|
70
|
+
# Root node was computed; we are done.
|
71
|
+
#
|
72
|
+
trace "Thread #{thread_index} got final answer"
|
73
|
+
result = root.result
|
74
|
+
end
|
75
|
+
node_finished_condition.signal
|
76
|
+
}
|
77
|
+
else
|
78
|
+
trace "Thread #{thread_index}: no node found; sleeping."
|
79
|
+
mutex.synchronize {
|
80
|
+
thread_wake_condition.wait(mutex)
|
81
|
+
}
|
82
|
+
end
|
83
|
+
end
|
84
|
+
trace "Thread #{thread_index} exiting"
|
85
|
+
}
|
86
|
+
}
|
87
|
+
|
88
|
+
trace "Main: waiting for threads to launch and block."
|
89
|
+
while true
|
90
|
+
break if mutex.synchronize {
|
91
|
+
num_threads_ready == num_threads
|
92
|
+
}
|
93
|
+
Thread.pass
|
94
|
+
end
|
95
|
+
|
96
|
+
trace "Main: entering main loop"
|
97
|
+
mutex.synchronize {
|
98
|
+
while true
|
99
|
+
trace "Main: waking threads"
|
100
|
+
thread_wake_condition.broadcast
|
101
|
+
|
102
|
+
if result
|
103
|
+
trace "Main: detected finish."
|
104
|
+
break
|
105
|
+
end
|
106
|
+
|
107
|
+
trace "Main: waiting for a node"
|
108
|
+
node_finished_condition.wait(mutex)
|
109
|
+
trace "Main: got a node"
|
110
|
+
end
|
111
|
+
}
|
112
|
+
|
113
|
+
trace "Main: waiting for threads to finish."
|
114
|
+
catch(:done) {
|
115
|
+
while true
|
116
|
+
mutex.synchronize {
|
117
|
+
throw :done if threads.all? { |thread|
|
118
|
+
thread.status == false
|
119
|
+
}
|
120
|
+
thread_wake_condition.broadcast
|
121
|
+
}
|
122
|
+
Thread.pass
|
123
|
+
end
|
124
|
+
}
|
125
|
+
|
126
|
+
trace "Main: computation done."
|
127
|
+
result
|
128
|
+
end
|
129
|
+
|
130
|
+
def find_node(node)
|
131
|
+
# --- only called inside mutex
|
132
|
+
trace "Looking for a node, starting with #{node.name}"
|
133
|
+
if node.result
|
134
|
+
#
|
135
|
+
# already computed
|
136
|
+
#
|
137
|
+
trace "#{node.name} has been computed"
|
138
|
+
nil
|
139
|
+
elsif node.children_results and node.try_lock
|
140
|
+
#
|
141
|
+
# Node is not computed and its children are computed;
|
142
|
+
# and we have the lock. Ready to compute.
|
143
|
+
#
|
144
|
+
node
|
145
|
+
else
|
146
|
+
#
|
147
|
+
# locked or children not computed; recurse to children
|
148
|
+
#
|
149
|
+
trace "Checking #{node.name}'s children"
|
150
|
+
node.each_child { |child|
|
151
|
+
if next_node = find_node(child)
|
152
|
+
return next_node
|
153
|
+
end
|
154
|
+
}
|
155
|
+
nil
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
def compute_node(node, use_fork, bucket)
|
160
|
+
if use_fork
|
161
|
+
trace "About to fork for node #{node.name}"
|
162
|
+
if bucket
|
163
|
+
#
|
164
|
+
# Use our assigned bucket to transfer the result.
|
165
|
+
#
|
166
|
+
fork_node(node) {
|
167
|
+
node.trace_compute
|
168
|
+
bucket.contents = node.compute
|
169
|
+
}
|
170
|
+
bucket.contents
|
171
|
+
else
|
172
|
+
#
|
173
|
+
# No bucket -- discarding result
|
174
|
+
#
|
175
|
+
fork_node(node) {
|
176
|
+
node.trace_compute
|
177
|
+
node.compute
|
178
|
+
}
|
179
|
+
true
|
180
|
+
end
|
181
|
+
else
|
182
|
+
#
|
183
|
+
# No fork
|
184
|
+
#
|
185
|
+
node.trace_compute
|
186
|
+
node.compute
|
187
|
+
end
|
188
|
+
end
|
189
|
+
|
190
|
+
def fork_node(node)
|
191
|
+
trace "About to fork for node #{node.name}"
|
192
|
+
process_id = RetriableFork.fork {
|
193
|
+
trace "Fork: process #{Process.pid}"
|
194
|
+
node.trace_compute
|
195
|
+
yield
|
196
|
+
trace "Fork: computation done"
|
197
|
+
}
|
198
|
+
trace "Waiting for process #{process_id}"
|
199
|
+
Process.wait(process_id)
|
200
|
+
trace "Process #{process_id} finished"
|
201
|
+
exitstatus = $?.exitstatus
|
202
|
+
if exitstatus != 0
|
203
|
+
trace "Process #{process_id} returned #{exitstatus}; exiting."
|
204
|
+
exit(1)
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
extend self
|
209
|
+
end
|
210
|
+
end
|
@@ -0,0 +1,151 @@
|
|
1
|
+
|
2
|
+
require 'drb'
|
3
|
+
require 'thread'
|
4
|
+
|
5
|
+
require 'rake/comp_tree/retriable_fork'
|
6
|
+
require 'rake/comp_tree/quix/diagnostic'
|
7
|
+
require 'rake/comp_tree/quix/builtin/kernel/tap'
|
8
|
+
|
9
|
+
module Rake::CompTree
|
10
|
+
module BucketIPC
|
11
|
+
class Bucket
|
12
|
+
include Quix::Diagnostic
|
13
|
+
include RetriableFork
|
14
|
+
|
15
|
+
def initialize(address, timeout, wait_interval)
|
16
|
+
trace "Making bucket with address #{address}"
|
17
|
+
|
18
|
+
@remote_pid = fork {
|
19
|
+
own_object = Class.new {
|
20
|
+
attr_accessor(:contents)
|
21
|
+
}.new
|
22
|
+
server = DRb.start_service(address, own_object)
|
23
|
+
debug {
|
24
|
+
server.verbose = true
|
25
|
+
}
|
26
|
+
DRb.thread.join
|
27
|
+
}
|
28
|
+
|
29
|
+
@remote_object = DRbObject.new_with_uri(address)
|
30
|
+
@address = address
|
31
|
+
@timeout = timeout
|
32
|
+
@wait_interval = wait_interval
|
33
|
+
end
|
34
|
+
|
35
|
+
attr_accessor(:timeout, :wait_interval)
|
36
|
+
attr_reader(:address)
|
37
|
+
|
38
|
+
def contents=(new_contents)
|
39
|
+
connect {
|
40
|
+
@remote_object.contents = new_contents
|
41
|
+
}
|
42
|
+
end
|
43
|
+
|
44
|
+
def contents
|
45
|
+
connect {
|
46
|
+
@remote_object.contents
|
47
|
+
}
|
48
|
+
end
|
49
|
+
|
50
|
+
def stop
|
51
|
+
Process.kill("TERM", @remote_pid)
|
52
|
+
end
|
53
|
+
|
54
|
+
private
|
55
|
+
|
56
|
+
def connect
|
57
|
+
begin
|
58
|
+
return yield
|
59
|
+
rescue DRb::DRbConnError
|
60
|
+
start = Time.now
|
61
|
+
begin
|
62
|
+
Kernel.sleep(@wait_interval)
|
63
|
+
return yield
|
64
|
+
rescue DRb::DRbConnError
|
65
|
+
if Time.now - start > @timeout
|
66
|
+
raise
|
67
|
+
end
|
68
|
+
retry
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
class DriverBase
|
75
|
+
def initialize(addresses, timeout, wait_interval)
|
76
|
+
begin
|
77
|
+
@buckets = addresses.map { |address|
|
78
|
+
Bucket.new(address, timeout, wait_interval)
|
79
|
+
}
|
80
|
+
if block_given?
|
81
|
+
yield @buckets
|
82
|
+
end
|
83
|
+
ensure
|
84
|
+
if block_given?
|
85
|
+
stop
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def stop
|
91
|
+
if defined?(@buckets)
|
92
|
+
@buckets.each { |bucket|
|
93
|
+
bucket.stop
|
94
|
+
}
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
class Driver < DriverBase
|
100
|
+
DEFAULTS = {
|
101
|
+
:timeout => 0.5,
|
102
|
+
:wait_interval => 0.05,
|
103
|
+
:port_start => 18181,
|
104
|
+
}
|
105
|
+
|
106
|
+
module BucketCounter
|
107
|
+
@mutex = Mutex.new
|
108
|
+
@count = 0
|
109
|
+
class << self
|
110
|
+
def increment_count
|
111
|
+
@mutex.synchronize {
|
112
|
+
@count += 1
|
113
|
+
}
|
114
|
+
end
|
115
|
+
|
116
|
+
def map_indexes(num_buckets)
|
117
|
+
Array.new.tap { |result|
|
118
|
+
num_buckets.times {
|
119
|
+
result << yield(increment_count)
|
120
|
+
}
|
121
|
+
}
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
def initialize(num_buckets, opts_in = {})
|
127
|
+
opts = DEFAULTS.merge(opts_in)
|
128
|
+
|
129
|
+
addresses =
|
130
|
+
if RetriableFork::HAVE_FORK
|
131
|
+
#
|
132
|
+
# Assume the existence of fork implies a unix machine.
|
133
|
+
#
|
134
|
+
require 'drb/unix'
|
135
|
+
basename = "drbunix://#{Dir.tmpdir}/bucket.#{Process.pid}.#{rand}"
|
136
|
+
BucketCounter.map_indexes(num_buckets) { |index|
|
137
|
+
"#{basename}.#{index}"
|
138
|
+
}
|
139
|
+
else
|
140
|
+
#
|
141
|
+
# Fallback: use the default socket.
|
142
|
+
#
|
143
|
+
BucketCounter.map_indexes(num_buckets) { |index|
|
144
|
+
"druby://localhost:#{opts[:port_start] + index}"
|
145
|
+
}
|
146
|
+
end
|
147
|
+
super(addresses, opts[:timeout], opts[:wait_interval])
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
@@ -0,0 +1,267 @@
|
|
1
|
+
|
2
|
+
require 'rake/comp_tree/bucket_ipc'
|
3
|
+
require 'rake/comp_tree/quix/diagnostic'
|
4
|
+
require 'rake/comp_tree/quix/kernel'
|
5
|
+
require 'rake/comp_tree/algorithm'
|
6
|
+
require 'rake/comp_tree/node'
|
7
|
+
require 'rake/comp_tree/task_node'
|
8
|
+
require 'rake/comp_tree/error'
|
9
|
+
|
10
|
+
require 'thread'
|
11
|
+
|
12
|
+
module Rake::CompTree
|
13
|
+
#
|
14
|
+
# Driver is the main interface to the computation tree. It is
|
15
|
+
# responsible for defining nodes and running computations.
|
16
|
+
#
|
17
|
+
class Driver
|
18
|
+
DEFAULTS = {
|
19
|
+
:threads => 1,
|
20
|
+
:fork => false,
|
21
|
+
:timeout => 5.0,
|
22
|
+
:wait_interval => 0.02,
|
23
|
+
}
|
24
|
+
|
25
|
+
include Quix::Diagnostic #:nodoc:
|
26
|
+
include Quix::Kernel #:nodoc:
|
27
|
+
|
28
|
+
#
|
29
|
+
# Begin a new computation tree.
|
30
|
+
#
|
31
|
+
# Options hash:
|
32
|
+
#
|
33
|
+
# <tt>:node_class</tt> -- (Class) Rake::CompTree::Node subclass from
|
34
|
+
# which nodes are created.
|
35
|
+
#
|
36
|
+
# <tt>:discard_result</tt> -- (boolean) If you are <em>not</em>
|
37
|
+
# interested in the final answer, but only in the actions which
|
38
|
+
# complete the computation, then set this to +true+. This is
|
39
|
+
# equivalent to saying <tt>:node_class => Rake::CompTree::TaskNode</tt>.
|
40
|
+
# (If you are forking processes, it is good to know that IPC is
|
41
|
+
# not needed to communicate the result.)
|
42
|
+
#
|
43
|
+
def initialize(opts = nil)
|
44
|
+
if opts and opts[:node_class] and opts[:discard_result]
|
45
|
+
raise(
|
46
|
+
Error::ArgumentError,
|
47
|
+
"#{self.class.name}.new: :discard_result and :node_class " +
|
48
|
+
"are mutually exclusive")
|
49
|
+
end
|
50
|
+
|
51
|
+
@node_class =
|
52
|
+
if opts and opts[:node_class]
|
53
|
+
opts[:node_class]
|
54
|
+
elsif opts and opts[:discard_result]
|
55
|
+
TaskNode
|
56
|
+
else
|
57
|
+
Node
|
58
|
+
end
|
59
|
+
|
60
|
+
@nodes = Hash.new
|
61
|
+
|
62
|
+
if block_given?
|
63
|
+
yield self
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
#
|
68
|
+
# Name-to-node hash.
|
69
|
+
#
|
70
|
+
attr_reader :nodes
|
71
|
+
|
72
|
+
#
|
73
|
+
# Define a computation node.
|
74
|
+
#
|
75
|
+
# There are three distinct forms of a node definition. In each of
|
76
|
+
# the following examples, a computation node named +area+ is
|
77
|
+
# defined which depends on the nodes +height+, +width+, +offset+.
|
78
|
+
#
|
79
|
+
# The method_missing form:
|
80
|
+
# driver.define_area(:width, :height, :offset) { |width, height, offset|
|
81
|
+
# width*height - offset
|
82
|
+
# }
|
83
|
+
#
|
84
|
+
# The eval form:
|
85
|
+
# driver.define_area :width, :height, :offset, %{
|
86
|
+
# width*height - offset
|
87
|
+
# }
|
88
|
+
# (Note the '%' before the brace.)
|
89
|
+
#
|
90
|
+
# The raw form:
|
91
|
+
# driver.define(:area, :width, :height, :offset) { |width, height, offset|
|
92
|
+
# width*height - offset
|
93
|
+
# }
|
94
|
+
#
|
95
|
+
def define(*args, &block)
|
96
|
+
parent_name = args.first
|
97
|
+
children_names = args[1..-1]
|
98
|
+
|
99
|
+
unless parent_name
|
100
|
+
raise Error::ArgumentError, "No name given for node"
|
101
|
+
end
|
102
|
+
|
103
|
+
#
|
104
|
+
# retrieve or create parent and children
|
105
|
+
#
|
106
|
+
parent =
|
107
|
+
if t = @nodes[parent_name]
|
108
|
+
t
|
109
|
+
else
|
110
|
+
@nodes[parent_name] = @node_class.new(parent_name)
|
111
|
+
end
|
112
|
+
|
113
|
+
if parent.function
|
114
|
+
raise Error::RedefinitionError, "Node #{parent.name} already defined."
|
115
|
+
end
|
116
|
+
parent.function = block
|
117
|
+
|
118
|
+
children = children_names.map { |child_name|
|
119
|
+
if t = @nodes[child_name]
|
120
|
+
t
|
121
|
+
else
|
122
|
+
@nodes[child_name] = @node_class.new(child_name)
|
123
|
+
end
|
124
|
+
}
|
125
|
+
|
126
|
+
#
|
127
|
+
# link
|
128
|
+
#
|
129
|
+
parent.children = children
|
130
|
+
children.each { |child|
|
131
|
+
child.parents << parent
|
132
|
+
}
|
133
|
+
end
|
134
|
+
|
135
|
+
#
|
136
|
+
# parsing/evaling helper
|
137
|
+
#
|
138
|
+
def evaling_define(*args) #:nodoc:
|
139
|
+
function_name = args[0]
|
140
|
+
function_arg_names = args[1..-2]
|
141
|
+
function_string = args.last.to_str
|
142
|
+
|
143
|
+
comma_separated = function_arg_names.map { |name|
|
144
|
+
name.to_s
|
145
|
+
}.join(",")
|
146
|
+
|
147
|
+
eval_me = %{
|
148
|
+
lambda { |#{comma_separated}|
|
149
|
+
#{function_string}
|
150
|
+
}
|
151
|
+
}
|
152
|
+
|
153
|
+
function = eval(eval_me, TOPLEVEL_BINDING)
|
154
|
+
|
155
|
+
define(function_name, *function_arg_names, &function)
|
156
|
+
end
|
157
|
+
|
158
|
+
def method_missing(symbol, *args, &block) #:nodoc:
|
159
|
+
if match = symbol.to_s.match(%r!\Adefine_(\w+)\Z!)
|
160
|
+
method_name = match.captures.first.to_sym
|
161
|
+
if block
|
162
|
+
define(method_name, *args, &block)
|
163
|
+
else
|
164
|
+
evaling_define(method_name, *args)
|
165
|
+
end
|
166
|
+
else
|
167
|
+
super(symbol, *args, &block)
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
#
|
172
|
+
# Mark this node and all its children as uncomputed.
|
173
|
+
#
|
174
|
+
# Arguments:
|
175
|
+
#
|
176
|
+
# +name+ -- (Symbol) node name.
|
177
|
+
#
|
178
|
+
def reset(name)
|
179
|
+
@nodes[name].reset
|
180
|
+
end
|
181
|
+
|
182
|
+
#
|
183
|
+
# Check for a cyclic graph below the given node. Raises
|
184
|
+
# Rake::CompTree::Error::CircularError if found.
|
185
|
+
#
|
186
|
+
# Arguments:
|
187
|
+
#
|
188
|
+
# +name+ -- (Symbol) node name.
|
189
|
+
#
|
190
|
+
def check_circular(name)
|
191
|
+
helper = lambda { |root, chain|
|
192
|
+
if chain.include? root
|
193
|
+
raise Error::CircularError,
|
194
|
+
"Circular dependency detected: #{root} => #{chain.last} => #{root}"
|
195
|
+
end
|
196
|
+
@nodes[root].children.each { |child|
|
197
|
+
helper.call(child.name, chain + [root])
|
198
|
+
}
|
199
|
+
}
|
200
|
+
helper.call(name, [])
|
201
|
+
end
|
202
|
+
|
203
|
+
#
|
204
|
+
# Compute this node.
|
205
|
+
#
|
206
|
+
# Arguments:
|
207
|
+
#
|
208
|
+
# +name+ -- (Symbol) node name.
|
209
|
+
#
|
210
|
+
# Options hash:
|
211
|
+
#
|
212
|
+
# <tt>:threads</tt> -- (Integer) Number of parallel threads.
|
213
|
+
#
|
214
|
+
# <tt>:fork</tt> -- (boolean) Whether to fork each computation
|
215
|
+
# node into its own process.
|
216
|
+
#
|
217
|
+
# Defaults options are taken from Driver::DEFAULTS.
|
218
|
+
#
|
219
|
+
def compute(name, opts = nil)
|
220
|
+
#
|
221
|
+
# Undocumented options:
|
222
|
+
#
|
223
|
+
# <tt>:wait_interval</tt> -- (seconds) (Obscure) How long to
|
224
|
+
# wait after an IPC failure.
|
225
|
+
#
|
226
|
+
# <tt>:timeout</tt> -- (seconds) (Obscure) Give up after this
|
227
|
+
# period of persistent IPC failures.
|
228
|
+
#
|
229
|
+
|
230
|
+
abort_on_exception {
|
231
|
+
compute_private(name, opts || Hash.new)
|
232
|
+
}
|
233
|
+
end
|
234
|
+
|
235
|
+
private
|
236
|
+
|
237
|
+
def compute_private(name, opts_in)
|
238
|
+
opts = DEFAULTS.merge(opts_in)
|
239
|
+
root = @nodes[name]
|
240
|
+
|
241
|
+
if opts[:threads] < 1
|
242
|
+
raise Error::ArgumentError, "threads is #{opts[:threads]}"
|
243
|
+
end
|
244
|
+
|
245
|
+
if opts[:threads] == 1
|
246
|
+
root.result = root.compute_now
|
247
|
+
elsif opts[:fork] and not @node_class.discard_result?
|
248
|
+
#
|
249
|
+
# Use buckets to send results across forks.
|
250
|
+
#
|
251
|
+
result = nil
|
252
|
+
BucketIPC::Driver.new(opts[:threads], opts) { |buckets|
|
253
|
+
result =
|
254
|
+
Algorithm.compute_multithreaded(
|
255
|
+
root, opts[:threads], opts[:fork], buckets)
|
256
|
+
}
|
257
|
+
result
|
258
|
+
else
|
259
|
+
#
|
260
|
+
# Multithreaded computation without fork.
|
261
|
+
#
|
262
|
+
Algorithm.compute_multithreaded(
|
263
|
+
root, opts[:threads], opts[:fork], nil)
|
264
|
+
end
|
265
|
+
end
|
266
|
+
end
|
267
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
|
2
|
+
module Rake::CompTree
|
3
|
+
module Error
|
4
|
+
# Base class for Rake::CompTree errors.
|
5
|
+
class Base < StandardError ; end
|
6
|
+
|
7
|
+
# Internal error inside Rake::CompTree. Please send a bug report.
|
8
|
+
class AssertionFailed < Base ; end
|
9
|
+
|
10
|
+
# Bad arguments were passed to a method.
|
11
|
+
class ArgumentError < Base ; end
|
12
|
+
|
13
|
+
#
|
14
|
+
# Attempt to redefine a Node.
|
15
|
+
#
|
16
|
+
# If you wish to only replace the function, set
|
17
|
+
# driver.nodes[name].function = some_new_lambda
|
18
|
+
#
|
19
|
+
class RedefinitionError < Base ; end
|
20
|
+
|
21
|
+
# A Cyclic graph was detected.
|
22
|
+
class CircularError < Base ; end
|
23
|
+
|
24
|
+
# No function was defined for this node.
|
25
|
+
class NoFunctionError < Base ; end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,165 @@
|
|
1
|
+
|
2
|
+
require 'rake/comp_tree/quix/diagnostic'
|
3
|
+
require 'thread'
|
4
|
+
|
5
|
+
module Rake::CompTree
|
6
|
+
#
|
7
|
+
# Base class for nodes in the computation tree.
|
8
|
+
#
|
9
|
+
class Node
|
10
|
+
include Quix::Diagnostic #:nodoc:
|
11
|
+
|
12
|
+
attr_reader :name #:nodoc:
|
13
|
+
|
14
|
+
attr_accessor :parents #:nodoc:
|
15
|
+
attr_accessor :children #:nodoc:
|
16
|
+
attr_accessor :function #:nodoc:
|
17
|
+
attr_accessor :result #:nodoc:
|
18
|
+
attr_accessor :shared_lock #:nodoc:
|
19
|
+
|
20
|
+
#
|
21
|
+
# Create a node
|
22
|
+
#
|
23
|
+
def initialize(name) #:nodoc:
|
24
|
+
@name = name
|
25
|
+
@mutex = Mutex.new
|
26
|
+
@children = []
|
27
|
+
@parents = []
|
28
|
+
reset_self
|
29
|
+
end
|
30
|
+
|
31
|
+
#
|
32
|
+
# Reset the computation for this node.
|
33
|
+
#
|
34
|
+
def reset_self #:nodoc:
|
35
|
+
@shared_lock = 0
|
36
|
+
@children_results = nil
|
37
|
+
@result = nil
|
38
|
+
end
|
39
|
+
|
40
|
+
#
|
41
|
+
# Reset the computation for this node and all children.
|
42
|
+
#
|
43
|
+
def reset #:nodoc:
|
44
|
+
each_downward { |node|
|
45
|
+
node.reset_self
|
46
|
+
}
|
47
|
+
end
|
48
|
+
|
49
|
+
def each_downward(&block) #:nodoc:
|
50
|
+
block.call(self)
|
51
|
+
@children.each { |child|
|
52
|
+
child.each_downward(&block)
|
53
|
+
}
|
54
|
+
end
|
55
|
+
|
56
|
+
def each_upward(&block) #:nodoc:
|
57
|
+
block.call(self)
|
58
|
+
@parents.each { |parent|
|
59
|
+
parent.each_upward(&block)
|
60
|
+
}
|
61
|
+
end
|
62
|
+
|
63
|
+
def each_child #:nodoc:
|
64
|
+
@children.each { |child|
|
65
|
+
yield(child)
|
66
|
+
}
|
67
|
+
end
|
68
|
+
|
69
|
+
#
|
70
|
+
# Force computation of all children; intended for
|
71
|
+
# single-threaded mode.
|
72
|
+
#
|
73
|
+
def compute_now #:nodoc:
|
74
|
+
unless @children_results
|
75
|
+
@children_results = @children.map { |child|
|
76
|
+
child.compute_now
|
77
|
+
}
|
78
|
+
end
|
79
|
+
compute
|
80
|
+
end
|
81
|
+
|
82
|
+
#
|
83
|
+
# If all children have been computed, return their results;
|
84
|
+
# otherwise return nil.
|
85
|
+
#
|
86
|
+
def children_results #:nodoc:
|
87
|
+
if @children_results
|
88
|
+
@children_results
|
89
|
+
else
|
90
|
+
results = @children.map { |child|
|
91
|
+
if child_result = child.result
|
92
|
+
child_result
|
93
|
+
else
|
94
|
+
return nil
|
95
|
+
end
|
96
|
+
}
|
97
|
+
@children_results = results
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
def trace_compute #:nodoc:
|
102
|
+
debug {
|
103
|
+
# --- own mutex
|
104
|
+
trace "Computing #{@name}"
|
105
|
+
raise Error::AssertionFailed if @result
|
106
|
+
raise Error::AssertionFailed unless @mutex.locked?
|
107
|
+
raise Error::AssertionFailed unless @children_results
|
108
|
+
}
|
109
|
+
end
|
110
|
+
|
111
|
+
#
|
112
|
+
# Compute this node; children must be computed and lock must be
|
113
|
+
# already acquired.
|
114
|
+
#
|
115
|
+
def compute #:nodoc:
|
116
|
+
unless defined?(@function) and @function
|
117
|
+
raise Error::NoFunctionError,
|
118
|
+
"No function was defined for node '#{@name.inspect}'"
|
119
|
+
end
|
120
|
+
@function.call(*@children_results)
|
121
|
+
end
|
122
|
+
|
123
|
+
def try_lock #:nodoc:
|
124
|
+
# --- shared tree mutex and own mutex
|
125
|
+
if @shared_lock == 0 and @mutex.try_lock
|
126
|
+
trace "Locking #{@name}"
|
127
|
+
each_upward { |node|
|
128
|
+
node.shared_lock += 1
|
129
|
+
trace "#{node.name} locked by #{@name}: level: #{node.shared_lock}"
|
130
|
+
}
|
131
|
+
true
|
132
|
+
else
|
133
|
+
false
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
def unlock #:nodoc:
|
138
|
+
# --- shared tree mutex and own mutex
|
139
|
+
debug {
|
140
|
+
raise Error::AssertionFailed unless @mutex.locked?
|
141
|
+
trace "Unlocking #{@name}"
|
142
|
+
}
|
143
|
+
each_upward { |node|
|
144
|
+
node.shared_lock -= 1
|
145
|
+
debug {
|
146
|
+
if node.shared_lock == 0
|
147
|
+
trace "#{node.name} unlocked by #{@name}"
|
148
|
+
end
|
149
|
+
}
|
150
|
+
}
|
151
|
+
@mutex.unlock
|
152
|
+
end
|
153
|
+
|
154
|
+
class << self
|
155
|
+
#
|
156
|
+
# Throw away the computation result?
|
157
|
+
#
|
158
|
+
# This Node base class always returns false.
|
159
|
+
#
|
160
|
+
def discard_result?
|
161
|
+
false
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
165
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
|
2
|
+
require 'rake/comp_tree/quix/builtin/kernel/tap'
|
3
|
+
|
4
|
+
module Rake::CompTree::Quix
|
5
|
+
module Diagnostic
|
6
|
+
def show(desc = nil, stream = STDOUT, &block)
|
7
|
+
if desc
|
8
|
+
stream.puts(desc)
|
9
|
+
end
|
10
|
+
if block
|
11
|
+
expression = block.call
|
12
|
+
eval(expression, block.binding).tap { |result|
|
13
|
+
stream.printf("%-16s => %s\n", expression, result.inspect)
|
14
|
+
}
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
if $DEBUG
|
19
|
+
def debug
|
20
|
+
yield
|
21
|
+
end
|
22
|
+
|
23
|
+
def debugging?
|
24
|
+
true
|
25
|
+
end
|
26
|
+
|
27
|
+
def trace(desc = nil, &block)
|
28
|
+
if desc
|
29
|
+
show("#{desc}.".sub(%r!\.\.+\Z!, ""), STDERR, &block)
|
30
|
+
else
|
31
|
+
show(nil, STDERR, &block)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
else
|
35
|
+
# non-$DEBUG
|
36
|
+
def debug ; end
|
37
|
+
def debugging? ; end
|
38
|
+
def trace(*args) ; end
|
39
|
+
end
|
40
|
+
|
41
|
+
extend self
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
@@ -0,0 +1,61 @@
|
|
1
|
+
|
2
|
+
require 'thread'
|
3
|
+
|
4
|
+
module Rake::CompTree::Quix
|
5
|
+
module Kernel
|
6
|
+
def let
|
7
|
+
yield self
|
8
|
+
end
|
9
|
+
|
10
|
+
def singleton_class
|
11
|
+
class << self
|
12
|
+
self
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
module Gensym
|
17
|
+
@mutex = Mutex.new
|
18
|
+
@count = 0
|
19
|
+
|
20
|
+
def gensym(prefix = nil)
|
21
|
+
count = Gensym.module_eval {
|
22
|
+
@mutex.synchronize {
|
23
|
+
@count += 1
|
24
|
+
}
|
25
|
+
}
|
26
|
+
"#{prefix || :G}_#{count}".to_sym
|
27
|
+
end
|
28
|
+
end
|
29
|
+
include Gensym
|
30
|
+
|
31
|
+
def call_private(method, *args, &block)
|
32
|
+
instance_eval { send(method, *args, &block) }
|
33
|
+
end
|
34
|
+
|
35
|
+
def with_warnings(value = true)
|
36
|
+
previous = $VERBOSE
|
37
|
+
$VERBOSE = value
|
38
|
+
begin
|
39
|
+
yield
|
40
|
+
ensure
|
41
|
+
$VERBOSE = previous
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def no_warnings(&block)
|
46
|
+
with_warnings(false, &block)
|
47
|
+
end
|
48
|
+
|
49
|
+
def abort_on_exception(value = true)
|
50
|
+
previous = Thread.abort_on_exception
|
51
|
+
Thread.abort_on_exception = value
|
52
|
+
begin
|
53
|
+
yield
|
54
|
+
ensure
|
55
|
+
Thread.abort_on_exception = previous
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
extend self
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
|
2
|
+
module Rake ; end
|
3
|
+
module Rake::CompTree
|
4
|
+
module RetriableFork
|
5
|
+
HAVE_FORK = lambda {
|
6
|
+
begin
|
7
|
+
process_id = fork { }
|
8
|
+
Process.wait(process_id)
|
9
|
+
rescue NotImplementedError
|
10
|
+
return false
|
11
|
+
end
|
12
|
+
true
|
13
|
+
}.call
|
14
|
+
|
15
|
+
def fork(retry_wait = 10, retry_max = 10, &block)
|
16
|
+
num_retries = 0
|
17
|
+
begin
|
18
|
+
Process.fork(&block)
|
19
|
+
rescue Errno::EAGAIN
|
20
|
+
num_retries += 1
|
21
|
+
if num_retries == retry_max
|
22
|
+
message = %Q{
|
23
|
+
****************************************************************
|
24
|
+
Maximum number of EAGAIN signals reached (#{retry_max})
|
25
|
+
****************************************************************
|
26
|
+
|
27
|
+
Either increase your process limit permission (consult your
|
28
|
+
OS manual) or run this script as superuser.
|
29
|
+
|
30
|
+
****************************************************************
|
31
|
+
}
|
32
|
+
STDERR.puts(message.gsub(%r!^[ \t]+!, ""))
|
33
|
+
raise
|
34
|
+
end
|
35
|
+
STDERR.puts "Caught EGAIN. Retrying in #{retry_wait} seconds."
|
36
|
+
sleep(retry_wait)
|
37
|
+
retry
|
38
|
+
end
|
39
|
+
end
|
40
|
+
module_function :fork
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
@@ -0,0 +1,22 @@
|
|
1
|
+
|
2
|
+
module Rake::CompTree
|
3
|
+
#
|
4
|
+
# TaskNode is a Node which discards its results
|
5
|
+
#
|
6
|
+
class TaskNode < Node
|
7
|
+
def compute #:nodoc:
|
8
|
+
@function.call
|
9
|
+
true
|
10
|
+
end
|
11
|
+
|
12
|
+
class << self
|
13
|
+
#
|
14
|
+
# TaskNode always returns true.
|
15
|
+
#
|
16
|
+
def discard_result?
|
17
|
+
true
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: drake
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.2.1.0.
|
4
|
+
version: 0.8.2.1.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- James M. Lawrence
|
@@ -69,6 +69,16 @@ files:
|
|
69
69
|
- lib/rake/tasklib.rb
|
70
70
|
- lib/rake/testtask.rb
|
71
71
|
- lib/rake.rb
|
72
|
+
- lib/rake/comp_tree/algorithm.rb
|
73
|
+
- lib/rake/comp_tree/bucket_ipc.rb
|
74
|
+
- lib/rake/comp_tree/driver.rb
|
75
|
+
- lib/rake/comp_tree/error.rb
|
76
|
+
- lib/rake/comp_tree/node.rb
|
77
|
+
- lib/rake/comp_tree/quix/builtin/kernel/tap.rb
|
78
|
+
- lib/rake/comp_tree/quix/diagnostic.rb
|
79
|
+
- lib/rake/comp_tree/quix/kernel.rb
|
80
|
+
- lib/rake/comp_tree/retriable_fork.rb
|
81
|
+
- lib/rake/comp_tree/task_node.rb
|
72
82
|
- test/capture_stdout.rb
|
73
83
|
- test/check_expansion.rb
|
74
84
|
- test/contrib/test_sys.rb
|