drake 0.8.2.1.0.5 → 0.8.2.1.0.6
Sign up to get free protection for your applications and to get access to all the features.
- data/CHANGES.drake +1 -1
- data/Rakefile +1 -0
- data/Rakefile.drake +2 -2
- data/lib/rake.rb +1 -1
- data/lib/rake/comp_tree/algorithm.rb +210 -0
- data/lib/rake/comp_tree/bucket_ipc.rb +151 -0
- data/lib/rake/comp_tree/driver.rb +267 -0
- data/lib/rake/comp_tree/error.rb +27 -0
- data/lib/rake/comp_tree/node.rb +165 -0
- data/lib/rake/comp_tree/quix/builtin/kernel/tap.rb +9 -0
- data/lib/rake/comp_tree/quix/diagnostic.rb +44 -0
- data/lib/rake/comp_tree/quix/kernel.rb +61 -0
- data/lib/rake/comp_tree/retriable_fork.rb +43 -0
- data/lib/rake/comp_tree/task_node.rb +22 -0
- metadata +11 -1
data/CHANGES.drake
CHANGED
data/Rakefile
CHANGED
data/Rakefile.drake
CHANGED
@@ -40,7 +40,7 @@ task :pull_contrib => [ :init_contrib, :run_pull_contrib ]
|
|
40
40
|
######################################################################
|
41
41
|
# drake_release
|
42
42
|
|
43
|
-
task :drake_prerelease => :clean do
|
43
|
+
task :drake_prerelease => [:clean, :gemspec] do
|
44
44
|
rm_rf("html")
|
45
45
|
rm_rf("pkg")
|
46
46
|
unless `git status` =~ %r!nothing to commit \(working directory clean\)!
|
@@ -77,7 +77,7 @@ task :drake_finish_release do
|
|
77
77
|
}
|
78
78
|
|
79
79
|
git("tag", "drake-" + SPEC.version.to_s)
|
80
|
-
git(
|
80
|
+
git(*%w(push --tags origin master))
|
81
81
|
end
|
82
82
|
|
83
83
|
task :drake_release =>
|
data/lib/rake.rb
CHANGED
@@ -0,0 +1,210 @@
|
|
1
|
+
|
2
|
+
require 'rake/comp_tree/quix/diagnostic'
|
3
|
+
require 'rake/comp_tree/retriable_fork'
|
4
|
+
|
5
|
+
module Rake::CompTree
|
6
|
+
module Algorithm
|
7
|
+
include Quix::Diagnostic
|
8
|
+
|
9
|
+
def compute_multithreaded(root, num_threads, use_fork, buckets)
|
10
|
+
trace "Computing #{root.name} with #{num_threads} threads"
|
11
|
+
result = nil
|
12
|
+
mutex = Mutex.new
|
13
|
+
node_finished_condition = ConditionVariable.new
|
14
|
+
thread_wake_condition = ConditionVariable.new
|
15
|
+
threads = []
|
16
|
+
|
17
|
+
# workaround: jruby gives "run" status for waiting on
|
18
|
+
# condition variable
|
19
|
+
num_threads_ready = 0
|
20
|
+
|
21
|
+
num_threads.times { |thread_index|
|
22
|
+
threads << Thread.new {
|
23
|
+
#
|
24
|
+
# wait for main thread
|
25
|
+
#
|
26
|
+
mutex.synchronize {
|
27
|
+
trace "Thread #{thread_index} waiting to start"
|
28
|
+
num_threads_ready += 1
|
29
|
+
thread_wake_condition.wait(mutex)
|
30
|
+
}
|
31
|
+
|
32
|
+
while true
|
33
|
+
trace "Thread #{thread_index} node search"
|
34
|
+
|
35
|
+
#
|
36
|
+
# Done! Thread will exit.
|
37
|
+
#
|
38
|
+
break if mutex.synchronize {
|
39
|
+
result
|
40
|
+
}
|
41
|
+
|
42
|
+
#
|
43
|
+
# Lock the tree and find a node. The node we
|
44
|
+
# obtain, if any, is already locked.
|
45
|
+
#
|
46
|
+
node = mutex.synchronize {
|
47
|
+
find_node(root)
|
48
|
+
}
|
49
|
+
|
50
|
+
if node
|
51
|
+
trace "Thread #{thread_index} found node #{node.name}"
|
52
|
+
|
53
|
+
node_result =
|
54
|
+
compute_node(
|
55
|
+
node,
|
56
|
+
use_fork,
|
57
|
+
buckets ? buckets[thread_index] : nil)
|
58
|
+
|
59
|
+
mutex.synchronize {
|
60
|
+
node.result = node_result
|
61
|
+
}
|
62
|
+
|
63
|
+
#
|
64
|
+
# remove locks for this node (shared lock and own lock)
|
65
|
+
#
|
66
|
+
mutex.synchronize {
|
67
|
+
node.unlock
|
68
|
+
if node == root
|
69
|
+
#
|
70
|
+
# Root node was computed; we are done.
|
71
|
+
#
|
72
|
+
trace "Thread #{thread_index} got final answer"
|
73
|
+
result = root.result
|
74
|
+
end
|
75
|
+
node_finished_condition.signal
|
76
|
+
}
|
77
|
+
else
|
78
|
+
trace "Thread #{thread_index}: no node found; sleeping."
|
79
|
+
mutex.synchronize {
|
80
|
+
thread_wake_condition.wait(mutex)
|
81
|
+
}
|
82
|
+
end
|
83
|
+
end
|
84
|
+
trace "Thread #{thread_index} exiting"
|
85
|
+
}
|
86
|
+
}
|
87
|
+
|
88
|
+
trace "Main: waiting for threads to launch and block."
|
89
|
+
while true
|
90
|
+
break if mutex.synchronize {
|
91
|
+
num_threads_ready == num_threads
|
92
|
+
}
|
93
|
+
Thread.pass
|
94
|
+
end
|
95
|
+
|
96
|
+
trace "Main: entering main loop"
|
97
|
+
mutex.synchronize {
|
98
|
+
while true
|
99
|
+
trace "Main: waking threads"
|
100
|
+
thread_wake_condition.broadcast
|
101
|
+
|
102
|
+
if result
|
103
|
+
trace "Main: detected finish."
|
104
|
+
break
|
105
|
+
end
|
106
|
+
|
107
|
+
trace "Main: waiting for a node"
|
108
|
+
node_finished_condition.wait(mutex)
|
109
|
+
trace "Main: got a node"
|
110
|
+
end
|
111
|
+
}
|
112
|
+
|
113
|
+
trace "Main: waiting for threads to finish."
|
114
|
+
catch(:done) {
|
115
|
+
while true
|
116
|
+
mutex.synchronize {
|
117
|
+
throw :done if threads.all? { |thread|
|
118
|
+
thread.status == false
|
119
|
+
}
|
120
|
+
thread_wake_condition.broadcast
|
121
|
+
}
|
122
|
+
Thread.pass
|
123
|
+
end
|
124
|
+
}
|
125
|
+
|
126
|
+
trace "Main: computation done."
|
127
|
+
result
|
128
|
+
end
|
129
|
+
|
130
|
+
def find_node(node)
|
131
|
+
# --- only called inside mutex
|
132
|
+
trace "Looking for a node, starting with #{node.name}"
|
133
|
+
if node.result
|
134
|
+
#
|
135
|
+
# already computed
|
136
|
+
#
|
137
|
+
trace "#{node.name} has been computed"
|
138
|
+
nil
|
139
|
+
elsif node.children_results and node.try_lock
|
140
|
+
#
|
141
|
+
# Node is not computed and its children are computed;
|
142
|
+
# and we have the lock. Ready to compute.
|
143
|
+
#
|
144
|
+
node
|
145
|
+
else
|
146
|
+
#
|
147
|
+
# locked or children not computed; recurse to children
|
148
|
+
#
|
149
|
+
trace "Checking #{node.name}'s children"
|
150
|
+
node.each_child { |child|
|
151
|
+
if next_node = find_node(child)
|
152
|
+
return next_node
|
153
|
+
end
|
154
|
+
}
|
155
|
+
nil
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
def compute_node(node, use_fork, bucket)
|
160
|
+
if use_fork
|
161
|
+
trace "About to fork for node #{node.name}"
|
162
|
+
if bucket
|
163
|
+
#
|
164
|
+
# Use our assigned bucket to transfer the result.
|
165
|
+
#
|
166
|
+
fork_node(node) {
|
167
|
+
node.trace_compute
|
168
|
+
bucket.contents = node.compute
|
169
|
+
}
|
170
|
+
bucket.contents
|
171
|
+
else
|
172
|
+
#
|
173
|
+
# No bucket -- discarding result
|
174
|
+
#
|
175
|
+
fork_node(node) {
|
176
|
+
node.trace_compute
|
177
|
+
node.compute
|
178
|
+
}
|
179
|
+
true
|
180
|
+
end
|
181
|
+
else
|
182
|
+
#
|
183
|
+
# No fork
|
184
|
+
#
|
185
|
+
node.trace_compute
|
186
|
+
node.compute
|
187
|
+
end
|
188
|
+
end
|
189
|
+
|
190
|
+
def fork_node(node)
|
191
|
+
trace "About to fork for node #{node.name}"
|
192
|
+
process_id = RetriableFork.fork {
|
193
|
+
trace "Fork: process #{Process.pid}"
|
194
|
+
node.trace_compute
|
195
|
+
yield
|
196
|
+
trace "Fork: computation done"
|
197
|
+
}
|
198
|
+
trace "Waiting for process #{process_id}"
|
199
|
+
Process.wait(process_id)
|
200
|
+
trace "Process #{process_id} finished"
|
201
|
+
exitstatus = $?.exitstatus
|
202
|
+
if exitstatus != 0
|
203
|
+
trace "Process #{process_id} returned #{exitstatus}; exiting."
|
204
|
+
exit(1)
|
205
|
+
end
|
206
|
+
end
|
207
|
+
|
208
|
+
extend self
|
209
|
+
end
|
210
|
+
end
|
@@ -0,0 +1,151 @@
|
|
1
|
+
|
2
|
+
require 'drb'
|
3
|
+
require 'thread'
|
4
|
+
|
5
|
+
require 'rake/comp_tree/retriable_fork'
|
6
|
+
require 'rake/comp_tree/quix/diagnostic'
|
7
|
+
require 'rake/comp_tree/quix/builtin/kernel/tap'
|
8
|
+
|
9
|
+
module Rake::CompTree
|
10
|
+
module BucketIPC
|
11
|
+
class Bucket
|
12
|
+
include Quix::Diagnostic
|
13
|
+
include RetriableFork
|
14
|
+
|
15
|
+
def initialize(address, timeout, wait_interval)
|
16
|
+
trace "Making bucket with address #{address}"
|
17
|
+
|
18
|
+
@remote_pid = fork {
|
19
|
+
own_object = Class.new {
|
20
|
+
attr_accessor(:contents)
|
21
|
+
}.new
|
22
|
+
server = DRb.start_service(address, own_object)
|
23
|
+
debug {
|
24
|
+
server.verbose = true
|
25
|
+
}
|
26
|
+
DRb.thread.join
|
27
|
+
}
|
28
|
+
|
29
|
+
@remote_object = DRbObject.new_with_uri(address)
|
30
|
+
@address = address
|
31
|
+
@timeout = timeout
|
32
|
+
@wait_interval = wait_interval
|
33
|
+
end
|
34
|
+
|
35
|
+
attr_accessor(:timeout, :wait_interval)
|
36
|
+
attr_reader(:address)
|
37
|
+
|
38
|
+
def contents=(new_contents)
|
39
|
+
connect {
|
40
|
+
@remote_object.contents = new_contents
|
41
|
+
}
|
42
|
+
end
|
43
|
+
|
44
|
+
def contents
|
45
|
+
connect {
|
46
|
+
@remote_object.contents
|
47
|
+
}
|
48
|
+
end
|
49
|
+
|
50
|
+
def stop
|
51
|
+
Process.kill("TERM", @remote_pid)
|
52
|
+
end
|
53
|
+
|
54
|
+
private
|
55
|
+
|
56
|
+
def connect
|
57
|
+
begin
|
58
|
+
return yield
|
59
|
+
rescue DRb::DRbConnError
|
60
|
+
start = Time.now
|
61
|
+
begin
|
62
|
+
Kernel.sleep(@wait_interval)
|
63
|
+
return yield
|
64
|
+
rescue DRb::DRbConnError
|
65
|
+
if Time.now - start > @timeout
|
66
|
+
raise
|
67
|
+
end
|
68
|
+
retry
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
class DriverBase
|
75
|
+
def initialize(addresses, timeout, wait_interval)
|
76
|
+
begin
|
77
|
+
@buckets = addresses.map { |address|
|
78
|
+
Bucket.new(address, timeout, wait_interval)
|
79
|
+
}
|
80
|
+
if block_given?
|
81
|
+
yield @buckets
|
82
|
+
end
|
83
|
+
ensure
|
84
|
+
if block_given?
|
85
|
+
stop
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
def stop
|
91
|
+
if defined?(@buckets)
|
92
|
+
@buckets.each { |bucket|
|
93
|
+
bucket.stop
|
94
|
+
}
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
class Driver < DriverBase
|
100
|
+
DEFAULTS = {
|
101
|
+
:timeout => 0.5,
|
102
|
+
:wait_interval => 0.05,
|
103
|
+
:port_start => 18181,
|
104
|
+
}
|
105
|
+
|
106
|
+
module BucketCounter
|
107
|
+
@mutex = Mutex.new
|
108
|
+
@count = 0
|
109
|
+
class << self
|
110
|
+
def increment_count
|
111
|
+
@mutex.synchronize {
|
112
|
+
@count += 1
|
113
|
+
}
|
114
|
+
end
|
115
|
+
|
116
|
+
def map_indexes(num_buckets)
|
117
|
+
Array.new.tap { |result|
|
118
|
+
num_buckets.times {
|
119
|
+
result << yield(increment_count)
|
120
|
+
}
|
121
|
+
}
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
def initialize(num_buckets, opts_in = {})
|
127
|
+
opts = DEFAULTS.merge(opts_in)
|
128
|
+
|
129
|
+
addresses =
|
130
|
+
if RetriableFork::HAVE_FORK
|
131
|
+
#
|
132
|
+
# Assume the existence of fork implies a unix machine.
|
133
|
+
#
|
134
|
+
require 'drb/unix'
|
135
|
+
basename = "drbunix://#{Dir.tmpdir}/bucket.#{Process.pid}.#{rand}"
|
136
|
+
BucketCounter.map_indexes(num_buckets) { |index|
|
137
|
+
"#{basename}.#{index}"
|
138
|
+
}
|
139
|
+
else
|
140
|
+
#
|
141
|
+
# Fallback: use the default socket.
|
142
|
+
#
|
143
|
+
BucketCounter.map_indexes(num_buckets) { |index|
|
144
|
+
"druby://localhost:#{opts[:port_start] + index}"
|
145
|
+
}
|
146
|
+
end
|
147
|
+
super(addresses, opts[:timeout], opts[:wait_interval])
|
148
|
+
end
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
@@ -0,0 +1,267 @@
|
|
1
|
+
|
2
|
+
require 'rake/comp_tree/bucket_ipc'
|
3
|
+
require 'rake/comp_tree/quix/diagnostic'
|
4
|
+
require 'rake/comp_tree/quix/kernel'
|
5
|
+
require 'rake/comp_tree/algorithm'
|
6
|
+
require 'rake/comp_tree/node'
|
7
|
+
require 'rake/comp_tree/task_node'
|
8
|
+
require 'rake/comp_tree/error'
|
9
|
+
|
10
|
+
require 'thread'
|
11
|
+
|
12
|
+
module Rake::CompTree
|
13
|
+
#
|
14
|
+
# Driver is the main interface to the computation tree. It is
|
15
|
+
# responsible for defining nodes and running computations.
|
16
|
+
#
|
17
|
+
class Driver
|
18
|
+
DEFAULTS = {
|
19
|
+
:threads => 1,
|
20
|
+
:fork => false,
|
21
|
+
:timeout => 5.0,
|
22
|
+
:wait_interval => 0.02,
|
23
|
+
}
|
24
|
+
|
25
|
+
include Quix::Diagnostic #:nodoc:
|
26
|
+
include Quix::Kernel #:nodoc:
|
27
|
+
|
28
|
+
#
|
29
|
+
# Begin a new computation tree.
|
30
|
+
#
|
31
|
+
# Options hash:
|
32
|
+
#
|
33
|
+
# <tt>:node_class</tt> -- (Class) Rake::CompTree::Node subclass from
|
34
|
+
# which nodes are created.
|
35
|
+
#
|
36
|
+
# <tt>:discard_result</tt> -- (boolean) If you are <em>not</em>
|
37
|
+
# interested in the final answer, but only in the actions which
|
38
|
+
# complete the computation, then set this to +true+. This is
|
39
|
+
# equivalent to saying <tt>:node_class => Rake::CompTree::TaskNode</tt>.
|
40
|
+
# (If you are forking processes, it is good to know that IPC is
|
41
|
+
# not needed to communicate the result.)
|
42
|
+
#
|
43
|
+
def initialize(opts = nil)
|
44
|
+
if opts and opts[:node_class] and opts[:discard_result]
|
45
|
+
raise(
|
46
|
+
Error::ArgumentError,
|
47
|
+
"#{self.class.name}.new: :discard_result and :node_class " +
|
48
|
+
"are mutually exclusive")
|
49
|
+
end
|
50
|
+
|
51
|
+
@node_class =
|
52
|
+
if opts and opts[:node_class]
|
53
|
+
opts[:node_class]
|
54
|
+
elsif opts and opts[:discard_result]
|
55
|
+
TaskNode
|
56
|
+
else
|
57
|
+
Node
|
58
|
+
end
|
59
|
+
|
60
|
+
@nodes = Hash.new
|
61
|
+
|
62
|
+
if block_given?
|
63
|
+
yield self
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
#
|
68
|
+
# Name-to-node hash.
|
69
|
+
#
|
70
|
+
attr_reader :nodes
|
71
|
+
|
72
|
+
#
|
73
|
+
# Define a computation node.
|
74
|
+
#
|
75
|
+
# There are three distinct forms of a node definition. In each of
|
76
|
+
# the following examples, a computation node named +area+ is
|
77
|
+
# defined which depends on the nodes +height+, +width+, +offset+.
|
78
|
+
#
|
79
|
+
# The method_missing form:
|
80
|
+
# driver.define_area(:width, :height, :offset) { |width, height, offset|
|
81
|
+
# width*height - offset
|
82
|
+
# }
|
83
|
+
#
|
84
|
+
# The eval form:
|
85
|
+
# driver.define_area :width, :height, :offset, %{
|
86
|
+
# width*height - offset
|
87
|
+
# }
|
88
|
+
# (Note the '%' before the brace.)
|
89
|
+
#
|
90
|
+
# The raw form:
|
91
|
+
# driver.define(:area, :width, :height, :offset) { |width, height, offset|
|
92
|
+
# width*height - offset
|
93
|
+
# }
|
94
|
+
#
|
95
|
+
def define(*args, &block)
|
96
|
+
parent_name = args.first
|
97
|
+
children_names = args[1..-1]
|
98
|
+
|
99
|
+
unless parent_name
|
100
|
+
raise Error::ArgumentError, "No name given for node"
|
101
|
+
end
|
102
|
+
|
103
|
+
#
|
104
|
+
# retrieve or create parent and children
|
105
|
+
#
|
106
|
+
parent =
|
107
|
+
if t = @nodes[parent_name]
|
108
|
+
t
|
109
|
+
else
|
110
|
+
@nodes[parent_name] = @node_class.new(parent_name)
|
111
|
+
end
|
112
|
+
|
113
|
+
if parent.function
|
114
|
+
raise Error::RedefinitionError, "Node #{parent.name} already defined."
|
115
|
+
end
|
116
|
+
parent.function = block
|
117
|
+
|
118
|
+
children = children_names.map { |child_name|
|
119
|
+
if t = @nodes[child_name]
|
120
|
+
t
|
121
|
+
else
|
122
|
+
@nodes[child_name] = @node_class.new(child_name)
|
123
|
+
end
|
124
|
+
}
|
125
|
+
|
126
|
+
#
|
127
|
+
# link
|
128
|
+
#
|
129
|
+
parent.children = children
|
130
|
+
children.each { |child|
|
131
|
+
child.parents << parent
|
132
|
+
}
|
133
|
+
end
|
134
|
+
|
135
|
+
#
|
136
|
+
# parsing/evaling helper
|
137
|
+
#
|
138
|
+
def evaling_define(*args) #:nodoc:
|
139
|
+
function_name = args[0]
|
140
|
+
function_arg_names = args[1..-2]
|
141
|
+
function_string = args.last.to_str
|
142
|
+
|
143
|
+
comma_separated = function_arg_names.map { |name|
|
144
|
+
name.to_s
|
145
|
+
}.join(",")
|
146
|
+
|
147
|
+
eval_me = %{
|
148
|
+
lambda { |#{comma_separated}|
|
149
|
+
#{function_string}
|
150
|
+
}
|
151
|
+
}
|
152
|
+
|
153
|
+
function = eval(eval_me, TOPLEVEL_BINDING)
|
154
|
+
|
155
|
+
define(function_name, *function_arg_names, &function)
|
156
|
+
end
|
157
|
+
|
158
|
+
def method_missing(symbol, *args, &block) #:nodoc:
|
159
|
+
if match = symbol.to_s.match(%r!\Adefine_(\w+)\Z!)
|
160
|
+
method_name = match.captures.first.to_sym
|
161
|
+
if block
|
162
|
+
define(method_name, *args, &block)
|
163
|
+
else
|
164
|
+
evaling_define(method_name, *args)
|
165
|
+
end
|
166
|
+
else
|
167
|
+
super(symbol, *args, &block)
|
168
|
+
end
|
169
|
+
end
|
170
|
+
|
171
|
+
#
|
172
|
+
# Mark this node and all its children as uncomputed.
|
173
|
+
#
|
174
|
+
# Arguments:
|
175
|
+
#
|
176
|
+
# +name+ -- (Symbol) node name.
|
177
|
+
#
|
178
|
+
def reset(name)
|
179
|
+
@nodes[name].reset
|
180
|
+
end
|
181
|
+
|
182
|
+
#
|
183
|
+
# Check for a cyclic graph below the given node. Raises
|
184
|
+
# Rake::CompTree::Error::CircularError if found.
|
185
|
+
#
|
186
|
+
# Arguments:
|
187
|
+
#
|
188
|
+
# +name+ -- (Symbol) node name.
|
189
|
+
#
|
190
|
+
def check_circular(name)
|
191
|
+
helper = lambda { |root, chain|
|
192
|
+
if chain.include? root
|
193
|
+
raise Error::CircularError,
|
194
|
+
"Circular dependency detected: #{root} => #{chain.last} => #{root}"
|
195
|
+
end
|
196
|
+
@nodes[root].children.each { |child|
|
197
|
+
helper.call(child.name, chain + [root])
|
198
|
+
}
|
199
|
+
}
|
200
|
+
helper.call(name, [])
|
201
|
+
end
|
202
|
+
|
203
|
+
#
|
204
|
+
# Compute this node.
|
205
|
+
#
|
206
|
+
# Arguments:
|
207
|
+
#
|
208
|
+
# +name+ -- (Symbol) node name.
|
209
|
+
#
|
210
|
+
# Options hash:
|
211
|
+
#
|
212
|
+
# <tt>:threads</tt> -- (Integer) Number of parallel threads.
|
213
|
+
#
|
214
|
+
# <tt>:fork</tt> -- (boolean) Whether to fork each computation
|
215
|
+
# node into its own process.
|
216
|
+
#
|
217
|
+
# Defaults options are taken from Driver::DEFAULTS.
|
218
|
+
#
|
219
|
+
def compute(name, opts = nil)
|
220
|
+
#
|
221
|
+
# Undocumented options:
|
222
|
+
#
|
223
|
+
# <tt>:wait_interval</tt> -- (seconds) (Obscure) How long to
|
224
|
+
# wait after an IPC failure.
|
225
|
+
#
|
226
|
+
# <tt>:timeout</tt> -- (seconds) (Obscure) Give up after this
|
227
|
+
# period of persistent IPC failures.
|
228
|
+
#
|
229
|
+
|
230
|
+
abort_on_exception {
|
231
|
+
compute_private(name, opts || Hash.new)
|
232
|
+
}
|
233
|
+
end
|
234
|
+
|
235
|
+
private
|
236
|
+
|
237
|
+
def compute_private(name, opts_in)
|
238
|
+
opts = DEFAULTS.merge(opts_in)
|
239
|
+
root = @nodes[name]
|
240
|
+
|
241
|
+
if opts[:threads] < 1
|
242
|
+
raise Error::ArgumentError, "threads is #{opts[:threads]}"
|
243
|
+
end
|
244
|
+
|
245
|
+
if opts[:threads] == 1
|
246
|
+
root.result = root.compute_now
|
247
|
+
elsif opts[:fork] and not @node_class.discard_result?
|
248
|
+
#
|
249
|
+
# Use buckets to send results across forks.
|
250
|
+
#
|
251
|
+
result = nil
|
252
|
+
BucketIPC::Driver.new(opts[:threads], opts) { |buckets|
|
253
|
+
result =
|
254
|
+
Algorithm.compute_multithreaded(
|
255
|
+
root, opts[:threads], opts[:fork], buckets)
|
256
|
+
}
|
257
|
+
result
|
258
|
+
else
|
259
|
+
#
|
260
|
+
# Multithreaded computation without fork.
|
261
|
+
#
|
262
|
+
Algorithm.compute_multithreaded(
|
263
|
+
root, opts[:threads], opts[:fork], nil)
|
264
|
+
end
|
265
|
+
end
|
266
|
+
end
|
267
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
|
2
|
+
module Rake::CompTree
|
3
|
+
module Error
|
4
|
+
# Base class for Rake::CompTree errors.
|
5
|
+
class Base < StandardError ; end
|
6
|
+
|
7
|
+
# Internal error inside Rake::CompTree. Please send a bug report.
|
8
|
+
class AssertionFailed < Base ; end
|
9
|
+
|
10
|
+
# Bad arguments were passed to a method.
|
11
|
+
class ArgumentError < Base ; end
|
12
|
+
|
13
|
+
#
|
14
|
+
# Attempt to redefine a Node.
|
15
|
+
#
|
16
|
+
# If you wish to only replace the function, set
|
17
|
+
# driver.nodes[name].function = some_new_lambda
|
18
|
+
#
|
19
|
+
class RedefinitionError < Base ; end
|
20
|
+
|
21
|
+
# A Cyclic graph was detected.
|
22
|
+
class CircularError < Base ; end
|
23
|
+
|
24
|
+
# No function was defined for this node.
|
25
|
+
class NoFunctionError < Base ; end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,165 @@
|
|
1
|
+
|
2
|
+
require 'rake/comp_tree/quix/diagnostic'
|
3
|
+
require 'thread'
|
4
|
+
|
5
|
+
module Rake::CompTree
|
6
|
+
#
|
7
|
+
# Base class for nodes in the computation tree.
|
8
|
+
#
|
9
|
+
class Node
|
10
|
+
include Quix::Diagnostic #:nodoc:
|
11
|
+
|
12
|
+
attr_reader :name #:nodoc:
|
13
|
+
|
14
|
+
attr_accessor :parents #:nodoc:
|
15
|
+
attr_accessor :children #:nodoc:
|
16
|
+
attr_accessor :function #:nodoc:
|
17
|
+
attr_accessor :result #:nodoc:
|
18
|
+
attr_accessor :shared_lock #:nodoc:
|
19
|
+
|
20
|
+
#
|
21
|
+
# Create a node
|
22
|
+
#
|
23
|
+
def initialize(name) #:nodoc:
|
24
|
+
@name = name
|
25
|
+
@mutex = Mutex.new
|
26
|
+
@children = []
|
27
|
+
@parents = []
|
28
|
+
reset_self
|
29
|
+
end
|
30
|
+
|
31
|
+
#
|
32
|
+
# Reset the computation for this node.
|
33
|
+
#
|
34
|
+
def reset_self #:nodoc:
|
35
|
+
@shared_lock = 0
|
36
|
+
@children_results = nil
|
37
|
+
@result = nil
|
38
|
+
end
|
39
|
+
|
40
|
+
#
|
41
|
+
# Reset the computation for this node and all children.
|
42
|
+
#
|
43
|
+
def reset #:nodoc:
|
44
|
+
each_downward { |node|
|
45
|
+
node.reset_self
|
46
|
+
}
|
47
|
+
end
|
48
|
+
|
49
|
+
def each_downward(&block) #:nodoc:
|
50
|
+
block.call(self)
|
51
|
+
@children.each { |child|
|
52
|
+
child.each_downward(&block)
|
53
|
+
}
|
54
|
+
end
|
55
|
+
|
56
|
+
def each_upward(&block) #:nodoc:
|
57
|
+
block.call(self)
|
58
|
+
@parents.each { |parent|
|
59
|
+
parent.each_upward(&block)
|
60
|
+
}
|
61
|
+
end
|
62
|
+
|
63
|
+
def each_child #:nodoc:
|
64
|
+
@children.each { |child|
|
65
|
+
yield(child)
|
66
|
+
}
|
67
|
+
end
|
68
|
+
|
69
|
+
#
|
70
|
+
# Force computation of all children; intended for
|
71
|
+
# single-threaded mode.
|
72
|
+
#
|
73
|
+
def compute_now #:nodoc:
|
74
|
+
unless @children_results
|
75
|
+
@children_results = @children.map { |child|
|
76
|
+
child.compute_now
|
77
|
+
}
|
78
|
+
end
|
79
|
+
compute
|
80
|
+
end
|
81
|
+
|
82
|
+
#
|
83
|
+
# If all children have been computed, return their results;
|
84
|
+
# otherwise return nil.
|
85
|
+
#
|
86
|
+
def children_results #:nodoc:
|
87
|
+
if @children_results
|
88
|
+
@children_results
|
89
|
+
else
|
90
|
+
results = @children.map { |child|
|
91
|
+
if child_result = child.result
|
92
|
+
child_result
|
93
|
+
else
|
94
|
+
return nil
|
95
|
+
end
|
96
|
+
}
|
97
|
+
@children_results = results
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
def trace_compute #:nodoc:
|
102
|
+
debug {
|
103
|
+
# --- own mutex
|
104
|
+
trace "Computing #{@name}"
|
105
|
+
raise Error::AssertionFailed if @result
|
106
|
+
raise Error::AssertionFailed unless @mutex.locked?
|
107
|
+
raise Error::AssertionFailed unless @children_results
|
108
|
+
}
|
109
|
+
end
|
110
|
+
|
111
|
+
#
|
112
|
+
# Compute this node; children must be computed and lock must be
|
113
|
+
# already acquired.
|
114
|
+
#
|
115
|
+
def compute #:nodoc:
|
116
|
+
unless defined?(@function) and @function
|
117
|
+
raise Error::NoFunctionError,
|
118
|
+
"No function was defined for node '#{@name.inspect}'"
|
119
|
+
end
|
120
|
+
@function.call(*@children_results)
|
121
|
+
end
|
122
|
+
|
123
|
+
def try_lock #:nodoc:
|
124
|
+
# --- shared tree mutex and own mutex
|
125
|
+
if @shared_lock == 0 and @mutex.try_lock
|
126
|
+
trace "Locking #{@name}"
|
127
|
+
each_upward { |node|
|
128
|
+
node.shared_lock += 1
|
129
|
+
trace "#{node.name} locked by #{@name}: level: #{node.shared_lock}"
|
130
|
+
}
|
131
|
+
true
|
132
|
+
else
|
133
|
+
false
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
def unlock #:nodoc:
|
138
|
+
# --- shared tree mutex and own mutex
|
139
|
+
debug {
|
140
|
+
raise Error::AssertionFailed unless @mutex.locked?
|
141
|
+
trace "Unlocking #{@name}"
|
142
|
+
}
|
143
|
+
each_upward { |node|
|
144
|
+
node.shared_lock -= 1
|
145
|
+
debug {
|
146
|
+
if node.shared_lock == 0
|
147
|
+
trace "#{node.name} unlocked by #{@name}"
|
148
|
+
end
|
149
|
+
}
|
150
|
+
}
|
151
|
+
@mutex.unlock
|
152
|
+
end
|
153
|
+
|
154
|
+
class << self
|
155
|
+
#
|
156
|
+
# Throw away the computation result?
|
157
|
+
#
|
158
|
+
# This Node base class always returns false.
|
159
|
+
#
|
160
|
+
def discard_result?
|
161
|
+
false
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
165
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
|
2
|
+
require 'rake/comp_tree/quix/builtin/kernel/tap'
|
3
|
+
|
4
|
+
module Rake::CompTree::Quix
|
5
|
+
module Diagnostic
|
6
|
+
def show(desc = nil, stream = STDOUT, &block)
|
7
|
+
if desc
|
8
|
+
stream.puts(desc)
|
9
|
+
end
|
10
|
+
if block
|
11
|
+
expression = block.call
|
12
|
+
eval(expression, block.binding).tap { |result|
|
13
|
+
stream.printf("%-16s => %s\n", expression, result.inspect)
|
14
|
+
}
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
if $DEBUG
|
19
|
+
def debug
|
20
|
+
yield
|
21
|
+
end
|
22
|
+
|
23
|
+
def debugging?
|
24
|
+
true
|
25
|
+
end
|
26
|
+
|
27
|
+
def trace(desc = nil, &block)
|
28
|
+
if desc
|
29
|
+
show("#{desc}.".sub(%r!\.\.+\Z!, ""), STDERR, &block)
|
30
|
+
else
|
31
|
+
show(nil, STDERR, &block)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
else
|
35
|
+
# non-$DEBUG
|
36
|
+
def debug ; end
|
37
|
+
def debugging? ; end
|
38
|
+
def trace(*args) ; end
|
39
|
+
end
|
40
|
+
|
41
|
+
extend self
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
@@ -0,0 +1,61 @@
|
|
1
|
+
|
2
|
+
require 'thread'
|
3
|
+
|
4
|
+
module Rake::CompTree::Quix
|
5
|
+
module Kernel
|
6
|
+
def let
|
7
|
+
yield self
|
8
|
+
end
|
9
|
+
|
10
|
+
def singleton_class
|
11
|
+
class << self
|
12
|
+
self
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
module Gensym
|
17
|
+
@mutex = Mutex.new
|
18
|
+
@count = 0
|
19
|
+
|
20
|
+
def gensym(prefix = nil)
|
21
|
+
count = Gensym.module_eval {
|
22
|
+
@mutex.synchronize {
|
23
|
+
@count += 1
|
24
|
+
}
|
25
|
+
}
|
26
|
+
"#{prefix || :G}_#{count}".to_sym
|
27
|
+
end
|
28
|
+
end
|
29
|
+
include Gensym
|
30
|
+
|
31
|
+
def call_private(method, *args, &block)
|
32
|
+
instance_eval { send(method, *args, &block) }
|
33
|
+
end
|
34
|
+
|
35
|
+
def with_warnings(value = true)
|
36
|
+
previous = $VERBOSE
|
37
|
+
$VERBOSE = value
|
38
|
+
begin
|
39
|
+
yield
|
40
|
+
ensure
|
41
|
+
$VERBOSE = previous
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def no_warnings(&block)
|
46
|
+
with_warnings(false, &block)
|
47
|
+
end
|
48
|
+
|
49
|
+
def abort_on_exception(value = true)
|
50
|
+
previous = Thread.abort_on_exception
|
51
|
+
Thread.abort_on_exception = value
|
52
|
+
begin
|
53
|
+
yield
|
54
|
+
ensure
|
55
|
+
Thread.abort_on_exception = previous
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
extend self
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
|
2
|
+
module Rake ; end
|
3
|
+
module Rake::CompTree
|
4
|
+
module RetriableFork
|
5
|
+
HAVE_FORK = lambda {
|
6
|
+
begin
|
7
|
+
process_id = fork { }
|
8
|
+
Process.wait(process_id)
|
9
|
+
rescue NotImplementedError
|
10
|
+
return false
|
11
|
+
end
|
12
|
+
true
|
13
|
+
}.call
|
14
|
+
|
15
|
+
def fork(retry_wait = 10, retry_max = 10, &block)
|
16
|
+
num_retries = 0
|
17
|
+
begin
|
18
|
+
Process.fork(&block)
|
19
|
+
rescue Errno::EAGAIN
|
20
|
+
num_retries += 1
|
21
|
+
if num_retries == retry_max
|
22
|
+
message = %Q{
|
23
|
+
****************************************************************
|
24
|
+
Maximum number of EAGAIN signals reached (#{retry_max})
|
25
|
+
****************************************************************
|
26
|
+
|
27
|
+
Either increase your process limit permission (consult your
|
28
|
+
OS manual) or run this script as superuser.
|
29
|
+
|
30
|
+
****************************************************************
|
31
|
+
}
|
32
|
+
STDERR.puts(message.gsub(%r!^[ \t]+!, ""))
|
33
|
+
raise
|
34
|
+
end
|
35
|
+
STDERR.puts "Caught EGAIN. Retrying in #{retry_wait} seconds."
|
36
|
+
sleep(retry_wait)
|
37
|
+
retry
|
38
|
+
end
|
39
|
+
end
|
40
|
+
module_function :fork
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
@@ -0,0 +1,22 @@
|
|
1
|
+
|
2
|
+
module Rake::CompTree
|
3
|
+
#
|
4
|
+
# TaskNode is a Node which discards its results
|
5
|
+
#
|
6
|
+
class TaskNode < Node
|
7
|
+
def compute #:nodoc:
|
8
|
+
@function.call
|
9
|
+
true
|
10
|
+
end
|
11
|
+
|
12
|
+
class << self
|
13
|
+
#
|
14
|
+
# TaskNode always returns true.
|
15
|
+
#
|
16
|
+
def discard_result?
|
17
|
+
true
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: drake
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.8.2.1.0.
|
4
|
+
version: 0.8.2.1.0.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- James M. Lawrence
|
@@ -69,6 +69,16 @@ files:
|
|
69
69
|
- lib/rake/tasklib.rb
|
70
70
|
- lib/rake/testtask.rb
|
71
71
|
- lib/rake.rb
|
72
|
+
- lib/rake/comp_tree/algorithm.rb
|
73
|
+
- lib/rake/comp_tree/bucket_ipc.rb
|
74
|
+
- lib/rake/comp_tree/driver.rb
|
75
|
+
- lib/rake/comp_tree/error.rb
|
76
|
+
- lib/rake/comp_tree/node.rb
|
77
|
+
- lib/rake/comp_tree/quix/builtin/kernel/tap.rb
|
78
|
+
- lib/rake/comp_tree/quix/diagnostic.rb
|
79
|
+
- lib/rake/comp_tree/quix/kernel.rb
|
80
|
+
- lib/rake/comp_tree/retriable_fork.rb
|
81
|
+
- lib/rake/comp_tree/task_node.rb
|
72
82
|
- test/capture_stdout.rb
|
73
83
|
- test/check_expansion.rb
|
74
84
|
- test/contrib/test_sys.rb
|