comp_tree 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. data/README +153 -0
  2. data/Rakefile +152 -0
  3. data/comp_tree.gemspec +38 -0
  4. data/contrib/quix/Rakefile +16 -0
  5. data/contrib/quix/install.rb +3 -0
  6. data/contrib/quix/lib/quix/builtin/dir/casefold_brackets.rb +7 -0
  7. data/contrib/quix/lib/quix/builtin/kernel/tap.rb +9 -0
  8. data/contrib/quix/lib/quix/builtin/module/include.rb +21 -0
  9. data/contrib/quix/lib/quix/builtin/module/private.rb +41 -0
  10. data/contrib/quix/lib/quix/config.rb +37 -0
  11. data/contrib/quix/lib/quix/cygwin.rb +60 -0
  12. data/contrib/quix/lib/quix/diagnostic.rb +44 -0
  13. data/contrib/quix/lib/quix/enumerable.rb +33 -0
  14. data/contrib/quix/lib/quix/fileutils.rb +37 -0
  15. data/contrib/quix/lib/quix/hash_struct.rb +27 -0
  16. data/contrib/quix/lib/quix/kernel.rb +61 -0
  17. data/contrib/quix/lib/quix/lazy_struct.rb +55 -0
  18. data/contrib/quix/lib/quix/simple_installer.rb +87 -0
  19. data/contrib/quix/lib/quix/string.rb +38 -0
  20. data/contrib/quix/lib/quix/subpackager.rb +52 -0
  21. data/contrib/quix/lib/quix/thread_local.rb +32 -0
  22. data/contrib/quix/lib/quix/vars.rb +138 -0
  23. data/contrib/quix/lib/quix.rb +32 -0
  24. data/contrib/quix/test/all.rb +12 -0
  25. data/contrib/quix/test/test_deps.rb +25 -0
  26. data/contrib/quix/test/test_include.rb +47 -0
  27. data/contrib/quix/test/test_private.rb +86 -0
  28. data/contrib/quix/test/test_root.rb +19 -0
  29. data/contrib/quix/test/test_struct.rb +48 -0
  30. data/contrib/quix/test/test_vars.rb +187 -0
  31. data/install.rb +3 -0
  32. data/lib/comp_tree/algorithm.rb +210 -0
  33. data/lib/comp_tree/bucket_ipc.rb +151 -0
  34. data/lib/comp_tree/driver.rb +267 -0
  35. data/lib/comp_tree/error.rb +27 -0
  36. data/lib/comp_tree/node.rb +165 -0
  37. data/lib/comp_tree/quix/builtin/kernel/tap.rb +33 -0
  38. data/lib/comp_tree/quix/diagnostic.rb +68 -0
  39. data/lib/comp_tree/quix/kernel.rb +85 -0
  40. data/lib/comp_tree/retriable_fork.rb +42 -0
  41. data/lib/comp_tree/task_node.rb +22 -0
  42. data/lib/comp_tree.rb +23 -0
  43. data/test/all.rb +12 -0
  44. data/test/test_bucketipc.rb +72 -0
  45. data/test/test_circular.rb +36 -0
  46. data/test/test_comp_tree.rb +364 -0
  47. data/test/test_exception.rb +97 -0
  48. metadata +120 -0
@@ -0,0 +1,267 @@
1
+
2
+ require 'comp_tree/bucket_ipc'
3
+ require 'comp_tree/quix/diagnostic'
4
+ require 'comp_tree/quix/kernel'
5
+ require 'comp_tree/algorithm'
6
+ require 'comp_tree/node'
7
+ require 'comp_tree/task_node'
8
+ require 'comp_tree/error'
9
+
10
+ require 'thread'
11
+
12
+ module CompTree
13
+ #
14
+ # Driver is the main interface to the computation tree. It is
15
+ # responsible for defining nodes and running computations.
16
+ #
17
+ class Driver
18
+ DEFAULTS = {
19
+ :threads => 1,
20
+ :fork => false,
21
+ :timeout => 5.0,
22
+ :wait_interval => 0.02,
23
+ }
24
+
25
+ include Quix::Diagnostic #:nodoc:
26
+ include Quix::Kernel #:nodoc:
27
+
28
+ #
29
+ # Begin a new computation tree.
30
+ #
31
+ # Options hash:
32
+ #
33
+ # <tt>:node_class</tt> -- (Class) CompTree::Node subclass from
34
+ # which nodes are created.
35
+ #
36
+ # <tt>:discard_result</tt> -- (boolean) If you are <em>not</em>
37
+ # interested in the final answer, but only in the actions which
38
+ # complete the computation, then set this to +true+. This is
39
+ # equivalent to saying <tt>:node_class => CompTree::TaskNode</tt>.
40
+ # (If you are forking processes, it is good to know that IPC is
41
+ # not needed to communicate the result.)
42
+ #
43
+ def initialize(opts = nil)
44
+ if opts and opts[:node_class] and opts[:discard_result]
45
+ raise(
46
+ Error::ArgumentError,
47
+ "#{self.class.name}.new: :discard_result and :node_class " +
48
+ "are mutually exclusive")
49
+ end
50
+
51
+ @node_class =
52
+ if opts and opts[:node_class]
53
+ opts[:node_class]
54
+ elsif opts and opts[:discard_result]
55
+ TaskNode
56
+ else
57
+ Node
58
+ end
59
+
60
+ @nodes = Hash.new
61
+
62
+ if block_given?
63
+ yield self
64
+ end
65
+ end
66
+
67
+ #
68
+ # Name-to-node hash.
69
+ #
70
+ attr_reader :nodes
71
+
72
+ #
73
+ # Define a computation node.
74
+ #
75
+ # There are three distinct forms of a node definition. In each of
76
+ # the following examples, a computation node named +area+ is
77
+ # defined which depends on the nodes +height+, +width+, +offset+.
78
+ #
79
+ # The method_missing form:
80
+ # driver.define_area(:width, :height, :offset) { |width, height, offset|
81
+ # width*height - offset
82
+ # }
83
+ #
84
+ # The eval form:
85
+ # driver.define_area :width, :height, :offset, %{
86
+ # width*height - offset
87
+ # }
88
+ # (Note the '%' before the brace.)
89
+ #
90
+ # The raw form:
91
+ # driver.define(:area, :width, :height, :offset) { |width, height, offset|
92
+ # width*height - offset
93
+ # }
94
+ #
95
+ def define(*args, &block)
96
+ parent_name = args.first
97
+ children_names = args[1..-1]
98
+
99
+ unless parent_name
100
+ raise Error::ArgumentError, "No name given for node"
101
+ end
102
+
103
+ #
104
+ # retrieve or create parent and children
105
+ #
106
+ parent =
107
+ if t = @nodes[parent_name]
108
+ t
109
+ else
110
+ @nodes[parent_name] = @node_class.new(parent_name)
111
+ end
112
+
113
+ if parent.function
114
+ raise Error::RedefinitionError, "Node #{parent.name} already defined."
115
+ end
116
+ parent.function = block
117
+
118
+ children = children_names.map { |child_name|
119
+ if t = @nodes[child_name]
120
+ t
121
+ else
122
+ @nodes[child_name] = @node_class.new(child_name)
123
+ end
124
+ }
125
+
126
+ #
127
+ # link
128
+ #
129
+ parent.children = children
130
+ children.each { |child|
131
+ child.parents << parent
132
+ }
133
+ end
134
+
135
+ #
136
+ # parsing/evaling helper
137
+ #
138
+ def evaling_define(*args) #:nodoc:
139
+ function_name = args[0]
140
+ function_arg_names = args[1..-2]
141
+ function_string = args.last.to_str
142
+
143
+ comma_separated = function_arg_names.map { |name|
144
+ name.to_s
145
+ }.join(",")
146
+
147
+ eval_me = %{
148
+ lambda { |#{comma_separated}|
149
+ #{function_string}
150
+ }
151
+ }
152
+
153
+ function = eval(eval_me, TOPLEVEL_BINDING)
154
+
155
+ define(function_name, *function_arg_names, &function)
156
+ end
157
+
158
+ def method_missing(symbol, *args, &block) #:nodoc:
159
+ if match = symbol.to_s.match(%r!\Adefine_(\w+)\Z!)
160
+ method_name = match.captures.first.to_sym
161
+ if block
162
+ define(method_name, *args, &block)
163
+ else
164
+ evaling_define(method_name, *args)
165
+ end
166
+ else
167
+ super(symbol, *args, &block)
168
+ end
169
+ end
170
+
171
+ #
172
+ # Mark this node and all its children as uncomputed.
173
+ #
174
+ # Arguments:
175
+ #
176
+ # +name+ -- (Symbol) node name.
177
+ #
178
+ def reset(name)
179
+ @nodes[name].reset
180
+ end
181
+
182
+ #
183
+ # Check for a cyclic graph below the given node. Raises
184
+ # CompTree::Error::CircularError if found.
185
+ #
186
+ # Arguments:
187
+ #
188
+ # +name+ -- (Symbol) node name.
189
+ #
190
+ def check_circular(name)
191
+ helper = lambda { |root, chain|
192
+ if chain.include? root
193
+ raise Error::CircularError,
194
+ "Circular dependency detected: #{root} => #{chain.last} => #{root}"
195
+ end
196
+ @nodes[root].children.each { |child|
197
+ helper.call(child.name, chain + [root])
198
+ }
199
+ }
200
+ helper.call(name, [])
201
+ end
202
+
203
+ #
204
+ # Compute this node.
205
+ #
206
+ # Arguments:
207
+ #
208
+ # +name+ -- (Symbol) node name.
209
+ #
210
+ # Options hash:
211
+ #
212
+ # <tt>:threads</tt> -- (Integer) Number of parallel threads.
213
+ #
214
+ # <tt>:fork</tt> -- (boolean) Whether to fork each computation
215
+ # node into its own process.
216
+ #
217
+ # Defaults options are taken from Driver::DEFAULTS.
218
+ #
219
+ def compute(name, opts = nil)
220
+ #
221
+ # Undocumented options:
222
+ #
223
+ # <tt>:wait_interval</tt> -- (seconds) (Obscure) How long to
224
+ # wait after an IPC failure.
225
+ #
226
+ # <tt>:timeout</tt> -- (seconds) (Obscure) Give up after this
227
+ # period of persistent IPC failures.
228
+ #
229
+
230
+ abort_on_exception {
231
+ compute_private(name, opts || Hash.new)
232
+ }
233
+ end
234
+
235
+ private
236
+
237
+ def compute_private(name, opts_in)
238
+ opts = DEFAULTS.merge(opts_in)
239
+ root = @nodes[name]
240
+
241
+ if opts[:threads] < 1
242
+ raise Error::ArgumentError, "threads is #{opts[:threads]}"
243
+ end
244
+
245
+ if opts[:threads] == 1
246
+ root.result = root.compute_now
247
+ elsif opts[:fork] and not @node_class.discard_result?
248
+ #
249
+ # Use buckets to send results across forks.
250
+ #
251
+ result = nil
252
+ BucketIPC::Driver.new(opts[:threads], opts) { |buckets|
253
+ result =
254
+ Algorithm.compute_multithreaded(
255
+ root, opts[:threads], opts[:fork], buckets)
256
+ }
257
+ result
258
+ else
259
+ #
260
+ # Multithreaded computation without fork.
261
+ #
262
+ Algorithm.compute_multithreaded(
263
+ root, opts[:threads], opts[:fork], nil)
264
+ end
265
+ end
266
+ end
267
+ end
@@ -0,0 +1,27 @@
1
+
2
+ module CompTree
3
+ module Error
4
+ # Base class for CompTree errors.
5
+ class Base < StandardError ; end
6
+
7
+ # Internal error inside CompTree. Please send a bug report.
8
+ class AssertionFailed < Base ; end
9
+
10
+ # Bad arguments were passed to a method.
11
+ class ArgumentError < Base ; end
12
+
13
+ #
14
+ # Attempt to redefine a Node.
15
+ #
16
+ # If you wish to only replace the function, set
17
+ # driver.nodes[name].function = some_new_lambda
18
+ #
19
+ class RedefinitionError < Base ; end
20
+
21
+ # A Cyclic graph was detected.
22
+ class CircularError < Base ; end
23
+
24
+ # No function was defined for this node.
25
+ class NoFunctionError < Base ; end
26
+ end
27
+ end
@@ -0,0 +1,165 @@
1
+
2
+ require 'comp_tree/quix/diagnostic'
3
+ require 'thread'
4
+
5
+ module CompTree
6
+ #
7
+ # Base class for nodes in the computation tree.
8
+ #
9
+ class Node
10
+ include Quix::Diagnostic #:nodoc:
11
+
12
+ attr_reader :name #:nodoc:
13
+
14
+ attr_accessor :parents #:nodoc:
15
+ attr_accessor :children #:nodoc:
16
+ attr_accessor :function #:nodoc:
17
+ attr_accessor :result #:nodoc:
18
+ attr_accessor :shared_lock #:nodoc:
19
+
20
+ #
21
+ # Create a node
22
+ #
23
+ def initialize(name) #:nodoc:
24
+ @name = name
25
+ @mutex = Mutex.new
26
+ @children = []
27
+ @parents = []
28
+ reset_self
29
+ end
30
+
31
+ #
32
+ # Reset the computation for this node.
33
+ #
34
+ def reset_self #:nodoc:
35
+ @shared_lock = 0
36
+ @children_results = nil
37
+ @result = nil
38
+ end
39
+
40
+ #
41
+ # Reset the computation for this node and all children.
42
+ #
43
+ def reset #:nodoc:
44
+ each_downward { |node|
45
+ node.reset_self
46
+ }
47
+ end
48
+
49
+ def each_downward(&block) #:nodoc:
50
+ block.call(self)
51
+ @children.each { |child|
52
+ child.each_downward(&block)
53
+ }
54
+ end
55
+
56
+ def each_upward(&block) #:nodoc:
57
+ block.call(self)
58
+ @parents.each { |parent|
59
+ parent.each_upward(&block)
60
+ }
61
+ end
62
+
63
+ def each_child #:nodoc:
64
+ @children.each { |child|
65
+ yield(child)
66
+ }
67
+ end
68
+
69
+ #
70
+ # Force computation of all children; intended for
71
+ # single-threaded mode.
72
+ #
73
+ def compute_now #:nodoc:
74
+ unless @children_results
75
+ @children_results = @children.map { |child|
76
+ child.compute_now
77
+ }
78
+ end
79
+ compute
80
+ end
81
+
82
+ #
83
+ # If all children have been computed, return their results;
84
+ # otherwise return nil.
85
+ #
86
+ def children_results #:nodoc:
87
+ if @children_results
88
+ @children_results
89
+ else
90
+ results = @children.map { |child|
91
+ if child_result = child.result
92
+ child_result
93
+ else
94
+ return nil
95
+ end
96
+ }
97
+ @children_results = results
98
+ end
99
+ end
100
+
101
+ def trace_compute #:nodoc:
102
+ debug {
103
+ # --- own mutex
104
+ trace "Computing #{@name}"
105
+ raise Error::AssertionFailed if @result
106
+ raise Error::AssertionFailed unless @mutex.locked?
107
+ raise Error::AssertionFailed unless @children_results
108
+ }
109
+ end
110
+
111
+ #
112
+ # Compute this node; children must be computed and lock must be
113
+ # already acquired.
114
+ #
115
+ def compute #:nodoc:
116
+ unless defined?(@function) and @function
117
+ raise Error::NoFunctionError,
118
+ "No function was defined for node '#{@name.inspect}'"
119
+ end
120
+ @function.call(*@children_results)
121
+ end
122
+
123
+ def try_lock #:nodoc:
124
+ # --- shared tree mutex and own mutex
125
+ if @shared_lock == 0 and @mutex.try_lock
126
+ trace "Locking #{@name}"
127
+ each_upward { |node|
128
+ node.shared_lock += 1
129
+ trace "#{node.name} locked by #{@name}: level: #{node.shared_lock}"
130
+ }
131
+ true
132
+ else
133
+ false
134
+ end
135
+ end
136
+
137
+ def unlock #:nodoc:
138
+ # --- shared tree mutex and own mutex
139
+ debug {
140
+ raise Error::AssertionFailed unless @mutex.locked?
141
+ trace "Unlocking #{@name}"
142
+ }
143
+ each_upward { |node|
144
+ node.shared_lock -= 1
145
+ debug {
146
+ if node.shared_lock == 0
147
+ trace "#{node.name} unlocked by #{@name}"
148
+ end
149
+ }
150
+ }
151
+ @mutex.unlock
152
+ end
153
+
154
+ class << self
155
+ #
156
+ # Throw away the computation result?
157
+ #
158
+ # This Node base class always returns false.
159
+ #
160
+ def discard_result?
161
+ false
162
+ end
163
+ end
164
+ end
165
+ end
@@ -0,0 +1,33 @@
1
+
2
+
3
+
4
+ ######################################################
5
+ #
6
+ # **** DO NOT EDIT ****
7
+ #
8
+ # **** THIS IS A GENERATED FILE *****
9
+ #
10
+ ######################################################
11
+
12
+
13
+
14
+ unless respond_to? :tap
15
+ module Kernel
16
+ def tap
17
+ yield self
18
+ self
19
+ end
20
+ end
21
+ end
22
+
23
+
24
+
25
+ ######################################################
26
+ #
27
+ # **** DO NOT EDIT ****
28
+ #
29
+ # **** THIS IS A GENERATED FILE *****
30
+ #
31
+ ######################################################
32
+
33
+
@@ -0,0 +1,68 @@
1
+
2
+
3
+
4
+ ######################################################
5
+ #
6
+ # **** DO NOT EDIT ****
7
+ #
8
+ # **** THIS IS A GENERATED FILE *****
9
+ #
10
+ ######################################################
11
+
12
+
13
+
14
+ require 'comp_tree/quix/builtin/kernel/tap'
15
+
16
+ module CompTree::Quix
17
+ module Diagnostic
18
+ def show(desc = nil, stream = STDOUT, &block)
19
+ if desc
20
+ stream.puts(desc)
21
+ end
22
+ if block
23
+ expression = block.call
24
+ eval(expression, block.binding).tap { |result|
25
+ stream.printf("%-16s => %s\n", expression, result.inspect)
26
+ }
27
+ end
28
+ end
29
+
30
+ if $DEBUG
31
+ def debug
32
+ yield
33
+ end
34
+
35
+ def debugging?
36
+ true
37
+ end
38
+
39
+ def trace(desc = nil, &block)
40
+ if desc
41
+ show("#{desc}.".sub(%r!\.\.+\Z!, ""), STDERR, &block)
42
+ else
43
+ show(nil, STDERR, &block)
44
+ end
45
+ end
46
+ else
47
+ # non-$DEBUG
48
+ def debug ; end
49
+ def debugging? ; end
50
+ def trace(*args) ; end
51
+ end
52
+
53
+ extend self
54
+ end
55
+ end
56
+
57
+
58
+
59
+
60
+ ######################################################
61
+ #
62
+ # **** DO NOT EDIT ****
63
+ #
64
+ # **** THIS IS A GENERATED FILE *****
65
+ #
66
+ ######################################################
67
+
68
+
@@ -0,0 +1,85 @@
1
+
2
+
3
+
4
+ ######################################################
5
+ #
6
+ # **** DO NOT EDIT ****
7
+ #
8
+ # **** THIS IS A GENERATED FILE *****
9
+ #
10
+ ######################################################
11
+
12
+
13
+
14
+ require 'thread'
15
+
16
+ module CompTree::Quix
17
+ module Kernel
18
+ def let
19
+ yield self
20
+ end
21
+
22
+ def singleton_class
23
+ class << self
24
+ self
25
+ end
26
+ end
27
+
28
+ module Gensym
29
+ @mutex = Mutex.new
30
+ @count = 0
31
+
32
+ def gensym(prefix = nil)
33
+ count = Gensym.module_eval {
34
+ @mutex.synchronize {
35
+ @count += 1
36
+ }
37
+ }
38
+ "#{prefix || :G}_#{count}_#{rand}".to_sym
39
+ end
40
+ end
41
+ include Gensym
42
+
43
+ def call_private(method, *args, &block)
44
+ instance_eval { send(method, *args, &block) }
45
+ end
46
+
47
+ def with_warnings(value = true)
48
+ previous = $VERBOSE
49
+ $VERBOSE = value
50
+ begin
51
+ yield
52
+ ensure
53
+ $VERBOSE = previous
54
+ end
55
+ end
56
+
57
+ def no_warnings(&block)
58
+ with_warnings(false, &block)
59
+ end
60
+
61
+ def abort_on_exception(value = true)
62
+ previous = Thread.abort_on_exception
63
+ Thread.abort_on_exception = value
64
+ begin
65
+ yield
66
+ ensure
67
+ Thread.abort_on_exception = previous
68
+ end
69
+ end
70
+
71
+ extend self
72
+ end
73
+ end
74
+
75
+
76
+
77
+ ######################################################
78
+ #
79
+ # **** DO NOT EDIT ****
80
+ #
81
+ # **** THIS IS A GENERATED FILE *****
82
+ #
83
+ ######################################################
84
+
85
+
@@ -0,0 +1,42 @@
1
+
2
+ module CompTree
3
+ module RetriableFork
4
+ HAVE_FORK = lambda {
5
+ begin
6
+ process_id = fork { }
7
+ Process.wait(process_id)
8
+ rescue NotImplementedError
9
+ return false
10
+ end
11
+ true
12
+ }.call
13
+
14
+ def fork(retry_wait = 10, retry_max = 10, &block)
15
+ num_retries = 0
16
+ begin
17
+ Process.fork(&block)
18
+ rescue Errno::EAGAIN
19
+ num_retries += 1
20
+ if num_retries == retry_max
21
+ message = %Q{
22
+ ****************************************************************
23
+ Maximum number of EAGAIN signals reached (#{retry_max})
24
+ ****************************************************************
25
+
26
+ Either increase your process limit permission (consult your
27
+ OS manual) or run this script as superuser.
28
+
29
+ ****************************************************************
30
+ }
31
+ STDERR.puts(message.gsub(%r!^[ \t]+!, ""))
32
+ raise
33
+ end
34
+ STDERR.puts "Caught EGAIN. Retrying in #{retry_wait} seconds."
35
+ sleep(retry_wait)
36
+ retry
37
+ end
38
+ end
39
+ module_function :fork
40
+ end
41
+ end
42
+
@@ -0,0 +1,22 @@
1
+
2
+ module CompTree
3
+ #
4
+ # TaskNode is a Node which discards its results
5
+ #
6
+ class TaskNode < Node
7
+ def compute #:nodoc:
8
+ @function.call
9
+ true
10
+ end
11
+
12
+ class << self
13
+ #
14
+ # TaskNode always returns true.
15
+ #
16
+ def discard_result?
17
+ true
18
+ end
19
+ end
20
+ end
21
+ end
22
+