comp_tree 0.5.2 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGES +24 -0
- data/README +19 -52
- data/Rakefile +1 -138
- data/comp_tree.gemspec +33 -30
- data/install.rb +3 -3
- data/lib/comp_tree/algorithm.rb +117 -156
- data/lib/comp_tree/driver.rb +39 -154
- data/lib/comp_tree/error.rb +18 -23
- data/lib/comp_tree/node.rb +46 -50
- data/lib/comp_tree.rb +56 -0
- data/rakelib/jumpstart/ruby.rb +51 -0
- data/{contrib/quix/lib/quix → rakelib/jumpstart}/simple_installer.rb +11 -13
- data/test/common.rb +29 -0
- data/test/test_basic.rb +189 -0
- data/test/test_circular.rb +34 -31
- data/test/test_drain.rb +38 -0
- data/test/test_exception.rb +37 -86
- data/test/test_flood.rb +14 -0
- data/test/test_grind.rb +77 -0
- data/test/test_sequential.rb +21 -0
- metadata +45 -58
- data/contrib/quix/Rakefile +0 -16
- data/contrib/quix/install.rb +0 -3
- data/contrib/quix/lib/quix/builtin/dir/casefold_brackets.rb +0 -7
- data/contrib/quix/lib/quix/builtin/kernel/tap.rb +0 -9
- data/contrib/quix/lib/quix/builtin/module/include.rb +0 -21
- data/contrib/quix/lib/quix/builtin/module/private.rb +0 -41
- data/contrib/quix/lib/quix/config.rb +0 -37
- data/contrib/quix/lib/quix/cygwin.rb +0 -60
- data/contrib/quix/lib/quix/diagnostic.rb +0 -44
- data/contrib/quix/lib/quix/enumerable.rb +0 -33
- data/contrib/quix/lib/quix/fileutils.rb +0 -37
- data/contrib/quix/lib/quix/hash_struct.rb +0 -27
- data/contrib/quix/lib/quix/kernel.rb +0 -61
- data/contrib/quix/lib/quix/lazy_struct.rb +0 -55
- data/contrib/quix/lib/quix/string.rb +0 -38
- data/contrib/quix/lib/quix/subpackager.rb +0 -52
- data/contrib/quix/lib/quix/thread_local.rb +0 -32
- data/contrib/quix/lib/quix/vars.rb +0 -138
- data/contrib/quix/lib/quix.rb +0 -32
- data/contrib/quix/test/all.rb +0 -12
- data/contrib/quix/test/test_deps.rb +0 -25
- data/contrib/quix/test/test_include.rb +0 -47
- data/contrib/quix/test/test_private.rb +0 -86
- data/contrib/quix/test/test_root.rb +0 -19
- data/contrib/quix/test/test_struct.rb +0 -48
- data/contrib/quix/test/test_vars.rb +0 -187
- data/lib/comp_tree/bucket_ipc.rb +0 -151
- data/lib/comp_tree/diagnostic.rb +0 -44
- data/lib/comp_tree/misc.rb +0 -61
- data/lib/comp_tree/retriable_fork.rb +0 -42
- data/lib/comp_tree/tap.rb +0 -9
- data/lib/comp_tree/task_node.rb +0 -22
- data/test/all.rb +0 -12
- data/test/test_bucketipc.rb +0 -72
- data/test/test_comp_tree.rb +0 -364
data/lib/comp_tree/bucket_ipc.rb
DELETED
@@ -1,151 +0,0 @@
|
|
1
|
-
|
2
|
-
require 'drb'
|
3
|
-
require 'thread'
|
4
|
-
|
5
|
-
require 'comp_tree/retriable_fork'
|
6
|
-
require 'comp_tree/diagnostic'
|
7
|
-
require 'comp_tree/tap'
|
8
|
-
|
9
|
-
module CompTree
|
10
|
-
module BucketIPC
|
11
|
-
class Bucket
|
12
|
-
include Diagnostic
|
13
|
-
include RetriableFork
|
14
|
-
|
15
|
-
def initialize(address, timeout, wait_interval)
|
16
|
-
trace "Making bucket with address #{address}"
|
17
|
-
|
18
|
-
@remote_pid = fork {
|
19
|
-
own_object = Class.new {
|
20
|
-
attr_accessor(:contents)
|
21
|
-
}.new
|
22
|
-
server = DRb.start_service(address, own_object)
|
23
|
-
debug {
|
24
|
-
server.verbose = true
|
25
|
-
}
|
26
|
-
DRb.thread.join
|
27
|
-
}
|
28
|
-
|
29
|
-
@remote_object = DRbObject.new_with_uri(address)
|
30
|
-
@address = address
|
31
|
-
@timeout = timeout
|
32
|
-
@wait_interval = wait_interval
|
33
|
-
end
|
34
|
-
|
35
|
-
attr_accessor(:timeout, :wait_interval)
|
36
|
-
attr_reader(:address)
|
37
|
-
|
38
|
-
def contents=(new_contents)
|
39
|
-
connect {
|
40
|
-
@remote_object.contents = new_contents
|
41
|
-
}
|
42
|
-
end
|
43
|
-
|
44
|
-
def contents
|
45
|
-
connect {
|
46
|
-
@remote_object.contents
|
47
|
-
}
|
48
|
-
end
|
49
|
-
|
50
|
-
def stop
|
51
|
-
Process.kill("TERM", @remote_pid)
|
52
|
-
end
|
53
|
-
|
54
|
-
private
|
55
|
-
|
56
|
-
def connect
|
57
|
-
begin
|
58
|
-
return yield
|
59
|
-
rescue DRb::DRbConnError
|
60
|
-
start = Time.now
|
61
|
-
begin
|
62
|
-
Kernel.sleep(@wait_interval)
|
63
|
-
return yield
|
64
|
-
rescue DRb::DRbConnError
|
65
|
-
if Time.now - start > @timeout
|
66
|
-
raise
|
67
|
-
end
|
68
|
-
retry
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|
72
|
-
end
|
73
|
-
|
74
|
-
class DriverBase
|
75
|
-
def initialize(addresses, timeout, wait_interval)
|
76
|
-
begin
|
77
|
-
@buckets = addresses.map { |address|
|
78
|
-
Bucket.new(address, timeout, wait_interval)
|
79
|
-
}
|
80
|
-
if block_given?
|
81
|
-
yield @buckets
|
82
|
-
end
|
83
|
-
ensure
|
84
|
-
if block_given?
|
85
|
-
stop
|
86
|
-
end
|
87
|
-
end
|
88
|
-
end
|
89
|
-
|
90
|
-
def stop
|
91
|
-
if defined?(@buckets)
|
92
|
-
@buckets.each { |bucket|
|
93
|
-
bucket.stop
|
94
|
-
}
|
95
|
-
end
|
96
|
-
end
|
97
|
-
end
|
98
|
-
|
99
|
-
class Driver < DriverBase
|
100
|
-
DEFAULTS = {
|
101
|
-
:timeout => 0.5,
|
102
|
-
:wait_interval => 0.05,
|
103
|
-
:port_start => 18181,
|
104
|
-
}
|
105
|
-
|
106
|
-
module BucketCounter
|
107
|
-
@mutex = Mutex.new
|
108
|
-
@count = 0
|
109
|
-
class << self
|
110
|
-
def increment_count
|
111
|
-
@mutex.synchronize {
|
112
|
-
@count += 1
|
113
|
-
}
|
114
|
-
end
|
115
|
-
|
116
|
-
def map_indexes(num_buckets)
|
117
|
-
Array.new.tap { |result|
|
118
|
-
num_buckets.times {
|
119
|
-
result << yield(increment_count)
|
120
|
-
}
|
121
|
-
}
|
122
|
-
end
|
123
|
-
end
|
124
|
-
end
|
125
|
-
|
126
|
-
def initialize(num_buckets, opts_in = {})
|
127
|
-
opts = DEFAULTS.merge(opts_in)
|
128
|
-
|
129
|
-
addresses =
|
130
|
-
if RetriableFork::HAVE_FORK
|
131
|
-
#
|
132
|
-
# Assume the existence of fork implies a unix machine.
|
133
|
-
#
|
134
|
-
require 'drb/unix'
|
135
|
-
basename = "drbunix://#{Dir.tmpdir}/bucket.#{Process.pid}.#{rand}"
|
136
|
-
BucketCounter.map_indexes(num_buckets) { |index|
|
137
|
-
"#{basename}.#{index}"
|
138
|
-
}
|
139
|
-
else
|
140
|
-
#
|
141
|
-
# Fallback: use the default socket.
|
142
|
-
#
|
143
|
-
BucketCounter.map_indexes(num_buckets) { |index|
|
144
|
-
"druby://localhost:#{opts[:port_start] + index}"
|
145
|
-
}
|
146
|
-
end
|
147
|
-
super(addresses, opts[:timeout], opts[:wait_interval])
|
148
|
-
end
|
149
|
-
end
|
150
|
-
end
|
151
|
-
end
|
data/lib/comp_tree/diagnostic.rb
DELETED
@@ -1,44 +0,0 @@
|
|
1
|
-
|
2
|
-
require 'comp_tree/tap'
|
3
|
-
|
4
|
-
module CompTree
|
5
|
-
module Diagnostic
|
6
|
-
def show(desc = nil, stream = STDOUT, &block)
|
7
|
-
if desc
|
8
|
-
stream.puts(desc)
|
9
|
-
end
|
10
|
-
if block
|
11
|
-
expression = block.call
|
12
|
-
eval(expression, block.binding).tap { |result|
|
13
|
-
stream.printf("%-16s => %s\n", expression, result.inspect)
|
14
|
-
}
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
if $DEBUG
|
19
|
-
def debug
|
20
|
-
yield
|
21
|
-
end
|
22
|
-
|
23
|
-
def debugging?
|
24
|
-
true
|
25
|
-
end
|
26
|
-
|
27
|
-
def trace(desc = nil, &block)
|
28
|
-
if desc
|
29
|
-
show("#{desc}.".sub(%r!\.\.+\Z!, ""), STDERR, &block)
|
30
|
-
else
|
31
|
-
show(nil, STDERR, &block)
|
32
|
-
end
|
33
|
-
end
|
34
|
-
else
|
35
|
-
# non-$DEBUG
|
36
|
-
def debug ; end
|
37
|
-
def debugging? ; end
|
38
|
-
def trace(*args) ; end
|
39
|
-
end
|
40
|
-
|
41
|
-
extend self
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
data/lib/comp_tree/misc.rb
DELETED
@@ -1,61 +0,0 @@
|
|
1
|
-
|
2
|
-
require 'thread'
|
3
|
-
|
4
|
-
module CompTree
|
5
|
-
module Misc
|
6
|
-
def let
|
7
|
-
yield self
|
8
|
-
end
|
9
|
-
|
10
|
-
def singleton_class
|
11
|
-
class << self
|
12
|
-
self
|
13
|
-
end
|
14
|
-
end
|
15
|
-
|
16
|
-
module Gensym
|
17
|
-
@mutex = Mutex.new
|
18
|
-
@count = 0
|
19
|
-
|
20
|
-
def gensym(prefix = nil)
|
21
|
-
count = Gensym.module_eval {
|
22
|
-
@mutex.synchronize {
|
23
|
-
@count += 1
|
24
|
-
}
|
25
|
-
}
|
26
|
-
"#{prefix || :G}_#{count}".to_sym
|
27
|
-
end
|
28
|
-
end
|
29
|
-
include Gensym
|
30
|
-
|
31
|
-
def call_private(method, *args, &block)
|
32
|
-
instance_eval { send(method, *args, &block) }
|
33
|
-
end
|
34
|
-
|
35
|
-
def with_warnings(value = true)
|
36
|
-
previous = $VERBOSE
|
37
|
-
$VERBOSE = value
|
38
|
-
begin
|
39
|
-
yield
|
40
|
-
ensure
|
41
|
-
$VERBOSE = previous
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
def no_warnings(&block)
|
46
|
-
with_warnings(false, &block)
|
47
|
-
end
|
48
|
-
|
49
|
-
def abort_on_exception(value = true)
|
50
|
-
previous = Thread.abort_on_exception
|
51
|
-
Thread.abort_on_exception = value
|
52
|
-
begin
|
53
|
-
yield
|
54
|
-
ensure
|
55
|
-
Thread.abort_on_exception = previous
|
56
|
-
end
|
57
|
-
end
|
58
|
-
|
59
|
-
extend self
|
60
|
-
end
|
61
|
-
end
|
@@ -1,42 +0,0 @@
|
|
1
|
-
|
2
|
-
module CompTree
|
3
|
-
module RetriableFork
|
4
|
-
HAVE_FORK = lambda {
|
5
|
-
begin
|
6
|
-
process_id = fork { }
|
7
|
-
Process.wait(process_id)
|
8
|
-
rescue NotImplementedError
|
9
|
-
return false
|
10
|
-
end
|
11
|
-
true
|
12
|
-
}.call
|
13
|
-
|
14
|
-
def fork(retry_wait = 10, retry_max = 10, &block)
|
15
|
-
num_retries = 0
|
16
|
-
begin
|
17
|
-
Process.fork(&block)
|
18
|
-
rescue Errno::EAGAIN
|
19
|
-
num_retries += 1
|
20
|
-
if num_retries == retry_max
|
21
|
-
message = %Q{
|
22
|
-
****************************************************************
|
23
|
-
Maximum number of EAGAIN signals reached (#{retry_max})
|
24
|
-
****************************************************************
|
25
|
-
|
26
|
-
Either increase your process limit permission (consult your
|
27
|
-
OS manual) or run this script as superuser.
|
28
|
-
|
29
|
-
****************************************************************
|
30
|
-
}
|
31
|
-
STDERR.puts(message.gsub(%r!^[ \t]+!, ""))
|
32
|
-
raise
|
33
|
-
end
|
34
|
-
STDERR.puts "Caught EGAIN. Retrying in #{retry_wait} seconds."
|
35
|
-
sleep(retry_wait)
|
36
|
-
retry
|
37
|
-
end
|
38
|
-
end
|
39
|
-
module_function :fork
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
data/lib/comp_tree/tap.rb
DELETED
data/lib/comp_tree/task_node.rb
DELETED
@@ -1,22 +0,0 @@
|
|
1
|
-
|
2
|
-
module CompTree
|
3
|
-
#
|
4
|
-
# TaskNode is a Node which discards its results
|
5
|
-
#
|
6
|
-
class TaskNode < Node
|
7
|
-
def compute #:nodoc:
|
8
|
-
@function.call
|
9
|
-
true
|
10
|
-
end
|
11
|
-
|
12
|
-
class << self
|
13
|
-
#
|
14
|
-
# TaskNode always returns true.
|
15
|
-
#
|
16
|
-
def discard_result?
|
17
|
-
true
|
18
|
-
end
|
19
|
-
end
|
20
|
-
end
|
21
|
-
end
|
22
|
-
|
data/test/all.rb
DELETED
@@ -1,12 +0,0 @@
|
|
1
|
-
|
2
|
-
require 'rbconfig'
|
3
|
-
|
4
|
-
Dir["#{File.dirname(__FILE__)}/test_*.rb"].map { |file|
|
5
|
-
File.expand_path(file)
|
6
|
-
}.each { |file|
|
7
|
-
# spawn separate processes to avoid EAGAIN signals on fork
|
8
|
-
ruby = File.join(
|
9
|
-
Config::CONFIG["bindir"],
|
10
|
-
Config::CONFIG["RUBY_INSTALL_NAME"])
|
11
|
-
system(ruby, file)
|
12
|
-
}
|
data/test/test_bucketipc.rb
DELETED
@@ -1,72 +0,0 @@
|
|
1
|
-
|
2
|
-
$LOAD_PATH.unshift(File.expand_path("#{File.dirname(__FILE__)}/../lib"))
|
3
|
-
|
4
|
-
require 'test/unit'
|
5
|
-
require 'comp_tree/bucket_ipc'
|
6
|
-
|
7
|
-
Thread.abort_on_exception = true
|
8
|
-
|
9
|
-
class BucketTest < Test::Unit::TestCase
|
10
|
-
include CompTree::RetriableFork
|
11
|
-
|
12
|
-
def each_bucket(num_buckets, &block)
|
13
|
-
if HAVE_FORK
|
14
|
-
CompTree::BucketIPC::Driver.new(num_buckets) { |buckets|
|
15
|
-
buckets.each { |bucket|
|
16
|
-
yield bucket
|
17
|
-
}
|
18
|
-
}
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
|
-
def test_1_no_fork
|
23
|
-
each_bucket(10) { |bucket|
|
24
|
-
local = bucket.contents = :before
|
25
|
-
bucket.contents = :after
|
26
|
-
assert_equal(local, :before)
|
27
|
-
assert_equal(bucket.contents, :after)
|
28
|
-
}
|
29
|
-
end
|
30
|
-
|
31
|
-
def test_2_fork
|
32
|
-
each_bucket(10) { |bucket|
|
33
|
-
local = bucket.contents = :before
|
34
|
-
process_id = fork {
|
35
|
-
bucket.contents = :after
|
36
|
-
}
|
37
|
-
Process.wait(process_id)
|
38
|
-
assert_equal(local, :before)
|
39
|
-
assert_equal(bucket.contents, :after)
|
40
|
-
}
|
41
|
-
end
|
42
|
-
|
43
|
-
def each_base_test
|
44
|
-
[
|
45
|
-
:test_1_no_fork,
|
46
|
-
:test_2_fork,
|
47
|
-
].each { |method|
|
48
|
-
yield method
|
49
|
-
}
|
50
|
-
end
|
51
|
-
|
52
|
-
def test_3_thread
|
53
|
-
each_base_test { |method|
|
54
|
-
Thread.new {
|
55
|
-
send(method)
|
56
|
-
}.join
|
57
|
-
}
|
58
|
-
end
|
59
|
-
|
60
|
-
def test_4_thread_flood
|
61
|
-
each_base_test { |method|
|
62
|
-
(0...10).map {
|
63
|
-
Thread.new {
|
64
|
-
send(method)
|
65
|
-
}
|
66
|
-
}.each { |thread|
|
67
|
-
thread.join
|
68
|
-
}
|
69
|
-
}
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|