tupelo 0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/COPYING +22 -0
- data/README.md +422 -0
- data/Rakefile +77 -0
- data/bench/pipeline.rb +25 -0
- data/bugs/take-write.rb +19 -0
- data/bugs/write-read.rb +15 -0
- data/example/add.rb +19 -0
- data/example/app-and-tup.rb +30 -0
- data/example/async-transaction.rb +16 -0
- data/example/balance-xfer-locking.rb +50 -0
- data/example/balance-xfer-retry.rb +55 -0
- data/example/balance-xfer.rb +33 -0
- data/example/boolean-match.rb +32 -0
- data/example/bounded-retry.rb +35 -0
- data/example/broker-locking.rb +43 -0
- data/example/broker-optimistic-async.rb +33 -0
- data/example/broker-optimistic.rb +41 -0
- data/example/broker-queue.rb +2 -0
- data/example/cancel.rb +17 -0
- data/example/concurrent-transactions.rb +39 -0
- data/example/custom-class.rb +29 -0
- data/example/custom-search.rb +27 -0
- data/example/fail-and-retry.rb +29 -0
- data/example/hash-tuples.rb +53 -0
- data/example/increment.rb +21 -0
- data/example/lock-mgr-with-queue.rb +75 -0
- data/example/lock-mgr.rb +62 -0
- data/example/map-reduce-v2.rb +96 -0
- data/example/map-reduce.rb +77 -0
- data/example/matching.rb +9 -0
- data/example/notify.rb +35 -0
- data/example/optimist.rb +20 -0
- data/example/pulse.rb +24 -0
- data/example/read-in-trans.rb +56 -0
- data/example/small-simplified.rb +18 -0
- data/example/small.rb +76 -0
- data/example/tcp.rb +35 -0
- data/example/timeout-trans.rb +21 -0
- data/example/timeout.rb +27 -0
- data/example/tiny-client.rb +14 -0
- data/example/tiny-server.rb +12 -0
- data/example/transaction-logic.rb +40 -0
- data/example/write-wait.rb +17 -0
- data/lib/tupelo/app.rb +121 -0
- data/lib/tupelo/archiver/tuplespace.rb +68 -0
- data/lib/tupelo/archiver/worker.rb +87 -0
- data/lib/tupelo/archiver.rb +86 -0
- data/lib/tupelo/client/common.rb +10 -0
- data/lib/tupelo/client/reader.rb +124 -0
- data/lib/tupelo/client/transaction.rb +455 -0
- data/lib/tupelo/client/tuplespace.rb +50 -0
- data/lib/tupelo/client/worker.rb +493 -0
- data/lib/tupelo/client.rb +44 -0
- data/lib/tupelo/version.rb +3 -0
- data/test/lib/mock-client.rb +38 -0
- data/test/lib/mock-msg.rb +47 -0
- data/test/lib/mock-queue.rb +42 -0
- data/test/lib/mock-seq.rb +50 -0
- data/test/lib/testable-worker.rb +24 -0
- data/test/stress/concurrent-transactions.rb +42 -0
- data/test/system/test-archiver.rb +35 -0
- data/test/unit/test-mock-queue.rb +93 -0
- data/test/unit/test-mock-seq.rb +39 -0
- data/test/unit/test-ops.rb +222 -0
- metadata +134 -0
data/lib/tupelo/app.rb
ADDED
@@ -0,0 +1,121 @@
|
|
1
|
+
require 'easy-serve'
|
2
|
+
require 'tupelo/client'
|
3
|
+
|
4
|
+
## this could be unified with the implementation of bin/tup, which is similar
|
5
|
+
|
6
|
+
module Tupelo
|
7
|
+
# Not an essential part of the library, but used to build up groups of
|
8
|
+
# processes for use in examples, tests, benchmarks, etc.
|
9
|
+
class AppBuilder
|
10
|
+
attr_reader :ez
|
11
|
+
|
12
|
+
# Does this app own (as child processes) the seq, cseq, and arc servers?
|
13
|
+
attr_reader :owns_servers
|
14
|
+
|
15
|
+
def initialize ez, owns_servers: nil
|
16
|
+
@ez = ez
|
17
|
+
@owns_servers = owns_servers
|
18
|
+
end
|
19
|
+
|
20
|
+
def log
|
21
|
+
ez.log
|
22
|
+
end
|
23
|
+
|
24
|
+
# Yields a client that runs in this process.
|
25
|
+
def local client_class = Client
|
26
|
+
ez.local :seqd, :cseqd, :arcd do |seqd, cseqd, arcd|
|
27
|
+
run_client client_class,
|
28
|
+
seq: seqd, cseq: cseqd, arc: arcd, log: log do |client|
|
29
|
+
yield client
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
# Yields a client that runs in a subprocess.
|
35
|
+
def child client_class = Client
|
36
|
+
ez.client :seqd, :cseqd, :arcd do |seqd, cseqd, arcd|
|
37
|
+
run_client client_class,
|
38
|
+
seq: seqd, cseq: cseqd, arc: arcd, log: log do |client|
|
39
|
+
yield client
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def run_client client_class, opts
|
45
|
+
log = opts[:log]
|
46
|
+
log.progname = "client <starting in #{log.progname}>"
|
47
|
+
client = client_class.new opts
|
48
|
+
client.start do
|
49
|
+
log.progname = "client #{client.client_id}"
|
50
|
+
end
|
51
|
+
yield client
|
52
|
+
ensure
|
53
|
+
client.stop if client # gracefully exit the tuplespace management thread
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
#blob_type: 'msgpack' # the default
|
58
|
+
#blob_type: 'marshal' # if you need to pass general ruby objects
|
59
|
+
#blob_type: 'yaml' # less general ruby objects, but cross-language
|
60
|
+
#blob_type: 'json' # more portable than yaml, but more restrictive
|
61
|
+
|
62
|
+
def self.application argv: ARGV,
|
63
|
+
servers_file: nil, blob_type: nil,
|
64
|
+
seqd_addr: [], cseqd_addr: [], arcd_addr: []
|
65
|
+
|
66
|
+
log_level = case
|
67
|
+
when argv.delete("--debug"); Logger::DEBUG
|
68
|
+
when argv.delete("--info"); Logger::INFO
|
69
|
+
when argv.delete("--warn"); Logger::WARN
|
70
|
+
when argv.delete("--error"); Logger::ERROR
|
71
|
+
when argv.delete("--fatal"); Logger::FATAL
|
72
|
+
else Logger::WARN
|
73
|
+
end
|
74
|
+
|
75
|
+
unless blob_type
|
76
|
+
%w{--marshal --yaml --json --msgpack}.each do |switch|
|
77
|
+
s = argv.delete(switch) and
|
78
|
+
blob_type ||= s.delete("--")
|
79
|
+
end
|
80
|
+
blob_type ||= "msgpack"
|
81
|
+
end
|
82
|
+
|
83
|
+
svrs = servers_file || argv.shift || "servers-#$$.yaml"
|
84
|
+
|
85
|
+
EasyServe.start(servers_file: svrs) do |ez|
|
86
|
+
log = ez.log
|
87
|
+
log.level = log_level
|
88
|
+
log.progname = "parent"
|
89
|
+
owns_servers = false
|
90
|
+
|
91
|
+
ez.start_servers do
|
92
|
+
owns_servers = true
|
93
|
+
|
94
|
+
arc_to_seq_sock, seq_to_arc_sock = UNIXSocket.pair
|
95
|
+
arc_to_cseq_sock, cseq_to_arc_sock = UNIXSocket.pair
|
96
|
+
|
97
|
+
ez.server :seqd, *seqd_addr do |svr|
|
98
|
+
require 'funl/message-sequencer'
|
99
|
+
seq = Funl::MessageSequencer.new svr, seq_to_arc_sock, log: log,
|
100
|
+
blob_type: blob_type
|
101
|
+
seq.start
|
102
|
+
end
|
103
|
+
|
104
|
+
ez.server :cseqd, *cseqd_addr do |svr|
|
105
|
+
require 'funl/client-sequencer'
|
106
|
+
cseq = Funl::ClientSequencer.new svr, cseq_to_arc_sock, log: log
|
107
|
+
cseq.start
|
108
|
+
end
|
109
|
+
|
110
|
+
ez.server :arcd, *arcd_addr do |svr|
|
111
|
+
require 'tupelo/archiver'
|
112
|
+
arc = Archiver.new svr, seq: arc_to_seq_sock,
|
113
|
+
cseq: arc_to_cseq_sock, log: log
|
114
|
+
arc.start
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
yield AppBuilder.new(ez, owns_servers: owns_servers)
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
class Tupelo::Archiver
|
2
|
+
class Tuplespace
|
3
|
+
include Enumerable
|
4
|
+
|
5
|
+
attr_reader :zero_tolerance
|
6
|
+
|
7
|
+
def initialize(zero_tolerance: Tupelo::Archiver::ZERO_TOLERANCE)
|
8
|
+
@counts = Hash.new(0) # tuple => count
|
9
|
+
@nzero = 0
|
10
|
+
@zero_tolerance = zero_tolerance
|
11
|
+
end
|
12
|
+
|
13
|
+
# note: multiple equal tuples are yielded once
|
14
|
+
def each
|
15
|
+
@counts.each do |tuple, count|
|
16
|
+
yield tuple, count if count > 0
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def insert tuple
|
21
|
+
@counts[tuple] += 1
|
22
|
+
end
|
23
|
+
|
24
|
+
def delete_once tuple
|
25
|
+
if @counts[tuple] > 0
|
26
|
+
@counts[tuple] -= 1
|
27
|
+
if @counts[tuple] == 0
|
28
|
+
@nzero += 1
|
29
|
+
clear_excess_zeros if @nzero > zero_tolerance
|
30
|
+
end
|
31
|
+
true
|
32
|
+
else
|
33
|
+
false
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def transaction inserts: [], deletes: []
|
38
|
+
deletes.each do |tuple|
|
39
|
+
delete_once tuple or raise "bug"
|
40
|
+
end
|
41
|
+
|
42
|
+
inserts.each do |tuple|
|
43
|
+
insert tuple.freeze ## freeze recursively
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def clear_excess_zeros
|
48
|
+
nd = (@nzero - zero_tolerance / 2)
|
49
|
+
@counts.delete_if {|tuple, count| count == 0 && (nd-=1) >= 0}
|
50
|
+
end
|
51
|
+
|
52
|
+
def find_distinct_matches_for tuples
|
53
|
+
h = Hash.new(0)
|
54
|
+
tuples.map do |tuple|
|
55
|
+
if @counts[tuple] > h[tuple]
|
56
|
+
h[tuple] += 1
|
57
|
+
tuple
|
58
|
+
else
|
59
|
+
nil
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
def find_match_for tuple
|
65
|
+
@counts[tuple] > 0 && tuple
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
@@ -0,0 +1,87 @@
|
|
1
|
+
require 'funl/history-worker'
|
2
|
+
|
3
|
+
class Tupelo::Archiver
|
4
|
+
class Worker < Tupelo::Client::Worker
|
5
|
+
include Funl::HistoryWorker
|
6
|
+
|
7
|
+
def handle_client_request req
|
8
|
+
case req
|
9
|
+
when Tupelo::Archiver::ForkRequest
|
10
|
+
handle_fork_request req
|
11
|
+
else
|
12
|
+
super
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def handle_fork_request req
|
17
|
+
stream = client.arc_server_stream_for req.io
|
18
|
+
|
19
|
+
begin
|
20
|
+
op, args = stream.read
|
21
|
+
rescue EOFError
|
22
|
+
log.debug {"#{stream.peer_name} disconnected from archiver"}
|
23
|
+
return
|
24
|
+
rescue => ex
|
25
|
+
log.error "in fork for #{stream || req.io}: #{ex.inspect}"
|
26
|
+
end
|
27
|
+
|
28
|
+
log.info {
|
29
|
+
"#{stream.peer_name} requested #{op.inspect}" +
|
30
|
+
(args ? " on #{args.inspect}" : "")}
|
31
|
+
|
32
|
+
fork do
|
33
|
+
begin
|
34
|
+
case op
|
35
|
+
when "new client"
|
36
|
+
raise "Unimplemented" ###
|
37
|
+
when "get range" ### handle this in Funl::HistoryWorker
|
38
|
+
raise "Unimplemented" ###
|
39
|
+
when GET_TUPLESPACE
|
40
|
+
send_tuplespace stream, args
|
41
|
+
else
|
42
|
+
raise "Unknown operation: #{op.inspect}"
|
43
|
+
end
|
44
|
+
rescue EOFError
|
45
|
+
log.debug {"#{stream.peer_name} disconnected from archiver"}
|
46
|
+
rescue => ex
|
47
|
+
log.error "in fork for #{stream || req.io}: #{ex.inspect}"
|
48
|
+
end
|
49
|
+
end
|
50
|
+
ensure
|
51
|
+
req.io.close
|
52
|
+
end
|
53
|
+
|
54
|
+
def send_tuplespace stream, templates
|
55
|
+
log.info {
|
56
|
+
"send_tuplespace to #{stream.peer_name} " +
|
57
|
+
"at tick #{global_tick.inspect} " +
|
58
|
+
(templates ? " with templates #{templates.inspect}" : "")}
|
59
|
+
|
60
|
+
stream << [global_tick]
|
61
|
+
|
62
|
+
if templates
|
63
|
+
templates = templates.map {|t| Tupelo::Client::Template.new t}
|
64
|
+
tuplespace.each do |tuple, count|
|
65
|
+
if templates.any? {|template| template === tuple}
|
66
|
+
count.times do
|
67
|
+
stream << tuple
|
68
|
+
## optimization: use stream.write_to_buffer
|
69
|
+
end
|
70
|
+
end
|
71
|
+
## optimize this if templates have simple form, such as
|
72
|
+
## [ [str1, nil, ...], [str2, nil, ...], ...]
|
73
|
+
end
|
74
|
+
else
|
75
|
+
tuplespace.each do |tuple, count|
|
76
|
+
count.times do ## just dump and send str * count?
|
77
|
+
stream << tuple ## optimize this, and cache the serial
|
78
|
+
## optimization: use stream.write_to_buffer
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
stream << nil # terminator
|
84
|
+
## stream.flush or close if write_to_buffer used above
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
require 'tupelo/client'
|
2
|
+
require 'funl/history-client'
|
3
|
+
|
4
|
+
class Tupelo::Archiver < Tupelo::Client; end
|
5
|
+
|
6
|
+
require 'tupelo/archiver/worker'
|
7
|
+
require 'tupelo/archiver/tuplespace'
|
8
|
+
|
9
|
+
module Tupelo
|
10
|
+
class Archiver
|
11
|
+
include Funl::HistoryClient
|
12
|
+
|
13
|
+
attr_reader :server
|
14
|
+
attr_reader :server_thread
|
15
|
+
|
16
|
+
# How many tuples with count=0 do we permit before cleaning up?
|
17
|
+
ZERO_TOLERANCE = 1000
|
18
|
+
|
19
|
+
def initialize server, **opts
|
20
|
+
super arc: nil, tuplespace: Tupelo::Archiver::Tuplespace, **opts
|
21
|
+
@server = server
|
22
|
+
end
|
23
|
+
|
24
|
+
# three kinds of requests:
|
25
|
+
#
|
26
|
+
# 1. fork a new client, with given Client class, and subselect
|
27
|
+
# using given templates
|
28
|
+
#
|
29
|
+
# 2. accept tcp/unix socket connection and fork, and then:
|
30
|
+
#
|
31
|
+
# a. dump subspace matching given templates OR
|
32
|
+
#
|
33
|
+
# b. dump all ops in a given range of the global sequence
|
34
|
+
# matching given templates
|
35
|
+
#
|
36
|
+
# the fork happens when tuplespace is consistent; we
|
37
|
+
# do this by passing cmd to worker thread, with conn
|
38
|
+
class ForkRequest
|
39
|
+
attr_reader :io
|
40
|
+
def initialize io
|
41
|
+
@io = io
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def make_worker
|
46
|
+
Tupelo::Archiver::Worker.new self
|
47
|
+
end
|
48
|
+
|
49
|
+
def start
|
50
|
+
## load from file?
|
51
|
+
super # start worker thread
|
52
|
+
@server_thread = Thread.new do
|
53
|
+
run
|
54
|
+
end
|
55
|
+
end
|
56
|
+
|
57
|
+
def stop
|
58
|
+
server_thread.kill if server_thread
|
59
|
+
super # stop worker thread
|
60
|
+
end
|
61
|
+
|
62
|
+
def run
|
63
|
+
loop do
|
64
|
+
## nonblock_accept?
|
65
|
+
Thread.new(server.accept) do |conn|
|
66
|
+
handle_conn conn
|
67
|
+
end
|
68
|
+
|
69
|
+
## periodically send worker request to dump space to file?
|
70
|
+
end
|
71
|
+
rescue => ex
|
72
|
+
log.error ex
|
73
|
+
raise
|
74
|
+
end
|
75
|
+
|
76
|
+
def handle_conn conn
|
77
|
+
log.debug {"accepted #{conn.inspect}"}
|
78
|
+
begin
|
79
|
+
worker << ForkRequest.new(conn)
|
80
|
+
rescue => ex
|
81
|
+
log.error ex
|
82
|
+
raise
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
@@ -0,0 +1,124 @@
|
|
1
|
+
require 'tupelo/client/common'
|
2
|
+
|
3
|
+
class Tupelo::Client
|
4
|
+
# include into class that defines #worker and #log
|
5
|
+
module Api
|
6
|
+
## need read with more complex predicates: |, &, etc
|
7
|
+
def read_wait template
|
8
|
+
waiter = Waiter.new(worker.make_template(template), self)
|
9
|
+
worker << waiter
|
10
|
+
result = waiter.wait
|
11
|
+
waiter = nil
|
12
|
+
result
|
13
|
+
ensure
|
14
|
+
worker << Unwaiter.new(waiter) if waiter
|
15
|
+
end
|
16
|
+
alias read read_wait
|
17
|
+
|
18
|
+
## need nonwaiting reader that accepts 2 or more templates
|
19
|
+
def read_nowait template
|
20
|
+
matcher = Matcher.new(worker.make_template(template), self)
|
21
|
+
worker << matcher
|
22
|
+
matcher.wait
|
23
|
+
end
|
24
|
+
|
25
|
+
# By default, reads *everything*.
|
26
|
+
def read_all template = Object
|
27
|
+
matcher = Matcher.new(worker.make_template(template), self, :all => true)
|
28
|
+
worker << matcher
|
29
|
+
a = []
|
30
|
+
while tuple = matcher.wait ## inefficient?
|
31
|
+
yield tuple if block_given?
|
32
|
+
a << tuple
|
33
|
+
end
|
34
|
+
a
|
35
|
+
end
|
36
|
+
|
37
|
+
def notifier
|
38
|
+
NotifyWaiter.new(self).tap {|n| n.toggle}
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
class WaiterBase
|
43
|
+
attr_reader :template
|
44
|
+
attr_reader :queue
|
45
|
+
|
46
|
+
def initialize template, client
|
47
|
+
@template = template
|
48
|
+
@queue = client.make_queue
|
49
|
+
@client = client
|
50
|
+
end
|
51
|
+
|
52
|
+
def gloms tuple
|
53
|
+
if template === tuple
|
54
|
+
peek tuple
|
55
|
+
true
|
56
|
+
else
|
57
|
+
false
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
def peek tuple
|
62
|
+
queue << tuple
|
63
|
+
end
|
64
|
+
|
65
|
+
def wait
|
66
|
+
@client.log.debug {"waiting for #{self}"}
|
67
|
+
r = queue.pop
|
68
|
+
@client.log.debug {"finished waiting for #{self}"}
|
69
|
+
r
|
70
|
+
end
|
71
|
+
|
72
|
+
def inspect
|
73
|
+
"<#{self.class}: #{template.inspect}>"
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
class Waiter < WaiterBase
|
78
|
+
end
|
79
|
+
|
80
|
+
class Matcher < WaiterBase
|
81
|
+
attr_reader :all # this is only cosmetic -- see #inspect
|
82
|
+
|
83
|
+
def initialize template, client, all: false
|
84
|
+
super template, client
|
85
|
+
@all = all
|
86
|
+
end
|
87
|
+
|
88
|
+
def fails
|
89
|
+
queue << nil
|
90
|
+
end
|
91
|
+
|
92
|
+
def inspect
|
93
|
+
e = all ? "all " : ""
|
94
|
+
t = template.inspect
|
95
|
+
"<#{self.class}: #{e}#{t}>"
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
# Instrumentation.
|
100
|
+
class NotifyWaiter
|
101
|
+
attr_reader :queue
|
102
|
+
|
103
|
+
def initialize client
|
104
|
+
@client = client
|
105
|
+
@queue = client.make_queue
|
106
|
+
end
|
107
|
+
|
108
|
+
def << event
|
109
|
+
queue << event
|
110
|
+
end
|
111
|
+
|
112
|
+
def wait
|
113
|
+
queue.pop
|
114
|
+
end
|
115
|
+
|
116
|
+
def toggle
|
117
|
+
@client.worker << self
|
118
|
+
end
|
119
|
+
|
120
|
+
def inspect
|
121
|
+
to_s
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|