tupelo 0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. checksums.yaml +7 -0
  2. data/COPYING +22 -0
  3. data/README.md +422 -0
  4. data/Rakefile +77 -0
  5. data/bench/pipeline.rb +25 -0
  6. data/bugs/take-write.rb +19 -0
  7. data/bugs/write-read.rb +15 -0
  8. data/example/add.rb +19 -0
  9. data/example/app-and-tup.rb +30 -0
  10. data/example/async-transaction.rb +16 -0
  11. data/example/balance-xfer-locking.rb +50 -0
  12. data/example/balance-xfer-retry.rb +55 -0
  13. data/example/balance-xfer.rb +33 -0
  14. data/example/boolean-match.rb +32 -0
  15. data/example/bounded-retry.rb +35 -0
  16. data/example/broker-locking.rb +43 -0
  17. data/example/broker-optimistic-async.rb +33 -0
  18. data/example/broker-optimistic.rb +41 -0
  19. data/example/broker-queue.rb +2 -0
  20. data/example/cancel.rb +17 -0
  21. data/example/concurrent-transactions.rb +39 -0
  22. data/example/custom-class.rb +29 -0
  23. data/example/custom-search.rb +27 -0
  24. data/example/fail-and-retry.rb +29 -0
  25. data/example/hash-tuples.rb +53 -0
  26. data/example/increment.rb +21 -0
  27. data/example/lock-mgr-with-queue.rb +75 -0
  28. data/example/lock-mgr.rb +62 -0
  29. data/example/map-reduce-v2.rb +96 -0
  30. data/example/map-reduce.rb +77 -0
  31. data/example/matching.rb +9 -0
  32. data/example/notify.rb +35 -0
  33. data/example/optimist.rb +20 -0
  34. data/example/pulse.rb +24 -0
  35. data/example/read-in-trans.rb +56 -0
  36. data/example/small-simplified.rb +18 -0
  37. data/example/small.rb +76 -0
  38. data/example/tcp.rb +35 -0
  39. data/example/timeout-trans.rb +21 -0
  40. data/example/timeout.rb +27 -0
  41. data/example/tiny-client.rb +14 -0
  42. data/example/tiny-server.rb +12 -0
  43. data/example/transaction-logic.rb +40 -0
  44. data/example/write-wait.rb +17 -0
  45. data/lib/tupelo/app.rb +121 -0
  46. data/lib/tupelo/archiver/tuplespace.rb +68 -0
  47. data/lib/tupelo/archiver/worker.rb +87 -0
  48. data/lib/tupelo/archiver.rb +86 -0
  49. data/lib/tupelo/client/common.rb +10 -0
  50. data/lib/tupelo/client/reader.rb +124 -0
  51. data/lib/tupelo/client/transaction.rb +455 -0
  52. data/lib/tupelo/client/tuplespace.rb +50 -0
  53. data/lib/tupelo/client/worker.rb +493 -0
  54. data/lib/tupelo/client.rb +44 -0
  55. data/lib/tupelo/version.rb +3 -0
  56. data/test/lib/mock-client.rb +38 -0
  57. data/test/lib/mock-msg.rb +47 -0
  58. data/test/lib/mock-queue.rb +42 -0
  59. data/test/lib/mock-seq.rb +50 -0
  60. data/test/lib/testable-worker.rb +24 -0
  61. data/test/stress/concurrent-transactions.rb +42 -0
  62. data/test/system/test-archiver.rb +35 -0
  63. data/test/unit/test-mock-queue.rb +93 -0
  64. data/test/unit/test-mock-seq.rb +39 -0
  65. data/test/unit/test-ops.rb +222 -0
  66. metadata +134 -0
data/lib/tupelo/app.rb ADDED
@@ -0,0 +1,121 @@
1
+ require 'easy-serve'
2
+ require 'tupelo/client'
3
+
4
+ ## this could be unified with the implementation of bin/tup, which is similar
5
+
6
+ module Tupelo
7
+ # Not an essential part of the library, but used to build up groups of
8
+ # processes for use in examples, tests, benchmarks, etc.
9
+ class AppBuilder
10
+ attr_reader :ez
11
+
12
+ # Does this app own (as child processes) the seq, cseq, and arc servers?
13
+ attr_reader :owns_servers
14
+
15
+ def initialize ez, owns_servers: nil
16
+ @ez = ez
17
+ @owns_servers = owns_servers
18
+ end
19
+
20
+ def log
21
+ ez.log
22
+ end
23
+
24
+ # Yields a client that runs in this process.
25
+ def local client_class = Client
26
+ ez.local :seqd, :cseqd, :arcd do |seqd, cseqd, arcd|
27
+ run_client client_class,
28
+ seq: seqd, cseq: cseqd, arc: arcd, log: log do |client|
29
+ yield client
30
+ end
31
+ end
32
+ end
33
+
34
+ # Yields a client that runs in a subprocess.
35
+ def child client_class = Client
36
+ ez.client :seqd, :cseqd, :arcd do |seqd, cseqd, arcd|
37
+ run_client client_class,
38
+ seq: seqd, cseq: cseqd, arc: arcd, log: log do |client|
39
+ yield client
40
+ end
41
+ end
42
+ end
43
+
44
+ def run_client client_class, opts
45
+ log = opts[:log]
46
+ log.progname = "client <starting in #{log.progname}>"
47
+ client = client_class.new opts
48
+ client.start do
49
+ log.progname = "client #{client.client_id}"
50
+ end
51
+ yield client
52
+ ensure
53
+ client.stop if client # gracefully exit the tuplespace management thread
54
+ end
55
+ end
56
+
57
+ #blob_type: 'msgpack' # the default
58
+ #blob_type: 'marshal' # if you need to pass general ruby objects
59
+ #blob_type: 'yaml' # less general ruby objects, but cross-language
60
+ #blob_type: 'json' # more portable than yaml, but more restrictive
61
+
62
+ def self.application argv: ARGV,
63
+ servers_file: nil, blob_type: nil,
64
+ seqd_addr: [], cseqd_addr: [], arcd_addr: []
65
+
66
+ log_level = case
67
+ when argv.delete("--debug"); Logger::DEBUG
68
+ when argv.delete("--info"); Logger::INFO
69
+ when argv.delete("--warn"); Logger::WARN
70
+ when argv.delete("--error"); Logger::ERROR
71
+ when argv.delete("--fatal"); Logger::FATAL
72
+ else Logger::WARN
73
+ end
74
+
75
+ unless blob_type
76
+ %w{--marshal --yaml --json --msgpack}.each do |switch|
77
+ s = argv.delete(switch) and
78
+ blob_type ||= s.delete("--")
79
+ end
80
+ blob_type ||= "msgpack"
81
+ end
82
+
83
+ svrs = servers_file || argv.shift || "servers-#$$.yaml"
84
+
85
+ EasyServe.start(servers_file: svrs) do |ez|
86
+ log = ez.log
87
+ log.level = log_level
88
+ log.progname = "parent"
89
+ owns_servers = false
90
+
91
+ ez.start_servers do
92
+ owns_servers = true
93
+
94
+ arc_to_seq_sock, seq_to_arc_sock = UNIXSocket.pair
95
+ arc_to_cseq_sock, cseq_to_arc_sock = UNIXSocket.pair
96
+
97
+ ez.server :seqd, *seqd_addr do |svr|
98
+ require 'funl/message-sequencer'
99
+ seq = Funl::MessageSequencer.new svr, seq_to_arc_sock, log: log,
100
+ blob_type: blob_type
101
+ seq.start
102
+ end
103
+
104
+ ez.server :cseqd, *cseqd_addr do |svr|
105
+ require 'funl/client-sequencer'
106
+ cseq = Funl::ClientSequencer.new svr, cseq_to_arc_sock, log: log
107
+ cseq.start
108
+ end
109
+
110
+ ez.server :arcd, *arcd_addr do |svr|
111
+ require 'tupelo/archiver'
112
+ arc = Archiver.new svr, seq: arc_to_seq_sock,
113
+ cseq: arc_to_cseq_sock, log: log
114
+ arc.start
115
+ end
116
+ end
117
+
118
+ yield AppBuilder.new(ez, owns_servers: owns_servers)
119
+ end
120
+ end
121
+ end
@@ -0,0 +1,68 @@
1
+ class Tupelo::Archiver
2
+ class Tuplespace
3
+ include Enumerable
4
+
5
+ attr_reader :zero_tolerance
6
+
7
+ def initialize(zero_tolerance: Tupelo::Archiver::ZERO_TOLERANCE)
8
+ @counts = Hash.new(0) # tuple => count
9
+ @nzero = 0
10
+ @zero_tolerance = zero_tolerance
11
+ end
12
+
13
+ # note: multiple equal tuples are yielded once
14
+ def each
15
+ @counts.each do |tuple, count|
16
+ yield tuple, count if count > 0
17
+ end
18
+ end
19
+
20
+ def insert tuple
21
+ @counts[tuple] += 1
22
+ end
23
+
24
+ def delete_once tuple
25
+ if @counts[tuple] > 0
26
+ @counts[tuple] -= 1
27
+ if @counts[tuple] == 0
28
+ @nzero += 1
29
+ clear_excess_zeros if @nzero > zero_tolerance
30
+ end
31
+ true
32
+ else
33
+ false
34
+ end
35
+ end
36
+
37
+ def transaction inserts: [], deletes: []
38
+ deletes.each do |tuple|
39
+ delete_once tuple or raise "bug"
40
+ end
41
+
42
+ inserts.each do |tuple|
43
+ insert tuple.freeze ## freeze recursively
44
+ end
45
+ end
46
+
47
+ def clear_excess_zeros
48
+ nd = (@nzero - zero_tolerance / 2)
49
+ @counts.delete_if {|tuple, count| count == 0 && (nd-=1) >= 0}
50
+ end
51
+
52
+ def find_distinct_matches_for tuples
53
+ h = Hash.new(0)
54
+ tuples.map do |tuple|
55
+ if @counts[tuple] > h[tuple]
56
+ h[tuple] += 1
57
+ tuple
58
+ else
59
+ nil
60
+ end
61
+ end
62
+ end
63
+
64
+ def find_match_for tuple
65
+ @counts[tuple] > 0 && tuple
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,87 @@
1
+ require 'funl/history-worker'
2
+
3
+ class Tupelo::Archiver
4
+ class Worker < Tupelo::Client::Worker
5
+ include Funl::HistoryWorker
6
+
7
+ def handle_client_request req
8
+ case req
9
+ when Tupelo::Archiver::ForkRequest
10
+ handle_fork_request req
11
+ else
12
+ super
13
+ end
14
+ end
15
+
16
+ def handle_fork_request req
17
+ stream = client.arc_server_stream_for req.io
18
+
19
+ begin
20
+ op, args = stream.read
21
+ rescue EOFError
22
+ log.debug {"#{stream.peer_name} disconnected from archiver"}
23
+ return
24
+ rescue => ex
25
+ log.error "in fork for #{stream || req.io}: #{ex.inspect}"
26
+ end
27
+
28
+ log.info {
29
+ "#{stream.peer_name} requested #{op.inspect}" +
30
+ (args ? " on #{args.inspect}" : "")}
31
+
32
+ fork do
33
+ begin
34
+ case op
35
+ when "new client"
36
+ raise "Unimplemented" ###
37
+ when "get range" ### handle this in Funl::HistoryWorker
38
+ raise "Unimplemented" ###
39
+ when GET_TUPLESPACE
40
+ send_tuplespace stream, args
41
+ else
42
+ raise "Unknown operation: #{op.inspect}"
43
+ end
44
+ rescue EOFError
45
+ log.debug {"#{stream.peer_name} disconnected from archiver"}
46
+ rescue => ex
47
+ log.error "in fork for #{stream || req.io}: #{ex.inspect}"
48
+ end
49
+ end
50
+ ensure
51
+ req.io.close
52
+ end
53
+
54
+ def send_tuplespace stream, templates
55
+ log.info {
56
+ "send_tuplespace to #{stream.peer_name} " +
57
+ "at tick #{global_tick.inspect} " +
58
+ (templates ? " with templates #{templates.inspect}" : "")}
59
+
60
+ stream << [global_tick]
61
+
62
+ if templates
63
+ templates = templates.map {|t| Tupelo::Client::Template.new t}
64
+ tuplespace.each do |tuple, count|
65
+ if templates.any? {|template| template === tuple}
66
+ count.times do
67
+ stream << tuple
68
+ ## optimization: use stream.write_to_buffer
69
+ end
70
+ end
71
+ ## optimize this if templates have simple form, such as
72
+ ## [ [str1, nil, ...], [str2, nil, ...], ...]
73
+ end
74
+ else
75
+ tuplespace.each do |tuple, count|
76
+ count.times do ## just dump and send str * count?
77
+ stream << tuple ## optimize this, and cache the serial
78
+ ## optimization: use stream.write_to_buffer
79
+ end
80
+ end
81
+ end
82
+
83
+ stream << nil # terminator
84
+ ## stream.flush or close if write_to_buffer used above
85
+ end
86
+ end
87
+ end
@@ -0,0 +1,86 @@
1
+ require 'tupelo/client'
2
+ require 'funl/history-client'
3
+
4
+ class Tupelo::Archiver < Tupelo::Client; end
5
+
6
+ require 'tupelo/archiver/worker'
7
+ require 'tupelo/archiver/tuplespace'
8
+
9
+ module Tupelo
10
+ class Archiver
11
+ include Funl::HistoryClient
12
+
13
+ attr_reader :server
14
+ attr_reader :server_thread
15
+
16
+ # How many tuples with count=0 do we permit before cleaning up?
17
+ ZERO_TOLERANCE = 1000
18
+
19
+ def initialize server, **opts
20
+ super arc: nil, tuplespace: Tupelo::Archiver::Tuplespace, **opts
21
+ @server = server
22
+ end
23
+
24
+ # three kinds of requests:
25
+ #
26
+ # 1. fork a new client, with given Client class, and subselect
27
+ # using given templates
28
+ #
29
+ # 2. accept tcp/unix socket connection and fork, and then:
30
+ #
31
+ # a. dump subspace matching given templates OR
32
+ #
33
+ # b. dump all ops in a given range of the global sequence
34
+ # matching given templates
35
+ #
36
+ # the fork happens when tuplespace is consistent; we
37
+ # do this by passing cmd to worker thread, with conn
38
+ class ForkRequest
39
+ attr_reader :io
40
+ def initialize io
41
+ @io = io
42
+ end
43
+ end
44
+
45
+ def make_worker
46
+ Tupelo::Archiver::Worker.new self
47
+ end
48
+
49
+ def start
50
+ ## load from file?
51
+ super # start worker thread
52
+ @server_thread = Thread.new do
53
+ run
54
+ end
55
+ end
56
+
57
+ def stop
58
+ server_thread.kill if server_thread
59
+ super # stop worker thread
60
+ end
61
+
62
+ def run
63
+ loop do
64
+ ## nonblock_accept?
65
+ Thread.new(server.accept) do |conn|
66
+ handle_conn conn
67
+ end
68
+
69
+ ## periodically send worker request to dump space to file?
70
+ end
71
+ rescue => ex
72
+ log.error ex
73
+ raise
74
+ end
75
+
76
+ def handle_conn conn
77
+ log.debug {"accepted #{conn.inspect}"}
78
+ begin
79
+ worker << ForkRequest.new(conn)
80
+ rescue => ex
81
+ log.error ex
82
+ raise
83
+ end
84
+ end
85
+ end
86
+ end
@@ -0,0 +1,10 @@
1
+ module Tupelo
2
+ class Client
3
+ class Unwaiter
4
+ attr_reader :waiter
5
+ def initialize waiter
6
+ @waiter = waiter
7
+ end
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,124 @@
1
+ require 'tupelo/client/common'
2
+
3
+ class Tupelo::Client
4
+ # include into class that defines #worker and #log
5
+ module Api
6
+ ## need read with more complex predicates: |, &, etc
7
+ def read_wait template
8
+ waiter = Waiter.new(worker.make_template(template), self)
9
+ worker << waiter
10
+ result = waiter.wait
11
+ waiter = nil
12
+ result
13
+ ensure
14
+ worker << Unwaiter.new(waiter) if waiter
15
+ end
16
+ alias read read_wait
17
+
18
+ ## need nonwaiting reader that accepts 2 or more templates
19
+ def read_nowait template
20
+ matcher = Matcher.new(worker.make_template(template), self)
21
+ worker << matcher
22
+ matcher.wait
23
+ end
24
+
25
+ # By default, reads *everything*.
26
+ def read_all template = Object
27
+ matcher = Matcher.new(worker.make_template(template), self, :all => true)
28
+ worker << matcher
29
+ a = []
30
+ while tuple = matcher.wait ## inefficient?
31
+ yield tuple if block_given?
32
+ a << tuple
33
+ end
34
+ a
35
+ end
36
+
37
+ def notifier
38
+ NotifyWaiter.new(self).tap {|n| n.toggle}
39
+ end
40
+ end
41
+
42
+ class WaiterBase
43
+ attr_reader :template
44
+ attr_reader :queue
45
+
46
+ def initialize template, client
47
+ @template = template
48
+ @queue = client.make_queue
49
+ @client = client
50
+ end
51
+
52
+ def gloms tuple
53
+ if template === tuple
54
+ peek tuple
55
+ true
56
+ else
57
+ false
58
+ end
59
+ end
60
+
61
+ def peek tuple
62
+ queue << tuple
63
+ end
64
+
65
+ def wait
66
+ @client.log.debug {"waiting for #{self}"}
67
+ r = queue.pop
68
+ @client.log.debug {"finished waiting for #{self}"}
69
+ r
70
+ end
71
+
72
+ def inspect
73
+ "<#{self.class}: #{template.inspect}>"
74
+ end
75
+ end
76
+
77
+ class Waiter < WaiterBase
78
+ end
79
+
80
+ class Matcher < WaiterBase
81
+ attr_reader :all # this is only cosmetic -- see #inspect
82
+
83
+ def initialize template, client, all: false
84
+ super template, client
85
+ @all = all
86
+ end
87
+
88
+ def fails
89
+ queue << nil
90
+ end
91
+
92
+ def inspect
93
+ e = all ? "all " : ""
94
+ t = template.inspect
95
+ "<#{self.class}: #{e}#{t}>"
96
+ end
97
+ end
98
+
99
+ # Instrumentation.
100
+ class NotifyWaiter
101
+ attr_reader :queue
102
+
103
+ def initialize client
104
+ @client = client
105
+ @queue = client.make_queue
106
+ end
107
+
108
+ def << event
109
+ queue << event
110
+ end
111
+
112
+ def wait
113
+ queue.pop
114
+ end
115
+
116
+ def toggle
117
+ @client.worker << self
118
+ end
119
+
120
+ def inspect
121
+ to_s
122
+ end
123
+ end
124
+ end