tupelo 0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. checksums.yaml +7 -0
  2. data/COPYING +22 -0
  3. data/README.md +422 -0
  4. data/Rakefile +77 -0
  5. data/bench/pipeline.rb +25 -0
  6. data/bugs/take-write.rb +19 -0
  7. data/bugs/write-read.rb +15 -0
  8. data/example/add.rb +19 -0
  9. data/example/app-and-tup.rb +30 -0
  10. data/example/async-transaction.rb +16 -0
  11. data/example/balance-xfer-locking.rb +50 -0
  12. data/example/balance-xfer-retry.rb +55 -0
  13. data/example/balance-xfer.rb +33 -0
  14. data/example/boolean-match.rb +32 -0
  15. data/example/bounded-retry.rb +35 -0
  16. data/example/broker-locking.rb +43 -0
  17. data/example/broker-optimistic-async.rb +33 -0
  18. data/example/broker-optimistic.rb +41 -0
  19. data/example/broker-queue.rb +2 -0
  20. data/example/cancel.rb +17 -0
  21. data/example/concurrent-transactions.rb +39 -0
  22. data/example/custom-class.rb +29 -0
  23. data/example/custom-search.rb +27 -0
  24. data/example/fail-and-retry.rb +29 -0
  25. data/example/hash-tuples.rb +53 -0
  26. data/example/increment.rb +21 -0
  27. data/example/lock-mgr-with-queue.rb +75 -0
  28. data/example/lock-mgr.rb +62 -0
  29. data/example/map-reduce-v2.rb +96 -0
  30. data/example/map-reduce.rb +77 -0
  31. data/example/matching.rb +9 -0
  32. data/example/notify.rb +35 -0
  33. data/example/optimist.rb +20 -0
  34. data/example/pulse.rb +24 -0
  35. data/example/read-in-trans.rb +56 -0
  36. data/example/small-simplified.rb +18 -0
  37. data/example/small.rb +76 -0
  38. data/example/tcp.rb +35 -0
  39. data/example/timeout-trans.rb +21 -0
  40. data/example/timeout.rb +27 -0
  41. data/example/tiny-client.rb +14 -0
  42. data/example/tiny-server.rb +12 -0
  43. data/example/transaction-logic.rb +40 -0
  44. data/example/write-wait.rb +17 -0
  45. data/lib/tupelo/app.rb +121 -0
  46. data/lib/tupelo/archiver/tuplespace.rb +68 -0
  47. data/lib/tupelo/archiver/worker.rb +87 -0
  48. data/lib/tupelo/archiver.rb +86 -0
  49. data/lib/tupelo/client/common.rb +10 -0
  50. data/lib/tupelo/client/reader.rb +124 -0
  51. data/lib/tupelo/client/transaction.rb +455 -0
  52. data/lib/tupelo/client/tuplespace.rb +50 -0
  53. data/lib/tupelo/client/worker.rb +493 -0
  54. data/lib/tupelo/client.rb +44 -0
  55. data/lib/tupelo/version.rb +3 -0
  56. data/test/lib/mock-client.rb +38 -0
  57. data/test/lib/mock-msg.rb +47 -0
  58. data/test/lib/mock-queue.rb +42 -0
  59. data/test/lib/mock-seq.rb +50 -0
  60. data/test/lib/testable-worker.rb +24 -0
  61. data/test/stress/concurrent-transactions.rb +42 -0
  62. data/test/system/test-archiver.rb +35 -0
  63. data/test/unit/test-mock-queue.rb +93 -0
  64. data/test/unit/test-mock-seq.rb +39 -0
  65. data/test/unit/test-ops.rb +222 -0
  66. metadata +134 -0
@@ -0,0 +1,96 @@
1
+ require 'tupelo/app'
2
+
3
+ N = 2 # how many cpus do you want to use for mappers?
4
+ VERBOSE = ARGV.delete "-v"
5
+
6
+ class Tupelo::Client
7
+ class Or
8
+ attr_reader :templates
9
+
10
+ def initialize worker, templates
11
+ @templates = templates.map {|template| worker.make_template(template)}
12
+ end
13
+
14
+ def === obj
15
+ templates.any? {|template| template === obj}
16
+ end
17
+ end
18
+
19
+ def or *templates
20
+ Or.new(worker, templates)
21
+ end
22
+ end
23
+
24
+ Tupelo.application do |app|
25
+ if VERBOSE
26
+ app.child do |client| # a debugger client, to see what's happening
27
+ note = client.notifier
28
+ puts "%4s %4s %10s %s" % %w{ tick cid status operation }
29
+ loop do
30
+ status, tick, cid, op = note.wait
31
+ unless status == :attempt
32
+ s = status == :failure ? "FAILED" : ""
33
+ puts "%4d %4d %10s %p" % [tick, cid, s, op]
34
+ end
35
+ end
36
+ end
37
+ end
38
+
39
+ app.child do |client|
40
+ document = "I will not map reduce in class\n" * 10
41
+ lineno = 0
42
+ document.each_line do |line|
43
+ lineno += 1
44
+ client.write line: line, lineno: lineno
45
+ # Note that tuples should be small, so if the data is large, the line
46
+ # should be a reference, not the actual data.
47
+ # Also, in a complex application you might want to add another
48
+ # key/value pair to avoid collisions (owner: ..., for example).
49
+ end
50
+
51
+ results = Hash.new(0)
52
+ lines_remaining = lineno
53
+ results_remaining = 0
54
+ result_template = client.or(
55
+ {word: String, count: Integer},
56
+ {lineno: Integer, result_count: Integer}
57
+ )
58
+
59
+ until lines_remaining == 0 and results_remaining == 0 do
60
+ result = client.take result_template
61
+
62
+ if result["word"]
63
+ results[result["word"]] += result["count"]
64
+ results_remaining -= 1
65
+ elsif result["lineno"]
66
+ lines_remaining -= 1
67
+ results_remaining += result["result_count"]
68
+ else
69
+ log.error "bad keys in result: #{result}"
70
+ end
71
+ end
72
+
73
+ client.log "DONE. results = #{results}"
74
+ client.log "Press ^C to exit"
75
+ end
76
+
77
+ N.times do |i|
78
+ app.child do |client|
79
+ client.log.progname = "mapper #{i}"
80
+
81
+ loop do
82
+ input = client.take line: String, lineno: Integer
83
+
84
+ h = Hash.new(0)
85
+ input["line"].split.each do |word|
86
+ h[word] += 1
87
+ end
88
+
89
+ h.each do |word, count|
90
+ client.write word: word, count: count
91
+ end
92
+ client.write lineno: input["lineno"], result_count: h.size
93
+ end
94
+ end
95
+ end
96
+ end
@@ -0,0 +1,77 @@
1
+ require 'tupelo/app'
2
+
3
+ N = 2 # how many cpus do you want to use for mappers?
4
+ VERBOSE = ARGV.delete "-v"
5
+
6
+ Tupelo.application do |app|
7
+ if VERBOSE
8
+ app.child do |client| # a debugger client, to see what's happening
9
+ note = client.notifier
10
+ puts "%4s %4s %10s %s" % %w{ tick cid status operation }
11
+ loop do
12
+ status, tick, cid, op = note.wait
13
+ unless status == :attempt
14
+ s = status == :failure ? "FAILED" : ""
15
+ puts "%4d %4d %10s %p" % [tick, cid, s, op]
16
+ end
17
+ end
18
+ end
19
+ end
20
+
21
+ app.child do |client|
22
+ document = "I will not map reduce in class\n" * 10
23
+ lineno = 0
24
+ document.each_line do |line|
25
+ lineno += 1
26
+ client.write ["wc input", lineno, line]
27
+ # Note that tuples should be small, so if the data is large, the line
28
+ # should be a reference, not the actual data.
29
+ end
30
+
31
+ results = Hash.new(0)
32
+ lines_remaining = lineno
33
+ results_remaining = 0
34
+ until lines_remaining == 0 and results_remaining == 0 do
35
+ event, *a = client.take [/wc (?:output|done)/, nil, nil]
36
+ # Using a regex is hacky. Better to use an "or" template. See
37
+ # boolean-match.rb. That's not a standard feature yet.
38
+ # Also, in real use it might be better to use hash tuples rather than
39
+ # arrays.
40
+ # See map-reduce-v2.rb.
41
+
42
+ case event
43
+ when "wc output"
44
+ word, count = a
45
+ results[word] += count
46
+ results_remaining -= 1
47
+ when "wc done"
48
+ lineno, result_count = a
49
+ lines_remaining -= 1
50
+ results_remaining += result_count
51
+ end
52
+ end
53
+
54
+ client.log "DONE. results = #{results}"
55
+ client.log "Press ^C to exit"
56
+ end
57
+
58
+ N.times do |i|
59
+ app.child do |client|
60
+ client.log.progname = "mapper #{i}"
61
+
62
+ loop do
63
+ _, lineno, line = client.take ["wc input", Integer, String]
64
+
65
+ h = Hash.new(0)
66
+ line.split.each do |word|
67
+ h[word] += 1
68
+ end
69
+
70
+ h.each do |word, count|
71
+ client.write ["wc output", word, count]
72
+ end
73
+ client.write ["wc done", lineno, h.size]
74
+ end
75
+ end
76
+ end
77
+ end
@@ -0,0 +1,9 @@
1
+ require 'tupelo/app'
2
+
3
+ Tupelo.application do |app|
4
+ app.local do |client|
5
+ client.write_wait ["foo", 42.5]
6
+ p client.read_all [/oo/, nil]
7
+ p client.read_all [nil, 5..95]
8
+ end
9
+ end
data/example/notify.rb ADDED
@@ -0,0 +1,35 @@
1
+ # See also bin/tspy
2
+
3
+ require 'tupelo/app'
4
+
5
+ Tupelo.application do |app|
6
+ app.child do |client|
7
+ Thread.new do
8
+ note = client.notifier
9
+ client.write ["start"]
10
+
11
+ client.log "%10s %10s %10s %s" % %w{ status tick client operation }
12
+ loop do
13
+ status, global_tick, client_id, op = note.wait
14
+ client.log "%10s %10d %10d %p" % [status, global_tick, client_id, op]
15
+ end
16
+ end
17
+
18
+ client.take ["finish"]
19
+ end
20
+
21
+ app.child do |client|
22
+ client.take ["start"]
23
+
24
+ client.write [1, 2]
25
+ client.write [3, 4]
26
+ client.write foo: "bar", baz: ["zap"]
27
+
28
+ client.transaction do |t|
29
+ x, y = t.take [Numeric, Numeric]
30
+ t.write [x, y, x + y]
31
+ end
32
+
33
+ client.write ["finish"]
34
+ end
35
+ end
@@ -0,0 +1,20 @@
1
+ require 'tupelo/app'
2
+
3
+ Tupelo.application do |app|
4
+ app.child do |client|
5
+ client.write [1]
6
+ sleep 0.1
7
+ client.take [1]
8
+ client.write [2]
9
+ end
10
+
11
+ app.child do |client|
12
+ final_i =
13
+ client.take([Integer]) do |optimistic_i|
14
+ client.log "optimistic_i = #{optimistic_i}"
15
+ sleep 0.2
16
+ end
17
+
18
+ client.log "final_i = #{final_i}"
19
+ end
20
+ end
data/example/pulse.rb ADDED
@@ -0,0 +1,24 @@
1
+ require 'tupelo/app'
2
+
3
+ Tupelo.application do |app|
4
+ app.child do |client|
5
+ client.take ['start']
6
+ 10.times do |i|
7
+ client.pulse [i]
8
+ sleep 0.1
9
+ end
10
+ client.write ['finish']
11
+ end
12
+
13
+ app.child do |client|
14
+ Thread.new do
15
+ loop do
16
+ client.log client.read [Integer]
17
+ client.log client.read_all [Integer]
18
+ end
19
+ end
20
+
21
+ client.write ['start']
22
+ client.take ['finish']
23
+ end
24
+ end
@@ -0,0 +1,56 @@
1
+ require 'tupelo/app'
2
+
3
+ Thread.abort_on_exception = true
4
+
5
+ N = 5
6
+
7
+ Tupelo.application do |app|
8
+ N.times do |i|
9
+ app.child do |client|
10
+ Thread.new do
11
+ step = 0
12
+ loop do
13
+ step += 1
14
+ client.transaction do |t|
15
+ t.read ["enabled", i]
16
+ t.write ["output", i, step]
17
+ end
18
+ sleep 0.2
19
+ end
20
+ end
21
+
22
+ client.read ["done"]
23
+ exit
24
+ end
25
+ end
26
+
27
+ app.child do |client|
28
+ t = Thread.new do
29
+ loop do
30
+ msg, i, step = client.take [nil, nil, nil]
31
+ printf "%20s from %2d at step %3d\n", msg, i, step
32
+ end
33
+ end
34
+
35
+ puts "Turning on 0 and 4"
36
+ client.write ["enabled", 0]
37
+ client.write ["enabled", 4]
38
+ sleep 2
39
+
40
+ puts "Turning off 0"
41
+ client.take ["enabled", 0]
42
+ sleep 2
43
+
44
+ puts "Turning off 4"
45
+ client.take ["enabled", 4]
46
+ sleep 2
47
+
48
+ puts "Turning on 1 and 3"
49
+ client.write ["enabled", 1]
50
+ client.write ["enabled", 3]
51
+ sleep 2
52
+
53
+ puts "Bye!"
54
+ client.write ["done"]
55
+ end
56
+ end
@@ -0,0 +1,18 @@
1
+ # Same as small.rb, but with the generic setup code factored out.
2
+
3
+ require 'tupelo/app'
4
+
5
+ Tupelo.application do |app|
6
+ app.child do |client|
7
+ client.write [2, 3, "frogs"]
8
+ _, s = client.take ["animals", nil]
9
+ puts s
10
+ end
11
+
12
+ app.child do |client|
13
+ x, y, s = client.take [Numeric, Numeric, String]
14
+ s2 = ([s] * (x + y)).join(" ")
15
+ client.write ["animals", s2]
16
+ end
17
+ end
18
+
data/example/small.rb ADDED
@@ -0,0 +1,76 @@
1
+ # An example of using EasyServe directly (the hard way) rather than tupelo/app.
2
+ # See small-simplified.rb for the difference.
3
+
4
+ require 'easy-serve'
5
+
6
+ log_level = case
7
+ when ARGV.delete("--debug"); Logger::DEBUG
8
+ when ARGV.delete("--info"); Logger::INFO
9
+ when ARGV.delete("--warn"); Logger::WARN
10
+ when ARGV.delete("--error"); Logger::ERROR
11
+ when ARGV.delete("--fatal"); Logger::FATAL
12
+ else Logger::WARN
13
+ end
14
+
15
+ EasyServe.start(servers_file: "small-servers.yaml") do |ez|
16
+ log = ez.log
17
+ log.level = log_level
18
+ log.progname = "parent"
19
+
20
+ ez.start_servers do
21
+ arc_to_seq_sock, seq_to_arc_sock = UNIXSocket.pair
22
+ arc_to_cseq_sock, cseq_to_arc_sock = UNIXSocket.pair
23
+
24
+ ez.server :seqd do |svr|
25
+ require 'funl/message-sequencer'
26
+ seq = Funl::MessageSequencer.new svr, seq_to_arc_sock, log: log,
27
+ blob_type: 'msgpack' # the default
28
+ #blob_type: 'marshal' # if you need to pass general ruby objects
29
+ #blob_type: 'yaml' # less general ruby objects, but cross-language
30
+ #blob_type: 'json' # more portable than yaml, but more restrictive
31
+ seq.start
32
+ end
33
+
34
+ ez.server :cseqd do |svr|
35
+ require 'funl/client-sequencer'
36
+ cseq = Funl::ClientSequencer.new svr, cseq_to_arc_sock, log: log
37
+ cseq.start
38
+ end
39
+
40
+ ez.server :arcd do |svr|
41
+ require 'tupelo/archiver'
42
+ arc = Archiver.new svr, seq: arc_to_seq_sock, cseq: arc_to_cseq_sock,
43
+ log: log
44
+ arc.start
45
+ end
46
+ end
47
+
48
+ def run_client opts
49
+ log = opts[:log]
50
+ log.progname = "client <starting in #{log.progname}>"
51
+ require 'tupelo/client'
52
+ client = Client.new opts
53
+ client.start do
54
+ log.progname = "client #{client.client_id}"
55
+ end
56
+ yield client
57
+ ensure
58
+ client.stop if client # gracefully exit the tuplespace management thread
59
+ end
60
+
61
+ ez.client :seqd, :cseqd, :arcd do |seqd, cseqd, arcd|
62
+ run_client seq: seqd, cseq: cseqd, arc: arcd, log: log do |client|
63
+ client.write [2, 3, "frogs"]
64
+ _, s = client.take ["animals", nil]
65
+ puts s
66
+ end
67
+ end
68
+
69
+ ez.client :seqd, :cseqd, :arcd do |seqd, cseqd, arcd|
70
+ run_client seq: seqd, cseq: cseqd, arc: arcd, log: log do |client|
71
+ x, y, s = client.take [Numeric, Numeric, String]
72
+ s2 = ([s] * (x + y)).join(" ")
73
+ client.write ["animals", s2]
74
+ end
75
+ end
76
+ end
data/example/tcp.rb ADDED
@@ -0,0 +1,35 @@
1
+ # Most examples in this dir use unix sockets for simplicity. With a little
2
+ # more effort, you can switch to tcp for remote access.
3
+ #
4
+ # Run this file like:
5
+ #
6
+ # ruby tcp.rb --info
7
+ #
8
+ # You can test with a local client:
9
+ #
10
+ # ../bin/tup tcp.yaml
11
+ #
12
+ # Copy tcp.yaml to a remote host, and in the remote copy edit the
13
+ # addr field to a hostname (or ip addr) isntead of 0.0.0.0.
14
+ #
15
+ # Then run a client like this:
16
+ #
17
+ # bin/tup remote-copy-of-tcp.yaml
18
+
19
+ require 'tupelo/app'
20
+
21
+ svr = "tcp.yaml" # copy this file to remote clients, setting host as needed
22
+ port = 9901 # Use 0 to let system choose free port
23
+
24
+ Tupelo.application servers_file: svr,
25
+ seqd_addr: [:tcp, '0.0.0.0', port],
26
+ cseqd_addr: [:tcp, '0.0.0.0', port + 1],
27
+ arcd_addr: [:tcp, '0.0.0.0', port + 2] do |app|
28
+ if app.owns_servers
29
+ puts "server started; ^C to stop"
30
+ puts "(run with --info to see events)" if app.log.level > Logger::INFO
31
+ sleep
32
+ else
33
+ abort "server seems to be running already; check file #{svr.inspect}"
34
+ end
35
+ end
@@ -0,0 +1,21 @@
1
+ # Use tupelo's scheduler to time out a transaction. See also timeout.rb.
2
+ # This has the advantage of using just one extra thread for all timeouts,
3
+ # rather than one thread per timeout.
4
+
5
+ require 'tupelo/app'
6
+
7
+ Tupelo.application do |app|
8
+ app.child do |client|
9
+ result =
10
+ begin
11
+ client.transaction timeout: 1 do |t|
12
+ t.take ["foo"]
13
+ end
14
+ rescue TimeoutError => ex
15
+ ex
16
+ end
17
+ client.log "result = #{result.inspect}"
18
+ end
19
+ end
20
+
21
+
@@ -0,0 +1,27 @@
1
+ # Use ruby's timeout lib to time out a read. For transactions there is also
2
+ # the timeout parameter (which is more efficient). See timeout-trans.rb.
3
+
4
+ require 'tupelo/app'
5
+
6
+ Tupelo.application do |app|
7
+ app.child do |client|
8
+ begin
9
+ n_sec = 2
10
+ Timeout.timeout n_sec do
11
+ client.log "waiting for non-existing tuple #{[0]}"
12
+ client.read [0]
13
+ end
14
+ rescue TimeoutError
15
+ end
16
+
17
+ r = client.read [1]
18
+ client.log "got #{r}"
19
+ end
20
+
21
+ app.child do |client|
22
+ sleep 1
23
+ client.log "writing [1]"
24
+ client.write [1]
25
+ end
26
+ end
27
+
@@ -0,0 +1,14 @@
1
+ require 'tupelo/app'
2
+
3
+ svr = "tiny-server.yaml"
4
+
5
+ Tupelo.application servers_file: svr do |app|
6
+ if app.owns_servers
7
+ abort "server not running"
8
+ end
9
+
10
+ app.child do |client|
11
+ client.write ["Hello", "world!"]
12
+ p client.take [nil, nil]
13
+ end
14
+ end
@@ -0,0 +1,12 @@
1
+ require 'tupelo/app'
2
+
3
+ svr = "tiny-server.yaml"
4
+
5
+ Tupelo.application servers_file: svr do |app|
6
+ if app.owns_servers
7
+ puts "server started"
8
+ sleep
9
+ else
10
+ abort "server seems to be running already; check file #{svr.inspect}"
11
+ end
12
+ end
@@ -0,0 +1,40 @@
1
+ require 'tupelo/app'
2
+
3
+ Tupelo.application do |app|
4
+ app.child do |client|
5
+ done = false
6
+ until done do
7
+ client.transaction do |t|
8
+ op, x, y = t.take [String, Numeric, Numeric]
9
+ # further operations within this transaction can depend on the above.
10
+
11
+ case op
12
+ when "+"
13
+ t.write [op, x, y, x + y]
14
+ when "*"
15
+ t.write [op, x, y, x * y]
16
+ when "select"
17
+ _, _, _, z = t.take [nil, nil, nil, x..y]
18
+ t.write [op, x, y, z]
19
+ when "stop"
20
+ done = true
21
+ end
22
+ end
23
+ end
24
+ end
25
+
26
+ app.child do |client|
27
+ client.write ["+", 1, 2]
28
+ results = client.read ["+", 1, 2, nil]
29
+ p results
30
+
31
+ client.write ["*", 3, 4]
32
+ results = client.read ["*", 3, 4, nil]
33
+ p results
34
+
35
+ client.write ["select", 10, 20]
36
+ p client.read ["select", 10, 20, nil]
37
+
38
+ client.write ["stop", 0, 0]
39
+ end
40
+ end
@@ -0,0 +1,17 @@
1
+ require 'tupelo/app'
2
+
3
+ Tupelo.application do |app|
4
+ app.local do |client|
5
+ client.write [1]
6
+ client.write [2]
7
+ w = client.write [3]
8
+ p client.read_all [nil]
9
+ w.wait # wait for the write to come back and be applied to the client
10
+ p client.read_all [nil]
11
+
12
+ client.write [4]
13
+ client.write [5]
14
+ client.write_wait [6]
15
+ p client.read_all [nil]
16
+ end
17
+ end