tupelo 0.9 → 0.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +31 -1
  3. data/bugs/read-take.rb +19 -0
  4. data/bugs/take-write.rb +7 -7
  5. data/example/app-and-tup.rb +11 -8
  6. data/example/async-transaction.rb +7 -7
  7. data/example/balance-xfer-locking.rb +13 -13
  8. data/example/balance-xfer-retry.rb +16 -16
  9. data/example/balance-xfer.rb +9 -9
  10. data/example/boolean-match.rb +5 -5
  11. data/example/bounded-retry.rb +9 -9
  12. data/example/broker-locking.rb +14 -14
  13. data/example/broker-optimistic.rb +7 -7
  14. data/example/cancel.rb +7 -7
  15. data/example/concurrent-transactions.rb +17 -17
  16. data/example/custom-class.rb +9 -9
  17. data/example/custom-search.rb +8 -8
  18. data/example/fail-and-retry.rb +11 -11
  19. data/example/hash-tuples.rb +12 -10
  20. data/example/increment.rb +8 -8
  21. data/example/load-balancer.rb +2 -1
  22. data/example/lock-mgr-with-queue.rb +18 -18
  23. data/example/lock-mgr.rb +18 -18
  24. data/example/map-reduce-v2.rb +11 -11
  25. data/example/map-reduce.rb +11 -11
  26. data/example/matching.rb +5 -5
  27. data/example/message-bus.rb +6 -3
  28. data/example/notify.rb +17 -17
  29. data/example/optimist.rb +9 -9
  30. data/example/parallel.rb +16 -8
  31. data/example/pregel/distributed.rb +129 -0
  32. data/example/pregel/pagerank.rb +72 -0
  33. data/example/pregel/pregel.rb +102 -0
  34. data/example/pregel/remote.rb +165 -0
  35. data/example/pulse.rb +10 -10
  36. data/example/read-in-trans.rb +15 -15
  37. data/example/subspace.rb +34 -0
  38. data/example/take-nowait.rb +1 -0
  39. data/example/tcp.rb +3 -3
  40. data/example/timeout-trans.rb +5 -5
  41. data/example/timeout.rb +10 -9
  42. data/example/tiny-client.rb +5 -5
  43. data/example/tiny-server.rb +2 -2
  44. data/example/transaction-logic.rb +16 -16
  45. data/example/wait-interrupt.rb +38 -0
  46. data/example/write-wait.rb +11 -11
  47. data/lib/tupelo/archiver/tuplespace.rb +5 -1
  48. data/lib/tupelo/archiver/worker.rb +25 -18
  49. data/lib/tupelo/client/reader.rb +2 -2
  50. data/lib/tupelo/client/transaction.rb +79 -36
  51. data/lib/tupelo/client/tuplespace.rb +1 -0
  52. data/lib/tupelo/client/worker.rb +107 -13
  53. data/lib/tupelo/client.rb +36 -2
  54. data/lib/tupelo/version.rb +1 -1
  55. data/test/lib/mock-client.rb +4 -0
  56. data/test/lib/testable-worker.rb +1 -1
  57. data/test/stress/concurrent-transactions.rb +15 -15
  58. data/test/system/test-archiver.rb +8 -8
  59. data/test/unit/test-ops.rb +56 -0
  60. metadata +72 -68
  61. data/bugs/write-read.rb +0 -15
  62. data/example/broker-queue.rb +0 -35
  63. data/example/child-of-child.rb +0 -34
@@ -2,13 +2,13 @@ require 'tupelo/app'
2
2
 
3
3
  svr = "tiny-server.yaml"
4
4
 
5
- Tupelo.application servers_file: svr do |app|
6
- if app.owns_servers
5
+ Tupelo.application servers_file: svr do
6
+ if owns_servers
7
7
  abort "server not running"
8
8
  end
9
9
 
10
- app.child do |client|
11
- client.write ["Hello", "world!"]
12
- p client.take [nil, nil]
10
+ child do
11
+ write ["Hello", "world!"]
12
+ p take [nil, nil]
13
13
  end
14
14
  end
@@ -2,8 +2,8 @@ require 'tupelo/app'
2
2
 
3
3
  svr = "tiny-server.yaml"
4
4
 
5
- Tupelo.application servers_file: svr do |app|
6
- if app.owns_servers
5
+ Tupelo.application servers_file: svr do
6
+ if owns_servers
7
7
  puts "server started"
8
8
  sleep
9
9
  else
@@ -1,21 +1,21 @@
1
1
  require 'tupelo/app'
2
2
 
3
- Tupelo.application do |app|
4
- app.child do |client|
3
+ Tupelo.application do
4
+ child do
5
5
  done = false
6
6
  until done do
7
- client.transaction do |t|
8
- op, x, y = t.take [String, Numeric, Numeric]
7
+ transaction do
8
+ op, x, y = take [String, Numeric, Numeric]
9
9
  # further operations within this transaction can depend on the above.
10
10
 
11
11
  case op
12
12
  when "+"
13
- t.write [op, x, y, x + y]
13
+ write [op, x, y, x + y]
14
14
  when "*"
15
- t.write [op, x, y, x * y]
15
+ write [op, x, y, x * y]
16
16
  when "select"
17
- _, _, _, z = t.take [nil, nil, nil, x..y]
18
- t.write [op, x, y, z]
17
+ _, _, _, z = take [nil, nil, nil, x..y]
18
+ write [op, x, y, z]
19
19
  when "stop"
20
20
  done = true
21
21
  end
@@ -23,18 +23,18 @@ Tupelo.application do |app|
23
23
  end
24
24
  end
25
25
 
26
- app.child do |client|
27
- client.write ["+", 1, 2]
28
- results = client.read ["+", 1, 2, nil]
26
+ child do
27
+ write ["+", 1, 2]
28
+ results = read ["+", 1, 2, nil]
29
29
  p results
30
30
 
31
- client.write ["*", 3, 4]
32
- results = client.read ["*", 3, 4, nil]
31
+ write ["*", 3, 4]
32
+ results = read ["*", 3, 4, nil]
33
33
  p results
34
34
 
35
- client.write ["select", 10, 20]
36
- p client.read ["select", 10, 20, nil]
35
+ write ["select", 10, 20]
36
+ p read ["select", 10, 20, nil]
37
37
 
38
- client.write ["stop", 0, 0]
38
+ write ["stop", 0, 0]
39
39
  end
40
40
  end
@@ -0,0 +1,38 @@
1
+ # This example shows how a transaction can be used to wait for some event,
2
+ # but interrupt the wait when some other event happens.
3
+
4
+ require 'tupelo/app'
5
+
6
+ Tupelo.application do
7
+ child do
8
+ log.progname = "host"
9
+
10
+ transaction do
11
+ mode = read(mode: nil)["mode"]
12
+ log "waiting for visitor to arrive by #{mode}"
13
+ read ["visitor arrives"]
14
+ end
15
+
16
+ log "welcome!"
17
+ end
18
+
19
+ child do
20
+ log.progname = "visitor"
21
+
22
+ log "I think I am coming by train"
23
+ write_wait mode: "train"
24
+ sleep 1
25
+
26
+ log "changing my mind, coming by plane"
27
+ transaction {take mode: nil; write mode: "plane"}
28
+ sleep 1
29
+
30
+ log "changing my mind again, coming by car"
31
+ transaction {take mode: nil; write mode: "car"}
32
+ sleep 1
33
+
34
+ log "hello!"
35
+ write_wait ["visitor arrives"]
36
+ end
37
+ end
38
+
@@ -1,17 +1,17 @@
1
1
  require 'tupelo/app'
2
2
 
3
- Tupelo.application do |app|
4
- app.local do |client|
5
- client.write [1]
6
- client.write [2]
7
- w = client.write [3]
8
- p client.read_all [nil]
3
+ Tupelo.application do
4
+ local do
5
+ write [1]
6
+ write [2]
7
+ w = write [3]
8
+ p read_all
9
9
  w.wait # wait for the write to come back and be applied to the client
10
- p client.read_all [nil]
10
+ p read_all
11
11
 
12
- client.write [4]
13
- client.write [5]
14
- client.write_wait [6]
15
- p client.read_all [nil]
12
+ write [4]
13
+ write [5]
14
+ write_wait [6]
15
+ p read_all
16
16
  end
17
17
  end
@@ -5,9 +5,13 @@ class Tupelo::Archiver
5
5
  attr_reader :zero_tolerance
6
6
 
7
7
  def initialize(zero_tolerance: Tupelo::Archiver::ZERO_TOLERANCE)
8
+ @zero_tolerance = zero_tolerance
9
+ clear
10
+ end
11
+
12
+ def clear
8
13
  @counts = Hash.new(0) # tuple => count
9
14
  @nzero = 0
10
- @zero_tolerance = zero_tolerance
11
15
  end
12
16
 
13
17
  # note: multiple equal tuples are yielded once
@@ -38,7 +38,7 @@ class Tupelo::Archiver
38
38
  stream = client.arc_server_stream_for req.io
39
39
 
40
40
  begin
41
- op, tags, tick = stream.read
41
+ op, sub_delta, tick = stream.read
42
42
  rescue EOFError
43
43
  log.debug {"#{stream.peer_name} disconnected from archiver"}
44
44
  return
@@ -48,18 +48,18 @@ class Tupelo::Archiver
48
48
 
49
49
  log.info {
50
50
  "#{stream.peer_name} requested #{op.inspect} at tick=#{tick}" +
51
- (tags ? " on #{tags}" : "")}
51
+ (sub_delta ? " for #{sub_delta}" : "")}
52
52
 
53
53
  if tick <= global_tick
54
- fork_for_op op, tags, tick, stream, req
54
+ fork_for_op op, sub_delta, tick, stream, req
55
55
  else
56
56
  at_tick tick do
57
- fork_for_op op, tags, tick, stream, req
57
+ fork_for_op op, sub_delta, tick, stream, req
58
58
  end
59
59
  end
60
60
  end
61
61
 
62
- def fork_for_op op, tags, tick, stream, req
62
+ def fork_for_op op, sub_delta, tick, stream, req
63
63
  fork do
64
64
  begin
65
65
  case op
@@ -68,7 +68,7 @@ class Tupelo::Archiver
68
68
  when "get range" ### handle this in Funl::HistoryWorker
69
69
  raise "Unimplemented" ###
70
70
  when GET_TUPLESPACE
71
- send_tuplespace stream, tags
71
+ send_tuplespace stream, sub_delta
72
72
  else
73
73
  raise "Unknown operation: #{op.inspect}"
74
74
  end
@@ -94,18 +94,32 @@ class Tupelo::Archiver
94
94
  end
95
95
  end
96
96
 
97
- def send_tuplespace stream, templates
97
+ def send_tuplespace stream, sub_delta
98
98
  log.info {
99
99
  "send_tuplespace to #{stream.peer_name} " +
100
100
  "at tick #{global_tick.inspect} " +
101
- (templates ? " with templates #{templates.inspect}" : "")}
101
+ (sub_delta ? " with sub_delta #{sub_delta.inspect}" : "")}
102
102
 
103
103
  stream << [global_tick]
104
104
 
105
- if templates
106
- templates = templates.map {|t| Tupelo::Client::Template.new t}
105
+ ## better: make use of sub_delta["subscribed_*"] to reduce what
106
+ ## has to be sent.
107
+
108
+ if sub_delta["request_all"]
107
109
  tuplespace.each do |tuple, count|
108
- if templates.any? {|template| template === tuple}
110
+ count.times do ## just dump and send str * count?
111
+ stream << tuple ## optimize this, and cache the serial
112
+ ## optimization: use stream.write_to_buffer
113
+ end
114
+ end
115
+
116
+ else
117
+ tags = sub_delta["request_tags"] ## use set
118
+ subs = subspaces.select {|sub| tags.include? sub.tag}
119
+
120
+ tuplespace.each do |tuple, count|
121
+ ## alternately, store tags with tuples (risk if dynamic spaces)
122
+ if subs.any? {|sub| sub === tuple}
109
123
  count.times do
110
124
  stream << tuple
111
125
  ## optimization: use stream.write_to_buffer
@@ -114,13 +128,6 @@ class Tupelo::Archiver
114
128
  ## optimize this if templates have simple form, such as
115
129
  ## [ [str1, nil, ...], [str2, nil, ...], ...]
116
130
  end
117
- else
118
- tuplespace.each do |tuple, count|
119
- count.times do ## just dump and send str * count?
120
- stream << tuple ## optimize this, and cache the serial
121
- ## optimization: use stream.write_to_buffer
122
- end
123
- end
124
131
  end
125
132
 
126
133
  stream << nil # terminator
@@ -61,9 +61,9 @@ class Tupelo::Client
61
61
  end
62
62
 
63
63
  def wait
64
- @client.log.debug {"waiting for #{self}"}
64
+ @client.log.debug {"waiting for #{inspect}"}
65
65
  r = queue.pop
66
- @client.log.debug {"finished waiting for #{self}"}
66
+ @client.log.debug {"finished waiting for #{inspect}"}
67
67
  r
68
68
  end
69
69
 
@@ -103,10 +103,13 @@ class Tupelo::Client
103
103
  attr_reader :pulses
104
104
  attr_reader :take_templates
105
105
  attr_reader :read_templates
106
- attr_reader :take_tuples
107
- attr_reader :read_tuples
106
+ attr_reader :take_tuples_for_remote
107
+ attr_reader :take_tuples_for_local
108
+ attr_reader :read_tuples_for_remote
109
+ attr_reader :read_tuples_for_local
108
110
  attr_reader :granted_tuples
109
111
  attr_reader :missing
112
+ attr_reader :tags
110
113
 
111
114
  STATES = [
112
115
  OPEN = :open, # initial state
@@ -140,10 +143,13 @@ class Tupelo::Client
140
143
  @pulses = []
141
144
  @take_templates = []
142
145
  @read_templates = []
143
- @take_tuples = []
144
- @read_tuples = []
146
+ @take_tuples_for_remote = []
147
+ @take_tuples_for_local = []
148
+ @read_tuples_for_remote = []
149
+ @read_tuples_for_local = []
145
150
  @granted_tuples = nil
146
151
  @missing = nil
152
+ @tags = nil
147
153
  @_take_nowait = nil
148
154
  @_read_nowait = nil
149
155
 
@@ -181,6 +187,7 @@ class Tupelo::Client
181
187
 
182
188
  ops = [ ["write", writes], ["pulse", pulses],
183
189
  ["take", take_templates], ["read", read_templates] ]
190
+ ## exclude templates that were satisfied locally by writes
184
191
  ops.map! do |label, tuples|
185
192
  ["#{label} #{tuples.map(&:inspect).join(", ")}"] unless tuples.empty?
186
193
  end
@@ -209,7 +216,10 @@ class Tupelo::Client
209
216
  raise TransactionStateError, "not open: #{inspect}" unless open? or
210
217
  failed?
211
218
  check_tuples tuples
212
- @writes.concat tuples
219
+ blobber = worker.blobber
220
+ @writes.concat tuples.map {|t| blobber.load(blobber.dump(t))}
221
+ # this is both to de-alias (esp. in case of marshal or yaml) and
222
+ # to convert symbols to strings (in case of msgpack or json)
213
223
  nil
214
224
  end
215
225
 
@@ -218,7 +228,8 @@ class Tupelo::Client
218
228
  raise TransactionStateError, "not open: #{inspect}" unless open? or
219
229
  failed?
220
230
  check_tuples tuples
221
- @pulses.concat tuples
231
+ blobber = worker.blobber
232
+ @pulses.concat tuples.map {|t| blobber.load(blobber.dump(t))}
222
233
  nil
223
234
  end
224
235
 
@@ -233,7 +244,7 @@ class Tupelo::Client
233
244
  log.debug {"asking worker to take #{template_spec.inspect}"}
234
245
  worker_push self
235
246
  wait
236
- return take_tuples.last
247
+ return take_tuples_for_local.last
237
248
  end
238
249
 
239
250
  def take_nowait template_spec
@@ -249,7 +260,7 @@ class Tupelo::Client
249
260
  log.debug {"asking worker to take_nowait #{template_spec.inspect}"}
250
261
  worker_push self
251
262
  wait
252
- return take_tuples[i]
263
+ return take_tuples_for_local[i]
253
264
  end
254
265
 
255
266
  # transaction applies only if template has a match
@@ -263,7 +274,7 @@ class Tupelo::Client
263
274
  log.debug {"asking worker to read #{template_spec.inspect}"}
264
275
  worker_push self
265
276
  wait
266
- return read_tuples.last
277
+ return read_tuples_for_local.last
267
278
  end
268
279
 
269
280
  def read_nowait template_spec
@@ -279,7 +290,7 @@ class Tupelo::Client
279
290
  log.debug {"asking worker to read #{template_spec.inspect}"}
280
291
  worker_push self
281
292
  wait
282
- return read_tuples[i]
293
+ return read_tuples_for_local[i]
283
294
  end
284
295
 
285
296
  def abort
@@ -299,8 +310,9 @@ class Tupelo::Client
299
310
  def commit
300
311
  if open?
301
312
  if @writes.empty? and @pulses.empty? and
302
- @take_tuples.empty? and @read_tuples.empty?
303
- @global_tick = global_tick
313
+ @take_tuples_for_remote.all? {|t| t.nil?} and
314
+ @read_tuples_for_remote.all? {|t| t.nil?}
315
+ @global_tick = worker.global_tick ## ok?
304
316
  done!
305
317
  log.info {"not committing empty transaction"}
306
318
  else
@@ -352,11 +364,16 @@ class Tupelo::Client
352
364
  end
353
365
  end
354
366
 
355
- def async
356
- raise ArgumentError, "must provide block" unless block_given?
367
+ def async &block
368
+ raise ArgumentError, "must provide block" unless block
357
369
  TransactionThread.new(self) do ## Fiber?
358
370
  begin
359
- val = yield self
371
+ val =
372
+ if block.arity == 0
373
+ instance_eval &block
374
+ else
375
+ yield self
376
+ end
360
377
  commit.wait
361
378
  val
362
379
  rescue TransactionFailure => ex
@@ -379,19 +396,21 @@ class Tupelo::Client
379
396
  raise unless in_worker_thread?
380
397
 
381
398
  if new_tuple
382
- return true if take_tuples.all? and read_tuples.all?
399
+ return true if take_tuples_for_local.all? and read_tuples_for_local.all?
383
400
 
384
- take_tuples.each_with_index do |tuple, i|
401
+ take_tuples_for_local.each_with_index do |tuple, i|
385
402
  if not tuple and take_templates[i] === new_tuple
386
- take_tuples[i] = new_tuple
403
+ take_tuples_for_local[i] = new_tuple
404
+ take_tuples_for_remote[i] = new_tuple
387
405
  log.debug {"prepared #{inspect} with #{new_tuple}"}
388
406
  break
389
407
  end
390
408
  end
391
409
 
392
- read_tuples.each_with_index do |tuple, i|
410
+ read_tuples_for_local.each_with_index do |tuple, i|
393
411
  if not tuple and read_templates[i] === new_tuple
394
- read_tuples[i] = new_tuple
412
+ read_tuples_for_local[i] = new_tuple
413
+ read_tuples_for_remote[i] = new_tuple
395
414
  log.debug {"prepared #{inspect} with #{new_tuple}"}
396
415
  end
397
416
  end
@@ -399,11 +418,22 @@ class Tupelo::Client
399
418
  else
400
419
  ## optimization: use tuple cache
401
420
  skip = nil
402
- (take_tuples.size...take_templates.size).each do |i|
403
- take_tuples[i] = worker.tuplespace.find_match_for(
404
- take_templates[i], distinct_from: take_tuples)
405
- if take_tuples[i]
406
- log.debug {"prepared #{inspect} with #{take_tuples[i]}"}
421
+ (take_tuples_for_local.size...take_templates.size).each do |i|
422
+ template = take_templates[i]
423
+
424
+ if wt = @writes.find {|tuple| template === tuple}
425
+ take_tuples_for_remote[i] = nil
426
+ take_tuples_for_local[i] = wt.dup
427
+ @writes.delete wt
428
+ next
429
+ end
430
+
431
+ take_tuples_for_local[i] = take_tuples_for_remote[i] =
432
+ worker.tuplespace.find_match_for(template,
433
+ distinct_from: take_tuples_for_local)
434
+
435
+ if take_tuples_for_local[i]
436
+ log.debug {"prepared #{inspect} with #{take_tuples_for_local[i]}"}
407
437
  else
408
438
  if @_take_nowait and @_take_nowait[i]
409
439
  (skip ||= []) << i
@@ -412,17 +442,28 @@ class Tupelo::Client
412
442
  end
413
443
 
414
444
  skip and skip.reverse_each do |i|
415
- take_tuples.delete_at i
445
+ take_tuples_for_local.delete_at i
446
+ take_tuples_for_remote.delete_at i
416
447
  take_templates.delete_at i
417
448
  @_take_nowait.delete i
418
449
  end
419
450
 
420
451
  skip = nil
421
- (read_tuples.size...read_templates.size).each do |i|
422
- read_tuples[i] = worker.tuplespace.find_match_for(
423
- read_templates[i])
424
- if read_tuples[i]
425
- log.debug {"prepared #{inspect} with #{read_tuples[i]}"}
452
+ (read_tuples_for_local.size...read_templates.size).each do |i|
453
+ template = read_templates[i]
454
+
455
+ if wt = @writes.find {|tuple| template === tuple}
456
+ read_tuples_for_remote[i] = nil
457
+ read_tuples_for_local[i] = wt.dup
458
+ next
459
+ end
460
+
461
+ read_tuples_for_local[i] = read_tuples_for_remote[i] =
462
+ worker.tuplespace.find_match_for(template,
463
+ distinct_from: take_tuples_for_local)
464
+
465
+ if read_tuples_for_local[i]
466
+ log.debug {"prepared #{inspect} with #{read_tuples_for_local[i]}"}
426
467
  else
427
468
  if @_read_nowait and @_read_nowait[i]
428
469
  (skip ||= []) << i
@@ -431,7 +472,8 @@ class Tupelo::Client
431
472
  end
432
473
 
433
474
  skip and skip.reverse_each do |i|
434
- read_tuples.delete_at i
475
+ read_tuples_for_local.delete_at i
476
+ read_tuples_for_remote.delete_at i
435
477
  read_templates.delete_at i
436
478
  @_read_nowait.delete i
437
479
  end
@@ -441,11 +483,12 @@ class Tupelo::Client
441
483
  ## convert cancelling take/write to read
442
484
  ## check that remaining take/read tuples do not cross a space boundary
443
485
 
444
- if take_tuples.all? and read_tuples.all?
486
+ if take_tuples_for_local.all? and read_tuples_for_local.all?
445
487
  @queue << true
446
488
  log.debug {
447
489
  "prepared #{inspect}, " +
448
- "take tuples: #{take_tuples}, read tuples: #{read_tuples}"}
490
+ "take tuples: #{take_tuples_for_local}, " +
491
+ "read tuples: #{read_tuples_for_local}"}
449
492
  end
450
493
 
451
494
  return true
@@ -455,7 +498,7 @@ class Tupelo::Client
455
498
  return false if closed? or failed? # might change during this method
456
499
  raise unless in_worker_thread?
457
500
 
458
- @take_tuples.each do |tuple|
501
+ @take_tuples_for_remote.each do |tuple|
459
502
  if tuple == missing_tuple ## might be false positive, but ok
460
503
  fail [missing_tuple]
461
504
  ## optimization: manage tuple cache
@@ -463,7 +506,7 @@ class Tupelo::Client
463
506
  end
464
507
  end
465
508
 
466
- @read_tuples.each do |tuple|
509
+ @read_tuples_for_remote.each do |tuple|
467
510
  if tuple == missing_tuple ## might be false positive, but ok
468
511
  fail [missing_tuple]
469
512
  return false
@@ -44,6 +44,7 @@ class Tupelo::Client
44
44
  def insert(*); self; end
45
45
  def find_distinct_matches_for(*); raise; end ##?
46
46
  def find_match_for(*); raise; end ##?
47
+ def clear; end
47
48
 
48
49
  ## should store space metadata, so outgoing writes can be tagged
49
50
  end