tupelo 0.21 → 0.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +171 -45
  3. data/bin/tup +51 -0
  4. data/example/counters/merge.rb +23 -3
  5. data/example/multi-tier/multi-sinatras.rb +5 -0
  6. data/example/riemann/event-subspace.rb +1 -4
  7. data/example/riemann/expiration-dbg.rb +2 -0
  8. data/example/riemann/producer.rb +4 -3
  9. data/example/riemann/v1/riemann.rb +2 -2
  10. data/example/riemann/v2/event-template.rb +71 -0
  11. data/example/riemann/v2/expirer.rb +1 -1
  12. data/example/riemann/v2/hash-store.rb +1 -0
  13. data/example/riemann/v2/ordered-event-store.rb +4 -1
  14. data/example/riemann/v2/riemann.rb +15 -8
  15. data/example/riemann/v2/sqlite-event-store.rb +117 -72
  16. data/example/sqlite/poi-store.rb +1 -1
  17. data/example/sqlite/poi-template.rb +2 -2
  18. data/example/sqlite/poi-v2.rb +2 -2
  19. data/example/subspaces/ramp.rb +9 -2
  20. data/example/tcp.rb +5 -0
  21. data/example/tiny-tcp-client.rb +15 -0
  22. data/example/tiny-tcp-service.rb +32 -0
  23. data/lib/tupelo/app.rb +4 -4
  24. data/lib/tupelo/app/builder.rb +2 -2
  25. data/lib/tupelo/app/irb-shell.rb +3 -3
  26. data/lib/tupelo/archiver.rb +0 -2
  27. data/lib/tupelo/archiver/tuplestore.rb +1 -1
  28. data/lib/tupelo/archiver/worker.rb +6 -6
  29. data/lib/tupelo/client.rb +2 -2
  30. data/lib/tupelo/client/reader.rb +3 -3
  31. data/lib/tupelo/client/scheduler.rb +1 -1
  32. data/lib/tupelo/client/subspace.rb +2 -2
  33. data/lib/tupelo/client/transaction.rb +28 -28
  34. data/lib/tupelo/client/tuplestore.rb +2 -2
  35. data/lib/tupelo/client/worker.rb +11 -10
  36. data/lib/tupelo/util/bin-circle.rb +8 -8
  37. data/lib/tupelo/util/boolean.rb +1 -1
  38. data/lib/tupelo/version.rb +1 -1
  39. data/test/lib/mock-client.rb +10 -10
  40. data/test/system/test-archiver.rb +2 -2
  41. data/test/unit/test-ops.rb +21 -21
  42. metadata +10 -20
  43. data/example/bingo/bingo-v2.rb +0 -20
  44. data/example/broker-queue.rb +0 -35
  45. data/example/child-of-child.rb +0 -34
  46. data/example/dataflow.rb +0 -21
  47. data/example/pregel/dist-opt.rb +0 -15
  48. data/example/riemann/v2/event-sql.rb +0 -56
  49. data/example/sqlite/tmp/poi-sqlite.rb +0 -35
  50. data/example/subspaces/addr-book-v1.rb +0 -104
  51. data/example/subspaces/addr-book-v2.rb +0 -16
  52. data/example/subspaces/sorted-set-space-OLD.rb +0 -130
  53. data/lib/tupelo/tuplets/persistent-archiver.rb +0 -86
  54. data/lib/tupelo/tuplets/persistent-archiver/tuplespace.rb +0 -91
  55. data/lib/tupelo/tuplets/persistent-archiver/worker.rb +0 -114
@@ -1,20 +0,0 @@
1
-
2
- __END__
3
- # dealer
4
- child passive: true do
5
- loop do
6
- # transaction -- add to v2
7
- _, player_id = take ["buy", nil]
8
-
9
- end
10
- end
11
-
12
- # player -- in v3 use subspace per player
13
- end
14
-
15
- ## how to buy 4 without increasing contention risk
16
-
17
- ## reduce contention for cards by randomizing or cons. hashing?
18
-
19
- ## swapping these lines -> more contention?
20
-
@@ -1,35 +0,0 @@
1
- # more like how you would do it in redis, except that the queue is not stored in
2
- # the central server, so operations on it are not a bottleneck, FWIW
3
-
4
- require 'tupelo/app'
5
-
6
- N_PLAYERS = 10
7
-
8
- Tupelo.application do
9
- N_PLAYERS.times do
10
- # sleep rand / 10 # reduce contention -- could also randomize inserts
11
- child do
12
- me = client_id
13
- write name: me
14
-
15
- you = transaction do
16
- game = read_nowait(
17
- player1: nil,
18
- player2: me)
19
- break game["player1"] if game
20
-
21
- unless take_nowait name: me
22
- raise Tupelo::Client::TransactionFailure
23
- end
24
-
25
- you = take(name: nil)["name"]
26
- write(
27
- player1: me,
28
- player2: you)
29
- you
30
- end
31
-
32
- log "now playing with #{you}"
33
- end
34
- end
35
- end
@@ -1,34 +0,0 @@
1
- require 'tupelo/app'
2
-
3
- ### need a programmatic way to start up clients
4
-
5
- Tupelo.application do |app|
6
-
7
- app.child do ## local still hangs
8
- 3.times do |i|
9
- app.child do
10
- write [i]
11
- log "wrote #{i}"
12
- end
13
- end
14
-
15
- 3.times do
16
- log take [nil]
17
- end
18
- end
19
- end
20
-
21
- __END__
22
-
23
- this hangs sometimes but not always:
24
-
25
- tick cid status operation
26
- A: client 3: wrote 0
27
- A: client 4: wrote 1
28
- 1 3 batch write [0]
29
- 2 4 batch write [1]
30
- A: client 2: [0]
31
- 3 2 atomic take [0]
32
- 4 2 atomic take [1]
33
- A: client 2: [1]
34
- A: client 5: wrote 2
@@ -1,21 +0,0 @@
1
- # http://slashdot.org/topic/bi/how-is-reactive-different-from-procedural-programming
2
- # http://developers.slashdot.org/story/14/01/13/2119202/how-reactive-programming-differs-from-procedural-programming
3
- # bud and bloom
4
-
5
- # TODO: DSL for expressing data dependency relations
6
-
7
- require 'tupelo/app'
8
-
9
- N_WORKERS = 2
10
-
11
- Tupelo.application do
12
- N_WORKERS.times do
13
- child passive: true do
14
-
15
- end
16
- end
17
-
18
- local do
19
- #write
20
- end
21
- end
@@ -1,15 +0,0 @@
1
- #
2
- # Minor optimization:
3
-
4
- class KeyMatcher
5
- def initialize i, n
6
- @i = i
7
- @n = n
8
- end
9
-
10
- def === id
11
- id % @n == @i
12
- end
13
- end
14
-
15
- vertex = take id: v_id_matcher, step: step, rank: nil, active: true
@@ -1,56 +0,0 @@
1
- require 'sequel'
2
-
3
- @db = Sequel.sqlite
4
- @db.create_table :events do
5
- primary_key :id # id is not significant to our app
6
- text :host, null: false ## need this ?
7
- text :service, null: false
8
- text :state
9
- number :time
10
- text :description
11
- number :metric
12
- number :ttl
13
-
14
- index [:host, :service]
15
- end
16
-
17
- @db.create_table :tags do
18
- foreign_key :event_id, :events
19
- text :tag
20
- index :tag
21
- primary_key [:event_id, :tag]
22
- end
23
-
24
- @db.create_table :customs do
25
- foreign_key :event_id, :events
26
- text :key
27
- text :value
28
- index :key
29
- primary_key [:event_id, :key]
30
- end
31
-
32
- require 'pp'
33
- #@db.tables.each do |table|
34
- # pp @db.schema(table)
35
- #end
36
-
37
- events = @db[:events]
38
- tags = @db[:tags]
39
- customs = @db[:customs]
40
-
41
- events << {host: "foo.bar", service: "httpd"}
42
- events << {host: "foo.bar", service: "sshd"}
43
- tags << {event_id: 1, tag: "red"}
44
- customs << {event_id: 1, key: "a", value: "1"}
45
- customs << {event_id: 1, key: "b", value: "2"}
46
- customs << {event_id: 2, key: "b", value: "3"}
47
-
48
- pp events.all
49
- pp tags.all
50
- pp customs.all
51
-
52
- full_events = events.join(:tags, :event_id => :id).join(:customs, :event_id => :events__id)
53
-
54
- p full_events
55
- pp full_events.all
56
-
@@ -1,35 +0,0 @@
1
- require 'sequel'
2
-
3
- db = Sequel.sqlite
4
- db.create_table "poi" do
5
- primary_key :id
6
- float :lat, null: false
7
- float :lng, null: false
8
- text :desc
9
- end
10
-
11
- #p db
12
- #p db.schema :poi
13
-
14
- poi = db[:poi]
15
-
16
- 10.times do |i|
17
- poi << {
18
- lat: rand,
19
- lng: rand,
20
- desc: "point #{i}"
21
- }
22
- end
23
-
24
- #p poi.all
25
- #exit
26
-
27
- poi << {lat: 1, lng: 2, desc: "a"}
28
- poi << {lat: 1, lng: 2, desc: "b"}
29
-
30
- #p poi.count
31
- p poi.select(:lat, :lng, :desc).
32
- where(lat: 0..1, lng: 0..1)
33
-
34
- p poi.select(:lat, :lng, :desc).
35
- where(lat: 0..1, lng: 0..1).limit(5).all
@@ -1,104 +0,0 @@
1
- ## TODO
2
- ##
3
- ## scaling params
4
-
5
- require 'tupelo/app'
6
-
7
- ab_tag = "my address book"
8
- ab_sort_field = 1
9
- ab_val_field = 2
10
- cmd_tag = "#{ab_tag} commands"
11
- resp_tag = "#{ab_tag} responses"
12
-
13
- Tupelo.application do
14
- local do
15
- # Subspace for tuples belonging to the addr book.
16
- define_subspace(
17
- tag: ab_tag,
18
- template: [
19
- {value: ab_tag},
20
- {type: "string"}, # name <-- ab_sort_field references this field
21
- nil # address; can be any object <-- ab_val_field
22
- ]
23
- )
24
-
25
- # Subspace for commands for fetch and delete.
26
- # We can't use #read and #take because then the requesting client
27
- # would have to subscribe to the ab_tag subspace.
28
- define_subspace(
29
- tag: cmd_tag,
30
- template: [
31
- {value: cmd_tag},
32
- {type: "string"}, # cmd name
33
- {type: "list"} # arguments
34
- ]
35
- )
36
-
37
- # Subspace for responses to commands. Identify the command this is in
38
- # response to by copying it (alternately, could use ids).
39
- define_subspace(
40
- tag: resp_tag,
41
- template: [
42
- {value: resp_tag},
43
- {type: "string"}, # cmd name
44
- {type: "list"}, # arguments
45
- nil # result of query -- type depends on command
46
- ]
47
- )
48
- end
49
-
50
- ## Could set N_SORTED_SET_SPACE > 1, but lookups are so fast it would
51
- ## just lead to contention and redundant computation. Redundancy is useful
52
- ## though.
53
-
54
- # Inserts are just writes, which are handled by Worker and SortedSetSpace,
55
- # so this child's app loop only needs to handle special commands: fetch and
56
- # delete, which are delegated to the SortedSetSpace.
57
- child tuplespace: [SortedSetSpace, ab_tag, ab_sort_field, ab_val_field],
58
- subscribe: [ab_tag, cmd_tag], passive: true do
59
- loop do
60
- transaction do
61
- _, cmd, args = take(subspace cmd_tag)
62
-
63
- case cmd
64
- when "delete"
65
- args.each do |name|
66
- take [ab_tag, name, nil]
67
- end
68
-
69
- when "fetch"
70
- name = args[0]
71
- _, _, addr = read [ab_tag, name, nil]
72
- write [resp_tag, name, args, addr]
73
-
74
- when "next", "prev"
75
- name = args[0]
76
- _, name2, addr = read SortedSetTemplate[ab_tag, cmd, name]
77
- write [resp_tag, name, args, name2, addr]
78
-
79
- when "first", "last"
80
- _, name, addr = read SortedSetTemplate[ab_tag, cmd]
81
- write [resp_tag, name, args, name, addr]
82
-
83
- else # maybe write an error message in a tuple
84
- log.error "bad command: #{cmd}"
85
- end
86
- end
87
- end
88
- end
89
-
90
- child subscribe: resp_tag do
91
- # write some ab entries
92
- write [ab_tag, "McFirst, Firsty", "123 W. Crescent Terrace"]
93
- write [ab_tag, "Secondismus, Deuce", "456 S. West Way"]
94
-
95
- # make some queries
96
- write [cmd_tag, "first", []]
97
- *, name, addr = take [resp_tag, "first", [], nil, nil]
98
- log "first entry: #{name} => #{addr}"
99
-
100
- write [cmd_tag, "next", [name]]
101
- *, name, addr = take [resp_tag, "next", [name], nil, nil]
102
- log "next entry: #{name} => #{addr}"
103
- end
104
- end
@@ -1,16 +0,0 @@
1
- require 'tupelo/app'
2
-
3
- Tupelo.application do
4
- local do
5
- define_subspace(
6
- tag: "address books",
7
- template: {
8
- book: {type: "string"},
9
- name: {type: "string"},
10
- address: nil
11
- }
12
- )
13
-
14
- define_subspace
15
- end
16
- end
@@ -1,130 +0,0 @@
1
- require 'rbtree'
2
-
3
- class SortedSetTemplate
4
- class << self
5
- alias [] new
6
- end
7
-
8
- # cmd can be "next", "prev", "first", "last"
9
- # for next/prev, args is ["name"]
10
- # for first/last, args is empty
11
- def initialize tag, cmd, *args
12
- @tag = tag
13
- @cmd = cmd
14
- @args = args
15
- end
16
-
17
- def === other
18
- raise ### should not need this?
19
- end
20
-
21
- def find_in rbtree
22
- case @cmd
23
- when "first"
24
- rbtree.first
25
- ###
26
- end
27
- end
28
-
29
- # A tuple store (in-memory) that is optimized for (key_string, object) pairs.
30
- # The object may be any serializable object (built up from numbers, booleans,
31
- # nil, strings, hashes and arrays).
32
- #
33
- # Unlike in a key-value store, a given key_string may occur more than once.
34
- # It is up to the application to decide whether to enforce key uniqueness or
35
- # not (for example, by taking (k,...) before writing (k,v).
36
- #
37
- # This store should be used only by clients that subscribe to a subspace
38
- # that can be represented as pairs. (See memo2.rb.)
39
- #
40
- # This store also manages meta tuples, which it keeps in an array, just like
41
- # the default Tuplespace class does.
42
- class SortedSetSpace
43
- include Enumerable
44
-
45
- attr_reader :tag, :hash, :metas
46
-
47
- def initialize tag
48
- @tag = tag
49
- clear
50
- end
51
-
52
- def clear
53
- @hash = Hash.new {|h,k| h[k] = []}
54
- # It's up to the application to enforce that these arrays have size <=1.
55
- @metas = []
56
- # We are automatically subscribed to tupelo metadata (subspace defs), so
57
- # we need to keep them somewhere.
58
- end
59
-
60
- def each
61
- hash.each do |k, vs|
62
- vs.each do |v|
63
- yield tag, k, v
64
- end
65
- end
66
- metas.each do |tuple|
67
- yield tuple
68
- end
69
- end
70
-
71
- def insert tuple
72
- if tuple.kind_of? Array
73
- # and tuple.size == 3 and tuple[0] == tag and tuple[1].kind_of? String
74
- # This is redundant, because of subscribe.
75
- t, k, v = tuple
76
- hash[k] << v
77
-
78
- else
79
- metas << tuple
80
- end
81
- end
82
-
83
- def delete_once tuple
84
- if tuple.kind_of? Array
85
- # and tuple.size == 3 and tuple[0] == tag and tuple[1].kind_of? String
86
- # This is redundant, because of subscribe.
87
- t, k, v = tuple
88
- if hash.key?(k) and hash[k].include? v
89
- hash[k].delete v
90
- hash.delete k if hash[k].empty?
91
- true
92
- else
93
- false
94
- end
95
-
96
- else
97
- if i=metas.index(tuple)
98
- delete_at i
99
- end
100
- end
101
- end
102
-
103
- def transaction inserts: [], deletes: [], tick: nil
104
- deletes.each do |tuple|
105
- delete_once tuple or raise "bug"
106
- end
107
-
108
- inserts.each do |tuple|
109
- insert tuple.freeze ## should be deep_freeze
110
- end
111
- end
112
-
113
- def find_distinct_matches_for templates
114
- templates.inject([]) do |tuples, template|
115
- tuples << find_match_for(template, distinct_from: tuples)
116
- end
117
- end
118
-
119
- def find_match_for template, distinct_from: []
120
- case template
121
- when SortedSetTemplate
122
- template.find_in rbtree, distinct_from: distinct_from ###
123
- else
124
- # fall back to linear search
125
- find do |tuple|
126
- template === tuple and not distinct_from.any? {|t| t.equal? tuple}
127
- end
128
- end
129
- end
130
- end