tupelo 0.21 → 0.22

Sign up to get free protection for your applications and to get access to all the features.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +171 -45
  3. data/bin/tup +51 -0
  4. data/example/counters/merge.rb +23 -3
  5. data/example/multi-tier/multi-sinatras.rb +5 -0
  6. data/example/riemann/event-subspace.rb +1 -4
  7. data/example/riemann/expiration-dbg.rb +2 -0
  8. data/example/riemann/producer.rb +4 -3
  9. data/example/riemann/v1/riemann.rb +2 -2
  10. data/example/riemann/v2/event-template.rb +71 -0
  11. data/example/riemann/v2/expirer.rb +1 -1
  12. data/example/riemann/v2/hash-store.rb +1 -0
  13. data/example/riemann/v2/ordered-event-store.rb +4 -1
  14. data/example/riemann/v2/riemann.rb +15 -8
  15. data/example/riemann/v2/sqlite-event-store.rb +117 -72
  16. data/example/sqlite/poi-store.rb +1 -1
  17. data/example/sqlite/poi-template.rb +2 -2
  18. data/example/sqlite/poi-v2.rb +2 -2
  19. data/example/subspaces/ramp.rb +9 -2
  20. data/example/tcp.rb +5 -0
  21. data/example/tiny-tcp-client.rb +15 -0
  22. data/example/tiny-tcp-service.rb +32 -0
  23. data/lib/tupelo/app.rb +4 -4
  24. data/lib/tupelo/app/builder.rb +2 -2
  25. data/lib/tupelo/app/irb-shell.rb +3 -3
  26. data/lib/tupelo/archiver.rb +0 -2
  27. data/lib/tupelo/archiver/tuplestore.rb +1 -1
  28. data/lib/tupelo/archiver/worker.rb +6 -6
  29. data/lib/tupelo/client.rb +2 -2
  30. data/lib/tupelo/client/reader.rb +3 -3
  31. data/lib/tupelo/client/scheduler.rb +1 -1
  32. data/lib/tupelo/client/subspace.rb +2 -2
  33. data/lib/tupelo/client/transaction.rb +28 -28
  34. data/lib/tupelo/client/tuplestore.rb +2 -2
  35. data/lib/tupelo/client/worker.rb +11 -10
  36. data/lib/tupelo/util/bin-circle.rb +8 -8
  37. data/lib/tupelo/util/boolean.rb +1 -1
  38. data/lib/tupelo/version.rb +1 -1
  39. data/test/lib/mock-client.rb +10 -10
  40. data/test/system/test-archiver.rb +2 -2
  41. data/test/unit/test-ops.rb +21 -21
  42. metadata +10 -20
  43. data/example/bingo/bingo-v2.rb +0 -20
  44. data/example/broker-queue.rb +0 -35
  45. data/example/child-of-child.rb +0 -34
  46. data/example/dataflow.rb +0 -21
  47. data/example/pregel/dist-opt.rb +0 -15
  48. data/example/riemann/v2/event-sql.rb +0 -56
  49. data/example/sqlite/tmp/poi-sqlite.rb +0 -35
  50. data/example/subspaces/addr-book-v1.rb +0 -104
  51. data/example/subspaces/addr-book-v2.rb +0 -16
  52. data/example/subspaces/sorted-set-space-OLD.rb +0 -130
  53. data/lib/tupelo/tuplets/persistent-archiver.rb +0 -86
  54. data/lib/tupelo/tuplets/persistent-archiver/tuplespace.rb +0 -91
  55. data/lib/tupelo/tuplets/persistent-archiver/worker.rb +0 -114
@@ -1,20 +0,0 @@
1
-
2
- __END__
3
- # dealer
4
- child passive: true do
5
- loop do
6
- # transaction -- add to v2
7
- _, player_id = take ["buy", nil]
8
-
9
- end
10
- end
11
-
12
- # player -- in v3 use subspace per player
13
- end
14
-
15
- ## how to buy 4 without increasing contention risk
16
-
17
- ## reduce contention for cards by randomizing or cons. hashing?
18
-
19
- ## swapping these lines -> more contention?
20
-
@@ -1,35 +0,0 @@
1
- # more like how you would do it in redis, except that the queue is not stored in
2
- # the central server, so operations on it are not a bottleneck, FWIW
3
-
4
- require 'tupelo/app'
5
-
6
- N_PLAYERS = 10
7
-
8
- Tupelo.application do
9
- N_PLAYERS.times do
10
- # sleep rand / 10 # reduce contention -- could also randomize inserts
11
- child do
12
- me = client_id
13
- write name: me
14
-
15
- you = transaction do
16
- game = read_nowait(
17
- player1: nil,
18
- player2: me)
19
- break game["player1"] if game
20
-
21
- unless take_nowait name: me
22
- raise Tupelo::Client::TransactionFailure
23
- end
24
-
25
- you = take(name: nil)["name"]
26
- write(
27
- player1: me,
28
- player2: you)
29
- you
30
- end
31
-
32
- log "now playing with #{you}"
33
- end
34
- end
35
- end
@@ -1,34 +0,0 @@
1
- require 'tupelo/app'
2
-
3
- ### need a programmatic way to start up clients
4
-
5
- Tupelo.application do |app|
6
-
7
- app.child do ## local still hangs
8
- 3.times do |i|
9
- app.child do
10
- write [i]
11
- log "wrote #{i}"
12
- end
13
- end
14
-
15
- 3.times do
16
- log take [nil]
17
- end
18
- end
19
- end
20
-
21
- __END__
22
-
23
- this hangs sometimes but not always:
24
-
25
- tick cid status operation
26
- A: client 3: wrote 0
27
- A: client 4: wrote 1
28
- 1 3 batch write [0]
29
- 2 4 batch write [1]
30
- A: client 2: [0]
31
- 3 2 atomic take [0]
32
- 4 2 atomic take [1]
33
- A: client 2: [1]
34
- A: client 5: wrote 2
@@ -1,21 +0,0 @@
1
- # http://slashdot.org/topic/bi/how-is-reactive-different-from-procedural-programming
2
- # http://developers.slashdot.org/story/14/01/13/2119202/how-reactive-programming-differs-from-procedural-programming
3
- # bud and bloom
4
-
5
- # TODO: DSL for expressing data dependency relations
6
-
7
- require 'tupelo/app'
8
-
9
- N_WORKERS = 2
10
-
11
- Tupelo.application do
12
- N_WORKERS.times do
13
- child passive: true do
14
-
15
- end
16
- end
17
-
18
- local do
19
- #write
20
- end
21
- end
@@ -1,15 +0,0 @@
1
- #
2
- # Minor optimization:
3
-
4
- class KeyMatcher
5
- def initialize i, n
6
- @i = i
7
- @n = n
8
- end
9
-
10
- def === id
11
- id % @n == @i
12
- end
13
- end
14
-
15
- vertex = take id: v_id_matcher, step: step, rank: nil, active: true
@@ -1,56 +0,0 @@
1
- require 'sequel'
2
-
3
- @db = Sequel.sqlite
4
- @db.create_table :events do
5
- primary_key :id # id is not significant to our app
6
- text :host, null: false ## need this ?
7
- text :service, null: false
8
- text :state
9
- number :time
10
- text :description
11
- number :metric
12
- number :ttl
13
-
14
- index [:host, :service]
15
- end
16
-
17
- @db.create_table :tags do
18
- foreign_key :event_id, :events
19
- text :tag
20
- index :tag
21
- primary_key [:event_id, :tag]
22
- end
23
-
24
- @db.create_table :customs do
25
- foreign_key :event_id, :events
26
- text :key
27
- text :value
28
- index :key
29
- primary_key [:event_id, :key]
30
- end
31
-
32
- require 'pp'
33
- #@db.tables.each do |table|
34
- # pp @db.schema(table)
35
- #end
36
-
37
- events = @db[:events]
38
- tags = @db[:tags]
39
- customs = @db[:customs]
40
-
41
- events << {host: "foo.bar", service: "httpd"}
42
- events << {host: "foo.bar", service: "sshd"}
43
- tags << {event_id: 1, tag: "red"}
44
- customs << {event_id: 1, key: "a", value: "1"}
45
- customs << {event_id: 1, key: "b", value: "2"}
46
- customs << {event_id: 2, key: "b", value: "3"}
47
-
48
- pp events.all
49
- pp tags.all
50
- pp customs.all
51
-
52
- full_events = events.join(:tags, :event_id => :id).join(:customs, :event_id => :events__id)
53
-
54
- p full_events
55
- pp full_events.all
56
-
@@ -1,35 +0,0 @@
1
- require 'sequel'
2
-
3
- db = Sequel.sqlite
4
- db.create_table "poi" do
5
- primary_key :id
6
- float :lat, null: false
7
- float :lng, null: false
8
- text :desc
9
- end
10
-
11
- #p db
12
- #p db.schema :poi
13
-
14
- poi = db[:poi]
15
-
16
- 10.times do |i|
17
- poi << {
18
- lat: rand,
19
- lng: rand,
20
- desc: "point #{i}"
21
- }
22
- end
23
-
24
- #p poi.all
25
- #exit
26
-
27
- poi << {lat: 1, lng: 2, desc: "a"}
28
- poi << {lat: 1, lng: 2, desc: "b"}
29
-
30
- #p poi.count
31
- p poi.select(:lat, :lng, :desc).
32
- where(lat: 0..1, lng: 0..1)
33
-
34
- p poi.select(:lat, :lng, :desc).
35
- where(lat: 0..1, lng: 0..1).limit(5).all
@@ -1,104 +0,0 @@
1
- ## TODO
2
- ##
3
- ## scaling params
4
-
5
- require 'tupelo/app'
6
-
7
- ab_tag = "my address book"
8
- ab_sort_field = 1
9
- ab_val_field = 2
10
- cmd_tag = "#{ab_tag} commands"
11
- resp_tag = "#{ab_tag} responses"
12
-
13
- Tupelo.application do
14
- local do
15
- # Subspace for tuples belonging to the addr book.
16
- define_subspace(
17
- tag: ab_tag,
18
- template: [
19
- {value: ab_tag},
20
- {type: "string"}, # name <-- ab_sort_field references this field
21
- nil # address; can be any object <-- ab_val_field
22
- ]
23
- )
24
-
25
- # Subspace for commands for fetch and delete.
26
- # We can't use #read and #take because then the requesting client
27
- # would have to subscribe to the ab_tag subspace.
28
- define_subspace(
29
- tag: cmd_tag,
30
- template: [
31
- {value: cmd_tag},
32
- {type: "string"}, # cmd name
33
- {type: "list"} # arguments
34
- ]
35
- )
36
-
37
- # Subspace for responses to commands. Identify the command this is in
38
- # response to by copying it (alternately, could use ids).
39
- define_subspace(
40
- tag: resp_tag,
41
- template: [
42
- {value: resp_tag},
43
- {type: "string"}, # cmd name
44
- {type: "list"}, # arguments
45
- nil # result of query -- type depends on command
46
- ]
47
- )
48
- end
49
-
50
- ## Could set N_SORTED_SET_SPACE > 1, but lookups are so fast it would
51
- ## just lead to contention and redundant computation. Redundancy is useful
52
- ## though.
53
-
54
- # Inserts are just writes, which are handled by Worker and SortedSetSpace,
55
- # so this child's app loop only needs to handle special commands: fetch and
56
- # delete, which are delegated to the SortedSetSpace.
57
- child tuplespace: [SortedSetSpace, ab_tag, ab_sort_field, ab_val_field],
58
- subscribe: [ab_tag, cmd_tag], passive: true do
59
- loop do
60
- transaction do
61
- _, cmd, args = take(subspace cmd_tag)
62
-
63
- case cmd
64
- when "delete"
65
- args.each do |name|
66
- take [ab_tag, name, nil]
67
- end
68
-
69
- when "fetch"
70
- name = args[0]
71
- _, _, addr = read [ab_tag, name, nil]
72
- write [resp_tag, name, args, addr]
73
-
74
- when "next", "prev"
75
- name = args[0]
76
- _, name2, addr = read SortedSetTemplate[ab_tag, cmd, name]
77
- write [resp_tag, name, args, name2, addr]
78
-
79
- when "first", "last"
80
- _, name, addr = read SortedSetTemplate[ab_tag, cmd]
81
- write [resp_tag, name, args, name, addr]
82
-
83
- else # maybe write an error message in a tuple
84
- log.error "bad command: #{cmd}"
85
- end
86
- end
87
- end
88
- end
89
-
90
- child subscribe: resp_tag do
91
- # write some ab entries
92
- write [ab_tag, "McFirst, Firsty", "123 W. Crescent Terrace"]
93
- write [ab_tag, "Secondismus, Deuce", "456 S. West Way"]
94
-
95
- # make some queries
96
- write [cmd_tag, "first", []]
97
- *, name, addr = take [resp_tag, "first", [], nil, nil]
98
- log "first entry: #{name} => #{addr}"
99
-
100
- write [cmd_tag, "next", [name]]
101
- *, name, addr = take [resp_tag, "next", [name], nil, nil]
102
- log "next entry: #{name} => #{addr}"
103
- end
104
- end
@@ -1,16 +0,0 @@
1
- require 'tupelo/app'
2
-
3
- Tupelo.application do
4
- local do
5
- define_subspace(
6
- tag: "address books",
7
- template: {
8
- book: {type: "string"},
9
- name: {type: "string"},
10
- address: nil
11
- }
12
- )
13
-
14
- define_subspace
15
- end
16
- end
@@ -1,130 +0,0 @@
1
- require 'rbtree'
2
-
3
- class SortedSetTemplate
4
- class << self
5
- alias [] new
6
- end
7
-
8
- # cmd can be "next", "prev", "first", "last"
9
- # for next/prev, args is ["name"]
10
- # for first/last, args is empty
11
- def initialize tag, cmd, *args
12
- @tag = tag
13
- @cmd = cmd
14
- @args = args
15
- end
16
-
17
- def === other
18
- raise ### should not need this?
19
- end
20
-
21
- def find_in rbtree
22
- case @cmd
23
- when "first"
24
- rbtree.first
25
- ###
26
- end
27
- end
28
-
29
- # A tuple store (in-memory) that is optimized for (key_string, object) pairs.
30
- # The object may be any serializable object (built up from numbers, booleans,
31
- # nil, strings, hashes and arrays).
32
- #
33
- # Unlike in a key-value store, a given key_string may occur more than once.
34
- # It is up to the application to decide whether to enforce key uniqueness or
35
- # not (for example, by taking (k,...) before writing (k,v).
36
- #
37
- # This store should be used only by clients that subscribe to a subspace
38
- # that can be represented as pairs. (See memo2.rb.)
39
- #
40
- # This store also manages meta tuples, which it keeps in an array, just like
41
- # the default Tuplespace class does.
42
- class SortedSetSpace
43
- include Enumerable
44
-
45
- attr_reader :tag, :hash, :metas
46
-
47
- def initialize tag
48
- @tag = tag
49
- clear
50
- end
51
-
52
- def clear
53
- @hash = Hash.new {|h,k| h[k] = []}
54
- # It's up to the application to enforce that these arrays have size <=1.
55
- @metas = []
56
- # We are automatically subscribed to tupelo metadata (subspace defs), so
57
- # we need to keep them somewhere.
58
- end
59
-
60
- def each
61
- hash.each do |k, vs|
62
- vs.each do |v|
63
- yield tag, k, v
64
- end
65
- end
66
- metas.each do |tuple|
67
- yield tuple
68
- end
69
- end
70
-
71
- def insert tuple
72
- if tuple.kind_of? Array
73
- # and tuple.size == 3 and tuple[0] == tag and tuple[1].kind_of? String
74
- # This is redundant, because of subscribe.
75
- t, k, v = tuple
76
- hash[k] << v
77
-
78
- else
79
- metas << tuple
80
- end
81
- end
82
-
83
- def delete_once tuple
84
- if tuple.kind_of? Array
85
- # and tuple.size == 3 and tuple[0] == tag and tuple[1].kind_of? String
86
- # This is redundant, because of subscribe.
87
- t, k, v = tuple
88
- if hash.key?(k) and hash[k].include? v
89
- hash[k].delete v
90
- hash.delete k if hash[k].empty?
91
- true
92
- else
93
- false
94
- end
95
-
96
- else
97
- if i=metas.index(tuple)
98
- delete_at i
99
- end
100
- end
101
- end
102
-
103
- def transaction inserts: [], deletes: [], tick: nil
104
- deletes.each do |tuple|
105
- delete_once tuple or raise "bug"
106
- end
107
-
108
- inserts.each do |tuple|
109
- insert tuple.freeze ## should be deep_freeze
110
- end
111
- end
112
-
113
- def find_distinct_matches_for templates
114
- templates.inject([]) do |tuples, template|
115
- tuples << find_match_for(template, distinct_from: tuples)
116
- end
117
- end
118
-
119
- def find_match_for template, distinct_from: []
120
- case template
121
- when SortedSetTemplate
122
- template.find_in rbtree, distinct_from: distinct_from ###
123
- else
124
- # fall back to linear search
125
- find do |tuple|
126
- template === tuple and not distinct_from.any? {|t| t.equal? tuple}
127
- end
128
- end
129
- end
130
- end