tupelo 0.21 → 0.22
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +171 -45
- data/bin/tup +51 -0
- data/example/counters/merge.rb +23 -3
- data/example/multi-tier/multi-sinatras.rb +5 -0
- data/example/riemann/event-subspace.rb +1 -4
- data/example/riemann/expiration-dbg.rb +2 -0
- data/example/riemann/producer.rb +4 -3
- data/example/riemann/v1/riemann.rb +2 -2
- data/example/riemann/v2/event-template.rb +71 -0
- data/example/riemann/v2/expirer.rb +1 -1
- data/example/riemann/v2/hash-store.rb +1 -0
- data/example/riemann/v2/ordered-event-store.rb +4 -1
- data/example/riemann/v2/riemann.rb +15 -8
- data/example/riemann/v2/sqlite-event-store.rb +117 -72
- data/example/sqlite/poi-store.rb +1 -1
- data/example/sqlite/poi-template.rb +2 -2
- data/example/sqlite/poi-v2.rb +2 -2
- data/example/subspaces/ramp.rb +9 -2
- data/example/tcp.rb +5 -0
- data/example/tiny-tcp-client.rb +15 -0
- data/example/tiny-tcp-service.rb +32 -0
- data/lib/tupelo/app.rb +4 -4
- data/lib/tupelo/app/builder.rb +2 -2
- data/lib/tupelo/app/irb-shell.rb +3 -3
- data/lib/tupelo/archiver.rb +0 -2
- data/lib/tupelo/archiver/tuplestore.rb +1 -1
- data/lib/tupelo/archiver/worker.rb +6 -6
- data/lib/tupelo/client.rb +2 -2
- data/lib/tupelo/client/reader.rb +3 -3
- data/lib/tupelo/client/scheduler.rb +1 -1
- data/lib/tupelo/client/subspace.rb +2 -2
- data/lib/tupelo/client/transaction.rb +28 -28
- data/lib/tupelo/client/tuplestore.rb +2 -2
- data/lib/tupelo/client/worker.rb +11 -10
- data/lib/tupelo/util/bin-circle.rb +8 -8
- data/lib/tupelo/util/boolean.rb +1 -1
- data/lib/tupelo/version.rb +1 -1
- data/test/lib/mock-client.rb +10 -10
- data/test/system/test-archiver.rb +2 -2
- data/test/unit/test-ops.rb +21 -21
- metadata +10 -20
- data/example/bingo/bingo-v2.rb +0 -20
- data/example/broker-queue.rb +0 -35
- data/example/child-of-child.rb +0 -34
- data/example/dataflow.rb +0 -21
- data/example/pregel/dist-opt.rb +0 -15
- data/example/riemann/v2/event-sql.rb +0 -56
- data/example/sqlite/tmp/poi-sqlite.rb +0 -35
- data/example/subspaces/addr-book-v1.rb +0 -104
- data/example/subspaces/addr-book-v2.rb +0 -16
- data/example/subspaces/sorted-set-space-OLD.rb +0 -130
- data/lib/tupelo/tuplets/persistent-archiver.rb +0 -86
- data/lib/tupelo/tuplets/persistent-archiver/tuplespace.rb +0 -91
- data/lib/tupelo/tuplets/persistent-archiver/worker.rb +0 -114
data/example/bingo/bingo-v2.rb
DELETED
@@ -1,20 +0,0 @@
|
|
1
|
-
|
2
|
-
__END__
|
3
|
-
# dealer
|
4
|
-
child passive: true do
|
5
|
-
loop do
|
6
|
-
# transaction -- add to v2
|
7
|
-
_, player_id = take ["buy", nil]
|
8
|
-
|
9
|
-
end
|
10
|
-
end
|
11
|
-
|
12
|
-
# player -- in v3 use subspace per player
|
13
|
-
end
|
14
|
-
|
15
|
-
## how to buy 4 without increasing contention risk
|
16
|
-
|
17
|
-
## reduce contention for cards by randomizing or cons. hashing?
|
18
|
-
|
19
|
-
## swapping these lines -> more contention?
|
20
|
-
|
data/example/broker-queue.rb
DELETED
@@ -1,35 +0,0 @@
|
|
1
|
-
# more like how you would do it in redis, except that the queue is not stored in
|
2
|
-
# the central server, so operations on it are not a bottleneck, FWIW
|
3
|
-
|
4
|
-
require 'tupelo/app'
|
5
|
-
|
6
|
-
N_PLAYERS = 10
|
7
|
-
|
8
|
-
Tupelo.application do
|
9
|
-
N_PLAYERS.times do
|
10
|
-
# sleep rand / 10 # reduce contention -- could also randomize inserts
|
11
|
-
child do
|
12
|
-
me = client_id
|
13
|
-
write name: me
|
14
|
-
|
15
|
-
you = transaction do
|
16
|
-
game = read_nowait(
|
17
|
-
player1: nil,
|
18
|
-
player2: me)
|
19
|
-
break game["player1"] if game
|
20
|
-
|
21
|
-
unless take_nowait name: me
|
22
|
-
raise Tupelo::Client::TransactionFailure
|
23
|
-
end
|
24
|
-
|
25
|
-
you = take(name: nil)["name"]
|
26
|
-
write(
|
27
|
-
player1: me,
|
28
|
-
player2: you)
|
29
|
-
you
|
30
|
-
end
|
31
|
-
|
32
|
-
log "now playing with #{you}"
|
33
|
-
end
|
34
|
-
end
|
35
|
-
end
|
data/example/child-of-child.rb
DELETED
@@ -1,34 +0,0 @@
|
|
1
|
-
require 'tupelo/app'
|
2
|
-
|
3
|
-
### need a programmatic way to start up clients
|
4
|
-
|
5
|
-
Tupelo.application do |app|
|
6
|
-
|
7
|
-
app.child do ## local still hangs
|
8
|
-
3.times do |i|
|
9
|
-
app.child do
|
10
|
-
write [i]
|
11
|
-
log "wrote #{i}"
|
12
|
-
end
|
13
|
-
end
|
14
|
-
|
15
|
-
3.times do
|
16
|
-
log take [nil]
|
17
|
-
end
|
18
|
-
end
|
19
|
-
end
|
20
|
-
|
21
|
-
__END__
|
22
|
-
|
23
|
-
this hangs sometimes but not always:
|
24
|
-
|
25
|
-
tick cid status operation
|
26
|
-
A: client 3: wrote 0
|
27
|
-
A: client 4: wrote 1
|
28
|
-
1 3 batch write [0]
|
29
|
-
2 4 batch write [1]
|
30
|
-
A: client 2: [0]
|
31
|
-
3 2 atomic take [0]
|
32
|
-
4 2 atomic take [1]
|
33
|
-
A: client 2: [1]
|
34
|
-
A: client 5: wrote 2
|
data/example/dataflow.rb
DELETED
@@ -1,21 +0,0 @@
|
|
1
|
-
# http://slashdot.org/topic/bi/how-is-reactive-different-from-procedural-programming
|
2
|
-
# http://developers.slashdot.org/story/14/01/13/2119202/how-reactive-programming-differs-from-procedural-programming
|
3
|
-
# bud and bloom
|
4
|
-
|
5
|
-
# TODO: DSL for expressing data dependency relations
|
6
|
-
|
7
|
-
require 'tupelo/app'
|
8
|
-
|
9
|
-
N_WORKERS = 2
|
10
|
-
|
11
|
-
Tupelo.application do
|
12
|
-
N_WORKERS.times do
|
13
|
-
child passive: true do
|
14
|
-
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
local do
|
19
|
-
#write
|
20
|
-
end
|
21
|
-
end
|
data/example/pregel/dist-opt.rb
DELETED
@@ -1,56 +0,0 @@
|
|
1
|
-
require 'sequel'
|
2
|
-
|
3
|
-
@db = Sequel.sqlite
|
4
|
-
@db.create_table :events do
|
5
|
-
primary_key :id # id is not significant to our app
|
6
|
-
text :host, null: false ## need this ?
|
7
|
-
text :service, null: false
|
8
|
-
text :state
|
9
|
-
number :time
|
10
|
-
text :description
|
11
|
-
number :metric
|
12
|
-
number :ttl
|
13
|
-
|
14
|
-
index [:host, :service]
|
15
|
-
end
|
16
|
-
|
17
|
-
@db.create_table :tags do
|
18
|
-
foreign_key :event_id, :events
|
19
|
-
text :tag
|
20
|
-
index :tag
|
21
|
-
primary_key [:event_id, :tag]
|
22
|
-
end
|
23
|
-
|
24
|
-
@db.create_table :customs do
|
25
|
-
foreign_key :event_id, :events
|
26
|
-
text :key
|
27
|
-
text :value
|
28
|
-
index :key
|
29
|
-
primary_key [:event_id, :key]
|
30
|
-
end
|
31
|
-
|
32
|
-
require 'pp'
|
33
|
-
#@db.tables.each do |table|
|
34
|
-
# pp @db.schema(table)
|
35
|
-
#end
|
36
|
-
|
37
|
-
events = @db[:events]
|
38
|
-
tags = @db[:tags]
|
39
|
-
customs = @db[:customs]
|
40
|
-
|
41
|
-
events << {host: "foo.bar", service: "httpd"}
|
42
|
-
events << {host: "foo.bar", service: "sshd"}
|
43
|
-
tags << {event_id: 1, tag: "red"}
|
44
|
-
customs << {event_id: 1, key: "a", value: "1"}
|
45
|
-
customs << {event_id: 1, key: "b", value: "2"}
|
46
|
-
customs << {event_id: 2, key: "b", value: "3"}
|
47
|
-
|
48
|
-
pp events.all
|
49
|
-
pp tags.all
|
50
|
-
pp customs.all
|
51
|
-
|
52
|
-
full_events = events.join(:tags, :event_id => :id).join(:customs, :event_id => :events__id)
|
53
|
-
|
54
|
-
p full_events
|
55
|
-
pp full_events.all
|
56
|
-
|
@@ -1,35 +0,0 @@
|
|
1
|
-
require 'sequel'
|
2
|
-
|
3
|
-
db = Sequel.sqlite
|
4
|
-
db.create_table "poi" do
|
5
|
-
primary_key :id
|
6
|
-
float :lat, null: false
|
7
|
-
float :lng, null: false
|
8
|
-
text :desc
|
9
|
-
end
|
10
|
-
|
11
|
-
#p db
|
12
|
-
#p db.schema :poi
|
13
|
-
|
14
|
-
poi = db[:poi]
|
15
|
-
|
16
|
-
10.times do |i|
|
17
|
-
poi << {
|
18
|
-
lat: rand,
|
19
|
-
lng: rand,
|
20
|
-
desc: "point #{i}"
|
21
|
-
}
|
22
|
-
end
|
23
|
-
|
24
|
-
#p poi.all
|
25
|
-
#exit
|
26
|
-
|
27
|
-
poi << {lat: 1, lng: 2, desc: "a"}
|
28
|
-
poi << {lat: 1, lng: 2, desc: "b"}
|
29
|
-
|
30
|
-
#p poi.count
|
31
|
-
p poi.select(:lat, :lng, :desc).
|
32
|
-
where(lat: 0..1, lng: 0..1)
|
33
|
-
|
34
|
-
p poi.select(:lat, :lng, :desc).
|
35
|
-
where(lat: 0..1, lng: 0..1).limit(5).all
|
@@ -1,104 +0,0 @@
|
|
1
|
-
## TODO
|
2
|
-
##
|
3
|
-
## scaling params
|
4
|
-
|
5
|
-
require 'tupelo/app'
|
6
|
-
|
7
|
-
ab_tag = "my address book"
|
8
|
-
ab_sort_field = 1
|
9
|
-
ab_val_field = 2
|
10
|
-
cmd_tag = "#{ab_tag} commands"
|
11
|
-
resp_tag = "#{ab_tag} responses"
|
12
|
-
|
13
|
-
Tupelo.application do
|
14
|
-
local do
|
15
|
-
# Subspace for tuples belonging to the addr book.
|
16
|
-
define_subspace(
|
17
|
-
tag: ab_tag,
|
18
|
-
template: [
|
19
|
-
{value: ab_tag},
|
20
|
-
{type: "string"}, # name <-- ab_sort_field references this field
|
21
|
-
nil # address; can be any object <-- ab_val_field
|
22
|
-
]
|
23
|
-
)
|
24
|
-
|
25
|
-
# Subspace for commands for fetch and delete.
|
26
|
-
# We can't use #read and #take because then the requesting client
|
27
|
-
# would have to subscribe to the ab_tag subspace.
|
28
|
-
define_subspace(
|
29
|
-
tag: cmd_tag,
|
30
|
-
template: [
|
31
|
-
{value: cmd_tag},
|
32
|
-
{type: "string"}, # cmd name
|
33
|
-
{type: "list"} # arguments
|
34
|
-
]
|
35
|
-
)
|
36
|
-
|
37
|
-
# Subspace for responses to commands. Identify the command this is in
|
38
|
-
# response to by copying it (alternately, could use ids).
|
39
|
-
define_subspace(
|
40
|
-
tag: resp_tag,
|
41
|
-
template: [
|
42
|
-
{value: resp_tag},
|
43
|
-
{type: "string"}, # cmd name
|
44
|
-
{type: "list"}, # arguments
|
45
|
-
nil # result of query -- type depends on command
|
46
|
-
]
|
47
|
-
)
|
48
|
-
end
|
49
|
-
|
50
|
-
## Could set N_SORTED_SET_SPACE > 1, but lookups are so fast it would
|
51
|
-
## just lead to contention and redundant computation. Redundancy is useful
|
52
|
-
## though.
|
53
|
-
|
54
|
-
# Inserts are just writes, which are handled by Worker and SortedSetSpace,
|
55
|
-
# so this child's app loop only needs to handle special commands: fetch and
|
56
|
-
# delete, which are delegated to the SortedSetSpace.
|
57
|
-
child tuplespace: [SortedSetSpace, ab_tag, ab_sort_field, ab_val_field],
|
58
|
-
subscribe: [ab_tag, cmd_tag], passive: true do
|
59
|
-
loop do
|
60
|
-
transaction do
|
61
|
-
_, cmd, args = take(subspace cmd_tag)
|
62
|
-
|
63
|
-
case cmd
|
64
|
-
when "delete"
|
65
|
-
args.each do |name|
|
66
|
-
take [ab_tag, name, nil]
|
67
|
-
end
|
68
|
-
|
69
|
-
when "fetch"
|
70
|
-
name = args[0]
|
71
|
-
_, _, addr = read [ab_tag, name, nil]
|
72
|
-
write [resp_tag, name, args, addr]
|
73
|
-
|
74
|
-
when "next", "prev"
|
75
|
-
name = args[0]
|
76
|
-
_, name2, addr = read SortedSetTemplate[ab_tag, cmd, name]
|
77
|
-
write [resp_tag, name, args, name2, addr]
|
78
|
-
|
79
|
-
when "first", "last"
|
80
|
-
_, name, addr = read SortedSetTemplate[ab_tag, cmd]
|
81
|
-
write [resp_tag, name, args, name, addr]
|
82
|
-
|
83
|
-
else # maybe write an error message in a tuple
|
84
|
-
log.error "bad command: #{cmd}"
|
85
|
-
end
|
86
|
-
end
|
87
|
-
end
|
88
|
-
end
|
89
|
-
|
90
|
-
child subscribe: resp_tag do
|
91
|
-
# write some ab entries
|
92
|
-
write [ab_tag, "McFirst, Firsty", "123 W. Crescent Terrace"]
|
93
|
-
write [ab_tag, "Secondismus, Deuce", "456 S. West Way"]
|
94
|
-
|
95
|
-
# make some queries
|
96
|
-
write [cmd_tag, "first", []]
|
97
|
-
*, name, addr = take [resp_tag, "first", [], nil, nil]
|
98
|
-
log "first entry: #{name} => #{addr}"
|
99
|
-
|
100
|
-
write [cmd_tag, "next", [name]]
|
101
|
-
*, name, addr = take [resp_tag, "next", [name], nil, nil]
|
102
|
-
log "next entry: #{name} => #{addr}"
|
103
|
-
end
|
104
|
-
end
|
@@ -1,130 +0,0 @@
|
|
1
|
-
require 'rbtree'
|
2
|
-
|
3
|
-
class SortedSetTemplate
|
4
|
-
class << self
|
5
|
-
alias [] new
|
6
|
-
end
|
7
|
-
|
8
|
-
# cmd can be "next", "prev", "first", "last"
|
9
|
-
# for next/prev, args is ["name"]
|
10
|
-
# for first/last, args is empty
|
11
|
-
def initialize tag, cmd, *args
|
12
|
-
@tag = tag
|
13
|
-
@cmd = cmd
|
14
|
-
@args = args
|
15
|
-
end
|
16
|
-
|
17
|
-
def === other
|
18
|
-
raise ### should not need this?
|
19
|
-
end
|
20
|
-
|
21
|
-
def find_in rbtree
|
22
|
-
case @cmd
|
23
|
-
when "first"
|
24
|
-
rbtree.first
|
25
|
-
###
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
# A tuple store (in-memory) that is optimized for (key_string, object) pairs.
|
30
|
-
# The object may be any serializable object (built up from numbers, booleans,
|
31
|
-
# nil, strings, hashes and arrays).
|
32
|
-
#
|
33
|
-
# Unlike in a key-value store, a given key_string may occur more than once.
|
34
|
-
# It is up to the application to decide whether to enforce key uniqueness or
|
35
|
-
# not (for example, by taking (k,...) before writing (k,v).
|
36
|
-
#
|
37
|
-
# This store should be used only by clients that subscribe to a subspace
|
38
|
-
# that can be represented as pairs. (See memo2.rb.)
|
39
|
-
#
|
40
|
-
# This store also manages meta tuples, which it keeps in an array, just like
|
41
|
-
# the default Tuplespace class does.
|
42
|
-
class SortedSetSpace
|
43
|
-
include Enumerable
|
44
|
-
|
45
|
-
attr_reader :tag, :hash, :metas
|
46
|
-
|
47
|
-
def initialize tag
|
48
|
-
@tag = tag
|
49
|
-
clear
|
50
|
-
end
|
51
|
-
|
52
|
-
def clear
|
53
|
-
@hash = Hash.new {|h,k| h[k] = []}
|
54
|
-
# It's up to the application to enforce that these arrays have size <=1.
|
55
|
-
@metas = []
|
56
|
-
# We are automatically subscribed to tupelo metadata (subspace defs), so
|
57
|
-
# we need to keep them somewhere.
|
58
|
-
end
|
59
|
-
|
60
|
-
def each
|
61
|
-
hash.each do |k, vs|
|
62
|
-
vs.each do |v|
|
63
|
-
yield tag, k, v
|
64
|
-
end
|
65
|
-
end
|
66
|
-
metas.each do |tuple|
|
67
|
-
yield tuple
|
68
|
-
end
|
69
|
-
end
|
70
|
-
|
71
|
-
def insert tuple
|
72
|
-
if tuple.kind_of? Array
|
73
|
-
# and tuple.size == 3 and tuple[0] == tag and tuple[1].kind_of? String
|
74
|
-
# This is redundant, because of subscribe.
|
75
|
-
t, k, v = tuple
|
76
|
-
hash[k] << v
|
77
|
-
|
78
|
-
else
|
79
|
-
metas << tuple
|
80
|
-
end
|
81
|
-
end
|
82
|
-
|
83
|
-
def delete_once tuple
|
84
|
-
if tuple.kind_of? Array
|
85
|
-
# and tuple.size == 3 and tuple[0] == tag and tuple[1].kind_of? String
|
86
|
-
# This is redundant, because of subscribe.
|
87
|
-
t, k, v = tuple
|
88
|
-
if hash.key?(k) and hash[k].include? v
|
89
|
-
hash[k].delete v
|
90
|
-
hash.delete k if hash[k].empty?
|
91
|
-
true
|
92
|
-
else
|
93
|
-
false
|
94
|
-
end
|
95
|
-
|
96
|
-
else
|
97
|
-
if i=metas.index(tuple)
|
98
|
-
delete_at i
|
99
|
-
end
|
100
|
-
end
|
101
|
-
end
|
102
|
-
|
103
|
-
def transaction inserts: [], deletes: [], tick: nil
|
104
|
-
deletes.each do |tuple|
|
105
|
-
delete_once tuple or raise "bug"
|
106
|
-
end
|
107
|
-
|
108
|
-
inserts.each do |tuple|
|
109
|
-
insert tuple.freeze ## should be deep_freeze
|
110
|
-
end
|
111
|
-
end
|
112
|
-
|
113
|
-
def find_distinct_matches_for templates
|
114
|
-
templates.inject([]) do |tuples, template|
|
115
|
-
tuples << find_match_for(template, distinct_from: tuples)
|
116
|
-
end
|
117
|
-
end
|
118
|
-
|
119
|
-
def find_match_for template, distinct_from: []
|
120
|
-
case template
|
121
|
-
when SortedSetTemplate
|
122
|
-
template.find_in rbtree, distinct_from: distinct_from ###
|
123
|
-
else
|
124
|
-
# fall back to linear search
|
125
|
-
find do |tuple|
|
126
|
-
template === tuple and not distinct_from.any? {|t| t.equal? tuple}
|
127
|
-
end
|
128
|
-
end
|
129
|
-
end
|
130
|
-
end
|