spinoza 0.1 → 0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +66 -3
- data/lib/spinoza/calvin/executor.rb +107 -0
- data/lib/spinoza/calvin/node.rb +44 -0
- data/lib/spinoza/calvin/readcaster.rb +50 -0
- data/lib/spinoza/calvin/scheduler.rb +134 -0
- data/lib/spinoza/calvin/sequencer.rb +74 -0
- data/lib/spinoza/common.rb +3 -0
- data/lib/spinoza/system/link.rb +33 -0
- data/lib/spinoza/system/lock-manager.rb +22 -8
- data/lib/spinoza/system/log.rb +95 -0
- data/lib/spinoza/system/meta-log.rb +103 -0
- data/lib/spinoza/system/model.rb +14 -0
- data/lib/spinoza/system/node.rb +56 -7
- data/lib/spinoza/system/operation.rb +22 -6
- data/lib/spinoza/system/store.rb +15 -14
- data/lib/spinoza/system/{table-spec.rb → table.rb} +10 -6
- data/lib/spinoza/system/timeline.rb +81 -0
- data/lib/spinoza/transaction.rb +170 -39
- data/lib/spinoza/version.rb +1 -1
- data/test/test-executor.rb +110 -0
- data/test/test-link.rb +43 -0
- data/test/test-log.rb +47 -0
- data/test/test-meta-log.rb +63 -0
- data/test/test-node.rb +35 -14
- data/test/test-readcaster.rb +87 -0
- data/test/test-scheduler.rb +163 -0
- data/test/test-sequencer.rb +78 -0
- data/test/test-timeline.rb +58 -0
- data/test/test-transaction.rb +75 -18
- metadata +42 -3
data/lib/spinoza/system/store.rb
CHANGED
@@ -1,44 +1,45 @@
|
|
1
1
|
require 'sequel'
|
2
|
+
require 'set'
|
2
3
|
require 'spinoza/system/operation'
|
3
4
|
|
4
5
|
# Represents the storage at one node. In our model, this consists of an
|
5
|
-
# in-memory sqlite database with some very simple tables.
|
6
|
+
# in-memory sqlite database with some very simple tables. The state of the
|
7
|
+
# store is exactly the sqlite database.
|
6
8
|
class Spinoza::Store
|
7
|
-
|
9
|
+
attr_reader :tables
|
10
|
+
|
11
|
+
def initialize *tables
|
8
12
|
@db = Sequel.sqlite
|
9
|
-
@table_specs = table_specs
|
10
13
|
|
11
|
-
|
12
|
-
@db.create_table
|
13
|
-
|
14
|
+
tables.each do |table|
|
15
|
+
@db.create_table table.name do
|
16
|
+
table.columns.each do |col|
|
14
17
|
case col.type
|
15
18
|
when "integer", "string", "float"
|
16
19
|
column col.name, col.type, primary_key: col.primary
|
17
20
|
else
|
18
21
|
raise ArgumentError,
|
19
|
-
"Bad col.type: #{col.type} in table #{
|
22
|
+
"Bad col.type: #{col.type} in table #{table.name}"
|
20
23
|
end
|
21
24
|
end
|
22
25
|
end
|
23
26
|
end
|
24
|
-
|
25
|
-
|
26
|
-
def tables
|
27
|
-
@db.tables
|
27
|
+
|
28
|
+
@tables = Set[*@db.tables]
|
28
29
|
end
|
29
30
|
|
30
31
|
def inspect
|
31
|
-
"<#{self.class.name}: #{tables.join(', ')}>"
|
32
|
+
"<#{self.class.name}: #{tables.to_a.join(', ')}>"
|
32
33
|
end
|
33
34
|
|
34
35
|
# Execute the operations on this store, skipping any that do not refer to
|
35
|
-
# a table in this store.
|
36
|
+
# a table in this store. Returns array of all read results.
|
36
37
|
def execute *operations
|
37
38
|
results = operations.map do |op|
|
38
39
|
if tables.include? op.table
|
39
40
|
op.execute @db[op.table]
|
40
41
|
end
|
41
42
|
end
|
42
|
-
results.grep(ReadResult)
|
43
|
+
results.grep(Spinoza::ReadResult)
|
43
44
|
end
|
44
45
|
end
|
@@ -1,11 +1,11 @@
|
|
1
1
|
require 'spinoza/common'
|
2
2
|
|
3
|
-
# Defines the schema for one table in all replicas.
|
4
|
-
class Spinoza::
|
3
|
+
# Defines the schema for one table in all replicas. No state.
|
4
|
+
class Spinoza::Table
|
5
5
|
attr_reader :name
|
6
|
-
attr_reader :
|
6
|
+
attr_reader :columns
|
7
7
|
|
8
|
-
class
|
8
|
+
class Column
|
9
9
|
attr_reader :name
|
10
10
|
attr_reader :type
|
11
11
|
attr_reader :primary
|
@@ -23,9 +23,13 @@ class Spinoza::TableSpec
|
|
23
23
|
#
|
24
24
|
def initialize name, **specs
|
25
25
|
@name = name
|
26
|
-
@
|
26
|
+
@columns = []
|
27
27
|
specs.each_with_index do |(col_name, col_type), i|
|
28
|
-
@
|
28
|
+
@columns << Column.new(col_name, col_type, (i==0))
|
29
29
|
end
|
30
30
|
end
|
31
|
+
|
32
|
+
class << self
|
33
|
+
alias [] new
|
34
|
+
end
|
31
35
|
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
require 'rbtree'
|
2
|
+
require 'spinoza/common'
|
3
|
+
|
4
|
+
# Events are the way we model the passage of time in a concurrent system. An
|
5
|
+
# event action initiates some action at some time. The action happens
|
6
|
+
# instantaneously on the timeline, except by scheduling later events.
|
7
|
+
class Spinoza::Event
|
8
|
+
attr_reader :time, :actor, :action, :data
|
9
|
+
|
10
|
+
class << self; alias [] new; end
|
11
|
+
|
12
|
+
def initialize time: raise, actor: raise, action: raise, **data
|
13
|
+
@time, @actor, @action, @data = time, actor, action, data
|
14
|
+
end
|
15
|
+
|
16
|
+
def dispatch
|
17
|
+
if data.empty?
|
18
|
+
actor.send action
|
19
|
+
else
|
20
|
+
actor.send action, **data
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
# A timeline of events, future and past, and a current time.
|
26
|
+
class Spinoza::Timeline
|
27
|
+
class Error < StandardError; end
|
28
|
+
|
29
|
+
attr_reader :now, :future, :history
|
30
|
+
|
31
|
+
def initialize
|
32
|
+
@history = MultiRBTree.new
|
33
|
+
@future = MultiRBTree.new
|
34
|
+
@now = 0.0
|
35
|
+
end
|
36
|
+
|
37
|
+
def schedule event
|
38
|
+
@future[event.time] = event
|
39
|
+
end
|
40
|
+
alias << schedule
|
41
|
+
|
42
|
+
# Dispatches the next event. If several events are scheduled at the same time,
|
43
|
+
# dispatches the one that was scheduled first. Returns nil if nothing is
|
44
|
+
# scheduled, otherwise returns current time, which is when the event was
|
45
|
+
# dispatched. Dispatched events are stored in a history timeline. Does not
|
46
|
+
# advance time if there is no event scheduled.
|
47
|
+
def step
|
48
|
+
time, event = @future.shift
|
49
|
+
return nil unless time
|
50
|
+
dispatch_event event
|
51
|
+
return now
|
52
|
+
end
|
53
|
+
|
54
|
+
# Dispatches all events in sequence for the next +dt+ units of time, in other
|
55
|
+
# words, all events scheduled in the interval `now..now+dt`. If several events
|
56
|
+
# are scheduled at the same time, dispatches the one that was scheduled first.
|
57
|
+
# Dispatched events are stored in a history timeline. Returns the time at the
|
58
|
+
# end of the interval, which may be later than the time of any dispatched
|
59
|
+
# event. Advances time even if no events exist in the interval.
|
60
|
+
def evolve dt
|
61
|
+
t_end = now + dt
|
62
|
+
loop do
|
63
|
+
time, event = @future.first
|
64
|
+
break if not time or time > t_end
|
65
|
+
@future.shift
|
66
|
+
dispatch_event event
|
67
|
+
yield event if block_given?
|
68
|
+
end
|
69
|
+
@now = t_end
|
70
|
+
end
|
71
|
+
|
72
|
+
private def dispatch_event event
|
73
|
+
if event.time < now
|
74
|
+
raise Error, "Event scheduled in the past: #{event}"
|
75
|
+
end
|
76
|
+
|
77
|
+
@now = event.time
|
78
|
+
@history[event.time] = event
|
79
|
+
event.dispatch
|
80
|
+
end
|
81
|
+
end
|
data/lib/spinoza/transaction.rb
CHANGED
@@ -1,56 +1,187 @@
|
|
1
1
|
require 'spinoza/system/operation'
|
2
|
+
require 'set'
|
2
3
|
|
3
|
-
|
4
|
-
|
4
|
+
# A transaction is just a list of operations, each on a specific table and key.
|
5
|
+
#
|
6
|
+
# The ACID guarantees are not provided by this class. Rather, this
|
7
|
+
# class is just common representation of a (very simple) kind of transaction
|
8
|
+
# that can be used by Calvin and other transaction engines built on top of
|
9
|
+
# the system model.
|
10
|
+
#
|
11
|
+
# Transactions submitted to Calvin are not as general as this class indicates.
|
12
|
+
# They should be split up into transactions each of which consists of reads
|
13
|
+
# followed by writes, and the row IDs of the writes should be independent of the
|
14
|
+
# read results. This is necessary for the Scheduler and the Readcaster
|
15
|
+
# protocols. (See the discussion of OLLP in the Calvin papers.)
|
16
|
+
#
|
17
|
+
# Like the operations they are composed of, transactions are stateless and do
|
18
|
+
# not reference any particular store. Hence they can be re-used in different
|
19
|
+
# replicas.
|
20
|
+
#
|
21
|
+
module Spinoza
|
22
|
+
def transaction(&b)
|
23
|
+
Transaction.new(&b)
|
24
|
+
end
|
25
|
+
|
26
|
+
class Transaction
|
27
|
+
class RowLocation
|
28
|
+
def initialize txn, table, key
|
29
|
+
@txn, @table, @key = txn, table, key
|
30
|
+
end
|
31
|
+
|
32
|
+
def insert row
|
33
|
+
if @key and not @key.empty?
|
34
|
+
raise ArgumentError, "Do not specify key in `at(...).insert(...)`."
|
35
|
+
end
|
36
|
+
@txn << InsertOperation.new(@txn, table: @table, row: row)
|
37
|
+
end
|
38
|
+
|
39
|
+
def update row
|
40
|
+
@txn << UpdateOperation.new(@txn, table: @table, row: row, key: @key)
|
41
|
+
end
|
42
|
+
|
43
|
+
def delete
|
44
|
+
@txn << DeleteOperation.new(@txn, table: @table, key: @key)
|
45
|
+
end
|
5
46
|
|
6
|
-
|
7
|
-
|
8
|
-
|
47
|
+
def read
|
48
|
+
@txn << ReadOperation.new(@txn, table: @table, key: @key)
|
49
|
+
end
|
9
50
|
end
|
10
|
-
|
11
|
-
|
12
|
-
|
51
|
+
|
52
|
+
# Build a transaction using a DSL. Example:
|
53
|
+
#
|
54
|
+
# txn = transaction do
|
55
|
+
# at(:persons, name: "Fred").update(age: 41, phrase: "yabba dabba doo")
|
56
|
+
# at(:persons, name: "Wilma").delete
|
57
|
+
# at(:persons, name: "Barney").read
|
58
|
+
# at(:persons).insert(name: "Betty", age: 65)
|
59
|
+
# end
|
60
|
+
#
|
61
|
+
# node.store.execute txn # ==> [ReadResult(...)]
|
62
|
+
#
|
63
|
+
# If the block takes an argument, then the Transaction instance is passed,
|
64
|
+
# and the block is not instance_eval-ed.
|
65
|
+
#
|
66
|
+
# If you create a transaction without using the block interface, you
|
67
|
+
# must call #closed! before using the transaction.
|
68
|
+
#
|
69
|
+
def initialize &block
|
70
|
+
@ops = []
|
71
|
+
@closed = false
|
72
|
+
@read_set = {}
|
73
|
+
@write_set = {}
|
74
|
+
|
75
|
+
if block
|
76
|
+
if block.arity == 0
|
77
|
+
instance_eval &block
|
78
|
+
else
|
79
|
+
yield self
|
80
|
+
end
|
81
|
+
closed!
|
82
|
+
end
|
13
83
|
end
|
14
84
|
|
15
|
-
|
16
|
-
|
85
|
+
class StateError < StandardError; end
|
86
|
+
|
87
|
+
def closed?
|
88
|
+
@closed
|
17
89
|
end
|
18
90
|
|
19
|
-
def
|
20
|
-
|
91
|
+
def closed!
|
92
|
+
assert_open!
|
93
|
+
@closed = true
|
21
94
|
end
|
22
95
|
|
23
|
-
def
|
24
|
-
|
96
|
+
def assert_closed!
|
97
|
+
unless closed?
|
98
|
+
raise StateError, "transaction must be closed to call that method."
|
99
|
+
end
|
25
100
|
end
|
26
|
-
end
|
27
101
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
102
|
+
def assert_open!
|
103
|
+
if closed?
|
104
|
+
raise StateError, "transaction must be open to call that method."
|
105
|
+
end
|
106
|
+
end
|
107
|
+
|
108
|
+
def at table, **key
|
109
|
+
RowLocation.new(self, table, key)
|
110
|
+
end
|
111
|
+
|
112
|
+
INSERT_KEY = :insert
|
113
|
+
|
114
|
+
def << op
|
115
|
+
assert_open!
|
116
|
+
|
117
|
+
case op
|
118
|
+
when ReadOperation
|
119
|
+
if reads_a_write?(op)
|
120
|
+
warn "reading your writes in a transaction may not be supported #{op}"
|
121
|
+
end
|
122
|
+
(@read_set[op.table] ||= Set[]) << op.key
|
123
|
+
when InsertOperation
|
124
|
+
(@write_set[op.table] ||= Set[]) << INSERT_KEY
|
47
125
|
else
|
48
|
-
|
126
|
+
(@write_set[op.table] ||= Set[]) << op.key
|
49
127
|
end
|
128
|
+
@ops << op
|
129
|
+
end
|
130
|
+
|
131
|
+
def reads_a_write? op
|
132
|
+
s = @write_set[op.table]
|
133
|
+
s && s.include?(op.key)
|
134
|
+
end
|
135
|
+
|
136
|
+
# {table => Set[key, ...]}
|
137
|
+
def read_set
|
138
|
+
assert_closed!
|
139
|
+
@read_set
|
140
|
+
end
|
141
|
+
|
142
|
+
# Set[table, table, ...]
|
143
|
+
def read_tables
|
144
|
+
@read_tables ||= Set[*read_set.keys]
|
145
|
+
end
|
146
|
+
|
147
|
+
# {table => Set[key|INSERT_KEY, ...]}
|
148
|
+
# where existence of INSERT_KEY means presence of inserts in txn
|
149
|
+
def write_set
|
150
|
+
assert_closed!
|
151
|
+
@write_set
|
152
|
+
end
|
153
|
+
|
154
|
+
# Set[table, table, ...]
|
155
|
+
def write_tables
|
156
|
+
@write_tables ||= Set[*write_set.keys]
|
157
|
+
end
|
158
|
+
|
159
|
+
def all_read_ops
|
160
|
+
assert_closed!
|
161
|
+
@all_read_ops ||= @ops.grep(ReadOperation)
|
162
|
+
end
|
163
|
+
|
164
|
+
def all_write_ops
|
165
|
+
assert_closed!
|
166
|
+
@all_write_ops ||= @ops - all_read_ops
|
167
|
+
end
|
168
|
+
|
169
|
+
def ops
|
170
|
+
assert_closed!
|
171
|
+
@ops
|
172
|
+
end
|
173
|
+
|
174
|
+
# returns true iff node_or_store contains elements of write set
|
175
|
+
def active? node_or_store
|
176
|
+
write_tables.intersect? node_or_store.tables
|
177
|
+
end
|
178
|
+
|
179
|
+
def all_reads_are_local? node_or_store
|
180
|
+
read_tables.subset? node_or_store.tables
|
181
|
+
end
|
182
|
+
|
183
|
+
def remote_read_tables node_or_store
|
184
|
+
read_tables - node_or_store.tables
|
50
185
|
end
|
51
|
-
end
|
52
|
-
|
53
|
-
def at table, **key
|
54
|
-
RowLocation.new(self, table, key)
|
55
186
|
end
|
56
187
|
end
|
data/lib/spinoza/version.rb
CHANGED
@@ -0,0 +1,110 @@
|
|
1
|
+
require 'minitest/autorun'
|
2
|
+
require 'spinoza/calvin/executor'
|
3
|
+
require 'spinoza/system/store'
|
4
|
+
require 'spinoza/system/table'
|
5
|
+
require 'spinoza/transaction'
|
6
|
+
|
7
|
+
class TestExecutor < Minitest::Test
|
8
|
+
include Spinoza
|
9
|
+
|
10
|
+
class MockReadcaster
|
11
|
+
def initialize store
|
12
|
+
@store = store
|
13
|
+
@tables = store.tables
|
14
|
+
end
|
15
|
+
|
16
|
+
def execute_local_reads txn
|
17
|
+
local_read_ops = txn.all_read_ops.select {|r| @tables.include? r.table}
|
18
|
+
@store.execute *local_read_ops
|
19
|
+
end
|
20
|
+
|
21
|
+
def serve_reads txn, local_read_results; end
|
22
|
+
end
|
23
|
+
|
24
|
+
def setup
|
25
|
+
@store = Store.new(
|
26
|
+
Table[:as, id: "integer", name: "string"],
|
27
|
+
Table[:bs, id: "integer", name: "string"])
|
28
|
+
|
29
|
+
@executor = Calvin::Executor.new(
|
30
|
+
store: @store,
|
31
|
+
readcaster: MockReadcaster.new(@store))
|
32
|
+
end
|
33
|
+
|
34
|
+
def test_passive_txn
|
35
|
+
txn = transaction do
|
36
|
+
at(:as, id: 1).read
|
37
|
+
at(:bs, id: 2).read
|
38
|
+
at(:cs).insert id: 3, name: "c3"
|
39
|
+
at(:ds, id: 4).read
|
40
|
+
end
|
41
|
+
|
42
|
+
assert @executor.passive? txn
|
43
|
+
|
44
|
+
result = @executor.execute_transaction txn
|
45
|
+
assert result
|
46
|
+
assert_equal 2, result.size
|
47
|
+
assert_equal txn.ops[0..1], result.map {|r| r.op}
|
48
|
+
assert @executor.ready?
|
49
|
+
end
|
50
|
+
|
51
|
+
def test_txn_with_no_remote_reads
|
52
|
+
txn = transaction do
|
53
|
+
at(:as).insert id: 1, name: "a1"
|
54
|
+
at(:bs, id: 2).read
|
55
|
+
at(:cs).insert id: 3, name: "c3"
|
56
|
+
end
|
57
|
+
|
58
|
+
refute @executor.passive? txn
|
59
|
+
assert @executor.all_reads_are_local? txn
|
60
|
+
|
61
|
+
result = @executor.execute_transaction txn
|
62
|
+
assert result
|
63
|
+
assert_equal 1, result.size
|
64
|
+
assert_equal txn.ops[1], result.map {|r| r.op}[0]
|
65
|
+
assert @executor.ready?
|
66
|
+
end
|
67
|
+
|
68
|
+
def test_txn_with_remote_reads
|
69
|
+
txn = transaction do
|
70
|
+
at(:as).insert id: 1, name: "a1"
|
71
|
+
at(:bs, id: 2).read
|
72
|
+
at(:cs, id: 3).read
|
73
|
+
at(:cs, id: 4).read
|
74
|
+
at(:ds, id: 5).read
|
75
|
+
end
|
76
|
+
|
77
|
+
refute @executor.passive? txn
|
78
|
+
|
79
|
+
result = @executor.execute_transaction txn
|
80
|
+
refute result
|
81
|
+
refute @executor.ready?
|
82
|
+
|
83
|
+
result = @executor.recv_remote_reads :cs, [
|
84
|
+
ReadResult.new(val: {id: 3}),
|
85
|
+
ReadResult.new(val: {id: 4})
|
86
|
+
]
|
87
|
+
refute result
|
88
|
+
refute @executor.ready?
|
89
|
+
|
90
|
+
# Same data, different replica:
|
91
|
+
result = @executor.recv_remote_reads :cs, [
|
92
|
+
ReadResult.new(val: {id: 3}),
|
93
|
+
ReadResult.new(val: {id: 4})
|
94
|
+
]
|
95
|
+
refute result
|
96
|
+
refute @executor.ready?
|
97
|
+
|
98
|
+
result = @executor.recv_remote_reads :ds, [
|
99
|
+
ReadResult.new(val: {id: 5})
|
100
|
+
]
|
101
|
+
assert result
|
102
|
+
assert @executor.ready?
|
103
|
+
assert_equal 4, result.size
|
104
|
+
|
105
|
+
local_read = result.shift
|
106
|
+
refute local_read.val # never wrote this one
|
107
|
+
|
108
|
+
assert_equal Set[3,4,5], Set[*result.map {|r| r.val[:id]}]
|
109
|
+
end
|
110
|
+
end
|