@evolu/common 8.17.0 → 8.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/Bytes.d.ts +31 -14
- package/dist/src/Bytes.d.ts.map +1 -1
- package/dist/src/Bytes.js +48 -8
- package/dist/src/Error.d.ts.map +1 -1
- package/dist/src/Error.js +5 -1
- package/dist/src/Polyfills.d.ts.map +1 -1
- package/dist/src/Polyfills.js +3 -2
- package/dist/src/Sqlite.d.ts +1 -1
- package/dist/src/Task.d.ts +4 -3
- package/dist/src/Task.d.ts.map +1 -1
- package/dist/src/Task.js +4 -3
- package/dist/src/index.d.ts +1 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/local-first/Db.d.ts +32 -3
- package/dist/src/local-first/Db.d.ts.map +1 -1
- package/dist/src/local-first/Db.js +30 -9
- package/dist/src/local-first/Evolu.d.ts +21 -6
- package/dist/src/local-first/Evolu.d.ts.map +1 -1
- package/dist/src/local-first/Evolu.js +5 -1
- package/dist/src/local-first/Protocol.d.ts +215 -93
- package/dist/src/local-first/Protocol.d.ts.map +1 -1
- package/dist/src/local-first/Protocol.js +799 -489
- package/dist/src/local-first/Relay.d.ts +9 -1
- package/dist/src/local-first/Relay.d.ts.map +1 -1
- package/dist/src/local-first/Relay.js +41 -36
- package/dist/src/local-first/Shared.d.ts +79 -53
- package/dist/src/local-first/Shared.d.ts.map +1 -1
- package/dist/src/local-first/Shared.js +67 -42
- package/dist/src/local-first/Storage.d.ts +80 -18
- package/dist/src/local-first/Storage.d.ts.map +1 -1
- package/dist/src/local-first/Storage.js +20 -4
- package/package.json +1 -1
- package/src/Bytes.test.ts +212 -0
- package/src/Bytes.ts +67 -17
- package/src/Error.ts +5 -1
- package/src/Polyfills.ts +3 -2
- package/src/Sqlite.ts +1 -1
- package/src/Task.ts +4 -3
- package/src/index.ts +4 -1
- package/src/local-first/Db.ts +76 -17
- package/src/local-first/Evolu.test.ts +43 -12
- package/src/local-first/Evolu.ts +34 -7
- package/src/local-first/Protocol.test.ts +1541 -44
- package/src/local-first/Protocol.ts +1070 -605
- package/src/local-first/Relay.ts +80 -69
- package/src/local-first/Shared.test.ts +37 -1
- package/src/local-first/Shared.ts +124 -73
- package/src/local-first/Storage.ts +100 -27
|
@@ -111,10 +111,10 @@
|
|
|
111
111
|
* whichever instance made it, even one disposed before the database worker
|
|
112
112
|
* answered. When a relay frame stores new messages, the tenant requests a round
|
|
113
113
|
* through each other transport claimed for the owner, so data learned from one
|
|
114
|
-
* relay reaches the others, except through a route that skipped a message
|
|
115
|
-
* described below. A closed transport reconciles
|
|
116
|
-
* replacement leader reconciles every transport again,
|
|
117
|
-
* reporting stored messages may have been lost.
|
|
114
|
+
* relay reaches the others, except through a route that skipped a message it
|
|
115
|
+
* received or could not send, as described below. A closed transport reconciles
|
|
116
|
+
* when it opens, and a replacement leader reconciles every transport again,
|
|
117
|
+
* because a response reporting stored messages may have been lost.
|
|
118
118
|
*
|
|
119
119
|
* Relays omit the sending socket when broadcasting an upload, so the uploader
|
|
120
120
|
* also delivers it as local Broadcast frames to every other tenant with
|
|
@@ -178,17 +178,17 @@
|
|
|
178
178
|
* request, a replacement leader, or storing messages from another transport,
|
|
179
179
|
* unless the route skipped a message. No failed or aborted result has arrived
|
|
180
180
|
* on the route since.
|
|
181
|
-
* - No result that skipped a received message
|
|
182
|
-
* round was last requested through it,
|
|
183
|
-
*
|
|
184
|
-
*
|
|
185
|
-
* skipped a message.
|
|
181
|
+
* - No result that skipped a received message or a stored change too large to
|
|
182
|
+
* send has arrived on the route since a round was last requested through it,
|
|
183
|
+
* and since that request, no messages have been stored from another
|
|
184
|
+
* transport, directly or through a sibling copy uploaded outside this
|
|
185
|
+
* transport, and no sibling copy has failed to apply or skipped a message.
|
|
186
186
|
*
|
|
187
187
|
* A route is settled when every condition except the last holds, so a complete
|
|
188
188
|
* route is settled too. A settled route that is incomplete has ended its
|
|
189
|
-
* reconciliation, but its relay may offer a message the database skipped,
|
|
190
|
-
*
|
|
191
|
-
* {@link SettledSyncRoute}.
|
|
189
|
+
* reconciliation, but its relay may offer a message the database skipped, the
|
|
190
|
+
* database may store a change too large to send, or messages stored elsewhere
|
|
191
|
+
* may not have reached it; it is published as {@link SettledSyncRoute}.
|
|
192
192
|
*
|
|
193
193
|
* A reconciliation chain ends only with a converged result, a failure, an
|
|
194
194
|
* abort, a dropped frame, or a continuation that finds the socket closed, and
|
|
@@ -207,20 +207,34 @@
|
|
|
207
207
|
* while the database worker creates a round is logged there and fails the
|
|
208
208
|
* round's routes with `SyncFailed` without a retry. A mutation that throws is
|
|
209
209
|
* rolled back and reported as an {@link UnknownError} to the tab that made it,
|
|
210
|
-
* or to every tab when its Evolu instance was disposed first.
|
|
211
|
-
* SQLite
|
|
212
|
-
*
|
|
213
|
-
*
|
|
210
|
+
* or to every tab when its Evolu instance was disposed first. A received batch
|
|
211
|
+
* SQLite fails to store, for example on a full disk, is rolled back and fails
|
|
212
|
+
* its route with an `UnknownError`. Other unexpected SQLite exceptions remain
|
|
213
|
+
* unsupported and can panic the database worker. A frame the relay silently
|
|
214
|
+
* drops, such as invalid data, leaves the count above zero until the liveness
|
|
215
|
+
* rule below replaces the socket.
|
|
214
216
|
*
|
|
215
|
-
* A result that skipped a
|
|
216
|
-
*
|
|
217
|
-
*
|
|
218
|
-
*
|
|
219
|
-
*
|
|
220
|
-
*
|
|
221
|
-
*
|
|
222
|
-
*
|
|
223
|
-
*
|
|
217
|
+
* A result that skipped a message records the error on the route as its
|
|
218
|
+
* `skippedError`, without ending its chain, so it requests no round. A message
|
|
219
|
+
* is skipped in two cases:
|
|
220
|
+
*
|
|
221
|
+
* - The relay offers a message the database could not decrypt, verify, or decode.
|
|
222
|
+
* The relay offers it again in every round through the route until the
|
|
223
|
+
* database stores a message with that timestamp.
|
|
224
|
+
* - The database stores a change it cannot send, a
|
|
225
|
+
* {@link ProtocolChangeTooLargeError}: one saved before `maxMutationSize`
|
|
226
|
+
* existed, or a crafted one received from a relay. Every route through a
|
|
227
|
+
* relay that lacks it skips it on every sync, so each such route stays
|
|
228
|
+
* settled.
|
|
229
|
+
*
|
|
230
|
+
* The messages received elsewhere that the last condition names request no
|
|
231
|
+
* round through such a route, even while a requested round checks it again,
|
|
232
|
+
* because each round would download every skipped message again, or skip again
|
|
233
|
+
* a change too large to send. The next round that an explicit request, a
|
|
234
|
+
* reopen, a replacement leader, or a failure sends through the route reconciles
|
|
235
|
+
* them. So while the database stores a change too large to send, messages from
|
|
236
|
+
* other relays reach a route only through such a round, for example after a
|
|
237
|
+
* reconnect or {@link Evolu.requestSync}.
|
|
224
238
|
*
|
|
225
239
|
* ### Liveness
|
|
226
240
|
*
|
|
@@ -842,8 +856,9 @@ export const initSharedWorker = (self) => async (run) => {
|
|
|
842
856
|
};
|
|
843
857
|
// Held while this worker runs. Its DbWorkers stop once they can take it,
|
|
844
858
|
// because a Dispose posted right before this worker closes can be lost, as
|
|
845
|
-
// in Firefox. Taken
|
|
846
|
-
//
|
|
859
|
+
// in Firefox (https://bugzilla.mozilla.org/show_bug.cgi?id=2077609). Taken
|
|
860
|
+
// before the build lock, so nothing delays the end of starting once that
|
|
861
|
+
// lock is held.
|
|
847
862
|
disposer.use(await run.ok(acquireLeaderLock(workerId)));
|
|
848
863
|
// Released after every tenant is disposed and has told its DbWorker to
|
|
849
864
|
// stop. Earlier releases take the same lock in their leader tab; see
|
|
@@ -1284,6 +1299,11 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, },
|
|
|
1284
1299
|
// tell tabs that connect later without starting another worker.
|
|
1285
1300
|
if (dbWorker.type === "Leading") {
|
|
1286
1301
|
assertNotSame(dbWorker.port, currentDbWorkerPort);
|
|
1302
|
+
// The leader may still hold the database lock: browsers deliver
|
|
1303
|
+
// a refusal before a worker requested later can lead, but the
|
|
1304
|
+
// tenant does not rely on that order. Tell it to dispose, so it
|
|
1305
|
+
// releases the lock.
|
|
1306
|
+
dbWorker.port.postMessage({ type: "Dispose" });
|
|
1287
1307
|
dbWorker.port[Symbol.dispose]();
|
|
1288
1308
|
}
|
|
1289
1309
|
currentDbWorkerPort[Symbol.dispose]();
|
|
@@ -1588,7 +1608,8 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, },
|
|
|
1588
1608
|
!hasQueuedWrite;
|
|
1589
1609
|
// Settling drops the failure, so the next one requests a round
|
|
1590
1610
|
// again. A route with a skip it is not rechecking settles without
|
|
1591
|
-
// completing, because its relay may offer the skipped message
|
|
1611
|
+
// completing, because its relay may offer the skipped message, the
|
|
1612
|
+
// database may store a change too large to send, or the relay may
|
|
1592
1613
|
// lack messages stored elsewhere.
|
|
1593
1614
|
const pending = routeToPending(route.progress);
|
|
1594
1615
|
route.progress =
|
|
@@ -1684,11 +1705,12 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, },
|
|
|
1684
1705
|
route.lastReceivedAt = now;
|
|
1685
1706
|
// A skipped message is recorded as the route's skip, not a
|
|
1686
1707
|
// failure, and does not end the round, whose response below is
|
|
1687
|
-
// still sent, so it requests no round. The relay offers
|
|
1688
|
-
//
|
|
1689
|
-
//
|
|
1690
|
-
//
|
|
1691
|
-
//
|
|
1708
|
+
// still sent, so it requests no round. The relay offers a message
|
|
1709
|
+
// the database could not store again in every later round, and
|
|
1710
|
+
// every round skips a stored change too large to send again, so
|
|
1711
|
+
// the route stays incomplete until a round requested through it
|
|
1712
|
+
// settles without skipping a message and without messages stored
|
|
1713
|
+
// elsewhere since that request.
|
|
1692
1714
|
if (skippedError !== null)
|
|
1693
1715
|
route.progress = {
|
|
1694
1716
|
...routeToPending(route.progress),
|
|
@@ -1712,12 +1734,14 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, },
|
|
|
1712
1734
|
}
|
|
1713
1735
|
else if (failure !== null || skippedError !== null) {
|
|
1714
1736
|
// A sibling's copy comes from this worker, not from a relay, so no
|
|
1715
|
-
// route shows its failure.
|
|
1716
|
-
//
|
|
1717
|
-
//
|
|
1718
|
-
//
|
|
1719
|
-
//
|
|
1720
|
-
//
|
|
1737
|
+
// route shows its failure. SQLite can fail to store it, for example
|
|
1738
|
+
// on a full disk, which returns an UnknownError. It is a Broadcast,
|
|
1739
|
+
// which carries no relay error, so any other error or a skip means
|
|
1740
|
+
// a bug, such as databases holding different keys for the owner.
|
|
1741
|
+
// Each is reported as an unexpected failure. A Failed result was
|
|
1742
|
+
// logged, which reports it already. An error is the failure, which
|
|
1743
|
+
// is never an abort, and like a route, the report leaves out its
|
|
1744
|
+
// frame.
|
|
1721
1745
|
const unexpected = error !== null ? failure : skippedError;
|
|
1722
1746
|
if (unexpected !== null)
|
|
1723
1747
|
deps.postConsoleEntryOrError({
|
|
@@ -1806,9 +1830,10 @@ const createEvoluTenant = ({ name, consoleLevel, sqliteSchema, encryptionKey, },
|
|
|
1806
1830
|
* `except`, after messages from elsewhere were stored or a sibling copy was
|
|
1807
1831
|
* not fully stored. Rounds toward the same transport coalesce whatever
|
|
1808
1832
|
* their source. A route that skipped a message gets none, because its relay
|
|
1809
|
-
* would offer that message again
|
|
1810
|
-
*
|
|
1811
|
-
*
|
|
1833
|
+
* would offer that message again or the round would skip a stored change
|
|
1834
|
+
* too large to send again. A route rechecking one stops rechecking, because
|
|
1835
|
+
* its round may have read the database before this event. A later requested
|
|
1836
|
+
* round through such a route reconciles it.
|
|
1812
1837
|
*/
|
|
1813
1838
|
const requestRoundsForReceivedMessages = (ownerId, { except, afterQueuedWrites, }) => {
|
|
1814
1839
|
const keys = [];
|
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
*/
|
|
6
6
|
import type { NonEmptyReadonlyArray } from "../Array.ts";
|
|
7
7
|
import type { Brand } from "../Brand.ts";
|
|
8
|
+
import type { UnknownError } from "../Error.ts";
|
|
8
9
|
import type { RandomDep } from "../Random.ts";
|
|
9
10
|
import type { SqliteDep } from "../Sqlite.ts";
|
|
10
11
|
import type { Task } from "../Task.ts";
|
|
@@ -32,7 +33,10 @@ export interface StorageConfig {
|
|
|
32
33
|
* asynchronous (for calling remote APIs).
|
|
33
34
|
*
|
|
34
35
|
* The callback returns a boolean rather than an error because error handling
|
|
35
|
-
* and logging are the responsibility of the callback implementation.
|
|
36
|
+
* and logging are the responsibility of the callback implementation. It must
|
|
37
|
+
* not throw or reject, because a throw is a defect that shuts the relay down
|
|
38
|
+
* for a supervisor to restart it; a callback calling a remote service catches
|
|
39
|
+
* its failure and returns whether to allow the write.
|
|
36
40
|
*
|
|
37
41
|
* Relay deployments configure this callback. Client applications observe a
|
|
38
42
|
* denied relay write as a {@link ProtocolQuotaError} in the `failure` of that
|
|
@@ -81,10 +85,9 @@ export interface StorageConfig {
|
|
|
81
85
|
* that satisfies this contract.
|
|
82
86
|
*
|
|
83
87
|
* {@link Storage.writeMessages} returns the {@link StorageWriteMessagesError}
|
|
84
|
-
* that made it store none of a batch. Implementations return
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
* to wire error codes.
|
|
88
|
+
* that made it store none of a batch. Implementations return it without
|
|
89
|
+
* reporting it; the caller owns reporting. The client protocol forwards these
|
|
90
|
+
* errors unchanged, while the relay protocol maps them to wire error codes.
|
|
88
91
|
*
|
|
89
92
|
* The Storage API is synchronous because SQLite's synchronous API is the
|
|
90
93
|
* fastest way to use SQLite. Synchronous bindings (like better-sqlite3) call
|
|
@@ -95,29 +98,55 @@ export interface StorageConfig {
|
|
|
95
98
|
* for async validation logic before writing to storage. The write operation
|
|
96
99
|
* itself remains synchronous.
|
|
97
100
|
*
|
|
101
|
+
* Indexes count an owner's timestamps in order from 0, and none may exceed the
|
|
102
|
+
* owner's size from {@link Storage.getSize}. Sync reads the size once per
|
|
103
|
+
* message and derives every index from it.
|
|
104
|
+
*
|
|
98
105
|
* @group Core
|
|
99
106
|
*/
|
|
100
107
|
export interface Storage {
|
|
108
|
+
/** Returns the number of the owner's timestamps. */
|
|
101
109
|
readonly getSize: (ownerId: OwnerIdBytes) => NonNegativeInt;
|
|
110
|
+
/**
|
|
111
|
+
* Returns the {@link Fingerprint} of the owner's timestamps from index `begin`
|
|
112
|
+
* up to, but not including, `end`, where `begin` is at most `end`.
|
|
113
|
+
*/
|
|
102
114
|
readonly fingerprint: (ownerId: OwnerIdBytes, begin: NonNegativeInt, end: NonNegativeInt) => Fingerprint;
|
|
103
115
|
/**
|
|
104
116
|
* Computes fingerprints with their upper bounds in one call.
|
|
105
117
|
*
|
|
118
|
+
* Each bucket is the end index of a range that starts at the previous bucket,
|
|
119
|
+
* or at 0 for the first, so the buckets must be ascending.
|
|
120
|
+
*
|
|
106
121
|
* This function can be replaced with many fingerprint/findLowerBound calls,
|
|
107
122
|
* but implementations can leverage it for batching and more efficient
|
|
108
123
|
* fingerprint computation.
|
|
109
124
|
*/
|
|
110
125
|
readonly fingerprintRanges: (ownerId: OwnerIdBytes, buckets: ReadonlyArray<NonNegativeInt>, upperBound?: RangeUpperBound) => ReadonlyArray<FingerprintRange>;
|
|
126
|
+
/**
|
|
127
|
+
* Returns the index of the owner's first timestamp not below `upperBound`,
|
|
128
|
+
* counted from the owner's first timestamp, or `end` when there is none or
|
|
129
|
+
* `upperBound` is {@link InfiniteUpperBound}.
|
|
130
|
+
*
|
|
131
|
+
* `begin` must be 0 or the result for an earlier bound not above
|
|
132
|
+
* `upperBound`, and `end` the owner's size, so the result lies in [`begin`,
|
|
133
|
+
* `end`]. When `end` is 0 or equals `begin`, the result is `end`. Sync meets
|
|
134
|
+
* this because the protocol rejects range upper bounds that decrease.
|
|
135
|
+
*/
|
|
111
136
|
readonly findLowerBound: (ownerId: OwnerIdBytes, begin: NonNegativeInt, end: NonNegativeInt, upperBound: RangeUpperBound) => NonNegativeInt;
|
|
137
|
+
/**
|
|
138
|
+
* Calls `callback` with the owner's timestamps and their indexes from `begin`
|
|
139
|
+
* up to, but not including, `end`, where `begin` is at most `end`, until the
|
|
140
|
+
* callback returns `false`.
|
|
141
|
+
*/
|
|
112
142
|
readonly iterate: (ownerId: OwnerIdBytes, begin: NonNegativeInt, end: NonNegativeInt, callback: (timestamp: TimestampBytes, index: NonNegativeInt) => boolean) => void;
|
|
113
143
|
/**
|
|
114
144
|
* Validates the {@link OwnerWriteKey} for the given {@link Owner}.
|
|
115
145
|
*
|
|
116
|
-
* Returns `true` if the write key is valid, `false` otherwise.
|
|
146
|
+
* Returns `true` if the write key is valid, `false` otherwise. A relay logs a
|
|
147
|
+
* throw and answers the request with {@link ProtocolWriteError}.
|
|
117
148
|
*/
|
|
118
149
|
readonly validateWriteKey: (ownerId: OwnerIdBytes, writeKey: OwnerWriteKey) => boolean;
|
|
119
|
-
/** Sets the {@link OwnerWriteKey} for the given {@link Owner}. */
|
|
120
|
-
readonly setWriteKey: (ownerId: OwnerIdBytes, writeKey: OwnerWriteKey) => void;
|
|
121
150
|
/**
|
|
122
151
|
* Write encrypted {@link CrdtMessage}s to storage.
|
|
123
152
|
*
|
|
@@ -131,7 +160,12 @@ export interface Storage {
|
|
|
131
160
|
* protocol logic handling during sync operations.
|
|
132
161
|
*/
|
|
133
162
|
readonly writeMessages: (ownerIdBytes: OwnerIdBytes, messages: NonEmptyReadonlyArray<EncryptedCrdtMessage>) => Task<void, StorageWriteMessagesError>;
|
|
134
|
-
/**
|
|
163
|
+
/**
|
|
164
|
+
* Read encrypted {@link DbChange}s from storage.
|
|
165
|
+
*
|
|
166
|
+
* The returned array must not be modified or reused later, because sync
|
|
167
|
+
* references it until it finishes its answer.
|
|
168
|
+
*/
|
|
135
169
|
readonly readDbChange: (ownerId: OwnerIdBytes, timestamp: TimestampBytes) => EncryptedDbChange;
|
|
136
170
|
/** Delete all data for the given {@link Owner}. */
|
|
137
171
|
readonly deleteOwner: (ownerId: OwnerIdBytes) => void;
|
|
@@ -152,7 +186,7 @@ export interface StorageDep {
|
|
|
152
186
|
export interface StorageQuotaError extends OwnerError, Typed<"StorageQuotaError"> {
|
|
153
187
|
}
|
|
154
188
|
/**
|
|
155
|
-
*
|
|
189
|
+
* Reasons why {@link Storage.writeMessages} stored none of a batch.
|
|
156
190
|
*
|
|
157
191
|
* The built-in relay storage stores opaque encrypted messages and rejects
|
|
158
192
|
* batches over quota. The built-in client storage decrypts and validates
|
|
@@ -162,15 +196,38 @@ export interface StorageQuotaError extends OwnerError, Typed<"StorageQuotaError"
|
|
|
162
196
|
* never a reason to reject a batch. The contract permits quota checks on either
|
|
163
197
|
* side.
|
|
164
198
|
*
|
|
199
|
+
* Both built-in storages return {@link UnknownError} when SQLite fails the
|
|
200
|
+
* write, for example on a full disk, after rolling it back.
|
|
201
|
+
*
|
|
165
202
|
* @group Core
|
|
166
203
|
*/
|
|
167
|
-
export type StorageWriteMessagesError = StorageQuotaError;
|
|
168
|
-
/**
|
|
169
|
-
* A
|
|
170
|
-
*
|
|
171
|
-
*
|
|
172
|
-
*
|
|
173
|
-
*
|
|
204
|
+
export type StorageWriteMessagesError = StorageQuotaError | UnknownError;
|
|
205
|
+
/**
|
|
206
|
+
* A summary of a range of {@link TimestampBytes} for comparing ranges cheaply.
|
|
207
|
+
*
|
|
208
|
+
* It is the XOR of the first {@link fingerprintSize} bytes of the SHA-256 hash
|
|
209
|
+
* of each timestamp in the range, or {@link zeroFingerprint} for an empty
|
|
210
|
+
* range.
|
|
211
|
+
*
|
|
212
|
+
* XOR is linear, so fingerprints are not collision resistant: any 97 timestamps
|
|
213
|
+
* contain a nonempty subset whose fingerprints XOR to zero. Anyone who can
|
|
214
|
+
* write to a relay for an owner, which includes every collaborator of a
|
|
215
|
+
* {@link SharedOwner}, can store timestamps whose fingerprints XOR to the
|
|
216
|
+
* fingerprint of a chosen change. Two ranges that differ only by that change
|
|
217
|
+
* and those timestamps then compare equal, so sync skips them, and the change
|
|
218
|
+
* does not move between that relay and a peer. This was demonstrated end to
|
|
219
|
+
* end, and nothing reports it. {@link Evolu.requestSync} does not heal it,
|
|
220
|
+
* because the next round compares the same fingerprints. In protocol version 1,
|
|
221
|
+
* the only recovery is syncing through a relay that does not hold those
|
|
222
|
+
* timestamps. Binding the count or hashing the result changes every
|
|
223
|
+
* fingerprint, so it needs a new protocol version.
|
|
224
|
+
*
|
|
225
|
+
* A write-key holder can also store a change under a timestamp another device
|
|
226
|
+
* has not used yet, such as a later one with that device's NodeId, because
|
|
227
|
+
* relays accept timestamps from any time. A relay keeps the first change stored
|
|
228
|
+
* for a timestamp, so it does not store the device's change when it arrives,
|
|
229
|
+
* and peers that sync through that relay later get the stored one instead. A
|
|
230
|
+
* new fingerprint does not close this.
|
|
174
231
|
*
|
|
175
232
|
* @group Ranges
|
|
176
233
|
*/
|
|
@@ -275,6 +332,9 @@ export interface EncryptedCrdtMessage {
|
|
|
275
332
|
/**
|
|
276
333
|
* Encrypted DbChange
|
|
277
334
|
*
|
|
335
|
+
* A 24-byte XChaCha20-Poly1305 nonce, the ciphertext length, and the ciphertext
|
|
336
|
+
* ending with its 16-byte Poly1305 tag, so at least 41 bytes.
|
|
337
|
+
*
|
|
278
338
|
* @group Messages
|
|
279
339
|
*/
|
|
280
340
|
export type EncryptedDbChange = Uint8Array & Brand<"EncryptedDbChange">;
|
|
@@ -367,7 +427,7 @@ export interface DbChange extends InferType<typeof DbChange> {
|
|
|
367
427
|
*
|
|
368
428
|
* @group SQLite
|
|
369
429
|
*/
|
|
370
|
-
export interface BaseSqliteStorage extends Omit<Storage, "validateWriteKey" | "
|
|
430
|
+
export interface BaseSqliteStorage extends Omit<Storage, "validateWriteKey" | "writeMessages" | "readDbChange"> {
|
|
371
431
|
/**
|
|
372
432
|
* Inserts a timestamp for an owner into the skiplist-based storage.
|
|
373
433
|
*
|
|
@@ -448,6 +508,8 @@ export declare const testFingerprintTimestamps: (timestamps: ReadonlyArray<Times
|
|
|
448
508
|
/**
|
|
449
509
|
* Reads the timestamp at a position within an owner's ordered timestamps.
|
|
450
510
|
*
|
|
511
|
+
* Throws when `index` is not below the owner's size.
|
|
512
|
+
*
|
|
451
513
|
* @group SQLite
|
|
452
514
|
*/
|
|
453
515
|
export declare const getTimestampByIndex: (deps: SqliteDep) => (ownerId: OwnerIdBytes, index: NonNegativeInt) => TimestampBytes;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"Storage.d.ts","sourceRoot":"","sources":["../../../src/local-first/Storage.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAGzD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"Storage.d.ts","sourceRoot":"","sources":["../../../src/local-first/Storage.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAGH,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAGzD,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAEhD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE9C,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAE9C,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,YAAY,CAAC;AAEvC,OAAO,EACL,OAAO,EAEP,EAAE,EACF,KAAK,SAAS,EAEd,cAAc,EACd,IAAI,EAGJ,KAAK,UAAU,EAGf,MAAM,EACN,KAAK,KAAK,EACV,KAAK,SAAS,EACd,KAAK,SAAS,EACf,MAAM,YAAY,CAAC;AACpB,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAE7C,OAAO,KAAK,EAAS,UAAU,EAAE,YAAY,EAAe,MAAM,YAAY,CAAC;AAC/E,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAIpD,OAAO,EAGL,SAAS,EACT,cAAc,EACf,MAAM,gBAAgB,CAAC;AAExB;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;OAqDG;IACH,QAAQ,CAAC,kBAAkB,EAAE,CAC3B,OAAO,EAAE,OAAO,EAChB,aAAa,EAAE,cAAc,KAC1B,SAAS,CAAC,OAAO,CAAC,CAAC;CACzB;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;GA2BG;AACH,MAAM,WAAW,OAAO;IACtB,oDAAoD;IACpD,QAAQ,CAAC,OAAO,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,cAAc,CAAC;IAE5D;;;OAGG;IACH,QAAQ,CAAC,WAAW,EAAE,CACpB,OAAO,EAAE,YAAY,EACrB,KAAK,EAAE,cAAc,EACrB,GAAG,EAAE,cAAc,KAChB,WAAW,CAAC;IAEjB;;;;;;;;;OASG;IACH,QAAQ,CAAC,iBAAiB,EAAE,CAC1B,OAAO,EAAE,YAAY,EACrB,OAAO,EAAE,aAAa,CAAC,cAAc,CAAC,EACtC,UAAU,CAAC,EAAE,eAAe,KACzB,aAAa,CAAC,gBAAgB,CAAC,CAAC;IAErC;;;;;;;;;OASG;IACH,QAAQ,CAAC,cAAc,EAAE,CACvB,OAAO,EAAE,YAAY,EACrB,KAAK,EAAE,cAAc,EACrB,GAAG,EAAE,cAAc,EACnB,UAAU,EAAE,eAAe,KACxB,cAAc,CAAC;IAEpB;;;;OAIG;IACH,QAAQ,CAAC,OAAO,EAAE,CAChB,OAAO,EAAE,YAAY,EACrB,KAAK,EAAE,cAAc,EACrB,GAAG,EAAE,cAAc,EACnB,QAAQ,EAAE,CAAC,SAAS,EAAE,cAAc,EAAE,KAAK,EAAE,cAAc,KAAK,OAAO,KACpE,IAAI,CAAC;IAEV;;;;;OAKG;IACH,QAAQ,CAAC,gBAAgB,EAAE,CACzB,OAAO,EAAE,YAAY,EACrB,QAAQ,EAAE,aAAa,KACpB,OAAO,CAAC;IAEb;;;;;;;;;;;OAWG;IACH,QAAQ,CAAC,aAAa,EAAE,CACtB,YAAY,EAAE,YAAY,EAC1B,QAAQ,EAAE,qBAAqB,CAAC,oBAAoB,CAAC,KAClD,IAAI,CAAC,IAAI,EAAE,yBAAyB,CAAC,CAAC;IAE3C;;;;;OAKG;IACH,QAAQ,CAAC,YAAY,EAAE,CACrB,OAAO,EAAE,YAAY,EACrB,SAAS,EAAE,cAAc,KACtB,iBAAiB,CAAC;IAEvB,mDAAmD;IACnD,QAAQ,CAAC,WAAW,EAAE,CAAC,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;CACvD;AAED;;;;GAIG;AACH,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;CAC3B;AAED;;;;GAIG;AACH,MAAM,WAAW,iBACf,SAAQ,UAAU,EAAE,KAAK,CAAC,mBAAmB,CAAC;CAAG;AAEnD;;;;;;;;;;;;;;;GAeG;AACH,MAAM,MAAM,yBAAyB,GAAG,iBAAiB,GAAG,YAAY,CAAC;AAEzE;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,MAAM,WAAW,GAAG,UAAU,GAAG,KAAK,CAAC,aAAa,CAAC,CAAC;AAE5D;;;;GAIG;AACH,eAAO,MAAM,eAAe,kFAA2C,CAAC;AAExE;;;;GAIG;AACH,eAAO,MAAM,eAAe,EAEvB,WAAW,CAAC;AAEjB;;;;GAIG;AACH,MAAM,WAAW,SAAS;IACxB,QAAQ,CAAC,UAAU,EAAE,eAAe,CAAC;CACtC;AAED;;;;;GAKG;AACH,MAAM,MAAM,eAAe,GAAG,cAAc,GAAG,kBAAkB,CAAC;AAElE;;;;GAIG;AACH,eAAO,MAAM,kBAAkB,eAE9B,CAAC;AACF;;;;GAIG;AACH,MAAM,MAAM,kBAAkB,GAAG,OAAO,kBAAkB,CAAC;AAE3D;;;;GAIG;AACH,eAAO,MAAM,SAAS;aACpB,WAAW,EAAE,CAAC;aACd,IAAI,EAAE,CAAC;aACP,UAAU,EAAE,CAAC;CACL,CAAC;AAEX;;;;GAIG;AACH,MAAM,MAAM,SAAS,GAAG,CAAC,OAAO,SAAS,CAAC,CAAC,MAAM,OAAO,SAAS,CAAC,CAAC;AAEnE;;;;GAIG;AACH,MAAM,WAAW,SAAU,SAAQ,SAAS;IAC1C,QAAQ,CAAC,IAAI,EAAE,OAAO,SAAS,CAAC,IAAI,CAAC;CACtC;AAED;;;;GAIG;AACH,MAAM,WAAW,gBAAiB,SAAQ,SAAS;IACjD,QAAQ,CAAC,IAAI,EAAE,OAAO,SAAS,CAAC,WAAW,CAAC;IAC5C,QAAQ,CAAC,WAAW,EAAE,WAAW,CAAC;CACnC;AAED;;;;GAIG;AACH,MAAM,WAAW,eAAgB,SAAQ,SAAS;IAChD,QAAQ,CAAC,IAAI,EAAE,OAAO,SAAS,CAAC,UAAU,CAAC;IAC3C,QAAQ,CAAC,UAAU,EAAE,aAAa,CAAC,cAAc,CAAC,CAAC;CACpD;AAED;;;;;GAKG;AACH,MAAM,MAAM,KAAK,GAAG,SAAS,GAAG,gBAAgB,GAAG,eAAe,CAAC;AAEnE;;;;GAIG;AACH,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,iBAAiB,CAAC;CACpC;AAED;;;;;;;GAOG;AACH,MAAM,MAAM,iBAAiB,GAAG,UAAU,GAAG,KAAK,CAAC,mBAAmB,CAAC,CAAC;AAExE;;;;;;;;GAQG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,SAAS,EAAE,SAAS,CAAC;IAC9B,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC;CAC3B;AAED;;;;GAIG;AACH,eAAO,MAAM,qBAAqB,OAC5B,EAAE,UACE,MAAM,QACR,MAAM,KACX,WAYD,CAAC;AAEH;;;;GAIG;AACH,eAAO,MAAM,cAAc,w+BAA4C,CAAC;AACxE,MAAM,MAAM,cAAc,GAAG,OAAO,cAAc,CAAC,MAAM,CAAC;AAE1D;;;;GAIG;AACH,eAAO,MAAM,mBAAmB,yjCAgB/B,CAAC;AACF,MAAM,MAAM,mBAAmB,GAAG,OAAO,mBAAmB,CAAC,MAAM,CAAC;AAEpE;;;;GAIG;AACH,MAAM,WAAW,wBAAyB,SAAQ,SAAS,CAAC,qBAAqB,CAAC;IAChF,QAAQ,CAAC,KAAK,EAAE,cAAc,CAAC;IAC/B,QAAQ,CAAC,cAAc,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;CAChD;AAED;;;;;GAKG;AACH,eAAO,MAAM,QAAQ,EAAE,UAAU,CAAC;IAChC,QAAQ,CAAC,KAAK,EAAE,OAAO,MAAM,CAAC;IAC9B,QAAQ,CAAC,EAAE,EAAE,OAAO,EAAE,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,OAAO,mBAAmB,CAAC;IAC5C,QAAQ,CAAC,QAAQ,EAAE,OAAO,OAAO,CAAC;IAClC,QAAQ,CAAC,QAAQ,EAAE,SAAS,CAAC,SAAS,CAAC,OAAO,OAAO,EAAE,OAAO,IAAI,CAAC,CAAC,CAAC;CACtE,CAMC,CAAC;AACH,MAAM,WAAW,QAAS,SAAQ,SAAS,CAAC,OAAO,QAAQ,CAAC;CAAG;AAE/D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AACH,MAAM,WAAW,iBAAkB,SAAQ,IAAI,CAC7C,OAAO,EACP,kBAAkB,GAAG,eAAe,GAAG,cAAc,CACtD;IACC;;;;;OAKG;IACH,QAAQ,CAAC,eAAe,EAAE,CACxB,OAAO,EAAE,YAAY,EACrB,SAAS,EAAE,cAAc,EACzB,QAAQ,EAAE,8BAA8B,KACrC,OAAO,CAAC;IAEb;;;OAGG;IACH,QAAQ,CAAC,qBAAqB,EAAE,CAC9B,YAAY,EAAE,YAAY,EAC1B,eAAe,EAAE,qBAAqB,CAAC,cAAc,CAAC,KACnD,aAAa,CAAC,cAAc,CAAC,CAAC;CACpC;AAED;;;;GAIG;AACH,MAAM,WAAW,oBAAoB;IACnC,QAAQ,CAAC,iBAAiB,EAAE,iBAAiB,CAAC;CAC/C;AAED;;;;GAIG;AACH,MAAM,MAAM,iBAAiB,GAAG,SAAS,GAAG,SAAS,CAAC;AAEtD;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,uBAAuB,SAC5B,iBAAiB,KACtB,iBAkGD,CAAC;AAMH;;;;GAIG;AACH,eAAO,MAAM,6BAA6B,SAAU,SAAS,KAAG,IAgE/D,CAAC;AAEF;;;;GAIG;AACH,MAAM,MAAM,8BAA8B,GAAG,QAAQ,GAAG,SAAS,GAAG,QAAQ,CAAC;AAE7E;;;;;;;GAOG;AACH,eAAO,MAAM,0BAA0B,cAC1B,cAAc,kBACT,cAAc,iBACf,cAAc,KAC5B,CACD,QAAQ,EAAE,8BAA8B,EACxC,cAAc,EAAE,cAAc,EAC9B,aAAa,EAAE,cAAc,CAS9B,CAAC;AAoeF;;;;GAIG;AACH,eAAO,MAAM,2BAA2B,cAC3B,cAAc,KACxB,WAGF,CAAC;AAEF;;;;;GAKG;AACH,eAAO,MAAM,yBAAyB,eACxB,aAAa,CAAC,cAAc,CAAC,KACxC,WAUE,CAAC;AAqYN;;;;;;GAMG;AACH,eAAO,MAAM,mBAAmB,SACvB,SAAS,eACN,YAAY,SAAS,cAAc,KAAG,cAgF/C,CAAC;AAEJ;;;;GAIG;AACH,eAAO,MAAM,uBAAuB,SAC3B,SAAS,oBAEA,YAAY,oBACR,cAAc,KAC/B;IACD,QAAQ,CAAC,WAAW,EAAE,cAAc,GAAG,IAAI,CAAC;IAC5C,QAAQ,CAAC,cAAc,EAAE,cAAc,CAAC;IACxC,QAAQ,CAAC,aAAa,EAAE,cAAc,CAAC;CA6BxC,CAAC;AAEJ;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB,SACpB,SAAS,oBAEA,YAAY,eACb,cAAc,kBACX,cAAc,iBACf,cAAc,KAC5B,IAYF,CAAC"}
|
|
@@ -119,6 +119,10 @@ export const createBaseSqliteStorage = (deps) => ({
|
|
|
119
119
|
// A batch can hold hundreds of thousands of timestamps, too many to spread
|
|
120
120
|
// into concatBytes.
|
|
121
121
|
const concatenatedTimestamps = concatByteArrays(timestampsBytes);
|
|
122
|
+
// A batch has no known size, so unlike the CTEs bounded by
|
|
123
|
+
// skiplistMaxLevel, the planner would make the owner's timestamps the outer
|
|
124
|
+
// loop. A cross join keeps the order as written, so each timestamp is
|
|
125
|
+
// looked up by primary key.
|
|
122
126
|
const result = deps.sqlite.exec(sql `
|
|
123
127
|
with recursive
|
|
124
128
|
split_timestamps(timestampBytes, pos) as (
|
|
@@ -135,7 +139,7 @@ export const createBaseSqliteStorage = (deps) => ({
|
|
|
135
139
|
select s.timestampBytes
|
|
136
140
|
from
|
|
137
141
|
split_timestamps s
|
|
138
|
-
join evolu_timestamp t
|
|
142
|
+
cross join evolu_timestamp t
|
|
139
143
|
on t.ownerId = ${ownerIdBytes} and s.timestampBytes = t.t;
|
|
140
144
|
`);
|
|
141
145
|
return result.rows.map((row) => row.timestampBytes);
|
|
@@ -183,6 +187,9 @@ export const createBaseSqliteStorage = (deps) => ({
|
|
|
183
187
|
deleteOwner: (ownerId) => {
|
|
184
188
|
deps.sqlite.exec(sql `
|
|
185
189
|
delete from evolu_timestamp where ownerId = ${ownerId};
|
|
190
|
+
`);
|
|
191
|
+
deps.sqlite.exec(sql `
|
|
192
|
+
delete from evolu_usage where ownerId = ${ownerId};
|
|
186
193
|
`);
|
|
187
194
|
},
|
|
188
195
|
});
|
|
@@ -956,6 +963,8 @@ const fingerprint = (deps) => (ownerId, begin, end) => {
|
|
|
956
963
|
*/
|
|
957
964
|
const fingerprintRanges = (deps) => (ownerId, buckets, upperBound = InfiniteUpperBound) => {
|
|
958
965
|
const bucketsJson = JSON.stringify(buckets);
|
|
966
|
+
// A bucket past the owner's timestamps would descend below level 1
|
|
967
|
+
// forever, so the walk stops there and the bucket has no row.
|
|
959
968
|
const result = deps.sqlite.exec(sql.prepared `
|
|
960
969
|
with
|
|
961
970
|
ml(ml) as (
|
|
@@ -1021,7 +1030,7 @@ const fingerprintRanges = (deps) => (ownerId, buckets, upperBound = InfiniteUppe
|
|
|
1021
1030
|
c1
|
|
1022
1031
|
left join evolu_timestamp as node
|
|
1023
1032
|
on not c1.b and node.ownerId = ${ownerId} and node.t = c1.nt
|
|
1024
|
-
where iif(c1.b, 1, c1.ic != c1.c)
|
|
1033
|
+
where iif(c1.b, 1, c1.ic != c1.c) and c1.dl >= 1
|
|
1025
1034
|
),
|
|
1026
1035
|
c2(h1, h2, t, rn) as (
|
|
1027
1036
|
select
|
|
@@ -1053,6 +1062,7 @@ const fingerprintRanges = (deps) => (ownerId, buckets, upperBound = InfiniteUppe
|
|
|
1053
1062
|
select b, cast(h1 as text) as h1, cast(h2 as text) as h2
|
|
1054
1063
|
from c3;
|
|
1055
1064
|
`);
|
|
1065
|
+
assert(result.rows.length === buckets.length, "bucket out of range");
|
|
1056
1066
|
const fingerprintRanges = result.rows.map((row, i, arr) => ({
|
|
1057
1067
|
type: RangeType.Fingerprint,
|
|
1058
1068
|
upperBound: i === arr.length - 1 ? upperBound : row.b,
|
|
@@ -1068,9 +1078,13 @@ const x = (a, b) => sql.raw(`(${a} | ${b}) - (${a} & ${b})`);
|
|
|
1068
1078
|
/**
|
|
1069
1079
|
* Reads the timestamp at a position within an owner's ordered timestamps.
|
|
1070
1080
|
*
|
|
1081
|
+
* Throws when `index` is not below the owner's size.
|
|
1082
|
+
*
|
|
1071
1083
|
* @group SQLite
|
|
1072
1084
|
*/
|
|
1073
1085
|
export const getTimestampByIndex = (deps) => (ownerId, index) => {
|
|
1086
|
+
// An index past the owner's timestamps would descend below level 1
|
|
1087
|
+
// forever, so the walk stops there and finds no row.
|
|
1074
1088
|
const result = deps.sqlite.exec(sql.prepared `
|
|
1075
1089
|
with
|
|
1076
1090
|
fi(b, cl, ic, pt, mt, nt, nc) as (
|
|
@@ -1136,13 +1150,15 @@ export const getTimestampByIndex = (deps) => (ownerId, index) => {
|
|
|
1136
1150
|
)
|
|
1137
1151
|
)
|
|
1138
1152
|
from fi
|
|
1139
|
-
where ic != ${index + 1}
|
|
1153
|
+
where ic != ${index + 1} and cl >= 1
|
|
1140
1154
|
)
|
|
1141
1155
|
select pt
|
|
1142
1156
|
from fi
|
|
1143
1157
|
where ic == ${index + 1};
|
|
1144
1158
|
`);
|
|
1145
|
-
|
|
1159
|
+
const row = result.rows.at(0);
|
|
1160
|
+
assert(row, "index out of range");
|
|
1161
|
+
return row.pt;
|
|
1146
1162
|
};
|
|
1147
1163
|
/**
|
|
1148
1164
|
* Reads owner usage from SQLite and returns default bounds when absent.
|