@syncular/server 0.59.0 → 0.60.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +20 -16
- package/dist/admin.d.ts +32 -20
- package/dist/admin.js +42 -40
- package/dist/context.d.ts +6 -8
- package/dist/context.js +1 -3
- package/dist/d1-storage.js +42 -24
- package/dist/errors.js +8 -0
- package/dist/events.d.ts +35 -17
- package/dist/frame-bytes.js +24 -6
- package/dist/handler.js +486 -223
- package/dist/local-authority.js +7 -9
- package/dist/operations.d.ts +5 -2
- package/dist/operations.js +20 -29
- package/dist/postgres-storage.d.ts +24 -19
- package/dist/postgres-storage.js +249 -168
- package/dist/pull.d.ts +68 -53
- package/dist/pull.js +258 -287
- package/dist/push.d.ts +12 -3
- package/dist/push.js +95 -32
- package/dist/realtime.d.ts +17 -11
- package/dist/realtime.js +65 -90
- package/dist/relational-rows.d.ts +0 -57
- package/dist/relational-rows.js +4 -100
- package/dist/seed.js +10 -15
- package/dist/segment-download.d.ts +4 -1
- package/dist/segment-download.js +4 -2
- package/dist/sqlite-dialect.d.ts +36 -59
- package/dist/sqlite-dialect.js +255 -159
- package/dist/sqlite-storage.js +24 -20
- package/dist/storage-query.d.ts +30 -1
- package/dist/storage-query.js +68 -0
- package/dist/storage.d.ts +156 -54
- package/dist/storage.js +15 -0
- package/package.json +2 -2
- package/src/admin.ts +87 -59
- package/src/context.ts +7 -11
- package/src/d1-storage.ts +49 -33
- package/src/errors.ts +8 -0
- package/src/events.ts +36 -17
- package/src/frame-bytes.ts +26 -6
- package/src/handler.ts +628 -286
- package/src/local-authority.ts +10 -11
- package/src/operations.ts +44 -42
- package/src/postgres-storage.ts +336 -226
- package/src/pull.ts +394 -414
- package/src/push.ts +116 -21
- package/src/realtime.ts +85 -129
- package/src/relational-rows.ts +4 -107
- package/src/seed.ts +13 -19
- package/src/segment-download.ts +8 -3
- package/src/sqlite-dialect.ts +363 -204
- package/src/sqlite-storage.ts +45 -25
- package/src/storage-query.ts +102 -0
- package/src/storage.ts +185 -58
package/dist/sqlite-dialect.js
CHANGED
|
@@ -2,10 +2,10 @@ import { bindAuthoritativePartition, prepareAuthoritativeQuery, } from './author
|
|
|
2
2
|
import { syncError } from './errors.js';
|
|
3
3
|
import { validateCommitPruneQuery } from './prune.js';
|
|
4
4
|
import { StorageQueryError } from './storage-errors.js';
|
|
5
|
-
import { deleteRowSql, deleteSqliteRowScopesSql, indexRowPageStatement, quoteIdent, scanRowPageSql, rowsReferencingBlobSql, selectRowSql, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN,
|
|
5
|
+
import { deleteRowSql, deleteSqliteRowScopesSql, indexRowPageStatement, quoteIdent, scanRowPageSql, rowsReferencingBlobSql, selectRowSql, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, upsertSql, upsertValues, } from './relational-rows.js';
|
|
6
6
|
import { matchesEffective } from './scopes.js';
|
|
7
7
|
import { clientRoundState, PARTITION_TOUCH_INTERVAL_MS, rowsLiveAfterDelete, } from './storage.js';
|
|
8
|
-
import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js';
|
|
8
|
+
import { assertScopeIndexedScan, readLiveWindowPages, resolveIndexRowScan, } from './storage-query.js';
|
|
9
9
|
/**
|
|
10
10
|
* Schema DDL, one statement per `;`-delimited chunk. Native SQLite applies
|
|
11
11
|
* the whole string via `db.exec(SQLITE_DDL)`; D1 applies each statement
|
|
@@ -19,21 +19,15 @@ import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js'
|
|
|
19
19
|
* the Postgres storage documents (§3.1, performance-by-
|
|
20
20
|
* construction).
|
|
21
21
|
*
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
* in `row_id` order and filtering, instead of seeking the PRIMARY KEY once
|
|
25
|
-
* per value. The per-row scope replacement (`deleteSqliteRowScopesSql`)
|
|
26
|
-
* binds `(partition, tbl, var, value, row_id)` and needs no secondary index.
|
|
27
|
-
* `DROP INDEX IF EXISTS` stays in the DDL so existing databases lose the
|
|
28
|
-
* index the next time the DDL runs: on every `SqliteServerStorage`
|
|
29
|
-
* construction, and on D1 in `migrate()` and the core phase of
|
|
30
|
-
* `migrateSchema`.
|
|
22
|
+
* Changes to the core tables of an existing database (added and dropped
|
|
23
|
+
* columns and indexes) live in `sqliteUpgradeDdl`, which runs after this DDL.
|
|
31
24
|
*/
|
|
32
25
|
export const SQLITE_DDL = `
|
|
33
26
|
CREATE TABLE IF NOT EXISTS sync_partitions(
|
|
34
27
|
partition TEXT PRIMARY KEY,
|
|
35
28
|
max_commit_seq INTEGER NOT NULL DEFAULT 0,
|
|
36
|
-
horizon_seq INTEGER NOT NULL DEFAULT 0
|
|
29
|
+
horizon_seq INTEGER NOT NULL DEFAULT 0,
|
|
30
|
+
delete_floor INTEGER NOT NULL DEFAULT 0
|
|
37
31
|
);
|
|
38
32
|
CREATE TABLE IF NOT EXISTS sync_partition_registry(
|
|
39
33
|
partition TEXT PRIMARY KEY,
|
|
@@ -46,7 +40,6 @@ CREATE TABLE IF NOT EXISTS sync_row_scopes(
|
|
|
46
40
|
var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,
|
|
47
41
|
PRIMARY KEY(partition, tbl, var, value, row_id)
|
|
48
42
|
);
|
|
49
|
-
DROP INDEX IF EXISTS sync_row_scopes_by_row;
|
|
50
43
|
CREATE TABLE IF NOT EXISTS sync_commits(
|
|
51
44
|
partition TEXT NOT NULL, commit_seq INTEGER NOT NULL,
|
|
52
45
|
client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,
|
|
@@ -72,8 +65,14 @@ CREATE TABLE IF NOT EXISTS sync_change_scopes(
|
|
|
72
65
|
CREATE TABLE IF NOT EXISTS sync_push_results(
|
|
73
66
|
partition TEXT NOT NULL, client_id TEXT NOT NULL,
|
|
74
67
|
client_commit_id TEXT NOT NULL, result TEXT NOT NULL,
|
|
68
|
+
outbox_id TEXT, client_seq INTEGER,
|
|
75
69
|
PRIMARY KEY(partition, client_id, client_commit_id)
|
|
76
70
|
);
|
|
71
|
+
CREATE TABLE IF NOT EXISTS sync_push_watermarks(
|
|
72
|
+
partition TEXT NOT NULL, client_id TEXT NOT NULL, outbox_id TEXT NOT NULL,
|
|
73
|
+
through_seq INTEGER NOT NULL,
|
|
74
|
+
PRIMARY KEY(partition, client_id, outbox_id)
|
|
75
|
+
);
|
|
77
76
|
CREATE TABLE IF NOT EXISTS sync_reactions(
|
|
78
77
|
partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,
|
|
79
78
|
type TEXT NOT NULL, version INTEGER NOT NULL, payload TEXT NOT NULL,
|
|
@@ -97,10 +96,12 @@ CREATE INDEX IF NOT EXISTS sync_reactions_dead_letter
|
|
|
97
96
|
CREATE TABLE IF NOT EXISTS sync_clients(
|
|
98
97
|
partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,
|
|
99
98
|
wire_version INTEGER NOT NULL DEFAULT 1,
|
|
100
|
-
cursor INTEGER NOT NULL,
|
|
99
|
+
cursor INTEGER NOT NULL,
|
|
101
100
|
updated_at_ms INTEGER NOT NULL,
|
|
102
101
|
last_round_at_ms INTEGER, last_error_code TEXT,
|
|
103
102
|
consecutive_failures INTEGER NOT NULL DEFAULT 0,
|
|
103
|
+
schema_version INTEGER, units TEXT, registration_digest BLOB,
|
|
104
|
+
catching_up TEXT NOT NULL DEFAULT '[]',
|
|
104
105
|
PRIMARY KEY(partition, client_id)
|
|
105
106
|
);
|
|
106
107
|
CREATE TABLE IF NOT EXISTS sync_blob_refs(
|
|
@@ -130,15 +131,97 @@ CREATE TABLE IF NOT EXISTS sync_writer_fence(
|
|
|
130
131
|
required_writer_version INTEGER NOT NULL
|
|
131
132
|
);
|
|
132
133
|
`;
|
|
133
|
-
/**
|
|
134
|
-
|
|
134
|
+
/**
|
|
135
|
+
* Columns added to core tables after their first release: table, column,
|
|
136
|
+
* type. A client record from before wire version 4 gains NULL `units` and
|
|
137
|
+
* `registration_digest`, so it holds no registration list (§9), and a push
|
|
138
|
+
* result gains NULL `outbox_id` and `client_seq`: it is a result stored
|
|
139
|
+
* before the upgrade, settled by the client's next push (§9). The
|
|
140
|
+
* watermark delete and the settlement range over the `(partition,
|
|
141
|
+
* client_id)` prefix of the primary key, which the watermark keeps to one
|
|
142
|
+
* request's results per outbox.
|
|
143
|
+
*/
|
|
144
|
+
const SQLITE_ADDED_COLUMNS = [
|
|
135
145
|
['sync_clients', 'wire_version', 'INTEGER NOT NULL DEFAULT 1'],
|
|
136
146
|
['sync_clients', 'last_round_at_ms', 'INTEGER'],
|
|
137
147
|
['sync_clients', 'last_error_code', 'TEXT'],
|
|
138
148
|
['sync_clients', 'consecutive_failures', 'INTEGER NOT NULL DEFAULT 0'],
|
|
149
|
+
['sync_clients', 'schema_version', 'INTEGER'],
|
|
150
|
+
['sync_clients', 'units', 'TEXT'],
|
|
151
|
+
['sync_clients', 'registration_digest', 'BLOB'],
|
|
152
|
+
['sync_clients', 'catching_up', "TEXT NOT NULL DEFAULT '[]'"],
|
|
139
153
|
['sync_tombstones', 'server_version', 'INTEGER'],
|
|
140
154
|
['sync_tombstones', 'scopes', 'TEXT'],
|
|
155
|
+
['sync_push_results', 'outbox_id', 'TEXT'],
|
|
156
|
+
['sync_push_results', 'client_seq', 'INTEGER'],
|
|
157
|
+
['sync_partitions', 'delete_floor', 'INTEGER NOT NULL DEFAULT 0'],
|
|
141
158
|
];
|
|
159
|
+
/** Core-table columns wire version 4 deleted: the version 3 subscription list. */
|
|
160
|
+
const SQLITE_DROPPED_COLUMNS = [['sync_clients', 'subscriptions']];
|
|
161
|
+
/**
|
|
162
|
+
* Indexes that `sqliteUpgradeDdl` creates, on a fresh database and an
|
|
163
|
+
* upgraded one alike. `sync_change_scopes_by_key` serves the §4.5 live
|
|
164
|
+
* scan: one range per (variable, value) across tables.
|
|
165
|
+
*/
|
|
166
|
+
const SQLITE_ADDED_INDEXES = [
|
|
167
|
+
[
|
|
168
|
+
'sync_change_scopes_by_key',
|
|
169
|
+
'CREATE INDEX IF NOT EXISTS sync_change_scopes_by_key ON sync_change_scopes(partition, var, value, commit_seq)',
|
|
170
|
+
],
|
|
171
|
+
];
|
|
172
|
+
/**
|
|
173
|
+
* Indexes that `sqliteUpgradeDdl` drops. With `sync_row_scopes_by_row`
|
|
174
|
+
* present the planner answered a scope scan of two or more values by walking
|
|
175
|
+
* it in `row_id` order and filtering, instead of seeking the PRIMARY KEY once
|
|
176
|
+
* per value; the per-row scope replacement (`deleteSqliteRowScopesSql`)
|
|
177
|
+
* binds the full key and needs no secondary index.
|
|
178
|
+
*/
|
|
179
|
+
const SQLITE_DROPPED_INDEXES = ['sync_row_scopes_by_row'];
|
|
180
|
+
/**
|
|
181
|
+
* One statement that reads what `sqliteUpgradeDdl` needs: the columns of
|
|
182
|
+
* every table with added or dropped columns, and which added indexes exist.
|
|
183
|
+
*/
|
|
184
|
+
export const SQLITE_UPGRADE_PROBE = {
|
|
185
|
+
sql: `SELECT t.value AS tbl, c.name AS name
|
|
186
|
+
FROM json_each(?) AS t, pragma_table_info(t.value) AS c
|
|
187
|
+
UNION ALL
|
|
188
|
+
SELECT '' AS tbl, name FROM sqlite_master
|
|
189
|
+
WHERE type='index' AND name IN (SELECT value FROM json_each(?))`,
|
|
190
|
+
params: [
|
|
191
|
+
JSON.stringify([
|
|
192
|
+
...new Set([...SQLITE_ADDED_COLUMNS, ...SQLITE_DROPPED_COLUMNS].map(([table]) => table)),
|
|
193
|
+
]),
|
|
194
|
+
JSON.stringify([
|
|
195
|
+
...SQLITE_ADDED_INDEXES.map(([name]) => name),
|
|
196
|
+
...SQLITE_DROPPED_INDEXES,
|
|
197
|
+
]),
|
|
198
|
+
],
|
|
199
|
+
};
|
|
200
|
+
/**
|
|
201
|
+
* The statements that bring the core tables of a database from an earlier
|
|
202
|
+
* release to the current DDL, given the `SQLITE_UPGRADE_PROBE` rows: missing
|
|
203
|
+
* columns, dropped columns, dropped indexes, then missing indexes. Empty for
|
|
204
|
+
* a current database. Run after `SQLITE_DDL`: on every `SqliteServerStorage`
|
|
205
|
+
* construction, and on D1 in `migrate()`, the core phase of `migrateSchema`
|
|
206
|
+
* and the first open of a database at the current schema version.
|
|
207
|
+
*/
|
|
208
|
+
export function sqliteUpgradeDdl(present) {
|
|
209
|
+
const has = (table, name) => present.some((row) => row.tbl === table && row.name === name);
|
|
210
|
+
return [
|
|
211
|
+
...SQLITE_ADDED_COLUMNS.filter(([table, name]) => !has(table, name)).map(([table, name, definition]) => `ALTER TABLE ${table} ADD COLUMN ${name} ${definition}`),
|
|
212
|
+
// A database from before wire version 4 (and the delete floor) starts
|
|
213
|
+
// the delete floor at the highest version its tombstones recorded
|
|
214
|
+
// (§2.2) and the horizon at its last commit (§9).
|
|
215
|
+
...(has('sync_partitions', 'delete_floor')
|
|
216
|
+
? []
|
|
217
|
+
: [
|
|
218
|
+
`UPDATE sync_partitions SET horizon_seq=max_commit_seq, delete_floor=coalesce((SELECT max(server_version) FROM sync_tombstones t WHERE t.partition=sync_partitions.partition),0)`,
|
|
219
|
+
]),
|
|
220
|
+
...SQLITE_DROPPED_COLUMNS.filter(([table, name]) => has(table, name)).map(([table, name]) => `ALTER TABLE ${table} DROP COLUMN ${name}`),
|
|
221
|
+
...SQLITE_DROPPED_INDEXES.filter((name) => has('', name)).map((name) => `DROP INDEX IF EXISTS ${name}`),
|
|
222
|
+
...SQLITE_ADDED_INDEXES.filter(([name]) => !has('', name)).map(([, ddl]) => ddl),
|
|
223
|
+
];
|
|
224
|
+
}
|
|
142
225
|
/** Split the DDL into individual statements (D1 applies them one by one). */
|
|
143
226
|
export function sqliteDdlStatements() {
|
|
144
227
|
return SQLITE_DDL.split(';')
|
|
@@ -297,77 +380,6 @@ export function toStoredChange(record) {
|
|
|
297
380
|
: {}),
|
|
298
381
|
};
|
|
299
382
|
}
|
|
300
|
-
/**
|
|
301
|
-
* Fold one `commitWindowPageSql` page into commits, preserving the exact
|
|
302
|
-
* semantics of the old per-candidate loop:
|
|
303
|
-
*
|
|
304
|
-
* - rows arrive ordered (commit_seq, idx); consecutive rows with the same
|
|
305
|
-
* `commit_seq` are one candidate commit;
|
|
306
|
-
* - every candidate advances the cursor (`lastSeq`) — including vanished
|
|
307
|
-
* commits (NULL meta) and commits whose changes all fail the exact
|
|
308
|
-
* multi-variable scope verification (`matchesEffective`);
|
|
309
|
-
* - a commit is emitted only with its matching changes, in `idx` order;
|
|
310
|
-
* - stops after the commit that accumulates at least `remaining` matching
|
|
311
|
-
* changes (never splitting a commit).
|
|
312
|
-
*
|
|
313
|
-
* `candidateCount` counts the processed candidates so the caller can detect
|
|
314
|
-
* a short page (window exhausted) exactly as it did with the old candidate
|
|
315
|
-
* query.
|
|
316
|
-
*/
|
|
317
|
-
export function collectCommitWindowPage(records, scopeFilter, remaining) {
|
|
318
|
-
const commits = [];
|
|
319
|
-
let delivered = 0;
|
|
320
|
-
let lastSeq = 0;
|
|
321
|
-
let candidateCount = 0;
|
|
322
|
-
let i = 0;
|
|
323
|
-
while (i < records.length) {
|
|
324
|
-
const head = records[i];
|
|
325
|
-
if (head === undefined)
|
|
326
|
-
break;
|
|
327
|
-
const seq = head.commit_seq;
|
|
328
|
-
candidateCount += 1;
|
|
329
|
-
lastSeq = seq;
|
|
330
|
-
const changes = [];
|
|
331
|
-
for (; i < records.length; i++) {
|
|
332
|
-
const record = records[i];
|
|
333
|
-
if (record === undefined || record.commit_seq !== seq)
|
|
334
|
-
break;
|
|
335
|
-
// NULL tbl: a candidate with no change rows for the table (LEFT JOIN
|
|
336
|
-
// contract) — it still advanced the cursor above.
|
|
337
|
-
if (record.tbl === null ||
|
|
338
|
-
record.row_id === null ||
|
|
339
|
-
record.op === null ||
|
|
340
|
-
record.scopes === null) {
|
|
341
|
-
continue;
|
|
342
|
-
}
|
|
343
|
-
const change = toStoredChange({
|
|
344
|
-
tbl: record.tbl,
|
|
345
|
-
row_id: record.row_id,
|
|
346
|
-
op: record.op,
|
|
347
|
-
row_version: record.row_version,
|
|
348
|
-
scopes: record.scopes,
|
|
349
|
-
payload: record.payload,
|
|
350
|
-
});
|
|
351
|
-
if (matchesEffective(change.scopes, scopeFilter))
|
|
352
|
-
changes.push(change);
|
|
353
|
-
}
|
|
354
|
-
// NULL meta: the commit vanished — cursor advanced, nothing emitted.
|
|
355
|
-
if (head.actor_id === null || head.created_at_ms === null)
|
|
356
|
-
continue;
|
|
357
|
-
if (changes.length === 0)
|
|
358
|
-
continue;
|
|
359
|
-
commits.push({
|
|
360
|
-
commitSeq: seq,
|
|
361
|
-
createdAtMs: head.created_at_ms,
|
|
362
|
-
actorId: head.actor_id,
|
|
363
|
-
changes,
|
|
364
|
-
});
|
|
365
|
-
delivered += changes.length;
|
|
366
|
-
if (delivered >= remaining)
|
|
367
|
-
break;
|
|
368
|
-
}
|
|
369
|
-
return { commits, delivered, lastSeq, candidateCount };
|
|
370
|
-
}
|
|
371
383
|
function toStoredReaction(record) {
|
|
372
384
|
return {
|
|
373
385
|
idempotencyKey: record.idempotency_key,
|
|
@@ -394,6 +406,7 @@ function toStoredReaction(record) {
|
|
|
394
406
|
: {}),
|
|
395
407
|
};
|
|
396
408
|
}
|
|
409
|
+
const CLIENT_RECORD_COLUMNS = 'client_id, actor_id, wire_version, cursor, updated_at_ms, schema_version, units, registration_digest, catching_up, last_round_at_ms, last_error_code, consecutive_failures';
|
|
397
410
|
function toClientRecord(record) {
|
|
398
411
|
return {
|
|
399
412
|
clientId: record.client_id,
|
|
@@ -401,7 +414,18 @@ function toClientRecord(record) {
|
|
|
401
414
|
wireVersion: record.wire_version,
|
|
402
415
|
cursor: record.cursor,
|
|
403
416
|
updatedAtMs: record.updated_at_ms,
|
|
404
|
-
|
|
417
|
+
...(record.schema_version !== null
|
|
418
|
+
? { schemaVersion: record.schema_version }
|
|
419
|
+
: {}),
|
|
420
|
+
...(record.units !== null && record.registration_digest !== null
|
|
421
|
+
? {
|
|
422
|
+
registration: {
|
|
423
|
+
units: JSON.parse(record.units),
|
|
424
|
+
digest: asUint8Array(record.registration_digest),
|
|
425
|
+
},
|
|
426
|
+
}
|
|
427
|
+
: {}),
|
|
428
|
+
catchingUp: JSON.parse(record.catching_up),
|
|
405
429
|
...clientRoundState(record.last_round_at_ms, record.last_error_code, record.consecutive_failures),
|
|
406
430
|
};
|
|
407
431
|
}
|
|
@@ -739,55 +763,46 @@ export class SqliteFamilyStorage {
|
|
|
739
763
|
};
|
|
740
764
|
}
|
|
741
765
|
async latestScopedChangeSeq(partition, query) {
|
|
742
|
-
const
|
|
743
|
-
const
|
|
744
|
-
|
|
745
|
-
: (query.scopeFilter[firstVariable] ?? []);
|
|
746
|
-
if (firstVariable === undefined || values.length === 0)
|
|
766
|
+
const variable = Object.keys(query.scopes).sort()[0];
|
|
767
|
+
const value = variable === undefined ? undefined : query.scopes[variable];
|
|
768
|
+
if (variable === undefined || value === undefined)
|
|
747
769
|
return 0;
|
|
748
|
-
|
|
770
|
+
// One index-only max over the (partition, tbl, var, value) prefix.
|
|
771
|
+
const [row] = await this.guardedAll('SELECT max(commit_seq) AS seq FROM sync_change_scopes WHERE partition=? AND tbl=? AND var=? AND value=?', [partition, query.table, variable, value]);
|
|
749
772
|
return row?.seq ?? 0;
|
|
750
773
|
}
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
const page = collectCommitWindowPage(records, query.scopeFilter, query.limitChanges - deliveredChanges);
|
|
784
|
-
commits.push(...page.commits);
|
|
785
|
-
deliveredChanges += page.delivered;
|
|
786
|
-
afterSeq = page.lastSeq;
|
|
787
|
-
if (page.candidateCount < batchSize)
|
|
788
|
-
break;
|
|
789
|
-
}
|
|
790
|
-
return commits;
|
|
774
|
+
readLiveWindow(partition, query) {
|
|
775
|
+
// One statement per page: the unit keys bind as one JSON array, and the
|
|
776
|
+
// CROSS JOIN keeps them the outer loop, so each key seeks one range of
|
|
777
|
+
// `sync_change_scopes_by_key` (INDEXED BY fails loudly when the index is
|
|
778
|
+
// missing). The candidates LEFT JOIN the commit meta and every change of
|
|
779
|
+
// the commit; a candidate whose commit vanished comes back with NULL
|
|
780
|
+
// meta and still advances the scan.
|
|
781
|
+
return readLiveWindowPages(query, async (keys, limit) => (await this.guardedAll(`SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
782
|
+
ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
783
|
+
FROM (SELECT DISTINCT s.commit_seq AS commit_seq
|
|
784
|
+
FROM json_each(?2) AS k
|
|
785
|
+
CROSS JOIN sync_change_scopes AS s INDEXED BY sync_change_scopes_by_key
|
|
786
|
+
ON s.partition=?1 AND s.var=json_extract(k.value, '$.var')
|
|
787
|
+
AND s.value=json_extract(k.value, '$.value')
|
|
788
|
+
AND s.commit_seq>json_extract(k.value, '$.after') AND s.commit_seq<=?3
|
|
789
|
+
ORDER BY s.commit_seq LIMIT ?4) AS c
|
|
790
|
+
LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=c.commit_seq
|
|
791
|
+
LEFT JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.commit_seq
|
|
792
|
+
ORDER BY c.commit_seq, ch.idx`, [partition, JSON.stringify(keys), query.throughSeq, limit])).map((record) => ({
|
|
793
|
+
commitSeq: record.commit_seq,
|
|
794
|
+
...(record.actor_id !== null && record.created_at_ms !== null
|
|
795
|
+
? {
|
|
796
|
+
meta: {
|
|
797
|
+
actorId: record.actor_id,
|
|
798
|
+
createdAtMs: record.created_at_ms,
|
|
799
|
+
},
|
|
800
|
+
}
|
|
801
|
+
: {}),
|
|
802
|
+
...(record.tbl !== null
|
|
803
|
+
? { change: toStoredChange({ ...record, tbl: record.tbl }) }
|
|
804
|
+
: {}),
|
|
805
|
+
})));
|
|
791
806
|
}
|
|
792
807
|
async scanRows(partition, query) {
|
|
793
808
|
const firstVariable = assertScopeIndexedScan(query);
|
|
@@ -837,21 +852,28 @@ export class SqliteFamilyStorage {
|
|
|
837
852
|
return (await this.guardedAll(statement.sql, statement.params)).map(toStoredRow);
|
|
838
853
|
}
|
|
839
854
|
async getClientRecord(partition, clientId) {
|
|
840
|
-
const [record] = await this.all(
|
|
855
|
+
const [record] = await this.all(`SELECT ${CLIENT_RECORD_COLUMNS} FROM sync_clients WHERE partition=? AND client_id=?`, [partition, clientId]);
|
|
841
856
|
return record === undefined ? undefined : toClientRecord(record);
|
|
842
857
|
}
|
|
843
858
|
async putClientRecord(partition, record, options) {
|
|
844
859
|
// One row per (partition, client_id): every non-key field is written
|
|
845
|
-
// from the new record
|
|
846
|
-
//
|
|
847
|
-
|
|
860
|
+
// from the new record, except the registration list and digest, which
|
|
861
|
+
// only a traveling list replaces (§4.5). `keepRoundOutcome` leaves the
|
|
862
|
+
// round-state columns alone (§1.5, §4.5).
|
|
863
|
+
const registration = record.registration;
|
|
864
|
+
await this.all(`INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, updated_at_ms, last_round_at_ms, schema_version, units, registration_digest, catching_up) VALUES (?,?,?,?,?,?,?,?,?,?,?)
|
|
848
865
|
ON CONFLICT(partition, client_id) DO UPDATE SET
|
|
849
866
|
actor_id=excluded.actor_id,
|
|
850
867
|
wire_version=excluded.wire_version,
|
|
851
868
|
cursor=excluded.cursor,
|
|
852
|
-
subscriptions=excluded.subscriptions,
|
|
853
869
|
updated_at_ms=excluded.updated_at_ms,
|
|
854
|
-
last_round_at_ms=excluded.last_round_at_ms
|
|
870
|
+
last_round_at_ms=excluded.last_round_at_ms,
|
|
871
|
+
schema_version=excluded.schema_version,${registration === undefined
|
|
872
|
+
? ''
|
|
873
|
+
: `
|
|
874
|
+
units=excluded.units,
|
|
875
|
+
registration_digest=excluded.registration_digest,`}
|
|
876
|
+
catching_up=excluded.catching_up${options?.keepRoundOutcome === true
|
|
855
877
|
? ''
|
|
856
878
|
: `,
|
|
857
879
|
last_error_code=NULL,
|
|
@@ -861,9 +883,12 @@ export class SqliteFamilyStorage {
|
|
|
861
883
|
record.actorId,
|
|
862
884
|
record.wireVersion,
|
|
863
885
|
record.cursor,
|
|
864
|
-
JSON.stringify(record.subscriptions),
|
|
865
|
-
record.updatedAtMs,
|
|
866
886
|
record.updatedAtMs,
|
|
887
|
+
record.lastRoundAtMs ?? record.updatedAtMs,
|
|
888
|
+
record.schemaVersion ?? null,
|
|
889
|
+
registration === undefined ? null : JSON.stringify(registration.units),
|
|
890
|
+
registration?.digest ?? null,
|
|
891
|
+
JSON.stringify(record.catchingUp),
|
|
867
892
|
]);
|
|
868
893
|
}
|
|
869
894
|
async recordClientRoundFailure(partition, clientId, actorId, errorCode, atMs) {
|
|
@@ -873,16 +898,17 @@ export class SqliteFamilyStorage {
|
|
|
873
898
|
WHERE partition=? AND client_id=? AND actor_id=?`, [atMs, errorCode, partition, clientId, actorId]);
|
|
874
899
|
}
|
|
875
900
|
async advanceClientCursor(partition, clientId, actorId, logEpoch, cursor, updatedAtMs) {
|
|
901
|
+
// §8.2: the ack never raises the record cursor above a catching-up
|
|
902
|
+
// unit's cursor, so pruning keeps that unit's history (§4.6).
|
|
876
903
|
await this.all(`UPDATE sync_clients
|
|
877
|
-
SET cursor=MAX(cursor,
|
|
904
|
+
SET cursor=MAX(cursor, MIN(?, coalesce((
|
|
905
|
+
SELECT min(json_extract(u.value, '$.cursor'))
|
|
906
|
+
FROM json_each(sync_clients.catching_up) AS u
|
|
907
|
+
WHERE json_extract(u.value, '$.cursor')>=0), ?))),
|
|
908
|
+
updated_at_ms=MAX(updated_at_ms, ?)
|
|
878
909
|
WHERE partition=? AND client_id=? AND actor_id=?
|
|
879
910
|
AND EXISTS (SELECT 1 FROM sync_partition_registry
|
|
880
|
-
WHERE partition=sync_clients.partition AND log_epoch=?)`, [cursor, updatedAtMs, partition, clientId, actorId, logEpoch]);
|
|
881
|
-
}
|
|
882
|
-
async updateClientCursor(partition, clientId, cursor, updatedAtMs) {
|
|
883
|
-
await this.all(`UPDATE sync_clients
|
|
884
|
-
SET cursor=MAX(cursor, ?), updated_at_ms=MAX(updated_at_ms, ?)
|
|
885
|
-
WHERE partition=? AND client_id=?`, [cursor, updatedAtMs, partition, clientId]);
|
|
911
|
+
WHERE partition=sync_clients.partition AND log_epoch=?)`, [cursor, cursor, updatedAtMs, partition, clientId, actorId, logEpoch]);
|
|
886
912
|
}
|
|
887
913
|
async getActiveClientCursorFloor(partition, cutoffMs) {
|
|
888
914
|
const [row] = await this.all('SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=? AND updated_at_ms>=? AND cursor>=0', [partition, cutoffMs]);
|
|
@@ -972,7 +998,7 @@ export class SqliteFamilyStorage {
|
|
|
972
998
|
}
|
|
973
999
|
// -- admin/console read surface --------------------------------------------
|
|
974
1000
|
async listClientRecords(partition) {
|
|
975
|
-
return (await this.all(
|
|
1001
|
+
return (await this.all(`SELECT ${CLIENT_RECORD_COLUMNS} FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC`, [partition])).map(toClientRecord);
|
|
976
1002
|
}
|
|
977
1003
|
async listCommitMetadata(partition, query) {
|
|
978
1004
|
// Newest-first window, resumed by `afterSeq` (exclusive lower bound) and
|
|
@@ -1102,13 +1128,23 @@ export class SqliteFamilyStorage {
|
|
|
1102
1128
|
result: deserializePushResult(record.result),
|
|
1103
1129
|
}));
|
|
1104
1130
|
}
|
|
1131
|
+
async listPushWatermarks(partition, clientCommitId) {
|
|
1132
|
+
// The clients come from the retained commit log, which has no index on
|
|
1133
|
+
// `client_commit_id`; the read is bounded by retention (§4.6).
|
|
1134
|
+
return (await this.all(`SELECT client_id, outbox_id, through_seq FROM sync_push_watermarks
|
|
1135
|
+
WHERE partition=? AND client_id IN (
|
|
1136
|
+
SELECT client_id FROM sync_commits WHERE partition=? AND client_commit_id=?)
|
|
1137
|
+
ORDER BY client_id, outbox_id`, [partition, partition, clientCommitId])).map((row) => ({
|
|
1138
|
+
clientId: row.client_id,
|
|
1139
|
+
outboxId: row.outbox_id,
|
|
1140
|
+
throughSeq: row.through_seq,
|
|
1141
|
+
}));
|
|
1142
|
+
}
|
|
1105
1143
|
async scopeActivity(partition, query) {
|
|
1106
1144
|
// One statement for the page: candidate (commit, table) pairs from the
|
|
1107
|
-
//
|
|
1108
|
-
//
|
|
1109
|
-
//
|
|
1110
|
-
// entries; only the LIMIT bounds the output. A candidate whose commit was
|
|
1111
|
-
// pruned has no metadata and is skipped.
|
|
1145
|
+
// `sync_change_scopes_by_key` range of the key, newest first, joined to
|
|
1146
|
+
// the commit metadata with the table's change count. A candidate whose
|
|
1147
|
+
// commit was pruned has no metadata and is skipped.
|
|
1112
1148
|
return (await this.all(`SELECT s.commit_seq, s.tbl, m.actor_id, m.created_at_ms,
|
|
1113
1149
|
(SELECT count(*) FROM sync_changes ch
|
|
1114
1150
|
WHERE ch.partition=? AND ch.commit_seq=s.commit_seq AND ch.tbl=s.tbl) AS n
|
|
@@ -1218,7 +1254,7 @@ export class SqliteFamilyTransaction {
|
|
|
1218
1254
|
change.table,
|
|
1219
1255
|
change.rowId,
|
|
1220
1256
|
change.op === 'upsert' ? 1 : 2,
|
|
1221
|
-
change.rowVersion
|
|
1257
|
+
change.rowVersion,
|
|
1222
1258
|
JSON.stringify(change.scopes),
|
|
1223
1259
|
change.payload ?? null,
|
|
1224
1260
|
]);
|
|
@@ -1233,7 +1269,7 @@ export class SqliteFamilyTransaction {
|
|
|
1233
1269
|
change.table,
|
|
1234
1270
|
change.rowId,
|
|
1235
1271
|
commitSeq,
|
|
1236
|
-
change.
|
|
1272
|
+
change.rowVersion,
|
|
1237
1273
|
JSON.stringify(change.scopes),
|
|
1238
1274
|
]);
|
|
1239
1275
|
}
|
|
@@ -1243,9 +1279,69 @@ export class SqliteFamilyTransaction {
|
|
|
1243
1279
|
});
|
|
1244
1280
|
return commitSeq;
|
|
1245
1281
|
}
|
|
1246
|
-
async putPushResult(clientId, clientCommitId, result) {
|
|
1282
|
+
async putPushResult(clientId, clientCommitId, result, sequence) {
|
|
1247
1283
|
this.assertOpen();
|
|
1248
|
-
this.run('INSERT OR IGNORE INTO sync_push_results(partition, client_id, client_commit_id, result) VALUES (
|
|
1284
|
+
this.run('INSERT OR IGNORE INTO sync_push_results(partition, client_id, client_commit_id, result, outbox_id, client_seq) VALUES (?,?,?,?,?,?)', [
|
|
1285
|
+
this.partition,
|
|
1286
|
+
clientId,
|
|
1287
|
+
clientCommitId,
|
|
1288
|
+
serializePushResult(result),
|
|
1289
|
+
sequence?.outboxId ?? null,
|
|
1290
|
+
sequence?.clientSeq ?? null,
|
|
1291
|
+
]);
|
|
1292
|
+
}
|
|
1293
|
+
/**
|
|
1294
|
+
* The §9 settlement and §6.3.2 steps 1 and 2 for one request, then the
|
|
1295
|
+
* idempotency read of its first commit. The reads run before the writes,
|
|
1296
|
+
* so D1, which buffers the writes until commit, answers exactly as SQLite:
|
|
1297
|
+
* the first commit's result counts as deleted when the writes delete it.
|
|
1298
|
+
*/
|
|
1299
|
+
async settleOutbox(clientId, clientCommitId, outbox) {
|
|
1300
|
+
const first = outbox.commits[0];
|
|
1301
|
+
const last = outbox.commits.at(-1);
|
|
1302
|
+
if (first === undefined ||
|
|
1303
|
+
last === undefined ||
|
|
1304
|
+
first.clientCommitId !== clientCommitId) {
|
|
1305
|
+
throw new Error("outbox settlement rides the request's first commit");
|
|
1306
|
+
}
|
|
1307
|
+
const p = this.partition;
|
|
1308
|
+
const [state] = await this.all(`SELECT (SELECT through_seq FROM sync_push_watermarks
|
|
1309
|
+
WHERE partition=?1 AND client_id=?2 AND outbox_id=?3) AS stored,
|
|
1310
|
+
EXISTS (SELECT 1 FROM sync_push_results WHERE partition=?1
|
|
1311
|
+
AND client_id=?2 AND client_commit_id=?5 AND outbox_id IS NULL) AS last_sequenceless,
|
|
1312
|
+
r.result AS result, r.outbox_id AS outbox_id, r.client_seq AS client_seq
|
|
1313
|
+
FROM (SELECT 1) LEFT JOIN sync_push_results AS r
|
|
1314
|
+
ON r.partition=?1 AND r.client_id=?2 AND r.client_commit_id=?4`, [p, clientId, outbox.outboxId, first.clientCommitId, last.clientCommitId]);
|
|
1315
|
+
const watermark = Math.max(state?.stored ?? 0, first.clientSeq);
|
|
1316
|
+
const commits = JSON.stringify(outbox.commits.map((commit) => [commit.clientCommitId, commit.clientSeq]));
|
|
1317
|
+
// §9 (b), then (a): with no pre-upgrade result on the request's last
|
|
1318
|
+
// commit, every remaining one outside the request left the outbox.
|
|
1319
|
+
if (state?.last_sequenceless !== 1) {
|
|
1320
|
+
this.run(`DELETE FROM sync_push_results
|
|
1321
|
+
WHERE partition=? AND client_id=? AND outbox_id IS NULL
|
|
1322
|
+
AND client_commit_id NOT IN (SELECT json_extract(value, '$[0]') FROM json_each(?))`, [p, clientId, commits]);
|
|
1323
|
+
}
|
|
1324
|
+
this.run(`UPDATE sync_push_results SET outbox_id=?3, client_seq=(
|
|
1325
|
+
SELECT json_extract(c.value, '$[1]') FROM json_each(?4) AS c
|
|
1326
|
+
WHERE json_extract(c.value, '$[0]')=sync_push_results.client_commit_id)
|
|
1327
|
+
WHERE partition=?1 AND client_id=?2 AND outbox_id IS NULL
|
|
1328
|
+
AND client_commit_id IN (SELECT json_extract(value, '$[0]') FROM json_each(?4))`, [p, clientId, outbox.outboxId, commits]);
|
|
1329
|
+
// §6.3.2 steps 1 and 2.
|
|
1330
|
+
this.run(`INSERT INTO sync_push_watermarks(partition, client_id, outbox_id, through_seq) VALUES (?,?,?,?)
|
|
1331
|
+
ON CONFLICT(partition, client_id, outbox_id) DO UPDATE SET through_seq=max(through_seq, excluded.through_seq)`, [p, clientId, outbox.outboxId, watermark]);
|
|
1332
|
+
this.run('DELETE FROM sync_push_results WHERE partition=? AND client_id=? AND outbox_id=? AND client_seq<?', [p, clientId, outbox.outboxId, watermark]);
|
|
1333
|
+
// The sequence step 2 sees: (a) gives a pre-upgrade result the first
|
|
1334
|
+
// commit's sequence; a result of this outbox keeps its stored one.
|
|
1335
|
+
const deleted = state?.outbox_id === null
|
|
1336
|
+
? first.clientSeq < watermark
|
|
1337
|
+
: state?.outbox_id === outbox.outboxId &&
|
|
1338
|
+
(state.client_seq ?? 0) < watermark;
|
|
1339
|
+
return {
|
|
1340
|
+
watermark,
|
|
1341
|
+
persisted: state?.result === null || state?.result === undefined || deleted
|
|
1342
|
+
? undefined
|
|
1343
|
+
: deserializePushResult(state.result),
|
|
1344
|
+
};
|
|
1249
1345
|
}
|
|
1250
1346
|
async enqueueReactions(reactions) {
|
|
1251
1347
|
this.assertOpen();
|