@syncular/server 0.59.0 → 0.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/README.md +20 -16
  2. package/dist/admin.d.ts +32 -20
  3. package/dist/admin.js +42 -40
  4. package/dist/context.d.ts +6 -8
  5. package/dist/context.js +1 -3
  6. package/dist/d1-storage.js +42 -24
  7. package/dist/errors.js +8 -0
  8. package/dist/events.d.ts +35 -17
  9. package/dist/frame-bytes.js +24 -6
  10. package/dist/handler.js +486 -223
  11. package/dist/local-authority.js +7 -9
  12. package/dist/operations.d.ts +5 -2
  13. package/dist/operations.js +20 -29
  14. package/dist/postgres-storage.d.ts +24 -19
  15. package/dist/postgres-storage.js +249 -168
  16. package/dist/pull.d.ts +68 -53
  17. package/dist/pull.js +258 -287
  18. package/dist/push.d.ts +12 -3
  19. package/dist/push.js +95 -32
  20. package/dist/realtime.d.ts +17 -11
  21. package/dist/realtime.js +65 -90
  22. package/dist/relational-rows.d.ts +0 -57
  23. package/dist/relational-rows.js +0 -97
  24. package/dist/seed.js +10 -15
  25. package/dist/segment-download.d.ts +4 -1
  26. package/dist/segment-download.js +4 -2
  27. package/dist/sqlite-dialect.d.ts +36 -59
  28. package/dist/sqlite-dialect.js +255 -159
  29. package/dist/sqlite-storage.js +24 -20
  30. package/dist/storage-query.d.ts +30 -1
  31. package/dist/storage-query.js +68 -0
  32. package/dist/storage.d.ts +156 -54
  33. package/dist/storage.js +15 -0
  34. package/package.json +2 -2
  35. package/src/admin.ts +87 -59
  36. package/src/context.ts +7 -11
  37. package/src/d1-storage.ts +49 -33
  38. package/src/errors.ts +8 -0
  39. package/src/events.ts +36 -17
  40. package/src/frame-bytes.ts +26 -6
  41. package/src/handler.ts +628 -286
  42. package/src/local-authority.ts +10 -11
  43. package/src/operations.ts +44 -42
  44. package/src/postgres-storage.ts +336 -226
  45. package/src/pull.ts +394 -414
  46. package/src/push.ts +116 -21
  47. package/src/realtime.ts +85 -129
  48. package/src/relational-rows.ts +0 -105
  49. package/src/seed.ts +13 -19
  50. package/src/segment-download.ts +8 -3
  51. package/src/sqlite-dialect.ts +363 -204
  52. package/src/sqlite-storage.ts +45 -25
  53. package/src/storage-query.ts +102 -0
  54. package/src/storage.ts +185 -58
@@ -2,10 +2,10 @@ import { bindAuthoritativePartition, prepareAuthoritativeQuery, } from './author
2
2
  import { syncError } from './errors.js';
3
3
  import { validateCommitPruneQuery } from './prune.js';
4
4
  import { StorageQueryError } from './storage-errors.js';
5
- import { deleteRowSql, deleteSqliteRowScopesSql, indexRowPageStatement, quoteIdent, scanRowPageSql, rowsReferencingBlobSql, selectRowSql, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, commitWindowPageSql, LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL, upsertSql, upsertValues, } from './relational-rows.js';
5
+ import { deleteRowSql, deleteSqliteRowScopesSql, indexRowPageStatement, quoteIdent, scanRowPageSql, rowsReferencingBlobSql, selectRowSql, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, upsertSql, upsertValues, } from './relational-rows.js';
6
6
  import { matchesEffective } from './scopes.js';
7
7
  import { clientRoundState, PARTITION_TOUCH_INTERVAL_MS, rowsLiveAfterDelete, } from './storage.js';
8
- import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js';
8
+ import { assertScopeIndexedScan, readLiveWindowPages, resolveIndexRowScan, } from './storage-query.js';
9
9
  /**
10
10
  * Schema DDL, one statement per `;`-delimited chunk. Native SQLite applies
11
11
  * the whole string via `db.exec(SQLITE_DDL)`; D1 applies each statement
@@ -19,21 +19,15 @@ import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js'
19
19
  * the Postgres storage documents (§3.1, performance-by-
20
20
  * construction).
21
21
  *
22
- * The old `sync_row_scopes_by_row` index is dropped: with it present the
23
- * planner answered a scope scan of two or more values by walking the index
24
- * in `row_id` order and filtering, instead of seeking the PRIMARY KEY once
25
- * per value. The per-row scope replacement (`deleteSqliteRowScopesSql`)
26
- * binds `(partition, tbl, var, value, row_id)` and needs no secondary index.
27
- * `DROP INDEX IF EXISTS` stays in the DDL so existing databases lose the
28
- * index the next time the DDL runs: on every `SqliteServerStorage`
29
- * construction, and on D1 in `migrate()` and the core phase of
30
- * `migrateSchema`.
22
+ * Changes to the core tables of an existing database (added and dropped
23
+ * columns and indexes) live in `sqliteUpgradeDdl`, which runs after this DDL.
31
24
  */
32
25
  export const SQLITE_DDL = `
33
26
  CREATE TABLE IF NOT EXISTS sync_partitions(
34
27
  partition TEXT PRIMARY KEY,
35
28
  max_commit_seq INTEGER NOT NULL DEFAULT 0,
36
- horizon_seq INTEGER NOT NULL DEFAULT 0
29
+ horizon_seq INTEGER NOT NULL DEFAULT 0,
30
+ delete_floor INTEGER NOT NULL DEFAULT 0
37
31
  );
38
32
  CREATE TABLE IF NOT EXISTS sync_partition_registry(
39
33
  partition TEXT PRIMARY KEY,
@@ -46,7 +40,6 @@ CREATE TABLE IF NOT EXISTS sync_row_scopes(
46
40
  var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,
47
41
  PRIMARY KEY(partition, tbl, var, value, row_id)
48
42
  );
49
- DROP INDEX IF EXISTS sync_row_scopes_by_row;
50
43
  CREATE TABLE IF NOT EXISTS sync_commits(
51
44
  partition TEXT NOT NULL, commit_seq INTEGER NOT NULL,
52
45
  client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,
@@ -72,8 +65,14 @@ CREATE TABLE IF NOT EXISTS sync_change_scopes(
72
65
  CREATE TABLE IF NOT EXISTS sync_push_results(
73
66
  partition TEXT NOT NULL, client_id TEXT NOT NULL,
74
67
  client_commit_id TEXT NOT NULL, result TEXT NOT NULL,
68
+ outbox_id TEXT, client_seq INTEGER,
75
69
  PRIMARY KEY(partition, client_id, client_commit_id)
76
70
  );
71
+ CREATE TABLE IF NOT EXISTS sync_push_watermarks(
72
+ partition TEXT NOT NULL, client_id TEXT NOT NULL, outbox_id TEXT NOT NULL,
73
+ through_seq INTEGER NOT NULL,
74
+ PRIMARY KEY(partition, client_id, outbox_id)
75
+ );
77
76
  CREATE TABLE IF NOT EXISTS sync_reactions(
78
77
  partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,
79
78
  type TEXT NOT NULL, version INTEGER NOT NULL, payload TEXT NOT NULL,
@@ -97,10 +96,12 @@ CREATE INDEX IF NOT EXISTS sync_reactions_dead_letter
97
96
  CREATE TABLE IF NOT EXISTS sync_clients(
98
97
  partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,
99
98
  wire_version INTEGER NOT NULL DEFAULT 1,
100
- cursor INTEGER NOT NULL, subscriptions TEXT NOT NULL,
99
+ cursor INTEGER NOT NULL,
101
100
  updated_at_ms INTEGER NOT NULL,
102
101
  last_round_at_ms INTEGER, last_error_code TEXT,
103
102
  consecutive_failures INTEGER NOT NULL DEFAULT 0,
103
+ schema_version INTEGER, units TEXT, registration_digest BLOB,
104
+ catching_up TEXT NOT NULL DEFAULT '[]',
104
105
  PRIMARY KEY(partition, client_id)
105
106
  );
106
107
  CREATE TABLE IF NOT EXISTS sync_blob_refs(
@@ -130,15 +131,97 @@ CREATE TABLE IF NOT EXISTS sync_writer_fence(
130
131
  required_writer_version INTEGER NOT NULL
131
132
  );
132
133
  `;
133
- /** Columns added to core tables after their first release: table, column, type. */
134
- export const SQLITE_ADDED_COLUMNS = [
134
+ /**
135
+ * Columns added to core tables after their first release: table, column,
136
+ * type. A client record from before wire version 4 gains NULL `units` and
137
+ * `registration_digest`, so it holds no registration list (§9), and a push
138
+ * result gains NULL `outbox_id` and `client_seq`: it is a result stored
139
+ * before the upgrade, settled by the client's next push (§9). The
140
+ * watermark delete and the settlement range over the `(partition,
141
+ * client_id)` prefix of the primary key, which the watermark keeps to one
142
+ * request's results per outbox.
143
+ */
144
+ const SQLITE_ADDED_COLUMNS = [
135
145
  ['sync_clients', 'wire_version', 'INTEGER NOT NULL DEFAULT 1'],
136
146
  ['sync_clients', 'last_round_at_ms', 'INTEGER'],
137
147
  ['sync_clients', 'last_error_code', 'TEXT'],
138
148
  ['sync_clients', 'consecutive_failures', 'INTEGER NOT NULL DEFAULT 0'],
149
+ ['sync_clients', 'schema_version', 'INTEGER'],
150
+ ['sync_clients', 'units', 'TEXT'],
151
+ ['sync_clients', 'registration_digest', 'BLOB'],
152
+ ['sync_clients', 'catching_up', "TEXT NOT NULL DEFAULT '[]'"],
139
153
  ['sync_tombstones', 'server_version', 'INTEGER'],
140
154
  ['sync_tombstones', 'scopes', 'TEXT'],
155
+ ['sync_push_results', 'outbox_id', 'TEXT'],
156
+ ['sync_push_results', 'client_seq', 'INTEGER'],
157
+ ['sync_partitions', 'delete_floor', 'INTEGER NOT NULL DEFAULT 0'],
141
158
  ];
159
+ /** Core-table columns wire version 4 deleted: the version 3 subscription list. */
160
+ const SQLITE_DROPPED_COLUMNS = [['sync_clients', 'subscriptions']];
161
+ /**
162
+ * Indexes that `sqliteUpgradeDdl` creates, on a fresh database and an
163
+ * upgraded one alike. `sync_change_scopes_by_key` serves the §4.5 live
164
+ * scan: one range per (variable, value) across tables.
165
+ */
166
+ const SQLITE_ADDED_INDEXES = [
167
+ [
168
+ 'sync_change_scopes_by_key',
169
+ 'CREATE INDEX IF NOT EXISTS sync_change_scopes_by_key ON sync_change_scopes(partition, var, value, commit_seq)',
170
+ ],
171
+ ];
172
+ /**
173
+ * Indexes that `sqliteUpgradeDdl` drops. With `sync_row_scopes_by_row`
174
+ * present the planner answered a scope scan of two or more values by walking
175
+ * it in `row_id` order and filtering, instead of seeking the PRIMARY KEY once
176
+ * per value; the per-row scope replacement (`deleteSqliteRowScopesSql`)
177
+ * binds the full key and needs no secondary index.
178
+ */
179
+ const SQLITE_DROPPED_INDEXES = ['sync_row_scopes_by_row'];
180
+ /**
181
+ * One statement that reads what `sqliteUpgradeDdl` needs: the columns of
182
+ * every table with added or dropped columns, and which added indexes exist.
183
+ */
184
+ export const SQLITE_UPGRADE_PROBE = {
185
+ sql: `SELECT t.value AS tbl, c.name AS name
186
+ FROM json_each(?) AS t, pragma_table_info(t.value) AS c
187
+ UNION ALL
188
+ SELECT '' AS tbl, name FROM sqlite_master
189
+ WHERE type='index' AND name IN (SELECT value FROM json_each(?))`,
190
+ params: [
191
+ JSON.stringify([
192
+ ...new Set([...SQLITE_ADDED_COLUMNS, ...SQLITE_DROPPED_COLUMNS].map(([table]) => table)),
193
+ ]),
194
+ JSON.stringify([
195
+ ...SQLITE_ADDED_INDEXES.map(([name]) => name),
196
+ ...SQLITE_DROPPED_INDEXES,
197
+ ]),
198
+ ],
199
+ };
200
+ /**
201
+ * The statements that bring the core tables of a database from an earlier
202
+ * release to the current DDL, given the `SQLITE_UPGRADE_PROBE` rows: missing
203
+ * columns, dropped columns, dropped indexes, then missing indexes. Empty for
204
+ * a current database. Run after `SQLITE_DDL`: on every `SqliteServerStorage`
205
+ * construction, and on D1 in `migrate()`, the core phase of `migrateSchema`
206
+ * and the first open of a database at the current schema version.
207
+ */
208
+ export function sqliteUpgradeDdl(present) {
209
+ const has = (table, name) => present.some((row) => row.tbl === table && row.name === name);
210
+ return [
211
+ ...SQLITE_ADDED_COLUMNS.filter(([table, name]) => !has(table, name)).map(([table, name, definition]) => `ALTER TABLE ${table} ADD COLUMN ${name} ${definition}`),
212
+ // A database from before wire version 4 (and the delete floor) starts
213
+ // the delete floor at the highest version its tombstones recorded
214
+ // (§2.2) and the horizon at its last commit (§9).
215
+ ...(has('sync_partitions', 'delete_floor')
216
+ ? []
217
+ : [
218
+ `UPDATE sync_partitions SET horizon_seq=max_commit_seq, delete_floor=coalesce((SELECT max(server_version) FROM sync_tombstones t WHERE t.partition=sync_partitions.partition),0)`,
219
+ ]),
220
+ ...SQLITE_DROPPED_COLUMNS.filter(([table, name]) => has(table, name)).map(([table, name]) => `ALTER TABLE ${table} DROP COLUMN ${name}`),
221
+ ...SQLITE_DROPPED_INDEXES.filter((name) => has('', name)).map((name) => `DROP INDEX IF EXISTS ${name}`),
222
+ ...SQLITE_ADDED_INDEXES.filter(([name]) => !has('', name)).map(([, ddl]) => ddl),
223
+ ];
224
+ }
142
225
  /** Split the DDL into individual statements (D1 applies them one by one). */
143
226
  export function sqliteDdlStatements() {
144
227
  return SQLITE_DDL.split(';')
@@ -297,77 +380,6 @@ export function toStoredChange(record) {
297
380
  : {}),
298
381
  };
299
382
  }
300
- /**
301
- * Fold one `commitWindowPageSql` page into commits, preserving the exact
302
- * semantics of the old per-candidate loop:
303
- *
304
- * - rows arrive ordered (commit_seq, idx); consecutive rows with the same
305
- * `commit_seq` are one candidate commit;
306
- * - every candidate advances the cursor (`lastSeq`) — including vanished
307
- * commits (NULL meta) and commits whose changes all fail the exact
308
- * multi-variable scope verification (`matchesEffective`);
309
- * - a commit is emitted only with its matching changes, in `idx` order;
310
- * - stops after the commit that accumulates at least `remaining` matching
311
- * changes (never splitting a commit).
312
- *
313
- * `candidateCount` counts the processed candidates so the caller can detect
314
- * a short page (window exhausted) exactly as it did with the old candidate
315
- * query.
316
- */
317
- export function collectCommitWindowPage(records, scopeFilter, remaining) {
318
- const commits = [];
319
- let delivered = 0;
320
- let lastSeq = 0;
321
- let candidateCount = 0;
322
- let i = 0;
323
- while (i < records.length) {
324
- const head = records[i];
325
- if (head === undefined)
326
- break;
327
- const seq = head.commit_seq;
328
- candidateCount += 1;
329
- lastSeq = seq;
330
- const changes = [];
331
- for (; i < records.length; i++) {
332
- const record = records[i];
333
- if (record === undefined || record.commit_seq !== seq)
334
- break;
335
- // NULL tbl: a candidate with no change rows for the table (LEFT JOIN
336
- // contract) — it still advanced the cursor above.
337
- if (record.tbl === null ||
338
- record.row_id === null ||
339
- record.op === null ||
340
- record.scopes === null) {
341
- continue;
342
- }
343
- const change = toStoredChange({
344
- tbl: record.tbl,
345
- row_id: record.row_id,
346
- op: record.op,
347
- row_version: record.row_version,
348
- scopes: record.scopes,
349
- payload: record.payload,
350
- });
351
- if (matchesEffective(change.scopes, scopeFilter))
352
- changes.push(change);
353
- }
354
- // NULL meta: the commit vanished — cursor advanced, nothing emitted.
355
- if (head.actor_id === null || head.created_at_ms === null)
356
- continue;
357
- if (changes.length === 0)
358
- continue;
359
- commits.push({
360
- commitSeq: seq,
361
- createdAtMs: head.created_at_ms,
362
- actorId: head.actor_id,
363
- changes,
364
- });
365
- delivered += changes.length;
366
- if (delivered >= remaining)
367
- break;
368
- }
369
- return { commits, delivered, lastSeq, candidateCount };
370
- }
371
383
  function toStoredReaction(record) {
372
384
  return {
373
385
  idempotencyKey: record.idempotency_key,
@@ -394,6 +406,7 @@ function toStoredReaction(record) {
394
406
  : {}),
395
407
  };
396
408
  }
409
+ const CLIENT_RECORD_COLUMNS = 'client_id, actor_id, wire_version, cursor, updated_at_ms, schema_version, units, registration_digest, catching_up, last_round_at_ms, last_error_code, consecutive_failures';
397
410
  function toClientRecord(record) {
398
411
  return {
399
412
  clientId: record.client_id,
@@ -401,7 +414,18 @@ function toClientRecord(record) {
401
414
  wireVersion: record.wire_version,
402
415
  cursor: record.cursor,
403
416
  updatedAtMs: record.updated_at_ms,
404
- subscriptions: JSON.parse(record.subscriptions),
417
+ ...(record.schema_version !== null
418
+ ? { schemaVersion: record.schema_version }
419
+ : {}),
420
+ ...(record.units !== null && record.registration_digest !== null
421
+ ? {
422
+ registration: {
423
+ units: JSON.parse(record.units),
424
+ digest: asUint8Array(record.registration_digest),
425
+ },
426
+ }
427
+ : {}),
428
+ catchingUp: JSON.parse(record.catching_up),
405
429
  ...clientRoundState(record.last_round_at_ms, record.last_error_code, record.consecutive_failures),
406
430
  };
407
431
  }
@@ -739,55 +763,46 @@ export class SqliteFamilyStorage {
739
763
  };
740
764
  }
741
765
  async latestScopedChangeSeq(partition, query) {
742
- const firstVariable = Object.keys(query.scopeFilter).sort()[0];
743
- const values = firstVariable === undefined
744
- ? []
745
- : (query.scopeFilter[firstVariable] ?? []);
746
- if (firstVariable === undefined || values.length === 0)
766
+ const variable = Object.keys(query.scopes).sort()[0];
767
+ const value = variable === undefined ? undefined : query.scopes[variable];
768
+ if (variable === undefined || value === undefined)
747
769
  return 0;
748
- const [row] = await this.guardedAll(LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL, [partition, query.table, firstVariable, JSON.stringify(values)]);
770
+ // One index-only max over the (partition, tbl, var, value) prefix.
771
+ const [row] = await this.guardedAll('SELECT max(commit_seq) AS seq FROM sync_change_scopes WHERE partition=? AND tbl=? AND var=? AND value=?', [partition, query.table, variable, value]);
749
772
  return row?.seq ?? 0;
750
773
  }
751
- async readCommitWindow(partition, query) {
752
- const variables = Object.keys(query.scopeFilter).sort();
753
- const firstVariable = variables[0];
754
- if (firstVariable === undefined)
755
- return [];
756
- const firstValues = query.scopeFilter[firstVariable] ?? [];
757
- if (firstValues.length === 0)
758
- return [];
759
- // Candidates via the inverted index (one variable) LEFT JOINed to the
760
- // commit meta + the table's changes — one statement per page, never two
761
- // per candidate (see `commitWindowPageSql`). Exact multi-variable
762
- // verification against the stored scope map in `collectCommitWindowPage`.
763
- const sql = commitWindowPageSql(firstValues.length, 'sqlite');
764
- const commits = [];
765
- let deliveredChanges = 0;
766
- let afterSeq = query.afterSeq;
767
- const batchSize = Math.max(64, query.limitChanges);
768
- while (deliveredChanges < query.limitChanges) {
769
- const records = await this.guardedAll(sql, [
770
- partition,
771
- query.table,
772
- firstVariable,
773
- JSON.stringify(firstValues),
774
- afterSeq,
775
- query.throughSeq,
776
- batchSize,
777
- partition,
778
- partition,
779
- query.table,
780
- ]);
781
- if (records.length === 0)
782
- break;
783
- const page = collectCommitWindowPage(records, query.scopeFilter, query.limitChanges - deliveredChanges);
784
- commits.push(...page.commits);
785
- deliveredChanges += page.delivered;
786
- afterSeq = page.lastSeq;
787
- if (page.candidateCount < batchSize)
788
- break;
789
- }
790
- return commits;
774
+ readLiveWindow(partition, query) {
775
+ // One statement per page: the unit keys bind as one JSON array, and the
776
+ // CROSS JOIN keeps them the outer loop, so each key seeks one range of
777
+ // `sync_change_scopes_by_key` (INDEXED BY fails loudly when the index is
778
+ // missing). The candidates LEFT JOIN the commit meta and every change of
779
+ // the commit; a candidate whose commit vanished comes back with NULL
780
+ // meta and still advances the scan.
781
+ return readLiveWindowPages(query, async (keys, limit) => (await this.guardedAll(`SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
782
+ ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
783
+ FROM (SELECT DISTINCT s.commit_seq AS commit_seq
784
+ FROM json_each(?2) AS k
785
+ CROSS JOIN sync_change_scopes AS s INDEXED BY sync_change_scopes_by_key
786
+ ON s.partition=?1 AND s.var=json_extract(k.value, '$.var')
787
+ AND s.value=json_extract(k.value, '$.value')
788
+ AND s.commit_seq>json_extract(k.value, '$.after') AND s.commit_seq<=?3
789
+ ORDER BY s.commit_seq LIMIT ?4) AS c
790
+ LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=c.commit_seq
791
+ LEFT JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.commit_seq
792
+ ORDER BY c.commit_seq, ch.idx`, [partition, JSON.stringify(keys), query.throughSeq, limit])).map((record) => ({
793
+ commitSeq: record.commit_seq,
794
+ ...(record.actor_id !== null && record.created_at_ms !== null
795
+ ? {
796
+ meta: {
797
+ actorId: record.actor_id,
798
+ createdAtMs: record.created_at_ms,
799
+ },
800
+ }
801
+ : {}),
802
+ ...(record.tbl !== null
803
+ ? { change: toStoredChange({ ...record, tbl: record.tbl }) }
804
+ : {}),
805
+ })));
791
806
  }
792
807
  async scanRows(partition, query) {
793
808
  const firstVariable = assertScopeIndexedScan(query);
@@ -837,21 +852,28 @@ export class SqliteFamilyStorage {
837
852
  return (await this.guardedAll(statement.sql, statement.params)).map(toStoredRow);
838
853
  }
839
854
  async getClientRecord(partition, clientId) {
840
- const [record] = await this.all('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=? AND client_id=?', [partition, clientId]);
855
+ const [record] = await this.all(`SELECT ${CLIENT_RECORD_COLUMNS} FROM sync_clients WHERE partition=? AND client_id=?`, [partition, clientId]);
841
856
  return record === undefined ? undefined : toClientRecord(record);
842
857
  }
843
858
  async putClientRecord(partition, record, options) {
844
859
  // One row per (partition, client_id): every non-key field is written
845
- // from the new record. `keepRoundOutcome` leaves the round-state columns
846
- // alone (§1.5, §4.5).
847
- await this.all(`INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms) VALUES (?,?,?,?,?,?,?,?)
860
+ // from the new record, except the registration list and digest, which
861
+ // only a traveling list replaces (§4.5). `keepRoundOutcome` leaves the
862
+ // round-state columns alone (§1.5, §4.5).
863
+ const registration = record.registration;
864
+ await this.all(`INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, updated_at_ms, last_round_at_ms, schema_version, units, registration_digest, catching_up) VALUES (?,?,?,?,?,?,?,?,?,?,?)
848
865
  ON CONFLICT(partition, client_id) DO UPDATE SET
849
866
  actor_id=excluded.actor_id,
850
867
  wire_version=excluded.wire_version,
851
868
  cursor=excluded.cursor,
852
- subscriptions=excluded.subscriptions,
853
869
  updated_at_ms=excluded.updated_at_ms,
854
- last_round_at_ms=excluded.last_round_at_ms${options?.keepRoundOutcome === true
870
+ last_round_at_ms=excluded.last_round_at_ms,
871
+ schema_version=excluded.schema_version,${registration === undefined
872
+ ? ''
873
+ : `
874
+ units=excluded.units,
875
+ registration_digest=excluded.registration_digest,`}
876
+ catching_up=excluded.catching_up${options?.keepRoundOutcome === true
855
877
  ? ''
856
878
  : `,
857
879
  last_error_code=NULL,
@@ -861,9 +883,12 @@ export class SqliteFamilyStorage {
861
883
  record.actorId,
862
884
  record.wireVersion,
863
885
  record.cursor,
864
- JSON.stringify(record.subscriptions),
865
- record.updatedAtMs,
866
886
  record.updatedAtMs,
887
+ record.lastRoundAtMs ?? record.updatedAtMs,
888
+ record.schemaVersion ?? null,
889
+ registration === undefined ? null : JSON.stringify(registration.units),
890
+ registration?.digest ?? null,
891
+ JSON.stringify(record.catchingUp),
867
892
  ]);
868
893
  }
869
894
  async recordClientRoundFailure(partition, clientId, actorId, errorCode, atMs) {
@@ -873,16 +898,17 @@ export class SqliteFamilyStorage {
873
898
  WHERE partition=? AND client_id=? AND actor_id=?`, [atMs, errorCode, partition, clientId, actorId]);
874
899
  }
875
900
  async advanceClientCursor(partition, clientId, actorId, logEpoch, cursor, updatedAtMs) {
901
+ // §8.2: the ack never raises the record cursor above a catching-up
902
+ // unit's cursor, so pruning keeps that unit's history (§4.6).
876
903
  await this.all(`UPDATE sync_clients
877
- SET cursor=MAX(cursor, ?), updated_at_ms=MAX(updated_at_ms, ?)
904
+ SET cursor=MAX(cursor, MIN(?, coalesce((
905
+ SELECT min(json_extract(u.value, '$.cursor'))
906
+ FROM json_each(sync_clients.catching_up) AS u
907
+ WHERE json_extract(u.value, '$.cursor')>=0), ?))),
908
+ updated_at_ms=MAX(updated_at_ms, ?)
878
909
  WHERE partition=? AND client_id=? AND actor_id=?
879
910
  AND EXISTS (SELECT 1 FROM sync_partition_registry
880
- WHERE partition=sync_clients.partition AND log_epoch=?)`, [cursor, updatedAtMs, partition, clientId, actorId, logEpoch]);
881
- }
882
- async updateClientCursor(partition, clientId, cursor, updatedAtMs) {
883
- await this.all(`UPDATE sync_clients
884
- SET cursor=MAX(cursor, ?), updated_at_ms=MAX(updated_at_ms, ?)
885
- WHERE partition=? AND client_id=?`, [cursor, updatedAtMs, partition, clientId]);
911
+ WHERE partition=sync_clients.partition AND log_epoch=?)`, [cursor, cursor, updatedAtMs, partition, clientId, actorId, logEpoch]);
886
912
  }
887
913
  async getActiveClientCursorFloor(partition, cutoffMs) {
888
914
  const [row] = await this.all('SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=? AND updated_at_ms>=? AND cursor>=0', [partition, cutoffMs]);
@@ -972,7 +998,7 @@ export class SqliteFamilyStorage {
972
998
  }
973
999
  // -- admin/console read surface --------------------------------------------
974
1000
  async listClientRecords(partition) {
975
- return (await this.all('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC', [partition])).map(toClientRecord);
1001
+ return (await this.all(`SELECT ${CLIENT_RECORD_COLUMNS} FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC`, [partition])).map(toClientRecord);
976
1002
  }
977
1003
  async listCommitMetadata(partition, query) {
978
1004
  // Newest-first window, resumed by `afterSeq` (exclusive lower bound) and
@@ -1102,13 +1128,23 @@ export class SqliteFamilyStorage {
1102
1128
  result: deserializePushResult(record.result),
1103
1129
  }));
1104
1130
  }
1131
+ async listPushWatermarks(partition, clientCommitId) {
1132
+ // The clients come from the retained commit log, which has no index on
1133
+ // `client_commit_id`; the read is bounded by retention (§4.6).
1134
+ return (await this.all(`SELECT client_id, outbox_id, through_seq FROM sync_push_watermarks
1135
+ WHERE partition=? AND client_id IN (
1136
+ SELECT client_id FROM sync_commits WHERE partition=? AND client_commit_id=?)
1137
+ ORDER BY client_id, outbox_id`, [partition, partition, clientCommitId])).map((row) => ({
1138
+ clientId: row.client_id,
1139
+ outboxId: row.outbox_id,
1140
+ throughSeq: row.through_seq,
1141
+ }));
1142
+ }
1105
1143
  async scopeActivity(partition, query) {
1106
1144
  // One statement for the page: candidate (commit, table) pairs from the
1107
- // change-scope index, joined to the commit metadata with the table's
1108
- // change count. `tbl` sits between `partition` and `var` in the index key,
1109
- // so the `var`/`value` filter ranges over the partition's change-scope
1110
- // entries; only the LIMIT bounds the output. A candidate whose commit was
1111
- // pruned has no metadata and is skipped.
1145
+ // `sync_change_scopes_by_key` range of the key, newest first, joined to
1146
+ // the commit metadata with the table's change count. A candidate whose
1147
+ // commit was pruned has no metadata and is skipped.
1112
1148
  return (await this.all(`SELECT s.commit_seq, s.tbl, m.actor_id, m.created_at_ms,
1113
1149
  (SELECT count(*) FROM sync_changes ch
1114
1150
  WHERE ch.partition=? AND ch.commit_seq=s.commit_seq AND ch.tbl=s.tbl) AS n
@@ -1218,7 +1254,7 @@ export class SqliteFamilyTransaction {
1218
1254
  change.table,
1219
1255
  change.rowId,
1220
1256
  change.op === 'upsert' ? 1 : 2,
1221
- change.rowVersion ?? null,
1257
+ change.rowVersion,
1222
1258
  JSON.stringify(change.scopes),
1223
1259
  change.payload ?? null,
1224
1260
  ]);
@@ -1233,7 +1269,7 @@ export class SqliteFamilyTransaction {
1233
1269
  change.table,
1234
1270
  change.rowId,
1235
1271
  commitSeq,
1236
- change.deletedVersion ?? null,
1272
+ change.rowVersion,
1237
1273
  JSON.stringify(change.scopes),
1238
1274
  ]);
1239
1275
  }
@@ -1243,9 +1279,69 @@ export class SqliteFamilyTransaction {
1243
1279
  });
1244
1280
  return commitSeq;
1245
1281
  }
1246
- async putPushResult(clientId, clientCommitId, result) {
1282
+ async putPushResult(clientId, clientCommitId, result, sequence) {
1247
1283
  this.assertOpen();
1248
- this.run('INSERT OR IGNORE INTO sync_push_results(partition, client_id, client_commit_id, result) VALUES (?,?,?,?)', [this.partition, clientId, clientCommitId, serializePushResult(result)]);
1284
+ this.run('INSERT OR IGNORE INTO sync_push_results(partition, client_id, client_commit_id, result, outbox_id, client_seq) VALUES (?,?,?,?,?,?)', [
1285
+ this.partition,
1286
+ clientId,
1287
+ clientCommitId,
1288
+ serializePushResult(result),
1289
+ sequence?.outboxId ?? null,
1290
+ sequence?.clientSeq ?? null,
1291
+ ]);
1292
+ }
1293
+ /**
1294
+ * The §9 settlement and §6.3.2 steps 1 and 2 for one request, then the
1295
+ * idempotency read of its first commit. The reads run before the writes,
1296
+ * so D1, which buffers the writes until commit, answers exactly as SQLite:
1297
+ * the first commit's result counts as deleted when the writes delete it.
1298
+ */
1299
+ async settleOutbox(clientId, clientCommitId, outbox) {
1300
+ const first = outbox.commits[0];
1301
+ const last = outbox.commits.at(-1);
1302
+ if (first === undefined ||
1303
+ last === undefined ||
1304
+ first.clientCommitId !== clientCommitId) {
1305
+ throw new Error("outbox settlement rides the request's first commit");
1306
+ }
1307
+ const p = this.partition;
1308
+ const [state] = await this.all(`SELECT (SELECT through_seq FROM sync_push_watermarks
1309
+ WHERE partition=?1 AND client_id=?2 AND outbox_id=?3) AS stored,
1310
+ EXISTS (SELECT 1 FROM sync_push_results WHERE partition=?1
1311
+ AND client_id=?2 AND client_commit_id=?5 AND outbox_id IS NULL) AS last_sequenceless,
1312
+ r.result AS result, r.outbox_id AS outbox_id, r.client_seq AS client_seq
1313
+ FROM (SELECT 1) LEFT JOIN sync_push_results AS r
1314
+ ON r.partition=?1 AND r.client_id=?2 AND r.client_commit_id=?4`, [p, clientId, outbox.outboxId, first.clientCommitId, last.clientCommitId]);
1315
+ const watermark = Math.max(state?.stored ?? 0, first.clientSeq);
1316
+ const commits = JSON.stringify(outbox.commits.map((commit) => [commit.clientCommitId, commit.clientSeq]));
1317
+ // §9 (b), then (a): with no pre-upgrade result on the request's last
1318
+ // commit, every remaining one outside the request left the outbox.
1319
+ if (state?.last_sequenceless !== 1) {
1320
+ this.run(`DELETE FROM sync_push_results
1321
+ WHERE partition=? AND client_id=? AND outbox_id IS NULL
1322
+ AND client_commit_id NOT IN (SELECT json_extract(value, '$[0]') FROM json_each(?))`, [p, clientId, commits]);
1323
+ }
1324
+ this.run(`UPDATE sync_push_results SET outbox_id=?3, client_seq=(
1325
+ SELECT json_extract(c.value, '$[1]') FROM json_each(?4) AS c
1326
+ WHERE json_extract(c.value, '$[0]')=sync_push_results.client_commit_id)
1327
+ WHERE partition=?1 AND client_id=?2 AND outbox_id IS NULL
1328
+ AND client_commit_id IN (SELECT json_extract(value, '$[0]') FROM json_each(?4))`, [p, clientId, outbox.outboxId, commits]);
1329
+ // §6.3.2 steps 1 and 2.
1330
+ this.run(`INSERT INTO sync_push_watermarks(partition, client_id, outbox_id, through_seq) VALUES (?,?,?,?)
1331
+ ON CONFLICT(partition, client_id, outbox_id) DO UPDATE SET through_seq=max(through_seq, excluded.through_seq)`, [p, clientId, outbox.outboxId, watermark]);
1332
+ this.run('DELETE FROM sync_push_results WHERE partition=? AND client_id=? AND outbox_id=? AND client_seq<?', [p, clientId, outbox.outboxId, watermark]);
1333
+ // The sequence step 2 sees: (a) gives a pre-upgrade result the first
1334
+ // commit's sequence; a result of this outbox keeps its stored one.
1335
+ const deleted = state?.outbox_id === null
1336
+ ? first.clientSeq < watermark
1337
+ : state?.outbox_id === outbox.outboxId &&
1338
+ (state.client_seq ?? 0) < watermark;
1339
+ return {
1340
+ watermark,
1341
+ persisted: state?.result === null || state?.result === undefined || deleted
1342
+ ? undefined
1343
+ : deserializePushResult(state.result),
1344
+ };
1249
1345
  }
1250
1346
  async enqueueReactions(reactions) {
1251
1347
  this.assertOpen();