@syncular/server 0.59.0 → 0.60.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/README.md +20 -16
  2. package/dist/admin.d.ts +32 -20
  3. package/dist/admin.js +42 -40
  4. package/dist/context.d.ts +6 -8
  5. package/dist/context.js +1 -3
  6. package/dist/d1-storage.js +42 -24
  7. package/dist/errors.js +8 -0
  8. package/dist/events.d.ts +35 -17
  9. package/dist/frame-bytes.js +24 -6
  10. package/dist/handler.js +486 -223
  11. package/dist/local-authority.js +7 -9
  12. package/dist/operations.d.ts +5 -2
  13. package/dist/operations.js +20 -29
  14. package/dist/postgres-storage.d.ts +24 -19
  15. package/dist/postgres-storage.js +249 -168
  16. package/dist/pull.d.ts +68 -53
  17. package/dist/pull.js +258 -287
  18. package/dist/push.d.ts +12 -3
  19. package/dist/push.js +95 -32
  20. package/dist/realtime.d.ts +17 -11
  21. package/dist/realtime.js +65 -90
  22. package/dist/relational-rows.d.ts +0 -57
  23. package/dist/relational-rows.js +4 -100
  24. package/dist/seed.js +10 -15
  25. package/dist/segment-download.d.ts +4 -1
  26. package/dist/segment-download.js +4 -2
  27. package/dist/sqlite-dialect.d.ts +36 -59
  28. package/dist/sqlite-dialect.js +255 -159
  29. package/dist/sqlite-storage.js +24 -20
  30. package/dist/storage-query.d.ts +30 -1
  31. package/dist/storage-query.js +68 -0
  32. package/dist/storage.d.ts +156 -54
  33. package/dist/storage.js +15 -0
  34. package/package.json +2 -2
  35. package/src/admin.ts +87 -59
  36. package/src/context.ts +7 -11
  37. package/src/d1-storage.ts +49 -33
  38. package/src/errors.ts +8 -0
  39. package/src/events.ts +36 -17
  40. package/src/frame-bytes.ts +26 -6
  41. package/src/handler.ts +628 -286
  42. package/src/local-authority.ts +10 -11
  43. package/src/operations.ts +44 -42
  44. package/src/postgres-storage.ts +336 -226
  45. package/src/pull.ts +394 -414
  46. package/src/push.ts +116 -21
  47. package/src/realtime.ts +85 -129
  48. package/src/relational-rows.ts +4 -107
  49. package/src/seed.ts +13 -19
  50. package/src/segment-download.ts +8 -3
  51. package/src/sqlite-dialect.ts +363 -204
  52. package/src/sqlite-storage.ts +45 -25
  53. package/src/storage-query.ts +102 -0
  54. package/src/storage.ts +185 -58
@@ -14,7 +14,7 @@
14
14
  * promises; the synchronous SQLite driver resolves them with rows it read at
15
15
  * call time. Both classes run the identical `test/storage-contract.ts`.
16
16
  */
17
- import type { PushOperationResult, ScopeMap } from '@syncular/core';
17
+ import type { PushOperationResult, UnitScopes } from '@syncular/core';
18
18
  import {
19
19
  bindAuthoritativePartition,
20
20
  type BoundAuthoritativeQuery,
@@ -37,8 +37,6 @@ import {
37
37
  SYNC_ROW_ID_COLUMN,
38
38
  SYNC_SCOPES_COLUMN,
39
39
  SYNC_VERSION_COLUMN,
40
- commitWindowPageSql,
41
- LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL,
42
40
  upsertSql,
43
41
  upsertValues,
44
42
  } from './relational-rows';
@@ -56,20 +54,23 @@ import type {
56
54
  ClientPushResult,
57
55
  ClientRecord,
58
56
  ClientRecordWriteOptions,
59
- ClientSubscription,
60
57
  CommitChangeSummary,
61
58
  CommitDetail,
62
59
  CommitMetadata,
63
60
  CommitMetadataQuery,
64
61
  CommitPruneQuery,
65
62
  CommitPruneResult,
66
- CommitWindowQuery,
67
63
  DurableJsonValue,
68
64
  IndexRowScanQuery,
65
+ LiveWindow,
66
+ LiveWindowQuery,
69
67
  NewCommit,
68
+ OutboxSequence,
69
+ OutboxSettlement,
70
70
  NewReaction,
71
71
  PartitionRegistryEntry,
72
72
  PrunedReactionCounts,
73
+ PushWatermark,
73
74
  ReactionClaimQuery,
74
75
  ReactionFailure,
75
76
  ReactionFailureUpdate,
@@ -84,13 +85,16 @@ import type {
84
85
  ScopeValueHistoryQuery,
85
86
  ScopeValueQuery,
86
87
  StoredChange,
87
- StoredCommit,
88
88
  StoredPushResult,
89
89
  StoredReaction,
90
90
  StoredRow,
91
91
  Tombstone,
92
92
  } from './storage';
93
- import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query';
93
+ import {
94
+ assertScopeIndexedScan,
95
+ readLiveWindowPages,
96
+ resolveIndexRowScan,
97
+ } from './storage-query';
94
98
 
95
99
  /**
96
100
  * Schema DDL, one statement per `;`-delimited chunk. Native SQLite applies
@@ -105,21 +109,15 @@ import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query';
105
109
  * the Postgres storage documents (§3.1, performance-by-
106
110
  * construction).
107
111
  *
108
- * The old `sync_row_scopes_by_row` index is dropped: with it present the
109
- * planner answered a scope scan of two or more values by walking the index
110
- * in `row_id` order and filtering, instead of seeking the PRIMARY KEY once
111
- * per value. The per-row scope replacement (`deleteSqliteRowScopesSql`)
112
- * binds `(partition, tbl, var, value, row_id)` and needs no secondary index.
113
- * `DROP INDEX IF EXISTS` stays in the DDL so existing databases lose the
114
- * index the next time the DDL runs: on every `SqliteServerStorage`
115
- * construction, and on D1 in `migrate()` and the core phase of
116
- * `migrateSchema`.
112
+ * Changes to the core tables of an existing database (added and dropped
113
+ * columns and indexes) live in `sqliteUpgradeDdl`, which runs after this DDL.
117
114
  */
118
115
  export const SQLITE_DDL = `
119
116
  CREATE TABLE IF NOT EXISTS sync_partitions(
120
117
  partition TEXT PRIMARY KEY,
121
118
  max_commit_seq INTEGER NOT NULL DEFAULT 0,
122
- horizon_seq INTEGER NOT NULL DEFAULT 0
119
+ horizon_seq INTEGER NOT NULL DEFAULT 0,
120
+ delete_floor INTEGER NOT NULL DEFAULT 0
123
121
  );
124
122
  CREATE TABLE IF NOT EXISTS sync_partition_registry(
125
123
  partition TEXT PRIMARY KEY,
@@ -132,7 +130,6 @@ CREATE TABLE IF NOT EXISTS sync_row_scopes(
132
130
  var TEXT NOT NULL, value TEXT NOT NULL, row_id TEXT NOT NULL,
133
131
  PRIMARY KEY(partition, tbl, var, value, row_id)
134
132
  );
135
- DROP INDEX IF EXISTS sync_row_scopes_by_row;
136
133
  CREATE TABLE IF NOT EXISTS sync_commits(
137
134
  partition TEXT NOT NULL, commit_seq INTEGER NOT NULL,
138
135
  client_id TEXT NOT NULL, client_commit_id TEXT NOT NULL,
@@ -158,8 +155,14 @@ CREATE TABLE IF NOT EXISTS sync_change_scopes(
158
155
  CREATE TABLE IF NOT EXISTS sync_push_results(
159
156
  partition TEXT NOT NULL, client_id TEXT NOT NULL,
160
157
  client_commit_id TEXT NOT NULL, result TEXT NOT NULL,
158
+ outbox_id TEXT, client_seq INTEGER,
161
159
  PRIMARY KEY(partition, client_id, client_commit_id)
162
160
  );
161
+ CREATE TABLE IF NOT EXISTS sync_push_watermarks(
162
+ partition TEXT NOT NULL, client_id TEXT NOT NULL, outbox_id TEXT NOT NULL,
163
+ through_seq INTEGER NOT NULL,
164
+ PRIMARY KEY(partition, client_id, outbox_id)
165
+ );
163
166
  CREATE TABLE IF NOT EXISTS sync_reactions(
164
167
  partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,
165
168
  type TEXT NOT NULL, version INTEGER NOT NULL, payload TEXT NOT NULL,
@@ -183,10 +186,12 @@ CREATE INDEX IF NOT EXISTS sync_reactions_dead_letter
183
186
  CREATE TABLE IF NOT EXISTS sync_clients(
184
187
  partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,
185
188
  wire_version INTEGER NOT NULL DEFAULT 1,
186
- cursor INTEGER NOT NULL, subscriptions TEXT NOT NULL,
189
+ cursor INTEGER NOT NULL,
187
190
  updated_at_ms INTEGER NOT NULL,
188
191
  last_round_at_ms INTEGER, last_error_code TEXT,
189
192
  consecutive_failures INTEGER NOT NULL DEFAULT 0,
193
+ schema_version INTEGER, units TEXT, registration_digest BLOB,
194
+ catching_up TEXT NOT NULL DEFAULT '[]',
190
195
  PRIMARY KEY(partition, client_id)
191
196
  );
192
197
  CREATE TABLE IF NOT EXISTS sync_blob_refs(
@@ -217,16 +222,119 @@ CREATE TABLE IF NOT EXISTS sync_writer_fence(
217
222
  );
218
223
  `;
219
224
 
220
- /** Columns added to core tables after their first release: table, column, type. */
221
- export const SQLITE_ADDED_COLUMNS = [
225
+ /**
226
+ * Columns added to core tables after their first release: table, column,
227
+ * type. A client record from before wire version 4 gains NULL `units` and
228
+ * `registration_digest`, so it holds no registration list (§9), and a push
229
+ * result gains NULL `outbox_id` and `client_seq`: it is a result stored
230
+ * before the upgrade, settled by the client's next push (§9). The
231
+ * watermark delete and the settlement range over the `(partition,
232
+ * client_id)` prefix of the primary key, which the watermark keeps to one
233
+ * request's results per outbox.
234
+ */
235
+ const SQLITE_ADDED_COLUMNS = [
222
236
  ['sync_clients', 'wire_version', 'INTEGER NOT NULL DEFAULT 1'],
223
237
  ['sync_clients', 'last_round_at_ms', 'INTEGER'],
224
238
  ['sync_clients', 'last_error_code', 'TEXT'],
225
239
  ['sync_clients', 'consecutive_failures', 'INTEGER NOT NULL DEFAULT 0'],
240
+ ['sync_clients', 'schema_version', 'INTEGER'],
241
+ ['sync_clients', 'units', 'TEXT'],
242
+ ['sync_clients', 'registration_digest', 'BLOB'],
243
+ ['sync_clients', 'catching_up', "TEXT NOT NULL DEFAULT '[]'"],
226
244
  ['sync_tombstones', 'server_version', 'INTEGER'],
227
245
  ['sync_tombstones', 'scopes', 'TEXT'],
246
+ ['sync_push_results', 'outbox_id', 'TEXT'],
247
+ ['sync_push_results', 'client_seq', 'INTEGER'],
248
+ ['sync_partitions', 'delete_floor', 'INTEGER NOT NULL DEFAULT 0'],
249
+ ] as const;
250
+
251
+ /** Core-table columns wire version 4 deleted: the version 3 subscription list. */
252
+ const SQLITE_DROPPED_COLUMNS = [['sync_clients', 'subscriptions']] as const;
253
+
254
+ /**
255
+ * Indexes that `sqliteUpgradeDdl` creates, on a fresh database and an
256
+ * upgraded one alike. `sync_change_scopes_by_key` serves the §4.5 live
257
+ * scan: one range per (variable, value) across tables.
258
+ */
259
+ const SQLITE_ADDED_INDEXES = [
260
+ [
261
+ 'sync_change_scopes_by_key',
262
+ 'CREATE INDEX IF NOT EXISTS sync_change_scopes_by_key ON sync_change_scopes(partition, var, value, commit_seq)',
263
+ ],
228
264
  ] as const;
229
265
 
266
+ /**
267
+ * Indexes that `sqliteUpgradeDdl` drops. With `sync_row_scopes_by_row`
268
+ * present the planner answered a scope scan of two or more values by walking
269
+ * it in `row_id` order and filtering, instead of seeking the PRIMARY KEY once
270
+ * per value; the per-row scope replacement (`deleteSqliteRowScopesSql`)
271
+ * binds the full key and needs no secondary index.
272
+ */
273
+ const SQLITE_DROPPED_INDEXES = ['sync_row_scopes_by_row'] as const;
274
+
275
+ /**
276
+ * One statement that reads what `sqliteUpgradeDdl` needs: the columns of
277
+ * every table with added or dropped columns, and which added indexes exist.
278
+ */
279
+ export const SQLITE_UPGRADE_PROBE: SqliteStatementSpec = {
280
+ sql: `SELECT t.value AS tbl, c.name AS name
281
+ FROM json_each(?) AS t, pragma_table_info(t.value) AS c
282
+ UNION ALL
283
+ SELECT '' AS tbl, name FROM sqlite_master
284
+ WHERE type='index' AND name IN (SELECT value FROM json_each(?))`,
285
+ params: [
286
+ JSON.stringify([
287
+ ...new Set(
288
+ [...SQLITE_ADDED_COLUMNS, ...SQLITE_DROPPED_COLUMNS].map(
289
+ ([table]) => table,
290
+ ),
291
+ ),
292
+ ]),
293
+ JSON.stringify([
294
+ ...SQLITE_ADDED_INDEXES.map(([name]) => name),
295
+ ...SQLITE_DROPPED_INDEXES,
296
+ ]),
297
+ ],
298
+ };
299
+
300
+ /**
301
+ * The statements that bring the core tables of a database from an earlier
302
+ * release to the current DDL, given the `SQLITE_UPGRADE_PROBE` rows: missing
303
+ * columns, dropped columns, dropped indexes, then missing indexes. Empty for
304
+ * a current database. Run after `SQLITE_DDL`: on every `SqliteServerStorage`
305
+ * construction, and on D1 in `migrate()`, the core phase of `migrateSchema`
306
+ * and the first open of a database at the current schema version.
307
+ */
308
+ export function sqliteUpgradeDdl(
309
+ present: readonly { readonly tbl: string; readonly name: string }[],
310
+ ): string[] {
311
+ const has = (table: string, name: string) =>
312
+ present.some((row) => row.tbl === table && row.name === name);
313
+ return [
314
+ ...SQLITE_ADDED_COLUMNS.filter(([table, name]) => !has(table, name)).map(
315
+ ([table, name, definition]) =>
316
+ `ALTER TABLE ${table} ADD COLUMN ${name} ${definition}`,
317
+ ),
318
+ // A database from before wire version 4 (and the delete floor) starts
319
+ // the delete floor at the highest version its tombstones recorded
320
+ // (§2.2) and the horizon at its last commit (§9).
321
+ ...(has('sync_partitions', 'delete_floor')
322
+ ? []
323
+ : [
324
+ `UPDATE sync_partitions SET horizon_seq=max_commit_seq, delete_floor=coalesce((SELECT max(server_version) FROM sync_tombstones t WHERE t.partition=sync_partitions.partition),0)`,
325
+ ]),
326
+ ...SQLITE_DROPPED_COLUMNS.filter(([table, name]) => has(table, name)).map(
327
+ ([table, name]) => `ALTER TABLE ${table} DROP COLUMN ${name}`,
328
+ ),
329
+ ...SQLITE_DROPPED_INDEXES.filter((name) => has('', name)).map(
330
+ (name) => `DROP INDEX IF EXISTS ${name}`,
331
+ ),
332
+ ...SQLITE_ADDED_INDEXES.filter(([name]) => !has('', name)).map(
333
+ ([, ddl]) => ddl,
334
+ ),
335
+ ];
336
+ }
337
+
230
338
  /** Split the DDL into individual statements (D1 applies them one by one). */
231
339
  export function sqliteDdlStatements(): string[] {
232
340
  return SQLITE_DDL.split(';')
@@ -436,102 +544,6 @@ export function toStoredChange(record: SqliteChangeRecord): StoredChange {
436
544
  };
437
545
  }
438
546
 
439
- /**
440
- * One result row of `commitWindowPageSql` (candidate LEFT JOIN commit meta
441
- * LEFT JOIN changes): meta/change columns are NULL when the joined row
442
- * vanished (see the builder's LEFT JOIN contract). `payload` is a BLOB —
443
- * Native SQLite hands back `Uint8Array`, D1 `ArrayBuffer`; `toStoredChange`
444
- * normalizes via `asUint8Array`.
445
- */
446
- export interface SqliteCommitWindowRecord {
447
- commit_seq: number;
448
- actor_id: string | null;
449
- created_at_ms: number | null;
450
- tbl: string | null;
451
- row_id: string | null;
452
- op: number | null;
453
- row_version: number | null;
454
- scopes: string | null;
455
- payload: Uint8Array | null;
456
- }
457
-
458
- /**
459
- * Fold one `commitWindowPageSql` page into commits, preserving the exact
460
- * semantics of the old per-candidate loop:
461
- *
462
- * - rows arrive ordered (commit_seq, idx); consecutive rows with the same
463
- * `commit_seq` are one candidate commit;
464
- * - every candidate advances the cursor (`lastSeq`) — including vanished
465
- * commits (NULL meta) and commits whose changes all fail the exact
466
- * multi-variable scope verification (`matchesEffective`);
467
- * - a commit is emitted only with its matching changes, in `idx` order;
468
- * - stops after the commit that accumulates at least `remaining` matching
469
- * changes (never splitting a commit).
470
- *
471
- * `candidateCount` counts the processed candidates so the caller can detect
472
- * a short page (window exhausted) exactly as it did with the old candidate
473
- * query.
474
- */
475
- export function collectCommitWindowPage(
476
- records: readonly SqliteCommitWindowRecord[],
477
- scopeFilter: ScopeMap,
478
- remaining: number,
479
- ): {
480
- commits: StoredCommit[];
481
- delivered: number;
482
- lastSeq: number;
483
- candidateCount: number;
484
- } {
485
- const commits: StoredCommit[] = [];
486
- let delivered = 0;
487
- let lastSeq = 0;
488
- let candidateCount = 0;
489
- let i = 0;
490
- while (i < records.length) {
491
- const head = records[i];
492
- if (head === undefined) break;
493
- const seq = head.commit_seq;
494
- candidateCount += 1;
495
- lastSeq = seq;
496
- const changes: StoredChange[] = [];
497
- for (; i < records.length; i++) {
498
- const record = records[i];
499
- if (record === undefined || record.commit_seq !== seq) break;
500
- // NULL tbl: a candidate with no change rows for the table (LEFT JOIN
501
- // contract) — it still advanced the cursor above.
502
- if (
503
- record.tbl === null ||
504
- record.row_id === null ||
505
- record.op === null ||
506
- record.scopes === null
507
- ) {
508
- continue;
509
- }
510
- const change = toStoredChange({
511
- tbl: record.tbl,
512
- row_id: record.row_id,
513
- op: record.op,
514
- row_version: record.row_version,
515
- scopes: record.scopes,
516
- payload: record.payload,
517
- });
518
- if (matchesEffective(change.scopes, scopeFilter)) changes.push(change);
519
- }
520
- // NULL meta: the commit vanished — cursor advanced, nothing emitted.
521
- if (head.actor_id === null || head.created_at_ms === null) continue;
522
- if (changes.length === 0) continue;
523
- commits.push({
524
- commitSeq: seq,
525
- createdAtMs: head.created_at_ms,
526
- actorId: head.actor_id,
527
- changes,
528
- });
529
- delivered += changes.length;
530
- if (delivered >= remaining) break;
531
- }
532
- return { commits, delivered, lastSeq, candidateCount };
533
- }
534
-
535
547
  interface SqliteReactionRecord {
536
548
  idempotency_key: string;
537
549
  type: string;
@@ -583,13 +595,19 @@ interface SqliteClientRecord {
583
595
  actor_id: string;
584
596
  wire_version: number;
585
597
  cursor: number;
586
- subscriptions: string;
587
598
  updated_at_ms: number;
599
+ schema_version: number | null;
600
+ units: string | null;
601
+ registration_digest: Uint8Array | null;
602
+ catching_up: string;
588
603
  last_round_at_ms: number | null;
589
604
  last_error_code: string | null;
590
605
  consecutive_failures: number;
591
606
  }
592
607
 
608
+ const CLIENT_RECORD_COLUMNS =
609
+ 'client_id, actor_id, wire_version, cursor, updated_at_ms, schema_version, units, registration_digest, catching_up, last_round_at_ms, last_error_code, consecutive_failures';
610
+
593
611
  function toClientRecord(record: SqliteClientRecord): ClientRecord {
594
612
  return {
595
613
  clientId: record.client_id,
@@ -597,7 +615,18 @@ function toClientRecord(record: SqliteClientRecord): ClientRecord {
597
615
  wireVersion: record.wire_version,
598
616
  cursor: record.cursor,
599
617
  updatedAtMs: record.updated_at_ms,
600
- subscriptions: JSON.parse(record.subscriptions) as ClientSubscription[],
618
+ ...(record.schema_version !== null
619
+ ? { schemaVersion: record.schema_version }
620
+ : {}),
621
+ ...(record.units !== null && record.registration_digest !== null
622
+ ? {
623
+ registration: {
624
+ units: JSON.parse(record.units) as UnitScopes[],
625
+ digest: asUint8Array(record.registration_digest),
626
+ },
627
+ }
628
+ : {}),
629
+ catchingUp: JSON.parse(record.catching_up) as ClientRecord['catchingUp'],
601
630
  ...clientRoundState(
602
631
  record.last_round_at_ms,
603
632
  record.last_error_code,
@@ -1135,64 +1164,71 @@ export abstract class SqliteFamilyStorage {
1135
1164
 
1136
1165
  async latestScopedChangeSeq(
1137
1166
  partition: string,
1138
- query: { readonly table: string; readonly scopeFilter: ScopeMap },
1167
+ query: { readonly table: string; readonly scopes: UnitScopes },
1139
1168
  ): Promise<number> {
1140
- const firstVariable = Object.keys(query.scopeFilter).sort()[0];
1141
- const values =
1142
- firstVariable === undefined
1143
- ? []
1144
- : (query.scopeFilter[firstVariable] ?? []);
1145
- if (firstVariable === undefined || values.length === 0) return 0;
1169
+ const variable = Object.keys(query.scopes).sort()[0];
1170
+ const value = variable === undefined ? undefined : query.scopes[variable];
1171
+ if (variable === undefined || value === undefined) return 0;
1172
+ // One index-only max over the (partition, tbl, var, value) prefix.
1146
1173
  const [row] = await this.guardedAll<{ seq: number | null }>(
1147
- LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL,
1148
- [partition, query.table, firstVariable, JSON.stringify(values)],
1174
+ 'SELECT max(commit_seq) AS seq FROM sync_change_scopes WHERE partition=? AND tbl=? AND var=? AND value=?',
1175
+ [partition, query.table, variable, value],
1149
1176
  );
1150
1177
  return row?.seq ?? 0;
1151
1178
  }
1152
1179
 
1153
- async readCommitWindow(
1180
+ readLiveWindow(
1154
1181
  partition: string,
1155
- query: CommitWindowQuery,
1156
- ): Promise<StoredCommit[]> {
1157
- const variables = Object.keys(query.scopeFilter).sort();
1158
- const firstVariable = variables[0];
1159
- if (firstVariable === undefined) return [];
1160
- const firstValues = query.scopeFilter[firstVariable] ?? [];
1161
- if (firstValues.length === 0) return [];
1162
- // Candidates via the inverted index (one variable) LEFT JOINed to the
1163
- // commit meta + the table's changes — one statement per page, never two
1164
- // per candidate (see `commitWindowPageSql`). Exact multi-variable
1165
- // verification against the stored scope map in `collectCommitWindowPage`.
1166
- const sql = commitWindowPageSql(firstValues.length, 'sqlite');
1167
- const commits: StoredCommit[] = [];
1168
- let deliveredChanges = 0;
1169
- let afterSeq = query.afterSeq;
1170
- const batchSize = Math.max(64, query.limitChanges);
1171
- while (deliveredChanges < query.limitChanges) {
1172
- const records = await this.guardedAll<SqliteCommitWindowRecord>(sql, [
1173
- partition,
1174
- query.table,
1175
- firstVariable,
1176
- JSON.stringify(firstValues),
1177
- afterSeq,
1178
- query.throughSeq,
1179
- batchSize,
1180
- partition,
1181
- partition,
1182
- query.table,
1183
- ]);
1184
- if (records.length === 0) break;
1185
- const page = collectCommitWindowPage(
1186
- records,
1187
- query.scopeFilter,
1188
- query.limitChanges - deliveredChanges,
1189
- );
1190
- commits.push(...page.commits);
1191
- deliveredChanges += page.delivered;
1192
- afterSeq = page.lastSeq;
1193
- if (page.candidateCount < batchSize) break;
1194
- }
1195
- return commits;
1182
+ query: LiveWindowQuery,
1183
+ ): Promise<LiveWindow> {
1184
+ // One statement per page: the unit keys bind as one JSON array, and the
1185
+ // CROSS JOIN keeps them the outer loop, so each key seeks one range of
1186
+ // `sync_change_scopes_by_key` (INDEXED BY fails loudly when the index is
1187
+ // missing). The candidates LEFT JOIN the commit meta and every change of
1188
+ // the commit; a candidate whose commit vanished comes back with NULL
1189
+ // meta and still advances the scan.
1190
+ return readLiveWindowPages(query, async (keys, limit) =>
1191
+ (
1192
+ await this.guardedAll<{
1193
+ commit_seq: number;
1194
+ actor_id: string | null;
1195
+ created_at_ms: number | null;
1196
+ tbl: string | null;
1197
+ row_id: string;
1198
+ op: number;
1199
+ row_version: number | null;
1200
+ scopes: string;
1201
+ payload: Uint8Array | null;
1202
+ }>(
1203
+ `SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
1204
+ ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
1205
+ FROM (SELECT DISTINCT s.commit_seq AS commit_seq
1206
+ FROM json_each(?2) AS k
1207
+ CROSS JOIN sync_change_scopes AS s INDEXED BY sync_change_scopes_by_key
1208
+ ON s.partition=?1 AND s.var=json_extract(k.value, '$.var')
1209
+ AND s.value=json_extract(k.value, '$.value')
1210
+ AND s.commit_seq>json_extract(k.value, '$.after') AND s.commit_seq<=?3
1211
+ ORDER BY s.commit_seq LIMIT ?4) AS c
1212
+ LEFT JOIN sync_commits AS m ON m.partition=?1 AND m.commit_seq=c.commit_seq
1213
+ LEFT JOIN sync_changes AS ch ON ch.partition=?1 AND ch.commit_seq=c.commit_seq
1214
+ ORDER BY c.commit_seq, ch.idx`,
1215
+ [partition, JSON.stringify(keys), query.throughSeq, limit],
1216
+ )
1217
+ ).map((record) => ({
1218
+ commitSeq: record.commit_seq,
1219
+ ...(record.actor_id !== null && record.created_at_ms !== null
1220
+ ? {
1221
+ meta: {
1222
+ actorId: record.actor_id,
1223
+ createdAtMs: record.created_at_ms,
1224
+ },
1225
+ }
1226
+ : {}),
1227
+ ...(record.tbl !== null
1228
+ ? { change: toStoredChange({ ...record, tbl: record.tbl }) }
1229
+ : {}),
1230
+ })),
1231
+ );
1196
1232
  }
1197
1233
 
1198
1234
  async scanRows(partition: string, query: RowScanQuery): Promise<StoredRow[]> {
@@ -1262,7 +1298,7 @@ export abstract class SqliteFamilyStorage {
1262
1298
  clientId: string,
1263
1299
  ): Promise<ClientRecord | undefined> {
1264
1300
  const [record] = await this.all<SqliteClientRecord>(
1265
- 'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=? AND client_id=?',
1301
+ `SELECT ${CLIENT_RECORD_COLUMNS} FROM sync_clients WHERE partition=? AND client_id=?`,
1266
1302
  [partition, clientId],
1267
1303
  );
1268
1304
  return record === undefined ? undefined : toClientRecord(record);
@@ -1274,17 +1310,26 @@ export abstract class SqliteFamilyStorage {
1274
1310
  options?: ClientRecordWriteOptions,
1275
1311
  ): Promise<void> {
1276
1312
  // One row per (partition, client_id): every non-key field is written
1277
- // from the new record. `keepRoundOutcome` leaves the round-state columns
1278
- // alone (§1.5, §4.5).
1313
+ // from the new record, except the registration list and digest, which
1314
+ // only a traveling list replaces (§4.5). `keepRoundOutcome` leaves the
1315
+ // round-state columns alone (§1.5, §4.5).
1316
+ const registration = record.registration;
1279
1317
  await this.all(
1280
- `INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms) VALUES (?,?,?,?,?,?,?,?)
1318
+ `INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, updated_at_ms, last_round_at_ms, schema_version, units, registration_digest, catching_up) VALUES (?,?,?,?,?,?,?,?,?,?,?)
1281
1319
  ON CONFLICT(partition, client_id) DO UPDATE SET
1282
1320
  actor_id=excluded.actor_id,
1283
1321
  wire_version=excluded.wire_version,
1284
1322
  cursor=excluded.cursor,
1285
- subscriptions=excluded.subscriptions,
1286
1323
  updated_at_ms=excluded.updated_at_ms,
1287
- last_round_at_ms=excluded.last_round_at_ms${
1324
+ last_round_at_ms=excluded.last_round_at_ms,
1325
+ schema_version=excluded.schema_version,${
1326
+ registration === undefined
1327
+ ? ''
1328
+ : `
1329
+ units=excluded.units,
1330
+ registration_digest=excluded.registration_digest,`
1331
+ }
1332
+ catching_up=excluded.catching_up${
1288
1333
  options?.keepRoundOutcome === true
1289
1334
  ? ''
1290
1335
  : `,
@@ -1297,9 +1342,12 @@ export abstract class SqliteFamilyStorage {
1297
1342
  record.actorId,
1298
1343
  record.wireVersion,
1299
1344
  record.cursor,
1300
- JSON.stringify(record.subscriptions),
1301
- record.updatedAtMs,
1302
1345
  record.updatedAtMs,
1346
+ record.lastRoundAtMs ?? record.updatedAtMs,
1347
+ record.schemaVersion ?? null,
1348
+ registration === undefined ? null : JSON.stringify(registration.units),
1349
+ registration?.digest ?? null,
1350
+ JSON.stringify(record.catchingUp),
1303
1351
  ],
1304
1352
  );
1305
1353
  }
@@ -1328,27 +1376,19 @@ export abstract class SqliteFamilyStorage {
1328
1376
  cursor: number,
1329
1377
  updatedAtMs: number,
1330
1378
  ): Promise<void> {
1379
+ // §8.2: the ack never raises the record cursor above a catching-up
1380
+ // unit's cursor, so pruning keeps that unit's history (§4.6).
1331
1381
  await this.all(
1332
1382
  `UPDATE sync_clients
1333
- SET cursor=MAX(cursor, ?), updated_at_ms=MAX(updated_at_ms, ?)
1383
+ SET cursor=MAX(cursor, MIN(?, coalesce((
1384
+ SELECT min(json_extract(u.value, '$.cursor'))
1385
+ FROM json_each(sync_clients.catching_up) AS u
1386
+ WHERE json_extract(u.value, '$.cursor')>=0), ?))),
1387
+ updated_at_ms=MAX(updated_at_ms, ?)
1334
1388
  WHERE partition=? AND client_id=? AND actor_id=?
1335
1389
  AND EXISTS (SELECT 1 FROM sync_partition_registry
1336
1390
  WHERE partition=sync_clients.partition AND log_epoch=?)`,
1337
- [cursor, updatedAtMs, partition, clientId, actorId, logEpoch],
1338
- );
1339
- }
1340
-
1341
- async updateClientCursor(
1342
- partition: string,
1343
- clientId: string,
1344
- cursor: number,
1345
- updatedAtMs: number,
1346
- ): Promise<void> {
1347
- await this.all(
1348
- `UPDATE sync_clients
1349
- SET cursor=MAX(cursor, ?), updated_at_ms=MAX(updated_at_ms, ?)
1350
- WHERE partition=? AND client_id=?`,
1351
- [cursor, updatedAtMs, partition, clientId],
1391
+ [cursor, cursor, updatedAtMs, partition, clientId, actorId, logEpoch],
1352
1392
  );
1353
1393
  }
1354
1394
 
@@ -1492,7 +1532,7 @@ export abstract class SqliteFamilyStorage {
1492
1532
  async listClientRecords(partition: string): Promise<ClientRecord[]> {
1493
1533
  return (
1494
1534
  await this.all<SqliteClientRecord>(
1495
- 'SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC',
1535
+ `SELECT ${CLIENT_RECORD_COLUMNS} FROM sync_clients WHERE partition=? ORDER BY updated_at_ms DESC`,
1496
1536
  [partition],
1497
1537
  )
1498
1538
  ).map(toClientRecord);
@@ -1701,16 +1741,39 @@ export abstract class SqliteFamilyStorage {
1701
1741
  }));
1702
1742
  }
1703
1743
 
1744
+ async listPushWatermarks(
1745
+ partition: string,
1746
+ clientCommitId: string,
1747
+ ): Promise<PushWatermark[]> {
1748
+ // The clients come from the retained commit log, which has no index on
1749
+ // `client_commit_id`; the read is bounded by retention (§4.6).
1750
+ return (
1751
+ await this.all<{
1752
+ client_id: string;
1753
+ outbox_id: string;
1754
+ through_seq: number;
1755
+ }>(
1756
+ `SELECT client_id, outbox_id, through_seq FROM sync_push_watermarks
1757
+ WHERE partition=? AND client_id IN (
1758
+ SELECT client_id FROM sync_commits WHERE partition=? AND client_commit_id=?)
1759
+ ORDER BY client_id, outbox_id`,
1760
+ [partition, partition, clientCommitId],
1761
+ )
1762
+ ).map((row) => ({
1763
+ clientId: row.client_id,
1764
+ outboxId: row.outbox_id,
1765
+ throughSeq: row.through_seq,
1766
+ }));
1767
+ }
1768
+
1704
1769
  async scopeActivity(
1705
1770
  partition: string,
1706
1771
  query: ScopeActivityQuery,
1707
1772
  ): Promise<ScopeCommitActivity[]> {
1708
1773
  // One statement for the page: candidate (commit, table) pairs from the
1709
- // change-scope index, joined to the commit metadata with the table's
1710
- // change count. `tbl` sits between `partition` and `var` in the index key,
1711
- // so the `var`/`value` filter ranges over the partition's change-scope
1712
- // entries; only the LIMIT bounds the output. A candidate whose commit was
1713
- // pruned has no metadata and is skipped.
1774
+ // `sync_change_scopes_by_key` range of the key, newest first, joined to
1775
+ // the commit metadata with the table's change count. A candidate whose
1776
+ // commit was pruned has no metadata and is skipped.
1714
1777
  return (
1715
1778
  await this.all<{
1716
1779
  commit_seq: number;
@@ -1892,7 +1955,7 @@ export abstract class SqliteFamilyTransaction {
1892
1955
  change.table,
1893
1956
  change.rowId,
1894
1957
  change.op === 'upsert' ? 1 : 2,
1895
- change.rowVersion ?? null,
1958
+ change.rowVersion,
1896
1959
  JSON.stringify(change.scopes),
1897
1960
  change.payload ?? null,
1898
1961
  ],
@@ -1912,7 +1975,7 @@ export abstract class SqliteFamilyTransaction {
1912
1975
  change.table,
1913
1976
  change.rowId,
1914
1977
  commitSeq,
1915
- change.deletedVersion ?? null,
1978
+ change.rowVersion,
1916
1979
  JSON.stringify(change.scopes),
1917
1980
  ],
1918
1981
  );
@@ -1931,12 +1994,108 @@ export abstract class SqliteFamilyTransaction {
1931
1994
  clientId: string,
1932
1995
  clientCommitId: string,
1933
1996
  result: StoredPushResult,
1997
+ sequence?: OutboxSequence,
1934
1998
  ): Promise<void> {
1935
1999
  this.assertOpen();
1936
2000
  this.run(
1937
- 'INSERT OR IGNORE INTO sync_push_results(partition, client_id, client_commit_id, result) VALUES (?,?,?,?)',
1938
- [this.partition, clientId, clientCommitId, serializePushResult(result)],
2001
+ 'INSERT OR IGNORE INTO sync_push_results(partition, client_id, client_commit_id, result, outbox_id, client_seq) VALUES (?,?,?,?,?,?)',
2002
+ [
2003
+ this.partition,
2004
+ clientId,
2005
+ clientCommitId,
2006
+ serializePushResult(result),
2007
+ sequence?.outboxId ?? null,
2008
+ sequence?.clientSeq ?? null,
2009
+ ],
2010
+ );
2011
+ }
2012
+
2013
+ /**
2014
+ * The §9 settlement and §6.3.2 steps 1 and 2 for one request, then the
2015
+ * idempotency read of its first commit. The reads run before the writes,
2016
+ * so D1, which buffers the writes until commit, answers exactly as SQLite:
2017
+ * the first commit's result counts as deleted when the writes delete it.
2018
+ */
2019
+ protected async settleOutbox(
2020
+ clientId: string,
2021
+ clientCommitId: string,
2022
+ outbox: OutboxSettlement,
2023
+ ): Promise<{
2024
+ readonly watermark: number;
2025
+ readonly persisted: StoredPushResult | undefined;
2026
+ }> {
2027
+ const first = outbox.commits[0];
2028
+ const last = outbox.commits.at(-1);
2029
+ if (
2030
+ first === undefined ||
2031
+ last === undefined ||
2032
+ first.clientCommitId !== clientCommitId
2033
+ ) {
2034
+ throw new Error("outbox settlement rides the request's first commit");
2035
+ }
2036
+ const p = this.partition;
2037
+ const [state] = await this.all<{
2038
+ stored: number | null;
2039
+ last_sequenceless: number;
2040
+ result: string | null;
2041
+ outbox_id: string | null;
2042
+ client_seq: number | null;
2043
+ }>(
2044
+ `SELECT (SELECT through_seq FROM sync_push_watermarks
2045
+ WHERE partition=?1 AND client_id=?2 AND outbox_id=?3) AS stored,
2046
+ EXISTS (SELECT 1 FROM sync_push_results WHERE partition=?1
2047
+ AND client_id=?2 AND client_commit_id=?5 AND outbox_id IS NULL) AS last_sequenceless,
2048
+ r.result AS result, r.outbox_id AS outbox_id, r.client_seq AS client_seq
2049
+ FROM (SELECT 1) LEFT JOIN sync_push_results AS r
2050
+ ON r.partition=?1 AND r.client_id=?2 AND r.client_commit_id=?4`,
2051
+ [p, clientId, outbox.outboxId, first.clientCommitId, last.clientCommitId],
2052
+ );
2053
+ const watermark = Math.max(state?.stored ?? 0, first.clientSeq);
2054
+ const commits = JSON.stringify(
2055
+ outbox.commits.map((commit) => [commit.clientCommitId, commit.clientSeq]),
2056
+ );
2057
+ // §9 (b), then (a): with no pre-upgrade result on the request's last
2058
+ // commit, every remaining one outside the request left the outbox.
2059
+ if (state?.last_sequenceless !== 1) {
2060
+ this.run(
2061
+ `DELETE FROM sync_push_results
2062
+ WHERE partition=? AND client_id=? AND outbox_id IS NULL
2063
+ AND client_commit_id NOT IN (SELECT json_extract(value, '$[0]') FROM json_each(?))`,
2064
+ [p, clientId, commits],
2065
+ );
2066
+ }
2067
+ this.run(
2068
+ `UPDATE sync_push_results SET outbox_id=?3, client_seq=(
2069
+ SELECT json_extract(c.value, '$[1]') FROM json_each(?4) AS c
2070
+ WHERE json_extract(c.value, '$[0]')=sync_push_results.client_commit_id)
2071
+ WHERE partition=?1 AND client_id=?2 AND outbox_id IS NULL
2072
+ AND client_commit_id IN (SELECT json_extract(value, '$[0]') FROM json_each(?4))`,
2073
+ [p, clientId, outbox.outboxId, commits],
2074
+ );
2075
+ // §6.3.2 steps 1 and 2.
2076
+ this.run(
2077
+ `INSERT INTO sync_push_watermarks(partition, client_id, outbox_id, through_seq) VALUES (?,?,?,?)
2078
+ ON CONFLICT(partition, client_id, outbox_id) DO UPDATE SET through_seq=max(through_seq, excluded.through_seq)`,
2079
+ [p, clientId, outbox.outboxId, watermark],
1939
2080
  );
2081
+ this.run(
2082
+ 'DELETE FROM sync_push_results WHERE partition=? AND client_id=? AND outbox_id=? AND client_seq<?',
2083
+ [p, clientId, outbox.outboxId, watermark],
2084
+ );
2085
+ // The sequence step 2 sees: (a) gives a pre-upgrade result the first
2086
+ // commit's sequence; a result of this outbox keeps its stored one.
2087
+ const deleted =
2088
+ state?.outbox_id === null
2089
+ ? first.clientSeq < watermark
2090
+ : state?.outbox_id === outbox.outboxId &&
2091
+ (state.client_seq ?? 0) < watermark;
2092
+ return {
2093
+ watermark,
2094
+ persisted:
2095
+ state?.result === null || state?.result === undefined || deleted
2096
+ ? undefined
2097
+ : deserializePushResult(state.result),
2098
+ };
1940
2099
  }
1941
2100
 
1942
2101
  async enqueueReactions(reactions: readonly NewReaction[]): Promise<void> {