@syncular/server 0.59.0 → 0.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. package/README.md +20 -16
  2. package/dist/admin.d.ts +32 -20
  3. package/dist/admin.js +42 -40
  4. package/dist/context.d.ts +6 -8
  5. package/dist/context.js +1 -3
  6. package/dist/d1-storage.js +42 -24
  7. package/dist/errors.js +8 -0
  8. package/dist/events.d.ts +35 -17
  9. package/dist/frame-bytes.js +24 -6
  10. package/dist/handler.js +486 -223
  11. package/dist/local-authority.js +7 -9
  12. package/dist/operations.d.ts +5 -2
  13. package/dist/operations.js +20 -29
  14. package/dist/postgres-storage.d.ts +24 -19
  15. package/dist/postgres-storage.js +249 -168
  16. package/dist/pull.d.ts +68 -53
  17. package/dist/pull.js +258 -287
  18. package/dist/push.d.ts +12 -3
  19. package/dist/push.js +95 -32
  20. package/dist/realtime.d.ts +17 -11
  21. package/dist/realtime.js +65 -90
  22. package/dist/relational-rows.d.ts +0 -57
  23. package/dist/relational-rows.js +0 -97
  24. package/dist/seed.js +10 -15
  25. package/dist/segment-download.d.ts +4 -1
  26. package/dist/segment-download.js +4 -2
  27. package/dist/sqlite-dialect.d.ts +36 -59
  28. package/dist/sqlite-dialect.js +255 -159
  29. package/dist/sqlite-storage.js +24 -20
  30. package/dist/storage-query.d.ts +30 -1
  31. package/dist/storage-query.js +68 -0
  32. package/dist/storage.d.ts +156 -54
  33. package/dist/storage.js +15 -0
  34. package/package.json +2 -2
  35. package/src/admin.ts +87 -59
  36. package/src/context.ts +7 -11
  37. package/src/d1-storage.ts +49 -33
  38. package/src/errors.ts +8 -0
  39. package/src/events.ts +36 -17
  40. package/src/frame-bytes.ts +26 -6
  41. package/src/handler.ts +628 -286
  42. package/src/local-authority.ts +10 -11
  43. package/src/operations.ts +44 -42
  44. package/src/postgres-storage.ts +336 -226
  45. package/src/pull.ts +394 -414
  46. package/src/push.ts +116 -21
  47. package/src/realtime.ts +85 -129
  48. package/src/relational-rows.ts +0 -105
  49. package/src/seed.ts +13 -19
  50. package/src/segment-download.ts +8 -3
  51. package/src/sqlite-dialect.ts +363 -204
  52. package/src/sqlite-storage.ts +45 -25
  53. package/src/storage-query.ts +102 -0
  54. package/src/storage.ts +185 -58
@@ -1,25 +1,30 @@
1
1
  import { validateCommitPruneQuery } from './prune.js';
2
2
  import { StorageQueryError } from './storage-errors.js';
3
- import { clientRoundState, PARTITION_TOUCH_INTERVAL_MS, rowsLiveAfterDelete, serveGateRefusal, } from './storage.js';
3
+ import { clientRoundState, PARTITION_TOUCH_INTERVAL_MS, commitDeleteFloor, rowsLiveAfterDelete, serveGateRefusal, } from './storage.js';
4
4
  import { serveNotReadyError } from './readiness.js';
5
5
  import { bindAuthoritativePartition, postgresPlaceholders, prepareAuthoritativeQuery, } from './authoritative-query.js';
6
6
  import { syncError } from './errors.js';
7
7
  import { asBytes, asNumber, } from './pg-executor.js';
8
- import { assertPhysicalColumns, assertStoredLayouts, COMMIT_WINDOW_BATCH_SQL, LATEST_SCOPED_CHANGE_BATCH_SQL, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, rowsReferencingBlobSql, selectRowSql, selectRowsSql, prefetchRowsBatchSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
8
+ import { assertPhysicalColumns, assertStoredLayouts, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, rowsReferencingBlobSql, selectRowSql, selectRowsSql, prefetchRowsBatchSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
9
9
  import { matchesEffective, sha256Hex } from './scopes.js';
10
10
  import { isPostgresConstraintError, StorageConstraintError, } from './storage-errors.js';
11
- import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js';
11
+ import { assertScopeIndexedScan, readLiveWindowPages, resolveIndexRowScan, } from './storage-query.js';
12
12
  /**
13
13
  * Schema DDL. Applied by `PostgresServerStorage.migrate()` (idempotent).
14
14
  *
15
15
  * Covering index design (the reason this file exists):
16
- * - `sync_change_scopes_pk (partition, tbl, var, value, commit_seq)` — the
17
- * candidate scan for `readCommitWindow` ranges on the (partition, tbl,
18
- * var, value IN …) prefix and returns `commit_seq` already ascending;
19
- * - `sync_row_scopes_pk (partition, tbl, var, value, row_id)` — same shape
20
- * for `scanRows`, ordered by `row_id`.
21
- * Both are the PRIMARY KEY, so they are the clustering/covering index for
22
- * their table.
16
+ * - `sync_change_scopes_by_key (partition, var, value, commit_seq)`: the
17
+ * §4.5 live scan ranges once per unit key on the (partition, var, value)
18
+ * prefix across tables and returns `commit_seq` already ascending;
19
+ * - `sync_change_scopes_pk (partition, tbl, var, value, commit_seq)`:
20
+ * the §4.7 scope pin, erasure and pruning name the table;
21
+ * - `sync_row_scopes_pk (partition, tbl, var, value, row_id)`: the same
22
+ * shape for `scanRows`, ordered by `row_id`.
23
+ *
24
+ * The §6.3.2 watermark delete and the §9 settlement of results stored
25
+ * before wire version 4 (NULL `outbox_id`) range over the `(partition,
26
+ * client_id)` prefix of `sync_push_results`' PRIMARY KEY, which the
27
+ * watermark keeps to one request's results per outbox.
23
28
  *
24
29
  * `sync_row_scopes_by_row (partition, tbl, row_id)` is the one secondary
25
30
  * scope index: `writeRowsOn` and `deleteRow` replace a row's scope entries
@@ -42,7 +47,8 @@ export const POSTGRES_DDL = `
42
47
  CREATE TABLE IF NOT EXISTS sync_partitions(
43
48
  partition TEXT PRIMARY KEY,
44
49
  max_commit_seq BIGINT NOT NULL DEFAULT 0,
45
- horizon_seq BIGINT NOT NULL DEFAULT 0
50
+ horizon_seq BIGINT NOT NULL DEFAULT 0,
51
+ delete_floor BIGINT NOT NULL DEFAULT 0
46
52
  );
47
53
  CREATE TABLE IF NOT EXISTS sync_partition_registry(
48
54
  partition TEXT PRIMARY KEY,
@@ -79,11 +85,21 @@ CREATE TABLE IF NOT EXISTS sync_change_scopes(
79
85
  var TEXT NOT NULL, value TEXT NOT NULL, commit_seq BIGINT NOT NULL,
80
86
  PRIMARY KEY(partition, tbl, var, value, commit_seq)
81
87
  );
88
+ CREATE INDEX IF NOT EXISTS sync_change_scopes_by_key
89
+ ON sync_change_scopes(partition, var, value, commit_seq);
82
90
  CREATE TABLE IF NOT EXISTS sync_push_results(
83
91
  partition TEXT NOT NULL, client_id TEXT NOT NULL,
84
92
  client_commit_id TEXT NOT NULL, result JSONB NOT NULL,
93
+ outbox_id TEXT, client_seq BIGINT,
85
94
  PRIMARY KEY(partition, client_id, client_commit_id)
86
95
  );
96
+ ALTER TABLE sync_push_results ADD COLUMN IF NOT EXISTS outbox_id TEXT;
97
+ ALTER TABLE sync_push_results ADD COLUMN IF NOT EXISTS client_seq BIGINT;
98
+ CREATE TABLE IF NOT EXISTS sync_push_watermarks(
99
+ partition TEXT NOT NULL, client_id TEXT NOT NULL, outbox_id TEXT NOT NULL,
100
+ through_seq BIGINT NOT NULL,
101
+ PRIMARY KEY(partition, client_id, outbox_id)
102
+ );
87
103
  CREATE TABLE IF NOT EXISTS sync_reactions(
88
104
  partition TEXT NOT NULL, idempotency_key TEXT NOT NULL,
89
105
  type TEXT NOT NULL, version INTEGER NOT NULL, payload JSONB NOT NULL,
@@ -107,10 +123,12 @@ CREATE INDEX IF NOT EXISTS sync_reactions_dead_letter
107
123
  CREATE TABLE IF NOT EXISTS sync_clients(
108
124
  partition TEXT NOT NULL, client_id TEXT NOT NULL, actor_id TEXT NOT NULL,
109
125
  wire_version INTEGER NOT NULL DEFAULT 1,
110
- cursor BIGINT NOT NULL, subscriptions JSONB NOT NULL,
126
+ cursor BIGINT NOT NULL,
111
127
  updated_at_ms BIGINT NOT NULL,
112
128
  last_round_at_ms BIGINT, last_error_code TEXT,
113
129
  consecutive_failures INTEGER NOT NULL DEFAULT 0,
130
+ schema_version INTEGER, units JSONB, registration_digest BYTEA,
131
+ catching_up JSONB NOT NULL DEFAULT '[]',
114
132
  PRIMARY KEY(partition, client_id)
115
133
  );
116
134
  ALTER TABLE sync_clients
@@ -119,6 +137,19 @@ ALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_round_at_ms BIGINT;
119
137
  ALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS last_error_code TEXT;
120
138
  ALTER TABLE sync_clients
121
139
  ADD COLUMN IF NOT EXISTS consecutive_failures INTEGER NOT NULL DEFAULT 0;
140
+ ALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS schema_version INTEGER;
141
+ ALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS units JSONB;
142
+ ALTER TABLE sync_clients ADD COLUMN IF NOT EXISTS registration_digest BYTEA;
143
+ ALTER TABLE sync_clients
144
+ ADD COLUMN IF NOT EXISTS catching_up JSONB NOT NULL DEFAULT '[]';
145
+ -- §9: a database from before wire version 4 (its client records still hold
146
+ -- subscriptions) starts the horizon at its last commit.
147
+ UPDATE sync_partitions SET horizon_seq = max_commit_seq
148
+ WHERE EXISTS (SELECT 1 FROM information_schema.columns
149
+ WHERE table_schema = current_schema()
150
+ AND table_name = 'sync_clients'
151
+ AND column_name = 'subscriptions');
152
+ ALTER TABLE sync_clients DROP COLUMN IF EXISTS subscriptions;
122
153
  CREATE TABLE IF NOT EXISTS sync_blob_refs(
123
154
  partition TEXT NOT NULL, tbl TEXT NOT NULL, row_id TEXT NOT NULL,
124
155
  blob_id TEXT NOT NULL,
@@ -133,6 +164,13 @@ CREATE TABLE IF NOT EXISTS sync_tombstones(
133
164
  );
134
165
  ALTER TABLE sync_tombstones ADD COLUMN IF NOT EXISTS server_version BIGINT;
135
166
  ALTER TABLE sync_tombstones ADD COLUMN IF NOT EXISTS scopes JSONB;
167
+ ALTER TABLE sync_partitions
168
+ ADD COLUMN IF NOT EXISTS delete_floor BIGINT NOT NULL DEFAULT 0;
169
+ UPDATE sync_partitions SET delete_floor = t.version
170
+ FROM (SELECT partition, max(server_version) AS version
171
+ FROM sync_tombstones GROUP BY partition) t
172
+ WHERE t.partition = sync_partitions.partition
173
+ AND t.version > sync_partitions.delete_floor;
136
174
  CREATE TABLE IF NOT EXISTS sync_backfill_checkpoints(
137
175
  partition TEXT NOT NULL, name TEXT NOT NULL,
138
176
  schema_version INTEGER NOT NULL,
@@ -271,7 +309,8 @@ async function readAdmissionOn(client, partition, clientId, clientCommitId, runn
271
309
  ];
272
310
  let statement = `SELECT ${SERVE_GATE_PROJECTION},
273
311
  (SELECT result FROM sync_push_results
274
- WHERE partition=$1 AND client_id=$3 AND client_commit_id=$4) AS result`;
312
+ WHERE partition=$1 AND client_id=$3 AND client_commit_id=$4) AS result,
313
+ (SELECT delete_floor FROM sync_partitions WHERE partition=$1) AS delete_floor`;
275
314
  if (prefetch !== undefined && prefetch.specs.length > 0) {
276
315
  params.push(JSON.stringify(prefetch.specs));
277
316
  // The gate row pads the read-ahead columns with typed NULLs so both
@@ -283,7 +322,7 @@ async function readAdmissionOn(client, partition, clientId, clientCommitId, runn
283
322
  NULL::bigint AS tombstone_seq, NULL::bigint AS tombstone_version,
284
323
  NULL::jsonb AS tombstone_scopes
285
324
  UNION ALL
286
- SELECT NULL::bigint, NULL::text, NULL::json, NULL::jsonb,
325
+ SELECT NULL::bigint, NULL::text, NULL::json, NULL::jsonb, NULL::bigint,
287
326
  p.tbl, p.lookup_id, p.row_id, p.server_version, p.scopes,
288
327
  p.payload, p.column_versions, p.tombstone_seq,
289
328
  p.tombstone_version, p.tombstone_scopes
@@ -297,6 +336,8 @@ async function readAdmissionOn(client, partition, clientId, clientCommitId, runn
297
336
  return {
298
337
  gate: toServeGate(gateRow),
299
338
  persisted: toPersistedPushResult(gateRow?.result),
339
+ // asNumber maps the NULL of a partition without a commit to 0.
340
+ deleteFloor: asNumber(gateRow?.delete_floor),
300
341
  prefetched: rows.filter((row) => row.tbl !== null && row.tbl !== undefined),
301
342
  };
302
343
  }
@@ -483,6 +524,31 @@ function toTombstone(seq, version, scopes) {
483
524
  : { scopes: asJson(scopes) }),
484
525
  };
485
526
  }
527
+ const CLIENT_RECORD_COLUMNS = 'client_id, actor_id, wire_version, cursor, updated_at_ms, schema_version, units, registration_digest, catching_up, last_round_at_ms, last_error_code, consecutive_failures';
528
+ function toClientRecord(record) {
529
+ return {
530
+ clientId: record.client_id,
531
+ actorId: record.actor_id,
532
+ wireVersion: asNumber(record.wire_version),
533
+ cursor: asNumber(record.cursor),
534
+ updatedAtMs: asNumber(record.updated_at_ms),
535
+ ...(record.schema_version === null
536
+ ? {}
537
+ : { schemaVersion: asNumber(record.schema_version) }),
538
+ ...(record.units === null || record.registration_digest === null
539
+ ? {}
540
+ : {
541
+ registration: {
542
+ units: asJson(record.units),
543
+ digest: asBytes(record.registration_digest),
544
+ },
545
+ }),
546
+ catchingUp: asJson(record.catching_up),
547
+ ...clientRoundState(record.last_round_at_ms === null
548
+ ? null
549
+ : asNumber(record.last_round_at_ms), record.last_error_code, asNumber(record.consecutive_failures)),
550
+ };
551
+ }
486
552
  function toStoredChange(record) {
487
553
  return {
488
554
  table: record.tbl,
@@ -973,7 +1039,7 @@ class PostgresTransaction {
973
1039
  }
974
1040
  return { rows: result.rows, maxCommitSeq: this.#cursorSeq };
975
1041
  }
976
- async preparePushApply(clientId, clientCommitId, runningSchemaVersion, targets = []) {
1042
+ async preparePushApply(clientId, clientCommitId, runningSchemaVersion, targets = [], outbox) {
977
1043
  this.#assertOpen();
978
1044
  // The lock is its own statement: a READ COMMITTED statement's snapshot is
979
1045
  // fixed at its start, so the admission read must run *after* a duplicate's
@@ -981,6 +1047,9 @@ class PostgresTransaction {
981
1047
  // and the read into one statement would miss a duplicate that committed
982
1048
  // while this transaction waited.
983
1049
  await this.lockPartition();
1050
+ const watermark = outbox === undefined
1051
+ ? undefined
1052
+ : await this.#settleOutbox(clientId, clientCommitId, outbox);
984
1053
  const tables = [];
985
1054
  const specs = [];
986
1055
  for (const target of targets) {
@@ -994,7 +1063,62 @@ class PostgresTransaction {
994
1063
  // read-ahead (when the caller already knows its targets).
995
1064
  const admission = await readAdmissionOn(this.#client, this.#partition, clientId, clientCommitId, runningSchemaVersion, specs.length === 0 ? undefined : { tables, specs });
996
1065
  this.#recordPrefetched(admission.prefetched);
997
- return { gate: admission.gate, persisted: admission.persisted };
1066
+ return {
1067
+ gate: admission.gate,
1068
+ persisted: admission.persisted,
1069
+ deleteFloor: admission.deleteFloor,
1070
+ ...(watermark === undefined ? {} : { watermark }),
1071
+ };
1072
+ }
1073
+ /**
1074
+ * The §9 settlement and §6.3.2 steps 1 and 2 under the partition lock,
1075
+ * before the candidate savepoint, so a rejection keeps them. Two
1076
+ * statements: the settlement touches disjoint rows (outside and inside the
1077
+ * request), and the watermark delete must see its sequences.
1078
+ */
1079
+ async #settleOutbox(clientId, clientCommitId, outbox) {
1080
+ const first = outbox.commits[0];
1081
+ const last = outbox.commits.at(-1);
1082
+ if (first === undefined ||
1083
+ last === undefined ||
1084
+ first.clientCommitId !== clientCommitId) {
1085
+ throw new Error("outbox settlement rides the request's first commit");
1086
+ }
1087
+ const commits = JSON.stringify(outbox.commits.map((commit) => ({
1088
+ id: commit.clientCommitId,
1089
+ seq: commit.clientSeq,
1090
+ })));
1091
+ // §9 (b): with no pre-upgrade result on the request's last commit, every
1092
+ // remaining one outside the request left the outbox; (a) assigns the
1093
+ // request's sequences to its pre-upgrade results.
1094
+ await this.#client.query(`WITH c AS (SELECT * FROM jsonb_to_recordset($4::text::jsonb) AS c(id text, seq bigint)),
1095
+ b AS (DELETE FROM sync_push_results
1096
+ WHERE partition=$1 AND client_id=$2 AND outbox_id IS NULL
1097
+ AND client_commit_id NOT IN (SELECT id FROM c)
1098
+ AND NOT EXISTS (SELECT 1 FROM sync_push_results
1099
+ WHERE partition=$1 AND client_id=$2 AND client_commit_id=$5
1100
+ AND outbox_id IS NULL)),
1101
+ a AS (UPDATE sync_push_results r SET outbox_id=$3, client_seq=c.seq FROM c
1102
+ WHERE r.partition=$1 AND r.client_id=$2 AND r.outbox_id IS NULL
1103
+ AND r.client_commit_id=c.id)
1104
+ SELECT 1`, [
1105
+ this.#partition,
1106
+ clientId,
1107
+ outbox.outboxId,
1108
+ commits,
1109
+ last.clientCommitId,
1110
+ ]);
1111
+ const { rows } = await this.#client.query(`WITH w AS (
1112
+ INSERT INTO sync_push_watermarks(partition, client_id, outbox_id, through_seq)
1113
+ VALUES ($1,$2,$3,$4)
1114
+ ON CONFLICT (partition, client_id, outbox_id) DO UPDATE
1115
+ SET through_seq=GREATEST(sync_push_watermarks.through_seq, EXCLUDED.through_seq)
1116
+ RETURNING through_seq),
1117
+ d AS (DELETE FROM sync_push_results
1118
+ WHERE partition=$1 AND client_id=$2 AND outbox_id=$3
1119
+ AND client_seq<(SELECT through_seq FROM w))
1120
+ SELECT through_seq FROM w`, [this.#partition, clientId, outbox.outboxId, first.clientSeq]);
1121
+ return asNumber(rows[0]?.through_seq);
998
1122
  }
999
1123
  async lockPartition() {
1000
1124
  this.#assertOpen();
@@ -1017,7 +1141,7 @@ class PostgresTransaction {
1017
1141
  // transaction, so a migration cannot interleave between them.
1018
1142
  return readServeGateOn(this.#client, this.#partition, runningSchemaVersion);
1019
1143
  }
1020
- async commitRejectedPushResult(clientId, clientCommitId, result) {
1144
+ async commitRejectedPushResult(clientId, clientCommitId, result, sequence) {
1021
1145
  this.#assertOpen();
1022
1146
  // Candidate rows that never reached the database are discarded with the
1023
1147
  // staging map; a savepoint exists only when one did.
@@ -1028,20 +1152,22 @@ class PostgresTransaction {
1028
1152
  await this.#client.query('RELEASE SAVEPOINT syncular_push_candidate');
1029
1153
  this.#pushApplySavepoint = false;
1030
1154
  }
1031
- const canonical = await this.#insertPushResult(clientId, clientCommitId, result);
1155
+ const canonical = await this.#insertPushResult(clientId, clientCommitId, result, sequence);
1032
1156
  await this.commit();
1033
1157
  return canonical;
1034
1158
  }
1035
1159
  /** Insert the §2.3 outcome and return the canonical persisted value. */
1036
- async #insertPushResult(clientId, clientCommitId, result) {
1037
- const { rows } = await this.#client.query(`INSERT INTO sync_push_results(partition, client_id, client_commit_id, result)
1038
- VALUES ($1,$2,$3,$4)
1160
+ async #insertPushResult(clientId, clientCommitId, result, sequence) {
1161
+ const { rows } = await this.#client.query(`INSERT INTO sync_push_results(partition, client_id, client_commit_id, result, outbox_id, client_seq)
1162
+ VALUES ($1,$2,$3,$4,$5,$6)
1039
1163
  ON CONFLICT (partition, client_id, client_commit_id) DO NOTHING
1040
1164
  RETURNING result`, [
1041
1165
  this.#partition,
1042
1166
  clientId,
1043
1167
  clientCommitId,
1044
1168
  JSON.stringify(serializePushResult(result)),
1169
+ sequence?.outboxId ?? null,
1170
+ sequence?.clientSeq ?? null,
1045
1171
  ]);
1046
1172
  if (rows[0] !== undefined) {
1047
1173
  return toPersistedPushResult(rows[0].result) ?? result;
@@ -1133,9 +1259,11 @@ class PostgresTransaction {
1133
1259
  // Allocate the next dense commitSeq under a per-partition row lock: the
1134
1260
  // UPDATE … RETURNING serializes concurrent pushes to this partition and
1135
1261
  // never leaves a gap on rollback (see the file header).
1136
- const allocate = `INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 1)
1262
+ // §2.2: the allocation also raises the delete floor to the commit's.
1263
+ const allocate = `INSERT INTO sync_partitions(partition, max_commit_seq, delete_floor) VALUES ($1, 1, $7)
1137
1264
  ON CONFLICT (partition) DO UPDATE
1138
- SET max_commit_seq = sync_partitions.max_commit_seq + 1
1265
+ SET max_commit_seq = sync_partitions.max_commit_seq + 1,
1266
+ delete_floor = GREATEST(sync_partitions.delete_floor, EXCLUDED.delete_floor)
1139
1267
  RETURNING max_commit_seq`;
1140
1268
  const commitRow = `INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms, writer_version)
1141
1269
  SELECT $1, allocated.max_commit_seq, $2, $3, $4, $5, $6 FROM allocated
@@ -1147,6 +1275,7 @@ class PostgresTransaction {
1147
1275
  commit.actorId,
1148
1276
  commit.createdAtMs,
1149
1277
  this.#writerVersion,
1278
+ commitDeleteFloor(commit.changes),
1150
1279
  ];
1151
1280
  if (commit.changes.length === 0) {
1152
1281
  const { rows } = await q.query(`WITH allocated AS (${allocate}) ${commitRow}`, head);
@@ -1168,11 +1297,10 @@ class PostgresTransaction {
1168
1297
  // parameters would otherwise store the string as a scalar. A commit can
1169
1298
  // delete one row twice, so a chunk's tombstone insert keeps the latest
1170
1299
  // delete per key and a later chunk overwrites it with its own. `version`
1171
- // is an upsert's row version or a delete's deleted version (§2.2); only
1172
- // the tombstone keeps the latter.
1300
+ // is the change's row version (§2.2), which a delete's tombstone keeps.
1173
1301
  const valuesOf = (params, slice, firstIdx) => slice.map((change, offset) => {
1174
1302
  const base = params.length;
1175
- params.push(firstIdx + offset, change.table, change.rowId, change.op === 'upsert' ? 1 : 2, change.rowVersion ?? change.deletedVersion ?? null, JSON.stringify(change.scopes), change.payload ?? null);
1303
+ params.push(firstIdx + offset, change.table, change.rowId, change.op === 'upsert' ? 1 : 2, change.rowVersion, JSON.stringify(change.scopes), change.payload ?? null);
1176
1304
  return `($${base + 1}::int,$${base + 2}::text,$${base + 3}::text,$${base + 4}::smallint,$${base + 5}::bigint,$${base + 6}::text::jsonb,$${base + 7}::bytea)`;
1177
1305
  });
1178
1306
  const scopesCte = `scoped AS (
@@ -1208,7 +1336,7 @@ class PostgresTransaction {
1208
1336
  ),
1209
1337
  inserted AS (
1210
1338
  INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
1211
- SELECT $1, allocated.max_commit_seq, v.idx, v.tbl, v.row_id, v.op, CASE WHEN v.op = 1 THEN v.version END, v.scopes, v.payload
1339
+ SELECT $1, allocated.max_commit_seq, v.idx, v.tbl, v.row_id, v.op, v.version, v.scopes, v.payload
1212
1340
  FROM allocated CROSS JOIN v
1213
1341
  RETURNING partition, idx, tbl, row_id, op, commit_seq, scopes
1214
1342
  ),
@@ -1226,7 +1354,7 @@ class PostgresTransaction {
1226
1354
  ),
1227
1355
  inserted AS (
1228
1356
  INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
1229
- SELECT $1, $2, v.idx, v.tbl, v.row_id, v.op, CASE WHEN v.op = 1 THEN v.version END, v.scopes, v.payload
1357
+ SELECT $1, $2, v.idx, v.tbl, v.row_id, v.op, v.version, v.scopes, v.payload
1230
1358
  FROM v
1231
1359
  RETURNING partition, idx, tbl, row_id, op, commit_seq, scopes
1232
1360
  ),
@@ -1236,15 +1364,17 @@ class PostgresTransaction {
1236
1364
  }
1237
1365
  return commitSeq;
1238
1366
  }
1239
- async putPushResult(clientId, clientCommitId, result) {
1367
+ async putPushResult(clientId, clientCommitId, result, sequence) {
1240
1368
  this.#assertOpen();
1241
- await this.#client.query(`INSERT INTO sync_push_results(partition, client_id, client_commit_id, result)
1242
- VALUES ($1,$2,$3,$4)
1369
+ await this.#client.query(`INSERT INTO sync_push_results(partition, client_id, client_commit_id, result, outbox_id, client_seq)
1370
+ VALUES ($1,$2,$3,$4,$5,$6)
1243
1371
  ON CONFLICT (partition, client_id, client_commit_id) DO NOTHING`, [
1244
1372
  this.#partition,
1245
1373
  clientId,
1246
1374
  clientCommitId,
1247
1375
  JSON.stringify(serializePushResult(result)),
1376
+ sequence?.outboxId ?? null,
1377
+ sequence?.clientSeq ?? null,
1248
1378
  ]);
1249
1379
  }
1250
1380
  async enqueueReactions(reactions) {
@@ -2032,101 +2162,53 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2032
2162
  };
2033
2163
  }
2034
2164
  async latestScopedChangeSeq(partition, query) {
2035
- const firstVariable = Object.keys(query.scopeFilter).sort()[0];
2036
- const values = firstVariable === undefined
2037
- ? []
2038
- : (query.scopeFilter[firstVariable] ?? []);
2039
- if (firstVariable === undefined || values.length === 0)
2165
+ const variable = Object.keys(query.scopes).sort()[0];
2166
+ const value = variable === undefined ? undefined : query.scopes[variable];
2167
+ if (variable === undefined || value === undefined)
2040
2168
  return 0;
2041
- // Concurrent reads of a pull share one statement across tables.
2042
- const [row] = await this.#batchedPage('latestScopedChange', LATEST_SCOPED_CHANGE_BATCH_SQL, partition, { tbl: query.table, var: firstVariable, vals: values });
2043
- return row === undefined ? 0 : asNumber(row.seq);
2044
- }
2045
- async readCommitWindow(partition, query) {
2046
- const variables = Object.keys(query.scopeFilter).sort();
2047
- const firstVariable = variables[0];
2048
- if (firstVariable === undefined)
2049
- return [];
2050
- const firstValues = query.scopeFilter[firstVariable] ?? [];
2051
- if (firstValues.length === 0)
2052
- return [];
2053
- // Candidates via the inverted index (one variable): the (partition, tbl,
2054
- // var, value, commit_seq) PK makes the candidate subquery an index range
2055
- // scan returning commit_seq already ascending (see postgres-explain
2056
- // .test.ts), LEFT JOINed to the commit meta + the table's changes. Pages
2057
- // of concurrent window reads share one statement across tables (see
2058
- // `COMMIT_WINDOW_BATCH_SQL`). Exact multi-variable verification against
2059
- // the stored scope map happens below.
2060
- const commits = [];
2061
- let deliveredChanges = 0;
2062
- let afterSeq = query.afterSeq;
2063
- const batchSize = Math.max(64, query.limitChanges);
2064
- while (deliveredChanges < query.limitChanges) {
2065
- const records = await this.#batchedPage('commitWindow', COMMIT_WINDOW_BATCH_SQL, partition, {
2066
- tbl: query.table,
2067
- var: firstVariable,
2068
- vals: firstValues,
2069
- after: afterSeq,
2070
- through: query.throughSeq,
2071
- lim: batchSize,
2072
- });
2073
- if (records.length === 0)
2074
- break;
2075
- // Fold the page: rows arrive ordered (commit_seq, idx); consecutive
2076
- // rows with the same commit_seq are one candidate. Every candidate
2077
- // advances the cursor — including vanished commits (NULL meta, LEFT
2078
- // JOIN contract) and commits whose changes all fail the exact
2079
- // multi-variable match.
2080
- let candidateCount = 0;
2081
- let i = 0;
2082
- while (i < records.length) {
2083
- const head = records[i];
2084
- if (head === undefined)
2085
- break;
2086
- const commitSeq = asNumber(head.commit_seq);
2087
- candidateCount += 1;
2088
- afterSeq = commitSeq;
2089
- const changes = [];
2090
- for (; i < records.length; i++) {
2091
- const record = records[i];
2092
- if (record === undefined || asNumber(record.commit_seq) !== commitSeq)
2093
- break;
2094
- // NULL tbl: a candidate with no change rows for the table.
2095
- if (record.tbl === null || record.row_id === null)
2096
- continue;
2097
- const change = toStoredChange({
2098
- tbl: record.tbl,
2099
- row_id: record.row_id,
2100
- op: record.op,
2101
- row_version: record.row_version,
2102
- scopes: record.scopes,
2103
- payload: record.payload,
2104
- });
2105
- if (matchesEffective(change.scopes, query.scopeFilter)) {
2106
- changes.push(change);
2107
- }
2108
- }
2109
- // NULL meta: the commit vanished — cursor advanced, nothing emitted.
2110
- if (head.actor_id === null)
2111
- continue;
2112
- if (changes.length === 0)
2113
- continue;
2114
- commits.push({
2115
- commitSeq,
2116
- createdAtMs: asNumber(head.created_at_ms),
2117
- actorId: head.actor_id,
2118
- changes,
2119
- });
2120
- deliveredChanges += changes.length;
2121
- if (deliveredChanges >= query.limitChanges)
2122
- break;
2123
- }
2124
- if (deliveredChanges >= query.limitChanges)
2125
- break;
2126
- if (candidateCount < batchSize)
2127
- break;
2128
- }
2129
- return commits;
2169
+ // Concurrent reads of a pull share one statement across tables: one
2170
+ // index-only max over the (partition, tbl, var, value) prefix per spec.
2171
+ const [row] = await this.#batchedPage('latestScopedChange', `SELECT s.i AS sub,
2172
+ (SELECT max(commit_seq) FROM sync_change_scopes
2173
+ WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=s.value) AS seq
2174
+ FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, value text)
2175
+ ORDER BY s.i`, partition, { tbl: query.table, var: variable, value });
2176
+ return row === undefined || row.seq === null ? 0 : asNumber(row.seq);
2177
+ }
2178
+ readLiveWindow(partition, query) {
2179
+ // One statement per page: one LATERAL range of
2180
+ // `sync_change_scopes_by_key` per unit key, each DISTINCT and LIMITed at
2181
+ // the index, merged in commit order and LIMITed again, then LEFT JOINed
2182
+ // to the commit meta and every change of the commit. A candidate whose
2183
+ // commit vanished comes back with NULL meta and still advances the scan.
2184
+ return readLiveWindowPages(query, async (keys, limit) => {
2185
+ const { rows } = await this.#exec.query(`SELECT c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
2186
+ ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
2187
+ FROM (SELECT DISTINCT s.commit_seq
2188
+ FROM jsonb_to_recordset($2::text::jsonb) AS k(var text, value text, after bigint)
2189
+ CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
2190
+ WHERE partition=$1 AND var=k.var AND value=k.value
2191
+ AND commit_seq>k.after AND commit_seq<=$3
2192
+ ORDER BY commit_seq LIMIT $4) s
2193
+ ORDER BY s.commit_seq LIMIT $4) c
2194
+ LEFT JOIN sync_commits m ON m.partition=$1 AND m.commit_seq=c.commit_seq
2195
+ LEFT JOIN sync_changes ch ON ch.partition=$1 AND ch.commit_seq=c.commit_seq
2196
+ ORDER BY c.commit_seq, ch.idx`, [partition, JSON.stringify(keys), query.throughSeq, limit]);
2197
+ return rows.map((record) => ({
2198
+ commitSeq: asNumber(record.commit_seq),
2199
+ ...(record.actor_id === null
2200
+ ? {}
2201
+ : {
2202
+ meta: {
2203
+ actorId: record.actor_id,
2204
+ createdAtMs: asNumber(record.created_at_ms),
2205
+ },
2206
+ }),
2207
+ ...(record.tbl === null
2208
+ ? {}
2209
+ : { change: toStoredChange({ ...record, tbl: record.tbl }) }),
2210
+ }));
2211
+ });
2130
2212
  }
2131
2213
  async scanRows(partition, query) {
2132
2214
  const firstVariable = assertScopeIndexedScan(query);
@@ -2173,31 +2255,24 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2173
2255
  return scanRowsByIndexOn(this.#exec, this.table(query.table), partition, query);
2174
2256
  }
2175
2257
  async getClientRecord(partition, clientId) {
2176
- const { rows } = await this.#exec.query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=$1 AND client_id=$2', [partition, clientId]);
2177
- const record = rows[0];
2178
- if (record === undefined)
2179
- return undefined;
2180
- return {
2181
- clientId: record.client_id,
2182
- actorId: record.actor_id,
2183
- wireVersion: asNumber(record.wire_version),
2184
- cursor: asNumber(record.cursor),
2185
- updatedAtMs: asNumber(record.updated_at_ms),
2186
- subscriptions: asJson(record.subscriptions),
2187
- ...clientRoundState(record.last_round_at_ms === null
2188
- ? null
2189
- : asNumber(record.last_round_at_ms), record.last_error_code, asNumber(record.consecutive_failures)),
2190
- };
2258
+ const { rows } = await this.#exec.query(`SELECT ${CLIENT_RECORD_COLUMNS} FROM sync_clients WHERE partition=$1 AND client_id=$2`, [partition, clientId]);
2259
+ return rows[0] === undefined ? undefined : toClientRecord(rows[0]);
2191
2260
  }
2192
2261
  async putClientRecord(partition, record, options) {
2193
- await this.#exec.query(`INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms)
2194
- VALUES ($1,$2,$3,$4,$5,$6,$7,$7)
2262
+ // Every non-key field comes from the new record, except the registration
2263
+ // list and digest, which only a traveling list replaces (§4.5).
2264
+ const registration = record.registration;
2265
+ await this.#exec.query(`INSERT INTO sync_clients(partition, client_id, actor_id, wire_version, cursor, updated_at_ms, last_round_at_ms, schema_version, units, registration_digest, catching_up)
2266
+ VALUES ($1,$2,$3,$4,$5,$6,$11,$7,$8::text::jsonb,$9,$10::text::jsonb)
2195
2267
  ON CONFLICT (partition, client_id) DO UPDATE
2196
2268
  SET actor_id=EXCLUDED.actor_id, wire_version=EXCLUDED.wire_version,
2197
2269
  cursor=EXCLUDED.cursor,
2198
- subscriptions=EXCLUDED.subscriptions,
2199
2270
  updated_at_ms=EXCLUDED.updated_at_ms,
2200
- last_round_at_ms=EXCLUDED.last_round_at_ms${options?.keepRoundOutcome === true
2271
+ last_round_at_ms=EXCLUDED.last_round_at_ms,
2272
+ schema_version=EXCLUDED.schema_version,${registration === undefined
2273
+ ? ''
2274
+ : '\n units=EXCLUDED.units, registration_digest=EXCLUDED.registration_digest,'}
2275
+ catching_up=EXCLUDED.catching_up${options?.keepRoundOutcome === true
2201
2276
  ? ''
2202
2277
  : ',\n last_error_code=NULL, consecutive_failures=0'}`, [
2203
2278
  partition,
@@ -2205,8 +2280,12 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2205
2280
  record.actorId,
2206
2281
  record.wireVersion,
2207
2282
  record.cursor,
2208
- JSON.stringify(record.subscriptions),
2209
2283
  record.updatedAtMs,
2284
+ record.schemaVersion ?? null,
2285
+ registration === undefined ? null : JSON.stringify(registration.units),
2286
+ registration?.digest ?? null,
2287
+ JSON.stringify(record.catchingUp),
2288
+ record.lastRoundAtMs ?? record.updatedAtMs,
2210
2289
  ]);
2211
2290
  }
2212
2291
  async recordClientRoundFailure(partition, clientId, actorId, errorCode, atMs) {
@@ -2216,17 +2295,18 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2216
2295
  WHERE partition=$3 AND client_id=$4 AND actor_id=$5`, [atMs, errorCode, partition, clientId, actorId]);
2217
2296
  }
2218
2297
  async advanceClientCursor(partition, clientId, actorId, logEpoch, cursor, updatedAtMs) {
2298
+ // §8.2: the ack never raises the record cursor above a catching-up
2299
+ // unit's cursor, so pruning keeps that unit's history (§4.6).
2219
2300
  await this.#exec.query(`UPDATE sync_clients
2220
- SET cursor=GREATEST(cursor, $1), updated_at_ms=GREATEST(updated_at_ms, $2)
2301
+ SET cursor=GREATEST(cursor, LEAST($1::bigint, COALESCE((
2302
+ SELECT min((u->>'cursor')::bigint)
2303
+ FROM jsonb_array_elements(sync_clients.catching_up) AS u
2304
+ WHERE (u->>'cursor')::bigint>=0), $1::bigint))),
2305
+ updated_at_ms=GREATEST(updated_at_ms, $2)
2221
2306
  WHERE partition=$3 AND client_id=$4 AND actor_id=$5
2222
2307
  AND EXISTS (SELECT 1 FROM sync_partition_registry
2223
2308
  WHERE partition=sync_clients.partition AND log_epoch=$6)`, [cursor, updatedAtMs, partition, clientId, actorId, logEpoch]);
2224
2309
  }
2225
- async updateClientCursor(partition, clientId, cursor, updatedAtMs) {
2226
- await this.#exec.query(`UPDATE sync_clients
2227
- SET cursor=GREATEST(cursor, $3), updated_at_ms=GREATEST(updated_at_ms, $4)
2228
- WHERE partition=$1 AND client_id=$2`, [partition, clientId, cursor, updatedAtMs]);
2229
- }
2230
2310
  async getActiveClientCursorFloor(partition, cutoffMs) {
2231
2311
  const { rows } = await this.#exec.query('SELECT MIN(cursor) AS cursor FROM sync_clients WHERE partition=$1 AND updated_at_ms>=$2 AND cursor>=0', [partition, cutoffMs]);
2232
2312
  return rows[0].cursor === null ? null : asNumber(rows[0].cursor);
@@ -2322,18 +2402,8 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2322
2402
  }
2323
2403
  // -- admin/console read surface --------------------------------------------
2324
2404
  async listClientRecords(partition) {
2325
- const { rows } = await this.#exec.query('SELECT client_id, actor_id, wire_version, cursor, subscriptions, updated_at_ms, last_round_at_ms, last_error_code, consecutive_failures FROM sync_clients WHERE partition=$1 ORDER BY updated_at_ms DESC', [partition]);
2326
- return rows.map((r) => ({
2327
- clientId: r.client_id,
2328
- actorId: r.actor_id,
2329
- wireVersion: asNumber(r.wire_version),
2330
- cursor: asNumber(r.cursor),
2331
- updatedAtMs: asNumber(r.updated_at_ms),
2332
- subscriptions: (typeof r.subscriptions === 'string'
2333
- ? JSON.parse(r.subscriptions)
2334
- : r.subscriptions),
2335
- ...clientRoundState(r.last_round_at_ms === null ? null : asNumber(r.last_round_at_ms), r.last_error_code, asNumber(r.consecutive_failures)),
2336
- }));
2405
+ const { rows } = await this.#exec.query(`SELECT ${CLIENT_RECORD_COLUMNS} FROM sync_clients WHERE partition=$1 ORDER BY updated_at_ms DESC`, [partition]);
2406
+ return rows.map(toClientRecord);
2337
2407
  }
2338
2408
  async listCommitMetadata(partition, query) {
2339
2409
  // Newest-first window, resumed by `afterSeq` (exclusive lower bound) and
@@ -2470,13 +2540,24 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2470
2540
  }
2471
2541
  });
2472
2542
  }
2543
+ async listPushWatermarks(partition, clientCommitId) {
2544
+ // The clients come from the retained commit log, which has no index on
2545
+ // `client_commit_id`; the read is bounded by retention (§4.6).
2546
+ const { rows } = await this.#exec.query(`SELECT client_id, outbox_id, through_seq FROM sync_push_watermarks
2547
+ WHERE partition=$1 AND client_id IN (
2548
+ SELECT client_id FROM sync_commits WHERE partition=$1 AND client_commit_id=$2)
2549
+ ORDER BY client_id, outbox_id`, [partition, clientCommitId]);
2550
+ return rows.map((row) => ({
2551
+ clientId: row.client_id,
2552
+ outboxId: row.outbox_id,
2553
+ throughSeq: asNumber(row.through_seq),
2554
+ }));
2555
+ }
2473
2556
  async scopeActivity(partition, query) {
2474
2557
  // One statement for the page: candidate (commit, table) pairs from the
2475
- // change-scope index, joined to the commit metadata with the table's
2476
- // change count. `tbl` sits between `partition` and `var` in the index key,
2477
- // so the `var`/`value` filter ranges over the partition's change-scope
2478
- // entries; only the LIMIT bounds the output. A candidate whose commit was
2479
- // pruned has no metadata and is skipped.
2558
+ // `sync_change_scopes_by_key` range of the key, newest first, joined to
2559
+ // the commit metadata with the table's change count. A candidate whose
2560
+ // commit was pruned has no metadata and is skipped.
2480
2561
  const { rows } = await this.#exec.query(`SELECT s.commit_seq, s.tbl, m.actor_id, m.created_at_ms,
2481
2562
  (SELECT count(*) FROM sync_changes ch
2482
2563
  WHERE ch.partition=$1 AND ch.commit_seq=s.commit_seq AND ch.tbl=s.tbl) AS n