@syncular/server 0.80.0 → 0.82.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -125,13 +125,13 @@ import type {
125
125
  IndexRowScanQuery,
126
126
  LiveWindow,
127
127
  LiveWindowQuery,
128
- NewChange,
129
128
  NewCommit,
130
129
  NewReaction,
131
130
  OutboxSequence,
132
131
  OutboxSettlement,
133
132
  PartitionRegistryEntry,
134
133
  PreparedPushApply,
134
+ PushApplyOptions,
135
135
  PrunedReactionCounts,
136
136
  PushApplyCommit,
137
137
  PushWatermark,
@@ -159,6 +159,7 @@ import type {
159
159
  Tombstone,
160
160
  } from './storage';
161
161
  import {
162
+ ClosedCommitConstraintError,
162
163
  isPostgresConstraintError,
163
164
  StorageConstraintError,
164
165
  } from './storage-errors';
@@ -358,15 +359,20 @@ interface SerializedResult {
358
359
  details?: import('@syncular/core').RejectionDetails;
359
360
  }
360
361
 
362
+ /** Lock the partition row; returns its `max_commit_seq`, which the lock freezes. */
361
363
  async function lockPartitionOn(
362
364
  client: PgQueryable,
363
365
  partition: string,
364
- ): Promise<void> {
365
- const locked = await client.query(
366
- 'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE',
367
- [partition],
368
- );
369
- if (locked.rows.length > 0) return;
366
+ ): Promise<number> {
367
+ const lock = () =>
368
+ client.query<{ max_commit_seq: unknown }>(
369
+ 'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE',
370
+ [partition],
371
+ );
372
+ const locked = await lock();
373
+ if (locked.rows[0] !== undefined) {
374
+ return asNumber(locked.rows[0].max_commit_seq);
375
+ }
370
376
  // The first writer initializes the partition. Concurrent initializers may
371
377
  // wait on this insert, so acquire the row lock again before applying writes.
372
378
  await client.query(
@@ -374,10 +380,7 @@ async function lockPartitionOn(
374
380
  ON CONFLICT (partition) DO NOTHING`,
375
381
  [partition],
376
382
  );
377
- await client.query(
378
- 'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE',
379
- [partition],
380
- );
383
+ return asNumber((await lock()).rows[0]?.max_commit_seq);
381
384
  }
382
385
 
383
386
  /**
@@ -488,6 +491,7 @@ interface ServeGateRow {
488
491
  pending: unknown;
489
492
  results?: unknown;
490
493
  delete_floor?: unknown;
494
+ watermark?: unknown;
491
495
  }
492
496
 
493
497
  /**
@@ -545,13 +549,22 @@ interface PrefetchRow {
545
549
  type AdmissionRow = ServeGateRow & Partial<PrefetchRow>;
546
550
 
547
551
  /**
548
- * The push admission read: the RFC 0007 serve gate, the §2.3 idempotency
549
- * records of every commit of the request, and (when the caller already knows
550
- * the commits' row targets) the read-ahead rows, in one statement with one
551
- * snapshot. The caller takes the partition lock first: under READ COMMITTED a
552
- * statement's snapshot is fixed at its start, so the read must be the
553
- * statement *after* the lock to observe a duplicate that committed while this
554
- * transaction waited for it.
552
+ * The push admission statement: the §9 settlement and §6.3.2 steps 1 and 2
553
+ * when the request carries an outbox, the RFC 0007 serve gate, the §2.3
554
+ * idempotency records of every commit of the request, and (when the caller
555
+ * already knows the commits' row targets) the read-ahead rows, in one
556
+ * statement with one snapshot. The caller takes the partition lock first:
557
+ * under READ COMMITTED a statement's snapshot is fixed at its start, so the
558
+ * read must be the statement *after* the lock to observe a duplicate that
559
+ * committed while this transaction waited for it.
560
+ *
561
+ * The settlement's writable CTEs see the same snapshot as the read, so they
562
+ * touch disjoint rows and the read leaves out the results they delete. §9
563
+ * (b): with no pre-upgrade result on the request's last commit, every
564
+ * remaining one outside the request left the outbox. (a) assigns the
565
+ * request's sequences to its pre-upgrade results at or above the new
566
+ * watermark; step 2 deletes the outbox's results below it, the pre-upgrade
567
+ * results of the request below it included.
555
568
  */
556
569
  async function readAdmissionOn(
557
570
  client: PgQueryable,
@@ -568,10 +581,12 @@ async function readAdmissionOn(
568
581
  }[];
569
582
  }
570
583
  | undefined,
584
+ outbox: OutboxSettlement | undefined,
571
585
  ): Promise<{
572
586
  gate: ServeGate;
573
587
  persisted: ReadonlyMap<string, unknown>;
574
588
  deleteFloor: number;
589
+ watermark: number | undefined;
575
590
  prefetched: readonly PrefetchRow[];
576
591
  }> {
577
592
  const params: unknown[] = [
@@ -580,10 +595,57 @@ async function readAdmissionOn(
580
595
  clientId,
581
596
  JSON.stringify(clientCommitIds),
582
597
  ];
583
- let statement = `SELECT ${SERVE_GATE_PROJECTION},
598
+ let settlement = '';
599
+ if (outbox !== undefined) {
600
+ const first = outbox.commits[0];
601
+ const last = outbox.commits.at(-1);
602
+ if (first === undefined || last === undefined) {
603
+ throw new Error('outbox settlement needs the request commits');
604
+ }
605
+ params.push(
606
+ outbox.outboxId,
607
+ JSON.stringify(
608
+ outbox.commits.map((commit) => ({
609
+ id: commit.clientCommitId,
610
+ seq: commit.clientSeq,
611
+ })),
612
+ ),
613
+ last.clientCommitId,
614
+ first.clientSeq,
615
+ );
616
+ settlement = `WITH c AS (SELECT * FROM jsonb_to_recordset($6::text::jsonb) AS c(id text, seq bigint)),
617
+ w AS (
618
+ INSERT INTO sync_push_watermarks(partition, client_id, outbox_id, through_seq)
619
+ VALUES ($1,$3,$5,$8)
620
+ ON CONFLICT (partition, client_id, outbox_id) DO UPDATE
621
+ SET through_seq=GREATEST(sync_push_watermarks.through_seq, EXCLUDED.through_seq)
622
+ RETURNING through_seq),
623
+ b AS (DELETE FROM sync_push_results
624
+ WHERE partition=$1 AND client_id=$3 AND outbox_id IS NULL
625
+ AND client_commit_id NOT IN (SELECT id FROM c)
626
+ AND NOT EXISTS (SELECT 1 FROM sync_push_results
627
+ WHERE partition=$1 AND client_id=$3 AND client_commit_id=$7
628
+ AND outbox_id IS NULL)),
629
+ a AS (UPDATE sync_push_results r SET outbox_id=$5, client_seq=c.seq FROM c, w
630
+ WHERE r.partition=$1 AND r.client_id=$3 AND r.outbox_id IS NULL
631
+ AND r.client_commit_id=c.id AND c.seq>=w.through_seq),
632
+ expired AS (DELETE FROM sync_push_results r USING w
633
+ WHERE r.partition=$1 AND r.client_id=$3
634
+ AND ((r.outbox_id=$5 AND r.client_seq<w.through_seq)
635
+ OR (r.outbox_id IS NULL AND EXISTS (SELECT 1 FROM c
636
+ WHERE c.id=r.client_commit_id AND c.seq<w.through_seq)))
637
+ RETURNING r.client_commit_id)
638
+ `;
639
+ }
640
+ let statement = `${settlement}SELECT ${SERVE_GATE_PROJECTION},
584
641
  (SELECT jsonb_object_agg(client_commit_id, result) FROM sync_push_results
585
- WHERE partition=$1 AND client_id=$3 AND client_commit_id IN (SELECT jsonb_array_elements_text($4::text::jsonb))) AS results,
586
- (SELECT delete_floor FROM sync_partitions WHERE partition=$1) AS delete_floor`;
642
+ WHERE partition=$1 AND client_id=$3 AND client_commit_id IN (SELECT jsonb_array_elements_text($4::text::jsonb))${
643
+ outbox === undefined
644
+ ? ''
645
+ : ' AND client_commit_id NOT IN (SELECT client_commit_id FROM expired)'
646
+ }) AS results,
647
+ (SELECT delete_floor FROM sync_partitions WHERE partition=$1) AS delete_floor,
648
+ ${outbox === undefined ? 'NULL::bigint' : '(SELECT through_seq FROM w)'} AS watermark`;
587
649
  if (prefetch !== undefined && prefetch.specs.length > 0) {
588
650
  params.push(JSON.stringify(prefetch.specs));
589
651
  // The gate row pads the read-ahead columns with typed NULLs so both
@@ -596,6 +658,7 @@ async function readAdmissionOn(
596
658
  NULL::jsonb AS tombstone_scopes
597
659
  UNION ALL
598
660
  SELECT NULL::bigint, NULL::text, NULL::json, NULL::jsonb, NULL::bigint,
661
+ NULL::bigint,
599
662
  p.tbl, p.lookup_id, p.row_id, p.server_version, p.scopes,
600
663
  p.payload, p.column_versions, p.tombstone_seq,
601
664
  p.tombstone_version, p.tombstone_scopes
@@ -619,6 +682,7 @@ async function readAdmissionOn(
619
682
  ),
620
683
  // asNumber maps the NULL of a partition without a commit to 0.
621
684
  deleteFloor: asNumber(gateRow?.delete_floor),
685
+ watermark: outbox === undefined ? undefined : asNumber(gateRow?.watermark),
622
686
  prefetched: rows.filter(
623
687
  (row): row is AdmissionRow & PrefetchRow =>
624
688
  row.tbl !== null && row.tbl !== undefined,
@@ -1079,145 +1143,294 @@ async function scanRowsByIndexOn(
1079
1143
  return rows.map(toStoredRow);
1080
1144
  }
1081
1145
 
1146
+ /** One row a transaction has staged and not yet written. */
1147
+ interface StagedRow {
1148
+ readonly row: StoredRow;
1149
+ /** `upsertValues` of the row, computed when it was staged. */
1150
+ readonly values: readonly unknown[];
1151
+ readonly opIndex?: number;
1152
+ }
1153
+
1082
1154
  /**
1083
- * Write rows of one table in one statement per chunk: the upsert, the scope
1084
- * index synced to each row's scope map (entries outside it deleted, missing
1085
- * ones inserted), and the §5.9.4 blob-reference index synced to the ids the
1086
- * row now references (entries outside the new set deleted, missing ones
1087
- * inserted). Row ids are distinct, so no index entry is touched twice and the
1088
- * writable CTEs cannot conflict. The scopes and blob ids bind as text before
1089
- * the JSONB cast (see `appendCommit`); the refs set is part of the same
1090
- * statement, so a blob-referencing commit costs no extra round trip per row.
1091
- * Every chunk is handed to `q` before the first one settles, so a statement
1092
- * issued after this call queues behind all of them on the transaction FIFO.
1155
+ * A staged commit append. `commitSeq` is known when the transaction holds
1156
+ * the partition lock; otherwise the append's own allocation decides it.
1093
1157
  */
1094
- function writeRowsOn(
1095
- q: PgQueryable,
1158
+ interface PendingAppend {
1159
+ readonly commit: NewCommit;
1160
+ readonly commitSeq?: number;
1161
+ }
1162
+
1163
+ /**
1164
+ * One writing statement: writable CTEs and the parameters they bind. The
1165
+ * CTEs touch disjoint rows, so their order inside the statement is free.
1166
+ */
1167
+ interface WriteStatement {
1168
+ readonly ctes: string[];
1169
+ readonly params: unknown[];
1170
+ /** The lowest operation index among the rows it writes. */
1171
+ opIndex?: number;
1172
+ /** True when the statement appends commits and returns the new cursor. */
1173
+ appends?: boolean;
1174
+ /** The `max_commit_seq` the statement's commit append must return. */
1175
+ allocates?: number;
1176
+ }
1177
+
1178
+ /**
1179
+ * Bound parameters per write statement. The Bind message counts parameters
1180
+ * in 16 bits; PostgreSQL reads up to 65,535, and PGlite 0.5 corrupts its
1181
+ * state above 32,767.
1182
+ */
1183
+ const PG_PARAMS_PER_STATEMENT = 30_000;
1184
+
1185
+ function bindParam(statement: WriteStatement, value: unknown): string {
1186
+ statement.params.push(value);
1187
+ return `$${statement.params.length}`;
1188
+ }
1189
+
1190
+ /**
1191
+ * The CTEs that write rows of one table: the upsert, the scope index synced
1192
+ * to each row's scope map (entries outside it deleted, missing ones
1193
+ * inserted), and the §5.9.4 blob-reference index synced to the ids the row
1194
+ * now references. Row ids are distinct, so no index entry is touched twice.
1195
+ * The scopes and blob ids bind as text before the JSONB cast; drivers that
1196
+ * encode JSONB parameters would otherwise store the string as a scalar.
1197
+ */
1198
+ function addRowWrites(
1199
+ statement: WriteStatement,
1096
1200
  compiled: CompiledTable,
1097
1201
  partition: string,
1098
- rows: readonly {
1099
- readonly row: StoredRow;
1100
- /** `upsertValues` of the row, computed when it was staged. */
1101
- readonly values: readonly unknown[];
1102
- }[],
1103
- blobRefs: readonly {
1104
- readonly rowId: string;
1105
- readonly blobIds: readonly string[];
1106
- }[] = [],
1107
- ): Promise<unknown> {
1108
- const width = tableColumnNames(compiled).length;
1109
- // PGlite corrupts its state above 32,767 bound parameters (see
1110
- // PG_CHANGES_PER_STATEMENT).
1111
- const chunk = Math.max(1, Math.floor(30_000 / width));
1112
- const pendingRefs = new Map(
1113
- blobRefs.map((entry) => [entry.rowId, entry.blobIds]),
1202
+ rows: readonly StagedRow[],
1203
+ blobRefs: readonly { row_id: string; blob_ids: readonly string[] }[],
1204
+ ): void {
1205
+ const k = statement.ctes.length;
1206
+ const offset = statement.params.length;
1207
+ statement.params.push(...rows.flatMap((entry) => entry.values));
1208
+ const scopes = bindParam(
1209
+ statement,
1210
+ JSON.stringify(
1211
+ rows.map(({ row }) => ({ row_id: row.rowId, scopes: row.scopes })),
1212
+ ),
1114
1213
  );
1115
- const writes: Promise<unknown>[] = [];
1116
- for (let first = 0; first < rows.length; first += chunk) {
1117
- const slice = rows.slice(first, first + chunk);
1118
- const params = slice.flatMap((entry) => entry.values);
1119
- const n = params.length;
1120
- const refs: { row_id: string; blob_ids: readonly string[] }[] = [];
1121
- for (const entry of slice) {
1122
- const blobIds = pendingRefs.get(entry.row.rowId);
1123
- if (blobIds === undefined) continue;
1124
- pendingRefs.delete(entry.row.rowId);
1125
- refs.push({ row_id: entry.row.rowId, blob_ids: blobIds });
1214
+ const p = bindParam(statement, partition);
1215
+ const tbl = bindParam(statement, compiled.name);
1216
+ statement.ctes.push(
1217
+ `u${k} AS (${upsertSql(compiled, 'postgres', rows.length, offset)})`,
1218
+ `b${k} AS (SELECT row_id, scopes FROM jsonb_to_recordset(${scopes}::text::jsonb) AS r(row_id text, scopes jsonb))`,
1219
+ `rs${k} AS (DELETE FROM sync_row_scopes existing USING b${k}
1220
+ WHERE existing.partition=${p} AND existing.tbl=${tbl}
1221
+ AND existing.row_id=b${k}.row_id
1222
+ AND b${k}.scopes->>existing.var IS DISTINCT FROM existing.value)`,
1223
+ `is${k} AS (INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
1224
+ SELECT ${p}, ${tbl}, entry.key, entry.value, b${k}.row_id
1225
+ FROM b${k} CROSS JOIN LATERAL jsonb_each_text(b${k}.scopes) AS entry
1226
+ ON CONFLICT DO NOTHING)`,
1227
+ );
1228
+ for (const entry of rows) {
1229
+ if (
1230
+ entry.opIndex !== undefined &&
1231
+ (statement.opIndex === undefined || entry.opIndex < statement.opIndex)
1232
+ ) {
1233
+ statement.opIndex = entry.opIndex;
1126
1234
  }
1127
- writes.push(
1128
- q.query(
1129
- `WITH upserted AS (${upsertSql(compiled, 'postgres', slice.length)}),
1130
- batch AS (
1131
- SELECT row_id, scopes
1132
- FROM jsonb_to_recordset($${n + 1}::text::jsonb) AS r(row_id text, scopes jsonb)
1133
- ),
1134
- scope AS (
1135
- SELECT batch.row_id, entry.key, entry.value
1136
- FROM batch CROSS JOIN LATERAL jsonb_each_text(batch.scopes) AS entry
1137
- ),
1138
- removed AS (
1139
- DELETE FROM sync_row_scopes existing
1140
- USING batch
1141
- WHERE existing.partition=$${n + 2} AND existing.tbl=$${n + 3}
1142
- AND existing.row_id=batch.row_id
1143
- AND batch.scopes->>existing.var IS DISTINCT FROM existing.value
1144
- ),
1145
- changed_refs AS (
1146
- SELECT DISTINCT row_id
1147
- FROM jsonb_to_recordset($${n + 4}::text::jsonb) AS r(row_id text, blob_ids jsonb)
1148
- ),
1149
- removed_refs AS (
1150
- DELETE FROM sync_blob_refs existing
1151
- USING changed_refs
1152
- WHERE existing.partition=$${n + 2} AND existing.tbl=$${n + 3}
1153
- AND existing.row_id=changed_refs.row_id
1154
- AND NOT EXISTS (
1155
- SELECT 1
1156
- FROM jsonb_to_recordset($${n + 4}::text::jsonb) AS r(row_id text, blob_ids jsonb)
1157
- CROSS JOIN LATERAL jsonb_array_elements_text(r.blob_ids) AS entry(value)
1158
- WHERE r.row_id=existing.row_id AND entry.value=existing.blob_id)
1159
- ),
1160
- inserted_refs AS (
1161
- INSERT INTO sync_blob_refs(partition, tbl, row_id, blob_id)
1162
- SELECT $${n + 2}, $${n + 3}, r.row_id, entry.value
1163
- FROM jsonb_to_recordset($${n + 4}::text::jsonb) AS r(row_id text, blob_ids jsonb)
1235
+ }
1236
+ if (blobRefs.length > 0) addBlobRefWrites(statement, p, tbl, blobRefs);
1237
+ }
1238
+
1239
+ /** Replace the §5.9.4 reference sets of rows of one table. */
1240
+ function addBlobRefWrites(
1241
+ statement: WriteStatement,
1242
+ p: string,
1243
+ tbl: string,
1244
+ blobRefs: readonly { row_id: string; blob_ids: readonly string[] }[],
1245
+ ): void {
1246
+ const k = statement.ctes.length;
1247
+ const refs = bindParam(statement, JSON.stringify(blobRefs));
1248
+ statement.ctes.push(
1249
+ `rr${k} AS (DELETE FROM sync_blob_refs existing
1250
+ WHERE existing.partition=${p} AND existing.tbl=${tbl}
1251
+ AND existing.row_id IN (SELECT r.row_id FROM jsonb_to_recordset(${refs}::text::jsonb) AS r(row_id text, blob_ids jsonb))
1252
+ AND NOT EXISTS (
1253
+ SELECT 1
1254
+ FROM jsonb_to_recordset(${refs}::text::jsonb) AS r(row_id text, blob_ids jsonb)
1164
1255
  CROSS JOIN LATERAL jsonb_array_elements_text(r.blob_ids) AS entry(value)
1165
- ON CONFLICT DO NOTHING
1166
- )
1167
- INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
1168
- SELECT $${n + 2}, $${n + 3}, key, value, row_id FROM scope
1169
- ON CONFLICT DO NOTHING`,
1170
- [
1171
- ...params,
1172
- JSON.stringify(
1173
- slice.map(({ row }) => ({ row_id: row.rowId, scopes: row.scopes })),
1174
- ),
1175
- partition,
1176
- compiled.name,
1177
- JSON.stringify(refs),
1178
- ],
1179
- ),
1256
+ WHERE r.row_id=existing.row_id AND entry.value=existing.blob_id))`,
1257
+ `ir${k} AS (INSERT INTO sync_blob_refs(partition, tbl, row_id, blob_id)
1258
+ SELECT ${p}, ${tbl}, r.row_id, entry.value
1259
+ FROM jsonb_to_recordset(${refs}::text::jsonb) AS r(row_id text, blob_ids jsonb)
1260
+ CROSS JOIN LATERAL jsonb_array_elements_text(r.blob_ids) AS entry(value)
1261
+ ON CONFLICT DO NOTHING)`,
1262
+ );
1263
+ }
1264
+
1265
+ /**
1266
+ * The CTEs that append commits with consecutive `commitSeq`s: the partition
1267
+ * cursor and delete floor (§2.2), the commit rows, their changes with the
1268
+ * inverted scope entries, and the §5 delete tombstones. The cursor advances
1269
+ * by the number of commits, and every row takes its commit's sequence from
1270
+ * the advanced cursor; the statement returns the cursor, which a caller that
1271
+ * computed the sequences under the partition lock checks. Every list binds
1272
+ * as one JSON text parameter, so the parameter count does not grow with the
1273
+ * change count. A tombstone keeps the change's row version (§2.2); a row
1274
+ * live again at the end of its commit (a §2.2 scope move) gets none, and a
1275
+ * later delete of the same row overwrites an earlier one.
1276
+ */
1277
+ function addAppends(
1278
+ statement: WriteStatement,
1279
+ partition: string,
1280
+ writerVersion: number,
1281
+ appends: readonly PendingAppend[],
1282
+ ): void {
1283
+ if (appends.length === 0) return;
1284
+ const p = bindParam(statement, partition);
1285
+ const changes: unknown[] = [];
1286
+ const tombstones = new Map<string, unknown>();
1287
+ let deleteFloor = 0;
1288
+ for (const [index, { commit }] of appends.entries()) {
1289
+ const k = index + 1;
1290
+ deleteFloor = Math.max(deleteFloor, commitDeleteFloor(commit.changes));
1291
+ const live = rowsLiveAfterDelete(commit.changes);
1292
+ for (const [idx, change] of commit.changes.entries()) {
1293
+ changes.push({
1294
+ k,
1295
+ idx,
1296
+ tbl: change.table,
1297
+ row_id: change.rowId,
1298
+ op: change.op === 'upsert' ? 1 : 2,
1299
+ version: change.rowVersion,
1300
+ scopes: change.scopes,
1301
+ payload: change.payload === undefined ? null : toBase64(change.payload),
1302
+ });
1303
+ const key = `${change.table}\u0000${change.rowId}`;
1304
+ if (change.op === 'delete' && !live.has(key)) {
1305
+ tombstones.set(key, {
1306
+ k,
1307
+ tbl: change.table,
1308
+ row_id: change.rowId,
1309
+ version: change.rowVersion,
1310
+ scopes: change.scopes,
1311
+ });
1312
+ }
1313
+ }
1314
+ }
1315
+ statement.appends = true;
1316
+ const expected = appends.at(-1)?.commitSeq;
1317
+ if (expected !== undefined) statement.allocates = expected;
1318
+ const n = bindParam(statement, appends.length);
1319
+ statement.ctes.push(
1320
+ `alloc AS (INSERT INTO sync_partitions(partition, max_commit_seq, delete_floor)
1321
+ VALUES (${p}, ${n}, ${bindParam(statement, deleteFloor)})
1322
+ ON CONFLICT (partition) DO UPDATE
1323
+ SET max_commit_seq=sync_partitions.max_commit_seq + EXCLUDED.max_commit_seq,
1324
+ delete_floor=GREATEST(sync_partitions.delete_floor, EXCLUDED.delete_floor)
1325
+ RETURNING max_commit_seq, max_commit_seq - ${n}::bigint AS base)`,
1326
+ `commits AS (INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms, writer_version)
1327
+ SELECT ${p}, alloc.base + c.k, c.client_id, c.client_commit_id, c.actor_id, c.created_at_ms, ${bindParam(statement, writerVersion)}
1328
+ FROM alloc CROSS JOIN jsonb_to_recordset(${bindParam(
1329
+ statement,
1330
+ JSON.stringify(
1331
+ appends.map(({ commit }, index) => ({
1332
+ k: index + 1,
1333
+ client_id: commit.clientId,
1334
+ client_commit_id: commit.clientCommitId,
1335
+ actor_id: commit.actorId,
1336
+ created_at_ms: commit.createdAtMs,
1337
+ })),
1338
+ ),
1339
+ )}::text::jsonb)
1340
+ AS c(k int, client_id text, client_commit_id text, actor_id text, created_at_ms bigint))`,
1341
+ );
1342
+ if (changes.length > 0) {
1343
+ statement.ctes.push(
1344
+ `changed AS (INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
1345
+ SELECT ${p}, alloc.base + c.k, c.idx, c.tbl, c.row_id, c.op, c.version, c.scopes, decode(c.payload, 'base64')
1346
+ FROM alloc CROSS JOIN jsonb_to_recordset(${bindParam(statement, JSON.stringify(changes))}::text::jsonb)
1347
+ AS c(k int, idx int, tbl text, row_id text, op smallint, version bigint, scopes jsonb, payload text)
1348
+ RETURNING commit_seq, tbl, scopes)`,
1349
+ `change_scopes AS (INSERT INTO sync_change_scopes(partition, tbl, var, value, commit_seq)
1350
+ SELECT ${p}, changed.tbl, scope.key, scope.value, changed.commit_seq
1351
+ FROM changed CROSS JOIN LATERAL jsonb_each_text(changed.scopes) AS scope
1352
+ ON CONFLICT DO NOTHING)`,
1180
1353
  );
1181
1354
  }
1182
- if (pendingRefs.size > 0) {
1183
- // Refs staged for a row this call did not carry (the row was written by an
1184
- // earlier flush): replace the reference set in one statement of its own.
1185
- writes.push(
1186
- q.query(
1187
- `WITH changed_refs AS (
1188
- SELECT DISTINCT row_id
1189
- FROM jsonb_to_recordset($3::text::jsonb) AS r(row_id text, blob_ids jsonb)
1190
- ),
1191
- removed_refs AS (
1192
- DELETE FROM sync_blob_refs existing
1193
- USING changed_refs
1194
- WHERE existing.partition=$1 AND existing.tbl=$2
1195
- AND existing.row_id=changed_refs.row_id
1196
- AND NOT EXISTS (
1197
- SELECT 1
1198
- FROM jsonb_to_recordset($3::text::jsonb) AS r(row_id text, blob_ids jsonb)
1199
- CROSS JOIN LATERAL jsonb_array_elements_text(r.blob_ids) AS entry(value)
1200
- WHERE r.row_id=existing.row_id AND entry.value=existing.blob_id)
1201
- )
1202
- INSERT INTO sync_blob_refs(partition, tbl, row_id, blob_id)
1203
- SELECT $1, $2, r.row_id, entry.value
1204
- FROM jsonb_to_recordset($3::text::jsonb) AS r(row_id text, blob_ids jsonb)
1205
- CROSS JOIN LATERAL jsonb_array_elements_text(r.blob_ids) AS entry(value)
1206
- ON CONFLICT DO NOTHING`,
1207
- [
1208
- partition,
1209
- compiled.name,
1210
- JSON.stringify(
1211
- [...pendingRefs].map(([row_id, blob_ids]) => ({
1212
- row_id,
1213
- blob_ids,
1214
- })),
1215
- ),
1216
- ],
1217
- ),
1355
+ if (tombstones.size > 0) {
1356
+ statement.ctes.push(
1357
+ `tombstoned AS (INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq, server_version, scopes)
1358
+ SELECT ${p}, t.tbl, t.row_id, alloc.base + t.k, t.version, t.scopes
1359
+ FROM alloc CROSS JOIN jsonb_to_recordset(${bindParam(statement, JSON.stringify([...tombstones.values()]))}::text::jsonb)
1360
+ AS t(k int, tbl text, row_id text, version bigint, scopes jsonb)
1361
+ ON CONFLICT (partition, tbl, row_id)
1362
+ DO UPDATE SET commit_seq=excluded.commit_seq,
1363
+ server_version=excluded.server_version,
1364
+ scopes=excluded.scopes)`,
1218
1365
  );
1219
1366
  }
1220
- return Promise.all(writes);
1367
+ }
1368
+
1369
+ /**
1370
+ * The statements that write staged rows, blob references, commit appends
1371
+ * and push results. Everything shares the first statement while the bound
1372
+ * parameters fit; a large row set continues in further statements of the
1373
+ * same shape.
1374
+ */
1375
+ function writeStatements(
1376
+ partition: string,
1377
+ writerVersion: number,
1378
+ resolveTable: (name: string) => CompiledTable,
1379
+ rows: ReadonlyMap<string, ReadonlyMap<string, StagedRow>>,
1380
+ blobRefs: ReadonlyMap<string, ReadonlyMap<string, readonly string[]>>,
1381
+ appends: readonly PendingAppend[],
1382
+ results: readonly unknown[],
1383
+ ): WriteStatement[] {
1384
+ let current: WriteStatement = { ctes: [], params: [] };
1385
+ const statements = [current];
1386
+ addAppends(current, partition, writerVersion, appends);
1387
+ if (results.length > 0) {
1388
+ current.ctes.push(
1389
+ `results AS (INSERT INTO sync_push_results(partition, client_id, client_commit_id, result, outbox_id, client_seq)
1390
+ SELECT ${bindParam(current, partition)}, r.client_id, r.client_commit_id, r.result, r.outbox_id, r.client_seq
1391
+ FROM jsonb_to_recordset(${bindParam(current, JSON.stringify(results))}::text::jsonb)
1392
+ AS r(client_id text, client_commit_id text, result jsonb, outbox_id text, client_seq bigint)
1393
+ ON CONFLICT (partition, client_id, client_commit_id) DO NOTHING)`,
1394
+ );
1395
+ }
1396
+ for (const table of new Set([...rows.keys(), ...blobRefs.keys()])) {
1397
+ const compiled = resolveTable(table);
1398
+ const width = tableColumnNames(compiled).length;
1399
+ const chunk = Math.max(
1400
+ 1,
1401
+ Math.floor((PG_PARAMS_PER_STATEMENT - 4) / width),
1402
+ );
1403
+ const pending = new Map(blobRefs.get(table));
1404
+ const staged = [...(rows.get(table)?.values() ?? [])];
1405
+ for (let first = 0; first < staged.length; first += chunk) {
1406
+ const slice = staged.slice(first, first + chunk);
1407
+ if (
1408
+ current.params.length + slice.length * width + 4 >
1409
+ PG_PARAMS_PER_STATEMENT
1410
+ ) {
1411
+ current = { ctes: [], params: [] };
1412
+ statements.push(current);
1413
+ }
1414
+ const refs: { row_id: string; blob_ids: readonly string[] }[] = [];
1415
+ for (const { row } of slice) {
1416
+ const blobIds = pending.get(row.rowId);
1417
+ if (blobIds === undefined) continue;
1418
+ pending.delete(row.rowId);
1419
+ refs.push({ row_id: row.rowId, blob_ids: blobIds });
1420
+ }
1421
+ addRowWrites(current, compiled, partition, slice, refs);
1422
+ }
1423
+ if (pending.size > 0) {
1424
+ // Refs staged for a row written by an earlier statement.
1425
+ addBlobRefWrites(
1426
+ current,
1427
+ bindParam(current, partition),
1428
+ bindParam(current, table),
1429
+ [...pending].map(([row_id, blob_ids]) => ({ row_id, blob_ids })),
1430
+ );
1431
+ }
1432
+ }
1433
+ return statements.filter((statement) => statement.ctes.length > 0);
1221
1434
  }
1222
1435
 
1223
1436
  async function getPushResultOn(
@@ -1271,7 +1484,7 @@ async function queryAuthoritativeOn(
1271
1484
 
1272
1485
  class PostgresTransaction implements StorageTransaction {
1273
1486
  #client: PgQueryable;
1274
- /** The FIFO without the staged-write prelude (see the constructor). */
1487
+ /** The FIFO without the pending-write prelude (see the constructor). */
1275
1488
  #serial: PgQueryable;
1276
1489
  #partition: string;
1277
1490
  /** Storage-owned relation rewrite, bound to this transaction's partition. */
@@ -1284,13 +1497,41 @@ class PostgresTransaction implements StorageTransaction {
1284
1497
  /** The storage's `chooseScanVariable` picks. */
1285
1498
  readonly #scanVariables: Map<string, string>;
1286
1499
  #open = true;
1500
+ /** Set by `preparePushApply`: commits close with a result and roll back alone. */
1501
+ #pushApply = false;
1502
+ /** Set by `preparePushApply` on the rerun after a closed commit's write failed. */
1503
+ #closeEagerly = false;
1287
1504
  /** True once the request's current commit has written behind its savepoint. */
1288
1505
  #pushApplySavepoint = false;
1289
1506
  /**
1290
- * The request's §2.3 results, written as one statement at `commit()`
1291
- * (§6.4): each commit closes with its result, and none is read back before
1292
- * the transaction ends.
1507
+ * The `commitSeq` the next appended commit takes. The partition lock read
1508
+ * sets it and freezes the cursor, so an append needs no allocation read.
1509
+ */
1510
+ #nextCommitSeq: number | undefined;
1511
+ /** The `commitSeq` the open push commit appended, until it closes. */
1512
+ #openCommitSeq: number | undefined;
1513
+ /**
1514
+ * The open commit's upserts staged per table name (row id → row and its
1515
+ * push operation index), its §5.9.4 reference sets, and its commit append.
1516
+ * They reach the database with the next statement that needs them, behind
1517
+ * the commit's savepoint, as one statement: a commit's rows cost one round
1518
+ * trip per statement instead of one per row. Tables with a unique index
1519
+ * write immediately: a violation must name its operation.
1520
+ */
1521
+ readonly #staged = new Map<string, Map<string, StagedRow>>();
1522
+ readonly #stagedBlobRefs = new Map<string, Map<string, readonly string[]>>();
1523
+ #stagedAppend: PendingAppend | undefined;
1524
+ /**
1525
+ * The writes of push commits that closed and are not yet written: rows (a
1526
+ * later commit's row replaces an earlier one), reference sets, appends in
1527
+ * `commitSeq` order, and the §2.3 results. They reach the database in one
1528
+ * statement ahead of the next statement of the transaction, outside any
1529
+ * commit's savepoint; a request whose commits never read their own writes
1530
+ * writes them all in one statement before COMMIT (§6.4).
1293
1531
  */
1532
+ readonly #closedRows = new Map<string, Map<string, StagedRow>>();
1533
+ readonly #closedBlobRefs = new Map<string, Map<string, readonly string[]>>();
1534
+ readonly #closedAppends: PendingAppend[] = [];
1294
1535
  readonly #pushResults: {
1295
1536
  readonly client_id: string;
1296
1537
  readonly client_commit_id: string;
@@ -1305,39 +1546,6 @@ class PostgresTransaction implements StorageTransaction {
1305
1546
  */
1306
1547
  readonly #prefetchedRows = new Map<string, StoredRow | null>();
1307
1548
  readonly #prefetchedTombstones = new Map<string, Tombstone | null>();
1308
- /**
1309
- * Upserts staged per table (row id → row and its push operation index).
1310
- * Every statement of this transaction writes them first (see the
1311
- * constructor), so reads and later writes observe them; a commit's rows
1312
- * reach the database as one statement per table instead of one per row.
1313
- * Tables with a unique index write immediately: a violation must name its
1314
- * operation.
1315
- */
1316
- readonly #staged = new Map<
1317
- CompiledTable,
1318
- Map<
1319
- string,
1320
- {
1321
- readonly row: StoredRow;
1322
- readonly values: readonly unknown[];
1323
- readonly opIndex?: number;
1324
- }
1325
- >
1326
- >();
1327
- /**
1328
- * §5.9.4 reference sets staged per table (row id → blob ids). They flush
1329
- * with the table's row statement, so a blob-referencing row costs no extra
1330
- * round trip; a ref set staged for an already-written row flushes on the
1331
- * next statement of its table.
1332
- */
1333
- readonly #stagedBlobRefs = new Map<string, Map<string, readonly string[]>>();
1334
- /**
1335
- * The partition cursor (`max_commit_seq`) read by the first registered
1336
- * query in this transaction. The partition lock freezes it, so later
1337
- * registered queries in the same transaction reuse the value instead of
1338
- * paying a round trip each.
1339
- */
1340
- #cursorSeq: number | undefined;
1341
1549
  /** Resolves/rejects the `transaction(fn)` wrapper (see `begin`). */
1342
1550
  #resolve: () => void;
1343
1551
  #reject: (error: unknown) => void;
@@ -1360,7 +1568,7 @@ class PostgresTransaction implements StorageTransaction {
1360
1568
  // issued through this transaction is chained on one FIFO — the same
1361
1569
  // shape as `SqliteServerStorage.queryAuthoritative`.
1362
1570
  let tail: Promise<void> = Promise.resolve();
1363
- const serial: PgQueryable = {
1571
+ this.#serial = {
1364
1572
  query: <Row>(text: string, params?: readonly unknown[]) => {
1365
1573
  const run = tail.then(() => client.query<Row>(text, params));
1366
1574
  tail = run.then(
@@ -1370,22 +1578,8 @@ class PostgresTransaction implements StorageTransaction {
1370
1578
  return run;
1371
1579
  },
1372
1580
  };
1373
- this.#serial = serial;
1374
1581
  this.#client = {
1375
- query: async <Row>(text: string, params?: readonly unknown[]) => {
1376
- // Queue the staged writes ahead of this statement synchronously, so
1377
- // a concurrently issued statement cannot overtake them.
1378
- const staged = this.#writeStaged();
1379
- const run = serial.query<Row>(text, params);
1380
- try {
1381
- await staged;
1382
- } catch (error) {
1383
- // The aborted transaction fails the queued statement too.
1384
- run.catch(() => undefined);
1385
- throw error;
1386
- }
1387
- return run;
1388
- },
1582
+ query: (text, params) => this.#statement(text, params, 'read'),
1389
1583
  };
1390
1584
  this.#partition = partition;
1391
1585
  this.#writerVersion = writerVersion;
@@ -1399,74 +1593,179 @@ class PostgresTransaction implements StorageTransaction {
1399
1593
  if (!this.#open) throw new Error('transaction already finished');
1400
1594
  }
1401
1595
 
1402
- #writeStaged(): Promise<unknown> {
1403
- if (this.#staged.size === 0 && this.#stagedBlobRefs.size === 0) {
1404
- return Promise.resolve();
1596
+ /**
1597
+ * Queue the pending writes and then one statement. `write` marks a
1598
+ * candidate write of the open commit, which needs the commit's savepoint
1599
+ * even when nothing is staged.
1600
+ */
1601
+ async #statement<Row>(
1602
+ text: string,
1603
+ params: readonly unknown[] | undefined,
1604
+ mode: 'read' | 'write',
1605
+ ): Promise<{ rows: Row[] }> {
1606
+ // Queue the pending writes ahead of this statement synchronously, so a
1607
+ // concurrently issued statement cannot overtake them.
1608
+ const pending = this.#flush(mode);
1609
+ const run = this.#serial.query<Row>(text, params);
1610
+ try {
1611
+ await pending;
1612
+ } catch (error) {
1613
+ // The aborted transaction fails the queued statement too.
1614
+ run.catch(() => undefined);
1615
+ throw error;
1405
1616
  }
1406
- const tables = new Set<string>([
1407
- ...[...this.#staged.keys()].map((table) => table.name),
1408
- ...this.#stagedBlobRefs.keys(),
1409
- ]);
1410
- const batches = [...tables].map((table) => {
1411
- const compiled = this.#resolveTable(table);
1412
- const rows = [...(this.#staged.get(compiled)?.values() ?? [])];
1413
- const blobRefs = [
1414
- ...(this.#stagedBlobRefs.get(table)?.entries() ?? []),
1415
- ].map(([rowId, blobIds]) => ({ rowId, blobIds }));
1416
- this.#staged.delete(compiled);
1417
- this.#stagedBlobRefs.delete(table);
1418
- return { table, compiled, rows, blobRefs };
1419
- });
1420
- const writes: Promise<unknown>[] = [];
1421
- // The candidate savepoint lands before the first candidate write, and
1422
- // only when a write actually reaches the database: a commit refused
1423
- // before its first write has no candidate rows to roll back, so its
1424
- // rejection finalization skips the savepoint round trips.
1425
- const savepoint = this.#ensurePushSavepoint();
1426
- if (savepoint !== undefined) writes.push(savepoint);
1427
- for (const { compiled, rows, blobRefs } of batches) {
1428
- writes.push(
1429
- writeRowsOn(
1430
- this.#serial,
1431
- compiled,
1432
- this.#partition,
1433
- rows,
1434
- blobRefs,
1435
- ).catch((error: unknown) => {
1436
- // Staged tables have no unique index, so a violation is a column
1437
- // constraint; it names the first staged operation of the table.
1438
- if (isPostgresConstraintError(error)) {
1439
- throw new StorageConstraintError(error, rows[0]?.opIndex);
1440
- }
1441
- throw error;
1442
- }),
1617
+ return run;
1618
+ }
1619
+
1620
+ /**
1621
+ * Queue every pending write, in order: the closed commits' writes, then
1622
+ * the open commit's savepoint (taken before its first write reaches the
1623
+ * database, so a refused commit that wrote nothing pays for none) and its
1624
+ * staged writes. `final` is the flush before COMMIT, which no rejection
1625
+ * follows.
1626
+ *
1627
+ * A relational constraint that fails on a closed commit's writes can no
1628
+ * longer reject that commit alone: the transaction is aborted, and
1629
+ * `ClosedCommitConstraintError` tells the push layer to rerun the request
1630
+ * with each commit written before it closes (§6.4).
1631
+ *
1632
+ * Resolves to the partition cursor the open commit's append returned.
1633
+ */
1634
+ async #flush(mode: 'read' | 'write' | 'final'): Promise<number | undefined> {
1635
+ const pending: Promise<unknown>[] = [];
1636
+ let cursor: Promise<number | undefined> = Promise.resolve(undefined);
1637
+ if (
1638
+ this.#closedRows.size > 0 ||
1639
+ this.#closedBlobRefs.size > 0 ||
1640
+ this.#closedAppends.length > 0 ||
1641
+ this.#pushResults.length > 0
1642
+ ) {
1643
+ const statements = writeStatements(
1644
+ this.#partition,
1645
+ this.#writerVersion,
1646
+ this.#resolveTable,
1647
+ this.#closedRows,
1648
+ this.#closedBlobRefs,
1649
+ this.#closedAppends,
1650
+ this.#pushResults,
1651
+ );
1652
+ this.#closedRows.clear();
1653
+ this.#closedBlobRefs.clear();
1654
+ this.#closedAppends.length = 0;
1655
+ this.#pushResults.length = 0;
1656
+ pending.push(
1657
+ this.#runWrites(statements, (error) =>
1658
+ isPostgresConstraintError(error)
1659
+ ? new ClosedCommitConstraintError(error)
1660
+ : error,
1661
+ ),
1443
1662
  );
1444
1663
  }
1445
- return Promise.all(writes);
1664
+ const staged =
1665
+ this.#staged.size > 0 ||
1666
+ this.#stagedBlobRefs.size > 0 ||
1667
+ this.#stagedAppend !== undefined;
1668
+ if (
1669
+ mode !== 'final' &&
1670
+ this.#pushApply &&
1671
+ !this.#pushApplySavepoint &&
1672
+ (staged || mode === 'write')
1673
+ ) {
1674
+ // Each commit takes its own savepoint under the same name: `ROLLBACK
1675
+ // TO` targets the newest, and an earlier commit's savepoint stays open
1676
+ // until the transaction ends, so no `RELEASE` round trip is needed.
1677
+ this.#pushApplySavepoint = true;
1678
+ pending.push(this.#serial.query('SAVEPOINT syncular_push_candidate'));
1679
+ }
1680
+ if (staged) {
1681
+ const statements = writeStatements(
1682
+ this.#partition,
1683
+ this.#writerVersion,
1684
+ this.#resolveTable,
1685
+ this.#staged,
1686
+ this.#stagedBlobRefs,
1687
+ this.#stagedAppend === undefined ? [] : [this.#stagedAppend],
1688
+ [],
1689
+ );
1690
+ this.#staged.clear();
1691
+ this.#stagedBlobRefs.clear();
1692
+ this.#stagedAppend = undefined;
1693
+ cursor = this.#runWrites(statements, (error, statement) =>
1694
+ isPostgresConstraintError(error)
1695
+ ? new StorageConstraintError(error, statement.opIndex)
1696
+ : error,
1697
+ );
1698
+ pending.push(cursor);
1699
+ }
1700
+ await Promise.all(pending);
1701
+ return cursor;
1446
1702
  }
1447
1703
 
1448
1704
  /**
1449
- * The current commit's candidate savepoint, taken before its first write.
1450
- * Every candidate write must be undoable by `rejectPushCommit`, and a
1451
- * refused commit that wrote nothing must not pay to create one. Each commit
1452
- * takes its own savepoint under the same name: `ROLLBACK TO` targets the
1453
- * newest, and an earlier commit's savepoint stays open until the
1454
- * transaction ends, so no `RELEASE` round trip is needed (§6.4).
1705
+ * Queue write statements on the FIFO. Resolves to the cursor a commit
1706
+ * append returned, checked against the sequences computed under the lock.
1455
1707
  */
1456
- #ensurePushSavepoint(): Promise<unknown> | undefined {
1457
- if (this.#pushApplySavepoint) return undefined;
1458
- this.#pushApplySavepoint = true;
1459
- return this.#serial.query('SAVEPOINT syncular_push_candidate');
1708
+ async #runWrites(
1709
+ statements: readonly WriteStatement[],
1710
+ mapError: (error: unknown, statement: WriteStatement) => unknown,
1711
+ ): Promise<number | undefined> {
1712
+ const cursors = await Promise.all(
1713
+ statements.map((statement) =>
1714
+ this.#serial
1715
+ .query<{ allocated: unknown }>(
1716
+ `WITH ${statement.ctes.join(',\n')} SELECT ${
1717
+ statement.appends === true
1718
+ ? '(SELECT max_commit_seq FROM alloc)'
1719
+ : 'NULL::bigint'
1720
+ } AS allocated`,
1721
+ statement.params,
1722
+ )
1723
+ .then(
1724
+ ({ rows }) => {
1725
+ if (statement.appends !== true) return undefined;
1726
+ const allocated = asNumber(rows[0]?.allocated);
1727
+ if (
1728
+ statement.allocates !== undefined &&
1729
+ allocated !== statement.allocates
1730
+ ) {
1731
+ throw new Error(
1732
+ 'partition commit sequence moved under the partition lock',
1733
+ );
1734
+ }
1735
+ return allocated;
1736
+ },
1737
+ (error: unknown) => {
1738
+ throw mapError(error, statement);
1739
+ },
1740
+ ),
1741
+ ),
1742
+ );
1743
+ return cursors.find((cursor) => cursor !== undefined);
1460
1744
  }
1461
1745
 
1746
+ /** The candidate row this transaction staged and has not yet written. */
1747
+ #stagedRow(table: string, rowId: string): StoredRow | undefined {
1748
+ return (
1749
+ this.#staged.get(table)?.get(rowId)?.row ??
1750
+ this.#closedRows.get(table)?.get(rowId)?.row
1751
+ );
1752
+ }
1753
+
1754
+ /**
1755
+ * A read by primary key. A staged row answers from memory, so the read
1756
+ * needs no pending write: it runs without flushing them, and a validator
1757
+ * that reads rows by id adds no savepoint or write round trip (§6.4).
1758
+ */
1462
1759
  getRow(table: string, rowId: string): Promise<StoredRow | undefined> {
1463
1760
  this.#assertOpen();
1761
+ const staged = this.#stagedRow(table, rowId);
1762
+ if (staged !== undefined) return Promise.resolve(staged);
1464
1763
  const prefetched = this.#prefetchedRows.get(`${table}\u0000${rowId}`);
1465
1764
  if (prefetched !== undefined) {
1466
1765
  return Promise.resolve(prefetched ?? undefined);
1467
1766
  }
1468
1767
  return getRowOn(
1469
- this.#client,
1768
+ this.#serial,
1470
1769
  this.#resolveTable(table),
1471
1770
  this.#partition,
1472
1771
  rowId,
@@ -1481,15 +1780,16 @@ class PostgresTransaction implements StorageTransaction {
1481
1780
  const rows = new Map<string, StoredRow>();
1482
1781
  const missing: string[] = [];
1483
1782
  for (const rowId of new Set(rowIds)) {
1783
+ const staged = this.#stagedRow(table, rowId);
1484
1784
  const prefetched = this.#prefetchedRows.get(`${table}\u0000${rowId}`);
1485
- if (prefetched === undefined) missing.push(rowId);
1785
+ if (staged !== undefined) rows.set(rowId, staged);
1786
+ else if (prefetched === undefined) missing.push(rowId);
1486
1787
  else if (prefetched !== null) rows.set(rowId, prefetched);
1487
1788
  }
1488
1789
  if (missing.length === 0) return rows;
1489
- // One statement for every id the admission read did not already answer.
1490
- // `#client` flushes this transaction's staged upserts first, so the rows
1491
- // are the final candidate state.
1492
- const { rows: records } = await this.#client.query<RowRecord>(
1790
+ // One statement for every id neither the staged writes nor the
1791
+ // admission read answered (see `getRow`).
1792
+ const { rows: records } = await this.#serial.query<RowRecord>(
1493
1793
  selectRowsSql(this.#resolveTable(table), 'postgres'),
1494
1794
  [this.#partition, JSON.stringify(missing)],
1495
1795
  );
@@ -1508,7 +1808,9 @@ class PostgresTransaction implements StorageTransaction {
1508
1808
  const specs: { tbl: string; ids: string[] }[] = [];
1509
1809
  for (const target of targets) {
1510
1810
  const ids = [...new Set(target.rowIds)].filter(
1511
- (rowId) => !this.#prefetchedRows.has(`${target.table}\u0000${rowId}`),
1811
+ (rowId) =>
1812
+ !this.#prefetchedRows.has(`${target.table}\u0000${rowId}`) &&
1813
+ this.#stagedRow(target.table, rowId) === undefined,
1512
1814
  );
1513
1815
  if (ids.length === 0) continue;
1514
1816
  tables.push(this.#resolveTable(target.table));
@@ -1568,10 +1870,10 @@ class PostgresTransaction implements StorageTransaction {
1568
1870
  async clearTombstone(table: string, rowId: string): Promise<void> {
1569
1871
  this.#assertOpen();
1570
1872
  this.#prefetchedTombstones.delete(`${table}\u0000${rowId}`);
1571
- await this.#ensurePushSavepoint();
1572
- await this.#client.query(
1873
+ await this.#statement(
1573
1874
  'DELETE FROM sync_tombstones WHERE partition=$1 AND tbl=$2 AND row_id=$3',
1574
1875
  [this.#partition, table, rowId],
1876
+ 'write',
1575
1877
  );
1576
1878
  }
1577
1879
 
@@ -1643,19 +1945,20 @@ class PostgresTransaction implements StorageTransaction {
1643
1945
  postgresPlaceholders(prepared.sql),
1644
1946
  prepared.params,
1645
1947
  );
1646
- // The partition lock freezes `max_commit_seq` for the whole transaction,
1647
- // so one read answers every registered query in it.
1648
- if (this.#cursorSeq === undefined) {
1649
- const cursor = await this.#client.query<{ max_commit_seq: unknown }>(
1650
- 'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1',
1651
- [this.#partition],
1652
- );
1653
- this.#cursorSeq =
1654
- cursor.rows[0] === undefined
1655
- ? 0
1656
- : asNumber(cursor.rows[0].max_commit_seq);
1948
+ // Under the partition lock the cursor is the locked `max_commit_seq` plus
1949
+ // this transaction's appends, which the query above flushed: no read.
1950
+ if (this.#nextCommitSeq !== undefined) {
1951
+ return { rows: result.rows, maxCommitSeq: this.#nextCommitSeq - 1 };
1657
1952
  }
1658
- return { rows: result.rows, maxCommitSeq: this.#cursorSeq };
1953
+ const cursor = await this.#client.query<{ max_commit_seq: unknown }>(
1954
+ 'SELECT max_commit_seq FROM sync_partitions WHERE partition=$1',
1955
+ [this.#partition],
1956
+ );
1957
+ // asNumber maps the absent row of a partition without a commit to 0.
1958
+ return {
1959
+ rows: result.rows,
1960
+ maxCommitSeq: asNumber(cursor.rows[0]?.max_commit_seq),
1961
+ };
1659
1962
  }
1660
1963
 
1661
1964
  async preparePushApply(
@@ -1663,18 +1966,17 @@ class PostgresTransaction implements StorageTransaction {
1663
1966
  commits: readonly PushApplyCommit[],
1664
1967
  runningSchemaVersion: number,
1665
1968
  outbox?: OutboxSettlement,
1969
+ options?: PushApplyOptions,
1666
1970
  ): Promise<PreparedPushApply> {
1667
1971
  this.#assertOpen();
1972
+ this.#pushApply = true;
1973
+ this.#closeEagerly = options?.closeEagerly === true;
1668
1974
  // The lock is its own statement: a READ COMMITTED statement's snapshot is
1669
1975
  // fixed at its start, so the admission read must run *after* a duplicate's
1670
1976
  // commit has released the partition lock to observe it. Folding the lock
1671
1977
  // and the read into one statement would miss a duplicate that committed
1672
1978
  // while this transaction waited.
1673
1979
  await this.lockPartition();
1674
- const watermark =
1675
- outbox === undefined
1676
- ? undefined
1677
- : await this.#settleOutbox(clientId, outbox);
1678
1980
  const ids = new Map<string, Set<string>>();
1679
1981
  for (const commit of commits) {
1680
1982
  for (const target of commit.targets ?? []) {
@@ -1689,8 +1991,10 @@ class PostgresTransaction implements StorageTransaction {
1689
1991
  }
1690
1992
  const tables = [...ids.keys()].map((table) => this.#resolveTable(table));
1691
1993
  const specs = [...ids].map(([tbl, rowIds]) => ({ tbl, ids: [...rowIds] }));
1692
- // One statement for the whole request: the serve gate, every commit's
1693
- // idempotency record, and the read-ahead of every commit's known rows.
1994
+ // One statement for the whole request: the outbox settlement, the serve
1995
+ // gate, every commit's idempotency record, and the read-ahead of every
1996
+ // commit's known rows. The settlement precedes every commit's savepoint,
1997
+ // so no rejection discards it.
1694
1998
  const admission = await readAdmissionOn(
1695
1999
  this.#client,
1696
2000
  this.#partition,
@@ -1698,6 +2002,7 @@ class PostgresTransaction implements StorageTransaction {
1698
2002
  commits.map((commit) => commit.clientCommitId),
1699
2003
  runningSchemaVersion,
1700
2004
  specs.length === 0 ? undefined : { tables, specs },
2005
+ outbox,
1701
2006
  );
1702
2007
  this.#recordPrefetched(admission.prefetched);
1703
2008
  return {
@@ -1705,73 +2010,16 @@ class PostgresTransaction implements StorageTransaction {
1705
2010
  persisted: (clientCommitId) =>
1706
2011
  toPersistedPushResult(admission.persisted.get(clientCommitId)),
1707
2012
  deleteFloor: admission.deleteFloor,
1708
- ...(watermark === undefined ? {} : { watermark }),
2013
+ ...(admission.watermark === undefined
2014
+ ? {}
2015
+ : { watermark: admission.watermark }),
1709
2016
  };
1710
2017
  }
1711
2018
 
1712
- /**
1713
- * The §9 settlement and §6.3.2 steps 1 and 2 under the partition lock,
1714
- * before the first commit's savepoint, so no rejection discards them. Two
1715
- * statements: the settlement touches disjoint rows (outside and inside the
1716
- * request), and the watermark delete must see its sequences.
1717
- */
1718
- async #settleOutbox(
1719
- clientId: string,
1720
- outbox: OutboxSettlement,
1721
- ): Promise<number> {
1722
- const first = outbox.commits[0];
1723
- const last = outbox.commits.at(-1);
1724
- if (first === undefined || last === undefined) {
1725
- throw new Error('outbox settlement needs the request commits');
1726
- }
1727
- const commits = JSON.stringify(
1728
- outbox.commits.map((commit) => ({
1729
- id: commit.clientCommitId,
1730
- seq: commit.clientSeq,
1731
- })),
1732
- );
1733
- // §9 (b): with no pre-upgrade result on the request's last commit, every
1734
- // remaining one outside the request left the outbox; (a) assigns the
1735
- // request's sequences to its pre-upgrade results.
1736
- await this.#client.query(
1737
- `WITH c AS (SELECT * FROM jsonb_to_recordset($4::text::jsonb) AS c(id text, seq bigint)),
1738
- b AS (DELETE FROM sync_push_results
1739
- WHERE partition=$1 AND client_id=$2 AND outbox_id IS NULL
1740
- AND client_commit_id NOT IN (SELECT id FROM c)
1741
- AND NOT EXISTS (SELECT 1 FROM sync_push_results
1742
- WHERE partition=$1 AND client_id=$2 AND client_commit_id=$5
1743
- AND outbox_id IS NULL)),
1744
- a AS (UPDATE sync_push_results r SET outbox_id=$3, client_seq=c.seq FROM c
1745
- WHERE r.partition=$1 AND r.client_id=$2 AND r.outbox_id IS NULL
1746
- AND r.client_commit_id=c.id)
1747
- SELECT 1`,
1748
- [
1749
- this.#partition,
1750
- clientId,
1751
- outbox.outboxId,
1752
- commits,
1753
- last.clientCommitId,
1754
- ],
1755
- );
1756
- const { rows } = await this.#client.query<{ through_seq: unknown }>(
1757
- `WITH w AS (
1758
- INSERT INTO sync_push_watermarks(partition, client_id, outbox_id, through_seq)
1759
- VALUES ($1,$2,$3,$4)
1760
- ON CONFLICT (partition, client_id, outbox_id) DO UPDATE
1761
- SET through_seq=GREATEST(sync_push_watermarks.through_seq, EXCLUDED.through_seq)
1762
- RETURNING through_seq),
1763
- d AS (DELETE FROM sync_push_results
1764
- WHERE partition=$1 AND client_id=$2 AND outbox_id=$3
1765
- AND client_seq<(SELECT through_seq FROM w))
1766
- SELECT through_seq FROM w`,
1767
- [this.#partition, clientId, outbox.outboxId, first.clientSeq],
1768
- );
1769
- return asNumber(rows[0]?.through_seq);
1770
- }
1771
-
1772
2019
  async lockPartition(): Promise<void> {
1773
2020
  this.#assertOpen();
1774
- await lockPartitionOn(this.#client, this.#partition);
2021
+ this.#nextCommitSeq =
2022
+ (await lockPartitionOn(this.#client, this.#partition)) + 1;
1775
2023
  }
1776
2024
 
1777
2025
  async pinSnapshot(): Promise<void> {
@@ -1790,15 +2038,15 @@ class PostgresTransaction implements StorageTransaction {
1790
2038
  nowMs: number,
1791
2039
  ): Promise<boolean> {
1792
2040
  this.#assertOpen();
1793
- await this.#ensurePushSavepoint();
1794
2041
  // `RETURNING`: `PgQueryable` reports returned rows only (no affected-row
1795
2042
  // count), so every affected-row check counts returned rows.
1796
- const result = await this.#client.query<{ name: string }>(
2043
+ const result = await this.#statement<{ name: string }>(
1797
2044
  `UPDATE sync_backfill_checkpoints
1798
2045
  SET watermark=$1, observed_rows=$2, updated_at_ms=$3
1799
2046
  WHERE partition=$4 AND name=$5 AND owner_epoch=$6
1800
2047
  RETURNING name`,
1801
2048
  [watermark, observedRows, nowMs, this.#partition, name, ownerEpoch],
2049
+ 'write',
1802
2050
  );
1803
2051
  return result.rows.length === 1;
1804
2052
  }
@@ -1817,13 +2065,20 @@ class PostgresTransaction implements StorageTransaction {
1817
2065
  sequence?: OutboxSequence,
1818
2066
  ): Promise<void> {
1819
2067
  this.#assertOpen();
1820
- // Candidate rows that never reached the database are discarded with the
1821
- // staging map; a savepoint exists only when one did. It stays open: the
2068
+ // Candidate writes that never reached the database are discarded with the
2069
+ // staging maps; a savepoint exists only when one did. It stays open: the
1822
2070
  // rollback returns the database to where the next commit starts (the
1823
- // results wait in memory), so the next commit writes behind it.
2071
+ // closed commits' writes wait in memory or precede it), so the next
2072
+ // commit writes behind it.
1824
2073
  this.#staged.clear();
1825
2074
  this.#stagedBlobRefs.clear();
2075
+ this.#stagedAppend = undefined;
2076
+ if (this.#openCommitSeq !== undefined) {
2077
+ this.#nextCommitSeq = this.#openCommitSeq;
2078
+ this.#openCommitSeq = undefined;
2079
+ }
1826
2080
  if (this.#pushApplySavepoint) {
2081
+ this.#pushApplySavepoint = false;
1827
2082
  await this.#serial.query('ROLLBACK TO SAVEPOINT syncular_push_candidate');
1828
2083
  }
1829
2084
  this.#recordPushResult(clientId, clientCommitId, result, sequence);
@@ -1854,19 +2109,20 @@ class PostgresTransaction implements StorageTransaction {
1854
2109
  const compiled = this.#resolveTable(table);
1855
2110
  // Decodes the payload: an undecodable row fails here, at its operation.
1856
2111
  const values = upsertValues(compiled, this.#partition, row, 'postgres');
2112
+ const entry = {
2113
+ row,
2114
+ values,
2115
+ ...(context === undefined ? {} : { opIndex: context.opIndex }),
2116
+ };
1857
2117
  if (!compiled.indexes.some((index) => index.unique === true)) {
1858
- let rows = this.#staged.get(compiled);
2118
+ let rows = this.#staged.get(table);
1859
2119
  if (rows === undefined) {
1860
2120
  rows = new Map();
1861
- this.#staged.set(compiled, rows);
2121
+ this.#staged.set(table, rows);
1862
2122
  }
1863
2123
  // A later upsert of the same row in this transaction replaces it.
1864
2124
  rows.delete(row.rowId);
1865
- rows.set(row.rowId, {
1866
- row,
1867
- values,
1868
- ...(context === undefined ? {} : { opIndex: context.opIndex }),
1869
- });
2125
+ rows.set(row.rowId, entry);
1870
2126
  return;
1871
2127
  }
1872
2128
  const refs = this.#stagedBlobRefs.get(table);
@@ -1875,15 +2131,25 @@ class PostgresTransaction implements StorageTransaction {
1875
2131
  refs.delete(row.rowId);
1876
2132
  if (refs.size === 0) this.#stagedBlobRefs.delete(table);
1877
2133
  }
1878
- await this.#ensurePushSavepoint();
2134
+ const statements = writeStatements(
2135
+ this.#partition,
2136
+ this.#writerVersion,
2137
+ this.#resolveTable,
2138
+ new Map([[table, new Map([[row.rowId, entry]])]]),
2139
+ blobIds === undefined
2140
+ ? new Map()
2141
+ : new Map([[table, new Map([[row.rowId, blobIds]])]]),
2142
+ [],
2143
+ [],
2144
+ );
1879
2145
  try {
1880
- await writeRowsOn(
1881
- this.#client,
1882
- compiled,
1883
- this.#partition,
1884
- [{ row, values }],
1885
- blobIds === undefined ? [] : [{ rowId: row.rowId, blobIds }],
1886
- );
2146
+ for (const statement of statements) {
2147
+ await this.#statement(
2148
+ `WITH ${statement.ctes.join(',\n')} SELECT 1`,
2149
+ statement.params,
2150
+ 'write',
2151
+ );
2152
+ }
1887
2153
  } catch (error) {
1888
2154
  if (isPostgresConstraintError(error)) {
1889
2155
  throw new StorageConstraintError(error, context?.opIndex);
@@ -1898,16 +2164,16 @@ class PostgresTransaction implements StorageTransaction {
1898
2164
  const refs = this.#stagedBlobRefs.get(table);
1899
2165
  refs?.delete(rowId);
1900
2166
  if (refs?.size === 0) this.#stagedBlobRefs.delete(table);
1901
- await this.#ensurePushSavepoint();
1902
2167
  // One statement: the row, its scope-index entries, and (§5.9.4: a
1903
2168
  // deleted row references no blobs) its blob references.
1904
- await this.#client.query(
2169
+ await this.#statement(
1905
2170
  `WITH deleted AS (${deleteRowSql(this.#resolveTable(table), 'postgres')}),
1906
2171
  unscoped AS (
1907
2172
  DELETE FROM sync_row_scopes WHERE partition=$1 AND tbl=$3 AND row_id=$2
1908
2173
  )
1909
2174
  DELETE FROM sync_blob_refs WHERE partition=$1 AND tbl=$3 AND row_id=$2`,
1910
2175
  [this.#partition, rowId, table],
2176
+ 'write',
1911
2177
  );
1912
2178
  }
1913
2179
 
@@ -1928,9 +2194,20 @@ class PostgresTransaction implements StorageTransaction {
1928
2194
  return Promise.resolve();
1929
2195
  }
1930
2196
 
2197
+ /**
2198
+ * Stage the append with the commit's rows. Under the partition lock the
2199
+ * cursor is frozen, so the next `commitSeq` is known without a statement
2200
+ * and a push commit's append waits for the next statement. Outside a push
2201
+ * apply the append is written at once (its allocation takes the lock when
2202
+ * no earlier statement did), so a relational constraint on the staged rows
2203
+ * fails this call.
2204
+ */
1931
2205
  async appendCommit(commit: NewCommit): Promise<number> {
1932
2206
  this.#assertOpen();
1933
- // The commit writes its delete tombstones below; every other prefetched
2207
+ if (this.#stagedAppend !== undefined || this.#openCommitSeq !== undefined) {
2208
+ throw new Error('the open commit already appended');
2209
+ }
2210
+ // The commit writes its delete tombstones; every other prefetched
1934
2211
  // tombstone still holds for the request's later commits.
1935
2212
  for (const change of commit.changes) {
1936
2213
  if (change.op === 'delete') {
@@ -1939,151 +2216,20 @@ class PostgresTransaction implements StorageTransaction {
1939
2216
  );
1940
2217
  }
1941
2218
  }
1942
- // The append advances the partition cursor; a registered query that runs
1943
- // after it must read the new value.
1944
- this.#cursorSeq = undefined;
1945
- const q = this.#client;
1946
- const p = this.#partition;
1947
- // Allocate the next dense commitSeq under a per-partition row lock: the
1948
- // UPDATE … RETURNING serializes concurrent pushes to this partition and
1949
- // never leaves a gap on rollback (see the file header).
1950
- // §2.2: the allocation also raises the delete floor to the commit's.
1951
- const allocate = `INSERT INTO sync_partitions(partition, max_commit_seq, delete_floor) VALUES ($1, 1, $7)
1952
- ON CONFLICT (partition) DO UPDATE
1953
- SET max_commit_seq = sync_partitions.max_commit_seq + 1,
1954
- delete_floor = GREATEST(sync_partitions.delete_floor, EXCLUDED.delete_floor)
1955
- RETURNING max_commit_seq`;
1956
- const commitRow = `INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms, writer_version)
1957
- SELECT $1, allocated.max_commit_seq, $2, $3, $4, $5, $6 FROM allocated
1958
- RETURNING commit_seq`;
1959
- const head = [
1960
- p,
1961
- commit.clientId,
1962
- commit.clientCommitId,
1963
- commit.actorId,
1964
- commit.createdAtMs,
1965
- this.#writerVersion,
1966
- commitDeleteFloor(commit.changes),
1967
- ];
1968
- if (commit.changes.length === 0) {
1969
- const { rows } = await q.query<{ commit_seq: unknown }>(
1970
- `WITH allocated AS (${allocate}) ${commitRow}`,
1971
- head,
1972
- );
1973
- return asNumber(rows[0]?.commit_seq);
1974
- }
1975
- // §2.2: a delete whose commit follows it with an upsert is a scope move.
1976
- // The row is live, so the tombstone insert below skips it.
1977
- const moved = JSON.stringify(
1978
- [...rowsLiveAfterDelete(commit.changes)].map((key) => {
1979
- const at = key.indexOf('\u0000');
1980
- return { tbl: key.slice(0, at), row_id: key.slice(at + 1) };
1981
- }),
1982
- );
1983
- // One writable-CTE chain per chunk of changes: the changes, their
1984
- // inverted scope entries, and the §5 delete tombstones. A statement binds
1985
- // at most a 16-bit count of parameters, so a chunk carries
1986
- // PG_CHANGES_PER_STATEMENT changes. The first chunk folds the allocation,
1987
- // the commit row, and the change-log write into one statement; later
1988
- // chunks run under the same commitSeq with their global idx. Serialized
1989
- // scopes bind as text before parsing JSONB; drivers that encode JSONB
1990
- // parameters would otherwise store the string as a scalar. A commit can
1991
- // delete one row twice, so a chunk's tombstone insert keeps the latest
1992
- // delete per key and a later chunk overwrites it with its own. `version`
1993
- // is the change's row version (§2.2), which a delete's tombstone keeps.
1994
- const valuesOf = (
1995
- params: unknown[],
1996
- slice: readonly NewChange[],
1997
- firstIdx: number,
1998
- ): string[] =>
1999
- slice.map((change, offset) => {
2000
- const base = params.length;
2001
- params.push(
2002
- firstIdx + offset,
2003
- change.table,
2004
- change.rowId,
2005
- change.op === 'upsert' ? 1 : 2,
2006
- change.rowVersion,
2007
- JSON.stringify(change.scopes),
2008
- change.payload ?? null,
2009
- );
2010
- return `($${base + 1}::int,$${base + 2}::text,$${base + 3}::text,$${base + 4}::smallint,$${base + 5}::bigint,$${base + 6}::text::jsonb,$${base + 7}::bytea)`;
2011
- });
2012
- const scopesCte = `scoped AS (
2013
- INSERT INTO sync_change_scopes(partition, tbl, var, value, commit_seq)
2014
- SELECT inserted.partition, inserted.tbl, scope.key, scope.value, inserted.commit_seq
2015
- FROM inserted CROSS JOIN LATERAL jsonb_each_text(inserted.scopes) AS scope
2016
- ON CONFLICT DO NOTHING
2017
- )`;
2018
- const tombstones = (
2019
- movedParam: string,
2020
- ) => `INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq, server_version, scopes)
2021
- SELECT DISTINCT ON (inserted.tbl, inserted.row_id) inserted.partition,
2022
- inserted.tbl, inserted.row_id, inserted.commit_seq,
2023
- v.version, inserted.scopes
2024
- FROM inserted JOIN v ON v.idx = inserted.idx
2025
- WHERE inserted.op = 2
2026
- AND NOT EXISTS (
2027
- SELECT 1
2028
- FROM jsonb_to_recordset(${movedParam}::text::jsonb) AS moved(tbl text, row_id text)
2029
- WHERE moved.tbl = inserted.tbl AND moved.row_id = inserted.row_id)
2030
- ORDER BY inserted.tbl, inserted.row_id, inserted.idx DESC
2031
- ON CONFLICT (partition, tbl, row_id)
2032
- DO UPDATE SET commit_seq = excluded.commit_seq,
2033
- server_version = excluded.server_version,
2034
- scopes = excluded.scopes`;
2035
- const firstSlice = commit.changes.slice(0, PG_CHANGES_PER_STATEMENT);
2036
- const params = [...head];
2037
- const values = valuesOf(params, firstSlice, 0);
2038
- params.push(moved);
2039
- const movedParam = `$${params.length}`;
2040
- const { rows } = await q.query<{ max_commit_seq: unknown }>(
2041
- `WITH allocated AS (${allocate}),
2042
- allocated_commit AS (${commitRow}),
2043
- v(idx, tbl, row_id, op, version, scopes, payload) AS (
2044
- VALUES ${values.join(',')}
2045
- ),
2046
- inserted AS (
2047
- INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
2048
- SELECT $1, allocated.max_commit_seq, v.idx, v.tbl, v.row_id, v.op, v.version, v.scopes, v.payload
2049
- FROM allocated CROSS JOIN v
2050
- RETURNING partition, idx, tbl, row_id, op, commit_seq, scopes
2051
- ),
2052
- ${scopesCte},
2053
- tombstoned AS (${tombstones(movedParam)})
2054
- SELECT max_commit_seq FROM allocated`,
2055
- params,
2056
- );
2057
- const commitSeq = asNumber(rows[0]?.max_commit_seq);
2058
- for (
2059
- let first = PG_CHANGES_PER_STATEMENT;
2060
- first < commit.changes.length;
2061
- first += PG_CHANGES_PER_STATEMENT
2062
- ) {
2063
- const chunkParams: unknown[] = [p, commitSeq];
2064
- const chunkValues = valuesOf(
2065
- chunkParams,
2066
- commit.changes.slice(first, first + PG_CHANGES_PER_STATEMENT),
2067
- first,
2068
- );
2069
- chunkParams.push(moved);
2070
- const chunkMovedParam = `$${chunkParams.length}`;
2071
- await q.query(
2072
- `WITH v(idx, tbl, row_id, op, version, scopes, payload) AS (
2073
- VALUES ${chunkValues.join(',')}
2074
- ),
2075
- inserted AS (
2076
- INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
2077
- SELECT $1, $2, v.idx, v.tbl, v.row_id, v.op, v.version, v.scopes, v.payload
2078
- FROM v
2079
- RETURNING partition, idx, tbl, row_id, op, commit_seq, scopes
2080
- ),
2081
- ${scopesCte},
2082
- tombstoned AS (${tombstones(chunkMovedParam)})
2083
- SELECT 1`,
2084
- chunkParams,
2085
- );
2219
+ const commitSeq = this.#nextCommitSeq;
2220
+ if (commitSeq === undefined) {
2221
+ this.#stagedAppend = { commit };
2222
+ const allocated = await this.#flush('write');
2223
+ if (allocated === undefined) {
2224
+ throw new Error('the commit append returned no cursor');
2225
+ }
2226
+ this.#nextCommitSeq = allocated + 1;
2227
+ return allocated;
2086
2228
  }
2229
+ this.#nextCommitSeq = commitSeq + 1;
2230
+ this.#stagedAppend = { commit, commitSeq };
2231
+ if (this.#pushApply) this.#openCommitSeq = commitSeq;
2232
+ else await this.#flush('write');
2087
2233
  return commitSeq;
2088
2234
  }
2089
2235
 
@@ -2094,16 +2240,40 @@ class PostgresTransaction implements StorageTransaction {
2094
2240
  sequence?: OutboxSequence,
2095
2241
  ): Promise<void> {
2096
2242
  this.#assertOpen();
2243
+ // On the rerun after a closed commit's write failed, the commit writes
2244
+ // behind its savepoint before it closes, so a relational constraint
2245
+ // rejects it alone (`StorageConstraintError`).
2246
+ if (this.#closeEagerly) await this.#flush('read');
2097
2247
  this.#recordPushResult(clientId, clientCommitId, result, sequence);
2098
- // The commit is closed: the next commit's first write takes its own
2099
- // savepoint.
2248
+ // The commit is closed: its writes join the closed commits', and the next
2249
+ // commit's first write takes its own savepoint.
2250
+ for (const [table, rows] of this.#staged) {
2251
+ const closed = this.#closedRows.get(table) ?? new Map();
2252
+ for (const [rowId, entry] of rows) {
2253
+ closed.delete(rowId);
2254
+ closed.set(rowId, entry);
2255
+ }
2256
+ this.#closedRows.set(table, closed);
2257
+ }
2258
+ for (const [table, refs] of this.#stagedBlobRefs) {
2259
+ const closed = this.#closedBlobRefs.get(table) ?? new Map();
2260
+ for (const [rowId, blobIds] of refs) closed.set(rowId, blobIds);
2261
+ this.#closedBlobRefs.set(table, closed);
2262
+ }
2263
+ if (this.#stagedAppend !== undefined) {
2264
+ this.#closedAppends.push(this.#stagedAppend);
2265
+ }
2266
+ this.#staged.clear();
2267
+ this.#stagedBlobRefs.clear();
2268
+ this.#stagedAppend = undefined;
2269
+ this.#openCommitSeq = undefined;
2100
2270
  this.#pushApplySavepoint = false;
2101
2271
  }
2102
2272
 
2103
2273
  async enqueueReactions(reactions: readonly NewReaction[]): Promise<void> {
2104
2274
  this.#assertOpen();
2105
2275
  for (const reaction of reactions) {
2106
- await this.#client.query(
2276
+ await this.#statement(
2107
2277
  `INSERT INTO sync_reactions(
2108
2278
  partition, idempotency_key, type, version, payload,
2109
2279
  source_client_id, source_client_commit_id, source_commit_seq,
@@ -2121,26 +2291,15 @@ class PostgresTransaction implements StorageTransaction {
2121
2291
  reaction.createdAtMs,
2122
2292
  reaction.maxAttempts,
2123
2293
  ],
2294
+ 'write',
2124
2295
  );
2125
2296
  }
2126
2297
  }
2127
2298
 
2128
2299
  async commit(): Promise<void> {
2129
2300
  this.#assertOpen();
2130
- if (this.#pushResults.length > 0) {
2131
- // Every result of the request in one statement; `#client` flushes the
2132
- // staged writes first.
2133
- await this.#client.query(
2134
- `INSERT INTO sync_push_results(partition, client_id, client_commit_id, result, outbox_id, client_seq)
2135
- SELECT $1, r.client_id, r.client_commit_id, r.result, r.outbox_id, r.client_seq
2136
- FROM jsonb_to_recordset($2::text::jsonb)
2137
- AS r(client_id text, client_commit_id text, result jsonb, outbox_id text, client_seq bigint)
2138
- ON CONFLICT (partition, client_id, client_commit_id) DO NOTHING`,
2139
- [this.#partition, JSON.stringify(this.#pushResults)],
2140
- );
2141
- this.#pushResults.length = 0;
2142
- }
2143
- await this.#writeStaged();
2301
+ // Every pending write in one statement where the parameters fit.
2302
+ await this.#flush('final');
2144
2303
  this.#open = false;
2145
2304
  // Signal the transaction wrapper to COMMIT; wait for it to actually land.
2146
2305
  this.#resolve();
@@ -2149,7 +2308,6 @@ class PostgresTransaction implements StorageTransaction {
2149
2308
 
2150
2309
  async rollback(): Promise<void> {
2151
2310
  if (!this.#open) return;
2152
- this.#staged.clear();
2153
2311
  this.#open = false;
2154
2312
  this.#reject(new RollbackSignal());
2155
2313
  // Swallow — a rollback is a normal outcome, not an error to the caller.
@@ -2171,11 +2329,6 @@ class RollbackSignal extends Error {
2171
2329
  }
2172
2330
  }
2173
2331
 
2174
- /** Changes per sync_changes statement: 2 + 7 × 4,096 = 28,674 bound
2175
- * parameters. The Bind message counts parameters in 16 bits; PostgreSQL reads
2176
- * up to 65,535, and PGlite 0.5 corrupts its state above 32,767. */
2177
- const PG_CHANGES_PER_STATEMENT = 4096;
2178
-
2179
2332
  /** One page read waiting in `PostgresServerStorage#batchedPage`. */
2180
2333
  interface PendingPage {
2181
2334
  readonly spec: Record<string, unknown>;