@syncular/server 0.80.0 → 0.82.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -45,7 +45,7 @@ import { syncError } from './errors.js';
45
45
  import { asBytes, asNumber, } from './pg-executor.js';
46
46
  import { assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, changedScopePatterns, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, rekeyCleanupSql, rekeyedScopes, rekeyRowScopesSql, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanProbeSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, rowsReferencingBlobSql, selectRowSql, selectRowsSql, prefetchRowsBatchSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
47
47
  import { matchesEffective, sha256Hex } from './scopes.js';
48
- import { isPostgresConstraintError, StorageConstraintError, } from './storage-errors.js';
48
+ import { ClosedCommitConstraintError, isPostgresConstraintError, StorageConstraintError, } from './storage-errors.js';
49
49
  import { chooseScanVariable, readLiveWindowPages, resolveIndexRowScan, } from './storage-query.js';
50
50
  /**
51
51
  * Schema DDL. Applied by `PostgresServerStorage.migrate()` (idempotent).
@@ -224,15 +224,18 @@ CREATE TABLE IF NOT EXISTS sync_writer_fence(
224
224
  required_writer_version INTEGER NOT NULL
225
225
  );
226
226
  `;
227
+ /** Lock the partition row; returns its `max_commit_seq`, which the lock freezes. */
227
228
  async function lockPartitionOn(client, partition) {
228
- const locked = await client.query('SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE', [partition]);
229
- if (locked.rows.length > 0)
230
- return;
229
+ const lock = () => client.query('SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE', [partition]);
230
+ const locked = await lock();
231
+ if (locked.rows[0] !== undefined) {
232
+ return asNumber(locked.rows[0].max_commit_seq);
233
+ }
231
234
  // The first writer initializes the partition. Concurrent initializers may
232
235
  // wait on this insert, so acquire the row lock again before applying writes.
233
236
  await client.query(`INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 0)
234
237
  ON CONFLICT (partition) DO NOTHING`, [partition]);
235
- await client.query('SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE', [partition]);
238
+ return asNumber((await lock()).rows[0]?.max_commit_seq);
236
239
  }
237
240
  /**
238
241
  * Insert the `declared` row and raise the fence, on the caller's open
@@ -331,25 +334,72 @@ async function readServeGateOn(client, partition, runningSchemaVersion) {
331
334
  return toServeGate(rows[0]);
332
335
  }
333
336
  /**
334
- * The push admission read: the RFC 0007 serve gate, the §2.3 idempotency
335
- * records of every commit of the request, and (when the caller already knows
336
- * the commits' row targets) the read-ahead rows, in one statement with one
337
- * snapshot. The caller takes the partition lock first: under READ COMMITTED a
338
- * statement's snapshot is fixed at its start, so the read must be the
339
- * statement *after* the lock to observe a duplicate that committed while this
340
- * transaction waited for it.
337
+ * The push admission statement: the §9 settlement and §6.3.2 steps 1 and 2
338
+ * when the request carries an outbox, the RFC 0007 serve gate, the §2.3
339
+ * idempotency records of every commit of the request, and (when the caller
340
+ * already knows the commits' row targets) the read-ahead rows, in one
341
+ * statement with one snapshot. The caller takes the partition lock first:
342
+ * under READ COMMITTED a statement's snapshot is fixed at its start, so the
343
+ * read must be the statement *after* the lock to observe a duplicate that
344
+ * committed while this transaction waited for it.
345
+ *
346
+ * The settlement's writable CTEs see the same snapshot as the read, so they
347
+ * touch disjoint rows and the read leaves out the results they delete. §9
348
+ * (b): with no pre-upgrade result on the request's last commit, every
349
+ * remaining one outside the request left the outbox. (a) assigns the
350
+ * request's sequences to its pre-upgrade results at or above the new
351
+ * watermark; step 2 deletes the outbox's results below it, the pre-upgrade
352
+ * results of the request below it included.
341
353
  */
342
- async function readAdmissionOn(client, partition, clientId, clientCommitIds, runningSchemaVersion, prefetch) {
354
+ async function readAdmissionOn(client, partition, clientId, clientCommitIds, runningSchemaVersion, prefetch, outbox) {
343
355
  const params = [
344
356
  partition,
345
357
  runningSchemaVersion,
346
358
  clientId,
347
359
  JSON.stringify(clientCommitIds),
348
360
  ];
349
- let statement = `SELECT ${SERVE_GATE_PROJECTION},
361
+ let settlement = '';
362
+ if (outbox !== undefined) {
363
+ const first = outbox.commits[0];
364
+ const last = outbox.commits.at(-1);
365
+ if (first === undefined || last === undefined) {
366
+ throw new Error('outbox settlement needs the request commits');
367
+ }
368
+ params.push(outbox.outboxId, JSON.stringify(outbox.commits.map((commit) => ({
369
+ id: commit.clientCommitId,
370
+ seq: commit.clientSeq,
371
+ }))), last.clientCommitId, first.clientSeq);
372
+ settlement = `WITH c AS (SELECT * FROM jsonb_to_recordset($6::text::jsonb) AS c(id text, seq bigint)),
373
+ w AS (
374
+ INSERT INTO sync_push_watermarks(partition, client_id, outbox_id, through_seq)
375
+ VALUES ($1,$3,$5,$8)
376
+ ON CONFLICT (partition, client_id, outbox_id) DO UPDATE
377
+ SET through_seq=GREATEST(sync_push_watermarks.through_seq, EXCLUDED.through_seq)
378
+ RETURNING through_seq),
379
+ b AS (DELETE FROM sync_push_results
380
+ WHERE partition=$1 AND client_id=$3 AND outbox_id IS NULL
381
+ AND client_commit_id NOT IN (SELECT id FROM c)
382
+ AND NOT EXISTS (SELECT 1 FROM sync_push_results
383
+ WHERE partition=$1 AND client_id=$3 AND client_commit_id=$7
384
+ AND outbox_id IS NULL)),
385
+ a AS (UPDATE sync_push_results r SET outbox_id=$5, client_seq=c.seq FROM c, w
386
+ WHERE r.partition=$1 AND r.client_id=$3 AND r.outbox_id IS NULL
387
+ AND r.client_commit_id=c.id AND c.seq>=w.through_seq),
388
+ expired AS (DELETE FROM sync_push_results r USING w
389
+ WHERE r.partition=$1 AND r.client_id=$3
390
+ AND ((r.outbox_id=$5 AND r.client_seq<w.through_seq)
391
+ OR (r.outbox_id IS NULL AND EXISTS (SELECT 1 FROM c
392
+ WHERE c.id=r.client_commit_id AND c.seq<w.through_seq)))
393
+ RETURNING r.client_commit_id)
394
+ `;
395
+ }
396
+ let statement = `${settlement}SELECT ${SERVE_GATE_PROJECTION},
350
397
  (SELECT jsonb_object_agg(client_commit_id, result) FROM sync_push_results
351
- WHERE partition=$1 AND client_id=$3 AND client_commit_id IN (SELECT jsonb_array_elements_text($4::text::jsonb))) AS results,
352
- (SELECT delete_floor FROM sync_partitions WHERE partition=$1) AS delete_floor`;
398
+ WHERE partition=$1 AND client_id=$3 AND client_commit_id IN (SELECT jsonb_array_elements_text($4::text::jsonb))${outbox === undefined
399
+ ? ''
400
+ : ' AND client_commit_id NOT IN (SELECT client_commit_id FROM expired)'}) AS results,
401
+ (SELECT delete_floor FROM sync_partitions WHERE partition=$1) AS delete_floor,
402
+ ${outbox === undefined ? 'NULL::bigint' : '(SELECT through_seq FROM w)'} AS watermark`;
353
403
  if (prefetch !== undefined && prefetch.specs.length > 0) {
354
404
  params.push(JSON.stringify(prefetch.specs));
355
405
  // The gate row pads the read-ahead columns with typed NULLs so both
@@ -362,6 +412,7 @@ async function readAdmissionOn(client, partition, clientId, clientCommitIds, run
362
412
  NULL::jsonb AS tombstone_scopes
363
413
  UNION ALL
364
414
  SELECT NULL::bigint, NULL::text, NULL::json, NULL::jsonb, NULL::bigint,
415
+ NULL::bigint,
365
416
  p.tbl, p.lookup_id, p.row_id, p.server_version, p.scopes,
366
417
  p.payload, p.column_versions, p.tombstone_seq,
367
418
  p.tombstone_version, p.tombstone_scopes
@@ -382,6 +433,7 @@ async function readAdmissionOn(client, partition, clientId, clientCommitIds, run
382
433
  : []),
383
434
  // asNumber maps the NULL of a partition without a commit to 0.
384
435
  deleteFloor: asNumber(gateRow?.delete_floor),
436
+ watermark: outbox === undefined ? undefined : asNumber(gateRow?.watermark),
385
437
  prefetched: rows.filter((row) => row.tbl !== null && row.tbl !== undefined),
386
438
  };
387
439
  }
@@ -696,116 +748,196 @@ async function scanRowsByIndexOn(q, compiled, partition, query) {
696
748
  return rows.map(toStoredRow);
697
749
  }
698
750
  /**
699
- * Write rows of one table in one statement per chunk: the upsert, the scope
700
- * index synced to each row's scope map (entries outside it deleted, missing
701
- * ones inserted), and the §5.9.4 blob-reference index synced to the ids the
702
- * row now references (entries outside the new set deleted, missing ones
703
- * inserted). Row ids are distinct, so no index entry is touched twice and the
704
- * writable CTEs cannot conflict. The scopes and blob ids bind as text before
705
- * the JSONB cast (see `appendCommit`); the refs set is part of the same
706
- * statement, so a blob-referencing commit costs no extra round trip per row.
707
- * Every chunk is handed to `q` before the first one settles, so a statement
708
- * issued after this call queues behind all of them on the transaction FIFO.
751
+ * Bound parameters per write statement. The Bind message counts parameters
752
+ * in 16 bits; PostgreSQL reads up to 65,535, and PGlite 0.5 corrupts its
753
+ * state above 32,767.
709
754
  */
710
- function writeRowsOn(q, compiled, partition, rows, blobRefs = []) {
711
- const width = tableColumnNames(compiled).length;
712
- // PGlite corrupts its state above 32,767 bound parameters (see
713
- // PG_CHANGES_PER_STATEMENT).
714
- const chunk = Math.max(1, Math.floor(30_000 / width));
715
- const pendingRefs = new Map(blobRefs.map((entry) => [entry.rowId, entry.blobIds]));
716
- const writes = [];
717
- for (let first = 0; first < rows.length; first += chunk) {
718
- const slice = rows.slice(first, first + chunk);
719
- const params = slice.flatMap((entry) => entry.values);
720
- const n = params.length;
721
- const refs = [];
722
- for (const entry of slice) {
723
- const blobIds = pendingRefs.get(entry.row.rowId);
724
- if (blobIds === undefined)
725
- continue;
726
- pendingRefs.delete(entry.row.rowId);
727
- refs.push({ row_id: entry.row.rowId, blob_ids: blobIds });
755
+ const PG_PARAMS_PER_STATEMENT = 30_000;
756
+ function bindParam(statement, value) {
757
+ statement.params.push(value);
758
+ return `$${statement.params.length}`;
759
+ }
760
+ /**
761
+ * The CTEs that write rows of one table: the upsert, the scope index synced
762
+ * to each row's scope map (entries outside it deleted, missing ones
763
+ * inserted), and the §5.9.4 blob-reference index synced to the ids the row
764
+ * now references. Row ids are distinct, so no index entry is touched twice.
765
+ * The scopes and blob ids bind as text before the JSONB cast; drivers that
766
+ * encode JSONB parameters would otherwise store the string as a scalar.
767
+ */
768
+ function addRowWrites(statement, compiled, partition, rows, blobRefs) {
769
+ const k = statement.ctes.length;
770
+ const offset = statement.params.length;
771
+ statement.params.push(...rows.flatMap((entry) => entry.values));
772
+ const scopes = bindParam(statement, JSON.stringify(rows.map(({ row }) => ({ row_id: row.rowId, scopes: row.scopes }))));
773
+ const p = bindParam(statement, partition);
774
+ const tbl = bindParam(statement, compiled.name);
775
+ statement.ctes.push(`u${k} AS (${upsertSql(compiled, 'postgres', rows.length, offset)})`, `b${k} AS (SELECT row_id, scopes FROM jsonb_to_recordset(${scopes}::text::jsonb) AS r(row_id text, scopes jsonb))`, `rs${k} AS (DELETE FROM sync_row_scopes existing USING b${k}
776
+ WHERE existing.partition=${p} AND existing.tbl=${tbl}
777
+ AND existing.row_id=b${k}.row_id
778
+ AND b${k}.scopes->>existing.var IS DISTINCT FROM existing.value)`, `is${k} AS (INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
779
+ SELECT ${p}, ${tbl}, entry.key, entry.value, b${k}.row_id
780
+ FROM b${k} CROSS JOIN LATERAL jsonb_each_text(b${k}.scopes) AS entry
781
+ ON CONFLICT DO NOTHING)`);
782
+ for (const entry of rows) {
783
+ if (entry.opIndex !== undefined &&
784
+ (statement.opIndex === undefined || entry.opIndex < statement.opIndex)) {
785
+ statement.opIndex = entry.opIndex;
728
786
  }
729
- writes.push(q.query(`WITH upserted AS (${upsertSql(compiled, 'postgres', slice.length)}),
730
- batch AS (
731
- SELECT row_id, scopes
732
- FROM jsonb_to_recordset($${n + 1}::text::jsonb) AS r(row_id text, scopes jsonb)
733
- ),
734
- scope AS (
735
- SELECT batch.row_id, entry.key, entry.value
736
- FROM batch CROSS JOIN LATERAL jsonb_each_text(batch.scopes) AS entry
737
- ),
738
- removed AS (
739
- DELETE FROM sync_row_scopes existing
740
- USING batch
741
- WHERE existing.partition=$${n + 2} AND existing.tbl=$${n + 3}
742
- AND existing.row_id=batch.row_id
743
- AND batch.scopes->>existing.var IS DISTINCT FROM existing.value
744
- ),
745
- changed_refs AS (
746
- SELECT DISTINCT row_id
747
- FROM jsonb_to_recordset($${n + 4}::text::jsonb) AS r(row_id text, blob_ids jsonb)
748
- ),
749
- removed_refs AS (
750
- DELETE FROM sync_blob_refs existing
751
- USING changed_refs
752
- WHERE existing.partition=$${n + 2} AND existing.tbl=$${n + 3}
753
- AND existing.row_id=changed_refs.row_id
754
- AND NOT EXISTS (
755
- SELECT 1
756
- FROM jsonb_to_recordset($${n + 4}::text::jsonb) AS r(row_id text, blob_ids jsonb)
757
- CROSS JOIN LATERAL jsonb_array_elements_text(r.blob_ids) AS entry(value)
758
- WHERE r.row_id=existing.row_id AND entry.value=existing.blob_id)
759
- ),
760
- inserted_refs AS (
761
- INSERT INTO sync_blob_refs(partition, tbl, row_id, blob_id)
762
- SELECT $${n + 2}, $${n + 3}, r.row_id, entry.value
763
- FROM jsonb_to_recordset($${n + 4}::text::jsonb) AS r(row_id text, blob_ids jsonb)
787
+ }
788
+ if (blobRefs.length > 0)
789
+ addBlobRefWrites(statement, p, tbl, blobRefs);
790
+ }
791
+ /** Replace the §5.9.4 reference sets of rows of one table. */
792
+ function addBlobRefWrites(statement, p, tbl, blobRefs) {
793
+ const k = statement.ctes.length;
794
+ const refs = bindParam(statement, JSON.stringify(blobRefs));
795
+ statement.ctes.push(`rr${k} AS (DELETE FROM sync_blob_refs existing
796
+ WHERE existing.partition=${p} AND existing.tbl=${tbl}
797
+ AND existing.row_id IN (SELECT r.row_id FROM jsonb_to_recordset(${refs}::text::jsonb) AS r(row_id text, blob_ids jsonb))
798
+ AND NOT EXISTS (
799
+ SELECT 1
800
+ FROM jsonb_to_recordset(${refs}::text::jsonb) AS r(row_id text, blob_ids jsonb)
764
801
  CROSS JOIN LATERAL jsonb_array_elements_text(r.blob_ids) AS entry(value)
765
- ON CONFLICT DO NOTHING
766
- )
767
- INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
768
- SELECT $${n + 2}, $${n + 3}, key, value, row_id FROM scope
769
- ON CONFLICT DO NOTHING`, [
770
- ...params,
771
- JSON.stringify(slice.map(({ row }) => ({ row_id: row.rowId, scopes: row.scopes }))),
772
- partition,
773
- compiled.name,
774
- JSON.stringify(refs),
775
- ]));
776
- }
777
- if (pendingRefs.size > 0) {
778
- // Refs staged for a row this call did not carry (the row was written by an
779
- // earlier flush): replace the reference set in one statement of its own.
780
- writes.push(q.query(`WITH changed_refs AS (
781
- SELECT DISTINCT row_id
782
- FROM jsonb_to_recordset($3::text::jsonb) AS r(row_id text, blob_ids jsonb)
783
- ),
784
- removed_refs AS (
785
- DELETE FROM sync_blob_refs existing
786
- USING changed_refs
787
- WHERE existing.partition=$1 AND existing.tbl=$2
788
- AND existing.row_id=changed_refs.row_id
789
- AND NOT EXISTS (
790
- SELECT 1
791
- FROM jsonb_to_recordset($3::text::jsonb) AS r(row_id text, blob_ids jsonb)
792
- CROSS JOIN LATERAL jsonb_array_elements_text(r.blob_ids) AS entry(value)
793
- WHERE r.row_id=existing.row_id AND entry.value=existing.blob_id)
794
- )
795
- INSERT INTO sync_blob_refs(partition, tbl, row_id, blob_id)
796
- SELECT $1, $2, r.row_id, entry.value
797
- FROM jsonb_to_recordset($3::text::jsonb) AS r(row_id text, blob_ids jsonb)
798
- CROSS JOIN LATERAL jsonb_array_elements_text(r.blob_ids) AS entry(value)
799
- ON CONFLICT DO NOTHING`, [
800
- partition,
801
- compiled.name,
802
- JSON.stringify([...pendingRefs].map(([row_id, blob_ids]) => ({
803
- row_id,
804
- blob_ids,
805
- }))),
806
- ]));
807
- }
808
- return Promise.all(writes);
802
+ WHERE r.row_id=existing.row_id AND entry.value=existing.blob_id))`, `ir${k} AS (INSERT INTO sync_blob_refs(partition, tbl, row_id, blob_id)
803
+ SELECT ${p}, ${tbl}, r.row_id, entry.value
804
+ FROM jsonb_to_recordset(${refs}::text::jsonb) AS r(row_id text, blob_ids jsonb)
805
+ CROSS JOIN LATERAL jsonb_array_elements_text(r.blob_ids) AS entry(value)
806
+ ON CONFLICT DO NOTHING)`);
807
+ }
808
+ /**
809
+ * The CTEs that append commits with consecutive `commitSeq`s: the partition
810
+ * cursor and delete floor (§2.2), the commit rows, their changes with the
811
+ * inverted scope entries, and the §5 delete tombstones. The cursor advances
812
+ * by the number of commits, and every row takes its commit's sequence from
813
+ * the advanced cursor; the statement returns the cursor, which a caller that
814
+ * computed the sequences under the partition lock checks. Every list binds
815
+ * as one JSON text parameter, so the parameter count does not grow with the
816
+ * change count. A tombstone keeps the change's row version (§2.2); a row
817
+ * live again at the end of its commit (a §2.2 scope move) gets none, and a
818
+ * later delete of the same row overwrites an earlier one.
819
+ */
820
+ function addAppends(statement, partition, writerVersion, appends) {
821
+ if (appends.length === 0)
822
+ return;
823
+ const p = bindParam(statement, partition);
824
+ const changes = [];
825
+ const tombstones = new Map();
826
+ let deleteFloor = 0;
827
+ for (const [index, { commit }] of appends.entries()) {
828
+ const k = index + 1;
829
+ deleteFloor = Math.max(deleteFloor, commitDeleteFloor(commit.changes));
830
+ const live = rowsLiveAfterDelete(commit.changes);
831
+ for (const [idx, change] of commit.changes.entries()) {
832
+ changes.push({
833
+ k,
834
+ idx,
835
+ tbl: change.table,
836
+ row_id: change.rowId,
837
+ op: change.op === 'upsert' ? 1 : 2,
838
+ version: change.rowVersion,
839
+ scopes: change.scopes,
840
+ payload: change.payload === undefined ? null : toBase64(change.payload),
841
+ });
842
+ const key = `${change.table}\u0000${change.rowId}`;
843
+ if (change.op === 'delete' && !live.has(key)) {
844
+ tombstones.set(key, {
845
+ k,
846
+ tbl: change.table,
847
+ row_id: change.rowId,
848
+ version: change.rowVersion,
849
+ scopes: change.scopes,
850
+ });
851
+ }
852
+ }
853
+ }
854
+ statement.appends = true;
855
+ const expected = appends.at(-1)?.commitSeq;
856
+ if (expected !== undefined)
857
+ statement.allocates = expected;
858
+ const n = bindParam(statement, appends.length);
859
+ statement.ctes.push(`alloc AS (INSERT INTO sync_partitions(partition, max_commit_seq, delete_floor)
860
+ VALUES (${p}, ${n}, ${bindParam(statement, deleteFloor)})
861
+ ON CONFLICT (partition) DO UPDATE
862
+ SET max_commit_seq=sync_partitions.max_commit_seq + EXCLUDED.max_commit_seq,
863
+ delete_floor=GREATEST(sync_partitions.delete_floor, EXCLUDED.delete_floor)
864
+ RETURNING max_commit_seq, max_commit_seq - ${n}::bigint AS base)`, `commits AS (INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms, writer_version)
865
+ SELECT ${p}, alloc.base + c.k, c.client_id, c.client_commit_id, c.actor_id, c.created_at_ms, ${bindParam(statement, writerVersion)}
866
+ FROM alloc CROSS JOIN jsonb_to_recordset(${bindParam(statement, JSON.stringify(appends.map(({ commit }, index) => ({
867
+ k: index + 1,
868
+ client_id: commit.clientId,
869
+ client_commit_id: commit.clientCommitId,
870
+ actor_id: commit.actorId,
871
+ created_at_ms: commit.createdAtMs,
872
+ }))))}::text::jsonb)
873
+ AS c(k int, client_id text, client_commit_id text, actor_id text, created_at_ms bigint))`);
874
+ if (changes.length > 0) {
875
+ statement.ctes.push(`changed AS (INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
876
+ SELECT ${p}, alloc.base + c.k, c.idx, c.tbl, c.row_id, c.op, c.version, c.scopes, decode(c.payload, 'base64')
877
+ FROM alloc CROSS JOIN jsonb_to_recordset(${bindParam(statement, JSON.stringify(changes))}::text::jsonb)
878
+ AS c(k int, idx int, tbl text, row_id text, op smallint, version bigint, scopes jsonb, payload text)
879
+ RETURNING commit_seq, tbl, scopes)`, `change_scopes AS (INSERT INTO sync_change_scopes(partition, tbl, var, value, commit_seq)
880
+ SELECT ${p}, changed.tbl, scope.key, scope.value, changed.commit_seq
881
+ FROM changed CROSS JOIN LATERAL jsonb_each_text(changed.scopes) AS scope
882
+ ON CONFLICT DO NOTHING)`);
883
+ }
884
+ if (tombstones.size > 0) {
885
+ statement.ctes.push(`tombstoned AS (INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq, server_version, scopes)
886
+ SELECT ${p}, t.tbl, t.row_id, alloc.base + t.k, t.version, t.scopes
887
+ FROM alloc CROSS JOIN jsonb_to_recordset(${bindParam(statement, JSON.stringify([...tombstones.values()]))}::text::jsonb)
888
+ AS t(k int, tbl text, row_id text, version bigint, scopes jsonb)
889
+ ON CONFLICT (partition, tbl, row_id)
890
+ DO UPDATE SET commit_seq=excluded.commit_seq,
891
+ server_version=excluded.server_version,
892
+ scopes=excluded.scopes)`);
893
+ }
894
+ }
895
+ /**
896
+ * The statements that write staged rows, blob references, commit appends
897
+ * and push results. Everything shares the first statement while the bound
898
+ * parameters fit; a large row set continues in further statements of the
899
+ * same shape.
900
+ */
901
+ function writeStatements(partition, writerVersion, resolveTable, rows, blobRefs, appends, results) {
902
+ let current = { ctes: [], params: [] };
903
+ const statements = [current];
904
+ addAppends(current, partition, writerVersion, appends);
905
+ if (results.length > 0) {
906
+ current.ctes.push(`results AS (INSERT INTO sync_push_results(partition, client_id, client_commit_id, result, outbox_id, client_seq)
907
+ SELECT ${bindParam(current, partition)}, r.client_id, r.client_commit_id, r.result, r.outbox_id, r.client_seq
908
+ FROM jsonb_to_recordset(${bindParam(current, JSON.stringify(results))}::text::jsonb)
909
+ AS r(client_id text, client_commit_id text, result jsonb, outbox_id text, client_seq bigint)
910
+ ON CONFLICT (partition, client_id, client_commit_id) DO NOTHING)`);
911
+ }
912
+ for (const table of new Set([...rows.keys(), ...blobRefs.keys()])) {
913
+ const compiled = resolveTable(table);
914
+ const width = tableColumnNames(compiled).length;
915
+ const chunk = Math.max(1, Math.floor((PG_PARAMS_PER_STATEMENT - 4) / width));
916
+ const pending = new Map(blobRefs.get(table));
917
+ const staged = [...(rows.get(table)?.values() ?? [])];
918
+ for (let first = 0; first < staged.length; first += chunk) {
919
+ const slice = staged.slice(first, first + chunk);
920
+ if (current.params.length + slice.length * width + 4 >
921
+ PG_PARAMS_PER_STATEMENT) {
922
+ current = { ctes: [], params: [] };
923
+ statements.push(current);
924
+ }
925
+ const refs = [];
926
+ for (const { row } of slice) {
927
+ const blobIds = pending.get(row.rowId);
928
+ if (blobIds === undefined)
929
+ continue;
930
+ pending.delete(row.rowId);
931
+ refs.push({ row_id: row.rowId, blob_ids: blobIds });
932
+ }
933
+ addRowWrites(current, compiled, partition, slice, refs);
934
+ }
935
+ if (pending.size > 0) {
936
+ // Refs staged for a row written by an earlier statement.
937
+ addBlobRefWrites(current, bindParam(current, partition), bindParam(current, table), [...pending].map(([row_id, blob_ids]) => ({ row_id, blob_ids })));
938
+ }
939
+ }
940
+ return statements.filter((statement) => statement.ctes.length > 0);
809
941
  }
810
942
  async function getPushResultOn(q, partition, clientId, clientCommitId) {
811
943
  const { rows } = await q.query('SELECT result FROM sync_push_results WHERE partition=$1 AND client_id=$2 AND client_commit_id=$3', [partition, clientId, clientCommitId]);
@@ -835,7 +967,7 @@ async function queryAuthoritativeOn(client, partition, prepared) {
835
967
  }
836
968
  class PostgresTransaction {
837
969
  #client;
838
- /** The FIFO without the staged-write prelude (see the constructor). */
970
+ /** The FIFO without the pending-write prelude (see the constructor). */
839
971
  #serial;
840
972
  #partition;
841
973
  /** Storage-owned relation rewrite, bound to this transaction's partition. */
@@ -846,13 +978,41 @@ class PostgresTransaction {
846
978
  /** The storage's `chooseScanVariable` picks. */
847
979
  #scanVariables;
848
980
  #open = true;
981
+ /** Set by `preparePushApply`: commits close with a result and roll back alone. */
982
+ #pushApply = false;
983
+ /** Set by `preparePushApply` on the rerun after a closed commit's write failed. */
984
+ #closeEagerly = false;
849
985
  /** True once the request's current commit has written behind its savepoint. */
850
986
  #pushApplySavepoint = false;
851
987
  /**
852
- * The request's §2.3 results, written as one statement at `commit()`
853
- * (§6.4): each commit closes with its result, and none is read back before
854
- * the transaction ends.
988
+ * The `commitSeq` the next appended commit takes. The partition lock read
989
+ * sets it and freezes the cursor, so an append needs no allocation read.
855
990
  */
991
+ #nextCommitSeq;
992
+ /** The `commitSeq` the open push commit appended, until it closes. */
993
+ #openCommitSeq;
994
+ /**
995
+ * The open commit's upserts staged per table name (row id → row and its
996
+ * push operation index), its §5.9.4 reference sets, and its commit append.
997
+ * They reach the database with the next statement that needs them, behind
998
+ * the commit's savepoint, as one statement: a commit's rows cost one round
999
+ * trip per statement instead of one per row. Tables with a unique index
1000
+ * write immediately: a violation must name its operation.
1001
+ */
1002
+ #staged = new Map();
1003
+ #stagedBlobRefs = new Map();
1004
+ #stagedAppend;
1005
+ /**
1006
+ * The writes of push commits that closed and are not yet written: rows (a
1007
+ * later commit's row replaces an earlier one), reference sets, appends in
1008
+ * `commitSeq` order, and the §2.3 results. They reach the database in one
1009
+ * statement ahead of the next statement of the transaction, outside any
1010
+ * commit's savepoint; a request whose commits never read their own writes
1011
+ * writes them all in one statement before COMMIT (§6.4).
1012
+ */
1013
+ #closedRows = new Map();
1014
+ #closedBlobRefs = new Map();
1015
+ #closedAppends = [];
856
1016
  #pushResults = [];
857
1017
  /**
858
1018
  * `prefetchRows` results keyed `table\u0000rowId`; `null` records a row
@@ -861,29 +1021,6 @@ class PostgresTransaction {
861
1021
  */
862
1022
  #prefetchedRows = new Map();
863
1023
  #prefetchedTombstones = new Map();
864
- /**
865
- * Upserts staged per table (row id → row and its push operation index).
866
- * Every statement of this transaction writes them first (see the
867
- * constructor), so reads and later writes observe them; a commit's rows
868
- * reach the database as one statement per table instead of one per row.
869
- * Tables with a unique index write immediately: a violation must name its
870
- * operation.
871
- */
872
- #staged = new Map();
873
- /**
874
- * §5.9.4 reference sets staged per table (row id → blob ids). They flush
875
- * with the table's row statement, so a blob-referencing row costs no extra
876
- * round trip; a ref set staged for an already-written row flushes on the
877
- * next statement of its table.
878
- */
879
- #stagedBlobRefs = new Map();
880
- /**
881
- * The partition cursor (`max_commit_seq`) read by the first registered
882
- * query in this transaction. The partition lock freezes it, so later
883
- * registered queries in the same transaction reuse the value instead of
884
- * paying a round trip each.
885
- */
886
- #cursorSeq;
887
1024
  /** Resolves/rejects the `transaction(fn)` wrapper (see `begin`). */
888
1025
  #resolve;
889
1026
  #reject;
@@ -896,30 +1033,15 @@ class PostgresTransaction {
896
1033
  // issued through this transaction is chained on one FIFO — the same
897
1034
  // shape as `SqliteServerStorage.queryAuthoritative`.
898
1035
  let tail = Promise.resolve();
899
- const serial = {
1036
+ this.#serial = {
900
1037
  query: (text, params) => {
901
1038
  const run = tail.then(() => client.query(text, params));
902
1039
  tail = run.then(() => undefined, () => undefined);
903
1040
  return run;
904
1041
  },
905
1042
  };
906
- this.#serial = serial;
907
1043
  this.#client = {
908
- query: async (text, params) => {
909
- // Queue the staged writes ahead of this statement synchronously, so
910
- // a concurrently issued statement cannot overtake them.
911
- const staged = this.#writeStaged();
912
- const run = serial.query(text, params);
913
- try {
914
- await staged;
915
- }
916
- catch (error) {
917
- // The aborted transaction fails the queued statement too.
918
- run.catch(() => undefined);
919
- throw error;
920
- }
921
- return run;
922
- },
1044
+ query: (text, params) => this.#statement(text, params, 'read'),
923
1045
  };
924
1046
  this.#partition = partition;
925
1047
  this.#writerVersion = writerVersion;
@@ -932,83 +1054,145 @@ class PostgresTransaction {
932
1054
  if (!this.#open)
933
1055
  throw new Error('transaction already finished');
934
1056
  }
935
- #writeStaged() {
936
- if (this.#staged.size === 0 && this.#stagedBlobRefs.size === 0) {
937
- return Promise.resolve();
1057
+ /**
1058
+ * Queue the pending writes and then one statement. `write` marks a
1059
+ * candidate write of the open commit, which needs the commit's savepoint
1060
+ * even when nothing is staged.
1061
+ */
1062
+ async #statement(text, params, mode) {
1063
+ // Queue the pending writes ahead of this statement synchronously, so a
1064
+ // concurrently issued statement cannot overtake them.
1065
+ const pending = this.#flush(mode);
1066
+ const run = this.#serial.query(text, params);
1067
+ try {
1068
+ await pending;
938
1069
  }
939
- const tables = new Set([
940
- ...[...this.#staged.keys()].map((table) => table.name),
941
- ...this.#stagedBlobRefs.keys(),
942
- ]);
943
- const batches = [...tables].map((table) => {
944
- const compiled = this.#resolveTable(table);
945
- const rows = [...(this.#staged.get(compiled)?.values() ?? [])];
946
- const blobRefs = [
947
- ...(this.#stagedBlobRefs.get(table)?.entries() ?? []),
948
- ].map(([rowId, blobIds]) => ({ rowId, blobIds }));
949
- this.#staged.delete(compiled);
950
- this.#stagedBlobRefs.delete(table);
951
- return { table, compiled, rows, blobRefs };
952
- });
953
- const writes = [];
954
- // The candidate savepoint lands before the first candidate write, and
955
- // only when a write actually reaches the database: a commit refused
956
- // before its first write has no candidate rows to roll back, so its
957
- // rejection finalization skips the savepoint round trips.
958
- const savepoint = this.#ensurePushSavepoint();
959
- if (savepoint !== undefined)
960
- writes.push(savepoint);
961
- for (const { compiled, rows, blobRefs } of batches) {
962
- writes.push(writeRowsOn(this.#serial, compiled, this.#partition, rows, blobRefs).catch((error) => {
963
- // Staged tables have no unique index, so a violation is a column
964
- // constraint; it names the first staged operation of the table.
965
- if (isPostgresConstraintError(error)) {
966
- throw new StorageConstraintError(error, rows[0]?.opIndex);
967
- }
968
- throw error;
969
- }));
1070
+ catch (error) {
1071
+ // The aborted transaction fails the queued statement too.
1072
+ run.catch(() => undefined);
1073
+ throw error;
970
1074
  }
971
- return Promise.all(writes);
1075
+ return run;
972
1076
  }
973
1077
  /**
974
- * The current commit's candidate savepoint, taken before its first write.
975
- * Every candidate write must be undoable by `rejectPushCommit`, and a
976
- * refused commit that wrote nothing must not pay to create one. Each commit
977
- * takes its own savepoint under the same name: `ROLLBACK TO` targets the
978
- * newest, and an earlier commit's savepoint stays open until the
979
- * transaction ends, so no `RELEASE` round trip is needed (§6.4).
1078
+ * Queue every pending write, in order: the closed commits' writes, then
1079
+ * the open commit's savepoint (taken before its first write reaches the
1080
+ * database, so a refused commit that wrote nothing pays for none) and its
1081
+ * staged writes. `final` is the flush before COMMIT, which no rejection
1082
+ * follows.
1083
+ *
1084
+ * A relational constraint that fails on a closed commit's writes can no
1085
+ * longer reject that commit alone: the transaction is aborted, and
1086
+ * `ClosedCommitConstraintError` tells the push layer to rerun the request
1087
+ * with each commit written before it closes (§6.4).
1088
+ *
1089
+ * Resolves to the partition cursor the open commit's append returned.
980
1090
  */
981
- #ensurePushSavepoint() {
982
- if (this.#pushApplySavepoint)
983
- return undefined;
984
- this.#pushApplySavepoint = true;
985
- return this.#serial.query('SAVEPOINT syncular_push_candidate');
1091
+ async #flush(mode) {
1092
+ const pending = [];
1093
+ let cursor = Promise.resolve(undefined);
1094
+ if (this.#closedRows.size > 0 ||
1095
+ this.#closedBlobRefs.size > 0 ||
1096
+ this.#closedAppends.length > 0 ||
1097
+ this.#pushResults.length > 0) {
1098
+ const statements = writeStatements(this.#partition, this.#writerVersion, this.#resolveTable, this.#closedRows, this.#closedBlobRefs, this.#closedAppends, this.#pushResults);
1099
+ this.#closedRows.clear();
1100
+ this.#closedBlobRefs.clear();
1101
+ this.#closedAppends.length = 0;
1102
+ this.#pushResults.length = 0;
1103
+ pending.push(this.#runWrites(statements, (error) => isPostgresConstraintError(error)
1104
+ ? new ClosedCommitConstraintError(error)
1105
+ : error));
1106
+ }
1107
+ const staged = this.#staged.size > 0 ||
1108
+ this.#stagedBlobRefs.size > 0 ||
1109
+ this.#stagedAppend !== undefined;
1110
+ if (mode !== 'final' &&
1111
+ this.#pushApply &&
1112
+ !this.#pushApplySavepoint &&
1113
+ (staged || mode === 'write')) {
1114
+ // Each commit takes its own savepoint under the same name: `ROLLBACK
1115
+ // TO` targets the newest, and an earlier commit's savepoint stays open
1116
+ // until the transaction ends, so no `RELEASE` round trip is needed.
1117
+ this.#pushApplySavepoint = true;
1118
+ pending.push(this.#serial.query('SAVEPOINT syncular_push_candidate'));
1119
+ }
1120
+ if (staged) {
1121
+ const statements = writeStatements(this.#partition, this.#writerVersion, this.#resolveTable, this.#staged, this.#stagedBlobRefs, this.#stagedAppend === undefined ? [] : [this.#stagedAppend], []);
1122
+ this.#staged.clear();
1123
+ this.#stagedBlobRefs.clear();
1124
+ this.#stagedAppend = undefined;
1125
+ cursor = this.#runWrites(statements, (error, statement) => isPostgresConstraintError(error)
1126
+ ? new StorageConstraintError(error, statement.opIndex)
1127
+ : error);
1128
+ pending.push(cursor);
1129
+ }
1130
+ await Promise.all(pending);
1131
+ return cursor;
986
1132
  }
1133
+ /**
1134
+ * Queue write statements on the FIFO. Resolves to the cursor a commit
1135
+ * append returned, checked against the sequences computed under the lock.
1136
+ */
1137
+ async #runWrites(statements, mapError) {
1138
+ const cursors = await Promise.all(statements.map((statement) => this.#serial
1139
+ .query(`WITH ${statement.ctes.join(',\n')} SELECT ${statement.appends === true
1140
+ ? '(SELECT max_commit_seq FROM alloc)'
1141
+ : 'NULL::bigint'} AS allocated`, statement.params)
1142
+ .then(({ rows }) => {
1143
+ if (statement.appends !== true)
1144
+ return undefined;
1145
+ const allocated = asNumber(rows[0]?.allocated);
1146
+ if (statement.allocates !== undefined &&
1147
+ allocated !== statement.allocates) {
1148
+ throw new Error('partition commit sequence moved under the partition lock');
1149
+ }
1150
+ return allocated;
1151
+ }, (error) => {
1152
+ throw mapError(error, statement);
1153
+ })));
1154
+ return cursors.find((cursor) => cursor !== undefined);
1155
+ }
1156
+ /** The candidate row this transaction staged and has not yet written. */
1157
+ #stagedRow(table, rowId) {
1158
+ return (this.#staged.get(table)?.get(rowId)?.row ??
1159
+ this.#closedRows.get(table)?.get(rowId)?.row);
1160
+ }
1161
+ /**
1162
+ * A read by primary key. A staged row answers from memory, so the read
1163
+ * needs no pending write: it runs without flushing them, and a validator
1164
+ * that reads rows by id adds no savepoint or write round trip (§6.4).
1165
+ */
987
1166
  getRow(table, rowId) {
988
1167
  this.#assertOpen();
1168
+ const staged = this.#stagedRow(table, rowId);
1169
+ if (staged !== undefined)
1170
+ return Promise.resolve(staged);
989
1171
  const prefetched = this.#prefetchedRows.get(`${table}\u0000${rowId}`);
990
1172
  if (prefetched !== undefined) {
991
1173
  return Promise.resolve(prefetched ?? undefined);
992
1174
  }
993
- return getRowOn(this.#client, this.#resolveTable(table), this.#partition, rowId);
1175
+ return getRowOn(this.#serial, this.#resolveTable(table), this.#partition, rowId);
994
1176
  }
995
1177
  async getRows(table, rowIds) {
996
1178
  this.#assertOpen();
997
1179
  const rows = new Map();
998
1180
  const missing = [];
999
1181
  for (const rowId of new Set(rowIds)) {
1182
+ const staged = this.#stagedRow(table, rowId);
1000
1183
  const prefetched = this.#prefetchedRows.get(`${table}\u0000${rowId}`);
1001
- if (prefetched === undefined)
1184
+ if (staged !== undefined)
1185
+ rows.set(rowId, staged);
1186
+ else if (prefetched === undefined)
1002
1187
  missing.push(rowId);
1003
1188
  else if (prefetched !== null)
1004
1189
  rows.set(rowId, prefetched);
1005
1190
  }
1006
1191
  if (missing.length === 0)
1007
1192
  return rows;
1008
- // One statement for every id the admission read did not already answer.
1009
- // `#client` flushes this transaction's staged upserts first, so the rows
1010
- // are the final candidate state.
1011
- const { rows: records } = await this.#client.query(selectRowsSql(this.#resolveTable(table), 'postgres'), [this.#partition, JSON.stringify(missing)]);
1193
+ // One statement for every id neither the staged writes nor the
1194
+ // admission read answered (see `getRow`).
1195
+ const { rows: records } = await this.#serial.query(selectRowsSql(this.#resolveTable(table), 'postgres'), [this.#partition, JSON.stringify(missing)]);
1012
1196
  for (const record of records)
1013
1197
  rows.set(record.row_id, toStoredRow(record));
1014
1198
  return rows;
@@ -1018,7 +1202,8 @@ class PostgresTransaction {
1018
1202
  const tables = [];
1019
1203
  const specs = [];
1020
1204
  for (const target of targets) {
1021
- const ids = [...new Set(target.rowIds)].filter((rowId) => !this.#prefetchedRows.has(`${target.table}\u0000${rowId}`));
1205
+ const ids = [...new Set(target.rowIds)].filter((rowId) => !this.#prefetchedRows.has(`${target.table}\u0000${rowId}`) &&
1206
+ this.#stagedRow(target.table, rowId) === undefined);
1022
1207
  if (ids.length === 0)
1023
1208
  continue;
1024
1209
  tables.push(this.#resolveTable(target.table));
@@ -1052,8 +1237,7 @@ class PostgresTransaction {
1052
1237
  async clearTombstone(table, rowId) {
1053
1238
  this.#assertOpen();
1054
1239
  this.#prefetchedTombstones.delete(`${table}\u0000${rowId}`);
1055
- await this.#ensurePushSavepoint();
1056
- await this.#client.query('DELETE FROM sync_tombstones WHERE partition=$1 AND tbl=$2 AND row_id=$3', [this.#partition, table, rowId]);
1240
+ await this.#statement('DELETE FROM sync_tombstones WHERE partition=$1 AND tbl=$2 AND row_id=$3', [this.#partition, table, rowId], 'write');
1057
1241
  }
1058
1242
  async scanRows(query) {
1059
1243
  this.#assertOpen();
@@ -1107,28 +1291,28 @@ class PostgresTransaction {
1107
1291
  // staged row and never waits for a second pool connection.
1108
1292
  const prepared = this.#prepareQuery(query);
1109
1293
  const result = await this.#client.query(postgresPlaceholders(prepared.sql), prepared.params);
1110
- // The partition lock freezes `max_commit_seq` for the whole transaction,
1111
- // so one read answers every registered query in it.
1112
- if (this.#cursorSeq === undefined) {
1113
- const cursor = await this.#client.query('SELECT max_commit_seq FROM sync_partitions WHERE partition=$1', [this.#partition]);
1114
- this.#cursorSeq =
1115
- cursor.rows[0] === undefined
1116
- ? 0
1117
- : asNumber(cursor.rows[0].max_commit_seq);
1294
+ // Under the partition lock the cursor is the locked `max_commit_seq` plus
1295
+ // this transaction's appends, which the query above flushed: no read.
1296
+ if (this.#nextCommitSeq !== undefined) {
1297
+ return { rows: result.rows, maxCommitSeq: this.#nextCommitSeq - 1 };
1118
1298
  }
1119
- return { rows: result.rows, maxCommitSeq: this.#cursorSeq };
1299
+ const cursor = await this.#client.query('SELECT max_commit_seq FROM sync_partitions WHERE partition=$1', [this.#partition]);
1300
+ // asNumber maps the absent row of a partition without a commit to 0.
1301
+ return {
1302
+ rows: result.rows,
1303
+ maxCommitSeq: asNumber(cursor.rows[0]?.max_commit_seq),
1304
+ };
1120
1305
  }
1121
- async preparePushApply(clientId, commits, runningSchemaVersion, outbox) {
1306
+ async preparePushApply(clientId, commits, runningSchemaVersion, outbox, options) {
1122
1307
  this.#assertOpen();
1308
+ this.#pushApply = true;
1309
+ this.#closeEagerly = options?.closeEagerly === true;
1123
1310
  // The lock is its own statement: a READ COMMITTED statement's snapshot is
1124
1311
  // fixed at its start, so the admission read must run *after* a duplicate's
1125
1312
  // commit has released the partition lock to observe it. Folding the lock
1126
1313
  // and the read into one statement would miss a duplicate that committed
1127
1314
  // while this transaction waited.
1128
1315
  await this.lockPartition();
1129
- const watermark = outbox === undefined
1130
- ? undefined
1131
- : await this.#settleOutbox(clientId, outbox);
1132
1316
  const ids = new Map();
1133
1317
  for (const commit of commits) {
1134
1318
  for (const target of commit.targets ?? []) {
@@ -1144,68 +1328,25 @@ class PostgresTransaction {
1144
1328
  }
1145
1329
  const tables = [...ids.keys()].map((table) => this.#resolveTable(table));
1146
1330
  const specs = [...ids].map(([tbl, rowIds]) => ({ tbl, ids: [...rowIds] }));
1147
- // One statement for the whole request: the serve gate, every commit's
1148
- // idempotency record, and the read-ahead of every commit's known rows.
1149
- const admission = await readAdmissionOn(this.#client, this.#partition, clientId, commits.map((commit) => commit.clientCommitId), runningSchemaVersion, specs.length === 0 ? undefined : { tables, specs });
1331
+ // One statement for the whole request: the outbox settlement, the serve
1332
+ // gate, every commit's idempotency record, and the read-ahead of every
1333
+ // commit's known rows. The settlement precedes every commit's savepoint,
1334
+ // so no rejection discards it.
1335
+ const admission = await readAdmissionOn(this.#client, this.#partition, clientId, commits.map((commit) => commit.clientCommitId), runningSchemaVersion, specs.length === 0 ? undefined : { tables, specs }, outbox);
1150
1336
  this.#recordPrefetched(admission.prefetched);
1151
1337
  return {
1152
1338
  gate: admission.gate,
1153
1339
  persisted: (clientCommitId) => toPersistedPushResult(admission.persisted.get(clientCommitId)),
1154
1340
  deleteFloor: admission.deleteFloor,
1155
- ...(watermark === undefined ? {} : { watermark }),
1341
+ ...(admission.watermark === undefined
1342
+ ? {}
1343
+ : { watermark: admission.watermark }),
1156
1344
  };
1157
1345
  }
1158
- /**
1159
- * The §9 settlement and §6.3.2 steps 1 and 2 under the partition lock,
1160
- * before the first commit's savepoint, so no rejection discards them. Two
1161
- * statements: the settlement touches disjoint rows (outside and inside the
1162
- * request), and the watermark delete must see its sequences.
1163
- */
1164
- async #settleOutbox(clientId, outbox) {
1165
- const first = outbox.commits[0];
1166
- const last = outbox.commits.at(-1);
1167
- if (first === undefined || last === undefined) {
1168
- throw new Error('outbox settlement needs the request commits');
1169
- }
1170
- const commits = JSON.stringify(outbox.commits.map((commit) => ({
1171
- id: commit.clientCommitId,
1172
- seq: commit.clientSeq,
1173
- })));
1174
- // §9 (b): with no pre-upgrade result on the request's last commit, every
1175
- // remaining one outside the request left the outbox; (a) assigns the
1176
- // request's sequences to its pre-upgrade results.
1177
- await this.#client.query(`WITH c AS (SELECT * FROM jsonb_to_recordset($4::text::jsonb) AS c(id text, seq bigint)),
1178
- b AS (DELETE FROM sync_push_results
1179
- WHERE partition=$1 AND client_id=$2 AND outbox_id IS NULL
1180
- AND client_commit_id NOT IN (SELECT id FROM c)
1181
- AND NOT EXISTS (SELECT 1 FROM sync_push_results
1182
- WHERE partition=$1 AND client_id=$2 AND client_commit_id=$5
1183
- AND outbox_id IS NULL)),
1184
- a AS (UPDATE sync_push_results r SET outbox_id=$3, client_seq=c.seq FROM c
1185
- WHERE r.partition=$1 AND r.client_id=$2 AND r.outbox_id IS NULL
1186
- AND r.client_commit_id=c.id)
1187
- SELECT 1`, [
1188
- this.#partition,
1189
- clientId,
1190
- outbox.outboxId,
1191
- commits,
1192
- last.clientCommitId,
1193
- ]);
1194
- const { rows } = await this.#client.query(`WITH w AS (
1195
- INSERT INTO sync_push_watermarks(partition, client_id, outbox_id, through_seq)
1196
- VALUES ($1,$2,$3,$4)
1197
- ON CONFLICT (partition, client_id, outbox_id) DO UPDATE
1198
- SET through_seq=GREATEST(sync_push_watermarks.through_seq, EXCLUDED.through_seq)
1199
- RETURNING through_seq),
1200
- d AS (DELETE FROM sync_push_results
1201
- WHERE partition=$1 AND client_id=$2 AND outbox_id=$3
1202
- AND client_seq<(SELECT through_seq FROM w))
1203
- SELECT through_seq FROM w`, [this.#partition, clientId, outbox.outboxId, first.clientSeq]);
1204
- return asNumber(rows[0]?.through_seq);
1205
- }
1206
1346
  async lockPartition() {
1207
1347
  this.#assertOpen();
1208
- await lockPartitionOn(this.#client, this.#partition);
1348
+ this.#nextCommitSeq =
1349
+ (await lockPartitionOn(this.#client, this.#partition)) + 1;
1209
1350
  }
1210
1351
  async pinSnapshot() {
1211
1352
  this.#assertOpen();
@@ -1214,13 +1355,12 @@ class PostgresTransaction {
1214
1355
  }
1215
1356
  async advanceCheckpoint(name, ownerEpoch, watermark, observedRows, nowMs) {
1216
1357
  this.#assertOpen();
1217
- await this.#ensurePushSavepoint();
1218
1358
  // `RETURNING`: `PgQueryable` reports returned rows only (no affected-row
1219
1359
  // count), so every affected-row check counts returned rows.
1220
- const result = await this.#client.query(`UPDATE sync_backfill_checkpoints
1360
+ const result = await this.#statement(`UPDATE sync_backfill_checkpoints
1221
1361
  SET watermark=$1, observed_rows=$2, updated_at_ms=$3
1222
1362
  WHERE partition=$4 AND name=$5 AND owner_epoch=$6
1223
- RETURNING name`, [watermark, observedRows, nowMs, this.#partition, name, ownerEpoch]);
1363
+ RETURNING name`, [watermark, observedRows, nowMs, this.#partition, name, ownerEpoch], 'write');
1224
1364
  return result.rows.length === 1;
1225
1365
  }
1226
1366
  readServeGate(runningSchemaVersion) {
@@ -1231,13 +1371,20 @@ class PostgresTransaction {
1231
1371
  }
1232
1372
  async rejectPushCommit(clientId, clientCommitId, result, sequence) {
1233
1373
  this.#assertOpen();
1234
- // Candidate rows that never reached the database are discarded with the
1235
- // staging map; a savepoint exists only when one did. It stays open: the
1374
+ // Candidate writes that never reached the database are discarded with the
1375
+ // staging maps; a savepoint exists only when one did. It stays open: the
1236
1376
  // rollback returns the database to where the next commit starts (the
1237
- // results wait in memory), so the next commit writes behind it.
1377
+ // closed commits' writes wait in memory or precede it), so the next
1378
+ // commit writes behind it.
1238
1379
  this.#staged.clear();
1239
1380
  this.#stagedBlobRefs.clear();
1381
+ this.#stagedAppend = undefined;
1382
+ if (this.#openCommitSeq !== undefined) {
1383
+ this.#nextCommitSeq = this.#openCommitSeq;
1384
+ this.#openCommitSeq = undefined;
1385
+ }
1240
1386
  if (this.#pushApplySavepoint) {
1387
+ this.#pushApplySavepoint = false;
1241
1388
  await this.#serial.query('ROLLBACK TO SAVEPOINT syncular_push_candidate');
1242
1389
  }
1243
1390
  this.#recordPushResult(clientId, clientCommitId, result, sequence);
@@ -1257,19 +1404,20 @@ class PostgresTransaction {
1257
1404
  const compiled = this.#resolveTable(table);
1258
1405
  // Decodes the payload: an undecodable row fails here, at its operation.
1259
1406
  const values = upsertValues(compiled, this.#partition, row, 'postgres');
1407
+ const entry = {
1408
+ row,
1409
+ values,
1410
+ ...(context === undefined ? {} : { opIndex: context.opIndex }),
1411
+ };
1260
1412
  if (!compiled.indexes.some((index) => index.unique === true)) {
1261
- let rows = this.#staged.get(compiled);
1413
+ let rows = this.#staged.get(table);
1262
1414
  if (rows === undefined) {
1263
1415
  rows = new Map();
1264
- this.#staged.set(compiled, rows);
1416
+ this.#staged.set(table, rows);
1265
1417
  }
1266
1418
  // A later upsert of the same row in this transaction replaces it.
1267
1419
  rows.delete(row.rowId);
1268
- rows.set(row.rowId, {
1269
- row,
1270
- values,
1271
- ...(context === undefined ? {} : { opIndex: context.opIndex }),
1272
- });
1420
+ rows.set(row.rowId, entry);
1273
1421
  return;
1274
1422
  }
1275
1423
  const refs = this.#stagedBlobRefs.get(table);
@@ -1279,9 +1427,13 @@ class PostgresTransaction {
1279
1427
  if (refs.size === 0)
1280
1428
  this.#stagedBlobRefs.delete(table);
1281
1429
  }
1282
- await this.#ensurePushSavepoint();
1430
+ const statements = writeStatements(this.#partition, this.#writerVersion, this.#resolveTable, new Map([[table, new Map([[row.rowId, entry]])]]), blobIds === undefined
1431
+ ? new Map()
1432
+ : new Map([[table, new Map([[row.rowId, blobIds]])]]), [], []);
1283
1433
  try {
1284
- await writeRowsOn(this.#client, compiled, this.#partition, [{ row, values }], blobIds === undefined ? [] : [{ rowId: row.rowId, blobIds }]);
1434
+ for (const statement of statements) {
1435
+ await this.#statement(`WITH ${statement.ctes.join(',\n')} SELECT 1`, statement.params, 'write');
1436
+ }
1285
1437
  }
1286
1438
  catch (error) {
1287
1439
  if (isPostgresConstraintError(error)) {
@@ -1297,14 +1449,13 @@ class PostgresTransaction {
1297
1449
  refs?.delete(rowId);
1298
1450
  if (refs?.size === 0)
1299
1451
  this.#stagedBlobRefs.delete(table);
1300
- await this.#ensurePushSavepoint();
1301
1452
  // One statement: the row, its scope-index entries, and (§5.9.4: a
1302
1453
  // deleted row references no blobs) its blob references.
1303
- await this.#client.query(`WITH deleted AS (${deleteRowSql(this.#resolveTable(table), 'postgres')}),
1454
+ await this.#statement(`WITH deleted AS (${deleteRowSql(this.#resolveTable(table), 'postgres')}),
1304
1455
  unscoped AS (
1305
1456
  DELETE FROM sync_row_scopes WHERE partition=$1 AND tbl=$3 AND row_id=$2
1306
1457
  )
1307
- DELETE FROM sync_blob_refs WHERE partition=$1 AND tbl=$3 AND row_id=$2`, [this.#partition, rowId, table]);
1458
+ DELETE FROM sync_blob_refs WHERE partition=$1 AND tbl=$3 AND row_id=$2`, [this.#partition, rowId, table], 'write');
1308
1459
  }
1309
1460
  setBlobRefs(table, rowId, blobIds) {
1310
1461
  this.#assertOpen();
@@ -1318,139 +1469,81 @@ class PostgresTransaction {
1318
1469
  rows.set(rowId, blobIds);
1319
1470
  return Promise.resolve();
1320
1471
  }
1472
+ /**
1473
+ * Stage the append with the commit's rows. Under the partition lock the
1474
+ * cursor is frozen, so the next `commitSeq` is known without a statement
1475
+ * and a push commit's append waits for the next statement. Outside a push
1476
+ * apply the append is written at once (its allocation takes the lock when
1477
+ * no earlier statement did), so a relational constraint on the staged rows
1478
+ * fails this call.
1479
+ */
1321
1480
  async appendCommit(commit) {
1322
1481
  this.#assertOpen();
1323
- // The commit writes its delete tombstones below; every other prefetched
1482
+ if (this.#stagedAppend !== undefined || this.#openCommitSeq !== undefined) {
1483
+ throw new Error('the open commit already appended');
1484
+ }
1485
+ // The commit writes its delete tombstones; every other prefetched
1324
1486
  // tombstone still holds for the request's later commits.
1325
1487
  for (const change of commit.changes) {
1326
1488
  if (change.op === 'delete') {
1327
1489
  this.#prefetchedTombstones.delete(`${change.table}\u0000${change.rowId}`);
1328
1490
  }
1329
1491
  }
1330
- // The append advances the partition cursor; a registered query that runs
1331
- // after it must read the new value.
1332
- this.#cursorSeq = undefined;
1333
- const q = this.#client;
1334
- const p = this.#partition;
1335
- // Allocate the next dense commitSeq under a per-partition row lock: the
1336
- // UPDATE … RETURNING serializes concurrent pushes to this partition and
1337
- // never leaves a gap on rollback (see the file header).
1338
- // §2.2: the allocation also raises the delete floor to the commit's.
1339
- const allocate = `INSERT INTO sync_partitions(partition, max_commit_seq, delete_floor) VALUES ($1, 1, $7)
1340
- ON CONFLICT (partition) DO UPDATE
1341
- SET max_commit_seq = sync_partitions.max_commit_seq + 1,
1342
- delete_floor = GREATEST(sync_partitions.delete_floor, EXCLUDED.delete_floor)
1343
- RETURNING max_commit_seq`;
1344
- const commitRow = `INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms, writer_version)
1345
- SELECT $1, allocated.max_commit_seq, $2, $3, $4, $5, $6 FROM allocated
1346
- RETURNING commit_seq`;
1347
- const head = [
1348
- p,
1349
- commit.clientId,
1350
- commit.clientCommitId,
1351
- commit.actorId,
1352
- commit.createdAtMs,
1353
- this.#writerVersion,
1354
- commitDeleteFloor(commit.changes),
1355
- ];
1356
- if (commit.changes.length === 0) {
1357
- const { rows } = await q.query(`WITH allocated AS (${allocate}) ${commitRow}`, head);
1358
- return asNumber(rows[0]?.commit_seq);
1359
- }
1360
- // §2.2: a delete whose commit follows it with an upsert is a scope move.
1361
- // The row is live, so the tombstone insert below skips it.
1362
- const moved = JSON.stringify([...rowsLiveAfterDelete(commit.changes)].map((key) => {
1363
- const at = key.indexOf('\u0000');
1364
- return { tbl: key.slice(0, at), row_id: key.slice(at + 1) };
1365
- }));
1366
- // One writable-CTE chain per chunk of changes: the changes, their
1367
- // inverted scope entries, and the §5 delete tombstones. A statement binds
1368
- // at most a 16-bit count of parameters, so a chunk carries
1369
- // PG_CHANGES_PER_STATEMENT changes. The first chunk folds the allocation,
1370
- // the commit row, and the change-log write into one statement; later
1371
- // chunks run under the same commitSeq with their global idx. Serialized
1372
- // scopes bind as text before parsing JSONB; drivers that encode JSONB
1373
- // parameters would otherwise store the string as a scalar. A commit can
1374
- // delete one row twice, so a chunk's tombstone insert keeps the latest
1375
- // delete per key and a later chunk overwrites it with its own. `version`
1376
- // is the change's row version (§2.2), which a delete's tombstone keeps.
1377
- const valuesOf = (params, slice, firstIdx) => slice.map((change, offset) => {
1378
- const base = params.length;
1379
- params.push(firstIdx + offset, change.table, change.rowId, change.op === 'upsert' ? 1 : 2, change.rowVersion, JSON.stringify(change.scopes), change.payload ?? null);
1380
- return `($${base + 1}::int,$${base + 2}::text,$${base + 3}::text,$${base + 4}::smallint,$${base + 5}::bigint,$${base + 6}::text::jsonb,$${base + 7}::bytea)`;
1381
- });
1382
- const scopesCte = `scoped AS (
1383
- INSERT INTO sync_change_scopes(partition, tbl, var, value, commit_seq)
1384
- SELECT inserted.partition, inserted.tbl, scope.key, scope.value, inserted.commit_seq
1385
- FROM inserted CROSS JOIN LATERAL jsonb_each_text(inserted.scopes) AS scope
1386
- ON CONFLICT DO NOTHING
1387
- )`;
1388
- const tombstones = (movedParam) => `INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq, server_version, scopes)
1389
- SELECT DISTINCT ON (inserted.tbl, inserted.row_id) inserted.partition,
1390
- inserted.tbl, inserted.row_id, inserted.commit_seq,
1391
- v.version, inserted.scopes
1392
- FROM inserted JOIN v ON v.idx = inserted.idx
1393
- WHERE inserted.op = 2
1394
- AND NOT EXISTS (
1395
- SELECT 1
1396
- FROM jsonb_to_recordset(${movedParam}::text::jsonb) AS moved(tbl text, row_id text)
1397
- WHERE moved.tbl = inserted.tbl AND moved.row_id = inserted.row_id)
1398
- ORDER BY inserted.tbl, inserted.row_id, inserted.idx DESC
1399
- ON CONFLICT (partition, tbl, row_id)
1400
- DO UPDATE SET commit_seq = excluded.commit_seq,
1401
- server_version = excluded.server_version,
1402
- scopes = excluded.scopes`;
1403
- const firstSlice = commit.changes.slice(0, PG_CHANGES_PER_STATEMENT);
1404
- const params = [...head];
1405
- const values = valuesOf(params, firstSlice, 0);
1406
- params.push(moved);
1407
- const movedParam = `$${params.length}`;
1408
- const { rows } = await q.query(`WITH allocated AS (${allocate}),
1409
- allocated_commit AS (${commitRow}),
1410
- v(idx, tbl, row_id, op, version, scopes, payload) AS (
1411
- VALUES ${values.join(',')}
1412
- ),
1413
- inserted AS (
1414
- INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
1415
- SELECT $1, allocated.max_commit_seq, v.idx, v.tbl, v.row_id, v.op, v.version, v.scopes, v.payload
1416
- FROM allocated CROSS JOIN v
1417
- RETURNING partition, idx, tbl, row_id, op, commit_seq, scopes
1418
- ),
1419
- ${scopesCte},
1420
- tombstoned AS (${tombstones(movedParam)})
1421
- SELECT max_commit_seq FROM allocated`, params);
1422
- const commitSeq = asNumber(rows[0]?.max_commit_seq);
1423
- for (let first = PG_CHANGES_PER_STATEMENT; first < commit.changes.length; first += PG_CHANGES_PER_STATEMENT) {
1424
- const chunkParams = [p, commitSeq];
1425
- const chunkValues = valuesOf(chunkParams, commit.changes.slice(first, first + PG_CHANGES_PER_STATEMENT), first);
1426
- chunkParams.push(moved);
1427
- const chunkMovedParam = `$${chunkParams.length}`;
1428
- await q.query(`WITH v(idx, tbl, row_id, op, version, scopes, payload) AS (
1429
- VALUES ${chunkValues.join(',')}
1430
- ),
1431
- inserted AS (
1432
- INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
1433
- SELECT $1, $2, v.idx, v.tbl, v.row_id, v.op, v.version, v.scopes, v.payload
1434
- FROM v
1435
- RETURNING partition, idx, tbl, row_id, op, commit_seq, scopes
1436
- ),
1437
- ${scopesCte},
1438
- tombstoned AS (${tombstones(chunkMovedParam)})
1439
- SELECT 1`, chunkParams);
1492
+ const commitSeq = this.#nextCommitSeq;
1493
+ if (commitSeq === undefined) {
1494
+ this.#stagedAppend = { commit };
1495
+ const allocated = await this.#flush('write');
1496
+ if (allocated === undefined) {
1497
+ throw new Error('the commit append returned no cursor');
1498
+ }
1499
+ this.#nextCommitSeq = allocated + 1;
1500
+ return allocated;
1440
1501
  }
1502
+ this.#nextCommitSeq = commitSeq + 1;
1503
+ this.#stagedAppend = { commit, commitSeq };
1504
+ if (this.#pushApply)
1505
+ this.#openCommitSeq = commitSeq;
1506
+ else
1507
+ await this.#flush('write');
1441
1508
  return commitSeq;
1442
1509
  }
1443
1510
  async putPushResult(clientId, clientCommitId, result, sequence) {
1444
1511
  this.#assertOpen();
1512
+ // On the rerun after a closed commit's write failed, the commit writes
1513
+ // behind its savepoint before it closes, so a relational constraint
1514
+ // rejects it alone (`StorageConstraintError`).
1515
+ if (this.#closeEagerly)
1516
+ await this.#flush('read');
1445
1517
  this.#recordPushResult(clientId, clientCommitId, result, sequence);
1446
- // The commit is closed: the next commit's first write takes its own
1447
- // savepoint.
1518
+ // The commit is closed: its writes join the closed commits', and the next
1519
+ // commit's first write takes its own savepoint.
1520
+ for (const [table, rows] of this.#staged) {
1521
+ const closed = this.#closedRows.get(table) ?? new Map();
1522
+ for (const [rowId, entry] of rows) {
1523
+ closed.delete(rowId);
1524
+ closed.set(rowId, entry);
1525
+ }
1526
+ this.#closedRows.set(table, closed);
1527
+ }
1528
+ for (const [table, refs] of this.#stagedBlobRefs) {
1529
+ const closed = this.#closedBlobRefs.get(table) ?? new Map();
1530
+ for (const [rowId, blobIds] of refs)
1531
+ closed.set(rowId, blobIds);
1532
+ this.#closedBlobRefs.set(table, closed);
1533
+ }
1534
+ if (this.#stagedAppend !== undefined) {
1535
+ this.#closedAppends.push(this.#stagedAppend);
1536
+ }
1537
+ this.#staged.clear();
1538
+ this.#stagedBlobRefs.clear();
1539
+ this.#stagedAppend = undefined;
1540
+ this.#openCommitSeq = undefined;
1448
1541
  this.#pushApplySavepoint = false;
1449
1542
  }
1450
1543
  async enqueueReactions(reactions) {
1451
1544
  this.#assertOpen();
1452
1545
  for (const reaction of reactions) {
1453
- await this.#client.query(`INSERT INTO sync_reactions(
1546
+ await this.#statement(`INSERT INTO sync_reactions(
1454
1547
  partition, idempotency_key, type, version, payload,
1455
1548
  source_client_id, source_client_commit_id, source_commit_seq,
1456
1549
  created_at_ms, available_at_ms, status, attempts, max_attempts
@@ -1465,22 +1558,13 @@ class PostgresTransaction {
1465
1558
  reaction.sourceCommitSeq,
1466
1559
  reaction.createdAtMs,
1467
1560
  reaction.maxAttempts,
1468
- ]);
1561
+ ], 'write');
1469
1562
  }
1470
1563
  }
1471
1564
  async commit() {
1472
1565
  this.#assertOpen();
1473
- if (this.#pushResults.length > 0) {
1474
- // Every result of the request in one statement; `#client` flushes the
1475
- // staged writes first.
1476
- await this.#client.query(`INSERT INTO sync_push_results(partition, client_id, client_commit_id, result, outbox_id, client_seq)
1477
- SELECT $1, r.client_id, r.client_commit_id, r.result, r.outbox_id, r.client_seq
1478
- FROM jsonb_to_recordset($2::text::jsonb)
1479
- AS r(client_id text, client_commit_id text, result jsonb, outbox_id text, client_seq bigint)
1480
- ON CONFLICT (partition, client_id, client_commit_id) DO NOTHING`, [this.#partition, JSON.stringify(this.#pushResults)]);
1481
- this.#pushResults.length = 0;
1482
- }
1483
- await this.#writeStaged();
1566
+ // Every pending write in one statement where the parameters fit.
1567
+ await this.#flush('final');
1484
1568
  this.#open = false;
1485
1569
  // Signal the transaction wrapper to COMMIT; wait for it to actually land.
1486
1570
  this.#resolve();
@@ -1489,7 +1573,6 @@ class PostgresTransaction {
1489
1573
  async rollback() {
1490
1574
  if (!this.#open)
1491
1575
  return;
1492
- this.#staged.clear();
1493
1576
  this.#open = false;
1494
1577
  this.#reject(new RollbackSignal());
1495
1578
  // Swallow — a rollback is a normal outcome, not an error to the caller.
@@ -1508,10 +1591,6 @@ class RollbackSignal extends Error {
1508
1591
  this.name = 'RollbackSignal';
1509
1592
  }
1510
1593
  }
1511
- /** Changes per sync_changes statement: 2 + 7 × 4,096 = 28,674 bound
1512
- * parameters. The Bind message counts parameters in 16 bits; PostgreSQL reads
1513
- * up to 65,535, and PGlite 0.5 corrupts its state above 32,767. */
1514
- const PG_CHANGES_PER_STATEMENT = 4096;
1515
1594
  export class PostgresServerStorage {
1516
1595
  #exec;
1517
1596
  /** Set by `ensureSchema`: app-table lookup for the relational row store. */