@syncular/server 0.81.0 → 0.82.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/authoritative-query.js +3 -1
- package/dist/operations.js +3 -6
- package/dist/postgres-storage.js +522 -443
- package/dist/push.d.ts +5 -0
- package/dist/push.js +18 -2
- package/dist/relational-rows.d.ts +5 -3
- package/dist/relational-rows.js +6 -4
- package/dist/storage-errors.d.ts +10 -0
- package/dist/storage-errors.js +12 -0
- package/dist/storage.d.ts +21 -6
- package/package.json +2 -2
- package/src/authoritative-query.ts +3 -1
- package/src/operations.ts +3 -7
- package/src/postgres-storage.ts +696 -538
- package/src/push.ts +39 -1
- package/src/relational-rows.ts +6 -3
- package/src/storage-errors.ts +14 -0
- package/src/storage.ts +22 -5
package/dist/postgres-storage.js
CHANGED
|
@@ -45,7 +45,7 @@ import { syncError } from './errors.js';
|
|
|
45
45
|
import { asBytes, asNumber, } from './pg-executor.js';
|
|
46
46
|
import { assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, changedScopePatterns, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, rekeyCleanupSql, rekeyedScopes, rekeyRowScopesSql, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanProbeSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, rowsReferencingBlobSql, selectRowSql, selectRowsSql, prefetchRowsBatchSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
47
47
|
import { matchesEffective, sha256Hex } from './scopes.js';
|
|
48
|
-
import { isPostgresConstraintError, StorageConstraintError, } from './storage-errors.js';
|
|
48
|
+
import { ClosedCommitConstraintError, isPostgresConstraintError, StorageConstraintError, } from './storage-errors.js';
|
|
49
49
|
import { chooseScanVariable, readLiveWindowPages, resolveIndexRowScan, } from './storage-query.js';
|
|
50
50
|
/**
|
|
51
51
|
* Schema DDL. Applied by `PostgresServerStorage.migrate()` (idempotent).
|
|
@@ -224,15 +224,18 @@ CREATE TABLE IF NOT EXISTS sync_writer_fence(
|
|
|
224
224
|
required_writer_version INTEGER NOT NULL
|
|
225
225
|
);
|
|
226
226
|
`;
|
|
227
|
+
/** Lock the partition row; returns its `max_commit_seq`, which the lock freezes. */
|
|
227
228
|
async function lockPartitionOn(client, partition) {
|
|
228
|
-
const
|
|
229
|
-
|
|
230
|
-
|
|
229
|
+
const lock = () => client.query('SELECT max_commit_seq FROM sync_partitions WHERE partition=$1 FOR UPDATE', [partition]);
|
|
230
|
+
const locked = await lock();
|
|
231
|
+
if (locked.rows[0] !== undefined) {
|
|
232
|
+
return asNumber(locked.rows[0].max_commit_seq);
|
|
233
|
+
}
|
|
231
234
|
// The first writer initializes the partition. Concurrent initializers may
|
|
232
235
|
// wait on this insert, so acquire the row lock again before applying writes.
|
|
233
236
|
await client.query(`INSERT INTO sync_partitions(partition, max_commit_seq) VALUES ($1, 0)
|
|
234
237
|
ON CONFLICT (partition) DO NOTHING`, [partition]);
|
|
235
|
-
await
|
|
238
|
+
return asNumber((await lock()).rows[0]?.max_commit_seq);
|
|
236
239
|
}
|
|
237
240
|
/**
|
|
238
241
|
* Insert the `declared` row and raise the fence, on the caller's open
|
|
@@ -331,25 +334,72 @@ async function readServeGateOn(client, partition, runningSchemaVersion) {
|
|
|
331
334
|
return toServeGate(rows[0]);
|
|
332
335
|
}
|
|
333
336
|
/**
|
|
334
|
-
* The push admission
|
|
335
|
-
*
|
|
336
|
-
*
|
|
337
|
-
*
|
|
338
|
-
* statement
|
|
339
|
-
*
|
|
340
|
-
*
|
|
337
|
+
* The push admission statement: the §9 settlement and §6.3.2 steps 1 and 2
|
|
338
|
+
* when the request carries an outbox, the RFC 0007 serve gate, the §2.3
|
|
339
|
+
* idempotency records of every commit of the request, and (when the caller
|
|
340
|
+
* already knows the commits' row targets) the read-ahead rows, in one
|
|
341
|
+
* statement with one snapshot. The caller takes the partition lock first:
|
|
342
|
+
* under READ COMMITTED a statement's snapshot is fixed at its start, so the
|
|
343
|
+
* read must be the statement *after* the lock to observe a duplicate that
|
|
344
|
+
* committed while this transaction waited for it.
|
|
345
|
+
*
|
|
346
|
+
* The settlement's writable CTEs see the same snapshot as the read, so they
|
|
347
|
+
* touch disjoint rows and the read leaves out the results they delete. §9
|
|
348
|
+
* (b): with no pre-upgrade result on the request's last commit, every
|
|
349
|
+
* remaining one outside the request left the outbox. (a) assigns the
|
|
350
|
+
* request's sequences to its pre-upgrade results at or above the new
|
|
351
|
+
* watermark; step 2 deletes the outbox's results below it, the pre-upgrade
|
|
352
|
+
* results of the request below it included.
|
|
341
353
|
*/
|
|
342
|
-
async function readAdmissionOn(client, partition, clientId, clientCommitIds, runningSchemaVersion, prefetch) {
|
|
354
|
+
async function readAdmissionOn(client, partition, clientId, clientCommitIds, runningSchemaVersion, prefetch, outbox) {
|
|
343
355
|
const params = [
|
|
344
356
|
partition,
|
|
345
357
|
runningSchemaVersion,
|
|
346
358
|
clientId,
|
|
347
359
|
JSON.stringify(clientCommitIds),
|
|
348
360
|
];
|
|
349
|
-
let
|
|
361
|
+
let settlement = '';
|
|
362
|
+
if (outbox !== undefined) {
|
|
363
|
+
const first = outbox.commits[0];
|
|
364
|
+
const last = outbox.commits.at(-1);
|
|
365
|
+
if (first === undefined || last === undefined) {
|
|
366
|
+
throw new Error('outbox settlement needs the request commits');
|
|
367
|
+
}
|
|
368
|
+
params.push(outbox.outboxId, JSON.stringify(outbox.commits.map((commit) => ({
|
|
369
|
+
id: commit.clientCommitId,
|
|
370
|
+
seq: commit.clientSeq,
|
|
371
|
+
}))), last.clientCommitId, first.clientSeq);
|
|
372
|
+
settlement = `WITH c AS (SELECT * FROM jsonb_to_recordset($6::text::jsonb) AS c(id text, seq bigint)),
|
|
373
|
+
w AS (
|
|
374
|
+
INSERT INTO sync_push_watermarks(partition, client_id, outbox_id, through_seq)
|
|
375
|
+
VALUES ($1,$3,$5,$8)
|
|
376
|
+
ON CONFLICT (partition, client_id, outbox_id) DO UPDATE
|
|
377
|
+
SET through_seq=GREATEST(sync_push_watermarks.through_seq, EXCLUDED.through_seq)
|
|
378
|
+
RETURNING through_seq),
|
|
379
|
+
b AS (DELETE FROM sync_push_results
|
|
380
|
+
WHERE partition=$1 AND client_id=$3 AND outbox_id IS NULL
|
|
381
|
+
AND client_commit_id NOT IN (SELECT id FROM c)
|
|
382
|
+
AND NOT EXISTS (SELECT 1 FROM sync_push_results
|
|
383
|
+
WHERE partition=$1 AND client_id=$3 AND client_commit_id=$7
|
|
384
|
+
AND outbox_id IS NULL)),
|
|
385
|
+
a AS (UPDATE sync_push_results r SET outbox_id=$5, client_seq=c.seq FROM c, w
|
|
386
|
+
WHERE r.partition=$1 AND r.client_id=$3 AND r.outbox_id IS NULL
|
|
387
|
+
AND r.client_commit_id=c.id AND c.seq>=w.through_seq),
|
|
388
|
+
expired AS (DELETE FROM sync_push_results r USING w
|
|
389
|
+
WHERE r.partition=$1 AND r.client_id=$3
|
|
390
|
+
AND ((r.outbox_id=$5 AND r.client_seq<w.through_seq)
|
|
391
|
+
OR (r.outbox_id IS NULL AND EXISTS (SELECT 1 FROM c
|
|
392
|
+
WHERE c.id=r.client_commit_id AND c.seq<w.through_seq)))
|
|
393
|
+
RETURNING r.client_commit_id)
|
|
394
|
+
`;
|
|
395
|
+
}
|
|
396
|
+
let statement = `${settlement}SELECT ${SERVE_GATE_PROJECTION},
|
|
350
397
|
(SELECT jsonb_object_agg(client_commit_id, result) FROM sync_push_results
|
|
351
|
-
WHERE partition=$1 AND client_id=$3 AND client_commit_id IN (SELECT jsonb_array_elements_text($4::text::jsonb))
|
|
352
|
-
|
|
398
|
+
WHERE partition=$1 AND client_id=$3 AND client_commit_id IN (SELECT jsonb_array_elements_text($4::text::jsonb))${outbox === undefined
|
|
399
|
+
? ''
|
|
400
|
+
: ' AND client_commit_id NOT IN (SELECT client_commit_id FROM expired)'}) AS results,
|
|
401
|
+
(SELECT delete_floor FROM sync_partitions WHERE partition=$1) AS delete_floor,
|
|
402
|
+
${outbox === undefined ? 'NULL::bigint' : '(SELECT through_seq FROM w)'} AS watermark`;
|
|
353
403
|
if (prefetch !== undefined && prefetch.specs.length > 0) {
|
|
354
404
|
params.push(JSON.stringify(prefetch.specs));
|
|
355
405
|
// The gate row pads the read-ahead columns with typed NULLs so both
|
|
@@ -362,6 +412,7 @@ async function readAdmissionOn(client, partition, clientId, clientCommitIds, run
|
|
|
362
412
|
NULL::jsonb AS tombstone_scopes
|
|
363
413
|
UNION ALL
|
|
364
414
|
SELECT NULL::bigint, NULL::text, NULL::json, NULL::jsonb, NULL::bigint,
|
|
415
|
+
NULL::bigint,
|
|
365
416
|
p.tbl, p.lookup_id, p.row_id, p.server_version, p.scopes,
|
|
366
417
|
p.payload, p.column_versions, p.tombstone_seq,
|
|
367
418
|
p.tombstone_version, p.tombstone_scopes
|
|
@@ -382,6 +433,7 @@ async function readAdmissionOn(client, partition, clientId, clientCommitIds, run
|
|
|
382
433
|
: []),
|
|
383
434
|
// asNumber maps the NULL of a partition without a commit to 0.
|
|
384
435
|
deleteFloor: asNumber(gateRow?.delete_floor),
|
|
436
|
+
watermark: outbox === undefined ? undefined : asNumber(gateRow?.watermark),
|
|
385
437
|
prefetched: rows.filter((row) => row.tbl !== null && row.tbl !== undefined),
|
|
386
438
|
};
|
|
387
439
|
}
|
|
@@ -620,11 +672,11 @@ async function rewriteRowsOn(q, table, oldLayout, rekey) {
|
|
|
620
672
|
// database) every statement is a round trip, and the bump holds the
|
|
621
673
|
// migration lock for its whole duration. The page's rows bind as one
|
|
622
674
|
// VALUES list with typed placeholders (`rewriteValues` order: projection
|
|
623
|
-
// columns, re-keyed scopes, payload, partition, row id). The scopes
|
|
624
|
-
// text before the JSONB cast (see `scanRowBatchSql`).
|
|
675
|
+
// columns, re-keyed scopes, payload, partition, row id). The scopes and
|
|
676
|
+
// `json` columns bind as text before the JSONB cast (see `scanRowBatchSql`).
|
|
625
677
|
const projected = table.materialize ? table.columns : [];
|
|
626
678
|
const types = [
|
|
627
|
-
...projected.map((column) => columnSqlType(column, 'postgres')),
|
|
679
|
+
...projected.map((column) => column.type === 'json' ? 'TEXT' : columnSqlType(column, 'postgres')),
|
|
628
680
|
...(rekey ? ['TEXT'] : []),
|
|
629
681
|
'BYTEA',
|
|
630
682
|
'TEXT',
|
|
@@ -632,7 +684,7 @@ async function rewriteRowsOn(q, table, oldLayout, rekey) {
|
|
|
632
684
|
];
|
|
633
685
|
const payloadAt = projected.length + (rekey ? 1 : 0);
|
|
634
686
|
const sets = [
|
|
635
|
-
...projected.map((column, i) => `${quoteIdent(column.name)}=v.c${i}`),
|
|
687
|
+
...projected.map((column, i) => `${quoteIdent(column.name)}=v.c${i}${column.type === 'json' ? '::jsonb' : ''}`),
|
|
636
688
|
...(rekey
|
|
637
689
|
? [`${quoteIdent(SYNC_SCOPES_COLUMN)}=v.c${projected.length}::jsonb`]
|
|
638
690
|
: []),
|
|
@@ -696,116 +748,196 @@ async function scanRowsByIndexOn(q, compiled, partition, query) {
|
|
|
696
748
|
return rows.map(toStoredRow);
|
|
697
749
|
}
|
|
698
750
|
/**
|
|
699
|
-
*
|
|
700
|
-
*
|
|
701
|
-
*
|
|
702
|
-
* row now references (entries outside the new set deleted, missing ones
|
|
703
|
-
* inserted). Row ids are distinct, so no index entry is touched twice and the
|
|
704
|
-
* writable CTEs cannot conflict. The scopes and blob ids bind as text before
|
|
705
|
-
* the JSONB cast (see `appendCommit`); the refs set is part of the same
|
|
706
|
-
* statement, so a blob-referencing commit costs no extra round trip per row.
|
|
707
|
-
* Every chunk is handed to `q` before the first one settles, so a statement
|
|
708
|
-
* issued after this call queues behind all of them on the transaction FIFO.
|
|
751
|
+
* Bound parameters per write statement. The Bind message counts parameters
|
|
752
|
+
* in 16 bits; PostgreSQL reads up to 65,535, and PGlite 0.5 corrupts its
|
|
753
|
+
* state above 32,767.
|
|
709
754
|
*/
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
755
|
+
const PG_PARAMS_PER_STATEMENT = 30_000;
|
|
756
|
+
function bindParam(statement, value) {
|
|
757
|
+
statement.params.push(value);
|
|
758
|
+
return `$${statement.params.length}`;
|
|
759
|
+
}
|
|
760
|
+
/**
|
|
761
|
+
* The CTEs that write rows of one table: the upsert, the scope index synced
|
|
762
|
+
* to each row's scope map (entries outside it deleted, missing ones
|
|
763
|
+
* inserted), and the §5.9.4 blob-reference index synced to the ids the row
|
|
764
|
+
* now references. Row ids are distinct, so no index entry is touched twice.
|
|
765
|
+
* The scopes and blob ids bind as text before the JSONB cast; drivers that
|
|
766
|
+
* encode JSONB parameters would otherwise store the string as a scalar.
|
|
767
|
+
*/
|
|
768
|
+
function addRowWrites(statement, compiled, partition, rows, blobRefs) {
|
|
769
|
+
const k = statement.ctes.length;
|
|
770
|
+
const offset = statement.params.length;
|
|
771
|
+
statement.params.push(...rows.flatMap((entry) => entry.values));
|
|
772
|
+
const scopes = bindParam(statement, JSON.stringify(rows.map(({ row }) => ({ row_id: row.rowId, scopes: row.scopes }))));
|
|
773
|
+
const p = bindParam(statement, partition);
|
|
774
|
+
const tbl = bindParam(statement, compiled.name);
|
|
775
|
+
statement.ctes.push(`u${k} AS (${upsertSql(compiled, 'postgres', rows.length, offset)})`, `b${k} AS (SELECT row_id, scopes FROM jsonb_to_recordset(${scopes}::text::jsonb) AS r(row_id text, scopes jsonb))`, `rs${k} AS (DELETE FROM sync_row_scopes existing USING b${k}
|
|
776
|
+
WHERE existing.partition=${p} AND existing.tbl=${tbl}
|
|
777
|
+
AND existing.row_id=b${k}.row_id
|
|
778
|
+
AND b${k}.scopes->>existing.var IS DISTINCT FROM existing.value)`, `is${k} AS (INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
|
|
779
|
+
SELECT ${p}, ${tbl}, entry.key, entry.value, b${k}.row_id
|
|
780
|
+
FROM b${k} CROSS JOIN LATERAL jsonb_each_text(b${k}.scopes) AS entry
|
|
781
|
+
ON CONFLICT DO NOTHING)`);
|
|
782
|
+
for (const entry of rows) {
|
|
783
|
+
if (entry.opIndex !== undefined &&
|
|
784
|
+
(statement.opIndex === undefined || entry.opIndex < statement.opIndex)) {
|
|
785
|
+
statement.opIndex = entry.opIndex;
|
|
728
786
|
}
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
AND batch.scopes->>existing.var IS DISTINCT FROM existing.value
|
|
744
|
-
),
|
|
745
|
-
changed_refs AS (
|
|
746
|
-
SELECT DISTINCT row_id
|
|
747
|
-
FROM jsonb_to_recordset($${n + 4}::text::jsonb) AS r(row_id text, blob_ids jsonb)
|
|
748
|
-
),
|
|
749
|
-
removed_refs AS (
|
|
750
|
-
DELETE FROM sync_blob_refs existing
|
|
751
|
-
USING changed_refs
|
|
752
|
-
WHERE existing.partition=$${n + 2} AND existing.tbl=$${n + 3}
|
|
753
|
-
AND existing.row_id=changed_refs.row_id
|
|
754
|
-
AND NOT EXISTS (
|
|
755
|
-
SELECT 1
|
|
756
|
-
FROM jsonb_to_recordset($${n + 4}::text::jsonb) AS r(row_id text, blob_ids jsonb)
|
|
757
|
-
CROSS JOIN LATERAL jsonb_array_elements_text(r.blob_ids) AS entry(value)
|
|
758
|
-
WHERE r.row_id=existing.row_id AND entry.value=existing.blob_id)
|
|
759
|
-
),
|
|
760
|
-
inserted_refs AS (
|
|
761
|
-
INSERT INTO sync_blob_refs(partition, tbl, row_id, blob_id)
|
|
762
|
-
SELECT $${n + 2}, $${n + 3}, r.row_id, entry.value
|
|
763
|
-
FROM jsonb_to_recordset($${n + 4}::text::jsonb) AS r(row_id text, blob_ids jsonb)
|
|
787
|
+
}
|
|
788
|
+
if (blobRefs.length > 0)
|
|
789
|
+
addBlobRefWrites(statement, p, tbl, blobRefs);
|
|
790
|
+
}
|
|
791
|
+
/** Replace the §5.9.4 reference sets of rows of one table. */
|
|
792
|
+
function addBlobRefWrites(statement, p, tbl, blobRefs) {
|
|
793
|
+
const k = statement.ctes.length;
|
|
794
|
+
const refs = bindParam(statement, JSON.stringify(blobRefs));
|
|
795
|
+
statement.ctes.push(`rr${k} AS (DELETE FROM sync_blob_refs existing
|
|
796
|
+
WHERE existing.partition=${p} AND existing.tbl=${tbl}
|
|
797
|
+
AND existing.row_id IN (SELECT r.row_id FROM jsonb_to_recordset(${refs}::text::jsonb) AS r(row_id text, blob_ids jsonb))
|
|
798
|
+
AND NOT EXISTS (
|
|
799
|
+
SELECT 1
|
|
800
|
+
FROM jsonb_to_recordset(${refs}::text::jsonb) AS r(row_id text, blob_ids jsonb)
|
|
764
801
|
CROSS JOIN LATERAL jsonb_array_elements_text(r.blob_ids) AS entry(value)
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
}
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
802
|
+
WHERE r.row_id=existing.row_id AND entry.value=existing.blob_id))`, `ir${k} AS (INSERT INTO sync_blob_refs(partition, tbl, row_id, blob_id)
|
|
803
|
+
SELECT ${p}, ${tbl}, r.row_id, entry.value
|
|
804
|
+
FROM jsonb_to_recordset(${refs}::text::jsonb) AS r(row_id text, blob_ids jsonb)
|
|
805
|
+
CROSS JOIN LATERAL jsonb_array_elements_text(r.blob_ids) AS entry(value)
|
|
806
|
+
ON CONFLICT DO NOTHING)`);
|
|
807
|
+
}
|
|
808
|
+
/**
|
|
809
|
+
* The CTEs that append commits with consecutive `commitSeq`s: the partition
|
|
810
|
+
* cursor and delete floor (§2.2), the commit rows, their changes with the
|
|
811
|
+
* inverted scope entries, and the §5 delete tombstones. The cursor advances
|
|
812
|
+
* by the number of commits, and every row takes its commit's sequence from
|
|
813
|
+
* the advanced cursor; the statement returns the cursor, which a caller that
|
|
814
|
+
* computed the sequences under the partition lock checks. Every list binds
|
|
815
|
+
* as one JSON text parameter, so the parameter count does not grow with the
|
|
816
|
+
* change count. A tombstone keeps the change's row version (§2.2); a row
|
|
817
|
+
* live again at the end of its commit (a §2.2 scope move) gets none, and a
|
|
818
|
+
* later delete of the same row overwrites an earlier one.
|
|
819
|
+
*/
|
|
820
|
+
function addAppends(statement, partition, writerVersion, appends) {
|
|
821
|
+
if (appends.length === 0)
|
|
822
|
+
return;
|
|
823
|
+
const p = bindParam(statement, partition);
|
|
824
|
+
const changes = [];
|
|
825
|
+
const tombstones = new Map();
|
|
826
|
+
let deleteFloor = 0;
|
|
827
|
+
for (const [index, { commit }] of appends.entries()) {
|
|
828
|
+
const k = index + 1;
|
|
829
|
+
deleteFloor = Math.max(deleteFloor, commitDeleteFloor(commit.changes));
|
|
830
|
+
const live = rowsLiveAfterDelete(commit.changes);
|
|
831
|
+
for (const [idx, change] of commit.changes.entries()) {
|
|
832
|
+
changes.push({
|
|
833
|
+
k,
|
|
834
|
+
idx,
|
|
835
|
+
tbl: change.table,
|
|
836
|
+
row_id: change.rowId,
|
|
837
|
+
op: change.op === 'upsert' ? 1 : 2,
|
|
838
|
+
version: change.rowVersion,
|
|
839
|
+
scopes: change.scopes,
|
|
840
|
+
payload: change.payload === undefined ? null : toBase64(change.payload),
|
|
841
|
+
});
|
|
842
|
+
const key = `${change.table}\u0000${change.rowId}`;
|
|
843
|
+
if (change.op === 'delete' && !live.has(key)) {
|
|
844
|
+
tombstones.set(key, {
|
|
845
|
+
k,
|
|
846
|
+
tbl: change.table,
|
|
847
|
+
row_id: change.rowId,
|
|
848
|
+
version: change.rowVersion,
|
|
849
|
+
scopes: change.scopes,
|
|
850
|
+
});
|
|
851
|
+
}
|
|
852
|
+
}
|
|
853
|
+
}
|
|
854
|
+
statement.appends = true;
|
|
855
|
+
const expected = appends.at(-1)?.commitSeq;
|
|
856
|
+
if (expected !== undefined)
|
|
857
|
+
statement.allocates = expected;
|
|
858
|
+
const n = bindParam(statement, appends.length);
|
|
859
|
+
statement.ctes.push(`alloc AS (INSERT INTO sync_partitions(partition, max_commit_seq, delete_floor)
|
|
860
|
+
VALUES (${p}, ${n}, ${bindParam(statement, deleteFloor)})
|
|
861
|
+
ON CONFLICT (partition) DO UPDATE
|
|
862
|
+
SET max_commit_seq=sync_partitions.max_commit_seq + EXCLUDED.max_commit_seq,
|
|
863
|
+
delete_floor=GREATEST(sync_partitions.delete_floor, EXCLUDED.delete_floor)
|
|
864
|
+
RETURNING max_commit_seq, max_commit_seq - ${n}::bigint AS base)`, `commits AS (INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms, writer_version)
|
|
865
|
+
SELECT ${p}, alloc.base + c.k, c.client_id, c.client_commit_id, c.actor_id, c.created_at_ms, ${bindParam(statement, writerVersion)}
|
|
866
|
+
FROM alloc CROSS JOIN jsonb_to_recordset(${bindParam(statement, JSON.stringify(appends.map(({ commit }, index) => ({
|
|
867
|
+
k: index + 1,
|
|
868
|
+
client_id: commit.clientId,
|
|
869
|
+
client_commit_id: commit.clientCommitId,
|
|
870
|
+
actor_id: commit.actorId,
|
|
871
|
+
created_at_ms: commit.createdAtMs,
|
|
872
|
+
}))))}::text::jsonb)
|
|
873
|
+
AS c(k int, client_id text, client_commit_id text, actor_id text, created_at_ms bigint))`);
|
|
874
|
+
if (changes.length > 0) {
|
|
875
|
+
statement.ctes.push(`changed AS (INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
|
|
876
|
+
SELECT ${p}, alloc.base + c.k, c.idx, c.tbl, c.row_id, c.op, c.version, c.scopes, decode(c.payload, 'base64')
|
|
877
|
+
FROM alloc CROSS JOIN jsonb_to_recordset(${bindParam(statement, JSON.stringify(changes))}::text::jsonb)
|
|
878
|
+
AS c(k int, idx int, tbl text, row_id text, op smallint, version bigint, scopes jsonb, payload text)
|
|
879
|
+
RETURNING commit_seq, tbl, scopes)`, `change_scopes AS (INSERT INTO sync_change_scopes(partition, tbl, var, value, commit_seq)
|
|
880
|
+
SELECT ${p}, changed.tbl, scope.key, scope.value, changed.commit_seq
|
|
881
|
+
FROM changed CROSS JOIN LATERAL jsonb_each_text(changed.scopes) AS scope
|
|
882
|
+
ON CONFLICT DO NOTHING)`);
|
|
883
|
+
}
|
|
884
|
+
if (tombstones.size > 0) {
|
|
885
|
+
statement.ctes.push(`tombstoned AS (INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq, server_version, scopes)
|
|
886
|
+
SELECT ${p}, t.tbl, t.row_id, alloc.base + t.k, t.version, t.scopes
|
|
887
|
+
FROM alloc CROSS JOIN jsonb_to_recordset(${bindParam(statement, JSON.stringify([...tombstones.values()]))}::text::jsonb)
|
|
888
|
+
AS t(k int, tbl text, row_id text, version bigint, scopes jsonb)
|
|
889
|
+
ON CONFLICT (partition, tbl, row_id)
|
|
890
|
+
DO UPDATE SET commit_seq=excluded.commit_seq,
|
|
891
|
+
server_version=excluded.server_version,
|
|
892
|
+
scopes=excluded.scopes)`);
|
|
893
|
+
}
|
|
894
|
+
}
|
|
895
|
+
/**
|
|
896
|
+
* The statements that write staged rows, blob references, commit appends
|
|
897
|
+
* and push results. Everything shares the first statement while the bound
|
|
898
|
+
* parameters fit; a large row set continues in further statements of the
|
|
899
|
+
* same shape.
|
|
900
|
+
*/
|
|
901
|
+
function writeStatements(partition, writerVersion, resolveTable, rows, blobRefs, appends, results) {
|
|
902
|
+
let current = { ctes: [], params: [] };
|
|
903
|
+
const statements = [current];
|
|
904
|
+
addAppends(current, partition, writerVersion, appends);
|
|
905
|
+
if (results.length > 0) {
|
|
906
|
+
current.ctes.push(`results AS (INSERT INTO sync_push_results(partition, client_id, client_commit_id, result, outbox_id, client_seq)
|
|
907
|
+
SELECT ${bindParam(current, partition)}, r.client_id, r.client_commit_id, r.result, r.outbox_id, r.client_seq
|
|
908
|
+
FROM jsonb_to_recordset(${bindParam(current, JSON.stringify(results))}::text::jsonb)
|
|
909
|
+
AS r(client_id text, client_commit_id text, result jsonb, outbox_id text, client_seq bigint)
|
|
910
|
+
ON CONFLICT (partition, client_id, client_commit_id) DO NOTHING)`);
|
|
911
|
+
}
|
|
912
|
+
for (const table of new Set([...rows.keys(), ...blobRefs.keys()])) {
|
|
913
|
+
const compiled = resolveTable(table);
|
|
914
|
+
const width = tableColumnNames(compiled).length;
|
|
915
|
+
const chunk = Math.max(1, Math.floor((PG_PARAMS_PER_STATEMENT - 4) / width));
|
|
916
|
+
const pending = new Map(blobRefs.get(table));
|
|
917
|
+
const staged = [...(rows.get(table)?.values() ?? [])];
|
|
918
|
+
for (let first = 0; first < staged.length; first += chunk) {
|
|
919
|
+
const slice = staged.slice(first, first + chunk);
|
|
920
|
+
if (current.params.length + slice.length * width + 4 >
|
|
921
|
+
PG_PARAMS_PER_STATEMENT) {
|
|
922
|
+
current = { ctes: [], params: [] };
|
|
923
|
+
statements.push(current);
|
|
924
|
+
}
|
|
925
|
+
const refs = [];
|
|
926
|
+
for (const { row } of slice) {
|
|
927
|
+
const blobIds = pending.get(row.rowId);
|
|
928
|
+
if (blobIds === undefined)
|
|
929
|
+
continue;
|
|
930
|
+
pending.delete(row.rowId);
|
|
931
|
+
refs.push({ row_id: row.rowId, blob_ids: blobIds });
|
|
932
|
+
}
|
|
933
|
+
addRowWrites(current, compiled, partition, slice, refs);
|
|
934
|
+
}
|
|
935
|
+
if (pending.size > 0) {
|
|
936
|
+
// Refs staged for a row written by an earlier statement.
|
|
937
|
+
addBlobRefWrites(current, bindParam(current, partition), bindParam(current, table), [...pending].map(([row_id, blob_ids]) => ({ row_id, blob_ids })));
|
|
938
|
+
}
|
|
939
|
+
}
|
|
940
|
+
return statements.filter((statement) => statement.ctes.length > 0);
|
|
809
941
|
}
|
|
810
942
|
async function getPushResultOn(q, partition, clientId, clientCommitId) {
|
|
811
943
|
const { rows } = await q.query('SELECT result FROM sync_push_results WHERE partition=$1 AND client_id=$2 AND client_commit_id=$3', [partition, clientId, clientCommitId]);
|
|
@@ -835,7 +967,7 @@ async function queryAuthoritativeOn(client, partition, prepared) {
|
|
|
835
967
|
}
|
|
836
968
|
class PostgresTransaction {
|
|
837
969
|
#client;
|
|
838
|
-
/** The FIFO without the
|
|
970
|
+
/** The FIFO without the pending-write prelude (see the constructor). */
|
|
839
971
|
#serial;
|
|
840
972
|
#partition;
|
|
841
973
|
/** Storage-owned relation rewrite, bound to this transaction's partition. */
|
|
@@ -846,13 +978,41 @@ class PostgresTransaction {
|
|
|
846
978
|
/** The storage's `chooseScanVariable` picks. */
|
|
847
979
|
#scanVariables;
|
|
848
980
|
#open = true;
|
|
981
|
+
/** Set by `preparePushApply`: commits close with a result and roll back alone. */
|
|
982
|
+
#pushApply = false;
|
|
983
|
+
/** Set by `preparePushApply` on the rerun after a closed commit's write failed. */
|
|
984
|
+
#closeEagerly = false;
|
|
849
985
|
/** True once the request's current commit has written behind its savepoint. */
|
|
850
986
|
#pushApplySavepoint = false;
|
|
851
987
|
/**
|
|
852
|
-
* The
|
|
853
|
-
*
|
|
854
|
-
* the transaction ends.
|
|
988
|
+
* The `commitSeq` the next appended commit takes. The partition lock read
|
|
989
|
+
* sets it and freezes the cursor, so an append needs no allocation read.
|
|
855
990
|
*/
|
|
991
|
+
#nextCommitSeq;
|
|
992
|
+
/** The `commitSeq` the open push commit appended, until it closes. */
|
|
993
|
+
#openCommitSeq;
|
|
994
|
+
/**
|
|
995
|
+
* The open commit's upserts staged per table name (row id → row and its
|
|
996
|
+
* push operation index), its §5.9.4 reference sets, and its commit append.
|
|
997
|
+
* They reach the database with the next statement that needs them, behind
|
|
998
|
+
* the commit's savepoint, as one statement: a commit's rows cost one round
|
|
999
|
+
* trip per statement instead of one per row. Tables with a unique index
|
|
1000
|
+
* write immediately: a violation must name its operation.
|
|
1001
|
+
*/
|
|
1002
|
+
#staged = new Map();
|
|
1003
|
+
#stagedBlobRefs = new Map();
|
|
1004
|
+
#stagedAppend;
|
|
1005
|
+
/**
|
|
1006
|
+
* The writes of push commits that closed and are not yet written: rows (a
|
|
1007
|
+
* later commit's row replaces an earlier one), reference sets, appends in
|
|
1008
|
+
* `commitSeq` order, and the §2.3 results. They reach the database in one
|
|
1009
|
+
* statement ahead of the next statement of the transaction, outside any
|
|
1010
|
+
* commit's savepoint; a request whose commits never read their own writes
|
|
1011
|
+
* writes them all in one statement before COMMIT (§6.4).
|
|
1012
|
+
*/
|
|
1013
|
+
#closedRows = new Map();
|
|
1014
|
+
#closedBlobRefs = new Map();
|
|
1015
|
+
#closedAppends = [];
|
|
856
1016
|
#pushResults = [];
|
|
857
1017
|
/**
|
|
858
1018
|
* `prefetchRows` results keyed `table\u0000rowId`; `null` records a row
|
|
@@ -861,29 +1021,6 @@ class PostgresTransaction {
|
|
|
861
1021
|
*/
|
|
862
1022
|
#prefetchedRows = new Map();
|
|
863
1023
|
#prefetchedTombstones = new Map();
|
|
864
|
-
/**
|
|
865
|
-
* Upserts staged per table (row id → row and its push operation index).
|
|
866
|
-
* Every statement of this transaction writes them first (see the
|
|
867
|
-
* constructor), so reads and later writes observe them; a commit's rows
|
|
868
|
-
* reach the database as one statement per table instead of one per row.
|
|
869
|
-
* Tables with a unique index write immediately: a violation must name its
|
|
870
|
-
* operation.
|
|
871
|
-
*/
|
|
872
|
-
#staged = new Map();
|
|
873
|
-
/**
|
|
874
|
-
* §5.9.4 reference sets staged per table (row id → blob ids). They flush
|
|
875
|
-
* with the table's row statement, so a blob-referencing row costs no extra
|
|
876
|
-
* round trip; a ref set staged for an already-written row flushes on the
|
|
877
|
-
* next statement of its table.
|
|
878
|
-
*/
|
|
879
|
-
#stagedBlobRefs = new Map();
|
|
880
|
-
/**
|
|
881
|
-
* The partition cursor (`max_commit_seq`) read by the first registered
|
|
882
|
-
* query in this transaction. The partition lock freezes it, so later
|
|
883
|
-
* registered queries in the same transaction reuse the value instead of
|
|
884
|
-
* paying a round trip each.
|
|
885
|
-
*/
|
|
886
|
-
#cursorSeq;
|
|
887
1024
|
/** Resolves/rejects the `transaction(fn)` wrapper (see `begin`). */
|
|
888
1025
|
#resolve;
|
|
889
1026
|
#reject;
|
|
@@ -896,30 +1033,15 @@ class PostgresTransaction {
|
|
|
896
1033
|
// issued through this transaction is chained on one FIFO — the same
|
|
897
1034
|
// shape as `SqliteServerStorage.queryAuthoritative`.
|
|
898
1035
|
let tail = Promise.resolve();
|
|
899
|
-
|
|
1036
|
+
this.#serial = {
|
|
900
1037
|
query: (text, params) => {
|
|
901
1038
|
const run = tail.then(() => client.query(text, params));
|
|
902
1039
|
tail = run.then(() => undefined, () => undefined);
|
|
903
1040
|
return run;
|
|
904
1041
|
},
|
|
905
1042
|
};
|
|
906
|
-
this.#serial = serial;
|
|
907
1043
|
this.#client = {
|
|
908
|
-
query:
|
|
909
|
-
// Queue the staged writes ahead of this statement synchronously, so
|
|
910
|
-
// a concurrently issued statement cannot overtake them.
|
|
911
|
-
const staged = this.#writeStaged();
|
|
912
|
-
const run = serial.query(text, params);
|
|
913
|
-
try {
|
|
914
|
-
await staged;
|
|
915
|
-
}
|
|
916
|
-
catch (error) {
|
|
917
|
-
// The aborted transaction fails the queued statement too.
|
|
918
|
-
run.catch(() => undefined);
|
|
919
|
-
throw error;
|
|
920
|
-
}
|
|
921
|
-
return run;
|
|
922
|
-
},
|
|
1044
|
+
query: (text, params) => this.#statement(text, params, 'read'),
|
|
923
1045
|
};
|
|
924
1046
|
this.#partition = partition;
|
|
925
1047
|
this.#writerVersion = writerVersion;
|
|
@@ -932,83 +1054,145 @@ class PostgresTransaction {
|
|
|
932
1054
|
if (!this.#open)
|
|
933
1055
|
throw new Error('transaction already finished');
|
|
934
1056
|
}
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
1057
|
+
/**
|
|
1058
|
+
* Queue the pending writes and then one statement. `write` marks a
|
|
1059
|
+
* candidate write of the open commit, which needs the commit's savepoint
|
|
1060
|
+
* even when nothing is staged.
|
|
1061
|
+
*/
|
|
1062
|
+
async #statement(text, params, mode) {
|
|
1063
|
+
// Queue the pending writes ahead of this statement synchronously, so a
|
|
1064
|
+
// concurrently issued statement cannot overtake them.
|
|
1065
|
+
const pending = this.#flush(mode);
|
|
1066
|
+
const run = this.#serial.query(text, params);
|
|
1067
|
+
try {
|
|
1068
|
+
await pending;
|
|
938
1069
|
}
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
const batches = [...tables].map((table) => {
|
|
944
|
-
const compiled = this.#resolveTable(table);
|
|
945
|
-
const rows = [...(this.#staged.get(compiled)?.values() ?? [])];
|
|
946
|
-
const blobRefs = [
|
|
947
|
-
...(this.#stagedBlobRefs.get(table)?.entries() ?? []),
|
|
948
|
-
].map(([rowId, blobIds]) => ({ rowId, blobIds }));
|
|
949
|
-
this.#staged.delete(compiled);
|
|
950
|
-
this.#stagedBlobRefs.delete(table);
|
|
951
|
-
return { table, compiled, rows, blobRefs };
|
|
952
|
-
});
|
|
953
|
-
const writes = [];
|
|
954
|
-
// The candidate savepoint lands before the first candidate write, and
|
|
955
|
-
// only when a write actually reaches the database: a commit refused
|
|
956
|
-
// before its first write has no candidate rows to roll back, so its
|
|
957
|
-
// rejection finalization skips the savepoint round trips.
|
|
958
|
-
const savepoint = this.#ensurePushSavepoint();
|
|
959
|
-
if (savepoint !== undefined)
|
|
960
|
-
writes.push(savepoint);
|
|
961
|
-
for (const { compiled, rows, blobRefs } of batches) {
|
|
962
|
-
writes.push(writeRowsOn(this.#serial, compiled, this.#partition, rows, blobRefs).catch((error) => {
|
|
963
|
-
// Staged tables have no unique index, so a violation is a column
|
|
964
|
-
// constraint; it names the first staged operation of the table.
|
|
965
|
-
if (isPostgresConstraintError(error)) {
|
|
966
|
-
throw new StorageConstraintError(error, rows[0]?.opIndex);
|
|
967
|
-
}
|
|
968
|
-
throw error;
|
|
969
|
-
}));
|
|
1070
|
+
catch (error) {
|
|
1071
|
+
// The aborted transaction fails the queued statement too.
|
|
1072
|
+
run.catch(() => undefined);
|
|
1073
|
+
throw error;
|
|
970
1074
|
}
|
|
971
|
-
return
|
|
1075
|
+
return run;
|
|
972
1076
|
}
|
|
973
1077
|
/**
|
|
974
|
-
*
|
|
975
|
-
*
|
|
976
|
-
* refused commit that wrote nothing
|
|
977
|
-
*
|
|
978
|
-
*
|
|
979
|
-
*
|
|
1078
|
+
* Queue every pending write, in order: the closed commits' writes, then
|
|
1079
|
+
* the open commit's savepoint (taken before its first write reaches the
|
|
1080
|
+
* database, so a refused commit that wrote nothing pays for none) and its
|
|
1081
|
+
* staged writes. `final` is the flush before COMMIT, which no rejection
|
|
1082
|
+
* follows.
|
|
1083
|
+
*
|
|
1084
|
+
* A relational constraint that fails on a closed commit's writes can no
|
|
1085
|
+
* longer reject that commit alone: the transaction is aborted, and
|
|
1086
|
+
* `ClosedCommitConstraintError` tells the push layer to rerun the request
|
|
1087
|
+
* with each commit written before it closes (§6.4).
|
|
1088
|
+
*
|
|
1089
|
+
* Resolves to the partition cursor the open commit's append returned.
|
|
980
1090
|
*/
|
|
981
|
-
#
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
this.#
|
|
985
|
-
|
|
1091
|
+
async #flush(mode) {
|
|
1092
|
+
const pending = [];
|
|
1093
|
+
let cursor = Promise.resolve(undefined);
|
|
1094
|
+
if (this.#closedRows.size > 0 ||
|
|
1095
|
+
this.#closedBlobRefs.size > 0 ||
|
|
1096
|
+
this.#closedAppends.length > 0 ||
|
|
1097
|
+
this.#pushResults.length > 0) {
|
|
1098
|
+
const statements = writeStatements(this.#partition, this.#writerVersion, this.#resolveTable, this.#closedRows, this.#closedBlobRefs, this.#closedAppends, this.#pushResults);
|
|
1099
|
+
this.#closedRows.clear();
|
|
1100
|
+
this.#closedBlobRefs.clear();
|
|
1101
|
+
this.#closedAppends.length = 0;
|
|
1102
|
+
this.#pushResults.length = 0;
|
|
1103
|
+
pending.push(this.#runWrites(statements, (error) => isPostgresConstraintError(error)
|
|
1104
|
+
? new ClosedCommitConstraintError(error)
|
|
1105
|
+
: error));
|
|
1106
|
+
}
|
|
1107
|
+
const staged = this.#staged.size > 0 ||
|
|
1108
|
+
this.#stagedBlobRefs.size > 0 ||
|
|
1109
|
+
this.#stagedAppend !== undefined;
|
|
1110
|
+
if (mode !== 'final' &&
|
|
1111
|
+
this.#pushApply &&
|
|
1112
|
+
!this.#pushApplySavepoint &&
|
|
1113
|
+
(staged || mode === 'write')) {
|
|
1114
|
+
// Each commit takes its own savepoint under the same name: `ROLLBACK
|
|
1115
|
+
// TO` targets the newest, and an earlier commit's savepoint stays open
|
|
1116
|
+
// until the transaction ends, so no `RELEASE` round trip is needed.
|
|
1117
|
+
this.#pushApplySavepoint = true;
|
|
1118
|
+
pending.push(this.#serial.query('SAVEPOINT syncular_push_candidate'));
|
|
1119
|
+
}
|
|
1120
|
+
if (staged) {
|
|
1121
|
+
const statements = writeStatements(this.#partition, this.#writerVersion, this.#resolveTable, this.#staged, this.#stagedBlobRefs, this.#stagedAppend === undefined ? [] : [this.#stagedAppend], []);
|
|
1122
|
+
this.#staged.clear();
|
|
1123
|
+
this.#stagedBlobRefs.clear();
|
|
1124
|
+
this.#stagedAppend = undefined;
|
|
1125
|
+
cursor = this.#runWrites(statements, (error, statement) => isPostgresConstraintError(error)
|
|
1126
|
+
? new StorageConstraintError(error, statement.opIndex)
|
|
1127
|
+
: error);
|
|
1128
|
+
pending.push(cursor);
|
|
1129
|
+
}
|
|
1130
|
+
await Promise.all(pending);
|
|
1131
|
+
return cursor;
|
|
986
1132
|
}
|
|
1133
|
+
/**
|
|
1134
|
+
* Queue write statements on the FIFO. Resolves to the cursor a commit
|
|
1135
|
+
* append returned, checked against the sequences computed under the lock.
|
|
1136
|
+
*/
|
|
1137
|
+
async #runWrites(statements, mapError) {
|
|
1138
|
+
const cursors = await Promise.all(statements.map((statement) => this.#serial
|
|
1139
|
+
.query(`WITH ${statement.ctes.join(',\n')} SELECT ${statement.appends === true
|
|
1140
|
+
? '(SELECT max_commit_seq FROM alloc)'
|
|
1141
|
+
: 'NULL::bigint'} AS allocated`, statement.params)
|
|
1142
|
+
.then(({ rows }) => {
|
|
1143
|
+
if (statement.appends !== true)
|
|
1144
|
+
return undefined;
|
|
1145
|
+
const allocated = asNumber(rows[0]?.allocated);
|
|
1146
|
+
if (statement.allocates !== undefined &&
|
|
1147
|
+
allocated !== statement.allocates) {
|
|
1148
|
+
throw new Error('partition commit sequence moved under the partition lock');
|
|
1149
|
+
}
|
|
1150
|
+
return allocated;
|
|
1151
|
+
}, (error) => {
|
|
1152
|
+
throw mapError(error, statement);
|
|
1153
|
+
})));
|
|
1154
|
+
return cursors.find((cursor) => cursor !== undefined);
|
|
1155
|
+
}
|
|
1156
|
+
/** The candidate row this transaction staged and has not yet written. */
|
|
1157
|
+
#stagedRow(table, rowId) {
|
|
1158
|
+
return (this.#staged.get(table)?.get(rowId)?.row ??
|
|
1159
|
+
this.#closedRows.get(table)?.get(rowId)?.row);
|
|
1160
|
+
}
|
|
1161
|
+
/**
|
|
1162
|
+
* A read by primary key. A staged row answers from memory, so the read
|
|
1163
|
+
* needs no pending write: it runs without flushing them, and a validator
|
|
1164
|
+
* that reads rows by id adds no savepoint or write round trip (§6.4).
|
|
1165
|
+
*/
|
|
987
1166
|
getRow(table, rowId) {
|
|
988
1167
|
this.#assertOpen();
|
|
1168
|
+
const staged = this.#stagedRow(table, rowId);
|
|
1169
|
+
if (staged !== undefined)
|
|
1170
|
+
return Promise.resolve(staged);
|
|
989
1171
|
const prefetched = this.#prefetchedRows.get(`${table}\u0000${rowId}`);
|
|
990
1172
|
if (prefetched !== undefined) {
|
|
991
1173
|
return Promise.resolve(prefetched ?? undefined);
|
|
992
1174
|
}
|
|
993
|
-
return getRowOn(this.#
|
|
1175
|
+
return getRowOn(this.#serial, this.#resolveTable(table), this.#partition, rowId);
|
|
994
1176
|
}
|
|
995
1177
|
async getRows(table, rowIds) {
|
|
996
1178
|
this.#assertOpen();
|
|
997
1179
|
const rows = new Map();
|
|
998
1180
|
const missing = [];
|
|
999
1181
|
for (const rowId of new Set(rowIds)) {
|
|
1182
|
+
const staged = this.#stagedRow(table, rowId);
|
|
1000
1183
|
const prefetched = this.#prefetchedRows.get(`${table}\u0000${rowId}`);
|
|
1001
|
-
if (
|
|
1184
|
+
if (staged !== undefined)
|
|
1185
|
+
rows.set(rowId, staged);
|
|
1186
|
+
else if (prefetched === undefined)
|
|
1002
1187
|
missing.push(rowId);
|
|
1003
1188
|
else if (prefetched !== null)
|
|
1004
1189
|
rows.set(rowId, prefetched);
|
|
1005
1190
|
}
|
|
1006
1191
|
if (missing.length === 0)
|
|
1007
1192
|
return rows;
|
|
1008
|
-
// One statement for every id the
|
|
1009
|
-
//
|
|
1010
|
-
|
|
1011
|
-
const { rows: records } = await this.#client.query(selectRowsSql(this.#resolveTable(table), 'postgres'), [this.#partition, JSON.stringify(missing)]);
|
|
1193
|
+
// One statement for every id neither the staged writes nor the
|
|
1194
|
+
// admission read answered (see `getRow`).
|
|
1195
|
+
const { rows: records } = await this.#serial.query(selectRowsSql(this.#resolveTable(table), 'postgres'), [this.#partition, JSON.stringify(missing)]);
|
|
1012
1196
|
for (const record of records)
|
|
1013
1197
|
rows.set(record.row_id, toStoredRow(record));
|
|
1014
1198
|
return rows;
|
|
@@ -1018,7 +1202,8 @@ class PostgresTransaction {
|
|
|
1018
1202
|
const tables = [];
|
|
1019
1203
|
const specs = [];
|
|
1020
1204
|
for (const target of targets) {
|
|
1021
|
-
const ids = [...new Set(target.rowIds)].filter((rowId) => !this.#prefetchedRows.has(`${target.table}\u0000${rowId}`)
|
|
1205
|
+
const ids = [...new Set(target.rowIds)].filter((rowId) => !this.#prefetchedRows.has(`${target.table}\u0000${rowId}`) &&
|
|
1206
|
+
this.#stagedRow(target.table, rowId) === undefined);
|
|
1022
1207
|
if (ids.length === 0)
|
|
1023
1208
|
continue;
|
|
1024
1209
|
tables.push(this.#resolveTable(target.table));
|
|
@@ -1052,8 +1237,7 @@ class PostgresTransaction {
|
|
|
1052
1237
|
async clearTombstone(table, rowId) {
|
|
1053
1238
|
this.#assertOpen();
|
|
1054
1239
|
this.#prefetchedTombstones.delete(`${table}\u0000${rowId}`);
|
|
1055
|
-
await this.#
|
|
1056
|
-
await this.#client.query('DELETE FROM sync_tombstones WHERE partition=$1 AND tbl=$2 AND row_id=$3', [this.#partition, table, rowId]);
|
|
1240
|
+
await this.#statement('DELETE FROM sync_tombstones WHERE partition=$1 AND tbl=$2 AND row_id=$3', [this.#partition, table, rowId], 'write');
|
|
1057
1241
|
}
|
|
1058
1242
|
async scanRows(query) {
|
|
1059
1243
|
this.#assertOpen();
|
|
@@ -1107,28 +1291,28 @@ class PostgresTransaction {
|
|
|
1107
1291
|
// staged row and never waits for a second pool connection.
|
|
1108
1292
|
const prepared = this.#prepareQuery(query);
|
|
1109
1293
|
const result = await this.#client.query(postgresPlaceholders(prepared.sql), prepared.params);
|
|
1110
|
-
//
|
|
1111
|
-
//
|
|
1112
|
-
if (this.#
|
|
1113
|
-
|
|
1114
|
-
this.#cursorSeq =
|
|
1115
|
-
cursor.rows[0] === undefined
|
|
1116
|
-
? 0
|
|
1117
|
-
: asNumber(cursor.rows[0].max_commit_seq);
|
|
1294
|
+
// Under the partition lock the cursor is the locked `max_commit_seq` plus
|
|
1295
|
+
// this transaction's appends, which the query above flushed: no read.
|
|
1296
|
+
if (this.#nextCommitSeq !== undefined) {
|
|
1297
|
+
return { rows: result.rows, maxCommitSeq: this.#nextCommitSeq - 1 };
|
|
1118
1298
|
}
|
|
1119
|
-
|
|
1299
|
+
const cursor = await this.#client.query('SELECT max_commit_seq FROM sync_partitions WHERE partition=$1', [this.#partition]);
|
|
1300
|
+
// asNumber maps the absent row of a partition without a commit to 0.
|
|
1301
|
+
return {
|
|
1302
|
+
rows: result.rows,
|
|
1303
|
+
maxCommitSeq: asNumber(cursor.rows[0]?.max_commit_seq),
|
|
1304
|
+
};
|
|
1120
1305
|
}
|
|
1121
|
-
async preparePushApply(clientId, commits, runningSchemaVersion, outbox) {
|
|
1306
|
+
async preparePushApply(clientId, commits, runningSchemaVersion, outbox, options) {
|
|
1122
1307
|
this.#assertOpen();
|
|
1308
|
+
this.#pushApply = true;
|
|
1309
|
+
this.#closeEagerly = options?.closeEagerly === true;
|
|
1123
1310
|
// The lock is its own statement: a READ COMMITTED statement's snapshot is
|
|
1124
1311
|
// fixed at its start, so the admission read must run *after* a duplicate's
|
|
1125
1312
|
// commit has released the partition lock to observe it. Folding the lock
|
|
1126
1313
|
// and the read into one statement would miss a duplicate that committed
|
|
1127
1314
|
// while this transaction waited.
|
|
1128
1315
|
await this.lockPartition();
|
|
1129
|
-
const watermark = outbox === undefined
|
|
1130
|
-
? undefined
|
|
1131
|
-
: await this.#settleOutbox(clientId, outbox);
|
|
1132
1316
|
const ids = new Map();
|
|
1133
1317
|
for (const commit of commits) {
|
|
1134
1318
|
for (const target of commit.targets ?? []) {
|
|
@@ -1144,68 +1328,25 @@ class PostgresTransaction {
|
|
|
1144
1328
|
}
|
|
1145
1329
|
const tables = [...ids.keys()].map((table) => this.#resolveTable(table));
|
|
1146
1330
|
const specs = [...ids].map(([tbl, rowIds]) => ({ tbl, ids: [...rowIds] }));
|
|
1147
|
-
// One statement for the whole request: the
|
|
1148
|
-
// idempotency record, and the read-ahead of every
|
|
1149
|
-
|
|
1331
|
+
// One statement for the whole request: the outbox settlement, the serve
|
|
1332
|
+
// gate, every commit's idempotency record, and the read-ahead of every
|
|
1333
|
+
// commit's known rows. The settlement precedes every commit's savepoint,
|
|
1334
|
+
// so no rejection discards it.
|
|
1335
|
+
const admission = await readAdmissionOn(this.#client, this.#partition, clientId, commits.map((commit) => commit.clientCommitId), runningSchemaVersion, specs.length === 0 ? undefined : { tables, specs }, outbox);
|
|
1150
1336
|
this.#recordPrefetched(admission.prefetched);
|
|
1151
1337
|
return {
|
|
1152
1338
|
gate: admission.gate,
|
|
1153
1339
|
persisted: (clientCommitId) => toPersistedPushResult(admission.persisted.get(clientCommitId)),
|
|
1154
1340
|
deleteFloor: admission.deleteFloor,
|
|
1155
|
-
...(watermark === undefined
|
|
1341
|
+
...(admission.watermark === undefined
|
|
1342
|
+
? {}
|
|
1343
|
+
: { watermark: admission.watermark }),
|
|
1156
1344
|
};
|
|
1157
1345
|
}
|
|
1158
|
-
/**
|
|
1159
|
-
* The §9 settlement and §6.3.2 steps 1 and 2 under the partition lock,
|
|
1160
|
-
* before the first commit's savepoint, so no rejection discards them. Two
|
|
1161
|
-
* statements: the settlement touches disjoint rows (outside and inside the
|
|
1162
|
-
* request), and the watermark delete must see its sequences.
|
|
1163
|
-
*/
|
|
1164
|
-
async #settleOutbox(clientId, outbox) {
|
|
1165
|
-
const first = outbox.commits[0];
|
|
1166
|
-
const last = outbox.commits.at(-1);
|
|
1167
|
-
if (first === undefined || last === undefined) {
|
|
1168
|
-
throw new Error('outbox settlement needs the request commits');
|
|
1169
|
-
}
|
|
1170
|
-
const commits = JSON.stringify(outbox.commits.map((commit) => ({
|
|
1171
|
-
id: commit.clientCommitId,
|
|
1172
|
-
seq: commit.clientSeq,
|
|
1173
|
-
})));
|
|
1174
|
-
// §9 (b): with no pre-upgrade result on the request's last commit, every
|
|
1175
|
-
// remaining one outside the request left the outbox; (a) assigns the
|
|
1176
|
-
// request's sequences to its pre-upgrade results.
|
|
1177
|
-
await this.#client.query(`WITH c AS (SELECT * FROM jsonb_to_recordset($4::text::jsonb) AS c(id text, seq bigint)),
|
|
1178
|
-
b AS (DELETE FROM sync_push_results
|
|
1179
|
-
WHERE partition=$1 AND client_id=$2 AND outbox_id IS NULL
|
|
1180
|
-
AND client_commit_id NOT IN (SELECT id FROM c)
|
|
1181
|
-
AND NOT EXISTS (SELECT 1 FROM sync_push_results
|
|
1182
|
-
WHERE partition=$1 AND client_id=$2 AND client_commit_id=$5
|
|
1183
|
-
AND outbox_id IS NULL)),
|
|
1184
|
-
a AS (UPDATE sync_push_results r SET outbox_id=$3, client_seq=c.seq FROM c
|
|
1185
|
-
WHERE r.partition=$1 AND r.client_id=$2 AND r.outbox_id IS NULL
|
|
1186
|
-
AND r.client_commit_id=c.id)
|
|
1187
|
-
SELECT 1`, [
|
|
1188
|
-
this.#partition,
|
|
1189
|
-
clientId,
|
|
1190
|
-
outbox.outboxId,
|
|
1191
|
-
commits,
|
|
1192
|
-
last.clientCommitId,
|
|
1193
|
-
]);
|
|
1194
|
-
const { rows } = await this.#client.query(`WITH w AS (
|
|
1195
|
-
INSERT INTO sync_push_watermarks(partition, client_id, outbox_id, through_seq)
|
|
1196
|
-
VALUES ($1,$2,$3,$4)
|
|
1197
|
-
ON CONFLICT (partition, client_id, outbox_id) DO UPDATE
|
|
1198
|
-
SET through_seq=GREATEST(sync_push_watermarks.through_seq, EXCLUDED.through_seq)
|
|
1199
|
-
RETURNING through_seq),
|
|
1200
|
-
d AS (DELETE FROM sync_push_results
|
|
1201
|
-
WHERE partition=$1 AND client_id=$2 AND outbox_id=$3
|
|
1202
|
-
AND client_seq<(SELECT through_seq FROM w))
|
|
1203
|
-
SELECT through_seq FROM w`, [this.#partition, clientId, outbox.outboxId, first.clientSeq]);
|
|
1204
|
-
return asNumber(rows[0]?.through_seq);
|
|
1205
|
-
}
|
|
1206
1346
|
async lockPartition() {
|
|
1207
1347
|
this.#assertOpen();
|
|
1208
|
-
|
|
1348
|
+
this.#nextCommitSeq =
|
|
1349
|
+
(await lockPartitionOn(this.#client, this.#partition)) + 1;
|
|
1209
1350
|
}
|
|
1210
1351
|
async pinSnapshot() {
|
|
1211
1352
|
this.#assertOpen();
|
|
@@ -1214,13 +1355,12 @@ class PostgresTransaction {
|
|
|
1214
1355
|
}
|
|
1215
1356
|
async advanceCheckpoint(name, ownerEpoch, watermark, observedRows, nowMs) {
|
|
1216
1357
|
this.#assertOpen();
|
|
1217
|
-
await this.#ensurePushSavepoint();
|
|
1218
1358
|
// `RETURNING`: `PgQueryable` reports returned rows only (no affected-row
|
|
1219
1359
|
// count), so every affected-row check counts returned rows.
|
|
1220
|
-
const result = await this.#
|
|
1360
|
+
const result = await this.#statement(`UPDATE sync_backfill_checkpoints
|
|
1221
1361
|
SET watermark=$1, observed_rows=$2, updated_at_ms=$3
|
|
1222
1362
|
WHERE partition=$4 AND name=$5 AND owner_epoch=$6
|
|
1223
|
-
RETURNING name`, [watermark, observedRows, nowMs, this.#partition, name, ownerEpoch]);
|
|
1363
|
+
RETURNING name`, [watermark, observedRows, nowMs, this.#partition, name, ownerEpoch], 'write');
|
|
1224
1364
|
return result.rows.length === 1;
|
|
1225
1365
|
}
|
|
1226
1366
|
readServeGate(runningSchemaVersion) {
|
|
@@ -1231,13 +1371,20 @@ class PostgresTransaction {
|
|
|
1231
1371
|
}
|
|
1232
1372
|
async rejectPushCommit(clientId, clientCommitId, result, sequence) {
|
|
1233
1373
|
this.#assertOpen();
|
|
1234
|
-
// Candidate
|
|
1235
|
-
// staging
|
|
1374
|
+
// Candidate writes that never reached the database are discarded with the
|
|
1375
|
+
// staging maps; a savepoint exists only when one did. It stays open: the
|
|
1236
1376
|
// rollback returns the database to where the next commit starts (the
|
|
1237
|
-
//
|
|
1377
|
+
// closed commits' writes wait in memory or precede it), so the next
|
|
1378
|
+
// commit writes behind it.
|
|
1238
1379
|
this.#staged.clear();
|
|
1239
1380
|
this.#stagedBlobRefs.clear();
|
|
1381
|
+
this.#stagedAppend = undefined;
|
|
1382
|
+
if (this.#openCommitSeq !== undefined) {
|
|
1383
|
+
this.#nextCommitSeq = this.#openCommitSeq;
|
|
1384
|
+
this.#openCommitSeq = undefined;
|
|
1385
|
+
}
|
|
1240
1386
|
if (this.#pushApplySavepoint) {
|
|
1387
|
+
this.#pushApplySavepoint = false;
|
|
1241
1388
|
await this.#serial.query('ROLLBACK TO SAVEPOINT syncular_push_candidate');
|
|
1242
1389
|
}
|
|
1243
1390
|
this.#recordPushResult(clientId, clientCommitId, result, sequence);
|
|
@@ -1257,19 +1404,20 @@ class PostgresTransaction {
|
|
|
1257
1404
|
const compiled = this.#resolveTable(table);
|
|
1258
1405
|
// Decodes the payload: an undecodable row fails here, at its operation.
|
|
1259
1406
|
const values = upsertValues(compiled, this.#partition, row, 'postgres');
|
|
1407
|
+
const entry = {
|
|
1408
|
+
row,
|
|
1409
|
+
values,
|
|
1410
|
+
...(context === undefined ? {} : { opIndex: context.opIndex }),
|
|
1411
|
+
};
|
|
1260
1412
|
if (!compiled.indexes.some((index) => index.unique === true)) {
|
|
1261
|
-
let rows = this.#staged.get(
|
|
1413
|
+
let rows = this.#staged.get(table);
|
|
1262
1414
|
if (rows === undefined) {
|
|
1263
1415
|
rows = new Map();
|
|
1264
|
-
this.#staged.set(
|
|
1416
|
+
this.#staged.set(table, rows);
|
|
1265
1417
|
}
|
|
1266
1418
|
// A later upsert of the same row in this transaction replaces it.
|
|
1267
1419
|
rows.delete(row.rowId);
|
|
1268
|
-
rows.set(row.rowId,
|
|
1269
|
-
row,
|
|
1270
|
-
values,
|
|
1271
|
-
...(context === undefined ? {} : { opIndex: context.opIndex }),
|
|
1272
|
-
});
|
|
1420
|
+
rows.set(row.rowId, entry);
|
|
1273
1421
|
return;
|
|
1274
1422
|
}
|
|
1275
1423
|
const refs = this.#stagedBlobRefs.get(table);
|
|
@@ -1279,9 +1427,13 @@ class PostgresTransaction {
|
|
|
1279
1427
|
if (refs.size === 0)
|
|
1280
1428
|
this.#stagedBlobRefs.delete(table);
|
|
1281
1429
|
}
|
|
1282
|
-
|
|
1430
|
+
const statements = writeStatements(this.#partition, this.#writerVersion, this.#resolveTable, new Map([[table, new Map([[row.rowId, entry]])]]), blobIds === undefined
|
|
1431
|
+
? new Map()
|
|
1432
|
+
: new Map([[table, new Map([[row.rowId, blobIds]])]]), [], []);
|
|
1283
1433
|
try {
|
|
1284
|
-
|
|
1434
|
+
for (const statement of statements) {
|
|
1435
|
+
await this.#statement(`WITH ${statement.ctes.join(',\n')} SELECT 1`, statement.params, 'write');
|
|
1436
|
+
}
|
|
1285
1437
|
}
|
|
1286
1438
|
catch (error) {
|
|
1287
1439
|
if (isPostgresConstraintError(error)) {
|
|
@@ -1297,14 +1449,13 @@ class PostgresTransaction {
|
|
|
1297
1449
|
refs?.delete(rowId);
|
|
1298
1450
|
if (refs?.size === 0)
|
|
1299
1451
|
this.#stagedBlobRefs.delete(table);
|
|
1300
|
-
await this.#ensurePushSavepoint();
|
|
1301
1452
|
// One statement: the row, its scope-index entries, and (§5.9.4: a
|
|
1302
1453
|
// deleted row references no blobs) its blob references.
|
|
1303
|
-
await this.#
|
|
1454
|
+
await this.#statement(`WITH deleted AS (${deleteRowSql(this.#resolveTable(table), 'postgres')}),
|
|
1304
1455
|
unscoped AS (
|
|
1305
1456
|
DELETE FROM sync_row_scopes WHERE partition=$1 AND tbl=$3 AND row_id=$2
|
|
1306
1457
|
)
|
|
1307
|
-
DELETE FROM sync_blob_refs WHERE partition=$1 AND tbl=$3 AND row_id=$2`, [this.#partition, rowId, table]);
|
|
1458
|
+
DELETE FROM sync_blob_refs WHERE partition=$1 AND tbl=$3 AND row_id=$2`, [this.#partition, rowId, table], 'write');
|
|
1308
1459
|
}
|
|
1309
1460
|
setBlobRefs(table, rowId, blobIds) {
|
|
1310
1461
|
this.#assertOpen();
|
|
@@ -1318,143 +1469,85 @@ class PostgresTransaction {
|
|
|
1318
1469
|
rows.set(rowId, blobIds);
|
|
1319
1470
|
return Promise.resolve();
|
|
1320
1471
|
}
|
|
1472
|
+
/**
|
|
1473
|
+
* Stage the append with the commit's rows. Under the partition lock the
|
|
1474
|
+
* cursor is frozen, so the next `commitSeq` is known without a statement
|
|
1475
|
+
* and a push commit's append waits for the next statement. Outside a push
|
|
1476
|
+
* apply the append is written at once (its allocation takes the lock when
|
|
1477
|
+
* no earlier statement did), so a relational constraint on the staged rows
|
|
1478
|
+
* fails this call.
|
|
1479
|
+
*/
|
|
1321
1480
|
async appendCommit(commit) {
|
|
1322
1481
|
this.#assertOpen();
|
|
1323
|
-
|
|
1482
|
+
if (this.#stagedAppend !== undefined || this.#openCommitSeq !== undefined) {
|
|
1483
|
+
throw new Error('the open commit already appended');
|
|
1484
|
+
}
|
|
1485
|
+
// The commit writes its delete tombstones; every other prefetched
|
|
1324
1486
|
// tombstone still holds for the request's later commits.
|
|
1325
1487
|
for (const change of commit.changes) {
|
|
1326
1488
|
if (change.op === 'delete') {
|
|
1327
1489
|
this.#prefetchedTombstones.delete(`${change.table}\u0000${change.rowId}`);
|
|
1328
1490
|
}
|
|
1329
1491
|
}
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1336
|
-
|
|
1337
|
-
|
|
1338
|
-
|
|
1339
|
-
const allocate = `INSERT INTO sync_partitions(partition, max_commit_seq, delete_floor) VALUES ($1, 1, $7)
|
|
1340
|
-
ON CONFLICT (partition) DO UPDATE
|
|
1341
|
-
SET max_commit_seq = sync_partitions.max_commit_seq + 1,
|
|
1342
|
-
delete_floor = GREATEST(sync_partitions.delete_floor, EXCLUDED.delete_floor)
|
|
1343
|
-
RETURNING max_commit_seq`;
|
|
1344
|
-
const commitRow = `INSERT INTO sync_commits(partition, commit_seq, client_id, client_commit_id, actor_id, created_at_ms, writer_version)
|
|
1345
|
-
SELECT $1, allocated.max_commit_seq, $2, $3, $4, $5, $6 FROM allocated
|
|
1346
|
-
RETURNING commit_seq`;
|
|
1347
|
-
const head = [
|
|
1348
|
-
p,
|
|
1349
|
-
commit.clientId,
|
|
1350
|
-
commit.clientCommitId,
|
|
1351
|
-
commit.actorId,
|
|
1352
|
-
commit.createdAtMs,
|
|
1353
|
-
this.#writerVersion,
|
|
1354
|
-
commitDeleteFloor(commit.changes),
|
|
1355
|
-
];
|
|
1356
|
-
if (commit.changes.length === 0) {
|
|
1357
|
-
const { rows } = await q.query(`WITH allocated AS (${allocate}) ${commitRow}`, head);
|
|
1358
|
-
return asNumber(rows[0]?.commit_seq);
|
|
1359
|
-
}
|
|
1360
|
-
// §2.2: a delete whose commit follows it with an upsert is a scope move.
|
|
1361
|
-
// The row is live, so the tombstone insert below skips it.
|
|
1362
|
-
const moved = JSON.stringify([...rowsLiveAfterDelete(commit.changes)].map((key) => {
|
|
1363
|
-
const at = key.indexOf('\u0000');
|
|
1364
|
-
return { tbl: key.slice(0, at), row_id: key.slice(at + 1) };
|
|
1365
|
-
}));
|
|
1366
|
-
// One writable-CTE chain per chunk of changes: the changes, their
|
|
1367
|
-
// inverted scope entries, and the §5 delete tombstones. A statement binds
|
|
1368
|
-
// at most a 16-bit count of parameters, so a chunk carries
|
|
1369
|
-
// PG_CHANGES_PER_STATEMENT changes. The first chunk folds the allocation,
|
|
1370
|
-
// the commit row, and the change-log write into one statement; later
|
|
1371
|
-
// chunks run under the same commitSeq with their global idx. Serialized
|
|
1372
|
-
// scopes bind as text before parsing JSONB; drivers that encode JSONB
|
|
1373
|
-
// parameters would otherwise store the string as a scalar. A commit can
|
|
1374
|
-
// delete one row twice, so a chunk's tombstone insert keeps the latest
|
|
1375
|
-
// delete per key and a later chunk overwrites it with its own. `version`
|
|
1376
|
-
// is the change's row version (§2.2), which a delete's tombstone keeps.
|
|
1377
|
-
const valuesOf = (params, slice, firstIdx) => slice.map((change, offset) => {
|
|
1378
|
-
const base = params.length;
|
|
1379
|
-
params.push(firstIdx + offset, change.table, change.rowId, change.op === 'upsert' ? 1 : 2, change.rowVersion, JSON.stringify(change.scopes), change.payload ?? null);
|
|
1380
|
-
return `($${base + 1}::int,$${base + 2}::text,$${base + 3}::text,$${base + 4}::smallint,$${base + 5}::bigint,$${base + 6}::text::jsonb,$${base + 7}::bytea)`;
|
|
1381
|
-
});
|
|
1382
|
-
const scopesCte = `scoped AS (
|
|
1383
|
-
INSERT INTO sync_change_scopes(partition, tbl, var, value, commit_seq)
|
|
1384
|
-
SELECT inserted.partition, inserted.tbl, scope.key, scope.value, inserted.commit_seq
|
|
1385
|
-
FROM inserted CROSS JOIN LATERAL jsonb_each_text(inserted.scopes) AS scope
|
|
1386
|
-
ON CONFLICT DO NOTHING
|
|
1387
|
-
)`;
|
|
1388
|
-
const tombstones = (movedParam) => `INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq, server_version, scopes)
|
|
1389
|
-
SELECT DISTINCT ON (inserted.tbl, inserted.row_id) inserted.partition,
|
|
1390
|
-
inserted.tbl, inserted.row_id, inserted.commit_seq,
|
|
1391
|
-
v.version, inserted.scopes
|
|
1392
|
-
FROM inserted JOIN v ON v.idx = inserted.idx
|
|
1393
|
-
WHERE inserted.op = 2
|
|
1394
|
-
AND NOT EXISTS (
|
|
1395
|
-
SELECT 1
|
|
1396
|
-
FROM jsonb_to_recordset(${movedParam}::text::jsonb) AS moved(tbl text, row_id text)
|
|
1397
|
-
WHERE moved.tbl = inserted.tbl AND moved.row_id = inserted.row_id)
|
|
1398
|
-
ORDER BY inserted.tbl, inserted.row_id, inserted.idx DESC
|
|
1399
|
-
ON CONFLICT (partition, tbl, row_id)
|
|
1400
|
-
DO UPDATE SET commit_seq = excluded.commit_seq,
|
|
1401
|
-
server_version = excluded.server_version,
|
|
1402
|
-
scopes = excluded.scopes`;
|
|
1403
|
-
const firstSlice = commit.changes.slice(0, PG_CHANGES_PER_STATEMENT);
|
|
1404
|
-
const params = [...head];
|
|
1405
|
-
const values = valuesOf(params, firstSlice, 0);
|
|
1406
|
-
params.push(moved);
|
|
1407
|
-
const movedParam = `$${params.length}`;
|
|
1408
|
-
const { rows } = await q.query(`WITH allocated AS (${allocate}),
|
|
1409
|
-
allocated_commit AS (${commitRow}),
|
|
1410
|
-
v(idx, tbl, row_id, op, version, scopes, payload) AS (
|
|
1411
|
-
VALUES ${values.join(',')}
|
|
1412
|
-
),
|
|
1413
|
-
inserted AS (
|
|
1414
|
-
INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
|
|
1415
|
-
SELECT $1, allocated.max_commit_seq, v.idx, v.tbl, v.row_id, v.op, v.version, v.scopes, v.payload
|
|
1416
|
-
FROM allocated CROSS JOIN v
|
|
1417
|
-
RETURNING partition, idx, tbl, row_id, op, commit_seq, scopes
|
|
1418
|
-
),
|
|
1419
|
-
${scopesCte},
|
|
1420
|
-
tombstoned AS (${tombstones(movedParam)})
|
|
1421
|
-
SELECT max_commit_seq FROM allocated`, params);
|
|
1422
|
-
const commitSeq = asNumber(rows[0]?.max_commit_seq);
|
|
1423
|
-
for (let first = PG_CHANGES_PER_STATEMENT; first < commit.changes.length; first += PG_CHANGES_PER_STATEMENT) {
|
|
1424
|
-
const chunkParams = [p, commitSeq];
|
|
1425
|
-
const chunkValues = valuesOf(chunkParams, commit.changes.slice(first, first + PG_CHANGES_PER_STATEMENT), first);
|
|
1426
|
-
chunkParams.push(moved);
|
|
1427
|
-
const chunkMovedParam = `$${chunkParams.length}`;
|
|
1428
|
-
await q.query(`WITH v(idx, tbl, row_id, op, version, scopes, payload) AS (
|
|
1429
|
-
VALUES ${chunkValues.join(',')}
|
|
1430
|
-
),
|
|
1431
|
-
inserted AS (
|
|
1432
|
-
INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
|
|
1433
|
-
SELECT $1, $2, v.idx, v.tbl, v.row_id, v.op, v.version, v.scopes, v.payload
|
|
1434
|
-
FROM v
|
|
1435
|
-
RETURNING partition, idx, tbl, row_id, op, commit_seq, scopes
|
|
1436
|
-
),
|
|
1437
|
-
${scopesCte},
|
|
1438
|
-
tombstoned AS (${tombstones(chunkMovedParam)})
|
|
1439
|
-
SELECT 1`, chunkParams);
|
|
1492
|
+
const commitSeq = this.#nextCommitSeq;
|
|
1493
|
+
if (commitSeq === undefined) {
|
|
1494
|
+
this.#stagedAppend = { commit };
|
|
1495
|
+
const allocated = await this.#flush('write');
|
|
1496
|
+
if (allocated === undefined) {
|
|
1497
|
+
throw new Error('the commit append returned no cursor');
|
|
1498
|
+
}
|
|
1499
|
+
this.#nextCommitSeq = allocated + 1;
|
|
1500
|
+
return allocated;
|
|
1440
1501
|
}
|
|
1502
|
+
this.#nextCommitSeq = commitSeq + 1;
|
|
1503
|
+
this.#stagedAppend = { commit, commitSeq };
|
|
1504
|
+
if (this.#pushApply)
|
|
1505
|
+
this.#openCommitSeq = commitSeq;
|
|
1506
|
+
else
|
|
1507
|
+
await this.#flush('write');
|
|
1441
1508
|
return commitSeq;
|
|
1442
1509
|
}
|
|
1443
1510
|
async putPushResult(clientId, clientCommitId, result, sequence) {
|
|
1444
1511
|
this.#assertOpen();
|
|
1512
|
+
// On the rerun after a closed commit's write failed, the commit writes
|
|
1513
|
+
// behind its savepoint before it closes, so a relational constraint
|
|
1514
|
+
// rejects it alone (`StorageConstraintError`).
|
|
1515
|
+
if (this.#closeEagerly)
|
|
1516
|
+
await this.#flush('read');
|
|
1445
1517
|
this.#recordPushResult(clientId, clientCommitId, result, sequence);
|
|
1446
|
-
// The commit is closed: the
|
|
1447
|
-
// savepoint.
|
|
1518
|
+
// The commit is closed: its writes join the closed commits', and the next
|
|
1519
|
+
// commit's first write takes its own savepoint.
|
|
1520
|
+
for (const [table, rows] of this.#staged) {
|
|
1521
|
+
const closed = this.#closedRows.get(table) ?? new Map();
|
|
1522
|
+
for (const [rowId, entry] of rows) {
|
|
1523
|
+
closed.delete(rowId);
|
|
1524
|
+
closed.set(rowId, entry);
|
|
1525
|
+
}
|
|
1526
|
+
this.#closedRows.set(table, closed);
|
|
1527
|
+
}
|
|
1528
|
+
for (const [table, refs] of this.#stagedBlobRefs) {
|
|
1529
|
+
const closed = this.#closedBlobRefs.get(table) ?? new Map();
|
|
1530
|
+
for (const [rowId, blobIds] of refs)
|
|
1531
|
+
closed.set(rowId, blobIds);
|
|
1532
|
+
this.#closedBlobRefs.set(table, closed);
|
|
1533
|
+
}
|
|
1534
|
+
if (this.#stagedAppend !== undefined) {
|
|
1535
|
+
this.#closedAppends.push(this.#stagedAppend);
|
|
1536
|
+
}
|
|
1537
|
+
this.#staged.clear();
|
|
1538
|
+
this.#stagedBlobRefs.clear();
|
|
1539
|
+
this.#stagedAppend = undefined;
|
|
1540
|
+
this.#openCommitSeq = undefined;
|
|
1448
1541
|
this.#pushApplySavepoint = false;
|
|
1449
1542
|
}
|
|
1450
1543
|
async enqueueReactions(reactions) {
|
|
1451
1544
|
this.#assertOpen();
|
|
1452
1545
|
for (const reaction of reactions) {
|
|
1453
|
-
await this.#
|
|
1546
|
+
await this.#statement(`INSERT INTO sync_reactions(
|
|
1454
1547
|
partition, idempotency_key, type, version, payload,
|
|
1455
1548
|
source_client_id, source_client_commit_id, source_commit_seq,
|
|
1456
1549
|
created_at_ms, available_at_ms, status, attempts, max_attempts
|
|
1457
|
-
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$9,'pending',0,$10)`, [
|
|
1550
|
+
) VALUES ($1,$2,$3,$4,$5::text::jsonb,$6,$7,$8,$9,$9,'pending',0,$10)`, [
|
|
1458
1551
|
this.#partition,
|
|
1459
1552
|
reaction.idempotencyKey,
|
|
1460
1553
|
reaction.type,
|
|
@@ -1465,22 +1558,13 @@ class PostgresTransaction {
|
|
|
1465
1558
|
reaction.sourceCommitSeq,
|
|
1466
1559
|
reaction.createdAtMs,
|
|
1467
1560
|
reaction.maxAttempts,
|
|
1468
|
-
]);
|
|
1561
|
+
], 'write');
|
|
1469
1562
|
}
|
|
1470
1563
|
}
|
|
1471
1564
|
async commit() {
|
|
1472
1565
|
this.#assertOpen();
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
// staged writes first.
|
|
1476
|
-
await this.#client.query(`INSERT INTO sync_push_results(partition, client_id, client_commit_id, result, outbox_id, client_seq)
|
|
1477
|
-
SELECT $1, r.client_id, r.client_commit_id, r.result, r.outbox_id, r.client_seq
|
|
1478
|
-
FROM jsonb_to_recordset($2::text::jsonb)
|
|
1479
|
-
AS r(client_id text, client_commit_id text, result jsonb, outbox_id text, client_seq bigint)
|
|
1480
|
-
ON CONFLICT (partition, client_id, client_commit_id) DO NOTHING`, [this.#partition, JSON.stringify(this.#pushResults)]);
|
|
1481
|
-
this.#pushResults.length = 0;
|
|
1482
|
-
}
|
|
1483
|
-
await this.#writeStaged();
|
|
1566
|
+
// Every pending write in one statement where the parameters fit.
|
|
1567
|
+
await this.#flush('final');
|
|
1484
1568
|
this.#open = false;
|
|
1485
1569
|
// Signal the transaction wrapper to COMMIT; wait for it to actually land.
|
|
1486
1570
|
this.#resolve();
|
|
@@ -1489,7 +1573,6 @@ class PostgresTransaction {
|
|
|
1489
1573
|
async rollback() {
|
|
1490
1574
|
if (!this.#open)
|
|
1491
1575
|
return;
|
|
1492
|
-
this.#staged.clear();
|
|
1493
1576
|
this.#open = false;
|
|
1494
1577
|
this.#reject(new RollbackSignal());
|
|
1495
1578
|
// Swallow — a rollback is a normal outcome, not an error to the caller.
|
|
@@ -1508,10 +1591,6 @@ class RollbackSignal extends Error {
|
|
|
1508
1591
|
this.name = 'RollbackSignal';
|
|
1509
1592
|
}
|
|
1510
1593
|
}
|
|
1511
|
-
/** Changes per sync_changes statement: 2 + 7 × 4,096 = 28,674 bound
|
|
1512
|
-
* parameters. The Bind message counts parameters in 16 bits; PostgreSQL reads
|
|
1513
|
-
* up to 65,535, and PGlite 0.5 corrupts its state above 32,767. */
|
|
1514
|
-
const PG_CHANGES_PER_STATEMENT = 4096;
|
|
1515
1594
|
export class PostgresServerStorage {
|
|
1516
1595
|
#exec;
|
|
1517
1596
|
/** Set by `ensureSchema`: app-table lookup for the relational row store. */
|
|
@@ -2186,7 +2265,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2186
2265
|
}
|
|
2187
2266
|
async failReaction(partition, idempotencyKey, update) {
|
|
2188
2267
|
const result = await this.#exec.query(`UPDATE sync_reactions
|
|
2189
|
-
SET status=$4, available_at_ms=$5, last_failure=$6,
|
|
2268
|
+
SET status=$4, available_at_ms=$5, last_failure=$6::text::jsonb,
|
|
2190
2269
|
lease_owner=NULL, lease_expires_at_ms=NULL
|
|
2191
2270
|
WHERE partition=$1 AND idempotency_key=$2
|
|
2192
2271
|
AND status='leased' AND lease_owner=$3
|