@syncular/server 0.26.4 → 0.26.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/postgres-storage.js +99 -45
- package/dist/push.js +19 -2
- package/dist/relational-rows.d.ts +6 -0
- package/dist/relational-rows.js +13 -0
- package/dist/storage.d.ts +8 -0
- package/package.json +2 -2
- package/src/postgres-storage.ts +123 -59
- package/src/push.ts +18 -2
- package/src/relational-rows.ts +14 -0
- package/src/storage.ts +8 -0
package/dist/postgres-storage.js
CHANGED
|
@@ -5,7 +5,7 @@ import { serveNotReadyError } from './readiness.js';
|
|
|
5
5
|
import { bindAuthoritativePartition, postgresPlaceholders, prepareAuthoritativeQuery, } from './authoritative-query.js';
|
|
6
6
|
import { syncError } from './errors.js';
|
|
7
7
|
import { asBytes, asNumber, } from './pg-executor.js';
|
|
8
|
-
import { assertPhysicalColumns, assertStoredLayouts, COMMIT_WINDOW_BATCH_SQL, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_POSTGRES, scanRowBatchSql, scanRowPageSql, schemaDdl, selectRowScopesSql, selectRowSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
8
|
+
import { assertPhysicalColumns, assertStoredLayouts, COMMIT_WINDOW_BATCH_SQL, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_POSTGRES, scanRowBatchSql, scanRowPageSql, schemaDdl, selectRowScopesSql, selectRowSql, prefetchRowsSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
9
9
|
import { matchesEffective } from './scopes.js';
|
|
10
10
|
import { isPostgresConstraintError, StorageConstraintError, } from './storage-errors.js';
|
|
11
11
|
import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js';
|
|
@@ -403,12 +403,28 @@ async function scanRowsByIndexOn(q, compiled, partition, query) {
|
|
|
403
403
|
return rows.map(toStoredRow);
|
|
404
404
|
}
|
|
405
405
|
async function writeRowOn(q, compiled, partition, row) {
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
406
|
+
// One statement per row write: the upsert, and the scope index synced to
|
|
407
|
+
// the row's scope map (entries outside it deleted, missing ones inserted).
|
|
408
|
+
// No index entry is touched twice, so the writable CTEs cannot conflict.
|
|
409
|
+
// The scopes bind as text before the JSONB cast (see `appendCommit`).
|
|
410
|
+
const params = upsertValues(compiled, partition, row, 'postgres');
|
|
411
|
+
const n = params.length;
|
|
412
|
+
await q.query(`WITH upserted AS (${upsertSql(compiled, 'postgres')}),
|
|
413
|
+
scope AS (SELECT key, value FROM jsonb_each_text($${n + 1}::text::jsonb)),
|
|
414
|
+
removed AS (
|
|
415
|
+
DELETE FROM sync_row_scopes
|
|
416
|
+
WHERE partition=$${n + 2} AND tbl=$${n + 3} AND row_id=$${n + 4}
|
|
417
|
+
AND (var, value) NOT IN (SELECT key, value FROM scope)
|
|
418
|
+
)
|
|
419
|
+
INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
|
|
420
|
+
SELECT $${n + 2}, $${n + 3}, key, value, $${n + 4} FROM scope
|
|
421
|
+
ON CONFLICT DO NOTHING`, [
|
|
422
|
+
...params,
|
|
423
|
+
JSON.stringify(row.scopes),
|
|
424
|
+
partition,
|
|
425
|
+
compiled.name,
|
|
426
|
+
row.rowId,
|
|
427
|
+
]);
|
|
412
428
|
}
|
|
413
429
|
async function getPushResultOn(q, partition, clientId, clientCommitId) {
|
|
414
430
|
const { rows } = await q.query('SELECT result FROM sync_push_results WHERE partition=$1 AND client_id=$2 AND client_commit_id=$3', [partition, clientId, clientCommitId]);
|
|
@@ -442,6 +458,13 @@ class PostgresTransaction {
|
|
|
442
458
|
#resolveTable;
|
|
443
459
|
#open = true;
|
|
444
460
|
#pushApplySavepoint = false;
|
|
461
|
+
/**
|
|
462
|
+
* `prefetchRows` results keyed `table\u0000rowId`; `null` records a row
|
|
463
|
+
* (or tombstone) read as absent. A write in this transaction drops the
|
|
464
|
+
* key, so the next read goes to the database again.
|
|
465
|
+
*/
|
|
466
|
+
#prefetchedRows = new Map();
|
|
467
|
+
#prefetchedTombstones = new Map();
|
|
445
468
|
/** Resolves/rejects the `transaction(fn)` wrapper (see `begin`). */
|
|
446
469
|
#resolve;
|
|
447
470
|
#reject;
|
|
@@ -473,16 +496,44 @@ class PostgresTransaction {
|
|
|
473
496
|
}
|
|
474
497
|
getRow(table, rowId) {
|
|
475
498
|
this.#assertOpen();
|
|
499
|
+
const prefetched = this.#prefetchedRows.get(`${table}\u0000${rowId}`);
|
|
500
|
+
if (prefetched !== undefined) {
|
|
501
|
+
return Promise.resolve(prefetched ?? undefined);
|
|
502
|
+
}
|
|
476
503
|
return getRowOn(this.#client, this.#resolveTable(table), this.#partition, rowId);
|
|
477
504
|
}
|
|
505
|
+
async prefetchRows(table, rowIds) {
|
|
506
|
+
this.#assertOpen();
|
|
507
|
+
const ids = [...new Set(rowIds)].filter((rowId) => !this.#prefetchedRows.has(`${table}\u0000${rowId}`));
|
|
508
|
+
if (ids.length === 0)
|
|
509
|
+
return;
|
|
510
|
+
const { rows } = await this.#client.query(prefetchRowsSql(this.#resolveTable(table)), [
|
|
511
|
+
this.#partition,
|
|
512
|
+
JSON.stringify(ids),
|
|
513
|
+
table,
|
|
514
|
+
]);
|
|
515
|
+
for (const record of rows) {
|
|
516
|
+
const key = `${table}\u0000${record.lookup_id}`;
|
|
517
|
+
this.#prefetchedRows.set(key, record.row_id === null
|
|
518
|
+
? null
|
|
519
|
+
: toStoredRow({ ...record, row_id: record.row_id }));
|
|
520
|
+
this.#prefetchedTombstones.set(key, record.tombstone_seq === null || record.tombstone_seq === undefined
|
|
521
|
+
? null
|
|
522
|
+
: asNumber(record.tombstone_seq));
|
|
523
|
+
}
|
|
524
|
+
}
|
|
478
525
|
async getTombstoneSeq(table, rowId) {
|
|
479
526
|
this.#assertOpen();
|
|
527
|
+
const prefetched = this.#prefetchedTombstones.get(`${table}\u0000${rowId}`);
|
|
528
|
+
if (prefetched !== undefined)
|
|
529
|
+
return prefetched ?? undefined;
|
|
480
530
|
const { rows } = await this.#client.query('SELECT commit_seq FROM sync_tombstones WHERE partition=$1 AND tbl=$2 AND row_id=$3', [this.#partition, table, rowId]);
|
|
481
531
|
const seq = rows[0]?.commit_seq;
|
|
482
532
|
return seq === null || seq === undefined ? undefined : asNumber(seq);
|
|
483
533
|
}
|
|
484
534
|
async clearTombstone(table, rowId) {
|
|
485
535
|
this.#assertOpen();
|
|
536
|
+
this.#prefetchedTombstones.delete(`${table}\u0000${rowId}`);
|
|
486
537
|
await this.#client.query('DELETE FROM sync_tombstones WHERE partition=$1 AND tbl=$2 AND row_id=$3', [this.#partition, table, rowId]);
|
|
487
538
|
}
|
|
488
539
|
getPushResult(clientId, clientCommitId) {
|
|
@@ -576,6 +627,7 @@ class PostgresTransaction {
|
|
|
576
627
|
}
|
|
577
628
|
async upsertRow(table, row, context) {
|
|
578
629
|
this.#assertOpen();
|
|
630
|
+
this.#prefetchedRows.delete(`${table}\u0000${row.rowId}`);
|
|
579
631
|
try {
|
|
580
632
|
await writeRowOn(this.#client, this.#resolveTable(table), this.#partition, row);
|
|
581
633
|
}
|
|
@@ -588,10 +640,14 @@ class PostgresTransaction {
|
|
|
588
640
|
}
|
|
589
641
|
async deleteRow(table, rowId) {
|
|
590
642
|
this.#assertOpen();
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
//
|
|
594
|
-
await this.#client.query(
|
|
643
|
+
this.#prefetchedRows.delete(`${table}\u0000${rowId}`);
|
|
644
|
+
// One statement: the row, its scope-index entries, and (§5.9.4: a
|
|
645
|
+
// deleted row references no blobs) its blob references.
|
|
646
|
+
await this.#client.query(`WITH deleted AS (${deleteRowSql(this.#resolveTable(table), 'postgres')}),
|
|
647
|
+
unscoped AS (
|
|
648
|
+
DELETE FROM sync_row_scopes WHERE partition=$1 AND tbl=$3 AND row_id=$2
|
|
649
|
+
)
|
|
650
|
+
DELETE FROM sync_blob_refs WHERE partition=$1 AND tbl=$3 AND row_id=$2`, [this.#partition, rowId, table]);
|
|
595
651
|
}
|
|
596
652
|
async setBlobRefs(table, rowId, blobIds) {
|
|
597
653
|
this.#assertOpen();
|
|
@@ -604,6 +660,8 @@ class PostgresTransaction {
|
|
|
604
660
|
}
|
|
605
661
|
async appendCommit(commit) {
|
|
606
662
|
this.#assertOpen();
|
|
663
|
+
// The commit writes its delete tombstones below.
|
|
664
|
+
this.#prefetchedTombstones.clear();
|
|
607
665
|
const q = this.#client;
|
|
608
666
|
const p = this.#partition;
|
|
609
667
|
// Allocate the next dense commitSeq under a per-partition row lock: the
|
|
@@ -626,40 +684,36 @@ class PostgresTransaction {
|
|
|
626
684
|
this.#writerVersion,
|
|
627
685
|
]);
|
|
628
686
|
const commitSeq = asNumber(rows[0]?.commit_seq);
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
JSON.stringify(change.scopes),
|
|
660
|
-
change.payload ?? null,
|
|
661
|
-
]);
|
|
662
|
-
}
|
|
687
|
+
if (commit.changes.length === 0)
|
|
688
|
+
return commitSeq;
|
|
689
|
+
// One statement for the whole commit: its changes, their inverted scope
|
|
690
|
+
// entries, and the §5 delete tombstones ride one writable-CTE chain.
|
|
691
|
+
// Serialized scopes bind as text before parsing JSONB; drivers that
|
|
692
|
+
// encode JSONB parameters would otherwise store the string as a scalar.
|
|
693
|
+
// A commit can delete one row twice, so the tombstone insert keeps one
|
|
694
|
+
// row per key (every candidate carries the same commit_seq).
|
|
695
|
+
const params = [p, commitSeq];
|
|
696
|
+
const values = commit.changes.map((change, idx) => {
|
|
697
|
+
const base = params.length;
|
|
698
|
+
params.push(idx, change.table, change.rowId, change.op === 'upsert' ? 1 : 2, change.rowVersion ?? null, JSON.stringify(change.scopes), change.payload ?? null);
|
|
699
|
+
return `($1,$2::bigint,$${base + 1}::int,$${base + 2},$${base + 3},$${base + 4}::smallint,$${base + 5}::bigint,$${base + 6}::text::jsonb,$${base + 7}::bytea)`;
|
|
700
|
+
});
|
|
701
|
+
await q.query(`WITH inserted AS (
|
|
702
|
+
INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
|
|
703
|
+
VALUES ${values.join(',')}
|
|
704
|
+
RETURNING partition, tbl, row_id, op, commit_seq, scopes
|
|
705
|
+
),
|
|
706
|
+
scoped AS (
|
|
707
|
+
INSERT INTO sync_change_scopes(partition, tbl, var, value, commit_seq)
|
|
708
|
+
SELECT inserted.partition, inserted.tbl, scope.key, scope.value, inserted.commit_seq
|
|
709
|
+
FROM inserted CROSS JOIN LATERAL jsonb_each_text(inserted.scopes) AS scope
|
|
710
|
+
ON CONFLICT DO NOTHING
|
|
711
|
+
)
|
|
712
|
+
INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq)
|
|
713
|
+
SELECT DISTINCT ON (tbl, row_id) partition, tbl, row_id, commit_seq
|
|
714
|
+
FROM inserted WHERE op = 2
|
|
715
|
+
ON CONFLICT (partition, tbl, row_id)
|
|
716
|
+
DO UPDATE SET commit_seq = excluded.commit_seq`, params);
|
|
663
717
|
return commitSeq;
|
|
664
718
|
}
|
|
665
719
|
async putPushResult(clientId, clientCommitId, result) {
|
package/dist/push.js
CHANGED
|
@@ -420,8 +420,12 @@ async function applyOperation(tx, schema, resolved, op, opIndex, blobCtx, crdtMe
|
|
|
420
420
|
const insertReject = await runValidator(validators, table, 'upsert', op.rowId, values, undefined, opIndex, tx, partition, actorId);
|
|
421
421
|
if (insertReject !== undefined)
|
|
422
422
|
return insertReject;
|
|
423
|
-
// §5.2: an explicit insert recreates — the tombstone goes with it.
|
|
424
|
-
|
|
423
|
+
// §5.2: an explicit insert recreates — the tombstone goes with it. An
|
|
424
|
+
// unversioned insert already proved above, under the partition lock, that
|
|
425
|
+
// no tombstone exists (this commit writes its own only in `appendCommit`).
|
|
426
|
+
if (op.baseVersion !== undefined) {
|
|
427
|
+
await tx.clearTombstone(op.table, op.rowId);
|
|
428
|
+
}
|
|
425
429
|
const newRow = {
|
|
426
430
|
rowId: op.rowId,
|
|
427
431
|
serverVersion: 1,
|
|
@@ -915,6 +919,19 @@ export async function processPushOperationsWithTrace(ctx, schema, resolved, clie
|
|
|
915
919
|
const stagedWrites = [];
|
|
916
920
|
let terminated;
|
|
917
921
|
const operations = await buildOperations(tx);
|
|
922
|
+
if (tx.prefetchRows !== undefined) {
|
|
923
|
+
const targets = new Map();
|
|
924
|
+
for (const op of operations) {
|
|
925
|
+
if (!schema.tables.has(op.table))
|
|
926
|
+
continue;
|
|
927
|
+
const ids = targets.get(op.table) ?? [];
|
|
928
|
+
ids.push(op.rowId);
|
|
929
|
+
targets.set(op.table, ids);
|
|
930
|
+
}
|
|
931
|
+
for (const [table, rowIds] of targets) {
|
|
932
|
+
await tx.prefetchRows(table, rowIds);
|
|
933
|
+
}
|
|
934
|
+
}
|
|
918
935
|
for (let opIndex = 0; opIndex < operations.length; opIndex++) {
|
|
919
936
|
const op = operations[opIndex];
|
|
920
937
|
if (op === undefined)
|
|
@@ -249,6 +249,12 @@ export declare function scanRowBatchSql(table: CompiledTable): string;
|
|
|
249
249
|
* `scanRowBatchSql`.
|
|
250
250
|
*/
|
|
251
251
|
export declare const COMMIT_WINDOW_BATCH_SQL = "SELECT s.i AS sub, c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,\n ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload\n FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, var text, vals text[], after bigint, through bigint, lim int)\n CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes\n WHERE partition=$1 AND tbl=$3 AND var=s.var AND value=ANY(s.vals)\n AND commit_seq>s.after AND commit_seq<=s.through\n ORDER BY commit_seq LIMIT s.lim) c\n LEFT JOIN sync_commits m\n ON m.partition=$1 AND m.commit_seq=c.commit_seq\n LEFT JOIN sync_changes ch\n ON ch.partition=$1 AND ch.commit_seq=c.commit_seq AND ch.tbl=$3\n ORDER BY s.i, c.commit_seq, ch.idx";
|
|
252
|
+
/**
|
|
253
|
+
* PostgreSQL push read-ahead: every requested row id with its stored row
|
|
254
|
+
* (NULL columns when absent, `selectRowSql` shape) and its §5 delete
|
|
255
|
+
* tombstone. Params: [partition, rowIds as JSON array text, tbl].
|
|
256
|
+
*/
|
|
257
|
+
export declare function prefetchRowsSql(table: CompiledTable): string;
|
|
252
258
|
/** SELECT a row's version + scope map (admin/blob authz). Params: [partition, rowId]. */
|
|
253
259
|
export declare function selectRowScopesSql(table: CompiledTable, dialect: RelationalDialect): string;
|
|
254
260
|
/** DELETE one stored row. Params: [partition, rowId]. */
|
package/dist/relational-rows.js
CHANGED
|
@@ -593,6 +593,19 @@ export const COMMIT_WINDOW_BATCH_SQL = `SELECT s.i AS sub, c.commit_seq AS commi
|
|
|
593
593
|
LEFT JOIN sync_changes ch
|
|
594
594
|
ON ch.partition=$1 AND ch.commit_seq=c.commit_seq AND ch.tbl=$3
|
|
595
595
|
ORDER BY s.i, c.commit_seq, ch.idx`;
|
|
596
|
+
/**
|
|
597
|
+
* PostgreSQL push read-ahead: every requested row id with its stored row
|
|
598
|
+
* (NULL columns when absent, `selectRowSql` shape) and its §5 delete
|
|
599
|
+
* tombstone. Params: [partition, rowIds as JSON array text, tbl].
|
|
600
|
+
*/
|
|
601
|
+
export function prefetchRowsSql(table) {
|
|
602
|
+
return `SELECT ids.id AS lookup_id, r.${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions, t.commit_seq AS tombstone_seq
|
|
603
|
+
FROM jsonb_array_elements_text($2::text::jsonb) AS ids(id)
|
|
604
|
+
LEFT JOIN ${quoteIdent(table.name)} r
|
|
605
|
+
ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=$1 AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=ids.id
|
|
606
|
+
LEFT JOIN sync_tombstones t
|
|
607
|
+
ON t.partition=$1 AND t.tbl=$3 AND t.row_id=ids.id`;
|
|
608
|
+
}
|
|
596
609
|
/** SELECT a row's version + scope map (admin/blob authz). Params: [partition, rowId]. */
|
|
597
610
|
export function selectRowScopesSql(table, dialect) {
|
|
598
611
|
const p = dialect === 'sqlite' ? ['?', '?'] : ['$1', '$2'];
|
package/dist/storage.d.ts
CHANGED
|
@@ -344,6 +344,14 @@ export interface AuthoritativeQueryResult {
|
|
|
344
344
|
*/
|
|
345
345
|
export interface StorageTransaction {
|
|
346
346
|
getRow(table: string, rowId: string): Promise<StoredRow | undefined>;
|
|
347
|
+
/**
|
|
348
|
+
* Optional read-ahead: the push layer names every row a commit's
|
|
349
|
+
* operations target, per table, before it applies them. A storage with
|
|
350
|
+
* per-statement network cost reads them in one statement and answers the
|
|
351
|
+
* following `getRow` calls from that snapshot until the row is written in
|
|
352
|
+
* this transaction.
|
|
353
|
+
*/
|
|
354
|
+
prefetchRows?(table: string, rowIds: readonly string[]): Promise<void>;
|
|
347
355
|
/**
|
|
348
356
|
* §5 delete-precedence tombstone read: the `commit_seq` of the newest
|
|
349
357
|
* applied delete of this row still within the pruning horizon, or
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.26.
|
|
3
|
+
"version": "0.26.5",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.26.
|
|
71
|
+
"@syncular/core": "0.26.5"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|
package/src/postgres-storage.ts
CHANGED
|
@@ -76,6 +76,7 @@ import {
|
|
|
76
76
|
schemaDdl,
|
|
77
77
|
selectRowScopesSql,
|
|
78
78
|
selectRowSql,
|
|
79
|
+
prefetchRowsSql,
|
|
79
80
|
selectRowsForRewriteSql,
|
|
80
81
|
upsertSql,
|
|
81
82
|
upsertValues,
|
|
@@ -697,21 +698,31 @@ async function writeRowOn(
|
|
|
697
698
|
partition: string,
|
|
698
699
|
row: StoredRow,
|
|
699
700
|
): Promise<void> {
|
|
701
|
+
// One statement per row write: the upsert, and the scope index synced to
|
|
702
|
+
// the row's scope map (entries outside it deleted, missing ones inserted).
|
|
703
|
+
// No index entry is touched twice, so the writable CTEs cannot conflict.
|
|
704
|
+
// The scopes bind as text before the JSONB cast (see `appendCommit`).
|
|
705
|
+
const params = upsertValues(compiled, partition, row, 'postgres');
|
|
706
|
+
const n = params.length;
|
|
700
707
|
await q.query(
|
|
701
|
-
upsertSql(compiled, 'postgres'),
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
708
|
+
`WITH upserted AS (${upsertSql(compiled, 'postgres')}),
|
|
709
|
+
scope AS (SELECT key, value FROM jsonb_each_text($${n + 1}::text::jsonb)),
|
|
710
|
+
removed AS (
|
|
711
|
+
DELETE FROM sync_row_scopes
|
|
712
|
+
WHERE partition=$${n + 2} AND tbl=$${n + 3} AND row_id=$${n + 4}
|
|
713
|
+
AND (var, value) NOT IN (SELECT key, value FROM scope)
|
|
714
|
+
)
|
|
715
|
+
INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
|
|
716
|
+
SELECT $${n + 2}, $${n + 3}, key, value, $${n + 4} FROM scope
|
|
717
|
+
ON CONFLICT DO NOTHING`,
|
|
718
|
+
[
|
|
719
|
+
...params,
|
|
720
|
+
JSON.stringify(row.scopes),
|
|
721
|
+
partition,
|
|
722
|
+
compiled.name,
|
|
723
|
+
row.rowId,
|
|
724
|
+
],
|
|
707
725
|
);
|
|
708
|
-
for (const [variable, value] of Object.entries(row.scopes)) {
|
|
709
|
-
await q.query(
|
|
710
|
-
`INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
|
|
711
|
-
VALUES ($1,$2,$3,$4,$5) ON CONFLICT DO NOTHING`,
|
|
712
|
-
[partition, compiled.name, variable, value, row.rowId],
|
|
713
|
-
);
|
|
714
|
-
}
|
|
715
726
|
}
|
|
716
727
|
|
|
717
728
|
async function getPushResultOn(
|
|
@@ -770,6 +781,13 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
770
781
|
#resolveTable: (name: string) => CompiledTable;
|
|
771
782
|
#open = true;
|
|
772
783
|
#pushApplySavepoint = false;
|
|
784
|
+
/**
|
|
785
|
+
* `prefetchRows` results keyed `table\u0000rowId`; `null` records a row
|
|
786
|
+
* (or tombstone) read as absent. A write in this transaction drops the
|
|
787
|
+
* key, so the next read goes to the database again.
|
|
788
|
+
*/
|
|
789
|
+
readonly #prefetchedRows = new Map<string, StoredRow | null>();
|
|
790
|
+
readonly #prefetchedTombstones = new Map<string, number | null>();
|
|
773
791
|
/** Resolves/rejects the `transaction(fn)` wrapper (see `begin`). */
|
|
774
792
|
#resolve: () => void;
|
|
775
793
|
#reject: (error: unknown) => void;
|
|
@@ -814,6 +832,10 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
814
832
|
|
|
815
833
|
getRow(table: string, rowId: string): Promise<StoredRow | undefined> {
|
|
816
834
|
this.#assertOpen();
|
|
835
|
+
const prefetched = this.#prefetchedRows.get(`${table}\u0000${rowId}`);
|
|
836
|
+
if (prefetched !== undefined) {
|
|
837
|
+
return Promise.resolve(prefetched ?? undefined);
|
|
838
|
+
}
|
|
817
839
|
return getRowOn(
|
|
818
840
|
this.#client,
|
|
819
841
|
this.#resolveTable(table),
|
|
@@ -822,11 +844,47 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
822
844
|
);
|
|
823
845
|
}
|
|
824
846
|
|
|
847
|
+
async prefetchRows(table: string, rowIds: readonly string[]): Promise<void> {
|
|
848
|
+
this.#assertOpen();
|
|
849
|
+
const ids = [...new Set(rowIds)].filter(
|
|
850
|
+
(rowId) => !this.#prefetchedRows.has(`${table}\u0000${rowId}`),
|
|
851
|
+
);
|
|
852
|
+
if (ids.length === 0) return;
|
|
853
|
+
const { rows } = await this.#client.query<
|
|
854
|
+
Omit<RowRecord, 'row_id'> & {
|
|
855
|
+
lookup_id: string;
|
|
856
|
+
row_id: string | null;
|
|
857
|
+
tombstone_seq: unknown;
|
|
858
|
+
}
|
|
859
|
+
>(prefetchRowsSql(this.#resolveTable(table)), [
|
|
860
|
+
this.#partition,
|
|
861
|
+
JSON.stringify(ids),
|
|
862
|
+
table,
|
|
863
|
+
]);
|
|
864
|
+
for (const record of rows) {
|
|
865
|
+
const key = `${table}\u0000${record.lookup_id}`;
|
|
866
|
+
this.#prefetchedRows.set(
|
|
867
|
+
key,
|
|
868
|
+
record.row_id === null
|
|
869
|
+
? null
|
|
870
|
+
: toStoredRow({ ...record, row_id: record.row_id }),
|
|
871
|
+
);
|
|
872
|
+
this.#prefetchedTombstones.set(
|
|
873
|
+
key,
|
|
874
|
+
record.tombstone_seq === null || record.tombstone_seq === undefined
|
|
875
|
+
? null
|
|
876
|
+
: asNumber(record.tombstone_seq),
|
|
877
|
+
);
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
|
|
825
881
|
async getTombstoneSeq(
|
|
826
882
|
table: string,
|
|
827
883
|
rowId: string,
|
|
828
884
|
): Promise<number | undefined> {
|
|
829
885
|
this.#assertOpen();
|
|
886
|
+
const prefetched = this.#prefetchedTombstones.get(`${table}\u0000${rowId}`);
|
|
887
|
+
if (prefetched !== undefined) return prefetched ?? undefined;
|
|
830
888
|
const { rows } = await this.#client.query<{ commit_seq: unknown }>(
|
|
831
889
|
'SELECT commit_seq FROM sync_tombstones WHERE partition=$1 AND tbl=$2 AND row_id=$3',
|
|
832
890
|
[this.#partition, table, rowId],
|
|
@@ -837,6 +895,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
837
895
|
|
|
838
896
|
async clearTombstone(table: string, rowId: string): Promise<void> {
|
|
839
897
|
this.#assertOpen();
|
|
898
|
+
this.#prefetchedTombstones.delete(`${table}\u0000${rowId}`);
|
|
840
899
|
await this.#client.query(
|
|
841
900
|
'DELETE FROM sync_tombstones WHERE partition=$1 AND tbl=$2 AND row_id=$3',
|
|
842
901
|
[this.#partition, table, rowId],
|
|
@@ -976,6 +1035,7 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
976
1035
|
context?: { readonly opIndex: number },
|
|
977
1036
|
): Promise<void> {
|
|
978
1037
|
this.#assertOpen();
|
|
1038
|
+
this.#prefetchedRows.delete(`${table}\u0000${row.rowId}`);
|
|
979
1039
|
try {
|
|
980
1040
|
await writeRowOn(
|
|
981
1041
|
this.#client,
|
|
@@ -993,18 +1053,16 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
993
1053
|
|
|
994
1054
|
async deleteRow(table: string, rowId: string): Promise<void> {
|
|
995
1055
|
this.#assertOpen();
|
|
1056
|
+
this.#prefetchedRows.delete(`${table}\u0000${rowId}`);
|
|
1057
|
+
// One statement: the row, its scope-index entries, and (§5.9.4: a
|
|
1058
|
+
// deleted row references no blobs) its blob references.
|
|
996
1059
|
await this.#client.query(
|
|
997
|
-
deleteRowSql(this.#resolveTable(table), 'postgres'),
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
|
|
1001
|
-
|
|
1002
|
-
[this.#partition,
|
|
1003
|
-
);
|
|
1004
|
-
// §5.9.4: a deleted row references no blobs.
|
|
1005
|
-
await this.#client.query(
|
|
1006
|
-
'DELETE FROM sync_blob_refs WHERE partition=$1 AND tbl=$2 AND row_id=$3',
|
|
1007
|
-
[this.#partition, table, rowId],
|
|
1060
|
+
`WITH deleted AS (${deleteRowSql(this.#resolveTable(table), 'postgres')}),
|
|
1061
|
+
unscoped AS (
|
|
1062
|
+
DELETE FROM sync_row_scopes WHERE partition=$1 AND tbl=$3 AND row_id=$2
|
|
1063
|
+
)
|
|
1064
|
+
DELETE FROM sync_blob_refs WHERE partition=$1 AND tbl=$3 AND row_id=$2`,
|
|
1065
|
+
[this.#partition, rowId, table],
|
|
1008
1066
|
);
|
|
1009
1067
|
}
|
|
1010
1068
|
|
|
@@ -1030,6 +1088,8 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1030
1088
|
|
|
1031
1089
|
async appendCommit(commit: NewCommit): Promise<number> {
|
|
1032
1090
|
this.#assertOpen();
|
|
1091
|
+
// The commit writes its delete tombstones below.
|
|
1092
|
+
this.#prefetchedTombstones.clear();
|
|
1033
1093
|
const q = this.#client;
|
|
1034
1094
|
const p = this.#partition;
|
|
1035
1095
|
// Allocate the next dense commitSeq under a per-partition row lock: the
|
|
@@ -1055,42 +1115,46 @@ class PostgresTransaction implements StorageTransaction {
|
|
|
1055
1115
|
],
|
|
1056
1116
|
);
|
|
1057
1117
|
const commitSeq = asNumber(rows[0]?.commit_seq);
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
|
|
1067
|
-
|
|
1068
|
-
|
|
1069
|
-
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
)
|
|
1077
|
-
INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq)
|
|
1078
|
-
SELECT partition, tbl, row_id, commit_seq FROM inserted WHERE op = 2
|
|
1079
|
-
ON CONFLICT (partition, tbl, row_id)
|
|
1080
|
-
DO UPDATE SET commit_seq = excluded.commit_seq`,
|
|
1081
|
-
[
|
|
1082
|
-
p,
|
|
1083
|
-
commitSeq,
|
|
1084
|
-
idx,
|
|
1085
|
-
change.table,
|
|
1086
|
-
change.rowId,
|
|
1087
|
-
change.op === 'upsert' ? 1 : 2,
|
|
1088
|
-
change.rowVersion ?? null,
|
|
1089
|
-
JSON.stringify(change.scopes),
|
|
1090
|
-
change.payload ?? null,
|
|
1091
|
-
],
|
|
1118
|
+
if (commit.changes.length === 0) return commitSeq;
|
|
1119
|
+
// One statement for the whole commit: its changes, their inverted scope
|
|
1120
|
+
// entries, and the §5 delete tombstones ride one writable-CTE chain.
|
|
1121
|
+
// Serialized scopes bind as text before parsing JSONB; drivers that
|
|
1122
|
+
// encode JSONB parameters would otherwise store the string as a scalar.
|
|
1123
|
+
// A commit can delete one row twice, so the tombstone insert keeps one
|
|
1124
|
+
// row per key (every candidate carries the same commit_seq).
|
|
1125
|
+
const params: unknown[] = [p, commitSeq];
|
|
1126
|
+
const values = commit.changes.map((change, idx) => {
|
|
1127
|
+
const base = params.length;
|
|
1128
|
+
params.push(
|
|
1129
|
+
idx,
|
|
1130
|
+
change.table,
|
|
1131
|
+
change.rowId,
|
|
1132
|
+
change.op === 'upsert' ? 1 : 2,
|
|
1133
|
+
change.rowVersion ?? null,
|
|
1134
|
+
JSON.stringify(change.scopes),
|
|
1135
|
+
change.payload ?? null,
|
|
1092
1136
|
);
|
|
1093
|
-
|
|
1137
|
+
return `($1,$2::bigint,$${base + 1}::int,$${base + 2},$${base + 3},$${base + 4}::smallint,$${base + 5}::bigint,$${base + 6}::text::jsonb,$${base + 7}::bytea)`;
|
|
1138
|
+
});
|
|
1139
|
+
await q.query(
|
|
1140
|
+
`WITH inserted AS (
|
|
1141
|
+
INSERT INTO sync_changes(partition, commit_seq, idx, tbl, row_id, op, row_version, scopes, payload)
|
|
1142
|
+
VALUES ${values.join(',')}
|
|
1143
|
+
RETURNING partition, tbl, row_id, op, commit_seq, scopes
|
|
1144
|
+
),
|
|
1145
|
+
scoped AS (
|
|
1146
|
+
INSERT INTO sync_change_scopes(partition, tbl, var, value, commit_seq)
|
|
1147
|
+
SELECT inserted.partition, inserted.tbl, scope.key, scope.value, inserted.commit_seq
|
|
1148
|
+
FROM inserted CROSS JOIN LATERAL jsonb_each_text(inserted.scopes) AS scope
|
|
1149
|
+
ON CONFLICT DO NOTHING
|
|
1150
|
+
)
|
|
1151
|
+
INSERT INTO sync_tombstones(partition, tbl, row_id, commit_seq)
|
|
1152
|
+
SELECT DISTINCT ON (tbl, row_id) partition, tbl, row_id, commit_seq
|
|
1153
|
+
FROM inserted WHERE op = 2
|
|
1154
|
+
ON CONFLICT (partition, tbl, row_id)
|
|
1155
|
+
DO UPDATE SET commit_seq = excluded.commit_seq`,
|
|
1156
|
+
params,
|
|
1157
|
+
);
|
|
1094
1158
|
return commitSeq;
|
|
1095
1159
|
}
|
|
1096
1160
|
|
package/src/push.ts
CHANGED
|
@@ -696,8 +696,12 @@ async function applyOperation(
|
|
|
696
696
|
actorId,
|
|
697
697
|
);
|
|
698
698
|
if (insertReject !== undefined) return insertReject;
|
|
699
|
-
// §5.2: an explicit insert recreates — the tombstone goes with it.
|
|
700
|
-
|
|
699
|
+
// §5.2: an explicit insert recreates — the tombstone goes with it. An
|
|
700
|
+
// unversioned insert already proved above, under the partition lock, that
|
|
701
|
+
// no tombstone exists (this commit writes its own only in `appendCommit`).
|
|
702
|
+
if (op.baseVersion !== undefined) {
|
|
703
|
+
await tx.clearTombstone(op.table, op.rowId);
|
|
704
|
+
}
|
|
701
705
|
const newRow = {
|
|
702
706
|
rowId: op.rowId,
|
|
703
707
|
serverVersion: 1,
|
|
@@ -1415,6 +1419,18 @@ export async function processPushOperationsWithTrace(
|
|
|
1415
1419
|
const stagedWrites: StagedWrite[] = [];
|
|
1416
1420
|
let terminated: PushOperationResult | undefined;
|
|
1417
1421
|
const operations = await buildOperations(tx);
|
|
1422
|
+
if (tx.prefetchRows !== undefined) {
|
|
1423
|
+
const targets = new Map<string, string[]>();
|
|
1424
|
+
for (const op of operations) {
|
|
1425
|
+
if (!schema.tables.has(op.table)) continue;
|
|
1426
|
+
const ids = targets.get(op.table) ?? [];
|
|
1427
|
+
ids.push(op.rowId);
|
|
1428
|
+
targets.set(op.table, ids);
|
|
1429
|
+
}
|
|
1430
|
+
for (const [table, rowIds] of targets) {
|
|
1431
|
+
await tx.prefetchRows(table, rowIds);
|
|
1432
|
+
}
|
|
1433
|
+
}
|
|
1418
1434
|
for (let opIndex = 0; opIndex < operations.length; opIndex++) {
|
|
1419
1435
|
const op = operations[opIndex];
|
|
1420
1436
|
if (op === undefined) continue;
|
package/src/relational-rows.ts
CHANGED
|
@@ -691,6 +691,20 @@ export const COMMIT_WINDOW_BATCH_SQL = `SELECT s.i AS sub, c.commit_seq AS commi
|
|
|
691
691
|
ON ch.partition=$1 AND ch.commit_seq=c.commit_seq AND ch.tbl=$3
|
|
692
692
|
ORDER BY s.i, c.commit_seq, ch.idx`;
|
|
693
693
|
|
|
694
|
+
/**
|
|
695
|
+
* PostgreSQL push read-ahead: every requested row id with its stored row
|
|
696
|
+
* (NULL columns when absent, `selectRowSql` shape) and its §5 delete
|
|
697
|
+
* tombstone. Params: [partition, rowIds as JSON array text, tbl].
|
|
698
|
+
*/
|
|
699
|
+
export function prefetchRowsSql(table: CompiledTable): string {
|
|
700
|
+
return `SELECT ids.id AS lookup_id, r.${quoteIdent(SYNC_ROW_ID_COLUMN)} AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions, t.commit_seq AS tombstone_seq
|
|
701
|
+
FROM jsonb_array_elements_text($2::text::jsonb) AS ids(id)
|
|
702
|
+
LEFT JOIN ${quoteIdent(table.name)} r
|
|
703
|
+
ON r.${quoteIdent(SYNC_PARTITION_COLUMN)}=$1 AND r.${quoteIdent(SYNC_ROW_ID_COLUMN)}=ids.id
|
|
704
|
+
LEFT JOIN sync_tombstones t
|
|
705
|
+
ON t.partition=$1 AND t.tbl=$3 AND t.row_id=ids.id`;
|
|
706
|
+
}
|
|
707
|
+
|
|
694
708
|
/** SELECT a row's version + scope map (admin/blob authz). Params: [partition, rowId]. */
|
|
695
709
|
export function selectRowScopesSql(
|
|
696
710
|
table: CompiledTable,
|
package/src/storage.ts
CHANGED
|
@@ -433,6 +433,14 @@ export interface AuthoritativeQueryResult {
|
|
|
433
433
|
*/
|
|
434
434
|
export interface StorageTransaction {
|
|
435
435
|
getRow(table: string, rowId: string): Promise<StoredRow | undefined>;
|
|
436
|
+
/**
|
|
437
|
+
* Optional read-ahead: the push layer names every row a commit's
|
|
438
|
+
* operations target, per table, before it applies them. A storage with
|
|
439
|
+
* per-statement network cost reads them in one statement and answers the
|
|
440
|
+
* following `getRow` calls from that snapshot until the row is written in
|
|
441
|
+
* this transaction.
|
|
442
|
+
*/
|
|
443
|
+
prefetchRows?(table: string, rowIds: readonly string[]): Promise<void>;
|
|
436
444
|
/**
|
|
437
445
|
* §5 delete-precedence tombstone read: the `commit_seq` of the newest
|
|
438
446
|
* applied delete of this row still within the pruning horizon, or
|