@syncular/server 0.35.0 → 0.37.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/postgres-storage.d.ts +7 -1
- package/dist/postgres-storage.js +113 -67
- package/dist/relational-rows.d.ts +11 -11
- package/dist/relational-rows.js +16 -16
- package/package.json +2 -2
- package/src/postgres-storage.ts +144 -96
- package/src/relational-rows.ts +16 -16
|
@@ -73,7 +73,13 @@ export declare const POSTGRES_DDL = "\nCREATE TABLE IF NOT EXISTS sync_partition
|
|
|
73
73
|
export declare class PostgresServerStorage implements ServerStorage {
|
|
74
74
|
#private;
|
|
75
75
|
constructor(exec: PgExecutor);
|
|
76
|
-
/**
|
|
76
|
+
/**
|
|
77
|
+
* Apply the schema DDL (idempotent). Call once before use. The last
|
|
78
|
+
* statement records a fingerprint of the applied DDL as the comment of
|
|
79
|
+
* `sync_schema_meta`; a database that carries the current fingerprint
|
|
80
|
+
* costs one catalog read instead of the DDL round trips, so a new storage
|
|
81
|
+
* instance (a Worker isolate, a Durable Object wake) starts fast.
|
|
82
|
+
*/
|
|
77
83
|
migrate(): Promise<void>;
|
|
78
84
|
/** Resolve a table's compiled schema; row operations require `ensureSchema`. */
|
|
79
85
|
table(name: string): CompiledTable;
|
package/dist/postgres-storage.js
CHANGED
|
@@ -6,7 +6,7 @@ import { bindAuthoritativePartition, postgresPlaceholders, prepareAuthoritativeQ
|
|
|
6
6
|
import { syncError } from './errors.js';
|
|
7
7
|
import { asBytes, asNumber, } from './pg-executor.js';
|
|
8
8
|
import { assertPhysicalColumns, assertStoredLayouts, COMMIT_WINDOW_BATCH_SQL, LATEST_SCOPED_CHANGE_BATCH_SQL, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanRowPageSql, schemaDdl, selectRowScopesSql, selectRowSql, prefetchRowsSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
9
|
-
import { matchesEffective } from './scopes.js';
|
|
9
|
+
import { matchesEffective, sha256Hex } from './scopes.js';
|
|
10
10
|
import { isPostgresConstraintError, StorageConstraintError, } from './storage-errors.js';
|
|
11
11
|
import { assertScopeIndexedScan, resolveIndexRowScan } from './storage-query.js';
|
|
12
12
|
/**
|
|
@@ -794,23 +794,24 @@ export class PostgresServerStorage {
|
|
|
794
794
|
#schemaVersion;
|
|
795
795
|
/**
|
|
796
796
|
* Page reads waiting for the end of the current microtask turn, keyed by
|
|
797
|
-
* statement and
|
|
797
|
+
* statement and partition. A pull starts the window read (or first
|
|
798
798
|
* snapshot page) of every subscription before awaiting any of them, so the
|
|
799
|
-
* reads of
|
|
800
|
-
*
|
|
799
|
+
* window reads of all tables leave as ONE set-based statement (snapshot
|
|
800
|
+
* pages as one per table, whose SQL names the table) instead of one round
|
|
801
|
+
* trip per subscription or table (SYNCULAR-PULL-ROUNDTRIPS-001).
|
|
801
802
|
*/
|
|
802
803
|
#pageBatches = new Map();
|
|
803
804
|
constructor(exec) {
|
|
804
805
|
this.#exec = exec;
|
|
805
806
|
}
|
|
806
807
|
/**
|
|
807
|
-
* Queue one page read into the batch for (`sql`, `partition
|
|
808
|
-
*
|
|
809
|
-
*
|
|
810
|
-
*
|
|
808
|
+
* Queue one page read into the batch for (`sql`, `partition`). The batch
|
|
809
|
+
* runs as a single statement with the queued specs bound as a JSON array
|
|
810
|
+
* (`$2`); each spec gets its position as `i`, and the result rows are
|
|
811
|
+
* split back by their `sub` column.
|
|
811
812
|
*/
|
|
812
|
-
#batchedPage(sql, partition,
|
|
813
|
-
const key = `${sql}\u0000${partition}
|
|
813
|
+
#batchedPage(sql, partition, spec) {
|
|
814
|
+
const key = `${sql}\u0000${partition}`;
|
|
814
815
|
let batch = this.#pageBatches.get(key);
|
|
815
816
|
if (batch === undefined) {
|
|
816
817
|
const queued = [];
|
|
@@ -819,7 +820,7 @@ export class PostgresServerStorage {
|
|
|
819
820
|
queueMicrotask(() => {
|
|
820
821
|
this.#pageBatches.delete(key);
|
|
821
822
|
const specs = JSON.stringify(queued.map((entry, i) => ({ ...entry.spec, i })));
|
|
822
|
-
this.#exec.query(sql, [partition, specs
|
|
823
|
+
this.#exec.query(sql, [partition, specs]).then(({ rows }) => {
|
|
823
824
|
const grouped = queued.map(() => []);
|
|
824
825
|
for (const row of rows)
|
|
825
826
|
grouped[asNumber(row.sub)]?.push(row);
|
|
@@ -839,22 +840,27 @@ export class PostgresServerStorage {
|
|
|
839
840
|
});
|
|
840
841
|
});
|
|
841
842
|
}
|
|
842
|
-
/**
|
|
843
|
+
/**
|
|
844
|
+
* Apply the schema DDL (idempotent). Call once before use. The last
|
|
845
|
+
* statement records a fingerprint of the applied DDL as the comment of
|
|
846
|
+
* `sync_schema_meta`; a database that carries the current fingerprint
|
|
847
|
+
* costs one catalog read instead of the DDL round trips, so a new storage
|
|
848
|
+
* instance (a Worker isolate, a Durable Object wake) starts fast.
|
|
849
|
+
*/
|
|
843
850
|
async migrate() {
|
|
844
851
|
// Split on the statement boundary so drivers that reject multi-statement
|
|
845
852
|
// query strings (node-postgres extended protocol) still apply each DDL.
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
await this.#exec.query(`CREATE OR REPLACE FUNCTION syncular_writer_fence()
|
|
853
|
+
// The fence function is sent whole because its dollar-quoted body
|
|
854
|
+
// contains `;`.
|
|
855
|
+
const statements = [
|
|
856
|
+
...POSTGRES_DDL.split(';')
|
|
857
|
+
.map((s) => s.trim())
|
|
858
|
+
.filter((s) => s.length > 0),
|
|
859
|
+
'ALTER TABLE sync_commits ADD COLUMN IF NOT EXISTS writer_version BIGINT',
|
|
860
|
+
// Database-side fence: an old binary's INSERT never reaches JS. An
|
|
861
|
+
// absent `sync_writer_fence` row makes the EXISTS false, so a partition
|
|
862
|
+
// with no barrier behaves exactly as before.
|
|
863
|
+
`CREATE OR REPLACE FUNCTION syncular_writer_fence()
|
|
858
864
|
RETURNS trigger AS $$
|
|
859
865
|
BEGIN
|
|
860
866
|
IF EXISTS (
|
|
@@ -867,11 +873,31 @@ BEGIN
|
|
|
867
873
|
END IF;
|
|
868
874
|
RETURN NEW;
|
|
869
875
|
END;
|
|
870
|
-
$$ LANGUAGE plpgsql
|
|
871
|
-
|
|
872
|
-
|
|
876
|
+
$$ LANGUAGE plpgsql`,
|
|
877
|
+
'DROP TRIGGER IF EXISTS sync_commits_writer_fence ON sync_commits',
|
|
878
|
+
`CREATE TRIGGER sync_commits_writer_fence
|
|
873
879
|
BEFORE INSERT ON sync_commits
|
|
874
|
-
FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()
|
|
880
|
+
FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
881
|
+
SCHEMA_META_DDL_POSTGRES,
|
|
882
|
+
];
|
|
883
|
+
const fingerprint = `syncular.storage:${await sha256Hex(new TextEncoder().encode(statements.join('\u0000')))}`;
|
|
884
|
+
// `obj_description` and `to_regclass` of a missing table are NULL, so
|
|
885
|
+
// the probe never fails. A core table dropped out of band reapplies the
|
|
886
|
+
// DDL even under the current fingerprint.
|
|
887
|
+
const { rows } = await this.#exec.query(`SELECT obj_description(to_regclass('sync_schema_meta'), 'pg_class') AS storage,
|
|
888
|
+
(SELECT count(*)::int
|
|
889
|
+
FROM jsonb_array_elements_text($1::text::jsonb) AS core(name)
|
|
890
|
+
WHERE to_regclass(core.name) IS NULL) AS missing`, [
|
|
891
|
+
JSON.stringify(statements.flatMap((statement) => /^CREATE TABLE IF NOT EXISTS (\w+)/.exec(statement)?.[1] ?? [])),
|
|
892
|
+
]);
|
|
893
|
+
if (rows[0]?.storage === fingerprint && rows[0].missing === 0)
|
|
894
|
+
return;
|
|
895
|
+
for (const statement of statements) {
|
|
896
|
+
await this.#exec.query(statement);
|
|
897
|
+
}
|
|
898
|
+
// The fingerprint is lowercase hex, safe to inline (COMMENT takes no
|
|
899
|
+
// bound parameter).
|
|
900
|
+
await this.#exec.query(`COMMENT ON TABLE sync_schema_meta IS '${fingerprint}'`);
|
|
875
901
|
}
|
|
876
902
|
/** Resolve a table's compiled schema; row operations require `ensureSchema`. */
|
|
877
903
|
table(name) {
|
|
@@ -886,51 +912,71 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
|
|
|
886
912
|
if (this.#schemaVersion === schema.version)
|
|
887
913
|
return;
|
|
888
914
|
await this.migrate();
|
|
915
|
+
// One statement reads the marker, the incomplete checkpoints of this
|
|
916
|
+
// version and the physical columns of every app table. The aggregates
|
|
917
|
+
// leave as text so every driver hands back the same JSON strings.
|
|
918
|
+
// Columns resolve through `to_regclass`, which follows the session's
|
|
919
|
+
// `search_path` exactly as the unqualified row queries do:
|
|
920
|
+
// `current_schema()` names only the first existing schema, so it would
|
|
921
|
+
// reject a table that resolves later in the path.
|
|
922
|
+
const { rows: probe } = await this.#exec.query(`SELECT meta.schema_version AS schema_version, meta.layouts AS layouts,
|
|
923
|
+
(SELECT coalesce(json_agg(json_build_object(
|
|
924
|
+
'partition', checkpoint.partition,
|
|
925
|
+
'name', checkpoint.name)), '[]'::json)::text
|
|
926
|
+
FROM sync_backfill_checkpoints checkpoint
|
|
927
|
+
WHERE checkpoint.schema_version=$1
|
|
928
|
+
AND checkpoint.state<>'activated') AS incomplete,
|
|
929
|
+
(SELECT coalesce(json_agg(json_build_object(
|
|
930
|
+
'tbl', tables.name,
|
|
931
|
+
'name', attribute.attname,
|
|
932
|
+
'type', format_type(attribute.atttypid, attribute.atttypmod),
|
|
933
|
+
'not_null', attribute.attnotnull,
|
|
934
|
+
'pk', (SELECT key.position::int
|
|
935
|
+
FROM pg_index pk,
|
|
936
|
+
unnest(pk.indkey::int2[]) WITH ORDINALITY
|
|
937
|
+
AS key(attnum, position)
|
|
938
|
+
WHERE pk.indrelid = attribute.attrelid
|
|
939
|
+
AND pk.indisprimary
|
|
940
|
+
AND key.attnum = attribute.attnum))
|
|
941
|
+
ORDER BY tables.name, attribute.attnum), '[]'::json)::text
|
|
942
|
+
FROM jsonb_array_elements_text($2::text::jsonb) AS tables(name)
|
|
943
|
+
JOIN pg_attribute attribute
|
|
944
|
+
ON attribute.attrelid = to_regclass(tables.name)
|
|
945
|
+
WHERE attribute.attnum > 0
|
|
946
|
+
AND NOT attribute.attisdropped) AS columns
|
|
947
|
+
FROM (SELECT 1) AS one
|
|
948
|
+
LEFT JOIN sync_schema_meta meta ON meta.id=1`, [schema.version, JSON.stringify([...schema.tables.keys()])]);
|
|
949
|
+
const marker = probe[0];
|
|
950
|
+
if (marker === undefined) {
|
|
951
|
+
throw new Error('schema probe returned no row');
|
|
952
|
+
}
|
|
889
953
|
// The barrier reads stored state: a checkpoint at this schema version that
|
|
890
954
|
// is not activated and that this caller did not declare means this process
|
|
891
955
|
// must not serve. An omitted or empty declaration set declares nothing and
|
|
892
956
|
// is refused here, never treated as a bypass.
|
|
893
957
|
const declarations = checkpoints ?? [];
|
|
894
|
-
const incomplete =
|
|
895
|
-
|
|
896
|
-
if (incomplete.rows.some((row) => !declarations.some((declaration) => declaration.partition === row.partition &&
|
|
958
|
+
const incomplete = JSON.parse(marker.incomplete);
|
|
959
|
+
if (incomplete.some((row) => !declarations.some((declaration) => declaration.partition === row.partition &&
|
|
897
960
|
declaration.name === row.name))) {
|
|
898
961
|
throw new StorageQueryError('sync.storage.checkpoint_incomplete');
|
|
899
962
|
}
|
|
900
|
-
|
|
901
|
-
const marker = await this.#exec.query('SELECT schema_version, layouts FROM sync_schema_meta WHERE id=1');
|
|
902
|
-
const stored = marker.rows[0] === undefined
|
|
963
|
+
const stored = marker.schema_version === null || marker.schema_version === undefined
|
|
903
964
|
? undefined
|
|
904
|
-
: asNumber(marker.
|
|
965
|
+
: asNumber(marker.schema_version);
|
|
905
966
|
if (stored !== undefined && stored > schema.version) {
|
|
906
967
|
throw new Error(`stored schema version ${stored} is newer than the configured schema (${schema.version}) — refusing to run an older server against a migrated database`);
|
|
907
968
|
}
|
|
908
969
|
if (stored !== undefined && stored === schema.version) {
|
|
909
|
-
assertStoredLayouts(schema, typeof marker.
|
|
910
|
-
|
|
911
|
-
: undefined);
|
|
912
|
-
// Resolve through `to_regclass`, which follows the session's
|
|
913
|
-
// `search_path` exactly as the unqualified row queries do:
|
|
914
|
-
// `current_schema()` names only the first existing schema, so it would
|
|
915
|
-
// reject a table that resolves later in the path.
|
|
970
|
+
assertStoredLayouts(schema, typeof marker.layouts === 'string' ? marker.layouts : undefined);
|
|
971
|
+
const columns = JSON.parse(marker.columns);
|
|
916
972
|
for (const table of schema.tables.values()) {
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
WHERE pk.indrelid = attribute.attrelid AND pk.indisprimary
|
|
925
|
-
AND key.attnum = attribute.attnum) AS pk
|
|
926
|
-
FROM pg_attribute attribute
|
|
927
|
-
WHERE attribute.attrelid = to_regclass($1)
|
|
928
|
-
AND attribute.attnum > 0 AND NOT attribute.attisdropped`, [table.name]);
|
|
929
|
-
assertPhysicalColumns(table, rows.map((row) => ({
|
|
930
|
-
name: row.name,
|
|
931
|
-
type: row.type,
|
|
932
|
-
notNull: row.not_null,
|
|
933
|
-
primaryKeyPosition: row.pk ?? 0,
|
|
973
|
+
assertPhysicalColumns(table, columns
|
|
974
|
+
.filter((column) => column.tbl === table.name)
|
|
975
|
+
.map((column) => ({
|
|
976
|
+
name: column.name,
|
|
977
|
+
type: column.type,
|
|
978
|
+
notNull: column.not_null,
|
|
979
|
+
primaryKeyPosition: column.pk ?? 0,
|
|
934
980
|
})), 'postgres');
|
|
935
981
|
}
|
|
936
982
|
}
|
|
@@ -941,9 +987,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
|
|
|
941
987
|
// backfill for flipped-on materialization) — all inside one
|
|
942
988
|
// transaction (Postgres DDL is transactional — a failed bump leaves
|
|
943
989
|
// no half-state).
|
|
944
|
-
const layouts = parseLayouts(typeof marker.
|
|
945
|
-
? marker.rows[0].layouts
|
|
946
|
-
: undefined);
|
|
990
|
+
const layouts = parseLayouts(typeof marker.layouts === 'string' ? marker.layouts : undefined);
|
|
947
991
|
const retiredTables = retiredTableNames(schema, layouts);
|
|
948
992
|
await this.#exec.transaction(async (client) => {
|
|
949
993
|
// RFC 0007 migration barrier: `LOCK TABLE ... EXCLUSIVE` is the FIRST
|
|
@@ -1431,8 +1475,8 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
|
|
|
1431
1475
|
: (query.scopeFilter[firstVariable] ?? []);
|
|
1432
1476
|
if (firstVariable === undefined || values.length === 0)
|
|
1433
1477
|
return 0;
|
|
1434
|
-
// Concurrent reads of
|
|
1435
|
-
const [row] = await this.#batchedPage(LATEST_SCOPED_CHANGE_BATCH_SQL, partition, query.table,
|
|
1478
|
+
// Concurrent reads of a pull share one statement across tables.
|
|
1479
|
+
const [row] = await this.#batchedPage(LATEST_SCOPED_CHANGE_BATCH_SQL, partition, { tbl: query.table, var: firstVariable, vals: values });
|
|
1436
1480
|
return row === undefined ? 0 : asNumber(row.seq);
|
|
1437
1481
|
}
|
|
1438
1482
|
async readCommitWindow(partition, query) {
|
|
@@ -1447,7 +1491,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
|
|
|
1447
1491
|
// var, value, commit_seq) PK makes the candidate subquery an index range
|
|
1448
1492
|
// scan returning commit_seq already ascending (see postgres-explain
|
|
1449
1493
|
// .test.ts), LEFT JOINed to the commit meta + the table's changes. Pages
|
|
1450
|
-
// of concurrent window reads
|
|
1494
|
+
// of concurrent window reads share one statement across tables (see
|
|
1451
1495
|
// `COMMIT_WINDOW_BATCH_SQL`). Exact multi-variable verification against
|
|
1452
1496
|
// the stored scope map happens below.
|
|
1453
1497
|
const commits = [];
|
|
@@ -1455,7 +1499,8 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
|
|
|
1455
1499
|
let afterSeq = query.afterSeq;
|
|
1456
1500
|
const batchSize = Math.max(64, query.limitChanges);
|
|
1457
1501
|
while (deliveredChanges < query.limitChanges) {
|
|
1458
|
-
const records = await this.#batchedPage(COMMIT_WINDOW_BATCH_SQL, partition,
|
|
1502
|
+
const records = await this.#batchedPage(COMMIT_WINDOW_BATCH_SQL, partition, {
|
|
1503
|
+
tbl: query.table,
|
|
1459
1504
|
var: firstVariable,
|
|
1460
1505
|
vals: firstValues,
|
|
1461
1506
|
after: afterSeq,
|
|
@@ -1535,7 +1580,8 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
|
|
|
1535
1580
|
let afterRowId = query.afterRowId ?? '';
|
|
1536
1581
|
const batchSize = Math.max(64, query.limit);
|
|
1537
1582
|
while (rows.length < query.limit) {
|
|
1538
|
-
const records = await this.#batchedPage(sql, partition,
|
|
1583
|
+
const records = await this.#batchedPage(sql, partition, {
|
|
1584
|
+
tbl: query.table,
|
|
1539
1585
|
var: firstVariable,
|
|
1540
1586
|
vals: firstValues,
|
|
1541
1587
|
after: afterRowId,
|
|
@@ -228,27 +228,27 @@ export declare function commitWindowPageSql(valueCount: number, dialect: Relatio
|
|
|
228
228
|
/**
|
|
229
229
|
* PostgreSQL set-based form of `scanRowPageSql`: one statement answers the
|
|
230
230
|
* page of every subscription scanning `table` in a pull. `$2` is a JSON array
|
|
231
|
-
* of `{i, var, vals, after, lim}` records; each record runs the same
|
|
231
|
+
* of `{i, tbl, var, vals, after, lim}` records; each record runs the same
|
|
232
232
|
* index-first candidate subquery through a LATERAL join, so every
|
|
233
233
|
* subscription keeps its own keyset cursor and LIMIT. Rows arrive ordered
|
|
234
234
|
* by (i, row_id).
|
|
235
235
|
*
|
|
236
|
-
* Params: [partition, specs (JSON text)
|
|
236
|
+
* Params: [partition, specs (JSON text)]. The specs bind as text before
|
|
237
237
|
* the JSONB cast, so a driver that JSON-encodes a string bound to a `jsonb`
|
|
238
238
|
* parameter (Bun.sql) cannot turn the array into a JSON string.
|
|
239
239
|
*/
|
|
240
240
|
export declare function scanRowBatchSql(table: CompiledTable): string;
|
|
241
241
|
/**
|
|
242
242
|
* PostgreSQL set-based form of `commitWindowPageSql`: one statement answers
|
|
243
|
-
* the window page of every subscription
|
|
244
|
-
* JSON array of `{i, var, vals, after, through, lim}` records, each
|
|
245
|
-
* the same index-first candidate subquery through a LATERAL join.
|
|
246
|
-
* arrive ordered by (i, commit_seq, idx).
|
|
243
|
+
* the window page of every subscription in a pull, across all tables. `$2`
|
|
244
|
+
* is a JSON array of `{i, tbl, var, vals, after, through, lim}` records, each
|
|
245
|
+
* running the same index-first candidate subquery through a LATERAL join.
|
|
246
|
+
* Rows arrive ordered by (i, commit_seq, idx).
|
|
247
247
|
*
|
|
248
|
-
* Params: [partition, specs (JSON text)
|
|
248
|
+
* Params: [partition, specs (JSON text)], the specs bound as text like
|
|
249
249
|
* `scanRowBatchSql`.
|
|
250
250
|
*/
|
|
251
|
-
export declare const COMMIT_WINDOW_BATCH_SQL = "SELECT s.i AS sub, c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,\n ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload\n FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, var text, vals text[], after bigint, through bigint, lim int)\n CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes\n WHERE partition=$1 AND tbl
|
|
251
|
+
export declare const COMMIT_WINDOW_BATCH_SQL = "SELECT s.i AS sub, c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,\n ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload\n FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], after bigint, through bigint, lim int)\n CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes\n WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=ANY(s.vals)\n AND commit_seq>s.after AND commit_seq<=s.through\n ORDER BY commit_seq LIMIT s.lim) c\n LEFT JOIN sync_commits m\n ON m.partition=$1 AND m.commit_seq=c.commit_seq\n LEFT JOIN sync_changes ch\n ON ch.partition=$1 AND ch.commit_seq=c.commit_seq AND ch.tbl=s.tbl\n ORDER BY s.i, c.commit_seq, ch.idx";
|
|
252
252
|
/**
|
|
253
253
|
* PostgreSQL push read-ahead: every requested row id with its stored row
|
|
254
254
|
* (NULL columns when absent, `selectRowSql` shape) and its §5 delete
|
|
@@ -267,10 +267,10 @@ export declare function prefetchRowsSql(table: CompiledTable): string;
|
|
|
267
267
|
export declare const LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL = "SELECT max((SELECT max(commit_seq) FROM sync_change_scopes\n WHERE partition=? AND tbl=? AND var=? AND value=v.value)) AS seq\n FROM json_each(?) AS v";
|
|
268
268
|
/**
|
|
269
269
|
* PostgreSQL set-based form of `LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL` for the
|
|
270
|
-
* subscriptions of
|
|
271
|
-
* `{i, var, vals}` records. Params: [partition, specs (JSON text)
|
|
270
|
+
* subscriptions of a pull, across all tables. `$2` is a JSON array of
|
|
271
|
+
* `{i, tbl, var, vals}` records. Params: [partition, specs (JSON text)].
|
|
272
272
|
*/
|
|
273
|
-
export declare const LATEST_SCOPED_CHANGE_BATCH_SQL = "SELECT s.i AS sub,\n (SELECT max(latest.seq) FROM (\n SELECT (SELECT max(commit_seq) FROM sync_change_scopes\n WHERE partition=$1 AND tbl
|
|
273
|
+
export declare const LATEST_SCOPED_CHANGE_BATCH_SQL = "SELECT s.i AS sub,\n (SELECT max(latest.seq) FROM (\n SELECT (SELECT max(commit_seq) FROM sync_change_scopes\n WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=v.value) AS seq\n FROM jsonb_array_elements_text(s.vals) AS v(value)) AS latest) AS seq\n FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals jsonb)\n ORDER BY s.i";
|
|
274
274
|
/** SELECT a row's version + scope map (admin/blob authz). Params: [partition, rowId]. */
|
|
275
275
|
export declare function selectRowScopesSql(table: CompiledTable, dialect: RelationalDialect): string;
|
|
276
276
|
/** DELETE one stored row. Params: [partition, rowId]. */
|
package/dist/relational-rows.js
CHANGED
|
@@ -551,20 +551,20 @@ export function commitWindowPageSql(valueCount, dialect) {
|
|
|
551
551
|
/**
|
|
552
552
|
* PostgreSQL set-based form of `scanRowPageSql`: one statement answers the
|
|
553
553
|
* page of every subscription scanning `table` in a pull. `$2` is a JSON array
|
|
554
|
-
* of `{i, var, vals, after, lim}` records; each record runs the same
|
|
554
|
+
* of `{i, tbl, var, vals, after, lim}` records; each record runs the same
|
|
555
555
|
* index-first candidate subquery through a LATERAL join, so every
|
|
556
556
|
* subscription keeps its own keyset cursor and LIMIT. Rows arrive ordered
|
|
557
557
|
* by (i, row_id).
|
|
558
558
|
*
|
|
559
|
-
* Params: [partition, specs (JSON text)
|
|
559
|
+
* Params: [partition, specs (JSON text)]. The specs bind as text before
|
|
560
560
|
* the JSONB cast, so a driver that JSON-encodes a string bound to a `jsonb`
|
|
561
561
|
* parameter (Bun.sql) cannot turn the array into a JSON string.
|
|
562
562
|
*/
|
|
563
563
|
export function scanRowBatchSql(table) {
|
|
564
564
|
return `SELECT s.i AS sub, c.row_id AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions
|
|
565
|
-
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, var text, vals text[], after text, lim int)
|
|
565
|
+
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], after text, lim int)
|
|
566
566
|
CROSS JOIN LATERAL (SELECT DISTINCT row_id FROM sync_row_scopes
|
|
567
|
-
WHERE partition=$1 AND tbl
|
|
567
|
+
WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=ANY(s.vals)
|
|
568
568
|
AND row_id>s.after
|
|
569
569
|
ORDER BY row_id LIMIT s.lim) c
|
|
570
570
|
LEFT JOIN ${quoteIdent(table.name)} r
|
|
@@ -573,25 +573,25 @@ export function scanRowBatchSql(table) {
|
|
|
573
573
|
}
|
|
574
574
|
/**
|
|
575
575
|
* PostgreSQL set-based form of `commitWindowPageSql`: one statement answers
|
|
576
|
-
* the window page of every subscription
|
|
577
|
-
* JSON array of `{i, var, vals, after, through, lim}` records, each
|
|
578
|
-
* the same index-first candidate subquery through a LATERAL join.
|
|
579
|
-
* arrive ordered by (i, commit_seq, idx).
|
|
576
|
+
* the window page of every subscription in a pull, across all tables. `$2`
|
|
577
|
+
* is a JSON array of `{i, tbl, var, vals, after, through, lim}` records, each
|
|
578
|
+
* running the same index-first candidate subquery through a LATERAL join.
|
|
579
|
+
* Rows arrive ordered by (i, commit_seq, idx).
|
|
580
580
|
*
|
|
581
|
-
* Params: [partition, specs (JSON text)
|
|
581
|
+
* Params: [partition, specs (JSON text)], the specs bound as text like
|
|
582
582
|
* `scanRowBatchSql`.
|
|
583
583
|
*/
|
|
584
584
|
export const COMMIT_WINDOW_BATCH_SQL = `SELECT s.i AS sub, c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
585
585
|
ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
586
|
-
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, var text, vals text[], after bigint, through bigint, lim int)
|
|
586
|
+
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], after bigint, through bigint, lim int)
|
|
587
587
|
CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
|
|
588
|
-
WHERE partition=$1 AND tbl
|
|
588
|
+
WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=ANY(s.vals)
|
|
589
589
|
AND commit_seq>s.after AND commit_seq<=s.through
|
|
590
590
|
ORDER BY commit_seq LIMIT s.lim) c
|
|
591
591
|
LEFT JOIN sync_commits m
|
|
592
592
|
ON m.partition=$1 AND m.commit_seq=c.commit_seq
|
|
593
593
|
LEFT JOIN sync_changes ch
|
|
594
|
-
ON ch.partition=$1 AND ch.commit_seq=c.commit_seq AND ch.tbl
|
|
594
|
+
ON ch.partition=$1 AND ch.commit_seq=c.commit_seq AND ch.tbl=s.tbl
|
|
595
595
|
ORDER BY s.i, c.commit_seq, ch.idx`;
|
|
596
596
|
/**
|
|
597
597
|
* PostgreSQL push read-ahead: every requested row id with its stored row
|
|
@@ -620,15 +620,15 @@ export const LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL = `SELECT max((SELECT max(commi
|
|
|
620
620
|
FROM json_each(?) AS v`;
|
|
621
621
|
/**
|
|
622
622
|
* PostgreSQL set-based form of `LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL` for the
|
|
623
|
-
* subscriptions of
|
|
624
|
-
* `{i, var, vals}` records. Params: [partition, specs (JSON text)
|
|
623
|
+
* subscriptions of a pull, across all tables. `$2` is a JSON array of
|
|
624
|
+
* `{i, tbl, var, vals}` records. Params: [partition, specs (JSON text)].
|
|
625
625
|
*/
|
|
626
626
|
export const LATEST_SCOPED_CHANGE_BATCH_SQL = `SELECT s.i AS sub,
|
|
627
627
|
(SELECT max(latest.seq) FROM (
|
|
628
628
|
SELECT (SELECT max(commit_seq) FROM sync_change_scopes
|
|
629
|
-
WHERE partition=$1 AND tbl
|
|
629
|
+
WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=v.value) AS seq
|
|
630
630
|
FROM jsonb_array_elements_text(s.vals) AS v(value)) AS latest) AS seq
|
|
631
|
-
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, var text, vals jsonb)
|
|
631
|
+
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals jsonb)
|
|
632
632
|
ORDER BY s.i`;
|
|
633
633
|
/** SELECT a row's version + scope map (admin/blob authz). Params: [partition, rowId]. */
|
|
634
634
|
export function selectRowScopesSql(table, dialect) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.37.0",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.
|
|
71
|
+
"@syncular/core": "0.37.0"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|
package/src/postgres-storage.ts
CHANGED
|
@@ -90,7 +90,7 @@ import {
|
|
|
90
90
|
upsertValues,
|
|
91
91
|
} from './relational-rows';
|
|
92
92
|
import type { CompiledSchema, CompiledTable } from './schema';
|
|
93
|
-
import { matchesEffective } from './scopes';
|
|
93
|
+
import { matchesEffective, sha256Hex } from './scopes';
|
|
94
94
|
import type {
|
|
95
95
|
AuthoritativeQueryRequest,
|
|
96
96
|
AuthoritativeQueryResult,
|
|
@@ -1279,10 +1279,11 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1279
1279
|
#schemaVersion: number | undefined;
|
|
1280
1280
|
/**
|
|
1281
1281
|
* Page reads waiting for the end of the current microtask turn, keyed by
|
|
1282
|
-
* statement and
|
|
1282
|
+
* statement and partition. A pull starts the window read (or first
|
|
1283
1283
|
* snapshot page) of every subscription before awaiting any of them, so the
|
|
1284
|
-
* reads of
|
|
1285
|
-
*
|
|
1284
|
+
* window reads of all tables leave as ONE set-based statement (snapshot
|
|
1285
|
+
* pages as one per table, whose SQL names the table) instead of one round
|
|
1286
|
+
* trip per subscription or table (SYNCULAR-PULL-ROUNDTRIPS-001).
|
|
1286
1287
|
*/
|
|
1287
1288
|
readonly #pageBatches = new Map<string, PendingPage[]>();
|
|
1288
1289
|
|
|
@@ -1291,18 +1292,17 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1291
1292
|
}
|
|
1292
1293
|
|
|
1293
1294
|
/**
|
|
1294
|
-
* Queue one page read into the batch for (`sql`, `partition
|
|
1295
|
-
*
|
|
1296
|
-
*
|
|
1297
|
-
*
|
|
1295
|
+
* Queue one page read into the batch for (`sql`, `partition`). The batch
|
|
1296
|
+
* runs as a single statement with the queued specs bound as a JSON array
|
|
1297
|
+
* (`$2`); each spec gets its position as `i`, and the result rows are
|
|
1298
|
+
* split back by their `sub` column.
|
|
1298
1299
|
*/
|
|
1299
1300
|
#batchedPage<Row>(
|
|
1300
1301
|
sql: string,
|
|
1301
1302
|
partition: string,
|
|
1302
|
-
table: string,
|
|
1303
1303
|
spec: Record<string, unknown>,
|
|
1304
1304
|
): Promise<Row[]> {
|
|
1305
|
-
const key = `${sql}\u0000${partition}
|
|
1305
|
+
const key = `${sql}\u0000${partition}`;
|
|
1306
1306
|
let batch = this.#pageBatches.get(key);
|
|
1307
1307
|
if (batch === undefined) {
|
|
1308
1308
|
const queued: PendingPage[] = [];
|
|
@@ -1313,7 +1313,7 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1313
1313
|
const specs = JSON.stringify(
|
|
1314
1314
|
queued.map((entry, i) => ({ ...entry.spec, i })),
|
|
1315
1315
|
);
|
|
1316
|
-
this.#exec.query<PgRow>(sql, [partition, specs
|
|
1316
|
+
this.#exec.query<PgRow>(sql, [partition, specs]).then(
|
|
1317
1317
|
({ rows }) => {
|
|
1318
1318
|
const grouped = queued.map((): PgRow[] => []);
|
|
1319
1319
|
for (const row of rows) grouped[asNumber(row.sub)]?.push(row);
|
|
@@ -1335,24 +1335,27 @@ export class PostgresServerStorage implements ServerStorage {
|
|
|
1335
1335
|
});
|
|
1336
1336
|
}
|
|
1337
1337
|
|
|
1338
|
-
/**
|
|
1338
|
+
/**
|
|
1339
|
+
* Apply the schema DDL (idempotent). Call once before use. The last
|
|
1340
|
+
* statement records a fingerprint of the applied DDL as the comment of
|
|
1341
|
+
* `sync_schema_meta`; a database that carries the current fingerprint
|
|
1342
|
+
* costs one catalog read instead of the DDL round trips, so a new storage
|
|
1343
|
+
* instance (a Worker isolate, a Durable Object wake) starts fast.
|
|
1344
|
+
*/
|
|
1339
1345
|
async migrate(): Promise<void> {
|
|
1340
1346
|
// Split on the statement boundary so drivers that reject multi-statement
|
|
1341
1347
|
// query strings (node-postgres extended protocol) still apply each DDL.
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
|
|
1346
|
-
|
|
1347
|
-
|
|
1348
|
-
await this.#exec.query(
|
|
1348
|
+
// The fence function is sent whole because its dollar-quoted body
|
|
1349
|
+
// contains `;`.
|
|
1350
|
+
const statements = [
|
|
1351
|
+
...POSTGRES_DDL.split(';')
|
|
1352
|
+
.map((s) => s.trim())
|
|
1353
|
+
.filter((s) => s.length > 0),
|
|
1349
1354
|
'ALTER TABLE sync_commits ADD COLUMN IF NOT EXISTS writer_version BIGINT',
|
|
1350
|
-
|
|
1351
|
-
|
|
1352
|
-
|
|
1353
|
-
|
|
1354
|
-
// the dollar-quoted body contains `;`.
|
|
1355
|
-
await this.#exec.query(`CREATE OR REPLACE FUNCTION syncular_writer_fence()
|
|
1355
|
+
// Database-side fence: an old binary's INSERT never reaches JS. An
|
|
1356
|
+
// absent `sync_writer_fence` row makes the EXISTS false, so a partition
|
|
1357
|
+
// with no barrier behaves exactly as before.
|
|
1358
|
+
`CREATE OR REPLACE FUNCTION syncular_writer_fence()
|
|
1356
1359
|
RETURNS trigger AS $$
|
|
1357
1360
|
BEGIN
|
|
1358
1361
|
IF EXISTS (
|
|
@@ -1365,13 +1368,45 @@ BEGIN
|
|
|
1365
1368
|
END IF;
|
|
1366
1369
|
RETURN NEW;
|
|
1367
1370
|
END;
|
|
1368
|
-
$$ LANGUAGE plpgsql
|
|
1369
|
-
await this.#exec.query(
|
|
1371
|
+
$$ LANGUAGE plpgsql`,
|
|
1370
1372
|
'DROP TRIGGER IF EXISTS sync_commits_writer_fence ON sync_commits',
|
|
1371
|
-
|
|
1372
|
-
await this.#exec.query(`CREATE TRIGGER sync_commits_writer_fence
|
|
1373
|
+
`CREATE TRIGGER sync_commits_writer_fence
|
|
1373
1374
|
BEFORE INSERT ON sync_commits
|
|
1374
|
-
FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()
|
|
1375
|
+
FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
1376
|
+
SCHEMA_META_DDL_POSTGRES,
|
|
1377
|
+
];
|
|
1378
|
+
const fingerprint = `syncular.storage:${await sha256Hex(
|
|
1379
|
+
new TextEncoder().encode(statements.join('\u0000')),
|
|
1380
|
+
)}`;
|
|
1381
|
+
// `obj_description` and `to_regclass` of a missing table are NULL, so
|
|
1382
|
+
// the probe never fails. A core table dropped out of band reapplies the
|
|
1383
|
+
// DDL even under the current fingerprint.
|
|
1384
|
+
const { rows } = await this.#exec.query<{
|
|
1385
|
+
storage: string | null;
|
|
1386
|
+
missing: number;
|
|
1387
|
+
}>(
|
|
1388
|
+
`SELECT obj_description(to_regclass('sync_schema_meta'), 'pg_class') AS storage,
|
|
1389
|
+
(SELECT count(*)::int
|
|
1390
|
+
FROM jsonb_array_elements_text($1::text::jsonb) AS core(name)
|
|
1391
|
+
WHERE to_regclass(core.name) IS NULL) AS missing`,
|
|
1392
|
+
[
|
|
1393
|
+
JSON.stringify(
|
|
1394
|
+
statements.flatMap(
|
|
1395
|
+
(statement) =>
|
|
1396
|
+
/^CREATE TABLE IF NOT EXISTS (\w+)/.exec(statement)?.[1] ?? [],
|
|
1397
|
+
),
|
|
1398
|
+
),
|
|
1399
|
+
],
|
|
1400
|
+
);
|
|
1401
|
+
if (rows[0]?.storage === fingerprint && rows[0].missing === 0) return;
|
|
1402
|
+
for (const statement of statements) {
|
|
1403
|
+
await this.#exec.query(statement);
|
|
1404
|
+
}
|
|
1405
|
+
// The fingerprint is lowercase hex, safe to inline (COMMENT takes no
|
|
1406
|
+
// bound parameter).
|
|
1407
|
+
await this.#exec.query(
|
|
1408
|
+
`COMMENT ON TABLE sync_schema_meta IS '${fingerprint}'`,
|
|
1409
|
+
);
|
|
1375
1410
|
}
|
|
1376
1411
|
|
|
1377
1412
|
/** Resolve a table's compiled schema; row operations require `ensureSchema`. */
|
|
@@ -1392,21 +1427,63 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
|
|
|
1392
1427
|
// Memoized fast path: same instance, same schema version.
|
|
1393
1428
|
if (this.#schemaVersion === schema.version) return;
|
|
1394
1429
|
await this.migrate();
|
|
1430
|
+
// One statement reads the marker, the incomplete checkpoints of this
|
|
1431
|
+
// version and the physical columns of every app table. The aggregates
|
|
1432
|
+
// leave as text so every driver hands back the same JSON strings.
|
|
1433
|
+
// Columns resolve through `to_regclass`, which follows the session's
|
|
1434
|
+
// `search_path` exactly as the unqualified row queries do:
|
|
1435
|
+
// `current_schema()` names only the first existing schema, so it would
|
|
1436
|
+
// reject a table that resolves later in the path.
|
|
1437
|
+
const { rows: probe } = await this.#exec.query<{
|
|
1438
|
+
schema_version: unknown;
|
|
1439
|
+
layouts: unknown;
|
|
1440
|
+
incomplete: string;
|
|
1441
|
+
columns: string;
|
|
1442
|
+
}>(
|
|
1443
|
+
`SELECT meta.schema_version AS schema_version, meta.layouts AS layouts,
|
|
1444
|
+
(SELECT coalesce(json_agg(json_build_object(
|
|
1445
|
+
'partition', checkpoint.partition,
|
|
1446
|
+
'name', checkpoint.name)), '[]'::json)::text
|
|
1447
|
+
FROM sync_backfill_checkpoints checkpoint
|
|
1448
|
+
WHERE checkpoint.schema_version=$1
|
|
1449
|
+
AND checkpoint.state<>'activated') AS incomplete,
|
|
1450
|
+
(SELECT coalesce(json_agg(json_build_object(
|
|
1451
|
+
'tbl', tables.name,
|
|
1452
|
+
'name', attribute.attname,
|
|
1453
|
+
'type', format_type(attribute.atttypid, attribute.atttypmod),
|
|
1454
|
+
'not_null', attribute.attnotnull,
|
|
1455
|
+
'pk', (SELECT key.position::int
|
|
1456
|
+
FROM pg_index pk,
|
|
1457
|
+
unnest(pk.indkey::int2[]) WITH ORDINALITY
|
|
1458
|
+
AS key(attnum, position)
|
|
1459
|
+
WHERE pk.indrelid = attribute.attrelid
|
|
1460
|
+
AND pk.indisprimary
|
|
1461
|
+
AND key.attnum = attribute.attnum))
|
|
1462
|
+
ORDER BY tables.name, attribute.attnum), '[]'::json)::text
|
|
1463
|
+
FROM jsonb_array_elements_text($2::text::jsonb) AS tables(name)
|
|
1464
|
+
JOIN pg_attribute attribute
|
|
1465
|
+
ON attribute.attrelid = to_regclass(tables.name)
|
|
1466
|
+
WHERE attribute.attnum > 0
|
|
1467
|
+
AND NOT attribute.attisdropped) AS columns
|
|
1468
|
+
FROM (SELECT 1) AS one
|
|
1469
|
+
LEFT JOIN sync_schema_meta meta ON meta.id=1`,
|
|
1470
|
+
[schema.version, JSON.stringify([...schema.tables.keys()])],
|
|
1471
|
+
);
|
|
1472
|
+
const marker = probe[0];
|
|
1473
|
+
if (marker === undefined) {
|
|
1474
|
+
throw new Error('schema probe returned no row');
|
|
1475
|
+
}
|
|
1395
1476
|
// The barrier reads stored state: a checkpoint at this schema version that
|
|
1396
1477
|
// is not activated and that this caller did not declare means this process
|
|
1397
1478
|
// must not serve. An omitted or empty declaration set declares nothing and
|
|
1398
1479
|
// is refused here, never treated as a bypass.
|
|
1399
1480
|
const declarations = checkpoints ?? [];
|
|
1400
|
-
const incomplete =
|
|
1481
|
+
const incomplete = JSON.parse(marker.incomplete) as {
|
|
1401
1482
|
partition: string;
|
|
1402
1483
|
name: string;
|
|
1403
|
-
}
|
|
1404
|
-
`SELECT partition, name FROM sync_backfill_checkpoints
|
|
1405
|
-
WHERE schema_version=$1 AND state<>'activated'`,
|
|
1406
|
-
[schema.version],
|
|
1407
|
-
);
|
|
1484
|
+
}[];
|
|
1408
1485
|
if (
|
|
1409
|
-
incomplete.
|
|
1486
|
+
incomplete.some(
|
|
1410
1487
|
(row) =>
|
|
1411
1488
|
!declarations.some(
|
|
1412
1489
|
(declaration) =>
|
|
@@ -1417,15 +1494,10 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
|
|
|
1417
1494
|
) {
|
|
1418
1495
|
throw new StorageQueryError('sync.storage.checkpoint_incomplete');
|
|
1419
1496
|
}
|
|
1420
|
-
await this.#exec.query(SCHEMA_META_DDL_POSTGRES);
|
|
1421
|
-
const marker = await this.#exec.query<{
|
|
1422
|
-
schema_version: unknown;
|
|
1423
|
-
layouts: unknown;
|
|
1424
|
-
}>('SELECT schema_version, layouts FROM sync_schema_meta WHERE id=1');
|
|
1425
1497
|
const stored =
|
|
1426
|
-
marker.
|
|
1498
|
+
marker.schema_version === null || marker.schema_version === undefined
|
|
1427
1499
|
? undefined
|
|
1428
|
-
: asNumber(marker.
|
|
1500
|
+
: asNumber(marker.schema_version);
|
|
1429
1501
|
if (stored !== undefined && stored > schema.version) {
|
|
1430
1502
|
throw new Error(
|
|
1431
1503
|
`stored schema version ${stored} is newer than the configured schema (${schema.version}) — refusing to run an older server against a migrated database`,
|
|
@@ -1434,43 +1506,26 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
|
|
|
1434
1506
|
if (stored !== undefined && stored === schema.version) {
|
|
1435
1507
|
assertStoredLayouts(
|
|
1436
1508
|
schema,
|
|
1437
|
-
typeof marker.
|
|
1438
|
-
? marker.rows[0].layouts
|
|
1439
|
-
: undefined,
|
|
1509
|
+
typeof marker.layouts === 'string' ? marker.layouts : undefined,
|
|
1440
1510
|
);
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1511
|
+
const columns = JSON.parse(marker.columns) as {
|
|
1512
|
+
tbl: string;
|
|
1513
|
+
name: string;
|
|
1514
|
+
type: string;
|
|
1515
|
+
not_null: boolean;
|
|
1516
|
+
pk: number | null;
|
|
1517
|
+
}[];
|
|
1445
1518
|
for (const table of schema.tables.values()) {
|
|
1446
|
-
const { rows } = await this.#exec.query<{
|
|
1447
|
-
name: string;
|
|
1448
|
-
type: string;
|
|
1449
|
-
not_null: boolean;
|
|
1450
|
-
pk: number | null;
|
|
1451
|
-
}>(
|
|
1452
|
-
`SELECT attribute.attname AS name,
|
|
1453
|
-
format_type(attribute.atttypid, attribute.atttypmod) AS type,
|
|
1454
|
-
attribute.attnotnull AS not_null,
|
|
1455
|
-
(SELECT key.position::int
|
|
1456
|
-
FROM pg_index pk,
|
|
1457
|
-
unnest(pk.indkey::int2[]) WITH ORDINALITY
|
|
1458
|
-
AS key(attnum, position)
|
|
1459
|
-
WHERE pk.indrelid = attribute.attrelid AND pk.indisprimary
|
|
1460
|
-
AND key.attnum = attribute.attnum) AS pk
|
|
1461
|
-
FROM pg_attribute attribute
|
|
1462
|
-
WHERE attribute.attrelid = to_regclass($1)
|
|
1463
|
-
AND attribute.attnum > 0 AND NOT attribute.attisdropped`,
|
|
1464
|
-
[table.name],
|
|
1465
|
-
);
|
|
1466
1519
|
assertPhysicalColumns(
|
|
1467
1520
|
table,
|
|
1468
|
-
|
|
1469
|
-
|
|
1470
|
-
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1521
|
+
columns
|
|
1522
|
+
.filter((column) => column.tbl === table.name)
|
|
1523
|
+
.map((column) => ({
|
|
1524
|
+
name: column.name,
|
|
1525
|
+
type: column.type,
|
|
1526
|
+
notNull: column.not_null,
|
|
1527
|
+
primaryKeyPosition: column.pk ?? 0,
|
|
1528
|
+
})),
|
|
1474
1529
|
'postgres',
|
|
1475
1530
|
);
|
|
1476
1531
|
}
|
|
@@ -1483,9 +1538,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
|
|
|
1483
1538
|
// transaction (Postgres DDL is transactional — a failed bump leaves
|
|
1484
1539
|
// no half-state).
|
|
1485
1540
|
const layouts = parseLayouts(
|
|
1486
|
-
typeof marker.
|
|
1487
|
-
? marker.rows[0].layouts
|
|
1488
|
-
: undefined,
|
|
1541
|
+
typeof marker.layouts === 'string' ? marker.layouts : undefined,
|
|
1489
1542
|
);
|
|
1490
1543
|
const retiredTables = retiredTableNames(schema, layouts);
|
|
1491
1544
|
await this.#exec.transaction(async (client) => {
|
|
@@ -2262,12 +2315,11 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
|
|
|
2262
2315
|
? []
|
|
2263
2316
|
: (query.scopeFilter[firstVariable] ?? []);
|
|
2264
2317
|
if (firstVariable === undefined || values.length === 0) return 0;
|
|
2265
|
-
// Concurrent reads of
|
|
2318
|
+
// Concurrent reads of a pull share one statement across tables.
|
|
2266
2319
|
const [row] = await this.#batchedPage<{ seq: unknown }>(
|
|
2267
2320
|
LATEST_SCOPED_CHANGE_BATCH_SQL,
|
|
2268
2321
|
partition,
|
|
2269
|
-
query.table,
|
|
2270
|
-
{ var: firstVariable, vals: values },
|
|
2322
|
+
{ tbl: query.table, var: firstVariable, vals: values },
|
|
2271
2323
|
);
|
|
2272
2324
|
return row === undefined ? 0 : asNumber(row.seq);
|
|
2273
2325
|
}
|
|
@@ -2285,7 +2337,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
|
|
|
2285
2337
|
// var, value, commit_seq) PK makes the candidate subquery an index range
|
|
2286
2338
|
// scan returning commit_seq already ascending (see postgres-explain
|
|
2287
2339
|
// .test.ts), LEFT JOINed to the commit meta + the table's changes. Pages
|
|
2288
|
-
// of concurrent window reads
|
|
2340
|
+
// of concurrent window reads share one statement across tables (see
|
|
2289
2341
|
// `COMMIT_WINDOW_BATCH_SQL`). Exact multi-variable verification against
|
|
2290
2342
|
// the stored scope map happens below.
|
|
2291
2343
|
const commits: StoredCommit[] = [];
|
|
@@ -2296,8 +2348,8 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
|
|
|
2296
2348
|
const records = await this.#batchedPage<CommitWindowRecord>(
|
|
2297
2349
|
COMMIT_WINDOW_BATCH_SQL,
|
|
2298
2350
|
partition,
|
|
2299
|
-
query.table,
|
|
2300
2351
|
{
|
|
2352
|
+
tbl: query.table,
|
|
2301
2353
|
var: firstVariable,
|
|
2302
2354
|
vals: firstValues,
|
|
2303
2355
|
after: afterSeq,
|
|
@@ -2370,17 +2422,13 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`);
|
|
|
2370
2422
|
let afterRowId = query.afterRowId ?? '';
|
|
2371
2423
|
const batchSize = Math.max(64, query.limit);
|
|
2372
2424
|
while (rows.length < query.limit) {
|
|
2373
|
-
const records = await this.#batchedPage<RowRecord>(
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
after: afterRowId,
|
|
2381
|
-
lim: batchSize,
|
|
2382
|
-
},
|
|
2383
|
-
);
|
|
2425
|
+
const records = await this.#batchedPage<RowRecord>(sql, partition, {
|
|
2426
|
+
tbl: query.table,
|
|
2427
|
+
var: firstVariable,
|
|
2428
|
+
vals: firstValues,
|
|
2429
|
+
after: afterRowId,
|
|
2430
|
+
lim: batchSize,
|
|
2431
|
+
});
|
|
2384
2432
|
if (records.length === 0) break;
|
|
2385
2433
|
for (const record of records) {
|
|
2386
2434
|
afterRowId = record.row_id;
|
package/src/relational-rows.ts
CHANGED
|
@@ -647,20 +647,20 @@ export function commitWindowPageSql(
|
|
|
647
647
|
/**
|
|
648
648
|
* PostgreSQL set-based form of `scanRowPageSql`: one statement answers the
|
|
649
649
|
* page of every subscription scanning `table` in a pull. `$2` is a JSON array
|
|
650
|
-
* of `{i, var, vals, after, lim}` records; each record runs the same
|
|
650
|
+
* of `{i, tbl, var, vals, after, lim}` records; each record runs the same
|
|
651
651
|
* index-first candidate subquery through a LATERAL join, so every
|
|
652
652
|
* subscription keeps its own keyset cursor and LIMIT. Rows arrive ordered
|
|
653
653
|
* by (i, row_id).
|
|
654
654
|
*
|
|
655
|
-
* Params: [partition, specs (JSON text)
|
|
655
|
+
* Params: [partition, specs (JSON text)]. The specs bind as text before
|
|
656
656
|
* the JSONB cast, so a driver that JSON-encodes a string bound to a `jsonb`
|
|
657
657
|
* parameter (Bun.sql) cannot turn the array into a JSON string.
|
|
658
658
|
*/
|
|
659
659
|
export function scanRowBatchSql(table: CompiledTable): string {
|
|
660
660
|
return `SELECT s.i AS sub, c.row_id AS row_id, r.${quoteIdent(SYNC_VERSION_COLUMN)} AS server_version, r.${quoteIdent(SYNC_SCOPES_COLUMN)} AS scopes, r.${quoteIdent(SYNC_PAYLOAD_COLUMN)} AS payload, r.${quoteIdent(SYNC_COLUMN_VERSIONS_COLUMN)} AS column_versions
|
|
661
|
-
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, var text, vals text[], after text, lim int)
|
|
661
|
+
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], after text, lim int)
|
|
662
662
|
CROSS JOIN LATERAL (SELECT DISTINCT row_id FROM sync_row_scopes
|
|
663
|
-
WHERE partition=$1 AND tbl
|
|
663
|
+
WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=ANY(s.vals)
|
|
664
664
|
AND row_id>s.after
|
|
665
665
|
ORDER BY row_id LIMIT s.lim) c
|
|
666
666
|
LEFT JOIN ${quoteIdent(table.name)} r
|
|
@@ -670,25 +670,25 @@ export function scanRowBatchSql(table: CompiledTable): string {
|
|
|
670
670
|
|
|
671
671
|
/**
|
|
672
672
|
* PostgreSQL set-based form of `commitWindowPageSql`: one statement answers
|
|
673
|
-
* the window page of every subscription
|
|
674
|
-
* JSON array of `{i, var, vals, after, through, lim}` records, each
|
|
675
|
-
* the same index-first candidate subquery through a LATERAL join.
|
|
676
|
-
* arrive ordered by (i, commit_seq, idx).
|
|
673
|
+
* the window page of every subscription in a pull, across all tables. `$2`
|
|
674
|
+
* is a JSON array of `{i, tbl, var, vals, after, through, lim}` records, each
|
|
675
|
+
* running the same index-first candidate subquery through a LATERAL join.
|
|
676
|
+
* Rows arrive ordered by (i, commit_seq, idx).
|
|
677
677
|
*
|
|
678
|
-
* Params: [partition, specs (JSON text)
|
|
678
|
+
* Params: [partition, specs (JSON text)], the specs bound as text like
|
|
679
679
|
* `scanRowBatchSql`.
|
|
680
680
|
*/
|
|
681
681
|
export const COMMIT_WINDOW_BATCH_SQL = `SELECT s.i AS sub, c.commit_seq AS commit_seq, m.actor_id AS actor_id, m.created_at_ms AS created_at_ms,
|
|
682
682
|
ch.tbl AS tbl, ch.row_id AS row_id, ch.op AS op, ch.row_version AS row_version, ch.scopes AS scopes, ch.payload AS payload
|
|
683
|
-
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, var text, vals text[], after bigint, through bigint, lim int)
|
|
683
|
+
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals text[], after bigint, through bigint, lim int)
|
|
684
684
|
CROSS JOIN LATERAL (SELECT DISTINCT commit_seq FROM sync_change_scopes
|
|
685
|
-
WHERE partition=$1 AND tbl
|
|
685
|
+
WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=ANY(s.vals)
|
|
686
686
|
AND commit_seq>s.after AND commit_seq<=s.through
|
|
687
687
|
ORDER BY commit_seq LIMIT s.lim) c
|
|
688
688
|
LEFT JOIN sync_commits m
|
|
689
689
|
ON m.partition=$1 AND m.commit_seq=c.commit_seq
|
|
690
690
|
LEFT JOIN sync_changes ch
|
|
691
|
-
ON ch.partition=$1 AND ch.commit_seq=c.commit_seq AND ch.tbl
|
|
691
|
+
ON ch.partition=$1 AND ch.commit_seq=c.commit_seq AND ch.tbl=s.tbl
|
|
692
692
|
ORDER BY s.i, c.commit_seq, ch.idx`;
|
|
693
693
|
|
|
694
694
|
/**
|
|
@@ -720,15 +720,15 @@ export const LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL = `SELECT max((SELECT max(commi
|
|
|
720
720
|
|
|
721
721
|
/**
|
|
722
722
|
* PostgreSQL set-based form of `LATEST_SCOPED_CHANGE_SEQ_SQLITE_SQL` for the
|
|
723
|
-
* subscriptions of
|
|
724
|
-
* `{i, var, vals}` records. Params: [partition, specs (JSON text)
|
|
723
|
+
* subscriptions of a pull, across all tables. `$2` is a JSON array of
|
|
724
|
+
* `{i, tbl, var, vals}` records. Params: [partition, specs (JSON text)].
|
|
725
725
|
*/
|
|
726
726
|
export const LATEST_SCOPED_CHANGE_BATCH_SQL = `SELECT s.i AS sub,
|
|
727
727
|
(SELECT max(latest.seq) FROM (
|
|
728
728
|
SELECT (SELECT max(commit_seq) FROM sync_change_scopes
|
|
729
|
-
WHERE partition=$1 AND tbl
|
|
729
|
+
WHERE partition=$1 AND tbl=s.tbl AND var=s.var AND value=v.value) AS seq
|
|
730
730
|
FROM jsonb_array_elements_text(s.vals) AS v(value)) AS latest) AS seq
|
|
731
|
-
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, var text, vals jsonb)
|
|
731
|
+
FROM jsonb_to_recordset($2::text::jsonb) AS s(i int, tbl text, var text, vals jsonb)
|
|
732
732
|
ORDER BY s.i`;
|
|
733
733
|
|
|
734
734
|
/** SELECT a row's version + scope map (admin/blob authz). Params: [partition, rowId]. */
|