@syncular/server 0.62.1 → 0.64.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/d1-storage.js +72 -7
- package/dist/postgres-storage.d.ts +1 -1
- package/dist/postgres-storage.js +83 -16
- package/dist/relational-rows.d.ts +42 -4
- package/dist/relational-rows.js +106 -5
- package/dist/sqlite-storage.js +31 -7
- package/dist/storage-errors.d.ts +1 -1
- package/dist/storage-errors.js +2 -0
- package/package.json +2 -2
- package/src/d1-storage.ts +108 -6
- package/src/postgres-storage.ts +66 -17
- package/src/relational-rows.ts +133 -5
- package/src/sqlite-storage.ts +45 -5
- package/src/storage-errors.ts +6 -0
package/dist/d1-storage.js
CHANGED
|
@@ -40,7 +40,7 @@ var _a;
|
|
|
40
40
|
* rather than a lock D1 does not expose.
|
|
41
41
|
*/
|
|
42
42
|
import { decodeRow } from '@syncular/core';
|
|
43
|
-
import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanRowPageSql, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowSql, selectRowsSql, selectRowsForRewriteSql, tableColumnNames, toSqlValue, } from './relational-rows.js';
|
|
43
|
+
import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, changedScopePatterns, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, rekeyCleanupSql, rekeyedScopes, rekeyRowScopesSql, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanRowPageSql, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowSql, selectRowsSql, selectRowsForRewriteSql, tableColumnNames, toSqlValue, } from './relational-rows.js';
|
|
44
44
|
import { matchesEffective } from './scopes.js';
|
|
45
45
|
import { asUint8Array, deserializePushResult, placeholders, SqliteFamilyStorage, SqliteFamilyTransaction, sqliteDdlStatements, SQLITE_UPGRADE_PROBE, sqliteUpgradeDdl, toStoredRow, toTombstone, } from './sqlite-dialect.js';
|
|
46
46
|
import { commitDeleteFloor } from './storage.js';
|
|
@@ -678,6 +678,15 @@ export class D1ServerStorage extends SqliteFamilyStorage {
|
|
|
678
678
|
let claim = await read(this.#db.prepare('SELECT * FROM sync_schema_migration WHERE id=1'));
|
|
679
679
|
if (claim === null && marker?.schema_version === schema.version) {
|
|
680
680
|
assertStoredLayouts(schema, marker.layouts);
|
|
681
|
+
if (!assertStoredScopePatterns(schema, marker.layouts)) {
|
|
682
|
+
// A marker from before 0.63 records no scope patterns: record the
|
|
683
|
+
// configured ones, which every later start compares against.
|
|
684
|
+
statementsExecuted++;
|
|
685
|
+
await this.#db
|
|
686
|
+
.prepare('UPDATE sync_schema_meta SET layouts=? WHERE id=1 AND schema_version=?')
|
|
687
|
+
.bind(layoutsOf(schema), schema.version)
|
|
688
|
+
.run();
|
|
689
|
+
}
|
|
681
690
|
statementsExecuted++;
|
|
682
691
|
const present = await this.#db
|
|
683
692
|
.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name IN (${placeholders(D1_CORE_TABLES.length)})`)
|
|
@@ -779,6 +788,7 @@ export class D1ServerStorage extends SqliteFamilyStorage {
|
|
|
779
788
|
throw new StorageQueryError('sync.storage.schema_migration_conflict');
|
|
780
789
|
}
|
|
781
790
|
const sourceLayouts = parseLayouts(claim.source_layouts);
|
|
791
|
+
const rekeyed = (changedScopePatterns(schema, claim.source_layouts) ?? []).map((change) => change.table);
|
|
782
792
|
let state = JSON.parse(claim.state);
|
|
783
793
|
// Reserve one statement to distinguish a stale checkpoint from a database
|
|
784
794
|
// failure. A losing concurrent step returns incomplete without retrying here.
|
|
@@ -837,8 +847,44 @@ export class D1ServerStorage extends SqliteFamilyStorage {
|
|
|
837
847
|
else if (state.phase === 'prepare') {
|
|
838
848
|
const table = tables[state.offset];
|
|
839
849
|
if (table !== undefined) {
|
|
840
|
-
|
|
850
|
+
// §3.1: every row of a re-keyed table needs a value for each new
|
|
851
|
+
// pattern. Check before any DDL (one more statement); a refusal
|
|
852
|
+
// releases the claim, so the refused bump changes nothing.
|
|
853
|
+
const oldLayout = sourceLayouts[table.name];
|
|
854
|
+
const verify = rekeyed.includes(table.name) && oldLayout !== undefined;
|
|
855
|
+
if (remaining < (verify ? 5 : 4))
|
|
841
856
|
break;
|
|
857
|
+
if (verify) {
|
|
858
|
+
statementsExecuted++;
|
|
859
|
+
const { results } = await this.#db
|
|
860
|
+
.prepare(`SELECT * FROM (${selectRowsForRewriteSql(table, 'sqlite')})
|
|
861
|
+
WHERE EXISTS (SELECT 1 FROM sync_schema_migration WHERE id=1 AND target=? AND state=?)`)
|
|
862
|
+
.bind(state.afterPartition, state.afterRowId, 500, target, claim.state)
|
|
863
|
+
.all();
|
|
864
|
+
try {
|
|
865
|
+
for (const row of results) {
|
|
866
|
+
rekeyedScopes(table, decodeRow(oldLayout, asUint8Array(row.payload)), row.row_id);
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
catch (error) {
|
|
870
|
+
await batch([
|
|
871
|
+
this.#db
|
|
872
|
+
.prepare('DELETE FROM sync_schema_migration WHERE id=1 AND target=? AND state=?')
|
|
873
|
+
.bind(target, claim.state),
|
|
874
|
+
]);
|
|
875
|
+
throw error;
|
|
876
|
+
}
|
|
877
|
+
const last = results.at(-1);
|
|
878
|
+
if (last !== undefined && results.length === 500) {
|
|
879
|
+
if (!(await save({
|
|
880
|
+
...state,
|
|
881
|
+
afterPartition: last.partition,
|
|
882
|
+
afterRowId: last.row_id,
|
|
883
|
+
}, [])))
|
|
884
|
+
break;
|
|
885
|
+
continue;
|
|
886
|
+
}
|
|
887
|
+
}
|
|
842
888
|
statementsExecuted += 2;
|
|
843
889
|
const columns = await this.#db
|
|
844
890
|
.prepare(`PRAGMA table_info(${quoteIdent(table.name)})`)
|
|
@@ -854,6 +900,8 @@ export class D1ServerStorage extends SqliteFamilyStorage {
|
|
|
854
900
|
if (!(await save({
|
|
855
901
|
...state,
|
|
856
902
|
offset: state.offset + 1,
|
|
903
|
+
afterPartition: '',
|
|
904
|
+
afterRowId: '',
|
|
857
905
|
columns: {
|
|
858
906
|
...state.columns,
|
|
859
907
|
...(columns.results.length > 0
|
|
@@ -880,13 +928,15 @@ export class D1ServerStorage extends SqliteFamilyStorage {
|
|
|
880
928
|
]));
|
|
881
929
|
const rewrites = tables.flatMap((table) => {
|
|
882
930
|
const plan = rewritePlan(table, sourceLayouts[table.name], columns.get(table.name));
|
|
883
|
-
|
|
931
|
+
const rekey = rekeyed.includes(table.name);
|
|
932
|
+
return plan.migrate || plan.backfill || rekey
|
|
884
933
|
? [
|
|
885
934
|
{
|
|
886
935
|
table: table.name,
|
|
887
936
|
...(plan.migrate
|
|
888
937
|
? { oldLayout: sourceLayouts[table.name] }
|
|
889
938
|
: {}),
|
|
939
|
+
...(rekey ? { rekey: true } : {}),
|
|
890
940
|
},
|
|
891
941
|
]
|
|
892
942
|
: [];
|
|
@@ -898,6 +948,7 @@ export class D1ServerStorage extends SqliteFamilyStorage {
|
|
|
898
948
|
dropTableDdl(name),
|
|
899
949
|
]),
|
|
900
950
|
...schemaDdl(schema, columns, 'sqlite', indexes),
|
|
951
|
+
...rekeyCleanupSql(rekeyed),
|
|
901
952
|
];
|
|
902
953
|
if (!(await save({
|
|
903
954
|
...state,
|
|
@@ -938,24 +989,38 @@ export class D1ServerStorage extends SqliteFamilyStorage {
|
|
|
938
989
|
const table = schema.tables.get(plan.table);
|
|
939
990
|
if (table === undefined)
|
|
940
991
|
throw new StorageQueryError('sync.storage.schema_migration_conflict');
|
|
941
|
-
|
|
992
|
+
// A re-keyed page adds one scope-index statement.
|
|
993
|
+
const rekey = plan.rekey === true;
|
|
994
|
+
if (remaining < (rekey ? 5 : 4))
|
|
942
995
|
break;
|
|
943
|
-
const limit = Math.min(32, remaining - 3);
|
|
996
|
+
const limit = Math.min(32, remaining - (rekey ? 4 : 3));
|
|
944
997
|
statementsExecuted++;
|
|
945
998
|
const { results } = await this.#db
|
|
946
999
|
.prepare(`SELECT * FROM (${selectRowsForRewriteSql(table, 'sqlite')})
|
|
947
1000
|
WHERE EXISTS (SELECT 1 FROM sync_schema_migration WHERE id=1 AND target=? AND state=?)`)
|
|
948
1001
|
.bind(state.afterPartition, state.afterRowId, limit, target, claim.state)
|
|
949
1002
|
.all();
|
|
1003
|
+
const entries = [];
|
|
950
1004
|
const statements = results.map((row) => {
|
|
951
1005
|
const bytes = asUint8Array(row.payload);
|
|
952
1006
|
const payload = plan.oldLayout !== undefined
|
|
953
1007
|
? migratePayload(plan.oldLayout, table, bytes)
|
|
954
1008
|
: bytes;
|
|
1009
|
+
const scopes = rekey
|
|
1010
|
+
? rekeyedScopes(table, decodeRow(table.columns, payload), row.row_id)
|
|
1011
|
+
: undefined;
|
|
1012
|
+
for (const [variable, value] of Object.entries(scopes ?? {})) {
|
|
1013
|
+
entries.push([row.partition, variable, value, row.row_id]);
|
|
1014
|
+
}
|
|
955
1015
|
return this.#db
|
|
956
|
-
.prepare(rewriteRowSql(table))
|
|
957
|
-
.bind(...rewriteValues(table, row.partition, row.row_id, payload, 'sqlite'));
|
|
1016
|
+
.prepare(rewriteRowSql(table, rekey))
|
|
1017
|
+
.bind(...rewriteValues(table, row.partition, row.row_id, payload, 'sqlite', scopes));
|
|
958
1018
|
});
|
|
1019
|
+
if (entries.length > 0) {
|
|
1020
|
+
statements.push(this.#db
|
|
1021
|
+
.prepare(rekeyRowScopesSql(table, 'sqlite'))
|
|
1022
|
+
.bind(JSON.stringify(entries)));
|
|
1023
|
+
}
|
|
959
1024
|
const last = results.at(-1);
|
|
960
1025
|
const done = results.length < limit;
|
|
961
1026
|
if (!(await save({
|
|
@@ -36,7 +36,7 @@ import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
|
|
|
36
36
|
* would leave gaps on rollback, which the pull-window arithmetic in §4.5
|
|
37
37
|
* does not tolerate). Cross-partition pushes never contend.
|
|
38
38
|
*/
|
|
39
|
-
import type
|
|
39
|
+
import { type UnitScopes } from '@syncular/core';
|
|
40
40
|
import { type PgExecutor } from './pg-executor.js';
|
|
41
41
|
import type { CompiledSchema, CompiledTable } from './schema.js';
|
|
42
42
|
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientPushResult, ClientRecord, ClientRecordWriteOptions, CommitDetail, CommitMetadata, CommitMetadataQuery, IndexRowScanQuery, LiveWindow, LiveWindowQuery, PartitionRegistryEntry, PrunedReactionCounts, PushWatermark, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowChangeRecord, RowChangesQuery, RowListQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ScopeValueHistoryQuery, ScopeValueQuery, ServerStorage, ServeGate, StorageTransaction, StoredCheckpoint, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
package/dist/postgres-storage.js
CHANGED
|
@@ -2,10 +2,48 @@ import { validateCommitPruneQuery } from './prune.js';
|
|
|
2
2
|
import { StorageQueryError } from './storage-errors.js';
|
|
3
3
|
import { clientRoundState, PARTITION_TOUCH_INTERVAL_MS, commitDeleteFloor, rowsLiveAfterDelete, serveGateRefusal, } from './storage.js';
|
|
4
4
|
import { serveNotReadyError } from './readiness.js';
|
|
5
|
+
/**
|
|
6
|
+
* Postgres server storage: the production database path.
|
|
7
|
+
*
|
|
8
|
+
* Semantics mirror `SqliteServerStorage` exactly (both run the shared
|
|
9
|
+
* storage contract in `test/storage-contract.ts`); the difference is that
|
|
10
|
+
* scope fanout MUST survive contact with Postgres — no scan-before-LIMIT,
|
|
11
|
+
* ever (performance-by-construction):
|
|
12
|
+
*
|
|
13
|
+
* - both the commit log and the current-row table carry a (table, var,
|
|
14
|
+
* value) inverted scope index;
|
|
15
|
+
* - `readLiveWindow`/`scanRows` select candidates through an index
|
|
16
|
+
* (`sync_change_scopes_by_key` / `sync_row_scopes`), ordered + LIMITed
|
|
17
|
+
* at the index, then verify the full multi-variable match against the
|
|
18
|
+
* stored scope map — never a log or table scan;
|
|
19
|
+
* - the candidate indexes are *covering* for their selection: the ordered
|
|
20
|
+
* column (`commit_seq` / `row_id`) sits in the index key after the scope
|
|
21
|
+
* key, so the planner does an index range scan and returns
|
|
22
|
+
* already-ordered candidates without a heap sort. `test/postgres-explain
|
|
23
|
+
* .test.ts` asserts an `Index` node (never `Seq Scan`) so the regression
|
|
24
|
+
* cannot silently return.
|
|
25
|
+
*
|
|
26
|
+
* Storage is written against the `PgExecutor` seam (zero runtime deps); the
|
|
27
|
+
* production driver (Bun.sql / node-postgres) and the test driver (pglite)
|
|
28
|
+
* are wired by the host. See `pg-executor.ts` and the server README.
|
|
29
|
+
*
|
|
30
|
+
* ## commitSeq allocation under concurrency
|
|
31
|
+
*
|
|
32
|
+
* Per-partition `commitSeq` is a dense, gap-free counter (§2.1), allocated
|
|
33
|
+
* inside the push transaction by `UPDATE sync_partitions SET
|
|
34
|
+
* max_commit_seq = max_commit_seq + 1 … RETURNING`. The `UPDATE` takes a
|
|
35
|
+
* row-level write lock on the partition row for the duration of the
|
|
36
|
+
* transaction, so two concurrent pushes to the same partition serialize on
|
|
37
|
+
* that row: the second blocks until the first commits, then reads the
|
|
38
|
+
* advanced counter. This keeps the sequence dense (a Postgres `SEQUENCE`
|
|
39
|
+
* would leave gaps on rollback, which the pull-window arithmetic in §4.5
|
|
40
|
+
* does not tolerate). Cross-partition pushes never contend.
|
|
41
|
+
*/
|
|
42
|
+
import { decodeRow, } from '@syncular/core';
|
|
5
43
|
import { bindAuthoritativePartition, postgresPlaceholders, prepareAuthoritativeQuery, } from './authoritative-query.js';
|
|
6
44
|
import { syncError } from './errors.js';
|
|
7
45
|
import { asBytes, asNumber, } from './pg-executor.js';
|
|
8
|
-
import { assertPhysicalColumns, assertStoredLayouts, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, rowsReferencingBlobSql, selectRowSql, selectRowsSql, prefetchRowsBatchSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
46
|
+
import { assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, changedScopePatterns, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, rekeyCleanupSql, rekeyedScopes, rekeyRowScopesSql, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, rowsReferencingBlobSql, selectRowSql, selectRowsSql, prefetchRowsBatchSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
9
47
|
import { matchesEffective, sha256Hex } from './scopes.js';
|
|
10
48
|
import { isPostgresConstraintError, StorageConstraintError, } from './storage-errors.js';
|
|
11
49
|
import { assertScopeIndexedScan, readLiveWindowPages, resolveIndexRowScan, } from './storage-query.js';
|
|
@@ -568,30 +606,35 @@ function toStoredChange(record) {
|
|
|
568
606
|
* transaction. When `oldLayout` is
|
|
569
607
|
* given every payload re-encodes under the current columns, and the
|
|
570
608
|
* projection (when materialized) refreshes from the payload either way.
|
|
609
|
+
* `rekey` recomputes and indexes the stored scopes (§3.1).
|
|
571
610
|
*/
|
|
572
|
-
async function rewriteRowsOn(q, table, oldLayout) {
|
|
611
|
+
async function rewriteRowsOn(q, table, oldLayout, rekey) {
|
|
573
612
|
const select = selectRowsForRewriteSql(table, 'postgres');
|
|
574
613
|
// One UPDATE per page: behind a network hop (Hyperdrive, a managed
|
|
575
614
|
// database) every statement is a round trip, and the bump holds the
|
|
576
615
|
// migration lock for its whole duration. The page's rows bind as one
|
|
577
616
|
// VALUES list with typed placeholders (`rewriteValues` order: projection
|
|
578
|
-
// columns, payload, partition, row id).
|
|
617
|
+
// columns, re-keyed scopes, payload, partition, row id). The scopes bind as
|
|
618
|
+
// text before the JSONB cast (see `scanRowBatchSql`).
|
|
619
|
+
const projected = table.materialize ? table.columns : [];
|
|
579
620
|
const types = [
|
|
580
|
-
...(
|
|
581
|
-
|
|
582
|
-
: []),
|
|
621
|
+
...projected.map((column) => columnSqlType(column, 'postgres')),
|
|
622
|
+
...(rekey ? ['TEXT'] : []),
|
|
583
623
|
'BYTEA',
|
|
584
624
|
'TEXT',
|
|
585
625
|
'TEXT',
|
|
586
626
|
];
|
|
587
|
-
const
|
|
627
|
+
const payloadAt = projected.length + (rekey ? 1 : 0);
|
|
588
628
|
const sets = [
|
|
589
629
|
...projected.map((column, i) => `${quoteIdent(column.name)}=v.c${i}`),
|
|
590
|
-
|
|
630
|
+
...(rekey
|
|
631
|
+
? [`${quoteIdent(SYNC_SCOPES_COLUMN)}=v.c${projected.length}::jsonb`]
|
|
632
|
+
: []),
|
|
633
|
+
`${quoteIdent(SYNC_PAYLOAD_COLUMN)}=v.c${payloadAt}`,
|
|
591
634
|
].join(', ');
|
|
592
635
|
const aliases = types.map((_, i) => `c${i}`).join(', ');
|
|
593
|
-
const partitionAlias = `c${
|
|
594
|
-
const rowIdAlias = `c${
|
|
636
|
+
const partitionAlias = `c${payloadAt + 1}`;
|
|
637
|
+
const rowIdAlias = `c${payloadAt + 2}`;
|
|
595
638
|
// A page binds types.length parameters per row; PGlite corrupts its state
|
|
596
639
|
// above 32,767 bound parameters (see PG_CHANGES_PER_STATEMENT).
|
|
597
640
|
const batch = Math.max(1, Math.min(500, Math.floor(30_000 / types.length)));
|
|
@@ -602,19 +645,31 @@ async function rewriteRowsOn(q, table, oldLayout) {
|
|
|
602
645
|
if (rows.length === 0)
|
|
603
646
|
break;
|
|
604
647
|
const params = [];
|
|
648
|
+
const entries = [];
|
|
605
649
|
const values = rows.map((row) => {
|
|
606
650
|
const bytes = asBytes(row.payload);
|
|
607
651
|
const payload = oldLayout !== undefined
|
|
608
652
|
? migratePayload(oldLayout, table, bytes)
|
|
609
653
|
: bytes;
|
|
654
|
+
const scopes = rekey
|
|
655
|
+
? rekeyedScopes(table, decodeRow(table.columns, payload), row.row_id)
|
|
656
|
+
: undefined;
|
|
657
|
+
for (const [variable, value] of Object.entries(scopes ?? {})) {
|
|
658
|
+
entries.push([row.partition, variable, value, row.row_id]);
|
|
659
|
+
}
|
|
610
660
|
const base = params.length;
|
|
611
|
-
params.push(...rewriteValues(table, row.partition, row.row_id, payload, 'postgres'));
|
|
661
|
+
params.push(...rewriteValues(table, row.partition, row.row_id, payload, 'postgres', scopes));
|
|
612
662
|
return `(${types.map((type, i) => `$${base + i + 1}::${type}`).join(',')})`;
|
|
613
663
|
});
|
|
614
664
|
await q.query(`UPDATE ${quoteIdent(table.name)} AS t SET ${sets}
|
|
615
665
|
FROM (VALUES ${values.join(',')}) AS v(${aliases})
|
|
616
666
|
WHERE t.${quoteIdent(SYNC_PARTITION_COLUMN)}=v.${partitionAlias}
|
|
617
667
|
AND t.${quoteIdent(SYNC_ROW_ID_COLUMN)}=v.${rowIdAlias}`, params);
|
|
668
|
+
if (entries.length > 0) {
|
|
669
|
+
await q.query(rekeyRowScopesSql(table, 'postgres'), [
|
|
670
|
+
JSON.stringify(entries),
|
|
671
|
+
]);
|
|
672
|
+
}
|
|
618
673
|
const last = rows[rows.length - 1];
|
|
619
674
|
if (last === undefined || rows.length < batch)
|
|
620
675
|
break;
|
|
@@ -1619,7 +1674,13 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1619
1674
|
throw new Error(`stored schema version ${stored} is newer than the configured schema (${schema.version}) — refusing to run an older server against a migrated database`);
|
|
1620
1675
|
}
|
|
1621
1676
|
if (stored !== undefined && stored === schema.version) {
|
|
1622
|
-
|
|
1677
|
+
const storedLayouts = typeof marker.layouts === 'string' ? marker.layouts : undefined;
|
|
1678
|
+
assertStoredLayouts(schema, storedLayouts);
|
|
1679
|
+
if (!assertStoredScopePatterns(schema, storedLayouts)) {
|
|
1680
|
+
// A marker from before 0.63 records no scope patterns: record the
|
|
1681
|
+
// configured ones, which every later start compares against.
|
|
1682
|
+
await this.#exec.query('UPDATE sync_schema_meta SET layouts=$1 WHERE id=1 AND schema_version=$2', [layoutsOf(schema), schema.version]);
|
|
1683
|
+
}
|
|
1623
1684
|
const columns = JSON.parse(marker.columns);
|
|
1624
1685
|
for (const table of schema.tables.values()) {
|
|
1625
1686
|
assertPhysicalColumns(table, columns
|
|
@@ -1678,9 +1739,11 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1678
1739
|
}
|
|
1679
1740
|
if (lockedVersion === schema.version)
|
|
1680
1741
|
return;
|
|
1681
|
-
const
|
|
1742
|
+
const lockedLayouts = typeof lockedMarker[0]?.layouts === 'string'
|
|
1682
1743
|
? lockedMarker[0].layouts
|
|
1683
|
-
: undefined
|
|
1744
|
+
: undefined;
|
|
1745
|
+
const rekeyed = (changedScopePatterns(schema, lockedLayouts) ?? []).map((change) => change.table);
|
|
1746
|
+
const layouts = parseLayouts(lockedLayouts);
|
|
1684
1747
|
const retiredTables = retiredTableNames(schema, layouts);
|
|
1685
1748
|
const existing = new Map();
|
|
1686
1749
|
const existingIndexes = new Map();
|
|
@@ -1709,12 +1772,16 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1709
1772
|
for (const statement of schemaDdl(schema, existing, 'postgres', existingIndexes)) {
|
|
1710
1773
|
await client.query(statement);
|
|
1711
1774
|
}
|
|
1775
|
+
for (const statement of rekeyCleanupSql(rekeyed)) {
|
|
1776
|
+
await client.query(statement);
|
|
1777
|
+
}
|
|
1712
1778
|
for (const table of schema.tables.values()) {
|
|
1713
1779
|
const oldLayout = layouts[table.name];
|
|
1714
1780
|
const plan = rewritePlan(table, oldLayout, existing.get(table.name));
|
|
1715
|
-
|
|
1781
|
+
const rekey = rekeyed.includes(table.name);
|
|
1782
|
+
if (!plan.migrate && !plan.backfill && !rekey)
|
|
1716
1783
|
continue;
|
|
1717
|
-
await rewriteRowsOn(client, table, plan.migrate ? oldLayout : undefined);
|
|
1784
|
+
await rewriteRowsOn(client, table, plan.migrate ? oldLayout : undefined, rekey);
|
|
1718
1785
|
}
|
|
1719
1786
|
await client.query(`INSERT INTO sync_schema_meta(id, schema_version, layouts) VALUES (1, $1, $2)
|
|
1720
1787
|
ON CONFLICT (id) DO UPDATE
|
|
@@ -298,6 +298,35 @@ export type StoredLayouts = Record<string, readonly StoredColumnLayout[]>;
|
|
|
298
298
|
/** The layouts JSON persisted alongside the schema version marker. */
|
|
299
299
|
export declare function layoutsOf(schema: CompiledSchema): string;
|
|
300
300
|
export declare function parseLayouts(json: string | null | undefined): StoredLayouts;
|
|
301
|
+
/**
|
|
302
|
+
* §3.1: the configured tables whose scope patterns differ from the recorded
|
|
303
|
+
* ones, or `undefined` when the marker predates recorded patterns (nothing to
|
|
304
|
+
* compare). A schema-version bump re-keys these tables.
|
|
305
|
+
*/
|
|
306
|
+
export declare function changedScopePatterns(schema: CompiledSchema, storedJson: string | null | undefined): {
|
|
307
|
+
table: string;
|
|
308
|
+
stored: string;
|
|
309
|
+
configured: string;
|
|
310
|
+
}[] | undefined;
|
|
311
|
+
/**
|
|
312
|
+
* §3.1: at the same schema version, refuse a configured table whose patterns
|
|
313
|
+
* differ from the recorded ones. Returns false when the marker predates
|
|
314
|
+
* recorded patterns, so the caller records them.
|
|
315
|
+
*/
|
|
316
|
+
export declare function assertStoredScopePatterns(schema: CompiledSchema, storedJson: string | null | undefined): boolean;
|
|
317
|
+
/**
|
|
318
|
+
* §3.1 re-key, before the row walk: drop the re-keyed tables' row scope index
|
|
319
|
+
* entries and tombstones, and raise every partition's horizon past the
|
|
320
|
+
* changes logged under the previous patterns. Empty when nothing re-keys.
|
|
321
|
+
*/
|
|
322
|
+
export declare function rekeyCleanupSql(tables: readonly string[]): string[];
|
|
323
|
+
/**
|
|
324
|
+
* §3.1 re-key: index the scopes of one rewritten page of rows. The entries
|
|
325
|
+
* bind as one JSON array of `[partition, variable, value, rowId]`, so the
|
|
326
|
+
* statement reads no row table (a planner that ranges a page over a declared
|
|
327
|
+
* index would scan the whole partition per page). Params: [entries].
|
|
328
|
+
*/
|
|
329
|
+
export declare function rekeyRowScopesSql(table: CompiledTable, dialect: RelationalDialect): string;
|
|
301
330
|
/**
|
|
302
331
|
* Version equality is not layout equality: a marker written by another build
|
|
303
332
|
* at the same version describes rows the running codec cannot decode. Refuse
|
|
@@ -334,13 +363,22 @@ export declare function migratePayload(oldLayout: readonly StoredColumnLayout[],
|
|
|
334
363
|
* Params: [afterPartition, afterRowId, limit].
|
|
335
364
|
*/
|
|
336
365
|
export declare function selectRowsForRewriteSql(table: CompiledTable, dialect: RelationalDialect): string;
|
|
366
|
+
/**
|
|
367
|
+
* §3.1 re-key: the row's stored scopes under the table's current patterns. A
|
|
368
|
+
* row without a value for one of them refuses the bump.
|
|
369
|
+
*/
|
|
370
|
+
export declare function rekeyedScopes(table: CompiledTable, values: readonly RowValue[], rowId: string): Record<string, string>;
|
|
337
371
|
/**
|
|
338
372
|
* SQLite/D1: rewrite one row during a migration: refresh the projection columns (when
|
|
339
|
-
* materialized)
|
|
373
|
+
* materialized), the stored scopes when the table re-keys (§3.1), and the
|
|
374
|
+
* payload. Bind with `rewriteValues`.
|
|
375
|
+
*/
|
|
376
|
+
export declare function rewriteRowSql(table: CompiledTable, rekey: boolean): string;
|
|
377
|
+
/**
|
|
378
|
+
* Bind values for `rewriteRowSql` from the (already migrated) payload:
|
|
379
|
+
* projection columns, the re-keyed scopes JSON, payload, partition, row id.
|
|
340
380
|
*/
|
|
341
|
-
export declare function
|
|
342
|
-
/** Bind values for `rewriteRowSql` from the (already migrated) payload. */
|
|
343
|
-
export declare function rewriteValues(table: CompiledTable, partition: string, rowId: string, payload: Uint8Array, dialect: RelationalDialect): RowValue[];
|
|
381
|
+
export declare function rewriteValues(table: CompiledTable, partition: string, rowId: string, payload: Uint8Array, dialect: RelationalDialect, scopes: Record<string, string> | undefined): RowValue[];
|
|
344
382
|
/**
|
|
345
383
|
* What the migration rewrite phase must do for one table, derived from the
|
|
346
384
|
* persisted layouts + the physical columns present before this run.
|
package/dist/relational-rows.js
CHANGED
|
@@ -48,6 +48,7 @@
|
|
|
48
48
|
* NOT NULL per the schema.
|
|
49
49
|
*/
|
|
50
50
|
import { decodeRow, encodeRow, } from '@syncular/core';
|
|
51
|
+
import { storedScopesForRow } from './scopes.js';
|
|
51
52
|
import { StorageQueryError } from './storage-errors.js';
|
|
52
53
|
export const SYNC_PARTITION_COLUMN = '_sync_partition';
|
|
53
54
|
export const SYNC_ROW_ID_COLUMN = '_sync_row_id';
|
|
@@ -701,19 +702,97 @@ export const SCHEMA_META_DDL_POSTGRES = `CREATE TABLE IF NOT EXISTS sync_schema_
|
|
|
701
702
|
/** The layouts JSON persisted alongside the schema version marker. */
|
|
702
703
|
export function layoutsOf(schema) {
|
|
703
704
|
const out = {};
|
|
705
|
+
const scopes = {};
|
|
704
706
|
for (const table of schema.tables.values()) {
|
|
705
707
|
out[table.name] = table.columns.map((column) => ({
|
|
706
708
|
name: column.name,
|
|
707
709
|
type: column.type,
|
|
708
710
|
nullable: column.nullable,
|
|
709
711
|
}));
|
|
712
|
+
scopes[table.name] = scopePatternsOf(table);
|
|
710
713
|
}
|
|
714
|
+
// `sync_` is a reserved table-name prefix, so this key names no table.
|
|
715
|
+
out.sync_scopes = scopes;
|
|
711
716
|
return JSON.stringify(out);
|
|
712
717
|
}
|
|
718
|
+
function scopePatternsOf(table) {
|
|
719
|
+
return table.scopePatterns
|
|
720
|
+
.map(({ prefix, variable, column }) => `${prefix}:{${variable}}@${column}`)
|
|
721
|
+
.sort();
|
|
722
|
+
}
|
|
713
723
|
export function parseLayouts(json) {
|
|
714
724
|
if (json === null || json === undefined || json.length === 0)
|
|
715
725
|
return {};
|
|
716
|
-
|
|
726
|
+
const { sync_scopes: _scopes, ...layouts } = JSON.parse(json);
|
|
727
|
+
return layouts;
|
|
728
|
+
}
|
|
729
|
+
/**
|
|
730
|
+
* §3.1: the configured tables whose scope patterns differ from the recorded
|
|
731
|
+
* ones, or `undefined` when the marker predates recorded patterns (nothing to
|
|
732
|
+
* compare). A schema-version bump re-keys these tables.
|
|
733
|
+
*/
|
|
734
|
+
export function changedScopePatterns(schema, storedJson) {
|
|
735
|
+
if (storedJson === null || storedJson === undefined || storedJson === '')
|
|
736
|
+
return [];
|
|
737
|
+
const stored = JSON.parse(storedJson).sync_scopes;
|
|
738
|
+
if (stored === undefined)
|
|
739
|
+
return undefined;
|
|
740
|
+
return [...schema.tables.values()].flatMap((table) => {
|
|
741
|
+
const before = stored[table.name];
|
|
742
|
+
const after = scopePatternsOf(table);
|
|
743
|
+
return before !== undefined && before.join('\n') !== after.join('\n')
|
|
744
|
+
? [
|
|
745
|
+
{
|
|
746
|
+
table: table.name,
|
|
747
|
+
stored: before.join(' '),
|
|
748
|
+
configured: after.join(' '),
|
|
749
|
+
},
|
|
750
|
+
]
|
|
751
|
+
: [];
|
|
752
|
+
});
|
|
753
|
+
}
|
|
754
|
+
/**
|
|
755
|
+
* §3.1: at the same schema version, refuse a configured table whose patterns
|
|
756
|
+
* differ from the recorded ones. Returns false when the marker predates
|
|
757
|
+
* recorded patterns, so the caller records them.
|
|
758
|
+
*/
|
|
759
|
+
export function assertStoredScopePatterns(schema, storedJson) {
|
|
760
|
+
const changed = changedScopePatterns(schema, storedJson);
|
|
761
|
+
if (changed?.[0] !== undefined) {
|
|
762
|
+
throw new StorageQueryError('sync.storage.scope_patterns_changed', changed[0]);
|
|
763
|
+
}
|
|
764
|
+
return changed !== undefined;
|
|
765
|
+
}
|
|
766
|
+
/**
|
|
767
|
+
* §3.1 re-key, before the row walk: drop the re-keyed tables' row scope index
|
|
768
|
+
* entries and tombstones, and raise every partition's horizon past the
|
|
769
|
+
* changes logged under the previous patterns. Empty when nothing re-keys.
|
|
770
|
+
*/
|
|
771
|
+
export function rekeyCleanupSql(tables) {
|
|
772
|
+
if (tables.length === 0)
|
|
773
|
+
return [];
|
|
774
|
+
const names = tables.map((name) => `'${name.replaceAll("'", "''")}'`);
|
|
775
|
+
return [
|
|
776
|
+
`DELETE FROM sync_row_scopes WHERE tbl IN (${names.join(',')})`,
|
|
777
|
+
`DELETE FROM sync_tombstones WHERE tbl IN (${names.join(',')})`,
|
|
778
|
+
'UPDATE sync_partitions SET horizon_seq=max_commit_seq WHERE horizon_seq<max_commit_seq',
|
|
779
|
+
];
|
|
780
|
+
}
|
|
781
|
+
/**
|
|
782
|
+
* §3.1 re-key: index the scopes of one rewritten page of rows. The entries
|
|
783
|
+
* bind as one JSON array of `[partition, variable, value, rowId]`, so the
|
|
784
|
+
* statement reads no row table (a planner that ranges a page over a declared
|
|
785
|
+
* index would scan the whole partition per page). Params: [entries].
|
|
786
|
+
*/
|
|
787
|
+
export function rekeyRowScopesSql(table, dialect) {
|
|
788
|
+
const name = `'${table.name.replaceAll("'", "''")}'`;
|
|
789
|
+
return dialect === 'sqlite'
|
|
790
|
+
? `INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
|
|
791
|
+
SELECT json_extract(e.value, '$[0]'), ${name}, json_extract(e.value, '$[1]'), json_extract(e.value, '$[2]'), json_extract(e.value, '$[3]')
|
|
792
|
+
FROM json_each(?) AS e`
|
|
793
|
+
: `INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
|
|
794
|
+
SELECT e->>0, ${name}, e->>1, e->>2, e->>3
|
|
795
|
+
FROM jsonb_array_elements($1::text::jsonb) AS e`;
|
|
717
796
|
}
|
|
718
797
|
/**
|
|
719
798
|
* Version equality is not layout equality: a marker written by another build
|
|
@@ -820,21 +899,41 @@ export function selectRowsForRewriteSql(table, dialect) {
|
|
|
820
899
|
ORDER BY ${quoteIdent(SYNC_PARTITION_COLUMN)}, ${quoteIdent(SYNC_ROW_ID_COLUMN)}
|
|
821
900
|
LIMIT ${p[2]}`;
|
|
822
901
|
}
|
|
902
|
+
/**
|
|
903
|
+
* §3.1 re-key: the row's stored scopes under the table's current patterns. A
|
|
904
|
+
* row without a value for one of them refuses the bump.
|
|
905
|
+
*/
|
|
906
|
+
export function rekeyedScopes(table, values, rowId) {
|
|
907
|
+
const extracted = storedScopesForRow(table, values);
|
|
908
|
+
if ('missing' in extracted) {
|
|
909
|
+
throw new StorageQueryError('sync.storage.scope_value_missing', {
|
|
910
|
+
table: table.name,
|
|
911
|
+
variable: extracted.missing,
|
|
912
|
+
rowId,
|
|
913
|
+
});
|
|
914
|
+
}
|
|
915
|
+
return extracted.scopes;
|
|
916
|
+
}
|
|
823
917
|
/**
|
|
824
918
|
* SQLite/D1: rewrite one row during a migration: refresh the projection columns (when
|
|
825
|
-
* materialized)
|
|
919
|
+
* materialized), the stored scopes when the table re-keys (§3.1), and the
|
|
920
|
+
* payload. Bind with `rewriteValues`.
|
|
826
921
|
*/
|
|
827
|
-
export function rewriteRowSql(table) {
|
|
922
|
+
export function rewriteRowSql(table, rekey) {
|
|
828
923
|
const sets = [
|
|
829
924
|
...(table.materialize
|
|
830
925
|
? table.columns.map((column) => `${quoteIdent(column.name)}=?`)
|
|
831
926
|
: []),
|
|
927
|
+
...(rekey ? [`${quoteIdent(SYNC_SCOPES_COLUMN)}=?`] : []),
|
|
832
928
|
`${quoteIdent(SYNC_PAYLOAD_COLUMN)}=?`,
|
|
833
929
|
];
|
|
834
930
|
return `UPDATE ${quoteIdent(table.name)} SET ${sets.join(', ')} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=? AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=?`;
|
|
835
931
|
}
|
|
836
|
-
/**
|
|
837
|
-
|
|
932
|
+
/**
|
|
933
|
+
* Bind values for `rewriteRowSql` from the (already migrated) payload:
|
|
934
|
+
* projection columns, the re-keyed scopes JSON, payload, partition, row id.
|
|
935
|
+
*/
|
|
936
|
+
export function rewriteValues(table, partition, rowId, payload, dialect, scopes) {
|
|
838
937
|
const out = [];
|
|
839
938
|
if (table.materialize) {
|
|
840
939
|
const values = decodeRow(table.columns, payload);
|
|
@@ -845,6 +944,8 @@ export function rewriteValues(table, partition, rowId, payload, dialect) {
|
|
|
845
944
|
out.push(toSqlValue(column, values[i] ?? null, dialect));
|
|
846
945
|
}
|
|
847
946
|
}
|
|
947
|
+
if (scopes !== undefined)
|
|
948
|
+
out.push(JSON.stringify(scopes));
|
|
848
949
|
out.push(payload, partition, rowId);
|
|
849
950
|
return out;
|
|
850
951
|
}
|