@syncular/server 0.63.0 → 0.64.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/d1-storage.js +63 -8
- package/dist/postgres-storage.d.ts +1 -1
- package/dist/postgres-storage.js +73 -14
- package/dist/relational-rows.d.ts +39 -8
- package/dist/relational-rows.js +86 -20
- package/dist/sqlite-storage.js +24 -8
- package/dist/storage-errors.d.ts +1 -1
- package/dist/storage-errors.js +1 -0
- package/package.json +2 -2
- package/src/d1-storage.ts +96 -7
- package/src/postgres-storage.ts +52 -12
- package/src/relational-rows.ts +102 -20
- package/src/sqlite-storage.ts +35 -6
- package/src/storage-errors.ts +3 -0
package/dist/d1-storage.js
CHANGED
|
@@ -40,7 +40,7 @@ var _a;
|
|
|
40
40
|
* rather than a lock D1 does not expose.
|
|
41
41
|
*/
|
|
42
42
|
import { decodeRow } from '@syncular/core';
|
|
43
|
-
import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanRowPageSql, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowSql, selectRowsSql, selectRowsForRewriteSql, tableColumnNames, toSqlValue, } from './relational-rows.js';
|
|
43
|
+
import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, changedScopePatterns, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, rekeyCleanupSql, rekeyedScopes, rekeyRowScopesSql, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanRowPageSql, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowSql, selectRowsSql, selectRowsForRewriteSql, tableColumnNames, toSqlValue, } from './relational-rows.js';
|
|
44
44
|
import { matchesEffective } from './scopes.js';
|
|
45
45
|
import { asUint8Array, deserializePushResult, placeholders, SqliteFamilyStorage, SqliteFamilyTransaction, sqliteDdlStatements, SQLITE_UPGRADE_PROBE, sqliteUpgradeDdl, toStoredRow, toTombstone, } from './sqlite-dialect.js';
|
|
46
46
|
import { commitDeleteFloor } from './storage.js';
|
|
@@ -744,7 +744,6 @@ export class D1ServerStorage extends SqliteFamilyStorage {
|
|
|
744
744
|
throw new StorageQueryError('sync.storage.schema_changed');
|
|
745
745
|
}
|
|
746
746
|
if (claim === null) {
|
|
747
|
-
assertStoredScopePatterns(schema, marker?.layouts);
|
|
748
747
|
const layouts = parseLayouts(marker?.layouts);
|
|
749
748
|
for (const table of tables) {
|
|
750
749
|
const oldLayout = layouts[table.name];
|
|
@@ -789,6 +788,7 @@ export class D1ServerStorage extends SqliteFamilyStorage {
|
|
|
789
788
|
throw new StorageQueryError('sync.storage.schema_migration_conflict');
|
|
790
789
|
}
|
|
791
790
|
const sourceLayouts = parseLayouts(claim.source_layouts);
|
|
791
|
+
const rekeyed = (changedScopePatterns(schema, claim.source_layouts) ?? []).map((change) => change.table);
|
|
792
792
|
let state = JSON.parse(claim.state);
|
|
793
793
|
// Reserve one statement to distinguish a stale checkpoint from a database
|
|
794
794
|
// failure. A losing concurrent step returns incomplete without retrying here.
|
|
@@ -847,8 +847,44 @@ export class D1ServerStorage extends SqliteFamilyStorage {
|
|
|
847
847
|
else if (state.phase === 'prepare') {
|
|
848
848
|
const table = tables[state.offset];
|
|
849
849
|
if (table !== undefined) {
|
|
850
|
-
|
|
850
|
+
// §3.1: every row of a re-keyed table needs a value for each new
|
|
851
|
+
// pattern. Check before any DDL (one more statement); a refusal
|
|
852
|
+
// releases the claim, so the refused bump changes nothing.
|
|
853
|
+
const oldLayout = sourceLayouts[table.name];
|
|
854
|
+
const verify = rekeyed.includes(table.name) && oldLayout !== undefined;
|
|
855
|
+
if (remaining < (verify ? 5 : 4))
|
|
851
856
|
break;
|
|
857
|
+
if (verify) {
|
|
858
|
+
statementsExecuted++;
|
|
859
|
+
const { results } = await this.#db
|
|
860
|
+
.prepare(`SELECT * FROM (${selectRowsForRewriteSql(table, 'sqlite')})
|
|
861
|
+
WHERE EXISTS (SELECT 1 FROM sync_schema_migration WHERE id=1 AND target=? AND state=?)`)
|
|
862
|
+
.bind(state.afterPartition, state.afterRowId, 500, target, claim.state)
|
|
863
|
+
.all();
|
|
864
|
+
try {
|
|
865
|
+
for (const row of results) {
|
|
866
|
+
rekeyedScopes(table, decodeRow(oldLayout, asUint8Array(row.payload)), row.row_id);
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
catch (error) {
|
|
870
|
+
await batch([
|
|
871
|
+
this.#db
|
|
872
|
+
.prepare('DELETE FROM sync_schema_migration WHERE id=1 AND target=? AND state=?')
|
|
873
|
+
.bind(target, claim.state),
|
|
874
|
+
]);
|
|
875
|
+
throw error;
|
|
876
|
+
}
|
|
877
|
+
const last = results.at(-1);
|
|
878
|
+
if (last !== undefined && results.length === 500) {
|
|
879
|
+
if (!(await save({
|
|
880
|
+
...state,
|
|
881
|
+
afterPartition: last.partition,
|
|
882
|
+
afterRowId: last.row_id,
|
|
883
|
+
}, [])))
|
|
884
|
+
break;
|
|
885
|
+
continue;
|
|
886
|
+
}
|
|
887
|
+
}
|
|
852
888
|
statementsExecuted += 2;
|
|
853
889
|
const columns = await this.#db
|
|
854
890
|
.prepare(`PRAGMA table_info(${quoteIdent(table.name)})`)
|
|
@@ -864,6 +900,8 @@ export class D1ServerStorage extends SqliteFamilyStorage {
|
|
|
864
900
|
if (!(await save({
|
|
865
901
|
...state,
|
|
866
902
|
offset: state.offset + 1,
|
|
903
|
+
afterPartition: '',
|
|
904
|
+
afterRowId: '',
|
|
867
905
|
columns: {
|
|
868
906
|
...state.columns,
|
|
869
907
|
...(columns.results.length > 0
|
|
@@ -890,13 +928,15 @@ export class D1ServerStorage extends SqliteFamilyStorage {
|
|
|
890
928
|
]));
|
|
891
929
|
const rewrites = tables.flatMap((table) => {
|
|
892
930
|
const plan = rewritePlan(table, sourceLayouts[table.name], columns.get(table.name));
|
|
893
|
-
|
|
931
|
+
const rekey = rekeyed.includes(table.name);
|
|
932
|
+
return plan.migrate || plan.backfill || rekey
|
|
894
933
|
? [
|
|
895
934
|
{
|
|
896
935
|
table: table.name,
|
|
897
936
|
...(plan.migrate
|
|
898
937
|
? { oldLayout: sourceLayouts[table.name] }
|
|
899
938
|
: {}),
|
|
939
|
+
...(rekey ? { rekey: true } : {}),
|
|
900
940
|
},
|
|
901
941
|
]
|
|
902
942
|
: [];
|
|
@@ -908,6 +948,7 @@ export class D1ServerStorage extends SqliteFamilyStorage {
|
|
|
908
948
|
dropTableDdl(name),
|
|
909
949
|
]),
|
|
910
950
|
...schemaDdl(schema, columns, 'sqlite', indexes),
|
|
951
|
+
...rekeyCleanupSql(rekeyed),
|
|
911
952
|
];
|
|
912
953
|
if (!(await save({
|
|
913
954
|
...state,
|
|
@@ -948,24 +989,38 @@ export class D1ServerStorage extends SqliteFamilyStorage {
|
|
|
948
989
|
const table = schema.tables.get(plan.table);
|
|
949
990
|
if (table === undefined)
|
|
950
991
|
throw new StorageQueryError('sync.storage.schema_migration_conflict');
|
|
951
|
-
|
|
992
|
+
// A re-keyed page adds one scope-index statement.
|
|
993
|
+
const rekey = plan.rekey === true;
|
|
994
|
+
if (remaining < (rekey ? 5 : 4))
|
|
952
995
|
break;
|
|
953
|
-
const limit = Math.min(32, remaining - 3);
|
|
996
|
+
const limit = Math.min(32, remaining - (rekey ? 4 : 3));
|
|
954
997
|
statementsExecuted++;
|
|
955
998
|
const { results } = await this.#db
|
|
956
999
|
.prepare(`SELECT * FROM (${selectRowsForRewriteSql(table, 'sqlite')})
|
|
957
1000
|
WHERE EXISTS (SELECT 1 FROM sync_schema_migration WHERE id=1 AND target=? AND state=?)`)
|
|
958
1001
|
.bind(state.afterPartition, state.afterRowId, limit, target, claim.state)
|
|
959
1002
|
.all();
|
|
1003
|
+
const entries = [];
|
|
960
1004
|
const statements = results.map((row) => {
|
|
961
1005
|
const bytes = asUint8Array(row.payload);
|
|
962
1006
|
const payload = plan.oldLayout !== undefined
|
|
963
1007
|
? migratePayload(plan.oldLayout, table, bytes)
|
|
964
1008
|
: bytes;
|
|
1009
|
+
const scopes = rekey
|
|
1010
|
+
? rekeyedScopes(table, decodeRow(table.columns, payload), row.row_id)
|
|
1011
|
+
: undefined;
|
|
1012
|
+
for (const [variable, value] of Object.entries(scopes ?? {})) {
|
|
1013
|
+
entries.push([row.partition, variable, value, row.row_id]);
|
|
1014
|
+
}
|
|
965
1015
|
return this.#db
|
|
966
|
-
.prepare(rewriteRowSql(table))
|
|
967
|
-
.bind(...rewriteValues(table, row.partition, row.row_id, payload, 'sqlite'));
|
|
1016
|
+
.prepare(rewriteRowSql(table, rekey))
|
|
1017
|
+
.bind(...rewriteValues(table, row.partition, row.row_id, payload, 'sqlite', scopes));
|
|
968
1018
|
});
|
|
1019
|
+
if (entries.length > 0) {
|
|
1020
|
+
statements.push(this.#db
|
|
1021
|
+
.prepare(rekeyRowScopesSql(table, 'sqlite'))
|
|
1022
|
+
.bind(JSON.stringify(entries)));
|
|
1023
|
+
}
|
|
969
1024
|
const last = results.at(-1);
|
|
970
1025
|
const done = results.length < limit;
|
|
971
1026
|
if (!(await save({
|
|
@@ -36,7 +36,7 @@ import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
|
|
|
36
36
|
* would leave gaps on rollback, which the pull-window arithmetic in §4.5
|
|
37
37
|
* does not tolerate). Cross-partition pushes never contend.
|
|
38
38
|
*/
|
|
39
|
-
import type
|
|
39
|
+
import { type UnitScopes } from '@syncular/core';
|
|
40
40
|
import { type PgExecutor } from './pg-executor.js';
|
|
41
41
|
import type { CompiledSchema, CompiledTable } from './schema.js';
|
|
42
42
|
import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientPushResult, ClientRecord, ClientRecordWriteOptions, CommitDetail, CommitMetadata, CommitMetadataQuery, IndexRowScanQuery, LiveWindow, LiveWindowQuery, PartitionRegistryEntry, PrunedReactionCounts, PushWatermark, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowChangeRecord, RowChangesQuery, RowListQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ScopeValueHistoryQuery, ScopeValueQuery, ServerStorage, ServeGate, StorageTransaction, StoredCheckpoint, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
|
package/dist/postgres-storage.js
CHANGED
|
@@ -2,10 +2,48 @@ import { validateCommitPruneQuery } from './prune.js';
|
|
|
2
2
|
import { StorageQueryError } from './storage-errors.js';
|
|
3
3
|
import { clientRoundState, PARTITION_TOUCH_INTERVAL_MS, commitDeleteFloor, rowsLiveAfterDelete, serveGateRefusal, } from './storage.js';
|
|
4
4
|
import { serveNotReadyError } from './readiness.js';
|
|
5
|
+
/**
|
|
6
|
+
* Postgres server storage: the production database path.
|
|
7
|
+
*
|
|
8
|
+
* Semantics mirror `SqliteServerStorage` exactly (both run the shared
|
|
9
|
+
* storage contract in `test/storage-contract.ts`); the difference is that
|
|
10
|
+
* scope fanout MUST survive contact with Postgres — no scan-before-LIMIT,
|
|
11
|
+
* ever (performance-by-construction):
|
|
12
|
+
*
|
|
13
|
+
* - both the commit log and the current-row table carry a (table, var,
|
|
14
|
+
* value) inverted scope index;
|
|
15
|
+
* - `readLiveWindow`/`scanRows` select candidates through an index
|
|
16
|
+
* (`sync_change_scopes_by_key` / `sync_row_scopes`), ordered + LIMITed
|
|
17
|
+
* at the index, then verify the full multi-variable match against the
|
|
18
|
+
* stored scope map — never a log or table scan;
|
|
19
|
+
* - the candidate indexes are *covering* for their selection: the ordered
|
|
20
|
+
* column (`commit_seq` / `row_id`) sits in the index key after the scope
|
|
21
|
+
* key, so the planner does an index range scan and returns
|
|
22
|
+
* already-ordered candidates without a heap sort. `test/postgres-explain
|
|
23
|
+
* .test.ts` asserts an `Index` node (never `Seq Scan`) so the regression
|
|
24
|
+
* cannot silently return.
|
|
25
|
+
*
|
|
26
|
+
* Storage is written against the `PgExecutor` seam (zero runtime deps); the
|
|
27
|
+
* production driver (Bun.sql / node-postgres) and the test driver (pglite)
|
|
28
|
+
* are wired by the host. See `pg-executor.ts` and the server README.
|
|
29
|
+
*
|
|
30
|
+
* ## commitSeq allocation under concurrency
|
|
31
|
+
*
|
|
32
|
+
* Per-partition `commitSeq` is a dense, gap-free counter (§2.1), allocated
|
|
33
|
+
* inside the push transaction by `UPDATE sync_partitions SET
|
|
34
|
+
* max_commit_seq = max_commit_seq + 1 … RETURNING`. The `UPDATE` takes a
|
|
35
|
+
* row-level write lock on the partition row for the duration of the
|
|
36
|
+
* transaction, so two concurrent pushes to the same partition serialize on
|
|
37
|
+
* that row: the second blocks until the first commits, then reads the
|
|
38
|
+
* advanced counter. This keeps the sequence dense (a Postgres `SEQUENCE`
|
|
39
|
+
* would leave gaps on rollback, which the pull-window arithmetic in §4.5
|
|
40
|
+
* does not tolerate). Cross-partition pushes never contend.
|
|
41
|
+
*/
|
|
42
|
+
import { decodeRow, } from '@syncular/core';
|
|
5
43
|
import { bindAuthoritativePartition, postgresPlaceholders, prepareAuthoritativeQuery, } from './authoritative-query.js';
|
|
6
44
|
import { syncError } from './errors.js';
|
|
7
45
|
import { asBytes, asNumber, } from './pg-executor.js';
|
|
8
|
-
import { assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, rowsReferencingBlobSql, selectRowSql, selectRowsSql, prefetchRowsBatchSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
46
|
+
import { assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, changedScopePatterns, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, rekeyCleanupSql, rekeyedScopes, rekeyRowScopesSql, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, rowsReferencingBlobSql, selectRowSql, selectRowsSql, prefetchRowsBatchSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
|
|
9
47
|
import { matchesEffective, sha256Hex } from './scopes.js';
|
|
10
48
|
import { isPostgresConstraintError, StorageConstraintError, } from './storage-errors.js';
|
|
11
49
|
import { assertScopeIndexedScan, readLiveWindowPages, resolveIndexRowScan, } from './storage-query.js';
|
|
@@ -568,30 +606,35 @@ function toStoredChange(record) {
|
|
|
568
606
|
* transaction. When `oldLayout` is
|
|
569
607
|
* given every payload re-encodes under the current columns, and the
|
|
570
608
|
* projection (when materialized) refreshes from the payload either way.
|
|
609
|
+
* `rekey` recomputes and indexes the stored scopes (§3.1).
|
|
571
610
|
*/
|
|
572
|
-
async function rewriteRowsOn(q, table, oldLayout) {
|
|
611
|
+
async function rewriteRowsOn(q, table, oldLayout, rekey) {
|
|
573
612
|
const select = selectRowsForRewriteSql(table, 'postgres');
|
|
574
613
|
// One UPDATE per page: behind a network hop (Hyperdrive, a managed
|
|
575
614
|
// database) every statement is a round trip, and the bump holds the
|
|
576
615
|
// migration lock for its whole duration. The page's rows bind as one
|
|
577
616
|
// VALUES list with typed placeholders (`rewriteValues` order: projection
|
|
578
|
-
// columns, payload, partition, row id).
|
|
617
|
+
// columns, re-keyed scopes, payload, partition, row id). The scopes bind as
|
|
618
|
+
// text before the JSONB cast (see `scanRowBatchSql`).
|
|
619
|
+
const projected = table.materialize ? table.columns : [];
|
|
579
620
|
const types = [
|
|
580
|
-
...(
|
|
581
|
-
|
|
582
|
-
: []),
|
|
621
|
+
...projected.map((column) => columnSqlType(column, 'postgres')),
|
|
622
|
+
...(rekey ? ['TEXT'] : []),
|
|
583
623
|
'BYTEA',
|
|
584
624
|
'TEXT',
|
|
585
625
|
'TEXT',
|
|
586
626
|
];
|
|
587
|
-
const
|
|
627
|
+
const payloadAt = projected.length + (rekey ? 1 : 0);
|
|
588
628
|
const sets = [
|
|
589
629
|
...projected.map((column, i) => `${quoteIdent(column.name)}=v.c${i}`),
|
|
590
|
-
|
|
630
|
+
...(rekey
|
|
631
|
+
? [`${quoteIdent(SYNC_SCOPES_COLUMN)}=v.c${projected.length}::jsonb`]
|
|
632
|
+
: []),
|
|
633
|
+
`${quoteIdent(SYNC_PAYLOAD_COLUMN)}=v.c${payloadAt}`,
|
|
591
634
|
].join(', ');
|
|
592
635
|
const aliases = types.map((_, i) => `c${i}`).join(', ');
|
|
593
|
-
const partitionAlias = `c${
|
|
594
|
-
const rowIdAlias = `c${
|
|
636
|
+
const partitionAlias = `c${payloadAt + 1}`;
|
|
637
|
+
const rowIdAlias = `c${payloadAt + 2}`;
|
|
595
638
|
// A page binds types.length parameters per row; PGlite corrupts its state
|
|
596
639
|
// above 32,767 bound parameters (see PG_CHANGES_PER_STATEMENT).
|
|
597
640
|
const batch = Math.max(1, Math.min(500, Math.floor(30_000 / types.length)));
|
|
@@ -602,19 +645,31 @@ async function rewriteRowsOn(q, table, oldLayout) {
|
|
|
602
645
|
if (rows.length === 0)
|
|
603
646
|
break;
|
|
604
647
|
const params = [];
|
|
648
|
+
const entries = [];
|
|
605
649
|
const values = rows.map((row) => {
|
|
606
650
|
const bytes = asBytes(row.payload);
|
|
607
651
|
const payload = oldLayout !== undefined
|
|
608
652
|
? migratePayload(oldLayout, table, bytes)
|
|
609
653
|
: bytes;
|
|
654
|
+
const scopes = rekey
|
|
655
|
+
? rekeyedScopes(table, decodeRow(table.columns, payload), row.row_id)
|
|
656
|
+
: undefined;
|
|
657
|
+
for (const [variable, value] of Object.entries(scopes ?? {})) {
|
|
658
|
+
entries.push([row.partition, variable, value, row.row_id]);
|
|
659
|
+
}
|
|
610
660
|
const base = params.length;
|
|
611
|
-
params.push(...rewriteValues(table, row.partition, row.row_id, payload, 'postgres'));
|
|
661
|
+
params.push(...rewriteValues(table, row.partition, row.row_id, payload, 'postgres', scopes));
|
|
612
662
|
return `(${types.map((type, i) => `$${base + i + 1}::${type}`).join(',')})`;
|
|
613
663
|
});
|
|
614
664
|
await q.query(`UPDATE ${quoteIdent(table.name)} AS t SET ${sets}
|
|
615
665
|
FROM (VALUES ${values.join(',')}) AS v(${aliases})
|
|
616
666
|
WHERE t.${quoteIdent(SYNC_PARTITION_COLUMN)}=v.${partitionAlias}
|
|
617
667
|
AND t.${quoteIdent(SYNC_ROW_ID_COLUMN)}=v.${rowIdAlias}`, params);
|
|
668
|
+
if (entries.length > 0) {
|
|
669
|
+
await q.query(rekeyRowScopesSql(table, 'postgres'), [
|
|
670
|
+
JSON.stringify(entries),
|
|
671
|
+
]);
|
|
672
|
+
}
|
|
618
673
|
const last = rows[rows.length - 1];
|
|
619
674
|
if (last === undefined || rows.length < batch)
|
|
620
675
|
break;
|
|
@@ -1687,7 +1742,7 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1687
1742
|
const lockedLayouts = typeof lockedMarker[0]?.layouts === 'string'
|
|
1688
1743
|
? lockedMarker[0].layouts
|
|
1689
1744
|
: undefined;
|
|
1690
|
-
|
|
1745
|
+
const rekeyed = (changedScopePatterns(schema, lockedLayouts) ?? []).map((change) => change.table);
|
|
1691
1746
|
const layouts = parseLayouts(lockedLayouts);
|
|
1692
1747
|
const retiredTables = retiredTableNames(schema, layouts);
|
|
1693
1748
|
const existing = new Map();
|
|
@@ -1717,12 +1772,16 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
1717
1772
|
for (const statement of schemaDdl(schema, existing, 'postgres', existingIndexes)) {
|
|
1718
1773
|
await client.query(statement);
|
|
1719
1774
|
}
|
|
1775
|
+
for (const statement of rekeyCleanupSql(rekeyed)) {
|
|
1776
|
+
await client.query(statement);
|
|
1777
|
+
}
|
|
1720
1778
|
for (const table of schema.tables.values()) {
|
|
1721
1779
|
const oldLayout = layouts[table.name];
|
|
1722
1780
|
const plan = rewritePlan(table, oldLayout, existing.get(table.name));
|
|
1723
|
-
|
|
1781
|
+
const rekey = rekeyed.includes(table.name);
|
|
1782
|
+
if (!plan.migrate && !plan.backfill && !rekey)
|
|
1724
1783
|
continue;
|
|
1725
|
-
await rewriteRowsOn(client, table, plan.migrate ? oldLayout : undefined);
|
|
1784
|
+
await rewriteRowsOn(client, table, plan.migrate ? oldLayout : undefined, rekey);
|
|
1726
1785
|
}
|
|
1727
1786
|
await client.query(`INSERT INTO sync_schema_meta(id, schema_version, layouts) VALUES (1, $1, $2)
|
|
1728
1787
|
ON CONFLICT (id) DO UPDATE
|
|
@@ -299,12 +299,34 @@ export type StoredLayouts = Record<string, readonly StoredColumnLayout[]>;
|
|
|
299
299
|
export declare function layoutsOf(schema: CompiledSchema): string;
|
|
300
300
|
export declare function parseLayouts(json: string | null | undefined): StoredLayouts;
|
|
301
301
|
/**
|
|
302
|
-
* §3.1:
|
|
303
|
-
*
|
|
304
|
-
*
|
|
305
|
-
|
|
302
|
+
* §3.1: the configured tables whose scope patterns differ from the recorded
|
|
303
|
+
* ones, or `undefined` when the marker predates recorded patterns (nothing to
|
|
304
|
+
* compare). A schema-version bump re-keys these tables.
|
|
305
|
+
*/
|
|
306
|
+
export declare function changedScopePatterns(schema: CompiledSchema, storedJson: string | null | undefined): {
|
|
307
|
+
table: string;
|
|
308
|
+
stored: string;
|
|
309
|
+
configured: string;
|
|
310
|
+
}[] | undefined;
|
|
311
|
+
/**
|
|
312
|
+
* §3.1: at the same schema version, refuse a configured table whose patterns
|
|
313
|
+
* differ from the recorded ones. Returns false when the marker predates
|
|
314
|
+
* recorded patterns, so the caller records them.
|
|
306
315
|
*/
|
|
307
316
|
export declare function assertStoredScopePatterns(schema: CompiledSchema, storedJson: string | null | undefined): boolean;
|
|
317
|
+
/**
|
|
318
|
+
* §3.1 re-key, before the row walk: drop the re-keyed tables' row scope index
|
|
319
|
+
* entries and tombstones, and raise every partition's horizon past the
|
|
320
|
+
* changes logged under the previous patterns. Empty when nothing re-keys.
|
|
321
|
+
*/
|
|
322
|
+
export declare function rekeyCleanupSql(tables: readonly string[]): string[];
|
|
323
|
+
/**
|
|
324
|
+
* §3.1 re-key: index the scopes of one rewritten page of rows. The entries
|
|
325
|
+
* bind as one JSON array of `[partition, variable, value, rowId]`, so the
|
|
326
|
+
* statement reads no row table (a planner that ranges a page over a declared
|
|
327
|
+
* index would scan the whole partition per page). Params: [entries].
|
|
328
|
+
*/
|
|
329
|
+
export declare function rekeyRowScopesSql(table: CompiledTable, dialect: RelationalDialect): string;
|
|
308
330
|
/**
|
|
309
331
|
* Version equality is not layout equality: a marker written by another build
|
|
310
332
|
* at the same version describes rows the running codec cannot decode. Refuse
|
|
@@ -341,13 +363,22 @@ export declare function migratePayload(oldLayout: readonly StoredColumnLayout[],
|
|
|
341
363
|
* Params: [afterPartition, afterRowId, limit].
|
|
342
364
|
*/
|
|
343
365
|
export declare function selectRowsForRewriteSql(table: CompiledTable, dialect: RelationalDialect): string;
|
|
366
|
+
/**
|
|
367
|
+
* §3.1 re-key: the row's stored scopes under the table's current patterns. A
|
|
368
|
+
* row without a value for one of them refuses the bump.
|
|
369
|
+
*/
|
|
370
|
+
export declare function rekeyedScopes(table: CompiledTable, values: readonly RowValue[], rowId: string): Record<string, string>;
|
|
344
371
|
/**
|
|
345
372
|
* SQLite/D1: rewrite one row during a migration: refresh the projection columns (when
|
|
346
|
-
* materialized)
|
|
373
|
+
* materialized), the stored scopes when the table re-keys (§3.1), and the
|
|
374
|
+
* payload. Bind with `rewriteValues`.
|
|
375
|
+
*/
|
|
376
|
+
export declare function rewriteRowSql(table: CompiledTable, rekey: boolean): string;
|
|
377
|
+
/**
|
|
378
|
+
* Bind values for `rewriteRowSql` from the (already migrated) payload:
|
|
379
|
+
* projection columns, the re-keyed scopes JSON, payload, partition, row id.
|
|
347
380
|
*/
|
|
348
|
-
export declare function
|
|
349
|
-
/** Bind values for `rewriteRowSql` from the (already migrated) payload. */
|
|
350
|
-
export declare function rewriteValues(table: CompiledTable, partition: string, rowId: string, payload: Uint8Array, dialect: RelationalDialect): RowValue[];
|
|
381
|
+
export declare function rewriteValues(table: CompiledTable, partition: string, rowId: string, payload: Uint8Array, dialect: RelationalDialect, scopes: Record<string, string> | undefined): RowValue[];
|
|
351
382
|
/**
|
|
352
383
|
* What the migration rewrite phase must do for one table, derived from the
|
|
353
384
|
* persisted layouts + the physical columns present before this run.
|
package/dist/relational-rows.js
CHANGED
|
@@ -48,6 +48,7 @@
|
|
|
48
48
|
* NOT NULL per the schema.
|
|
49
49
|
*/
|
|
50
50
|
import { decodeRow, encodeRow, } from '@syncular/core';
|
|
51
|
+
import { storedScopesForRow } from './scopes.js';
|
|
51
52
|
import { StorageQueryError } from './storage-errors.js';
|
|
52
53
|
export const SYNC_PARTITION_COLUMN = '_sync_partition';
|
|
53
54
|
export const SYNC_ROW_ID_COLUMN = '_sync_row_id';
|
|
@@ -726,29 +727,72 @@ export function parseLayouts(json) {
|
|
|
726
727
|
return layouts;
|
|
727
728
|
}
|
|
728
729
|
/**
|
|
729
|
-
* §3.1:
|
|
730
|
-
*
|
|
731
|
-
*
|
|
732
|
-
* predates recorded patterns (nothing to compare), so the caller records them.
|
|
730
|
+
* §3.1: the configured tables whose scope patterns differ from the recorded
|
|
731
|
+
* ones, or `undefined` when the marker predates recorded patterns (nothing to
|
|
732
|
+
* compare). A schema-version bump re-keys these tables.
|
|
733
733
|
*/
|
|
734
|
-
export function
|
|
734
|
+
export function changedScopePatterns(schema, storedJson) {
|
|
735
735
|
if (storedJson === null || storedJson === undefined || storedJson === '')
|
|
736
|
-
return
|
|
736
|
+
return [];
|
|
737
737
|
const stored = JSON.parse(storedJson).sync_scopes;
|
|
738
738
|
if (stored === undefined)
|
|
739
|
-
return
|
|
740
|
-
|
|
739
|
+
return undefined;
|
|
740
|
+
return [...schema.tables.values()].flatMap((table) => {
|
|
741
741
|
const before = stored[table.name];
|
|
742
742
|
const after = scopePatternsOf(table);
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
743
|
+
return before !== undefined && before.join('\n') !== after.join('\n')
|
|
744
|
+
? [
|
|
745
|
+
{
|
|
746
|
+
table: table.name,
|
|
747
|
+
stored: before.join(' '),
|
|
748
|
+
configured: after.join(' '),
|
|
749
|
+
},
|
|
750
|
+
]
|
|
751
|
+
: [];
|
|
752
|
+
});
|
|
753
|
+
}
|
|
754
|
+
/**
|
|
755
|
+
* §3.1: at the same schema version, refuse a configured table whose patterns
|
|
756
|
+
* differ from the recorded ones. Returns false when the marker predates
|
|
757
|
+
* recorded patterns, so the caller records them.
|
|
758
|
+
*/
|
|
759
|
+
export function assertStoredScopePatterns(schema, storedJson) {
|
|
760
|
+
const changed = changedScopePatterns(schema, storedJson);
|
|
761
|
+
if (changed?.[0] !== undefined) {
|
|
762
|
+
throw new StorageQueryError('sync.storage.scope_patterns_changed', changed[0]);
|
|
750
763
|
}
|
|
751
|
-
return
|
|
764
|
+
return changed !== undefined;
|
|
765
|
+
}
|
|
766
|
+
/**
|
|
767
|
+
* §3.1 re-key, before the row walk: drop the re-keyed tables' row scope index
|
|
768
|
+
* entries and tombstones, and raise every partition's horizon past the
|
|
769
|
+
* changes logged under the previous patterns. Empty when nothing re-keys.
|
|
770
|
+
*/
|
|
771
|
+
export function rekeyCleanupSql(tables) {
|
|
772
|
+
if (tables.length === 0)
|
|
773
|
+
return [];
|
|
774
|
+
const names = tables.map((name) => `'${name.replaceAll("'", "''")}'`);
|
|
775
|
+
return [
|
|
776
|
+
`DELETE FROM sync_row_scopes WHERE tbl IN (${names.join(',')})`,
|
|
777
|
+
`DELETE FROM sync_tombstones WHERE tbl IN (${names.join(',')})`,
|
|
778
|
+
'UPDATE sync_partitions SET horizon_seq=max_commit_seq WHERE horizon_seq<max_commit_seq',
|
|
779
|
+
];
|
|
780
|
+
}
|
|
781
|
+
/**
|
|
782
|
+
* §3.1 re-key: index the scopes of one rewritten page of rows. The entries
|
|
783
|
+
* bind as one JSON array of `[partition, variable, value, rowId]`, so the
|
|
784
|
+
* statement reads no row table (a planner that ranges a page over a declared
|
|
785
|
+
* index would scan the whole partition per page). Params: [entries].
|
|
786
|
+
*/
|
|
787
|
+
export function rekeyRowScopesSql(table, dialect) {
|
|
788
|
+
const name = `'${table.name.replaceAll("'", "''")}'`;
|
|
789
|
+
return dialect === 'sqlite'
|
|
790
|
+
? `INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
|
|
791
|
+
SELECT json_extract(e.value, '$[0]'), ${name}, json_extract(e.value, '$[1]'), json_extract(e.value, '$[2]'), json_extract(e.value, '$[3]')
|
|
792
|
+
FROM json_each(?) AS e`
|
|
793
|
+
: `INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
|
|
794
|
+
SELECT e->>0, ${name}, e->>1, e->>2, e->>3
|
|
795
|
+
FROM jsonb_array_elements($1::text::jsonb) AS e`;
|
|
752
796
|
}
|
|
753
797
|
/**
|
|
754
798
|
* Version equality is not layout equality: a marker written by another build
|
|
@@ -855,21 +899,41 @@ export function selectRowsForRewriteSql(table, dialect) {
|
|
|
855
899
|
ORDER BY ${quoteIdent(SYNC_PARTITION_COLUMN)}, ${quoteIdent(SYNC_ROW_ID_COLUMN)}
|
|
856
900
|
LIMIT ${p[2]}`;
|
|
857
901
|
}
|
|
902
|
+
/**
|
|
903
|
+
* §3.1 re-key: the row's stored scopes under the table's current patterns. A
|
|
904
|
+
* row without a value for one of them refuses the bump.
|
|
905
|
+
*/
|
|
906
|
+
export function rekeyedScopes(table, values, rowId) {
|
|
907
|
+
const extracted = storedScopesForRow(table, values);
|
|
908
|
+
if ('missing' in extracted) {
|
|
909
|
+
throw new StorageQueryError('sync.storage.scope_value_missing', {
|
|
910
|
+
table: table.name,
|
|
911
|
+
variable: extracted.missing,
|
|
912
|
+
rowId,
|
|
913
|
+
});
|
|
914
|
+
}
|
|
915
|
+
return extracted.scopes;
|
|
916
|
+
}
|
|
858
917
|
/**
|
|
859
918
|
* SQLite/D1: rewrite one row during a migration: refresh the projection columns (when
|
|
860
|
-
* materialized)
|
|
919
|
+
* materialized), the stored scopes when the table re-keys (§3.1), and the
|
|
920
|
+
* payload. Bind with `rewriteValues`.
|
|
861
921
|
*/
|
|
862
|
-
export function rewriteRowSql(table) {
|
|
922
|
+
export function rewriteRowSql(table, rekey) {
|
|
863
923
|
const sets = [
|
|
864
924
|
...(table.materialize
|
|
865
925
|
? table.columns.map((column) => `${quoteIdent(column.name)}=?`)
|
|
866
926
|
: []),
|
|
927
|
+
...(rekey ? [`${quoteIdent(SYNC_SCOPES_COLUMN)}=?`] : []),
|
|
867
928
|
`${quoteIdent(SYNC_PAYLOAD_COLUMN)}=?`,
|
|
868
929
|
];
|
|
869
930
|
return `UPDATE ${quoteIdent(table.name)} SET ${sets.join(', ')} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=? AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=?`;
|
|
870
931
|
}
|
|
871
|
-
/**
|
|
872
|
-
|
|
932
|
+
/**
|
|
933
|
+
* Bind values for `rewriteRowSql` from the (already migrated) payload:
|
|
934
|
+
* projection columns, the re-keyed scopes JSON, payload, partition, row id.
|
|
935
|
+
*/
|
|
936
|
+
export function rewriteValues(table, partition, rowId, payload, dialect, scopes) {
|
|
873
937
|
const out = [];
|
|
874
938
|
if (table.materialize) {
|
|
875
939
|
const values = decodeRow(table.columns, payload);
|
|
@@ -880,6 +944,8 @@ export function rewriteValues(table, partition, rowId, payload, dialect) {
|
|
|
880
944
|
out.push(toSqlValue(column, values[i] ?? null, dialect));
|
|
881
945
|
}
|
|
882
946
|
}
|
|
947
|
+
if (scopes !== undefined)
|
|
948
|
+
out.push(JSON.stringify(scopes));
|
|
883
949
|
out.push(payload, partition, rowId);
|
|
884
950
|
return out;
|
|
885
951
|
}
|
package/dist/sqlite-storage.js
CHANGED
|
@@ -9,10 +9,11 @@
|
|
|
9
9
|
* select candidates from the index and verify the full multi-variable
|
|
10
10
|
* match against the stored scope map — never a log scan.
|
|
11
11
|
*/
|
|
12
|
+
import { decodeRow } from '@syncular/core';
|
|
12
13
|
import { isSqliteConstraintError, retainRollbackFailure, rollbackAfterError, StorageConstraintError, StorageQueryError, } from './storage-errors.js';
|
|
13
14
|
import { commitDeleteFloor, serveGateRefusal } from './storage.js';
|
|
14
15
|
import { serveNotReadyError } from './readiness.js';
|
|
15
|
-
import { assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, dropTableDdl, layoutsOf, migratePayload, parseLayouts, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowsForRewriteSql, } from './relational-rows.js';
|
|
16
|
+
import { assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, changedScopePatterns, dropTableDdl, layoutsOf, migratePayload, parseLayouts, rekeyCleanupSql, rekeyedScopes, rekeyRowScopesSql, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowsForRewriteSql, } from './relational-rows.js';
|
|
16
17
|
import { SQLITE_DDL, SQLITE_UPGRADE_PROBE, SqliteFamilyStorage, SqliteFamilyTransaction, sqliteUpgradeDdl, } from './sqlite-dialect.js';
|
|
17
18
|
import { SqliteAdapterRequiredError, } from './sqlite-driver.js';
|
|
18
19
|
function toStoredCheckpoint(record) {
|
|
@@ -413,7 +414,7 @@ END`);
|
|
|
413
414
|
this.schemaVersion = schema.version;
|
|
414
415
|
return;
|
|
415
416
|
}
|
|
416
|
-
|
|
417
|
+
const rekeyed = (changedScopePatterns(schema, locked?.layouts) ?? []).map((change) => change.table);
|
|
417
418
|
const layouts = parseLayouts(locked?.layouts);
|
|
418
419
|
const retiredTables = retiredTableNames(schema, layouts);
|
|
419
420
|
for (const tableName of retiredTables) {
|
|
@@ -428,12 +429,16 @@ END`);
|
|
|
428
429
|
for (const statement of schemaDdl(schema, existing, 'sqlite', existingIndexes)) {
|
|
429
430
|
this.db.exec(statement);
|
|
430
431
|
}
|
|
432
|
+
for (const statement of rekeyCleanupSql(rekeyed)) {
|
|
433
|
+
this.db.exec(statement);
|
|
434
|
+
}
|
|
431
435
|
for (const table of schema.tables.values()) {
|
|
432
436
|
const oldLayout = layouts[table.name];
|
|
433
437
|
const plan = rewritePlan(table, oldLayout, existing.get(table.name));
|
|
434
|
-
|
|
438
|
+
const rekey = rekeyed.includes(table.name);
|
|
439
|
+
if (!plan.migrate && !plan.backfill && !rekey)
|
|
435
440
|
continue;
|
|
436
|
-
this.#rewriteRows(table, plan.migrate ? oldLayout : undefined);
|
|
441
|
+
this.#rewriteRows(table, plan.migrate ? oldLayout : undefined, rekey);
|
|
437
442
|
}
|
|
438
443
|
this.db
|
|
439
444
|
.query('INSERT INTO sync_schema_meta(id, schema_version, layouts) VALUES (1, ?, ?) ON CONFLICT(id) DO UPDATE SET schema_version=excluded.schema_version, layouts=excluded.layouts')
|
|
@@ -471,11 +476,13 @@ END`);
|
|
|
471
476
|
* Migration rewrite: keyset-paged walk of a row table. When `oldLayout` is
|
|
472
477
|
* given every payload re-encodes under
|
|
473
478
|
* the current columns, and the projection (when materialized) refreshes
|
|
474
|
-
* from the payload either way.
|
|
479
|
+
* from the payload either way. `rekey` recomputes and indexes the stored
|
|
480
|
+
* scopes (§3.1).
|
|
475
481
|
*/
|
|
476
|
-
#rewriteRows(table, oldLayout) {
|
|
482
|
+
#rewriteRows(table, oldLayout, rekey) {
|
|
477
483
|
const select = selectRowsForRewriteSql(table, 'sqlite');
|
|
478
|
-
const update = this.db.query(rewriteRowSql(table));
|
|
484
|
+
const update = this.db.query(rewriteRowSql(table, rekey));
|
|
485
|
+
const index = this.db.query(rekeyRowScopesSql(table, 'sqlite'));
|
|
479
486
|
const BATCH = 500;
|
|
480
487
|
let afterPartition = '';
|
|
481
488
|
let afterRowId = '';
|
|
@@ -485,12 +492,21 @@ END`);
|
|
|
485
492
|
.all(afterPartition, afterRowId, BATCH);
|
|
486
493
|
if (rows.length === 0)
|
|
487
494
|
break;
|
|
495
|
+
const entries = [];
|
|
488
496
|
for (const row of rows) {
|
|
489
497
|
const payload = oldLayout !== undefined
|
|
490
498
|
? migratePayload(oldLayout, table, row.payload)
|
|
491
499
|
: row.payload;
|
|
492
|
-
|
|
500
|
+
const scopes = rekey
|
|
501
|
+
? rekeyedScopes(table, decodeRow(table.columns, payload), row.row_id)
|
|
502
|
+
: undefined;
|
|
503
|
+
for (const [variable, value] of Object.entries(scopes ?? {})) {
|
|
504
|
+
entries.push([row.partition, variable, value, row.row_id]);
|
|
505
|
+
}
|
|
506
|
+
update.run(...rewriteValues(table, row.partition, row.row_id, payload, 'sqlite', scopes));
|
|
493
507
|
}
|
|
508
|
+
if (entries.length > 0)
|
|
509
|
+
index.run(JSON.stringify(entries));
|
|
494
510
|
const last = rows[rows.length - 1];
|
|
495
511
|
if (last === undefined || rows.length < BATCH)
|
|
496
512
|
break;
|
package/dist/storage-errors.d.ts
CHANGED
|
@@ -9,7 +9,7 @@ export declare class StorageConstraintError extends Error {
|
|
|
9
9
|
constructor(cause: unknown, opIndex?: number);
|
|
10
10
|
}
|
|
11
11
|
/** Stable, privacy-safe failures for trusted server storage queries. */
|
|
12
|
-
export type StorageQueryErrorCode = 'sync.storage.schema_migration_pending' | 'sync.storage.schema_migration_conflict' | 'sync.storage.schema_changed' | 'sync.storage.invalid_migration_budget' | 'sync.storage.scan_requires_scope' | 'sync.storage.index_not_found' | 'sync.storage.index_not_materialized' | 'sync.storage.index_value_count_mismatch' | 'sync.storage.invalid_limit' | 'sync.storage.prune_epoch_mismatch' | 'sync.storage.partition_unregistered' | 'sync.storage.invalid_prune_cursor' | 'sync.storage.checkpoint_not_declared' | 'sync.storage.checkpoint_unsupported' | 'sync.storage.checkpoint_fence_missing' | 'sync.storage.checkpoint_incomplete' | 'sync.storage.stored_layout_mismatch' | 'sync.storage.scope_patterns_changed' | 'sync.storage.physical_layout_mismatch' | 'sync.storage.transaction_query_unsupported' | 'sync.storage.partition_lock_unsupported' | 'sync.storage.query_over_staged_writes' | 'sync.storage.erase_unsupported' | 'sync.storage.invalid_erase_query';
|
|
12
|
+
export type StorageQueryErrorCode = 'sync.storage.schema_migration_pending' | 'sync.storage.schema_migration_conflict' | 'sync.storage.schema_changed' | 'sync.storage.invalid_migration_budget' | 'sync.storage.scan_requires_scope' | 'sync.storage.index_not_found' | 'sync.storage.index_not_materialized' | 'sync.storage.index_value_count_mismatch' | 'sync.storage.invalid_limit' | 'sync.storage.prune_epoch_mismatch' | 'sync.storage.partition_unregistered' | 'sync.storage.invalid_prune_cursor' | 'sync.storage.checkpoint_not_declared' | 'sync.storage.checkpoint_unsupported' | 'sync.storage.checkpoint_fence_missing' | 'sync.storage.checkpoint_incomplete' | 'sync.storage.stored_layout_mismatch' | 'sync.storage.scope_patterns_changed' | 'sync.storage.scope_value_missing' | 'sync.storage.physical_layout_mismatch' | 'sync.storage.transaction_query_unsupported' | 'sync.storage.partition_lock_unsupported' | 'sync.storage.query_over_staged_writes' | 'sync.storage.erase_unsupported' | 'sync.storage.invalid_erase_query';
|
|
13
13
|
/**
|
|
14
14
|
* Host-only query error. Messages never include identifiers, values, SQL,
|
|
15
15
|
* paths, or row data; callers branch on `code`, never message text. The
|
package/dist/storage-errors.js
CHANGED
|
@@ -25,6 +25,7 @@ const STORAGE_QUERY_MESSAGES = {
|
|
|
25
25
|
'sync.storage.checkpoint_incomplete': 'a declared backfill checkpoint is not activated and this caller did not declare it',
|
|
26
26
|
'sync.storage.stored_layout_mismatch': 'stored schema layouts disagree with the configured schema at the same version',
|
|
27
27
|
'sync.storage.scope_patterns_changed': 'a table declares scope patterns other than the ones its stored rows are indexed under',
|
|
28
|
+
'sync.storage.scope_value_missing': 'a stored row has no value for a scope column the schema bump re-keys it under',
|
|
28
29
|
'sync.storage.physical_layout_mismatch': 'a synced table does not match the storage layout the running code reads and writes',
|
|
29
30
|
'sync.storage.transaction_query_unsupported': 'this storage transaction cannot run registered queries',
|
|
30
31
|
'sync.storage.partition_lock_unsupported': 'this storage transaction cannot serialize partition authority',
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.64.0",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.
|
|
71
|
+
"@syncular/core": "0.64.0"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|
package/src/d1-storage.ts
CHANGED
|
@@ -45,12 +45,16 @@ import {
|
|
|
45
45
|
assertPhysicalColumns,
|
|
46
46
|
assertStoredLayouts,
|
|
47
47
|
assertStoredScopePatterns,
|
|
48
|
+
changedScopePatterns,
|
|
48
49
|
dropTableDdl,
|
|
49
50
|
indexRowPageStatement,
|
|
50
51
|
layoutsOf,
|
|
51
52
|
migratePayload,
|
|
52
53
|
parseLayouts,
|
|
53
54
|
quoteIdent,
|
|
55
|
+
rekeyCleanupSql,
|
|
56
|
+
rekeyedScopes,
|
|
57
|
+
rekeyRowScopesSql,
|
|
54
58
|
retiredTableNames,
|
|
55
59
|
rewritePlan,
|
|
56
60
|
rewriteRowSql,
|
|
@@ -836,7 +840,11 @@ interface D1MigrationState {
|
|
|
836
840
|
columns: Record<string, string[]>;
|
|
837
841
|
indexes: Record<string, Record<string, string>>;
|
|
838
842
|
ddl: string[];
|
|
839
|
-
rewrites: Array<{
|
|
843
|
+
rewrites: Array<{
|
|
844
|
+
table: string;
|
|
845
|
+
oldLayout?: readonly StoredColumnLayout[];
|
|
846
|
+
rekey?: true;
|
|
847
|
+
}>;
|
|
840
848
|
afterPartition: string;
|
|
841
849
|
afterRowId: string;
|
|
842
850
|
}
|
|
@@ -1130,7 +1138,6 @@ export class D1ServerStorage
|
|
|
1130
1138
|
throw new StorageQueryError('sync.storage.schema_changed');
|
|
1131
1139
|
}
|
|
1132
1140
|
if (claim === null) {
|
|
1133
|
-
assertStoredScopePatterns(schema, marker?.layouts);
|
|
1134
1141
|
const layouts = parseLayouts(marker?.layouts);
|
|
1135
1142
|
for (const table of tables) {
|
|
1136
1143
|
const oldLayout = layouts[table.name];
|
|
@@ -1180,6 +1187,9 @@ export class D1ServerStorage
|
|
|
1180
1187
|
throw new StorageQueryError('sync.storage.schema_migration_conflict');
|
|
1181
1188
|
}
|
|
1182
1189
|
const sourceLayouts = parseLayouts(claim.source_layouts);
|
|
1190
|
+
const rekeyed = (
|
|
1191
|
+
changedScopePatterns(schema, claim.source_layouts) ?? []
|
|
1192
|
+
).map((change) => change.table);
|
|
1183
1193
|
let state = JSON.parse(claim.state) as D1MigrationState;
|
|
1184
1194
|
// Reserve one statement to distinguish a stale checkpoint from a database
|
|
1185
1195
|
// failure. A losing concurrent step returns incomplete without retrying here.
|
|
@@ -1252,7 +1262,60 @@ export class D1ServerStorage
|
|
|
1252
1262
|
} else if (state.phase === 'prepare') {
|
|
1253
1263
|
const table = tables[state.offset];
|
|
1254
1264
|
if (table !== undefined) {
|
|
1255
|
-
|
|
1265
|
+
// §3.1: every row of a re-keyed table needs a value for each new
|
|
1266
|
+
// pattern. Check before any DDL (one more statement); a refusal
|
|
1267
|
+
// releases the claim, so the refused bump changes nothing.
|
|
1268
|
+
const oldLayout = sourceLayouts[table.name];
|
|
1269
|
+
const verify =
|
|
1270
|
+
rekeyed.includes(table.name) && oldLayout !== undefined;
|
|
1271
|
+
if (remaining < (verify ? 5 : 4)) break;
|
|
1272
|
+
if (verify) {
|
|
1273
|
+
statementsExecuted++;
|
|
1274
|
+
const { results } = await this.#db
|
|
1275
|
+
.prepare(`SELECT * FROM (${selectRowsForRewriteSql(table, 'sqlite')})
|
|
1276
|
+
WHERE EXISTS (SELECT 1 FROM sync_schema_migration WHERE id=1 AND target=? AND state=?)`)
|
|
1277
|
+
.bind(
|
|
1278
|
+
state.afterPartition,
|
|
1279
|
+
state.afterRowId,
|
|
1280
|
+
500,
|
|
1281
|
+
target,
|
|
1282
|
+
claim.state,
|
|
1283
|
+
)
|
|
1284
|
+
.all<{ partition: string; row_id: string; payload: unknown }>();
|
|
1285
|
+
try {
|
|
1286
|
+
for (const row of results) {
|
|
1287
|
+
rekeyedScopes(
|
|
1288
|
+
table,
|
|
1289
|
+
decodeRow(oldLayout, asUint8Array(row.payload)),
|
|
1290
|
+
row.row_id,
|
|
1291
|
+
);
|
|
1292
|
+
}
|
|
1293
|
+
} catch (error) {
|
|
1294
|
+
await batch([
|
|
1295
|
+
this.#db
|
|
1296
|
+
.prepare(
|
|
1297
|
+
'DELETE FROM sync_schema_migration WHERE id=1 AND target=? AND state=?',
|
|
1298
|
+
)
|
|
1299
|
+
.bind(target, claim.state),
|
|
1300
|
+
]);
|
|
1301
|
+
throw error;
|
|
1302
|
+
}
|
|
1303
|
+
const last = results.at(-1);
|
|
1304
|
+
if (last !== undefined && results.length === 500) {
|
|
1305
|
+
if (
|
|
1306
|
+
!(await save(
|
|
1307
|
+
{
|
|
1308
|
+
...state,
|
|
1309
|
+
afterPartition: last.partition,
|
|
1310
|
+
afterRowId: last.row_id,
|
|
1311
|
+
},
|
|
1312
|
+
[],
|
|
1313
|
+
))
|
|
1314
|
+
)
|
|
1315
|
+
break;
|
|
1316
|
+
continue;
|
|
1317
|
+
}
|
|
1318
|
+
}
|
|
1256
1319
|
statementsExecuted += 2;
|
|
1257
1320
|
const columns = await this.#db
|
|
1258
1321
|
.prepare(`PRAGMA table_info(${quoteIdent(table.name)})`)
|
|
@@ -1274,6 +1337,8 @@ export class D1ServerStorage
|
|
|
1274
1337
|
{
|
|
1275
1338
|
...state,
|
|
1276
1339
|
offset: state.offset + 1,
|
|
1340
|
+
afterPartition: '',
|
|
1341
|
+
afterRowId: '',
|
|
1277
1342
|
columns: {
|
|
1278
1343
|
...state.columns,
|
|
1279
1344
|
...(columns.results.length > 0
|
|
@@ -1313,13 +1378,15 @@ export class D1ServerStorage
|
|
|
1313
1378
|
sourceLayouts[table.name],
|
|
1314
1379
|
columns.get(table.name),
|
|
1315
1380
|
);
|
|
1316
|
-
|
|
1381
|
+
const rekey = rekeyed.includes(table.name);
|
|
1382
|
+
return plan.migrate || plan.backfill || rekey
|
|
1317
1383
|
? [
|
|
1318
1384
|
{
|
|
1319
1385
|
table: table.name,
|
|
1320
1386
|
...(plan.migrate
|
|
1321
1387
|
? { oldLayout: sourceLayouts[table.name] }
|
|
1322
1388
|
: {}),
|
|
1389
|
+
...(rekey ? { rekey: true as const } : {}),
|
|
1323
1390
|
},
|
|
1324
1391
|
]
|
|
1325
1392
|
: [];
|
|
@@ -1331,6 +1398,7 @@ export class D1ServerStorage
|
|
|
1331
1398
|
dropTableDdl(name),
|
|
1332
1399
|
]),
|
|
1333
1400
|
...schemaDdl(schema, columns, 'sqlite', indexes),
|
|
1401
|
+
...rekeyCleanupSql(rekeyed),
|
|
1334
1402
|
];
|
|
1335
1403
|
if (
|
|
1336
1404
|
!(await save(
|
|
@@ -1389,8 +1457,10 @@ export class D1ServerStorage
|
|
|
1389
1457
|
const table = schema.tables.get(plan.table);
|
|
1390
1458
|
if (table === undefined)
|
|
1391
1459
|
throw new StorageQueryError('sync.storage.schema_migration_conflict');
|
|
1392
|
-
|
|
1393
|
-
const
|
|
1460
|
+
// A re-keyed page adds one scope-index statement.
|
|
1461
|
+
const rekey = plan.rekey === true;
|
|
1462
|
+
if (remaining < (rekey ? 5 : 4)) break;
|
|
1463
|
+
const limit = Math.min(32, remaining - (rekey ? 4 : 3));
|
|
1394
1464
|
statementsExecuted++;
|
|
1395
1465
|
const { results } = await this.#db
|
|
1396
1466
|
.prepare(`SELECT * FROM (${selectRowsForRewriteSql(table, 'sqlite')})
|
|
@@ -1403,14 +1473,25 @@ export class D1ServerStorage
|
|
|
1403
1473
|
claim.state,
|
|
1404
1474
|
)
|
|
1405
1475
|
.all<{ partition: string; row_id: string; payload: unknown }>();
|
|
1476
|
+
const entries: string[][] = [];
|
|
1406
1477
|
const statements = results.map((row) => {
|
|
1407
1478
|
const bytes = asUint8Array(row.payload);
|
|
1408
1479
|
const payload =
|
|
1409
1480
|
plan.oldLayout !== undefined
|
|
1410
1481
|
? migratePayload(plan.oldLayout, table, bytes)
|
|
1411
1482
|
: bytes;
|
|
1483
|
+
const scopes = rekey
|
|
1484
|
+
? rekeyedScopes(
|
|
1485
|
+
table,
|
|
1486
|
+
decodeRow(table.columns, payload),
|
|
1487
|
+
row.row_id,
|
|
1488
|
+
)
|
|
1489
|
+
: undefined;
|
|
1490
|
+
for (const [variable, value] of Object.entries(scopes ?? {})) {
|
|
1491
|
+
entries.push([row.partition, variable, value, row.row_id]);
|
|
1492
|
+
}
|
|
1412
1493
|
return this.#db
|
|
1413
|
-
.prepare(rewriteRowSql(table))
|
|
1494
|
+
.prepare(rewriteRowSql(table, rekey))
|
|
1414
1495
|
.bind(
|
|
1415
1496
|
...rewriteValues(
|
|
1416
1497
|
table,
|
|
@@ -1418,9 +1499,17 @@ export class D1ServerStorage
|
|
|
1418
1499
|
row.row_id,
|
|
1419
1500
|
payload,
|
|
1420
1501
|
'sqlite',
|
|
1502
|
+
scopes,
|
|
1421
1503
|
),
|
|
1422
1504
|
);
|
|
1423
1505
|
});
|
|
1506
|
+
if (entries.length > 0) {
|
|
1507
|
+
statements.push(
|
|
1508
|
+
this.#db
|
|
1509
|
+
.prepare(rekeyRowScopesSql(table, 'sqlite'))
|
|
1510
|
+
.bind(JSON.stringify(entries)),
|
|
1511
|
+
);
|
|
1512
|
+
}
|
|
1424
1513
|
const last = results.at(-1);
|
|
1425
1514
|
const done = results.length < limit;
|
|
1426
1515
|
if (
|
package/src/postgres-storage.ts
CHANGED
|
@@ -46,7 +46,11 @@ import { serveNotReadyError } from './readiness';
|
|
|
46
46
|
* would leave gaps on rollback, which the pull-window arithmetic in §4.5
|
|
47
47
|
* does not tolerate). Cross-partition pushes never contend.
|
|
48
48
|
*/
|
|
49
|
-
import
|
|
49
|
+
import {
|
|
50
|
+
decodeRow,
|
|
51
|
+
type PushOperationResult,
|
|
52
|
+
type UnitScopes,
|
|
53
|
+
} from '@syncular/core';
|
|
50
54
|
import {
|
|
51
55
|
bindAuthoritativePartition,
|
|
52
56
|
type BoundAuthoritativeQuery,
|
|
@@ -65,6 +69,7 @@ import {
|
|
|
65
69
|
assertPhysicalColumns,
|
|
66
70
|
assertStoredLayouts,
|
|
67
71
|
assertStoredScopePatterns,
|
|
72
|
+
changedScopePatterns,
|
|
68
73
|
deleteRowSql,
|
|
69
74
|
dropTableDdl,
|
|
70
75
|
indexRowPageStatement,
|
|
@@ -72,6 +77,9 @@ import {
|
|
|
72
77
|
migratePayload,
|
|
73
78
|
parseLayouts,
|
|
74
79
|
quoteIdent,
|
|
80
|
+
rekeyCleanupSql,
|
|
81
|
+
rekeyedScopes,
|
|
82
|
+
rekeyRowScopesSql,
|
|
75
83
|
tableColumnNames,
|
|
76
84
|
retiredTableNames,
|
|
77
85
|
rewritePlan,
|
|
@@ -933,34 +941,40 @@ function toStoredChange(record: ChangeRecord): StoredChange {
|
|
|
933
941
|
* transaction. When `oldLayout` is
|
|
934
942
|
* given every payload re-encodes under the current columns, and the
|
|
935
943
|
* projection (when materialized) refreshes from the payload either way.
|
|
944
|
+
* `rekey` recomputes and indexes the stored scopes (§3.1).
|
|
936
945
|
*/
|
|
937
946
|
async function rewriteRowsOn(
|
|
938
947
|
q: PgQueryable,
|
|
939
948
|
table: CompiledTable,
|
|
940
949
|
oldLayout: readonly StoredColumnLayout[] | undefined,
|
|
950
|
+
rekey: boolean,
|
|
941
951
|
): Promise<void> {
|
|
942
952
|
const select = selectRowsForRewriteSql(table, 'postgres');
|
|
943
953
|
// One UPDATE per page: behind a network hop (Hyperdrive, a managed
|
|
944
954
|
// database) every statement is a round trip, and the bump holds the
|
|
945
955
|
// migration lock for its whole duration. The page's rows bind as one
|
|
946
956
|
// VALUES list with typed placeholders (`rewriteValues` order: projection
|
|
947
|
-
// columns, payload, partition, row id).
|
|
957
|
+
// columns, re-keyed scopes, payload, partition, row id). The scopes bind as
|
|
958
|
+
// text before the JSONB cast (see `scanRowBatchSql`).
|
|
959
|
+
const projected = table.materialize ? table.columns : [];
|
|
948
960
|
const types = [
|
|
949
|
-
...(
|
|
950
|
-
|
|
951
|
-
: []),
|
|
961
|
+
...projected.map((column) => columnSqlType(column, 'postgres')),
|
|
962
|
+
...(rekey ? ['TEXT'] : []),
|
|
952
963
|
'BYTEA',
|
|
953
964
|
'TEXT',
|
|
954
965
|
'TEXT',
|
|
955
966
|
];
|
|
956
|
-
const
|
|
967
|
+
const payloadAt = projected.length + (rekey ? 1 : 0);
|
|
957
968
|
const sets = [
|
|
958
969
|
...projected.map((column, i) => `${quoteIdent(column.name)}=v.c${i}`),
|
|
959
|
-
|
|
970
|
+
...(rekey
|
|
971
|
+
? [`${quoteIdent(SYNC_SCOPES_COLUMN)}=v.c${projected.length}::jsonb`]
|
|
972
|
+
: []),
|
|
973
|
+
`${quoteIdent(SYNC_PAYLOAD_COLUMN)}=v.c${payloadAt}`,
|
|
960
974
|
].join(', ');
|
|
961
975
|
const aliases = types.map((_, i) => `c${i}`).join(', ');
|
|
962
|
-
const partitionAlias = `c${
|
|
963
|
-
const rowIdAlias = `c${
|
|
976
|
+
const partitionAlias = `c${payloadAt + 1}`;
|
|
977
|
+
const rowIdAlias = `c${payloadAt + 2}`;
|
|
964
978
|
// A page binds types.length parameters per row; PGlite corrupts its state
|
|
965
979
|
// above 32,767 bound parameters (see PG_CHANGES_PER_STATEMENT).
|
|
966
980
|
const batch = Math.max(1, Math.min(500, Math.floor(30_000 / types.length)));
|
|
@@ -974,15 +988,29 @@ async function rewriteRowsOn(
|
|
|
974
988
|
}>(select, [afterPartition, afterRowId, batch]);
|
|
975
989
|
if (rows.length === 0) break;
|
|
976
990
|
const params: unknown[] = [];
|
|
991
|
+
const entries: string[][] = [];
|
|
977
992
|
const values = rows.map((row) => {
|
|
978
993
|
const bytes = asBytes(row.payload);
|
|
979
994
|
const payload =
|
|
980
995
|
oldLayout !== undefined
|
|
981
996
|
? migratePayload(oldLayout, table, bytes)
|
|
982
997
|
: bytes;
|
|
998
|
+
const scopes = rekey
|
|
999
|
+
? rekeyedScopes(table, decodeRow(table.columns, payload), row.row_id)
|
|
1000
|
+
: undefined;
|
|
1001
|
+
for (const [variable, value] of Object.entries(scopes ?? {})) {
|
|
1002
|
+
entries.push([row.partition, variable, value, row.row_id]);
|
|
1003
|
+
}
|
|
983
1004
|
const base = params.length;
|
|
984
1005
|
params.push(
|
|
985
|
-
...rewriteValues(
|
|
1006
|
+
...rewriteValues(
|
|
1007
|
+
table,
|
|
1008
|
+
row.partition,
|
|
1009
|
+
row.row_id,
|
|
1010
|
+
payload,
|
|
1011
|
+
'postgres',
|
|
1012
|
+
scopes,
|
|
1013
|
+
),
|
|
986
1014
|
);
|
|
987
1015
|
return `(${types.map((type, i) => `$${base + i + 1}::${type}`).join(',')})`;
|
|
988
1016
|
});
|
|
@@ -993,6 +1021,11 @@ async function rewriteRowsOn(
|
|
|
993
1021
|
AND t.${quoteIdent(SYNC_ROW_ID_COLUMN)}=v.${rowIdAlias}`,
|
|
994
1022
|
params,
|
|
995
1023
|
);
|
|
1024
|
+
if (entries.length > 0) {
|
|
1025
|
+
await q.query(rekeyRowScopesSql(table, 'postgres'), [
|
|
1026
|
+
JSON.stringify(entries),
|
|
1027
|
+
]);
|
|
1028
|
+
}
|
|
996
1029
|
const last = rows[rows.length - 1];
|
|
997
1030
|
if (last === undefined || rows.length < batch) break;
|
|
998
1031
|
afterPartition = last.partition;
|
|
@@ -2456,7 +2489,9 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2456
2489
|
typeof lockedMarker[0]?.layouts === 'string'
|
|
2457
2490
|
? lockedMarker[0].layouts
|
|
2458
2491
|
: undefined;
|
|
2459
|
-
|
|
2492
|
+
const rekeyed = (changedScopePatterns(schema, lockedLayouts) ?? []).map(
|
|
2493
|
+
(change) => change.table,
|
|
2494
|
+
);
|
|
2460
2495
|
const layouts = parseLayouts(lockedLayouts);
|
|
2461
2496
|
const retiredTables = retiredTableNames(schema, layouts);
|
|
2462
2497
|
const existing = new Map<string, ReadonlySet<string>>();
|
|
@@ -2503,14 +2538,19 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2503
2538
|
)) {
|
|
2504
2539
|
await client.query(statement);
|
|
2505
2540
|
}
|
|
2541
|
+
for (const statement of rekeyCleanupSql(rekeyed)) {
|
|
2542
|
+
await client.query(statement);
|
|
2543
|
+
}
|
|
2506
2544
|
for (const table of schema.tables.values()) {
|
|
2507
2545
|
const oldLayout = layouts[table.name];
|
|
2508
2546
|
const plan = rewritePlan(table, oldLayout, existing.get(table.name));
|
|
2509
|
-
|
|
2547
|
+
const rekey = rekeyed.includes(table.name);
|
|
2548
|
+
if (!plan.migrate && !plan.backfill && !rekey) continue;
|
|
2510
2549
|
await rewriteRowsOn(
|
|
2511
2550
|
client,
|
|
2512
2551
|
table,
|
|
2513
2552
|
plan.migrate ? oldLayout : undefined,
|
|
2553
|
+
rekey,
|
|
2514
2554
|
);
|
|
2515
2555
|
}
|
|
2516
2556
|
await client.query(
|
package/src/relational-rows.ts
CHANGED
|
@@ -60,6 +60,7 @@ import type {
|
|
|
60
60
|
ServerIndexSchema,
|
|
61
61
|
} from './schema';
|
|
62
62
|
import type { StoredRow } from './storage';
|
|
63
|
+
import { storedScopesForRow } from './scopes';
|
|
63
64
|
import { StorageQueryError } from './storage-errors';
|
|
64
65
|
|
|
65
66
|
export type RelationalDialect = 'sqlite' | 'postgres';
|
|
@@ -903,35 +904,89 @@ export function parseLayouts(json: string | null | undefined): StoredLayouts {
|
|
|
903
904
|
}
|
|
904
905
|
|
|
905
906
|
/**
|
|
906
|
-
* §3.1:
|
|
907
|
-
*
|
|
908
|
-
*
|
|
909
|
-
* predates recorded patterns (nothing to compare), so the caller records them.
|
|
907
|
+
* §3.1: the configured tables whose scope patterns differ from the recorded
|
|
908
|
+
* ones, or `undefined` when the marker predates recorded patterns (nothing to
|
|
909
|
+
* compare). A schema-version bump re-keys these tables.
|
|
910
910
|
*/
|
|
911
|
-
export function
|
|
911
|
+
export function changedScopePatterns(
|
|
912
912
|
schema: CompiledSchema,
|
|
913
913
|
storedJson: string | null | undefined,
|
|
914
|
-
):
|
|
914
|
+
): { table: string; stored: string; configured: string }[] | undefined {
|
|
915
915
|
if (storedJson === null || storedJson === undefined || storedJson === '')
|
|
916
|
-
return
|
|
916
|
+
return [];
|
|
917
917
|
const stored = (
|
|
918
918
|
JSON.parse(storedJson) as {
|
|
919
919
|
sync_scopes?: Record<string, readonly string[]>;
|
|
920
920
|
}
|
|
921
921
|
).sync_scopes;
|
|
922
|
-
if (stored === undefined) return
|
|
923
|
-
|
|
922
|
+
if (stored === undefined) return undefined;
|
|
923
|
+
return [...schema.tables.values()].flatMap((table) => {
|
|
924
924
|
const before = stored[table.name];
|
|
925
925
|
const after = scopePatternsOf(table);
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
926
|
+
return before !== undefined && before.join('\n') !== after.join('\n')
|
|
927
|
+
? [
|
|
928
|
+
{
|
|
929
|
+
table: table.name,
|
|
930
|
+
stored: before.join(' '),
|
|
931
|
+
configured: after.join(' '),
|
|
932
|
+
},
|
|
933
|
+
]
|
|
934
|
+
: [];
|
|
935
|
+
});
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
/**
|
|
939
|
+
* §3.1: at the same schema version, refuse a configured table whose patterns
|
|
940
|
+
* differ from the recorded ones. Returns false when the marker predates
|
|
941
|
+
* recorded patterns, so the caller records them.
|
|
942
|
+
*/
|
|
943
|
+
export function assertStoredScopePatterns(
|
|
944
|
+
schema: CompiledSchema,
|
|
945
|
+
storedJson: string | null | undefined,
|
|
946
|
+
): boolean {
|
|
947
|
+
const changed = changedScopePatterns(schema, storedJson);
|
|
948
|
+
if (changed?.[0] !== undefined) {
|
|
949
|
+
throw new StorageQueryError(
|
|
950
|
+
'sync.storage.scope_patterns_changed',
|
|
951
|
+
changed[0],
|
|
952
|
+
);
|
|
933
953
|
}
|
|
934
|
-
return
|
|
954
|
+
return changed !== undefined;
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
/**
|
|
958
|
+
* §3.1 re-key, before the row walk: drop the re-keyed tables' row scope index
|
|
959
|
+
* entries and tombstones, and raise every partition's horizon past the
|
|
960
|
+
* changes logged under the previous patterns. Empty when nothing re-keys.
|
|
961
|
+
*/
|
|
962
|
+
export function rekeyCleanupSql(tables: readonly string[]): string[] {
|
|
963
|
+
if (tables.length === 0) return [];
|
|
964
|
+
const names = tables.map((name) => `'${name.replaceAll("'", "''")}'`);
|
|
965
|
+
return [
|
|
966
|
+
`DELETE FROM sync_row_scopes WHERE tbl IN (${names.join(',')})`,
|
|
967
|
+
`DELETE FROM sync_tombstones WHERE tbl IN (${names.join(',')})`,
|
|
968
|
+
'UPDATE sync_partitions SET horizon_seq=max_commit_seq WHERE horizon_seq<max_commit_seq',
|
|
969
|
+
];
|
|
970
|
+
}
|
|
971
|
+
|
|
972
|
+
/**
|
|
973
|
+
* §3.1 re-key: index the scopes of one rewritten page of rows. The entries
|
|
974
|
+
* bind as one JSON array of `[partition, variable, value, rowId]`, so the
|
|
975
|
+
* statement reads no row table (a planner that ranges a page over a declared
|
|
976
|
+
* index would scan the whole partition per page). Params: [entries].
|
|
977
|
+
*/
|
|
978
|
+
export function rekeyRowScopesSql(
|
|
979
|
+
table: CompiledTable,
|
|
980
|
+
dialect: RelationalDialect,
|
|
981
|
+
): string {
|
|
982
|
+
const name = `'${table.name.replaceAll("'", "''")}'`;
|
|
983
|
+
return dialect === 'sqlite'
|
|
984
|
+
? `INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
|
|
985
|
+
SELECT json_extract(e.value, '$[0]'), ${name}, json_extract(e.value, '$[1]'), json_extract(e.value, '$[2]'), json_extract(e.value, '$[3]')
|
|
986
|
+
FROM json_each(?) AS e`
|
|
987
|
+
: `INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
|
|
988
|
+
SELECT e->>0, ${name}, e->>1, e->>2, e->>3
|
|
989
|
+
FROM jsonb_array_elements($1::text::jsonb) AS e`;
|
|
935
990
|
}
|
|
936
991
|
|
|
937
992
|
/**
|
|
@@ -1071,27 +1126,53 @@ export function selectRowsForRewriteSql(
|
|
|
1071
1126
|
LIMIT ${p[2]}`;
|
|
1072
1127
|
}
|
|
1073
1128
|
|
|
1129
|
+
/**
|
|
1130
|
+
* §3.1 re-key: the row's stored scopes under the table's current patterns. A
|
|
1131
|
+
* row without a value for one of them refuses the bump.
|
|
1132
|
+
*/
|
|
1133
|
+
export function rekeyedScopes(
|
|
1134
|
+
table: CompiledTable,
|
|
1135
|
+
values: readonly RowValue[],
|
|
1136
|
+
rowId: string,
|
|
1137
|
+
): Record<string, string> {
|
|
1138
|
+
const extracted = storedScopesForRow(table, values);
|
|
1139
|
+
if ('missing' in extracted) {
|
|
1140
|
+
throw new StorageQueryError('sync.storage.scope_value_missing', {
|
|
1141
|
+
table: table.name,
|
|
1142
|
+
variable: extracted.missing,
|
|
1143
|
+
rowId,
|
|
1144
|
+
});
|
|
1145
|
+
}
|
|
1146
|
+
return extracted.scopes;
|
|
1147
|
+
}
|
|
1148
|
+
|
|
1074
1149
|
/**
|
|
1075
1150
|
* SQLite/D1: rewrite one row during a migration: refresh the projection columns (when
|
|
1076
|
-
* materialized)
|
|
1151
|
+
* materialized), the stored scopes when the table re-keys (§3.1), and the
|
|
1152
|
+
* payload. Bind with `rewriteValues`.
|
|
1077
1153
|
*/
|
|
1078
|
-
export function rewriteRowSql(table: CompiledTable): string {
|
|
1154
|
+
export function rewriteRowSql(table: CompiledTable, rekey: boolean): string {
|
|
1079
1155
|
const sets = [
|
|
1080
1156
|
...(table.materialize
|
|
1081
1157
|
? table.columns.map((column) => `${quoteIdent(column.name)}=?`)
|
|
1082
1158
|
: []),
|
|
1159
|
+
...(rekey ? [`${quoteIdent(SYNC_SCOPES_COLUMN)}=?`] : []),
|
|
1083
1160
|
`${quoteIdent(SYNC_PAYLOAD_COLUMN)}=?`,
|
|
1084
1161
|
];
|
|
1085
1162
|
return `UPDATE ${quoteIdent(table.name)} SET ${sets.join(', ')} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=? AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=?`;
|
|
1086
1163
|
}
|
|
1087
1164
|
|
|
1088
|
-
/**
|
|
1165
|
+
/**
|
|
1166
|
+
* Bind values for `rewriteRowSql` from the (already migrated) payload:
|
|
1167
|
+
* projection columns, the re-keyed scopes JSON, payload, partition, row id.
|
|
1168
|
+
*/
|
|
1089
1169
|
export function rewriteValues(
|
|
1090
1170
|
table: CompiledTable,
|
|
1091
1171
|
partition: string,
|
|
1092
1172
|
rowId: string,
|
|
1093
1173
|
payload: Uint8Array,
|
|
1094
1174
|
dialect: RelationalDialect,
|
|
1175
|
+
scopes: Record<string, string> | undefined,
|
|
1095
1176
|
): RowValue[] {
|
|
1096
1177
|
const out: RowValue[] = [];
|
|
1097
1178
|
if (table.materialize) {
|
|
@@ -1102,6 +1183,7 @@ export function rewriteValues(
|
|
|
1102
1183
|
out.push(toSqlValue(column, values[i] ?? null, dialect));
|
|
1103
1184
|
}
|
|
1104
1185
|
}
|
|
1186
|
+
if (scopes !== undefined) out.push(JSON.stringify(scopes));
|
|
1105
1187
|
out.push(payload, partition, rowId);
|
|
1106
1188
|
return out;
|
|
1107
1189
|
}
|
package/src/sqlite-storage.ts
CHANGED
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
* select candidates from the index and verify the full multi-variable
|
|
10
10
|
* match against the stored scope map — never a log scan.
|
|
11
11
|
*/
|
|
12
|
+
import { decodeRow } from '@syncular/core';
|
|
12
13
|
import {
|
|
13
14
|
isSqliteConstraintError,
|
|
14
15
|
retainRollbackFailure,
|
|
@@ -22,10 +23,14 @@ import {
|
|
|
22
23
|
assertPhysicalColumns,
|
|
23
24
|
assertStoredLayouts,
|
|
24
25
|
assertStoredScopePatterns,
|
|
26
|
+
changedScopePatterns,
|
|
25
27
|
dropTableDdl,
|
|
26
28
|
layoutsOf,
|
|
27
29
|
migratePayload,
|
|
28
30
|
parseLayouts,
|
|
31
|
+
rekeyCleanupSql,
|
|
32
|
+
rekeyedScopes,
|
|
33
|
+
rekeyRowScopesSql,
|
|
29
34
|
retiredTableNames,
|
|
30
35
|
rewritePlan,
|
|
31
36
|
rewriteRowSql,
|
|
@@ -628,7 +633,9 @@ END`);
|
|
|
628
633
|
this.schemaVersion = schema.version;
|
|
629
634
|
return;
|
|
630
635
|
}
|
|
631
|
-
|
|
636
|
+
const rekeyed = (
|
|
637
|
+
changedScopePatterns(schema, locked?.layouts) ?? []
|
|
638
|
+
).map((change) => change.table);
|
|
632
639
|
const layouts = parseLayouts(locked?.layouts);
|
|
633
640
|
const retiredTables = retiredTableNames(schema, layouts);
|
|
634
641
|
for (const tableName of retiredTables) {
|
|
@@ -648,11 +655,15 @@ END`);
|
|
|
648
655
|
)) {
|
|
649
656
|
this.db.exec(statement);
|
|
650
657
|
}
|
|
658
|
+
for (const statement of rekeyCleanupSql(rekeyed)) {
|
|
659
|
+
this.db.exec(statement);
|
|
660
|
+
}
|
|
651
661
|
for (const table of schema.tables.values()) {
|
|
652
662
|
const oldLayout = layouts[table.name];
|
|
653
663
|
const plan = rewritePlan(table, oldLayout, existing.get(table.name));
|
|
654
|
-
|
|
655
|
-
|
|
664
|
+
const rekey = rekeyed.includes(table.name);
|
|
665
|
+
if (!plan.migrate && !plan.backfill && !rekey) continue;
|
|
666
|
+
this.#rewriteRows(table, plan.migrate ? oldLayout : undefined, rekey);
|
|
656
667
|
}
|
|
657
668
|
this.db
|
|
658
669
|
.query(
|
|
@@ -699,14 +710,17 @@ END`);
|
|
|
699
710
|
* Migration rewrite: keyset-paged walk of a row table. When `oldLayout` is
|
|
700
711
|
* given every payload re-encodes under
|
|
701
712
|
* the current columns, and the projection (when materialized) refreshes
|
|
702
|
-
* from the payload either way.
|
|
713
|
+
* from the payload either way. `rekey` recomputes and indexes the stored
|
|
714
|
+
* scopes (§3.1).
|
|
703
715
|
*/
|
|
704
716
|
#rewriteRows(
|
|
705
717
|
table: CompiledTable,
|
|
706
718
|
oldLayout: readonly StoredColumnLayout[] | undefined,
|
|
719
|
+
rekey: boolean,
|
|
707
720
|
): void {
|
|
708
721
|
const select = selectRowsForRewriteSql(table, 'sqlite');
|
|
709
|
-
const update = this.db.query(rewriteRowSql(table));
|
|
722
|
+
const update = this.db.query(rewriteRowSql(table, rekey));
|
|
723
|
+
const index = this.db.query(rekeyRowScopesSql(table, 'sqlite'));
|
|
710
724
|
const BATCH = 500;
|
|
711
725
|
let afterPartition = '';
|
|
712
726
|
let afterRowId = '';
|
|
@@ -718,15 +732,30 @@ END`);
|
|
|
718
732
|
>(select)
|
|
719
733
|
.all(afterPartition, afterRowId, BATCH);
|
|
720
734
|
if (rows.length === 0) break;
|
|
735
|
+
const entries: string[][] = [];
|
|
721
736
|
for (const row of rows) {
|
|
722
737
|
const payload =
|
|
723
738
|
oldLayout !== undefined
|
|
724
739
|
? migratePayload(oldLayout, table, row.payload)
|
|
725
740
|
: row.payload;
|
|
741
|
+
const scopes = rekey
|
|
742
|
+
? rekeyedScopes(table, decodeRow(table.columns, payload), row.row_id)
|
|
743
|
+
: undefined;
|
|
744
|
+
for (const [variable, value] of Object.entries(scopes ?? {})) {
|
|
745
|
+
entries.push([row.partition, variable, value, row.row_id]);
|
|
746
|
+
}
|
|
726
747
|
update.run(
|
|
727
|
-
...rewriteValues(
|
|
748
|
+
...rewriteValues(
|
|
749
|
+
table,
|
|
750
|
+
row.partition,
|
|
751
|
+
row.row_id,
|
|
752
|
+
payload,
|
|
753
|
+
'sqlite',
|
|
754
|
+
scopes,
|
|
755
|
+
),
|
|
728
756
|
);
|
|
729
757
|
}
|
|
758
|
+
if (entries.length > 0) index.run(JSON.stringify(entries));
|
|
730
759
|
const last = rows[rows.length - 1];
|
|
731
760
|
if (last === undefined || rows.length < BATCH) break;
|
|
732
761
|
afterPartition = last.partition;
|
package/src/storage-errors.ts
CHANGED
|
@@ -33,6 +33,7 @@ export type StorageQueryErrorCode =
|
|
|
33
33
|
| 'sync.storage.checkpoint_incomplete'
|
|
34
34
|
| 'sync.storage.stored_layout_mismatch'
|
|
35
35
|
| 'sync.storage.scope_patterns_changed'
|
|
36
|
+
| 'sync.storage.scope_value_missing'
|
|
36
37
|
| 'sync.storage.physical_layout_mismatch'
|
|
37
38
|
| 'sync.storage.transaction_query_unsupported'
|
|
38
39
|
| 'sync.storage.partition_lock_unsupported'
|
|
@@ -68,6 +69,8 @@ const STORAGE_QUERY_MESSAGES: Readonly<Record<StorageQueryErrorCode, string>> =
|
|
|
68
69
|
'stored schema layouts disagree with the configured schema at the same version',
|
|
69
70
|
'sync.storage.scope_patterns_changed':
|
|
70
71
|
'a table declares scope patterns other than the ones its stored rows are indexed under',
|
|
72
|
+
'sync.storage.scope_value_missing':
|
|
73
|
+
'a stored row has no value for a scope column the schema bump re-keys it under',
|
|
71
74
|
'sync.storage.physical_layout_mismatch':
|
|
72
75
|
'a synced table does not match the storage layout the running code reads and writes',
|
|
73
76
|
'sync.storage.transaction_query_unsupported':
|