@syncular/server 0.62.1 → 0.64.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -40,7 +40,7 @@ var _a;
40
40
  * rather than a lock D1 does not expose.
41
41
  */
42
42
  import { decodeRow } from '@syncular/core';
43
- import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanRowPageSql, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowSql, selectRowsSql, selectRowsForRewriteSql, tableColumnNames, toSqlValue, } from './relational-rows.js';
43
+ import { assertAppendOnlyMigration, assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, changedScopePatterns, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, rekeyCleanupSql, rekeyedScopes, rekeyRowScopesSql, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, scanRowPageSql, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowSql, selectRowsSql, selectRowsForRewriteSql, tableColumnNames, toSqlValue, } from './relational-rows.js';
44
44
  import { matchesEffective } from './scopes.js';
45
45
  import { asUint8Array, deserializePushResult, placeholders, SqliteFamilyStorage, SqliteFamilyTransaction, sqliteDdlStatements, SQLITE_UPGRADE_PROBE, sqliteUpgradeDdl, toStoredRow, toTombstone, } from './sqlite-dialect.js';
46
46
  import { commitDeleteFloor } from './storage.js';
@@ -678,6 +678,15 @@ export class D1ServerStorage extends SqliteFamilyStorage {
678
678
  let claim = await read(this.#db.prepare('SELECT * FROM sync_schema_migration WHERE id=1'));
679
679
  if (claim === null && marker?.schema_version === schema.version) {
680
680
  assertStoredLayouts(schema, marker.layouts);
681
+ if (!assertStoredScopePatterns(schema, marker.layouts)) {
682
+ // A marker from before 0.63 records no scope patterns: record the
683
+ // configured ones, which every later start compares against.
684
+ statementsExecuted++;
685
+ await this.#db
686
+ .prepare('UPDATE sync_schema_meta SET layouts=? WHERE id=1 AND schema_version=?')
687
+ .bind(layoutsOf(schema), schema.version)
688
+ .run();
689
+ }
681
690
  statementsExecuted++;
682
691
  const present = await this.#db
683
692
  .prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name IN (${placeholders(D1_CORE_TABLES.length)})`)
@@ -779,6 +788,7 @@ export class D1ServerStorage extends SqliteFamilyStorage {
779
788
  throw new StorageQueryError('sync.storage.schema_migration_conflict');
780
789
  }
781
790
  const sourceLayouts = parseLayouts(claim.source_layouts);
791
+ const rekeyed = (changedScopePatterns(schema, claim.source_layouts) ?? []).map((change) => change.table);
782
792
  let state = JSON.parse(claim.state);
783
793
  // Reserve one statement to distinguish a stale checkpoint from a database
784
794
  // failure. A losing concurrent step returns incomplete without retrying here.
@@ -837,8 +847,44 @@ export class D1ServerStorage extends SqliteFamilyStorage {
837
847
  else if (state.phase === 'prepare') {
838
848
  const table = tables[state.offset];
839
849
  if (table !== undefined) {
840
- if (remaining < 4)
850
+ // §3.1: every row of a re-keyed table needs a value for each new
851
+ // pattern. Check before any DDL (one more statement); a refusal
852
+ // releases the claim, so the refused bump changes nothing.
853
+ const oldLayout = sourceLayouts[table.name];
854
+ const verify = rekeyed.includes(table.name) && oldLayout !== undefined;
855
+ if (remaining < (verify ? 5 : 4))
841
856
  break;
857
+ if (verify) {
858
+ statementsExecuted++;
859
+ const { results } = await this.#db
860
+ .prepare(`SELECT * FROM (${selectRowsForRewriteSql(table, 'sqlite')})
861
+ WHERE EXISTS (SELECT 1 FROM sync_schema_migration WHERE id=1 AND target=? AND state=?)`)
862
+ .bind(state.afterPartition, state.afterRowId, 500, target, claim.state)
863
+ .all();
864
+ try {
865
+ for (const row of results) {
866
+ rekeyedScopes(table, decodeRow(oldLayout, asUint8Array(row.payload)), row.row_id);
867
+ }
868
+ }
869
+ catch (error) {
870
+ await batch([
871
+ this.#db
872
+ .prepare('DELETE FROM sync_schema_migration WHERE id=1 AND target=? AND state=?')
873
+ .bind(target, claim.state),
874
+ ]);
875
+ throw error;
876
+ }
877
+ const last = results.at(-1);
878
+ if (last !== undefined && results.length === 500) {
879
+ if (!(await save({
880
+ ...state,
881
+ afterPartition: last.partition,
882
+ afterRowId: last.row_id,
883
+ }, [])))
884
+ break;
885
+ continue;
886
+ }
887
+ }
842
888
  statementsExecuted += 2;
843
889
  const columns = await this.#db
844
890
  .prepare(`PRAGMA table_info(${quoteIdent(table.name)})`)
@@ -854,6 +900,8 @@ export class D1ServerStorage extends SqliteFamilyStorage {
854
900
  if (!(await save({
855
901
  ...state,
856
902
  offset: state.offset + 1,
903
+ afterPartition: '',
904
+ afterRowId: '',
857
905
  columns: {
858
906
  ...state.columns,
859
907
  ...(columns.results.length > 0
@@ -880,13 +928,15 @@ export class D1ServerStorage extends SqliteFamilyStorage {
880
928
  ]));
881
929
  const rewrites = tables.flatMap((table) => {
882
930
  const plan = rewritePlan(table, sourceLayouts[table.name], columns.get(table.name));
883
- return plan.migrate || plan.backfill
931
+ const rekey = rekeyed.includes(table.name);
932
+ return plan.migrate || plan.backfill || rekey
884
933
  ? [
885
934
  {
886
935
  table: table.name,
887
936
  ...(plan.migrate
888
937
  ? { oldLayout: sourceLayouts[table.name] }
889
938
  : {}),
939
+ ...(rekey ? { rekey: true } : {}),
890
940
  },
891
941
  ]
892
942
  : [];
@@ -898,6 +948,7 @@ export class D1ServerStorage extends SqliteFamilyStorage {
898
948
  dropTableDdl(name),
899
949
  ]),
900
950
  ...schemaDdl(schema, columns, 'sqlite', indexes),
951
+ ...rekeyCleanupSql(rekeyed),
901
952
  ];
902
953
  if (!(await save({
903
954
  ...state,
@@ -938,24 +989,38 @@ export class D1ServerStorage extends SqliteFamilyStorage {
938
989
  const table = schema.tables.get(plan.table);
939
990
  if (table === undefined)
940
991
  throw new StorageQueryError('sync.storage.schema_migration_conflict');
941
- if (remaining < 4)
992
+ // A re-keyed page adds one scope-index statement.
993
+ const rekey = plan.rekey === true;
994
+ if (remaining < (rekey ? 5 : 4))
942
995
  break;
943
- const limit = Math.min(32, remaining - 3);
996
+ const limit = Math.min(32, remaining - (rekey ? 4 : 3));
944
997
  statementsExecuted++;
945
998
  const { results } = await this.#db
946
999
  .prepare(`SELECT * FROM (${selectRowsForRewriteSql(table, 'sqlite')})
947
1000
  WHERE EXISTS (SELECT 1 FROM sync_schema_migration WHERE id=1 AND target=? AND state=?)`)
948
1001
  .bind(state.afterPartition, state.afterRowId, limit, target, claim.state)
949
1002
  .all();
1003
+ const entries = [];
950
1004
  const statements = results.map((row) => {
951
1005
  const bytes = asUint8Array(row.payload);
952
1006
  const payload = plan.oldLayout !== undefined
953
1007
  ? migratePayload(plan.oldLayout, table, bytes)
954
1008
  : bytes;
1009
+ const scopes = rekey
1010
+ ? rekeyedScopes(table, decodeRow(table.columns, payload), row.row_id)
1011
+ : undefined;
1012
+ for (const [variable, value] of Object.entries(scopes ?? {})) {
1013
+ entries.push([row.partition, variable, value, row.row_id]);
1014
+ }
955
1015
  return this.#db
956
- .prepare(rewriteRowSql(table))
957
- .bind(...rewriteValues(table, row.partition, row.row_id, payload, 'sqlite'));
1016
+ .prepare(rewriteRowSql(table, rekey))
1017
+ .bind(...rewriteValues(table, row.partition, row.row_id, payload, 'sqlite', scopes));
958
1018
  });
1019
+ if (entries.length > 0) {
1020
+ statements.push(this.#db
1021
+ .prepare(rekeyRowScopesSql(table, 'sqlite'))
1022
+ .bind(JSON.stringify(entries)));
1023
+ }
959
1024
  const last = results.at(-1);
960
1025
  const done = results.length < limit;
961
1026
  if (!(await save({
@@ -36,7 +36,7 @@ import type { CommitPruneQuery, CommitPruneResult } from './storage.js';
36
36
  * would leave gaps on rollback, which the pull-window arithmetic in §4.5
37
37
  * does not tolerate). Cross-partition pushes never contend.
38
38
  */
39
- import type { UnitScopes } from '@syncular/core';
39
+ import { type UnitScopes } from '@syncular/core';
40
40
  import { type PgExecutor } from './pg-executor.js';
41
41
  import type { CompiledSchema, CompiledTable } from './schema.js';
42
42
  import type { AuthoritativeQueryRequest, AuthoritativeQueryResult, CheckpointDeclaration, ClientCursorInfo, ClientPushResult, ClientRecord, ClientRecordWriteOptions, CommitDetail, CommitMetadata, CommitMetadataQuery, IndexRowScanQuery, LiveWindow, LiveWindowQuery, PartitionRegistryEntry, PrunedReactionCounts, PushWatermark, ReactionClaimQuery, ReactionFailureUpdate, ReactionListQuery, ReactionPruneQuery, RowChangeRecord, RowChangesQuery, RowListQuery, RowScanQuery, ScopeActivityQuery, ScopeCommitActivity, ScopeValueHistoryQuery, ScopeValueQuery, ServerStorage, ServeGate, StorageTransaction, StoredCheckpoint, StoredPushResult, StoredReaction, StoredRow } from './storage.js';
@@ -2,10 +2,48 @@ import { validateCommitPruneQuery } from './prune.js';
2
2
  import { StorageQueryError } from './storage-errors.js';
3
3
  import { clientRoundState, PARTITION_TOUCH_INTERVAL_MS, commitDeleteFloor, rowsLiveAfterDelete, serveGateRefusal, } from './storage.js';
4
4
  import { serveNotReadyError } from './readiness.js';
5
+ /**
6
+ * Postgres server storage: the production database path.
7
+ *
8
+ * Semantics mirror `SqliteServerStorage` exactly (both run the shared
9
+ * storage contract in `test/storage-contract.ts`); the difference is that
10
+ * scope fanout MUST survive contact with Postgres — no scan-before-LIMIT,
11
+ * ever (performance-by-construction):
12
+ *
13
+ * - both the commit log and the current-row table carry a (table, var,
14
+ * value) inverted scope index;
15
+ * - `readLiveWindow`/`scanRows` select candidates through an index
16
+ * (`sync_change_scopes_by_key` / `sync_row_scopes`), ordered + LIMITed
17
+ * at the index, then verify the full multi-variable match against the
18
+ * stored scope map — never a log or table scan;
19
+ * - the candidate indexes are *covering* for their selection: the ordered
20
+ * column (`commit_seq` / `row_id`) sits in the index key after the scope
21
+ * key, so the planner does an index range scan and returns
22
+ * already-ordered candidates without a heap sort. `test/postgres-explain
23
+ * .test.ts` asserts an `Index` node (never `Seq Scan`) so the regression
24
+ * cannot silently return.
25
+ *
26
+ * Storage is written against the `PgExecutor` seam (zero runtime deps); the
27
+ * production driver (Bun.sql / node-postgres) and the test driver (pglite)
28
+ * are wired by the host. See `pg-executor.ts` and the server README.
29
+ *
30
+ * ## commitSeq allocation under concurrency
31
+ *
32
+ * Per-partition `commitSeq` is a dense, gap-free counter (§2.1), allocated
33
+ * inside the push transaction by `UPDATE sync_partitions SET
34
+ * max_commit_seq = max_commit_seq + 1 … RETURNING`. The `UPDATE` takes a
35
+ * row-level write lock on the partition row for the duration of the
36
+ * transaction, so two concurrent pushes to the same partition serialize on
37
+ * that row: the second blocks until the first commits, then reads the
38
+ * advanced counter. This keeps the sequence dense (a Postgres `SEQUENCE`
39
+ * would leave gaps on rollback, which the pull-window arithmetic in §4.5
40
+ * does not tolerate). Cross-partition pushes never contend.
41
+ */
42
+ import { decodeRow, } from '@syncular/core';
5
43
  import { bindAuthoritativePartition, postgresPlaceholders, prepareAuthoritativeQuery, } from './authoritative-query.js';
6
44
  import { syncError } from './errors.js';
7
45
  import { asBytes, asNumber, } from './pg-executor.js';
8
- import { assertPhysicalColumns, assertStoredLayouts, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, rowsReferencingBlobSql, selectRowSql, selectRowsSql, prefetchRowsBatchSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
46
+ import { assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, changedScopePatterns, deleteRowSql, dropTableDdl, indexRowPageStatement, layoutsOf, migratePayload, parseLayouts, quoteIdent, rekeyCleanupSql, rekeyedScopes, rekeyRowScopesSql, tableColumnNames, retiredTableNames, rewritePlan, columnSqlType, rewriteValues, SCHEMA_META_DDL_POSTGRES, SYNC_COLUMN_VERSIONS_COLUMN, SYNC_PARTITION_COLUMN, SYNC_PAYLOAD_COLUMN, SYNC_ROW_ID_COLUMN, SYNC_SCOPES_COLUMN, SYNC_VERSION_COLUMN, scanRowBatchSql, scanRowPageSql, indexSignature, schemaDdl, serverIndexDdl, rowsReferencingBlobSql, selectRowSql, selectRowsSql, prefetchRowsBatchSql, selectRowsForRewriteSql, upsertSql, upsertValues, } from './relational-rows.js';
9
47
  import { matchesEffective, sha256Hex } from './scopes.js';
10
48
  import { isPostgresConstraintError, StorageConstraintError, } from './storage-errors.js';
11
49
  import { assertScopeIndexedScan, readLiveWindowPages, resolveIndexRowScan, } from './storage-query.js';
@@ -568,30 +606,35 @@ function toStoredChange(record) {
568
606
  * transaction. When `oldLayout` is
569
607
  * given every payload re-encodes under the current columns, and the
570
608
  * projection (when materialized) refreshes from the payload either way.
609
+ * `rekey` recomputes and indexes the stored scopes (§3.1).
571
610
  */
572
- async function rewriteRowsOn(q, table, oldLayout) {
611
+ async function rewriteRowsOn(q, table, oldLayout, rekey) {
573
612
  const select = selectRowsForRewriteSql(table, 'postgres');
574
613
  // One UPDATE per page: behind a network hop (Hyperdrive, a managed
575
614
  // database) every statement is a round trip, and the bump holds the
576
615
  // migration lock for its whole duration. The page's rows bind as one
577
616
  // VALUES list with typed placeholders (`rewriteValues` order: projection
578
- // columns, payload, partition, row id).
617
+ // columns, re-keyed scopes, payload, partition, row id). The scopes bind as
618
+ // text before the JSONB cast (see `scanRowBatchSql`).
619
+ const projected = table.materialize ? table.columns : [];
579
620
  const types = [
580
- ...(table.materialize
581
- ? table.columns.map((column) => columnSqlType(column, 'postgres'))
582
- : []),
621
+ ...projected.map((column) => columnSqlType(column, 'postgres')),
622
+ ...(rekey ? ['TEXT'] : []),
583
623
  'BYTEA',
584
624
  'TEXT',
585
625
  'TEXT',
586
626
  ];
587
- const projected = table.materialize ? table.columns : [];
627
+ const payloadAt = projected.length + (rekey ? 1 : 0);
588
628
  const sets = [
589
629
  ...projected.map((column, i) => `${quoteIdent(column.name)}=v.c${i}`),
590
- `${quoteIdent(SYNC_PAYLOAD_COLUMN)}=v.c${projected.length}`,
630
+ ...(rekey
631
+ ? [`${quoteIdent(SYNC_SCOPES_COLUMN)}=v.c${projected.length}::jsonb`]
632
+ : []),
633
+ `${quoteIdent(SYNC_PAYLOAD_COLUMN)}=v.c${payloadAt}`,
591
634
  ].join(', ');
592
635
  const aliases = types.map((_, i) => `c${i}`).join(', ');
593
- const partitionAlias = `c${projected.length + 1}`;
594
- const rowIdAlias = `c${projected.length + 2}`;
636
+ const partitionAlias = `c${payloadAt + 1}`;
637
+ const rowIdAlias = `c${payloadAt + 2}`;
595
638
  // A page binds types.length parameters per row; PGlite corrupts its state
596
639
  // above 32,767 bound parameters (see PG_CHANGES_PER_STATEMENT).
597
640
  const batch = Math.max(1, Math.min(500, Math.floor(30_000 / types.length)));
@@ -602,19 +645,31 @@ async function rewriteRowsOn(q, table, oldLayout) {
602
645
  if (rows.length === 0)
603
646
  break;
604
647
  const params = [];
648
+ const entries = [];
605
649
  const values = rows.map((row) => {
606
650
  const bytes = asBytes(row.payload);
607
651
  const payload = oldLayout !== undefined
608
652
  ? migratePayload(oldLayout, table, bytes)
609
653
  : bytes;
654
+ const scopes = rekey
655
+ ? rekeyedScopes(table, decodeRow(table.columns, payload), row.row_id)
656
+ : undefined;
657
+ for (const [variable, value] of Object.entries(scopes ?? {})) {
658
+ entries.push([row.partition, variable, value, row.row_id]);
659
+ }
610
660
  const base = params.length;
611
- params.push(...rewriteValues(table, row.partition, row.row_id, payload, 'postgres'));
661
+ params.push(...rewriteValues(table, row.partition, row.row_id, payload, 'postgres', scopes));
612
662
  return `(${types.map((type, i) => `$${base + i + 1}::${type}`).join(',')})`;
613
663
  });
614
664
  await q.query(`UPDATE ${quoteIdent(table.name)} AS t SET ${sets}
615
665
  FROM (VALUES ${values.join(',')}) AS v(${aliases})
616
666
  WHERE t.${quoteIdent(SYNC_PARTITION_COLUMN)}=v.${partitionAlias}
617
667
  AND t.${quoteIdent(SYNC_ROW_ID_COLUMN)}=v.${rowIdAlias}`, params);
668
+ if (entries.length > 0) {
669
+ await q.query(rekeyRowScopesSql(table, 'postgres'), [
670
+ JSON.stringify(entries),
671
+ ]);
672
+ }
618
673
  const last = rows[rows.length - 1];
619
674
  if (last === undefined || rows.length < batch)
620
675
  break;
@@ -1619,7 +1674,13 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
1619
1674
  throw new Error(`stored schema version ${stored} is newer than the configured schema (${schema.version}) — refusing to run an older server against a migrated database`);
1620
1675
  }
1621
1676
  if (stored !== undefined && stored === schema.version) {
1622
- assertStoredLayouts(schema, typeof marker.layouts === 'string' ? marker.layouts : undefined);
1677
+ const storedLayouts = typeof marker.layouts === 'string' ? marker.layouts : undefined;
1678
+ assertStoredLayouts(schema, storedLayouts);
1679
+ if (!assertStoredScopePatterns(schema, storedLayouts)) {
1680
+ // A marker from before 0.63 records no scope patterns: record the
1681
+ // configured ones, which every later start compares against.
1682
+ await this.#exec.query('UPDATE sync_schema_meta SET layouts=$1 WHERE id=1 AND schema_version=$2', [layoutsOf(schema), schema.version]);
1683
+ }
1623
1684
  const columns = JSON.parse(marker.columns);
1624
1685
  for (const table of schema.tables.values()) {
1625
1686
  assertPhysicalColumns(table, columns
@@ -1678,9 +1739,11 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
1678
1739
  }
1679
1740
  if (lockedVersion === schema.version)
1680
1741
  return;
1681
- const layouts = parseLayouts(typeof lockedMarker[0]?.layouts === 'string'
1742
+ const lockedLayouts = typeof lockedMarker[0]?.layouts === 'string'
1682
1743
  ? lockedMarker[0].layouts
1683
- : undefined);
1744
+ : undefined;
1745
+ const rekeyed = (changedScopePatterns(schema, lockedLayouts) ?? []).map((change) => change.table);
1746
+ const layouts = parseLayouts(lockedLayouts);
1684
1747
  const retiredTables = retiredTableNames(schema, layouts);
1685
1748
  const existing = new Map();
1686
1749
  const existingIndexes = new Map();
@@ -1709,12 +1772,16 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
1709
1772
  for (const statement of schemaDdl(schema, existing, 'postgres', existingIndexes)) {
1710
1773
  await client.query(statement);
1711
1774
  }
1775
+ for (const statement of rekeyCleanupSql(rekeyed)) {
1776
+ await client.query(statement);
1777
+ }
1712
1778
  for (const table of schema.tables.values()) {
1713
1779
  const oldLayout = layouts[table.name];
1714
1780
  const plan = rewritePlan(table, oldLayout, existing.get(table.name));
1715
- if (!plan.migrate && !plan.backfill)
1781
+ const rekey = rekeyed.includes(table.name);
1782
+ if (!plan.migrate && !plan.backfill && !rekey)
1716
1783
  continue;
1717
- await rewriteRowsOn(client, table, plan.migrate ? oldLayout : undefined);
1784
+ await rewriteRowsOn(client, table, plan.migrate ? oldLayout : undefined, rekey);
1718
1785
  }
1719
1786
  await client.query(`INSERT INTO sync_schema_meta(id, schema_version, layouts) VALUES (1, $1, $2)
1720
1787
  ON CONFLICT (id) DO UPDATE
@@ -298,6 +298,35 @@ export type StoredLayouts = Record<string, readonly StoredColumnLayout[]>;
298
298
  /** The layouts JSON persisted alongside the schema version marker. */
299
299
  export declare function layoutsOf(schema: CompiledSchema): string;
300
300
  export declare function parseLayouts(json: string | null | undefined): StoredLayouts;
301
+ /**
302
+ * §3.1: the configured tables whose scope patterns differ from the recorded
303
+ * ones, or `undefined` when the marker predates recorded patterns (nothing to
304
+ * compare). A schema-version bump re-keys these tables.
305
+ */
306
+ export declare function changedScopePatterns(schema: CompiledSchema, storedJson: string | null | undefined): {
307
+ table: string;
308
+ stored: string;
309
+ configured: string;
310
+ }[] | undefined;
311
+ /**
312
+ * §3.1: at the same schema version, refuse a configured table whose patterns
313
+ * differ from the recorded ones. Returns false when the marker predates
314
+ * recorded patterns, so the caller records them.
315
+ */
316
+ export declare function assertStoredScopePatterns(schema: CompiledSchema, storedJson: string | null | undefined): boolean;
317
+ /**
318
+ * §3.1 re-key, before the row walk: drop the re-keyed tables' row scope index
319
+ * entries and tombstones, and raise every partition's horizon past the
320
+ * changes logged under the previous patterns. Empty when nothing re-keys.
321
+ */
322
+ export declare function rekeyCleanupSql(tables: readonly string[]): string[];
323
+ /**
324
+ * §3.1 re-key: index the scopes of one rewritten page of rows. The entries
325
+ * bind as one JSON array of `[partition, variable, value, rowId]`, so the
326
+ * statement reads no row table (a planner that ranges a page over a declared
327
+ * index would scan the whole partition per page). Params: [entries].
328
+ */
329
+ export declare function rekeyRowScopesSql(table: CompiledTable, dialect: RelationalDialect): string;
301
330
  /**
302
331
  * Version equality is not layout equality: a marker written by another build
303
332
  * at the same version describes rows the running codec cannot decode. Refuse
@@ -334,13 +363,22 @@ export declare function migratePayload(oldLayout: readonly StoredColumnLayout[],
334
363
  * Params: [afterPartition, afterRowId, limit].
335
364
  */
336
365
  export declare function selectRowsForRewriteSql(table: CompiledTable, dialect: RelationalDialect): string;
366
+ /**
367
+ * §3.1 re-key: the row's stored scopes under the table's current patterns. A
368
+ * row without a value for one of them refuses the bump.
369
+ */
370
+ export declare function rekeyedScopes(table: CompiledTable, values: readonly RowValue[], rowId: string): Record<string, string>;
337
371
  /**
338
372
  * SQLite/D1: rewrite one row during a migration: refresh the projection columns (when
339
- * materialized) and the payload. Bind with `rewriteValues`.
373
+ * materialized), the stored scopes when the table re-keys (§3.1), and the
374
+ * payload. Bind with `rewriteValues`.
375
+ */
376
+ export declare function rewriteRowSql(table: CompiledTable, rekey: boolean): string;
377
+ /**
378
+ * Bind values for `rewriteRowSql` from the (already migrated) payload:
379
+ * projection columns, the re-keyed scopes JSON, payload, partition, row id.
340
380
  */
341
- export declare function rewriteRowSql(table: CompiledTable): string;
342
- /** Bind values for `rewriteRowSql` from the (already migrated) payload. */
343
- export declare function rewriteValues(table: CompiledTable, partition: string, rowId: string, payload: Uint8Array, dialect: RelationalDialect): RowValue[];
381
+ export declare function rewriteValues(table: CompiledTable, partition: string, rowId: string, payload: Uint8Array, dialect: RelationalDialect, scopes: Record<string, string> | undefined): RowValue[];
344
382
  /**
345
383
  * What the migration rewrite phase must do for one table, derived from the
346
384
  * persisted layouts + the physical columns present before this run.
@@ -48,6 +48,7 @@
48
48
  * NOT NULL per the schema.
49
49
  */
50
50
  import { decodeRow, encodeRow, } from '@syncular/core';
51
+ import { storedScopesForRow } from './scopes.js';
51
52
  import { StorageQueryError } from './storage-errors.js';
52
53
  export const SYNC_PARTITION_COLUMN = '_sync_partition';
53
54
  export const SYNC_ROW_ID_COLUMN = '_sync_row_id';
@@ -701,19 +702,97 @@ export const SCHEMA_META_DDL_POSTGRES = `CREATE TABLE IF NOT EXISTS sync_schema_
701
702
  /** The layouts JSON persisted alongside the schema version marker. */
702
703
  export function layoutsOf(schema) {
703
704
  const out = {};
705
+ const scopes = {};
704
706
  for (const table of schema.tables.values()) {
705
707
  out[table.name] = table.columns.map((column) => ({
706
708
  name: column.name,
707
709
  type: column.type,
708
710
  nullable: column.nullable,
709
711
  }));
712
+ scopes[table.name] = scopePatternsOf(table);
710
713
  }
714
+ // `sync_` is a reserved table-name prefix, so this key names no table.
715
+ out.sync_scopes = scopes;
711
716
  return JSON.stringify(out);
712
717
  }
718
+ function scopePatternsOf(table) {
719
+ return table.scopePatterns
720
+ .map(({ prefix, variable, column }) => `${prefix}:{${variable}}@${column}`)
721
+ .sort();
722
+ }
713
723
  export function parseLayouts(json) {
714
724
  if (json === null || json === undefined || json.length === 0)
715
725
  return {};
716
- return JSON.parse(json);
726
+ const { sync_scopes: _scopes, ...layouts } = JSON.parse(json);
727
+ return layouts;
728
+ }
729
+ /**
730
+ * §3.1: the configured tables whose scope patterns differ from the recorded
731
+ * ones, or `undefined` when the marker predates recorded patterns (nothing to
732
+ * compare). A schema-version bump re-keys these tables.
733
+ */
734
+ export function changedScopePatterns(schema, storedJson) {
735
+ if (storedJson === null || storedJson === undefined || storedJson === '')
736
+ return [];
737
+ const stored = JSON.parse(storedJson).sync_scopes;
738
+ if (stored === undefined)
739
+ return undefined;
740
+ return [...schema.tables.values()].flatMap((table) => {
741
+ const before = stored[table.name];
742
+ const after = scopePatternsOf(table);
743
+ return before !== undefined && before.join('\n') !== after.join('\n')
744
+ ? [
745
+ {
746
+ table: table.name,
747
+ stored: before.join(' '),
748
+ configured: after.join(' '),
749
+ },
750
+ ]
751
+ : [];
752
+ });
753
+ }
754
+ /**
755
+ * §3.1: at the same schema version, refuse a configured table whose patterns
756
+ * differ from the recorded ones. Returns false when the marker predates
757
+ * recorded patterns, so the caller records them.
758
+ */
759
+ export function assertStoredScopePatterns(schema, storedJson) {
760
+ const changed = changedScopePatterns(schema, storedJson);
761
+ if (changed?.[0] !== undefined) {
762
+ throw new StorageQueryError('sync.storage.scope_patterns_changed', changed[0]);
763
+ }
764
+ return changed !== undefined;
765
+ }
766
+ /**
767
+ * §3.1 re-key, before the row walk: drop the re-keyed tables' row scope index
768
+ * entries and tombstones, and raise every partition's horizon past the
769
+ * changes logged under the previous patterns. Empty when nothing re-keys.
770
+ */
771
+ export function rekeyCleanupSql(tables) {
772
+ if (tables.length === 0)
773
+ return [];
774
+ const names = tables.map((name) => `'${name.replaceAll("'", "''")}'`);
775
+ return [
776
+ `DELETE FROM sync_row_scopes WHERE tbl IN (${names.join(',')})`,
777
+ `DELETE FROM sync_tombstones WHERE tbl IN (${names.join(',')})`,
778
+ 'UPDATE sync_partitions SET horizon_seq=max_commit_seq WHERE horizon_seq<max_commit_seq',
779
+ ];
780
+ }
781
+ /**
782
+ * §3.1 re-key: index the scopes of one rewritten page of rows. The entries
783
+ * bind as one JSON array of `[partition, variable, value, rowId]`, so the
784
+ * statement reads no row table (a planner that ranges a page over a declared
785
+ * index would scan the whole partition per page). Params: [entries].
786
+ */
787
+ export function rekeyRowScopesSql(table, dialect) {
788
+ const name = `'${table.name.replaceAll("'", "''")}'`;
789
+ return dialect === 'sqlite'
790
+ ? `INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
791
+ SELECT json_extract(e.value, '$[0]'), ${name}, json_extract(e.value, '$[1]'), json_extract(e.value, '$[2]'), json_extract(e.value, '$[3]')
792
+ FROM json_each(?) AS e`
793
+ : `INSERT INTO sync_row_scopes(partition, tbl, var, value, row_id)
794
+ SELECT e->>0, ${name}, e->>1, e->>2, e->>3
795
+ FROM jsonb_array_elements($1::text::jsonb) AS e`;
717
796
  }
718
797
  /**
719
798
  * Version equality is not layout equality: a marker written by another build
@@ -820,21 +899,41 @@ export function selectRowsForRewriteSql(table, dialect) {
820
899
  ORDER BY ${quoteIdent(SYNC_PARTITION_COLUMN)}, ${quoteIdent(SYNC_ROW_ID_COLUMN)}
821
900
  LIMIT ${p[2]}`;
822
901
  }
902
+ /**
903
+ * §3.1 re-key: the row's stored scopes under the table's current patterns. A
904
+ * row without a value for one of them refuses the bump.
905
+ */
906
+ export function rekeyedScopes(table, values, rowId) {
907
+ const extracted = storedScopesForRow(table, values);
908
+ if ('missing' in extracted) {
909
+ throw new StorageQueryError('sync.storage.scope_value_missing', {
910
+ table: table.name,
911
+ variable: extracted.missing,
912
+ rowId,
913
+ });
914
+ }
915
+ return extracted.scopes;
916
+ }
823
917
  /**
824
918
  * SQLite/D1: rewrite one row during a migration: refresh the projection columns (when
825
- * materialized) and the payload. Bind with `rewriteValues`.
919
+ * materialized), the stored scopes when the table re-keys (§3.1), and the
920
+ * payload. Bind with `rewriteValues`.
826
921
  */
827
- export function rewriteRowSql(table) {
922
+ export function rewriteRowSql(table, rekey) {
828
923
  const sets = [
829
924
  ...(table.materialize
830
925
  ? table.columns.map((column) => `${quoteIdent(column.name)}=?`)
831
926
  : []),
927
+ ...(rekey ? [`${quoteIdent(SYNC_SCOPES_COLUMN)}=?`] : []),
832
928
  `${quoteIdent(SYNC_PAYLOAD_COLUMN)}=?`,
833
929
  ];
834
930
  return `UPDATE ${quoteIdent(table.name)} SET ${sets.join(', ')} WHERE ${quoteIdent(SYNC_PARTITION_COLUMN)}=? AND ${quoteIdent(SYNC_ROW_ID_COLUMN)}=?`;
835
931
  }
836
- /** Bind values for `rewriteRowSql` from the (already migrated) payload. */
837
- export function rewriteValues(table, partition, rowId, payload, dialect) {
932
+ /**
933
+ * Bind values for `rewriteRowSql` from the (already migrated) payload:
934
+ * projection columns, the re-keyed scopes JSON, payload, partition, row id.
935
+ */
936
+ export function rewriteValues(table, partition, rowId, payload, dialect, scopes) {
838
937
  const out = [];
839
938
  if (table.materialize) {
840
939
  const values = decodeRow(table.columns, payload);
@@ -845,6 +944,8 @@ export function rewriteValues(table, partition, rowId, payload, dialect) {
845
944
  out.push(toSqlValue(column, values[i] ?? null, dialect));
846
945
  }
847
946
  }
947
+ if (scopes !== undefined)
948
+ out.push(JSON.stringify(scopes));
848
949
  out.push(payload, partition, rowId);
849
950
  return out;
850
951
  }