@syncular/server 0.62.1 → 0.64.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -9,10 +9,11 @@
9
9
  * select candidates from the index and verify the full multi-variable
10
10
  * match against the stored scope map — never a log scan.
11
11
  */
12
+ import { decodeRow } from '@syncular/core';
12
13
  import { isSqliteConstraintError, retainRollbackFailure, rollbackAfterError, StorageConstraintError, StorageQueryError, } from './storage-errors.js';
13
14
  import { commitDeleteFloor, serveGateRefusal } from './storage.js';
14
15
  import { serveNotReadyError } from './readiness.js';
15
- import { assertPhysicalColumns, assertStoredLayouts, dropTableDdl, layoutsOf, migratePayload, parseLayouts, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowsForRewriteSql, } from './relational-rows.js';
16
+ import { assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, changedScopePatterns, dropTableDdl, layoutsOf, migratePayload, parseLayouts, rekeyCleanupSql, rekeyedScopes, rekeyRowScopesSql, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowsForRewriteSql, } from './relational-rows.js';
16
17
  import { SQLITE_DDL, SQLITE_UPGRADE_PROBE, SqliteFamilyStorage, SqliteFamilyTransaction, sqliteUpgradeDdl, } from './sqlite-dialect.js';
17
18
  import { SqliteAdapterRequiredError, } from './sqlite-driver.js';
18
19
  function toStoredCheckpoint(record) {
@@ -330,6 +331,13 @@ END`);
330
331
  }
331
332
  if (marker !== null && marker.schema_version === schema.version) {
332
333
  assertStoredLayouts(schema, marker.layouts);
334
+ if (!assertStoredScopePatterns(schema, marker.layouts)) {
335
+ // A marker from before 0.63 records no scope patterns: record the
336
+ // configured ones, which every later start compares against.
337
+ this.db
338
+ .query('UPDATE sync_schema_meta SET layouts=? WHERE id=1 AND schema_version=?')
339
+ .run(layoutsOf(schema), schema.version);
340
+ }
333
341
  for (const table of schema.tables.values()) {
334
342
  const escapedTableName = table.name.replaceAll('"', '""');
335
343
  const columns = this.db
@@ -406,6 +414,7 @@ END`);
406
414
  this.schemaVersion = schema.version;
407
415
  return;
408
416
  }
417
+ const rekeyed = (changedScopePatterns(schema, locked?.layouts) ?? []).map((change) => change.table);
409
418
  const layouts = parseLayouts(locked?.layouts);
410
419
  const retiredTables = retiredTableNames(schema, layouts);
411
420
  for (const tableName of retiredTables) {
@@ -420,12 +429,16 @@ END`);
420
429
  for (const statement of schemaDdl(schema, existing, 'sqlite', existingIndexes)) {
421
430
  this.db.exec(statement);
422
431
  }
432
+ for (const statement of rekeyCleanupSql(rekeyed)) {
433
+ this.db.exec(statement);
434
+ }
423
435
  for (const table of schema.tables.values()) {
424
436
  const oldLayout = layouts[table.name];
425
437
  const plan = rewritePlan(table, oldLayout, existing.get(table.name));
426
- if (!plan.migrate && !plan.backfill)
438
+ const rekey = rekeyed.includes(table.name);
439
+ if (!plan.migrate && !plan.backfill && !rekey)
427
440
  continue;
428
- this.#rewriteRows(table, plan.migrate ? oldLayout : undefined);
441
+ this.#rewriteRows(table, plan.migrate ? oldLayout : undefined, rekey);
429
442
  }
430
443
  this.db
431
444
  .query('INSERT INTO sync_schema_meta(id, schema_version, layouts) VALUES (1, ?, ?) ON CONFLICT(id) DO UPDATE SET schema_version=excluded.schema_version, layouts=excluded.layouts')
@@ -463,11 +476,13 @@ END`);
463
476
  * Migration rewrite: keyset-paged walk of a row table. When `oldLayout` is
464
477
  * given every payload re-encodes under
465
478
  * the current columns, and the projection (when materialized) refreshes
466
- * from the payload either way.
479
+ * from the payload either way. `rekey` recomputes and indexes the stored
480
+ * scopes (§3.1).
467
481
  */
468
- #rewriteRows(table, oldLayout) {
482
+ #rewriteRows(table, oldLayout, rekey) {
469
483
  const select = selectRowsForRewriteSql(table, 'sqlite');
470
- const update = this.db.query(rewriteRowSql(table));
484
+ const update = this.db.query(rewriteRowSql(table, rekey));
485
+ const index = this.db.query(rekeyRowScopesSql(table, 'sqlite'));
471
486
  const BATCH = 500;
472
487
  let afterPartition = '';
473
488
  let afterRowId = '';
@@ -477,12 +492,21 @@ END`);
477
492
  .all(afterPartition, afterRowId, BATCH);
478
493
  if (rows.length === 0)
479
494
  break;
495
+ const entries = [];
480
496
  for (const row of rows) {
481
497
  const payload = oldLayout !== undefined
482
498
  ? migratePayload(oldLayout, table, row.payload)
483
499
  : row.payload;
484
- update.run(...rewriteValues(table, row.partition, row.row_id, payload, 'sqlite'));
500
+ const scopes = rekey
501
+ ? rekeyedScopes(table, decodeRow(table.columns, payload), row.row_id)
502
+ : undefined;
503
+ for (const [variable, value] of Object.entries(scopes ?? {})) {
504
+ entries.push([row.partition, variable, value, row.row_id]);
505
+ }
506
+ update.run(...rewriteValues(table, row.partition, row.row_id, payload, 'sqlite', scopes));
485
507
  }
508
+ if (entries.length > 0)
509
+ index.run(JSON.stringify(entries));
486
510
  const last = rows[rows.length - 1];
487
511
  if (last === undefined || rows.length < BATCH)
488
512
  break;
@@ -9,7 +9,7 @@ export declare class StorageConstraintError extends Error {
9
9
  constructor(cause: unknown, opIndex?: number);
10
10
  }
11
11
  /** Stable, privacy-safe failures for trusted server storage queries. */
12
- export type StorageQueryErrorCode = 'sync.storage.schema_migration_pending' | 'sync.storage.schema_migration_conflict' | 'sync.storage.schema_changed' | 'sync.storage.invalid_migration_budget' | 'sync.storage.scan_requires_scope' | 'sync.storage.index_not_found' | 'sync.storage.index_not_materialized' | 'sync.storage.index_value_count_mismatch' | 'sync.storage.invalid_limit' | 'sync.storage.prune_epoch_mismatch' | 'sync.storage.partition_unregistered' | 'sync.storage.invalid_prune_cursor' | 'sync.storage.checkpoint_not_declared' | 'sync.storage.checkpoint_unsupported' | 'sync.storage.checkpoint_fence_missing' | 'sync.storage.checkpoint_incomplete' | 'sync.storage.stored_layout_mismatch' | 'sync.storage.physical_layout_mismatch' | 'sync.storage.transaction_query_unsupported' | 'sync.storage.partition_lock_unsupported' | 'sync.storage.query_over_staged_writes' | 'sync.storage.erase_unsupported' | 'sync.storage.invalid_erase_query';
12
+ export type StorageQueryErrorCode = 'sync.storage.schema_migration_pending' | 'sync.storage.schema_migration_conflict' | 'sync.storage.schema_changed' | 'sync.storage.invalid_migration_budget' | 'sync.storage.scan_requires_scope' | 'sync.storage.index_not_found' | 'sync.storage.index_not_materialized' | 'sync.storage.index_value_count_mismatch' | 'sync.storage.invalid_limit' | 'sync.storage.prune_epoch_mismatch' | 'sync.storage.partition_unregistered' | 'sync.storage.invalid_prune_cursor' | 'sync.storage.checkpoint_not_declared' | 'sync.storage.checkpoint_unsupported' | 'sync.storage.checkpoint_fence_missing' | 'sync.storage.checkpoint_incomplete' | 'sync.storage.stored_layout_mismatch' | 'sync.storage.scope_patterns_changed' | 'sync.storage.scope_value_missing' | 'sync.storage.physical_layout_mismatch' | 'sync.storage.transaction_query_unsupported' | 'sync.storage.partition_lock_unsupported' | 'sync.storage.query_over_staged_writes' | 'sync.storage.erase_unsupported' | 'sync.storage.invalid_erase_query';
13
13
  /**
14
14
  * Host-only query error. Messages never include identifiers, values, SQL,
15
15
  * paths, or row data; callers branch on `code`, never message text. The
@@ -24,6 +24,8 @@ const STORAGE_QUERY_MESSAGES = {
24
24
  'sync.storage.checkpoint_fence_missing': 'checkpoint activation requires the writer fence raised at declaration',
25
25
  'sync.storage.checkpoint_incomplete': 'a declared backfill checkpoint is not activated and this caller did not declare it',
26
26
  'sync.storage.stored_layout_mismatch': 'stored schema layouts disagree with the configured schema at the same version',
27
+ 'sync.storage.scope_patterns_changed': 'a table declares scope patterns other than the ones its stored rows are indexed under',
28
+ 'sync.storage.scope_value_missing': 'a stored row has no value for a scope column the schema bump re-keys it under',
27
29
  'sync.storage.physical_layout_mismatch': 'a synced table does not match the storage layout the running code reads and writes',
28
30
  'sync.storage.transaction_query_unsupported': 'this storage transaction cannot run registered queries',
29
31
  'sync.storage.partition_lock_unsupported': 'this storage transaction cannot serialize partition authority',
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@syncular/server",
3
- "version": "0.62.1",
3
+ "version": "0.64.0",
4
4
  "description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
5
5
  "license": "Apache-2.0",
6
6
  "author": "Benjamin Kniffler",
@@ -68,7 +68,7 @@
68
68
  "verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
69
69
  },
70
70
  "dependencies": {
71
- "@syncular/core": "0.62.1"
71
+ "@syncular/core": "0.64.0"
72
72
  },
73
73
  "devDependencies": {
74
74
  "@electric-sql/pglite": "^0.5.4"
package/src/d1-storage.ts CHANGED
@@ -44,12 +44,17 @@ import {
44
44
  assertAppendOnlyMigration,
45
45
  assertPhysicalColumns,
46
46
  assertStoredLayouts,
47
+ assertStoredScopePatterns,
48
+ changedScopePatterns,
47
49
  dropTableDdl,
48
50
  indexRowPageStatement,
49
51
  layoutsOf,
50
52
  migratePayload,
51
53
  parseLayouts,
52
54
  quoteIdent,
55
+ rekeyCleanupSql,
56
+ rekeyedScopes,
57
+ rekeyRowScopesSql,
53
58
  retiredTableNames,
54
59
  rewritePlan,
55
60
  rewriteRowSql,
@@ -835,7 +840,11 @@ interface D1MigrationState {
835
840
  columns: Record<string, string[]>;
836
841
  indexes: Record<string, Record<string, string>>;
837
842
  ddl: string[];
838
- rewrites: Array<{ table: string; oldLayout?: readonly StoredColumnLayout[] }>;
843
+ rewrites: Array<{
844
+ table: string;
845
+ oldLayout?: readonly StoredColumnLayout[];
846
+ rekey?: true;
847
+ }>;
839
848
  afterPartition: string;
840
849
  afterRowId: string;
841
850
  }
@@ -1052,6 +1061,17 @@ export class D1ServerStorage
1052
1061
  );
1053
1062
  if (claim === null && marker?.schema_version === schema.version) {
1054
1063
  assertStoredLayouts(schema, marker.layouts);
1064
+ if (!assertStoredScopePatterns(schema, marker.layouts)) {
1065
+ // A marker from before 0.63 records no scope patterns: record the
1066
+ // configured ones, which every later start compares against.
1067
+ statementsExecuted++;
1068
+ await this.#db
1069
+ .prepare(
1070
+ 'UPDATE sync_schema_meta SET layouts=? WHERE id=1 AND schema_version=?',
1071
+ )
1072
+ .bind(layoutsOf(schema), schema.version)
1073
+ .run();
1074
+ }
1055
1075
  statementsExecuted++;
1056
1076
  const present = await this.#db
1057
1077
  .prepare(
@@ -1167,6 +1187,9 @@ export class D1ServerStorage
1167
1187
  throw new StorageQueryError('sync.storage.schema_migration_conflict');
1168
1188
  }
1169
1189
  const sourceLayouts = parseLayouts(claim.source_layouts);
1190
+ const rekeyed = (
1191
+ changedScopePatterns(schema, claim.source_layouts) ?? []
1192
+ ).map((change) => change.table);
1170
1193
  let state = JSON.parse(claim.state) as D1MigrationState;
1171
1194
  // Reserve one statement to distinguish a stale checkpoint from a database
1172
1195
  // failure. A losing concurrent step returns incomplete without retrying here.
@@ -1239,7 +1262,60 @@ export class D1ServerStorage
1239
1262
  } else if (state.phase === 'prepare') {
1240
1263
  const table = tables[state.offset];
1241
1264
  if (table !== undefined) {
1242
- if (remaining < 4) break;
1265
+ // §3.1: every row of a re-keyed table needs a value for each new
1266
+ // pattern. Check before any DDL (one more statement); a refusal
1267
+ // releases the claim, so the refused bump changes nothing.
1268
+ const oldLayout = sourceLayouts[table.name];
1269
+ const verify =
1270
+ rekeyed.includes(table.name) && oldLayout !== undefined;
1271
+ if (remaining < (verify ? 5 : 4)) break;
1272
+ if (verify) {
1273
+ statementsExecuted++;
1274
+ const { results } = await this.#db
1275
+ .prepare(`SELECT * FROM (${selectRowsForRewriteSql(table, 'sqlite')})
1276
+ WHERE EXISTS (SELECT 1 FROM sync_schema_migration WHERE id=1 AND target=? AND state=?)`)
1277
+ .bind(
1278
+ state.afterPartition,
1279
+ state.afterRowId,
1280
+ 500,
1281
+ target,
1282
+ claim.state,
1283
+ )
1284
+ .all<{ partition: string; row_id: string; payload: unknown }>();
1285
+ try {
1286
+ for (const row of results) {
1287
+ rekeyedScopes(
1288
+ table,
1289
+ decodeRow(oldLayout, asUint8Array(row.payload)),
1290
+ row.row_id,
1291
+ );
1292
+ }
1293
+ } catch (error) {
1294
+ await batch([
1295
+ this.#db
1296
+ .prepare(
1297
+ 'DELETE FROM sync_schema_migration WHERE id=1 AND target=? AND state=?',
1298
+ )
1299
+ .bind(target, claim.state),
1300
+ ]);
1301
+ throw error;
1302
+ }
1303
+ const last = results.at(-1);
1304
+ if (last !== undefined && results.length === 500) {
1305
+ if (
1306
+ !(await save(
1307
+ {
1308
+ ...state,
1309
+ afterPartition: last.partition,
1310
+ afterRowId: last.row_id,
1311
+ },
1312
+ [],
1313
+ ))
1314
+ )
1315
+ break;
1316
+ continue;
1317
+ }
1318
+ }
1243
1319
  statementsExecuted += 2;
1244
1320
  const columns = await this.#db
1245
1321
  .prepare(`PRAGMA table_info(${quoteIdent(table.name)})`)
@@ -1261,6 +1337,8 @@ export class D1ServerStorage
1261
1337
  {
1262
1338
  ...state,
1263
1339
  offset: state.offset + 1,
1340
+ afterPartition: '',
1341
+ afterRowId: '',
1264
1342
  columns: {
1265
1343
  ...state.columns,
1266
1344
  ...(columns.results.length > 0
@@ -1300,13 +1378,15 @@ export class D1ServerStorage
1300
1378
  sourceLayouts[table.name],
1301
1379
  columns.get(table.name),
1302
1380
  );
1303
- return plan.migrate || plan.backfill
1381
+ const rekey = rekeyed.includes(table.name);
1382
+ return plan.migrate || plan.backfill || rekey
1304
1383
  ? [
1305
1384
  {
1306
1385
  table: table.name,
1307
1386
  ...(plan.migrate
1308
1387
  ? { oldLayout: sourceLayouts[table.name] }
1309
1388
  : {}),
1389
+ ...(rekey ? { rekey: true as const } : {}),
1310
1390
  },
1311
1391
  ]
1312
1392
  : [];
@@ -1318,6 +1398,7 @@ export class D1ServerStorage
1318
1398
  dropTableDdl(name),
1319
1399
  ]),
1320
1400
  ...schemaDdl(schema, columns, 'sqlite', indexes),
1401
+ ...rekeyCleanupSql(rekeyed),
1321
1402
  ];
1322
1403
  if (
1323
1404
  !(await save(
@@ -1376,8 +1457,10 @@ export class D1ServerStorage
1376
1457
  const table = schema.tables.get(plan.table);
1377
1458
  if (table === undefined)
1378
1459
  throw new StorageQueryError('sync.storage.schema_migration_conflict');
1379
- if (remaining < 4) break;
1380
- const limit = Math.min(32, remaining - 3);
1460
+ // A re-keyed page adds one scope-index statement.
1461
+ const rekey = plan.rekey === true;
1462
+ if (remaining < (rekey ? 5 : 4)) break;
1463
+ const limit = Math.min(32, remaining - (rekey ? 4 : 3));
1381
1464
  statementsExecuted++;
1382
1465
  const { results } = await this.#db
1383
1466
  .prepare(`SELECT * FROM (${selectRowsForRewriteSql(table, 'sqlite')})
@@ -1390,14 +1473,25 @@ export class D1ServerStorage
1390
1473
  claim.state,
1391
1474
  )
1392
1475
  .all<{ partition: string; row_id: string; payload: unknown }>();
1476
+ const entries: string[][] = [];
1393
1477
  const statements = results.map((row) => {
1394
1478
  const bytes = asUint8Array(row.payload);
1395
1479
  const payload =
1396
1480
  plan.oldLayout !== undefined
1397
1481
  ? migratePayload(plan.oldLayout, table, bytes)
1398
1482
  : bytes;
1483
+ const scopes = rekey
1484
+ ? rekeyedScopes(
1485
+ table,
1486
+ decodeRow(table.columns, payload),
1487
+ row.row_id,
1488
+ )
1489
+ : undefined;
1490
+ for (const [variable, value] of Object.entries(scopes ?? {})) {
1491
+ entries.push([row.partition, variable, value, row.row_id]);
1492
+ }
1399
1493
  return this.#db
1400
- .prepare(rewriteRowSql(table))
1494
+ .prepare(rewriteRowSql(table, rekey))
1401
1495
  .bind(
1402
1496
  ...rewriteValues(
1403
1497
  table,
@@ -1405,9 +1499,17 @@ export class D1ServerStorage
1405
1499
  row.row_id,
1406
1500
  payload,
1407
1501
  'sqlite',
1502
+ scopes,
1408
1503
  ),
1409
1504
  );
1410
1505
  });
1506
+ if (entries.length > 0) {
1507
+ statements.push(
1508
+ this.#db
1509
+ .prepare(rekeyRowScopesSql(table, 'sqlite'))
1510
+ .bind(JSON.stringify(entries)),
1511
+ );
1512
+ }
1411
1513
  const last = results.at(-1);
1412
1514
  const done = results.length < limit;
1413
1515
  if (
@@ -46,7 +46,11 @@ import { serveNotReadyError } from './readiness';
46
46
  * would leave gaps on rollback, which the pull-window arithmetic in §4.5
47
47
  * does not tolerate). Cross-partition pushes never contend.
48
48
  */
49
- import type { PushOperationResult, UnitScopes } from '@syncular/core';
49
+ import {
50
+ decodeRow,
51
+ type PushOperationResult,
52
+ type UnitScopes,
53
+ } from '@syncular/core';
50
54
  import {
51
55
  bindAuthoritativePartition,
52
56
  type BoundAuthoritativeQuery,
@@ -64,6 +68,8 @@ import {
64
68
  import {
65
69
  assertPhysicalColumns,
66
70
  assertStoredLayouts,
71
+ assertStoredScopePatterns,
72
+ changedScopePatterns,
67
73
  deleteRowSql,
68
74
  dropTableDdl,
69
75
  indexRowPageStatement,
@@ -71,6 +77,9 @@ import {
71
77
  migratePayload,
72
78
  parseLayouts,
73
79
  quoteIdent,
80
+ rekeyCleanupSql,
81
+ rekeyedScopes,
82
+ rekeyRowScopesSql,
74
83
  tableColumnNames,
75
84
  retiredTableNames,
76
85
  rewritePlan,
@@ -932,34 +941,40 @@ function toStoredChange(record: ChangeRecord): StoredChange {
932
941
  * transaction. When `oldLayout` is
933
942
  * given every payload re-encodes under the current columns, and the
934
943
  * projection (when materialized) refreshes from the payload either way.
944
+ * `rekey` recomputes and indexes the stored scopes (§3.1).
935
945
  */
936
946
  async function rewriteRowsOn(
937
947
  q: PgQueryable,
938
948
  table: CompiledTable,
939
949
  oldLayout: readonly StoredColumnLayout[] | undefined,
950
+ rekey: boolean,
940
951
  ): Promise<void> {
941
952
  const select = selectRowsForRewriteSql(table, 'postgres');
942
953
  // One UPDATE per page: behind a network hop (Hyperdrive, a managed
943
954
  // database) every statement is a round trip, and the bump holds the
944
955
  // migration lock for its whole duration. The page's rows bind as one
945
956
  // VALUES list with typed placeholders (`rewriteValues` order: projection
946
- // columns, payload, partition, row id).
957
+ // columns, re-keyed scopes, payload, partition, row id). The scopes bind as
958
+ // text before the JSONB cast (see `scanRowBatchSql`).
959
+ const projected = table.materialize ? table.columns : [];
947
960
  const types = [
948
- ...(table.materialize
949
- ? table.columns.map((column) => columnSqlType(column, 'postgres'))
950
- : []),
961
+ ...projected.map((column) => columnSqlType(column, 'postgres')),
962
+ ...(rekey ? ['TEXT'] : []),
951
963
  'BYTEA',
952
964
  'TEXT',
953
965
  'TEXT',
954
966
  ];
955
- const projected = table.materialize ? table.columns : [];
967
+ const payloadAt = projected.length + (rekey ? 1 : 0);
956
968
  const sets = [
957
969
  ...projected.map((column, i) => `${quoteIdent(column.name)}=v.c${i}`),
958
- `${quoteIdent(SYNC_PAYLOAD_COLUMN)}=v.c${projected.length}`,
970
+ ...(rekey
971
+ ? [`${quoteIdent(SYNC_SCOPES_COLUMN)}=v.c${projected.length}::jsonb`]
972
+ : []),
973
+ `${quoteIdent(SYNC_PAYLOAD_COLUMN)}=v.c${payloadAt}`,
959
974
  ].join(', ');
960
975
  const aliases = types.map((_, i) => `c${i}`).join(', ');
961
- const partitionAlias = `c${projected.length + 1}`;
962
- const rowIdAlias = `c${projected.length + 2}`;
976
+ const partitionAlias = `c${payloadAt + 1}`;
977
+ const rowIdAlias = `c${payloadAt + 2}`;
963
978
  // A page binds types.length parameters per row; PGlite corrupts its state
964
979
  // above 32,767 bound parameters (see PG_CHANGES_PER_STATEMENT).
965
980
  const batch = Math.max(1, Math.min(500, Math.floor(30_000 / types.length)));
@@ -973,15 +988,29 @@ async function rewriteRowsOn(
973
988
  }>(select, [afterPartition, afterRowId, batch]);
974
989
  if (rows.length === 0) break;
975
990
  const params: unknown[] = [];
991
+ const entries: string[][] = [];
976
992
  const values = rows.map((row) => {
977
993
  const bytes = asBytes(row.payload);
978
994
  const payload =
979
995
  oldLayout !== undefined
980
996
  ? migratePayload(oldLayout, table, bytes)
981
997
  : bytes;
998
+ const scopes = rekey
999
+ ? rekeyedScopes(table, decodeRow(table.columns, payload), row.row_id)
1000
+ : undefined;
1001
+ for (const [variable, value] of Object.entries(scopes ?? {})) {
1002
+ entries.push([row.partition, variable, value, row.row_id]);
1003
+ }
982
1004
  const base = params.length;
983
1005
  params.push(
984
- ...rewriteValues(table, row.partition, row.row_id, payload, 'postgres'),
1006
+ ...rewriteValues(
1007
+ table,
1008
+ row.partition,
1009
+ row.row_id,
1010
+ payload,
1011
+ 'postgres',
1012
+ scopes,
1013
+ ),
985
1014
  );
986
1015
  return `(${types.map((type, i) => `$${base + i + 1}::${type}`).join(',')})`;
987
1016
  });
@@ -992,6 +1021,11 @@ async function rewriteRowsOn(
992
1021
  AND t.${quoteIdent(SYNC_ROW_ID_COLUMN)}=v.${rowIdAlias}`,
993
1022
  params,
994
1023
  );
1024
+ if (entries.length > 0) {
1025
+ await q.query(rekeyRowScopesSql(table, 'postgres'), [
1026
+ JSON.stringify(entries),
1027
+ ]);
1028
+ }
995
1029
  const last = rows[rows.length - 1];
996
1030
  if (last === undefined || rows.length < batch) break;
997
1031
  afterPartition = last.partition;
@@ -2361,10 +2395,17 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2361
2395
  );
2362
2396
  }
2363
2397
  if (stored !== undefined && stored === schema.version) {
2364
- assertStoredLayouts(
2365
- schema,
2366
- typeof marker.layouts === 'string' ? marker.layouts : undefined,
2367
- );
2398
+ const storedLayouts =
2399
+ typeof marker.layouts === 'string' ? marker.layouts : undefined;
2400
+ assertStoredLayouts(schema, storedLayouts);
2401
+ if (!assertStoredScopePatterns(schema, storedLayouts)) {
2402
+ // A marker from before 0.63 records no scope patterns: record the
2403
+ // configured ones, which every later start compares against.
2404
+ await this.#exec.query(
2405
+ 'UPDATE sync_schema_meta SET layouts=$1 WHERE id=1 AND schema_version=$2',
2406
+ [layoutsOf(schema), schema.version],
2407
+ );
2408
+ }
2368
2409
  const columns = JSON.parse(marker.columns) as {
2369
2410
  tbl: string;
2370
2411
  name: string;
@@ -2444,11 +2485,14 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2444
2485
  );
2445
2486
  }
2446
2487
  if (lockedVersion === schema.version) return;
2447
- const layouts = parseLayouts(
2488
+ const lockedLayouts =
2448
2489
  typeof lockedMarker[0]?.layouts === 'string'
2449
2490
  ? lockedMarker[0].layouts
2450
- : undefined,
2491
+ : undefined;
2492
+ const rekeyed = (changedScopePatterns(schema, lockedLayouts) ?? []).map(
2493
+ (change) => change.table,
2451
2494
  );
2495
+ const layouts = parseLayouts(lockedLayouts);
2452
2496
  const retiredTables = retiredTableNames(schema, layouts);
2453
2497
  const existing = new Map<string, ReadonlySet<string>>();
2454
2498
  const existingIndexes = new Map<string, ReadonlyMap<string, string>>();
@@ -2494,14 +2538,19 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
2494
2538
  )) {
2495
2539
  await client.query(statement);
2496
2540
  }
2541
+ for (const statement of rekeyCleanupSql(rekeyed)) {
2542
+ await client.query(statement);
2543
+ }
2497
2544
  for (const table of schema.tables.values()) {
2498
2545
  const oldLayout = layouts[table.name];
2499
2546
  const plan = rewritePlan(table, oldLayout, existing.get(table.name));
2500
- if (!plan.migrate && !plan.backfill) continue;
2547
+ const rekey = rekeyed.includes(table.name);
2548
+ if (!plan.migrate && !plan.backfill && !rekey) continue;
2501
2549
  await rewriteRowsOn(
2502
2550
  client,
2503
2551
  table,
2504
2552
  plan.migrate ? oldLayout : undefined,
2553
+ rekey,
2505
2554
  );
2506
2555
  }
2507
2556
  await client.query(