@syncular/server 0.62.1 → 0.64.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/d1-storage.js +72 -7
- package/dist/postgres-storage.d.ts +1 -1
- package/dist/postgres-storage.js +83 -16
- package/dist/relational-rows.d.ts +42 -4
- package/dist/relational-rows.js +106 -5
- package/dist/sqlite-storage.js +31 -7
- package/dist/storage-errors.d.ts +1 -1
- package/dist/storage-errors.js +2 -0
- package/package.json +2 -2
- package/src/d1-storage.ts +108 -6
- package/src/postgres-storage.ts +66 -17
- package/src/relational-rows.ts +133 -5
- package/src/sqlite-storage.ts +45 -5
- package/src/storage-errors.ts +6 -0
package/dist/sqlite-storage.js
CHANGED
|
@@ -9,10 +9,11 @@
|
|
|
9
9
|
* select candidates from the index and verify the full multi-variable
|
|
10
10
|
* match against the stored scope map — never a log scan.
|
|
11
11
|
*/
|
|
12
|
+
import { decodeRow } from '@syncular/core';
|
|
12
13
|
import { isSqliteConstraintError, retainRollbackFailure, rollbackAfterError, StorageConstraintError, StorageQueryError, } from './storage-errors.js';
|
|
13
14
|
import { commitDeleteFloor, serveGateRefusal } from './storage.js';
|
|
14
15
|
import { serveNotReadyError } from './readiness.js';
|
|
15
|
-
import { assertPhysicalColumns, assertStoredLayouts, dropTableDdl, layoutsOf, migratePayload, parseLayouts, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowsForRewriteSql, } from './relational-rows.js';
|
|
16
|
+
import { assertPhysicalColumns, assertStoredLayouts, assertStoredScopePatterns, changedScopePatterns, dropTableDdl, layoutsOf, migratePayload, parseLayouts, rekeyCleanupSql, rekeyedScopes, rekeyRowScopesSql, retiredTableNames, rewritePlan, rewriteRowSql, rewriteValues, SCHEMA_META_DDL_SQLITE, schemaDdl, serverIndexDdl, SQLITE_TABLE_INDEXES_SQL, sqliteIndexSignatures, selectRowsForRewriteSql, } from './relational-rows.js';
|
|
16
17
|
import { SQLITE_DDL, SQLITE_UPGRADE_PROBE, SqliteFamilyStorage, SqliteFamilyTransaction, sqliteUpgradeDdl, } from './sqlite-dialect.js';
|
|
17
18
|
import { SqliteAdapterRequiredError, } from './sqlite-driver.js';
|
|
18
19
|
function toStoredCheckpoint(record) {
|
|
@@ -330,6 +331,13 @@ END`);
|
|
|
330
331
|
}
|
|
331
332
|
if (marker !== null && marker.schema_version === schema.version) {
|
|
332
333
|
assertStoredLayouts(schema, marker.layouts);
|
|
334
|
+
if (!assertStoredScopePatterns(schema, marker.layouts)) {
|
|
335
|
+
// A marker from before 0.63 records no scope patterns: record the
|
|
336
|
+
// configured ones, which every later start compares against.
|
|
337
|
+
this.db
|
|
338
|
+
.query('UPDATE sync_schema_meta SET layouts=? WHERE id=1 AND schema_version=?')
|
|
339
|
+
.run(layoutsOf(schema), schema.version);
|
|
340
|
+
}
|
|
333
341
|
for (const table of schema.tables.values()) {
|
|
334
342
|
const escapedTableName = table.name.replaceAll('"', '""');
|
|
335
343
|
const columns = this.db
|
|
@@ -406,6 +414,7 @@ END`);
|
|
|
406
414
|
this.schemaVersion = schema.version;
|
|
407
415
|
return;
|
|
408
416
|
}
|
|
417
|
+
const rekeyed = (changedScopePatterns(schema, locked?.layouts) ?? []).map((change) => change.table);
|
|
409
418
|
const layouts = parseLayouts(locked?.layouts);
|
|
410
419
|
const retiredTables = retiredTableNames(schema, layouts);
|
|
411
420
|
for (const tableName of retiredTables) {
|
|
@@ -420,12 +429,16 @@ END`);
|
|
|
420
429
|
for (const statement of schemaDdl(schema, existing, 'sqlite', existingIndexes)) {
|
|
421
430
|
this.db.exec(statement);
|
|
422
431
|
}
|
|
432
|
+
for (const statement of rekeyCleanupSql(rekeyed)) {
|
|
433
|
+
this.db.exec(statement);
|
|
434
|
+
}
|
|
423
435
|
for (const table of schema.tables.values()) {
|
|
424
436
|
const oldLayout = layouts[table.name];
|
|
425
437
|
const plan = rewritePlan(table, oldLayout, existing.get(table.name));
|
|
426
|
-
|
|
438
|
+
const rekey = rekeyed.includes(table.name);
|
|
439
|
+
if (!plan.migrate && !plan.backfill && !rekey)
|
|
427
440
|
continue;
|
|
428
|
-
this.#rewriteRows(table, plan.migrate ? oldLayout : undefined);
|
|
441
|
+
this.#rewriteRows(table, plan.migrate ? oldLayout : undefined, rekey);
|
|
429
442
|
}
|
|
430
443
|
this.db
|
|
431
444
|
.query('INSERT INTO sync_schema_meta(id, schema_version, layouts) VALUES (1, ?, ?) ON CONFLICT(id) DO UPDATE SET schema_version=excluded.schema_version, layouts=excluded.layouts')
|
|
@@ -463,11 +476,13 @@ END`);
|
|
|
463
476
|
* Migration rewrite: keyset-paged walk of a row table. When `oldLayout` is
|
|
464
477
|
* given every payload re-encodes under
|
|
465
478
|
* the current columns, and the projection (when materialized) refreshes
|
|
466
|
-
* from the payload either way.
|
|
479
|
+
* from the payload either way. `rekey` recomputes and indexes the stored
|
|
480
|
+
* scopes (§3.1).
|
|
467
481
|
*/
|
|
468
|
-
#rewriteRows(table, oldLayout) {
|
|
482
|
+
#rewriteRows(table, oldLayout, rekey) {
|
|
469
483
|
const select = selectRowsForRewriteSql(table, 'sqlite');
|
|
470
|
-
const update = this.db.query(rewriteRowSql(table));
|
|
484
|
+
const update = this.db.query(rewriteRowSql(table, rekey));
|
|
485
|
+
const index = this.db.query(rekeyRowScopesSql(table, 'sqlite'));
|
|
471
486
|
const BATCH = 500;
|
|
472
487
|
let afterPartition = '';
|
|
473
488
|
let afterRowId = '';
|
|
@@ -477,12 +492,21 @@ END`);
|
|
|
477
492
|
.all(afterPartition, afterRowId, BATCH);
|
|
478
493
|
if (rows.length === 0)
|
|
479
494
|
break;
|
|
495
|
+
const entries = [];
|
|
480
496
|
for (const row of rows) {
|
|
481
497
|
const payload = oldLayout !== undefined
|
|
482
498
|
? migratePayload(oldLayout, table, row.payload)
|
|
483
499
|
: row.payload;
|
|
484
|
-
|
|
500
|
+
const scopes = rekey
|
|
501
|
+
? rekeyedScopes(table, decodeRow(table.columns, payload), row.row_id)
|
|
502
|
+
: undefined;
|
|
503
|
+
for (const [variable, value] of Object.entries(scopes ?? {})) {
|
|
504
|
+
entries.push([row.partition, variable, value, row.row_id]);
|
|
505
|
+
}
|
|
506
|
+
update.run(...rewriteValues(table, row.partition, row.row_id, payload, 'sqlite', scopes));
|
|
485
507
|
}
|
|
508
|
+
if (entries.length > 0)
|
|
509
|
+
index.run(JSON.stringify(entries));
|
|
486
510
|
const last = rows[rows.length - 1];
|
|
487
511
|
if (last === undefined || rows.length < BATCH)
|
|
488
512
|
break;
|
package/dist/storage-errors.d.ts
CHANGED
|
@@ -9,7 +9,7 @@ export declare class StorageConstraintError extends Error {
|
|
|
9
9
|
constructor(cause: unknown, opIndex?: number);
|
|
10
10
|
}
|
|
11
11
|
/** Stable, privacy-safe failures for trusted server storage queries. */
|
|
12
|
-
export type StorageQueryErrorCode = 'sync.storage.schema_migration_pending' | 'sync.storage.schema_migration_conflict' | 'sync.storage.schema_changed' | 'sync.storage.invalid_migration_budget' | 'sync.storage.scan_requires_scope' | 'sync.storage.index_not_found' | 'sync.storage.index_not_materialized' | 'sync.storage.index_value_count_mismatch' | 'sync.storage.invalid_limit' | 'sync.storage.prune_epoch_mismatch' | 'sync.storage.partition_unregistered' | 'sync.storage.invalid_prune_cursor' | 'sync.storage.checkpoint_not_declared' | 'sync.storage.checkpoint_unsupported' | 'sync.storage.checkpoint_fence_missing' | 'sync.storage.checkpoint_incomplete' | 'sync.storage.stored_layout_mismatch' | 'sync.storage.physical_layout_mismatch' | 'sync.storage.transaction_query_unsupported' | 'sync.storage.partition_lock_unsupported' | 'sync.storage.query_over_staged_writes' | 'sync.storage.erase_unsupported' | 'sync.storage.invalid_erase_query';
|
|
12
|
+
export type StorageQueryErrorCode = 'sync.storage.schema_migration_pending' | 'sync.storage.schema_migration_conflict' | 'sync.storage.schema_changed' | 'sync.storage.invalid_migration_budget' | 'sync.storage.scan_requires_scope' | 'sync.storage.index_not_found' | 'sync.storage.index_not_materialized' | 'sync.storage.index_value_count_mismatch' | 'sync.storage.invalid_limit' | 'sync.storage.prune_epoch_mismatch' | 'sync.storage.partition_unregistered' | 'sync.storage.invalid_prune_cursor' | 'sync.storage.checkpoint_not_declared' | 'sync.storage.checkpoint_unsupported' | 'sync.storage.checkpoint_fence_missing' | 'sync.storage.checkpoint_incomplete' | 'sync.storage.stored_layout_mismatch' | 'sync.storage.scope_patterns_changed' | 'sync.storage.scope_value_missing' | 'sync.storage.physical_layout_mismatch' | 'sync.storage.transaction_query_unsupported' | 'sync.storage.partition_lock_unsupported' | 'sync.storage.query_over_staged_writes' | 'sync.storage.erase_unsupported' | 'sync.storage.invalid_erase_query';
|
|
13
13
|
/**
|
|
14
14
|
* Host-only query error. Messages never include identifiers, values, SQL,
|
|
15
15
|
* paths, or row data; callers branch on `code`, never message text. The
|
package/dist/storage-errors.js
CHANGED
|
@@ -24,6 +24,8 @@ const STORAGE_QUERY_MESSAGES = {
|
|
|
24
24
|
'sync.storage.checkpoint_fence_missing': 'checkpoint activation requires the writer fence raised at declaration',
|
|
25
25
|
'sync.storage.checkpoint_incomplete': 'a declared backfill checkpoint is not activated and this caller did not declare it',
|
|
26
26
|
'sync.storage.stored_layout_mismatch': 'stored schema layouts disagree with the configured schema at the same version',
|
|
27
|
+
'sync.storage.scope_patterns_changed': 'a table declares scope patterns other than the ones its stored rows are indexed under',
|
|
28
|
+
'sync.storage.scope_value_missing': 'a stored row has no value for a scope column the schema bump re-keys it under',
|
|
27
29
|
'sync.storage.physical_layout_mismatch': 'a synced table does not match the storage layout the running code reads and writes',
|
|
28
30
|
'sync.storage.transaction_query_unsupported': 'this storage transaction cannot run registered queries',
|
|
29
31
|
'sync.storage.partition_lock_unsupported': 'this storage transaction cannot serialize partition authority',
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@syncular/server",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.64.0",
|
|
4
4
|
"description": "Syncular server: handleSyncRequest + storage/auth interfaces for the sync protocol",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"author": "Benjamin Kniffler",
|
|
@@ -68,7 +68,7 @@
|
|
|
68
68
|
"verify:node": "cd ../.. && bun build ./packages/server/test/sqlite-runtime/verify-node.mjs --target=node --conditions=bun --outfile=./packages/server/.verify-node.built.mjs && node ./packages/server/.verify-node.built.mjs"
|
|
69
69
|
},
|
|
70
70
|
"dependencies": {
|
|
71
|
-
"@syncular/core": "0.
|
|
71
|
+
"@syncular/core": "0.64.0"
|
|
72
72
|
},
|
|
73
73
|
"devDependencies": {
|
|
74
74
|
"@electric-sql/pglite": "^0.5.4"
|
package/src/d1-storage.ts
CHANGED
|
@@ -44,12 +44,17 @@ import {
|
|
|
44
44
|
assertAppendOnlyMigration,
|
|
45
45
|
assertPhysicalColumns,
|
|
46
46
|
assertStoredLayouts,
|
|
47
|
+
assertStoredScopePatterns,
|
|
48
|
+
changedScopePatterns,
|
|
47
49
|
dropTableDdl,
|
|
48
50
|
indexRowPageStatement,
|
|
49
51
|
layoutsOf,
|
|
50
52
|
migratePayload,
|
|
51
53
|
parseLayouts,
|
|
52
54
|
quoteIdent,
|
|
55
|
+
rekeyCleanupSql,
|
|
56
|
+
rekeyedScopes,
|
|
57
|
+
rekeyRowScopesSql,
|
|
53
58
|
retiredTableNames,
|
|
54
59
|
rewritePlan,
|
|
55
60
|
rewriteRowSql,
|
|
@@ -835,7 +840,11 @@ interface D1MigrationState {
|
|
|
835
840
|
columns: Record<string, string[]>;
|
|
836
841
|
indexes: Record<string, Record<string, string>>;
|
|
837
842
|
ddl: string[];
|
|
838
|
-
rewrites: Array<{
|
|
843
|
+
rewrites: Array<{
|
|
844
|
+
table: string;
|
|
845
|
+
oldLayout?: readonly StoredColumnLayout[];
|
|
846
|
+
rekey?: true;
|
|
847
|
+
}>;
|
|
839
848
|
afterPartition: string;
|
|
840
849
|
afterRowId: string;
|
|
841
850
|
}
|
|
@@ -1052,6 +1061,17 @@ export class D1ServerStorage
|
|
|
1052
1061
|
);
|
|
1053
1062
|
if (claim === null && marker?.schema_version === schema.version) {
|
|
1054
1063
|
assertStoredLayouts(schema, marker.layouts);
|
|
1064
|
+
if (!assertStoredScopePatterns(schema, marker.layouts)) {
|
|
1065
|
+
// A marker from before 0.63 records no scope patterns: record the
|
|
1066
|
+
// configured ones, which every later start compares against.
|
|
1067
|
+
statementsExecuted++;
|
|
1068
|
+
await this.#db
|
|
1069
|
+
.prepare(
|
|
1070
|
+
'UPDATE sync_schema_meta SET layouts=? WHERE id=1 AND schema_version=?',
|
|
1071
|
+
)
|
|
1072
|
+
.bind(layoutsOf(schema), schema.version)
|
|
1073
|
+
.run();
|
|
1074
|
+
}
|
|
1055
1075
|
statementsExecuted++;
|
|
1056
1076
|
const present = await this.#db
|
|
1057
1077
|
.prepare(
|
|
@@ -1167,6 +1187,9 @@ export class D1ServerStorage
|
|
|
1167
1187
|
throw new StorageQueryError('sync.storage.schema_migration_conflict');
|
|
1168
1188
|
}
|
|
1169
1189
|
const sourceLayouts = parseLayouts(claim.source_layouts);
|
|
1190
|
+
const rekeyed = (
|
|
1191
|
+
changedScopePatterns(schema, claim.source_layouts) ?? []
|
|
1192
|
+
).map((change) => change.table);
|
|
1170
1193
|
let state = JSON.parse(claim.state) as D1MigrationState;
|
|
1171
1194
|
// Reserve one statement to distinguish a stale checkpoint from a database
|
|
1172
1195
|
// failure. A losing concurrent step returns incomplete without retrying here.
|
|
@@ -1239,7 +1262,60 @@ export class D1ServerStorage
|
|
|
1239
1262
|
} else if (state.phase === 'prepare') {
|
|
1240
1263
|
const table = tables[state.offset];
|
|
1241
1264
|
if (table !== undefined) {
|
|
1242
|
-
|
|
1265
|
+
// §3.1: every row of a re-keyed table needs a value for each new
|
|
1266
|
+
// pattern. Check before any DDL (one more statement); a refusal
|
|
1267
|
+
// releases the claim, so the refused bump changes nothing.
|
|
1268
|
+
const oldLayout = sourceLayouts[table.name];
|
|
1269
|
+
const verify =
|
|
1270
|
+
rekeyed.includes(table.name) && oldLayout !== undefined;
|
|
1271
|
+
if (remaining < (verify ? 5 : 4)) break;
|
|
1272
|
+
if (verify) {
|
|
1273
|
+
statementsExecuted++;
|
|
1274
|
+
const { results } = await this.#db
|
|
1275
|
+
.prepare(`SELECT * FROM (${selectRowsForRewriteSql(table, 'sqlite')})
|
|
1276
|
+
WHERE EXISTS (SELECT 1 FROM sync_schema_migration WHERE id=1 AND target=? AND state=?)`)
|
|
1277
|
+
.bind(
|
|
1278
|
+
state.afterPartition,
|
|
1279
|
+
state.afterRowId,
|
|
1280
|
+
500,
|
|
1281
|
+
target,
|
|
1282
|
+
claim.state,
|
|
1283
|
+
)
|
|
1284
|
+
.all<{ partition: string; row_id: string; payload: unknown }>();
|
|
1285
|
+
try {
|
|
1286
|
+
for (const row of results) {
|
|
1287
|
+
rekeyedScopes(
|
|
1288
|
+
table,
|
|
1289
|
+
decodeRow(oldLayout, asUint8Array(row.payload)),
|
|
1290
|
+
row.row_id,
|
|
1291
|
+
);
|
|
1292
|
+
}
|
|
1293
|
+
} catch (error) {
|
|
1294
|
+
await batch([
|
|
1295
|
+
this.#db
|
|
1296
|
+
.prepare(
|
|
1297
|
+
'DELETE FROM sync_schema_migration WHERE id=1 AND target=? AND state=?',
|
|
1298
|
+
)
|
|
1299
|
+
.bind(target, claim.state),
|
|
1300
|
+
]);
|
|
1301
|
+
throw error;
|
|
1302
|
+
}
|
|
1303
|
+
const last = results.at(-1);
|
|
1304
|
+
if (last !== undefined && results.length === 500) {
|
|
1305
|
+
if (
|
|
1306
|
+
!(await save(
|
|
1307
|
+
{
|
|
1308
|
+
...state,
|
|
1309
|
+
afterPartition: last.partition,
|
|
1310
|
+
afterRowId: last.row_id,
|
|
1311
|
+
},
|
|
1312
|
+
[],
|
|
1313
|
+
))
|
|
1314
|
+
)
|
|
1315
|
+
break;
|
|
1316
|
+
continue;
|
|
1317
|
+
}
|
|
1318
|
+
}
|
|
1243
1319
|
statementsExecuted += 2;
|
|
1244
1320
|
const columns = await this.#db
|
|
1245
1321
|
.prepare(`PRAGMA table_info(${quoteIdent(table.name)})`)
|
|
@@ -1261,6 +1337,8 @@ export class D1ServerStorage
|
|
|
1261
1337
|
{
|
|
1262
1338
|
...state,
|
|
1263
1339
|
offset: state.offset + 1,
|
|
1340
|
+
afterPartition: '',
|
|
1341
|
+
afterRowId: '',
|
|
1264
1342
|
columns: {
|
|
1265
1343
|
...state.columns,
|
|
1266
1344
|
...(columns.results.length > 0
|
|
@@ -1300,13 +1378,15 @@ export class D1ServerStorage
|
|
|
1300
1378
|
sourceLayouts[table.name],
|
|
1301
1379
|
columns.get(table.name),
|
|
1302
1380
|
);
|
|
1303
|
-
|
|
1381
|
+
const rekey = rekeyed.includes(table.name);
|
|
1382
|
+
return plan.migrate || plan.backfill || rekey
|
|
1304
1383
|
? [
|
|
1305
1384
|
{
|
|
1306
1385
|
table: table.name,
|
|
1307
1386
|
...(plan.migrate
|
|
1308
1387
|
? { oldLayout: sourceLayouts[table.name] }
|
|
1309
1388
|
: {}),
|
|
1389
|
+
...(rekey ? { rekey: true as const } : {}),
|
|
1310
1390
|
},
|
|
1311
1391
|
]
|
|
1312
1392
|
: [];
|
|
@@ -1318,6 +1398,7 @@ export class D1ServerStorage
|
|
|
1318
1398
|
dropTableDdl(name),
|
|
1319
1399
|
]),
|
|
1320
1400
|
...schemaDdl(schema, columns, 'sqlite', indexes),
|
|
1401
|
+
...rekeyCleanupSql(rekeyed),
|
|
1321
1402
|
];
|
|
1322
1403
|
if (
|
|
1323
1404
|
!(await save(
|
|
@@ -1376,8 +1457,10 @@ export class D1ServerStorage
|
|
|
1376
1457
|
const table = schema.tables.get(plan.table);
|
|
1377
1458
|
if (table === undefined)
|
|
1378
1459
|
throw new StorageQueryError('sync.storage.schema_migration_conflict');
|
|
1379
|
-
|
|
1380
|
-
const
|
|
1460
|
+
// A re-keyed page adds one scope-index statement.
|
|
1461
|
+
const rekey = plan.rekey === true;
|
|
1462
|
+
if (remaining < (rekey ? 5 : 4)) break;
|
|
1463
|
+
const limit = Math.min(32, remaining - (rekey ? 4 : 3));
|
|
1381
1464
|
statementsExecuted++;
|
|
1382
1465
|
const { results } = await this.#db
|
|
1383
1466
|
.prepare(`SELECT * FROM (${selectRowsForRewriteSql(table, 'sqlite')})
|
|
@@ -1390,14 +1473,25 @@ export class D1ServerStorage
|
|
|
1390
1473
|
claim.state,
|
|
1391
1474
|
)
|
|
1392
1475
|
.all<{ partition: string; row_id: string; payload: unknown }>();
|
|
1476
|
+
const entries: string[][] = [];
|
|
1393
1477
|
const statements = results.map((row) => {
|
|
1394
1478
|
const bytes = asUint8Array(row.payload);
|
|
1395
1479
|
const payload =
|
|
1396
1480
|
plan.oldLayout !== undefined
|
|
1397
1481
|
? migratePayload(plan.oldLayout, table, bytes)
|
|
1398
1482
|
: bytes;
|
|
1483
|
+
const scopes = rekey
|
|
1484
|
+
? rekeyedScopes(
|
|
1485
|
+
table,
|
|
1486
|
+
decodeRow(table.columns, payload),
|
|
1487
|
+
row.row_id,
|
|
1488
|
+
)
|
|
1489
|
+
: undefined;
|
|
1490
|
+
for (const [variable, value] of Object.entries(scopes ?? {})) {
|
|
1491
|
+
entries.push([row.partition, variable, value, row.row_id]);
|
|
1492
|
+
}
|
|
1399
1493
|
return this.#db
|
|
1400
|
-
.prepare(rewriteRowSql(table))
|
|
1494
|
+
.prepare(rewriteRowSql(table, rekey))
|
|
1401
1495
|
.bind(
|
|
1402
1496
|
...rewriteValues(
|
|
1403
1497
|
table,
|
|
@@ -1405,9 +1499,17 @@ export class D1ServerStorage
|
|
|
1405
1499
|
row.row_id,
|
|
1406
1500
|
payload,
|
|
1407
1501
|
'sqlite',
|
|
1502
|
+
scopes,
|
|
1408
1503
|
),
|
|
1409
1504
|
);
|
|
1410
1505
|
});
|
|
1506
|
+
if (entries.length > 0) {
|
|
1507
|
+
statements.push(
|
|
1508
|
+
this.#db
|
|
1509
|
+
.prepare(rekeyRowScopesSql(table, 'sqlite'))
|
|
1510
|
+
.bind(JSON.stringify(entries)),
|
|
1511
|
+
);
|
|
1512
|
+
}
|
|
1411
1513
|
const last = results.at(-1);
|
|
1412
1514
|
const done = results.length < limit;
|
|
1413
1515
|
if (
|
package/src/postgres-storage.ts
CHANGED
|
@@ -46,7 +46,11 @@ import { serveNotReadyError } from './readiness';
|
|
|
46
46
|
* would leave gaps on rollback, which the pull-window arithmetic in §4.5
|
|
47
47
|
* does not tolerate). Cross-partition pushes never contend.
|
|
48
48
|
*/
|
|
49
|
-
import
|
|
49
|
+
import {
|
|
50
|
+
decodeRow,
|
|
51
|
+
type PushOperationResult,
|
|
52
|
+
type UnitScopes,
|
|
53
|
+
} from '@syncular/core';
|
|
50
54
|
import {
|
|
51
55
|
bindAuthoritativePartition,
|
|
52
56
|
type BoundAuthoritativeQuery,
|
|
@@ -64,6 +68,8 @@ import {
|
|
|
64
68
|
import {
|
|
65
69
|
assertPhysicalColumns,
|
|
66
70
|
assertStoredLayouts,
|
|
71
|
+
assertStoredScopePatterns,
|
|
72
|
+
changedScopePatterns,
|
|
67
73
|
deleteRowSql,
|
|
68
74
|
dropTableDdl,
|
|
69
75
|
indexRowPageStatement,
|
|
@@ -71,6 +77,9 @@ import {
|
|
|
71
77
|
migratePayload,
|
|
72
78
|
parseLayouts,
|
|
73
79
|
quoteIdent,
|
|
80
|
+
rekeyCleanupSql,
|
|
81
|
+
rekeyedScopes,
|
|
82
|
+
rekeyRowScopesSql,
|
|
74
83
|
tableColumnNames,
|
|
75
84
|
retiredTableNames,
|
|
76
85
|
rewritePlan,
|
|
@@ -932,34 +941,40 @@ function toStoredChange(record: ChangeRecord): StoredChange {
|
|
|
932
941
|
* transaction. When `oldLayout` is
|
|
933
942
|
* given every payload re-encodes under the current columns, and the
|
|
934
943
|
* projection (when materialized) refreshes from the payload either way.
|
|
944
|
+
* `rekey` recomputes and indexes the stored scopes (§3.1).
|
|
935
945
|
*/
|
|
936
946
|
async function rewriteRowsOn(
|
|
937
947
|
q: PgQueryable,
|
|
938
948
|
table: CompiledTable,
|
|
939
949
|
oldLayout: readonly StoredColumnLayout[] | undefined,
|
|
950
|
+
rekey: boolean,
|
|
940
951
|
): Promise<void> {
|
|
941
952
|
const select = selectRowsForRewriteSql(table, 'postgres');
|
|
942
953
|
// One UPDATE per page: behind a network hop (Hyperdrive, a managed
|
|
943
954
|
// database) every statement is a round trip, and the bump holds the
|
|
944
955
|
// migration lock for its whole duration. The page's rows bind as one
|
|
945
956
|
// VALUES list with typed placeholders (`rewriteValues` order: projection
|
|
946
|
-
// columns, payload, partition, row id).
|
|
957
|
+
// columns, re-keyed scopes, payload, partition, row id). The scopes bind as
|
|
958
|
+
// text before the JSONB cast (see `scanRowBatchSql`).
|
|
959
|
+
const projected = table.materialize ? table.columns : [];
|
|
947
960
|
const types = [
|
|
948
|
-
...(
|
|
949
|
-
|
|
950
|
-
: []),
|
|
961
|
+
...projected.map((column) => columnSqlType(column, 'postgres')),
|
|
962
|
+
...(rekey ? ['TEXT'] : []),
|
|
951
963
|
'BYTEA',
|
|
952
964
|
'TEXT',
|
|
953
965
|
'TEXT',
|
|
954
966
|
];
|
|
955
|
-
const
|
|
967
|
+
const payloadAt = projected.length + (rekey ? 1 : 0);
|
|
956
968
|
const sets = [
|
|
957
969
|
...projected.map((column, i) => `${quoteIdent(column.name)}=v.c${i}`),
|
|
958
|
-
|
|
970
|
+
...(rekey
|
|
971
|
+
? [`${quoteIdent(SYNC_SCOPES_COLUMN)}=v.c${projected.length}::jsonb`]
|
|
972
|
+
: []),
|
|
973
|
+
`${quoteIdent(SYNC_PAYLOAD_COLUMN)}=v.c${payloadAt}`,
|
|
959
974
|
].join(', ');
|
|
960
975
|
const aliases = types.map((_, i) => `c${i}`).join(', ');
|
|
961
|
-
const partitionAlias = `c${
|
|
962
|
-
const rowIdAlias = `c${
|
|
976
|
+
const partitionAlias = `c${payloadAt + 1}`;
|
|
977
|
+
const rowIdAlias = `c${payloadAt + 2}`;
|
|
963
978
|
// A page binds types.length parameters per row; PGlite corrupts its state
|
|
964
979
|
// above 32,767 bound parameters (see PG_CHANGES_PER_STATEMENT).
|
|
965
980
|
const batch = Math.max(1, Math.min(500, Math.floor(30_000 / types.length)));
|
|
@@ -973,15 +988,29 @@ async function rewriteRowsOn(
|
|
|
973
988
|
}>(select, [afterPartition, afterRowId, batch]);
|
|
974
989
|
if (rows.length === 0) break;
|
|
975
990
|
const params: unknown[] = [];
|
|
991
|
+
const entries: string[][] = [];
|
|
976
992
|
const values = rows.map((row) => {
|
|
977
993
|
const bytes = asBytes(row.payload);
|
|
978
994
|
const payload =
|
|
979
995
|
oldLayout !== undefined
|
|
980
996
|
? migratePayload(oldLayout, table, bytes)
|
|
981
997
|
: bytes;
|
|
998
|
+
const scopes = rekey
|
|
999
|
+
? rekeyedScopes(table, decodeRow(table.columns, payload), row.row_id)
|
|
1000
|
+
: undefined;
|
|
1001
|
+
for (const [variable, value] of Object.entries(scopes ?? {})) {
|
|
1002
|
+
entries.push([row.partition, variable, value, row.row_id]);
|
|
1003
|
+
}
|
|
982
1004
|
const base = params.length;
|
|
983
1005
|
params.push(
|
|
984
|
-
...rewriteValues(
|
|
1006
|
+
...rewriteValues(
|
|
1007
|
+
table,
|
|
1008
|
+
row.partition,
|
|
1009
|
+
row.row_id,
|
|
1010
|
+
payload,
|
|
1011
|
+
'postgres',
|
|
1012
|
+
scopes,
|
|
1013
|
+
),
|
|
985
1014
|
);
|
|
986
1015
|
return `(${types.map((type, i) => `$${base + i + 1}::${type}`).join(',')})`;
|
|
987
1016
|
});
|
|
@@ -992,6 +1021,11 @@ async function rewriteRowsOn(
|
|
|
992
1021
|
AND t.${quoteIdent(SYNC_ROW_ID_COLUMN)}=v.${rowIdAlias}`,
|
|
993
1022
|
params,
|
|
994
1023
|
);
|
|
1024
|
+
if (entries.length > 0) {
|
|
1025
|
+
await q.query(rekeyRowScopesSql(table, 'postgres'), [
|
|
1026
|
+
JSON.stringify(entries),
|
|
1027
|
+
]);
|
|
1028
|
+
}
|
|
995
1029
|
const last = rows[rows.length - 1];
|
|
996
1030
|
if (last === undefined || rows.length < batch) break;
|
|
997
1031
|
afterPartition = last.partition;
|
|
@@ -2361,10 +2395,17 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2361
2395
|
);
|
|
2362
2396
|
}
|
|
2363
2397
|
if (stored !== undefined && stored === schema.version) {
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
)
|
|
2398
|
+
const storedLayouts =
|
|
2399
|
+
typeof marker.layouts === 'string' ? marker.layouts : undefined;
|
|
2400
|
+
assertStoredLayouts(schema, storedLayouts);
|
|
2401
|
+
if (!assertStoredScopePatterns(schema, storedLayouts)) {
|
|
2402
|
+
// A marker from before 0.63 records no scope patterns: record the
|
|
2403
|
+
// configured ones, which every later start compares against.
|
|
2404
|
+
await this.#exec.query(
|
|
2405
|
+
'UPDATE sync_schema_meta SET layouts=$1 WHERE id=1 AND schema_version=$2',
|
|
2406
|
+
[layoutsOf(schema), schema.version],
|
|
2407
|
+
);
|
|
2408
|
+
}
|
|
2368
2409
|
const columns = JSON.parse(marker.columns) as {
|
|
2369
2410
|
tbl: string;
|
|
2370
2411
|
name: string;
|
|
@@ -2444,11 +2485,14 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2444
2485
|
);
|
|
2445
2486
|
}
|
|
2446
2487
|
if (lockedVersion === schema.version) return;
|
|
2447
|
-
const
|
|
2488
|
+
const lockedLayouts =
|
|
2448
2489
|
typeof lockedMarker[0]?.layouts === 'string'
|
|
2449
2490
|
? lockedMarker[0].layouts
|
|
2450
|
-
: undefined
|
|
2491
|
+
: undefined;
|
|
2492
|
+
const rekeyed = (changedScopePatterns(schema, lockedLayouts) ?? []).map(
|
|
2493
|
+
(change) => change.table,
|
|
2451
2494
|
);
|
|
2495
|
+
const layouts = parseLayouts(lockedLayouts);
|
|
2452
2496
|
const retiredTables = retiredTableNames(schema, layouts);
|
|
2453
2497
|
const existing = new Map<string, ReadonlySet<string>>();
|
|
2454
2498
|
const existingIndexes = new Map<string, ReadonlyMap<string, string>>();
|
|
@@ -2494,14 +2538,19 @@ FOR EACH ROW EXECUTE FUNCTION syncular_writer_fence()`,
|
|
|
2494
2538
|
)) {
|
|
2495
2539
|
await client.query(statement);
|
|
2496
2540
|
}
|
|
2541
|
+
for (const statement of rekeyCleanupSql(rekeyed)) {
|
|
2542
|
+
await client.query(statement);
|
|
2543
|
+
}
|
|
2497
2544
|
for (const table of schema.tables.values()) {
|
|
2498
2545
|
const oldLayout = layouts[table.name];
|
|
2499
2546
|
const plan = rewritePlan(table, oldLayout, existing.get(table.name));
|
|
2500
|
-
|
|
2547
|
+
const rekey = rekeyed.includes(table.name);
|
|
2548
|
+
if (!plan.migrate && !plan.backfill && !rekey) continue;
|
|
2501
2549
|
await rewriteRowsOn(
|
|
2502
2550
|
client,
|
|
2503
2551
|
table,
|
|
2504
2552
|
plan.migrate ? oldLayout : undefined,
|
|
2553
|
+
rekey,
|
|
2505
2554
|
);
|
|
2506
2555
|
}
|
|
2507
2556
|
await client.query(
|