@spooky-sync/core 0.0.1-canary.322 → 0.0.1-canary.323
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +4 -0
- package/dist/index.js +117 -3
- package/dist/types.d.ts +10 -0
- package/package.json +3 -3
- package/src/client/services.ts +2 -0
- package/src/services/database/cache-engine.ts +10 -0
- package/src/services/database/sqlite-cache-engine.ts +26 -0
- package/src/services/database/sqlite-column-reset.integration.test.ts +100 -0
- package/src/services/database/sqlite-column-reset.ts +91 -0
- package/src/services/stream-processor/index.ts +24 -0
- package/src/services/stream-processor/stream-processor.prime.test.ts +36 -0
package/dist/index.d.ts
CHANGED
|
@@ -303,6 +303,10 @@ interface CircuitPrimeContext {
|
|
|
303
303
|
/** Encoded ids with an unsettled local mutation (their `_00_rv` was bumped
|
|
304
304
|
* locally and may exceed the server's next version). */
|
|
305
305
|
pendingIds: Set<string>;
|
|
306
|
+
/** Each table's client column set (`columnFingerprints`). An engine that
|
|
307
|
+
* keeps bodies across schema changes resets the versions of a table whose
|
|
308
|
+
* set changed before they are read, so its rows are fetched again whole. */
|
|
309
|
+
columnFingerprints?: Record<string, string>;
|
|
306
310
|
/** Receives every `(id, rv)` the prime put into the circuit, per table, so
|
|
307
311
|
* the sync layer can skip re-downloading bodies it already has. */
|
|
308
312
|
onVersions?: (table: string, entries: [string, number][]) => void;
|
package/dist/index.js
CHANGED
|
@@ -2188,8 +2188,8 @@ function trimHistory(history, cap) {
|
|
|
2188
2188
|
|
|
2189
2189
|
//#endregion
|
|
2190
2190
|
//#region src/modules/devtools/index.ts
|
|
2191
|
-
const CORE_VERSION = "0.0.1-canary.
|
|
2192
|
-
const WASM_VERSION = "0.0.1-canary.
|
|
2191
|
+
const CORE_VERSION = "0.0.1-canary.323";
|
|
2192
|
+
const WASM_VERSION = "0.0.1-canary.323";
|
|
2193
2193
|
const SURREAL_VERSION = "3.0.3";
|
|
2194
2194
|
var DevToolsService = class DevToolsService {
|
|
2195
2195
|
version = CORE_VERSION;
|
|
@@ -9627,6 +9627,84 @@ var PortSqliteTransport = class extends BaseTransport {
|
|
|
9627
9627
|
}
|
|
9628
9628
|
};
|
|
9629
9629
|
|
|
9630
|
+
//#endregion
|
|
9631
|
+
//#region src/services/database/sqlite-column-reset.ts
|
|
9632
|
+
/**
|
|
9633
|
+
* Cached bodies outlive a schema change on the SQLite engine, and that loses
|
|
9634
|
+
* columns for good.
|
|
9635
|
+
*
|
|
9636
|
+
* A body is landed through `cleanRecord`, which keeps only the columns the
|
|
9637
|
+
* running app's schema declares, and is stored at the server's `_00_rv`. The
|
|
9638
|
+
* engine is schemaless (no migrator, nothing is recreated when the schema
|
|
9639
|
+
* changes), so a tab on an older build that fetched a row after the server
|
|
9640
|
+
* added and backfilled a column stored it WITHOUT that column at the new
|
|
9641
|
+
* version. The next build knows the column, but its local version equals the
|
|
9642
|
+
* server's, so the fetch plan never asks for the row again and the column
|
|
9643
|
+
* stays missing until something else bumps the row. (WhitePawn, 2026-10-11:
|
|
9644
|
+
* every puzzle read as unconverted because `layout` never arrived.)
|
|
9645
|
+
*
|
|
9646
|
+
* So the store remembers each table's column set (`_00_schema`, row
|
|
9647
|
+
* `columns`). On boot, before the prime reads the versions, a table whose set
|
|
9648
|
+
* changed, or every table when no set was recorded yet, gets its cached
|
|
9649
|
+
* `_00_rv` reset to 0. The bodies stay and keep rendering; any view that names
|
|
9650
|
+
* one of those rows sees it as stale and fetches it again, whole. Rows with an
|
|
9651
|
+
* unsettled local write keep their version: their echo is a newer body anyway.
|
|
9652
|
+
*/
|
|
9653
|
+
/** `_00_schema` row holding the recorded column sets. */
|
|
9654
|
+
const COLUMNS_ROW_ID = "columns";
|
|
9655
|
+
/**
|
|
9656
|
+
* The column set of every table, as `cleanRecord` sees it: the sorted declared
|
|
9657
|
+
* column names. Plain names rather than a hash, so a stored set says what it is.
|
|
9658
|
+
*/
|
|
9659
|
+
function columnFingerprints(tables) {
|
|
9660
|
+
const out = {};
|
|
9661
|
+
for (const t of tables) out[t.name] = Object.keys(t.columns).sort().join(",");
|
|
9662
|
+
return out;
|
|
9663
|
+
}
|
|
9664
|
+
const sameSets = (a, b) => {
|
|
9665
|
+
if (!a) return false;
|
|
9666
|
+
const ka = Object.keys(a);
|
|
9667
|
+
return ka.length === Object.keys(b).length && ka.every((k) => a[k] === b[k]);
|
|
9668
|
+
};
|
|
9669
|
+
/**
|
|
9670
|
+
* What to run for the column sets `current`, given the `stored` ones (null when
|
|
9671
|
+
* none were recorded), the tables that exist in the store, and the encoded ids
|
|
9672
|
+
* (`table:id`) with an unsettled local write.
|
|
9673
|
+
*/
|
|
9674
|
+
function planColumnReset(stored, current, existingTables, keep) {
|
|
9675
|
+
if (sameSets(stored, current)) return {
|
|
9676
|
+
changed: [],
|
|
9677
|
+
statements: []
|
|
9678
|
+
};
|
|
9679
|
+
const keepByTable = /* @__PURE__ */ new Map();
|
|
9680
|
+
for (const id of keep) {
|
|
9681
|
+
const table = id.slice(0, id.indexOf(":"));
|
|
9682
|
+
if (!keepByTable.has(table)) keepByTable.set(table, []);
|
|
9683
|
+
keepByTable.get(table).push(id);
|
|
9684
|
+
}
|
|
9685
|
+
const changed = [];
|
|
9686
|
+
const statements = [];
|
|
9687
|
+
for (const [table, set] of Object.entries(current)) {
|
|
9688
|
+
if (stored && stored[table] === set) continue;
|
|
9689
|
+
if (!existingTables.has(table)) continue;
|
|
9690
|
+
changed.push(table);
|
|
9691
|
+
const reset = `UPDATE ${`"${table.replace(/"/g, "\"\"")}"`} SET data = json_set(data, '$._00_rv', 0) WHERE json_extract(data, '$._00_rv') != 0`;
|
|
9692
|
+
const kept = keepByTable.get(table);
|
|
9693
|
+
statements.push(kept ? {
|
|
9694
|
+
sql: `${reset} AND id NOT IN (SELECT value FROM json_each(?))`,
|
|
9695
|
+
bind: [JSON.stringify(kept)]
|
|
9696
|
+
} : { sql: reset });
|
|
9697
|
+
}
|
|
9698
|
+
statements.push({
|
|
9699
|
+
sql: `INSERT OR REPLACE INTO "_00_schema"(id, data) VALUES (?, ?)`,
|
|
9700
|
+
bind: [COLUMNS_ROW_ID, JSON.stringify(current)]
|
|
9701
|
+
});
|
|
9702
|
+
return {
|
|
9703
|
+
changed,
|
|
9704
|
+
statements
|
|
9705
|
+
};
|
|
9706
|
+
}
|
|
9707
|
+
|
|
9630
9708
|
//#endregion
|
|
9631
9709
|
//#region src/services/database/sqlite-cache-engine.ts
|
|
9632
9710
|
/**
|
|
@@ -10259,6 +10337,28 @@ var SqliteCacheEngine = class {
|
|
|
10259
10337
|
}
|
|
10260
10338
|
return out;
|
|
10261
10339
|
}
|
|
10340
|
+
/**
|
|
10341
|
+
* Reset the cached `_00_rv` of every table whose column set changed since
|
|
10342
|
+
* the last boot, so its rows are fetched again whole. See
|
|
10343
|
+
* `sqlite-column-reset.ts`. Returns the tables reset.
|
|
10344
|
+
*/
|
|
10345
|
+
async resetChangedTables(fingerprints, keep) {
|
|
10346
|
+
const { rows } = await this.call("exec", {
|
|
10347
|
+
sql: `SELECT data FROM "_00_schema" WHERE id = ?`,
|
|
10348
|
+
bind: [COLUMNS_ROW_ID]
|
|
10349
|
+
});
|
|
10350
|
+
let stored = null;
|
|
10351
|
+
try {
|
|
10352
|
+
const raw = rows?.[0]?.data;
|
|
10353
|
+
if (typeof raw === "string") stored = JSON.parse(raw);
|
|
10354
|
+
} catch {
|
|
10355
|
+
stored = null;
|
|
10356
|
+
}
|
|
10357
|
+
const { rows: tables } = await this.call("exec", { sql: `SELECT name FROM sqlite_master WHERE type = 'table'` });
|
|
10358
|
+
const plan = planColumnReset(stored, fingerprints, new Set((tables ?? []).map((t) => t.name)), keep);
|
|
10359
|
+
if (plan.statements.length > 0) await this.call("batch", plan.statements);
|
|
10360
|
+
return plan.changed;
|
|
10361
|
+
}
|
|
10262
10362
|
async getSnapshot(key) {
|
|
10263
10363
|
const { rows } = await this.call("exec", {
|
|
10264
10364
|
sql: `SELECT id, data FROM "${SNAPSHOT_TABLE}" WHERE id IN (?, ?)`,
|
|
@@ -10955,6 +11055,19 @@ var StreamProcessorService = class {
|
|
|
10955
11055
|
const epoch = this.db.epoch;
|
|
10956
11056
|
const alive = () => this.processor === processor && generation === this.stateGeneration && epoch === this.db.epoch;
|
|
10957
11057
|
const t0 = performance.now();
|
|
11058
|
+
if (ctx.columnFingerprints && typeof this.db.resetChangedTables === "function") try {
|
|
11059
|
+
const reset = await this.db.resetChangedTables(ctx.columnFingerprints, ctx.pendingIds);
|
|
11060
|
+
if (!alive()) return;
|
|
11061
|
+
if (reset.length > 0) this.logger.info({
|
|
11062
|
+
tables: reset,
|
|
11063
|
+
Category: "sp00ky-client::StreamProcessorService::primeFromLocal"
|
|
11064
|
+
}, "Columns changed since these rows were cached; they will be fetched again");
|
|
11065
|
+
} catch (e) {
|
|
11066
|
+
this.logger.warn({
|
|
11067
|
+
error: e,
|
|
11068
|
+
Category: "sp00ky-client::StreamProcessorService::primeFromLocal"
|
|
11069
|
+
}, "Column change check failed; cached versions kept");
|
|
11070
|
+
}
|
|
10958
11071
|
let restored = false;
|
|
10959
11072
|
if (this.persistCircuit && typeof this.db.getSnapshot === "function" && typeof processor.load_store_state === "function") {
|
|
10960
11073
|
const snapshot = await this.db.getSnapshot(CIRCUIT_SNAPSHOT_KEY);
|
|
@@ -14865,7 +14978,7 @@ function createTabsCoordinator(config, s, host) {
|
|
|
14865
14978
|
return new TabsCoordinator({
|
|
14866
14979
|
tabId: s.tabId,
|
|
14867
14980
|
fingerprint: computeTabsFingerprint({
|
|
14868
|
-
coreVersion: "0.0.1-canary.
|
|
14981
|
+
coreVersion: "0.0.1-canary.323",
|
|
14869
14982
|
schemaHash: hash53(config.schemaSurql),
|
|
14870
14983
|
endpoint: config.database.endpoint ?? "",
|
|
14871
14984
|
namespace: config.database.namespace,
|
|
@@ -15005,6 +15118,7 @@ function createAdapters(config, s, host, lateModules) {
|
|
|
15005
15118
|
],
|
|
15006
15119
|
schemaHash: String(hash53(config.schemaSurql)),
|
|
15007
15120
|
pendingIds: new Set(pendingIds),
|
|
15121
|
+
columnFingerprints: columnFingerprints(config.schema.tables),
|
|
15008
15122
|
onVersions: (_table, entries) => host.dispatch({
|
|
15009
15123
|
type: "VersionsPrimed",
|
|
15010
15124
|
entries
|
package/dist/types.d.ts
CHANGED
|
@@ -451,6 +451,16 @@ interface LocalStore extends LocalCacheEngine {
|
|
|
451
451
|
* without it boots the circuit empty, as before.
|
|
452
452
|
*/
|
|
453
453
|
scanVersions?(tables: string[]): Promise<Record<string, [string, number][]>>;
|
|
454
|
+
/**
|
|
455
|
+
* Reset the cached `_00_rv` of every table whose column set (`fingerprints`,
|
|
456
|
+
* see `columnFingerprints`) changed since the store last recorded it, except
|
|
457
|
+
* the rows in `keep`, and record the new sets. Returns the tables reset.
|
|
458
|
+
* For engines that keep bodies across schema changes: a body landed under an
|
|
459
|
+
* older schema lacks the columns it did not know, at the server's version,
|
|
460
|
+
* and would never be fetched again. OPTIONAL: an engine that rebuilds its
|
|
461
|
+
* store on a schema change (SurrealDB) does not need it.
|
|
462
|
+
*/
|
|
463
|
+
resetChangedTables?(fingerprints: Record<string, string>, keep: Set<string>): Promise<string[]>;
|
|
454
464
|
/**
|
|
455
465
|
* Circuit snapshot storage, keyed. Lives in the same durable store as the
|
|
456
466
|
* rows (OPFS SQLite), so it is per-bucket by construction, atomic, and
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@spooky-sync/core",
|
|
3
|
-
"version": "0.0.1-canary.
|
|
3
|
+
"version": "0.0.1-canary.323",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"sideEffects": false,
|
|
6
6
|
"main": "./dist/index.js",
|
|
@@ -75,8 +75,8 @@
|
|
|
75
75
|
}
|
|
76
76
|
},
|
|
77
77
|
"dependencies": {
|
|
78
|
-
"@spooky-sync/query-builder": "0.0.1-canary.
|
|
79
|
-
"@spooky-sync/ssp-wasm": "0.0.1-canary.
|
|
78
|
+
"@spooky-sync/query-builder": "0.0.1-canary.323",
|
|
79
|
+
"@spooky-sync/ssp-wasm": "0.0.1-canary.323",
|
|
80
80
|
"@sqlite.org/sqlite-wasm": "3.53.0-build1",
|
|
81
81
|
"@surrealdb/wasm": "^3.0.3",
|
|
82
82
|
"blurhash": "^2.0.5",
|
package/src/client/services.ts
CHANGED
|
@@ -21,6 +21,7 @@ import { TabsCoordinator, type CoordinatorHooks } from '../services/tabs/coordin
|
|
|
21
21
|
import type { LeaderSyncHub, SyncForwarder } from '../services/tabs/coordinator';
|
|
22
22
|
import { computeTabsFingerprint, hash53 } from '../services/tabs/protocol';
|
|
23
23
|
import type { SqliteCacheEngine } from '../services/database/sqlite-cache-engine';
|
|
24
|
+
import { columnFingerprints } from '../services/database/sqlite-column-reset';
|
|
24
25
|
import type { BlobCache } from '../services/blobs/index';
|
|
25
26
|
import { MemoryBlobStore, createBlobCache, resolveBlobBudget } from '../services/blobs/index';
|
|
26
27
|
import type { Adapters } from '../kernel/interpreter';
|
|
@@ -308,6 +309,7 @@ export function createAdapters<S extends SchemaStructure>(config: Sp00kyConfig<S
|
|
|
308
309
|
tables: [...config.schema.tables.map((t) => t.name), '_00_user_feature', '_00_app_release'],
|
|
309
310
|
schemaHash: String(hash53(config.schemaSurql)),
|
|
310
311
|
pendingIds: new Set(pendingIds),
|
|
312
|
+
columnFingerprints: columnFingerprints(config.schema.tables),
|
|
311
313
|
onVersions: (_table, entries) => host.dispatch({ type: 'VersionsPrimed', entries }),
|
|
312
314
|
}),
|
|
313
315
|
'ssp.reset': () => s.streamProcessor.reset(),
|
|
@@ -151,6 +151,16 @@ export interface LocalStore extends LocalCacheEngine {
|
|
|
151
151
|
* without it boots the circuit empty, as before.
|
|
152
152
|
*/
|
|
153
153
|
scanVersions?(tables: string[]): Promise<Record<string, [string, number][]>>;
|
|
154
|
+
/**
|
|
155
|
+
* Reset the cached `_00_rv` of every table whose column set (`fingerprints`,
|
|
156
|
+
* see `columnFingerprints`) changed since the store last recorded it, except
|
|
157
|
+
* the rows in `keep`, and record the new sets. Returns the tables reset.
|
|
158
|
+
* For engines that keep bodies across schema changes: a body landed under an
|
|
159
|
+
* older schema lacks the columns it did not know, at the server's version,
|
|
160
|
+
* and would never be fetched again. OPTIONAL: an engine that rebuilds its
|
|
161
|
+
* store on a schema change (SurrealDB) does not need it.
|
|
162
|
+
*/
|
|
163
|
+
resetChangedTables?(fingerprints: Record<string, string>, keep: Set<string>): Promise<string[]>;
|
|
154
164
|
/**
|
|
155
165
|
* Circuit snapshot storage, keyed. Lives in the same durable store as the
|
|
156
166
|
* rows (OPFS SQLite), so it is per-bucket by construction, atomic, and
|
|
@@ -30,6 +30,7 @@ import {
|
|
|
30
30
|
type SqliteTransport,
|
|
31
31
|
} from './sqlite-transport';
|
|
32
32
|
import { PROMOTION_OPEN_OPTIONS } from './sqlite-open';
|
|
33
|
+
import { COLUMNS_ROW_ID, planColumnReset } from './sqlite-column-reset';
|
|
33
34
|
import type { EngineTx, Id, LocalStore, OrderBy, RelationFetch, Row } from './cache-engine';
|
|
34
35
|
import type { EngineStorageDiagnostics } from '../../modules/devtools/storage-info';
|
|
35
36
|
|
|
@@ -898,6 +899,31 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
898
899
|
return out;
|
|
899
900
|
}
|
|
900
901
|
|
|
902
|
+
/**
|
|
903
|
+
* Reset the cached `_00_rv` of every table whose column set changed since
|
|
904
|
+
* the last boot, so its rows are fetched again whole. See
|
|
905
|
+
* `sqlite-column-reset.ts`. Returns the tables reset.
|
|
906
|
+
*/
|
|
907
|
+
async resetChangedTables(fingerprints: Record<string, string>, keep: Set<string>): Promise<string[]> {
|
|
908
|
+
const { rows } = await this.call<{ rows: { data: unknown }[] }>('exec', {
|
|
909
|
+
sql: `SELECT data FROM "_00_schema" WHERE id = ?`,
|
|
910
|
+
bind: [COLUMNS_ROW_ID],
|
|
911
|
+
});
|
|
912
|
+
let stored: Record<string, string> | null = null;
|
|
913
|
+
try {
|
|
914
|
+
const raw = rows?.[0]?.data;
|
|
915
|
+
if (typeof raw === 'string') stored = JSON.parse(raw) as Record<string, string>;
|
|
916
|
+
} catch {
|
|
917
|
+
stored = null;
|
|
918
|
+
}
|
|
919
|
+
const { rows: tables } = await this.call<{ rows: { name: string }[] }>('exec', {
|
|
920
|
+
sql: `SELECT name FROM sqlite_master WHERE type = 'table'`,
|
|
921
|
+
});
|
|
922
|
+
const plan = planColumnReset(stored, fingerprints, new Set((tables ?? []).map((t) => t.name)), keep);
|
|
923
|
+
if (plan.statements.length > 0) await this.call('batch', plan.statements);
|
|
924
|
+
return plan.changed;
|
|
925
|
+
}
|
|
926
|
+
|
|
901
927
|
async getSnapshot(key: string): Promise<StoredSnapshot | null> {
|
|
902
928
|
const { rows } = await this.call<{ rows: { id: string; data: unknown }[] }>('exec', {
|
|
903
929
|
sql: `SELECT id, data FROM "${SNAPSHOT_TABLE}" WHERE id IN (?, ?)`,
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
|
|
3
|
+
import { COLUMNS_ROW_ID, columnFingerprints, planColumnReset } from './sqlite-column-reset';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* A body landed under an older schema lacks the columns that schema did not
|
|
7
|
+
* know, at the server's version. Without a reset the next build never fetches
|
|
8
|
+
* it again (WhitePawn's puzzles stuck on "Preparing" for want of `layout`).
|
|
9
|
+
* These run the planned statements against a REAL in-memory SQLite, the same
|
|
10
|
+
* build the worker loads.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
let db: any;
|
|
14
|
+
|
|
15
|
+
const rows = (sql: string, bind?: unknown[]) =>
|
|
16
|
+
db.exec({ sql, bind, rowMode: 'object', returnValue: 'resultRows' }) as Record<string, unknown>[];
|
|
17
|
+
|
|
18
|
+
const versions = (table: string) =>
|
|
19
|
+
Object.fromEntries(
|
|
20
|
+
rows(`SELECT id, json_extract(data, '$._00_rv') AS rv FROM "${table}"`).map((r) => [r.id, r.rv])
|
|
21
|
+
);
|
|
22
|
+
|
|
23
|
+
const storedSets = () => {
|
|
24
|
+
const r = rows(`SELECT data FROM "_00_schema" WHERE id = ?`, [COLUMNS_ROW_ID]);
|
|
25
|
+
return r[0] ? (JSON.parse(r[0].data as string) as Record<string, string>) : null;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
const existing = () => new Set(rows(`SELECT name FROM sqlite_master WHERE type = 'table'`).map((r) => r.name as string));
|
|
29
|
+
|
|
30
|
+
function run(current: Record<string, string>, keep: string[] = []) {
|
|
31
|
+
const plan = planColumnReset(storedSets(), current, existing(), keep);
|
|
32
|
+
db.exec({ sql: 'BEGIN' });
|
|
33
|
+
for (const s of plan.statements) db.exec({ sql: s.sql, bind: s.bind });
|
|
34
|
+
db.exec({ sql: 'COMMIT' });
|
|
35
|
+
return plan;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
beforeEach(async () => {
|
|
39
|
+
const sqlite3: any = await sqlite3InitModule();
|
|
40
|
+
db = new sqlite3.oo1.DB(':memory:', 'c');
|
|
41
|
+
for (const t of ['_00_schema', 'puzzle', 'game']) {
|
|
42
|
+
db.exec({ sql: `CREATE TABLE "${t}" (id TEXT PRIMARY KEY, data TEXT NOT NULL)` });
|
|
43
|
+
}
|
|
44
|
+
const put = (t: string, id: string, body: object) =>
|
|
45
|
+
db.exec({ sql: `INSERT INTO "${t}"(id, data) VALUES(?, ?)`, bind: [id, JSON.stringify(body)] });
|
|
46
|
+
// Landed by the old build: no `layout`, at the server's version 6.
|
|
47
|
+
put('puzzle', 'puzzle:a', { title: 'A', _00_rv: 6 });
|
|
48
|
+
put('puzzle', 'puzzle:b', { title: 'B', _00_rv: 6 });
|
|
49
|
+
put('game', 'game:1', { white: 'x', _00_rv: 3 });
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
afterEach(() => db?.close());
|
|
53
|
+
|
|
54
|
+
const OLD = { puzzle: 'title', game: 'white' };
|
|
55
|
+
const NEW = { puzzle: 'layout,title', game: 'white' };
|
|
56
|
+
|
|
57
|
+
describe('column reset on the SQLite cache', () => {
|
|
58
|
+
it('fingerprints are the sorted declared columns', () => {
|
|
59
|
+
expect(columnFingerprints([{ name: 'puzzle', columns: { title: {}, layout: {} } }])).toEqual({
|
|
60
|
+
puzzle: 'layout,title',
|
|
61
|
+
});
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
it('first boot with no recorded sets resets every cached table once', () => {
|
|
65
|
+
const plan = run(NEW);
|
|
66
|
+
expect(plan.changed.sort()).toEqual(['game', 'puzzle']);
|
|
67
|
+
expect(versions('puzzle')).toEqual({ 'puzzle:a': 0, 'puzzle:b': 0 });
|
|
68
|
+
expect(versions('game')).toEqual({ 'game:1': 0 });
|
|
69
|
+
expect(storedSets()).toEqual(NEW);
|
|
70
|
+
// Bodies stay: they keep rendering until the refetch lands.
|
|
71
|
+
expect(rows(`SELECT data FROM puzzle WHERE id = 'puzzle:a'`)[0].data).toContain('"title":"A"');
|
|
72
|
+
|
|
73
|
+
// The next boot of the same build changes nothing.
|
|
74
|
+
const again = run(NEW);
|
|
75
|
+
expect(again.statements).toEqual([]);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it('a new column resets only that table', () => {
|
|
79
|
+
run(OLD);
|
|
80
|
+
db.exec({ sql: `UPDATE puzzle SET data = json_set(data, '$._00_rv', 6)` });
|
|
81
|
+
db.exec({ sql: `UPDATE game SET data = json_set(data, '$._00_rv', 3)` });
|
|
82
|
+
|
|
83
|
+
const plan = run(NEW);
|
|
84
|
+
expect(plan.changed).toEqual(['puzzle']);
|
|
85
|
+
expect(versions('puzzle')).toEqual({ 'puzzle:a': 0, 'puzzle:b': 0 });
|
|
86
|
+
expect(versions('game')).toEqual({ 'game:1': 3 });
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
it('keeps the version of a row with an unsettled local write', () => {
|
|
90
|
+
const plan = run(NEW, ['puzzle:b', 'game:9']);
|
|
91
|
+
expect(plan.changed).toContain('puzzle');
|
|
92
|
+
expect(versions('puzzle')).toEqual({ 'puzzle:a': 0, 'puzzle:b': 6 });
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
it('records a table the store has no rows for without touching it', () => {
|
|
96
|
+
const plan = run({ ...NEW, puzzle_chapter: 'puzzle,sort,title' });
|
|
97
|
+
expect(plan.changed).not.toContain('puzzle_chapter');
|
|
98
|
+
expect(storedSets()).toEqual({ ...NEW, puzzle_chapter: 'puzzle,sort,title' });
|
|
99
|
+
});
|
|
100
|
+
});
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cached bodies outlive a schema change on the SQLite engine, and that loses
|
|
3
|
+
* columns for good.
|
|
4
|
+
*
|
|
5
|
+
* A body is landed through `cleanRecord`, which keeps only the columns the
|
|
6
|
+
* running app's schema declares, and is stored at the server's `_00_rv`. The
|
|
7
|
+
* engine is schemaless (no migrator, nothing is recreated when the schema
|
|
8
|
+
* changes), so a tab on an older build that fetched a row after the server
|
|
9
|
+
* added and backfilled a column stored it WITHOUT that column at the new
|
|
10
|
+
* version. The next build knows the column, but its local version equals the
|
|
11
|
+
* server's, so the fetch plan never asks for the row again and the column
|
|
12
|
+
* stays missing until something else bumps the row. (WhitePawn, 2026-10-11:
|
|
13
|
+
* every puzzle read as unconverted because `layout` never arrived.)
|
|
14
|
+
*
|
|
15
|
+
* So the store remembers each table's column set (`_00_schema`, row
|
|
16
|
+
* `columns`). On boot, before the prime reads the versions, a table whose set
|
|
17
|
+
* changed, or every table when no set was recorded yet, gets its cached
|
|
18
|
+
* `_00_rv` reset to 0. The bodies stay and keep rendering; any view that names
|
|
19
|
+
* one of those rows sees it as stale and fetches it again, whole. Rows with an
|
|
20
|
+
* unsettled local write keep their version: their echo is a newer body anyway.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
/** `_00_schema` row holding the recorded column sets. */
|
|
24
|
+
export const COLUMNS_ROW_ID = 'columns';
|
|
25
|
+
|
|
26
|
+
export interface ColumnResetPlan {
|
|
27
|
+
/** Tables with cached rows whose versions are reset. */
|
|
28
|
+
changed: string[];
|
|
29
|
+
/** One transaction: the resets, then the new column sets. Empty when nothing changed. */
|
|
30
|
+
statements: { sql: string; bind?: unknown[] }[];
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* The column set of every table, as `cleanRecord` sees it: the sorted declared
|
|
35
|
+
* column names. Plain names rather than a hash, so a stored set says what it is.
|
|
36
|
+
*/
|
|
37
|
+
export function columnFingerprints(
|
|
38
|
+
tables: readonly { name: string; columns: Record<string, unknown> }[]
|
|
39
|
+
): Record<string, string> {
|
|
40
|
+
const out: Record<string, string> = {};
|
|
41
|
+
for (const t of tables) out[t.name] = Object.keys(t.columns).sort().join(',');
|
|
42
|
+
return out;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const sameSets = (a: Record<string, string> | null, b: Record<string, string>): boolean => {
|
|
46
|
+
if (!a) return false;
|
|
47
|
+
const ka = Object.keys(a);
|
|
48
|
+
return ka.length === Object.keys(b).length && ka.every((k) => a[k] === b[k]);
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* What to run for the column sets `current`, given the `stored` ones (null when
|
|
53
|
+
* none were recorded), the tables that exist in the store, and the encoded ids
|
|
54
|
+
* (`table:id`) with an unsettled local write.
|
|
55
|
+
*/
|
|
56
|
+
export function planColumnReset(
|
|
57
|
+
stored: Record<string, string> | null,
|
|
58
|
+
current: Record<string, string>,
|
|
59
|
+
existingTables: ReadonlySet<string>,
|
|
60
|
+
keep: Iterable<string>
|
|
61
|
+
): ColumnResetPlan {
|
|
62
|
+
if (sameSets(stored, current)) return { changed: [], statements: [] };
|
|
63
|
+
|
|
64
|
+
const keepByTable = new Map<string, string[]>();
|
|
65
|
+
for (const id of keep) {
|
|
66
|
+
const table = id.slice(0, id.indexOf(':'));
|
|
67
|
+
if (!keepByTable.has(table)) keepByTable.set(table, []);
|
|
68
|
+
keepByTable.get(table)!.push(id);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const changed: string[] = [];
|
|
72
|
+
const statements: ColumnResetPlan['statements'] = [];
|
|
73
|
+
for (const [table, set] of Object.entries(current)) {
|
|
74
|
+
if (stored && stored[table] === set) continue;
|
|
75
|
+
if (!existingTables.has(table)) continue;
|
|
76
|
+
changed.push(table);
|
|
77
|
+
const quoted = `"${table.replace(/"/g, '""')}"`;
|
|
78
|
+
const reset = `UPDATE ${quoted} SET data = json_set(data, '$._00_rv', 0) WHERE json_extract(data, '$._00_rv') != 0`;
|
|
79
|
+
const kept = keepByTable.get(table);
|
|
80
|
+
statements.push(
|
|
81
|
+
kept
|
|
82
|
+
? { sql: `${reset} AND id NOT IN (SELECT value FROM json_each(?))`, bind: [JSON.stringify(kept)] }
|
|
83
|
+
: { sql: reset }
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
statements.push({
|
|
87
|
+
sql: `INSERT OR REPLACE INTO "_00_schema"(id, data) VALUES (?, ?)`,
|
|
88
|
+
bind: [COLUMNS_ROW_ID, JSON.stringify(current)],
|
|
89
|
+
});
|
|
90
|
+
return { changed, statements };
|
|
91
|
+
}
|
|
@@ -87,6 +87,10 @@ export interface CircuitPrimeContext {
|
|
|
87
87
|
/** Encoded ids with an unsettled local mutation (their `_00_rv` was bumped
|
|
88
88
|
* locally and may exceed the server's next version). */
|
|
89
89
|
pendingIds: Set<string>;
|
|
90
|
+
/** Each table's client column set (`columnFingerprints`). An engine that
|
|
91
|
+
* keeps bodies across schema changes resets the versions of a table whose
|
|
92
|
+
* set changed before they are read, so its rows are fetched again whole. */
|
|
93
|
+
columnFingerprints?: Record<string, string>;
|
|
90
94
|
/** Receives every `(id, rv)` the prime put into the circuit, per table, so
|
|
91
95
|
* the sync layer can skip re-downloading bodies it already has. */
|
|
92
96
|
onVersions?: (table: string, entries: [string, number][]) => void;
|
|
@@ -507,6 +511,26 @@ export class StreamProcessorService {
|
|
|
507
511
|
this.processor === processor && generation === this.stateGeneration && epoch === this.db.epoch;
|
|
508
512
|
const t0 = performance.now();
|
|
509
513
|
|
|
514
|
+
// Before anything reads a version: a table whose columns changed since its
|
|
515
|
+
// bodies were cached reads as stale from here on (see sqlite-column-reset).
|
|
516
|
+
if (ctx.columnFingerprints && typeof this.db.resetChangedTables === 'function') {
|
|
517
|
+
try {
|
|
518
|
+
const reset = await this.db.resetChangedTables(ctx.columnFingerprints, ctx.pendingIds);
|
|
519
|
+
if (!alive()) return;
|
|
520
|
+
if (reset.length > 0) {
|
|
521
|
+
this.logger.info(
|
|
522
|
+
{ tables: reset, Category: 'sp00ky-client::StreamProcessorService::primeFromLocal' },
|
|
523
|
+
'Columns changed since these rows were cached; they will be fetched again'
|
|
524
|
+
);
|
|
525
|
+
}
|
|
526
|
+
} catch (e) {
|
|
527
|
+
this.logger.warn(
|
|
528
|
+
{ error: e, Category: 'sp00ky-client::StreamProcessorService::primeFromLocal' },
|
|
529
|
+
'Column change check failed; cached versions kept'
|
|
530
|
+
);
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
|
|
510
534
|
let restored = false;
|
|
511
535
|
if (
|
|
512
536
|
this.persistCircuit &&
|
|
@@ -106,6 +106,42 @@ describe('StreamProcessorService.primeFromLocal', () => {
|
|
|
106
106
|
await expect(service.whenPrimed()).resolves.toBeUndefined();
|
|
107
107
|
});
|
|
108
108
|
|
|
109
|
+
it('resets tables whose columns changed BEFORE reading the versions it reports', async () => {
|
|
110
|
+
const { service, db } = makeService();
|
|
111
|
+
const order: string[] = [];
|
|
112
|
+
// The engine zeroes `thing`'s cached versions (a column was added).
|
|
113
|
+
const reset = vi.fn(async () => {
|
|
114
|
+
order.push('reset');
|
|
115
|
+
db.scanVersions.mockImplementation(async (tables: string[]) => {
|
|
116
|
+
order.push('scan');
|
|
117
|
+
return Object.fromEntries(tables.map((t) => [t, t === 'thing' ? [['thing:a', 0], ['thing:b', 0]] : [['other:x', 5]]]));
|
|
118
|
+
});
|
|
119
|
+
return ['thing'];
|
|
120
|
+
});
|
|
121
|
+
(db as any).resetChangedTables = reset;
|
|
122
|
+
await service.init();
|
|
123
|
+
const c = { ...ctx(), pendingIds: new Set(['thing:b']), columnFingerprints: { thing: 'layout,title', other: 'title' } };
|
|
124
|
+
await service.primeFromLocal(c);
|
|
125
|
+
expect(reset).toHaveBeenCalledWith({ thing: 'layout,title', other: 'title' }, new Set(['thing:b']));
|
|
126
|
+
expect(order).toEqual(['reset', 'scan']);
|
|
127
|
+
// Reported as stale, so the fetch plan pulls the bodies again.
|
|
128
|
+
expect(c.onVersions).toHaveBeenCalledWith('thing', [['thing:a', 0]]);
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
it('keeps priming when the column check fails', async () => {
|
|
132
|
+
const { service, db } = makeService();
|
|
133
|
+
(db as any).resetChangedTables = vi.fn(async () => {
|
|
134
|
+
throw new Error('worker gone');
|
|
135
|
+
});
|
|
136
|
+
await service.init();
|
|
137
|
+
const c = { ...ctx(), columnFingerprints: { thing: 'title' } };
|
|
138
|
+
await service.primeFromLocal(c);
|
|
139
|
+
expect(c.onVersions).toHaveBeenCalledWith('thing', [
|
|
140
|
+
['thing:a', 1],
|
|
141
|
+
['thing:b', 2],
|
|
142
|
+
]);
|
|
143
|
+
});
|
|
144
|
+
|
|
109
145
|
it('with a snapshot, restores it under the registered views and only fetches what reconcile asks for', async () => {
|
|
110
146
|
const { service, db, received } = makeService({ snapshot: { bytes: new Uint8Array([7]), meta: goodMeta } });
|
|
111
147
|
await service.init();
|