@spooky-sync/core 0.0.1-canary.322 → 0.0.1-canary.324
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +4 -3
- package/dist/index.d.ts +156 -161
- package/dist/index.js +910 -482
- package/dist/types.d.ts +17 -6
- package/package.json +4 -4
- package/src/boot/preload.saga.test.ts +29 -0
- package/src/boot/preload.saga.ts +11 -2
- package/src/client/services.ts +3 -1
- package/src/client/sp00ky-client.test.ts +4 -1
- package/src/client/sp00ky-client.ts +37 -4
- package/src/modules/crdt/acceptance.ts +75 -0
- package/src/modules/crdt/bytes.ts +50 -0
- package/src/modules/crdt/crdt-field.ts +48 -255
- package/src/modules/crdt/crdt.test.ts +365 -0
- package/src/modules/crdt/hash.ts +103 -0
- package/src/modules/crdt/index.ts +610 -363
- package/src/modules/crdt/mirror.ts +37 -0
- package/src/mutation/push.saga.test.ts +16 -0
- package/src/mutation/push.saga.ts +9 -1
- package/src/query/anon-query-isolation.integration.test.ts +31 -0
- package/src/query/env.ts +9 -0
- package/src/services/database/cache-engine.ts +10 -0
- package/src/services/database/local-migrator.ts +5 -3
- package/src/services/database/sqlite-cache-engine.ts +28 -0
- package/src/services/database/sqlite-column-reset.integration.test.ts +100 -0
- package/src/services/database/sqlite-column-reset.ts +91 -0
- package/src/services/stream-processor/index.ts +24 -0
- package/src/services/stream-processor/stream-processor.prime.test.ts +36 -0
- package/src/testing/fake-services.ts +1 -1
- package/src/types.ts +7 -6
- package/src/utils/surql.ts +3 -4
- package/src/modules/crdt/crdt-hydration.test.ts +0 -210
- package/src/modules/crdt/crdt-reconnect.test.ts +0 -195
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { Container, LoroDoc, LoroList, LoroMap } from 'loro-crdt';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* The plain text a `@crdt` field shows as its column value, rendered the same
|
|
5
|
+
* way the SSP fold renders it (`packages/ssp-node/src/crdt/mirror.rs`).
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
export type CrdtKind = 'text' | 'prosemirror';
|
|
9
|
+
|
|
10
|
+
const PM_ROOT = 'doc';
|
|
11
|
+
const PM_NODE_NAME = 'nodeName';
|
|
12
|
+
const PM_CHILDREN = 'children';
|
|
13
|
+
const PM_LINE_BREAKS = new Set(['hard_break', 'hardBreak']);
|
|
14
|
+
|
|
15
|
+
export function renderText(doc: LoroDoc, field: string, kind: CrdtKind): string {
|
|
16
|
+
return kind === 'text' ? doc.getText(field).toString() : nodeText(doc.getMap(PM_ROOT));
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** A node holding text runs is inline content (its children concatenate); a
|
|
20
|
+
* node holding only nodes is a block container (its children are lines). */
|
|
21
|
+
function nodeText(node: LoroMap): string {
|
|
22
|
+
const name = node.get(PM_NODE_NAME);
|
|
23
|
+
if (typeof name === 'string' && PM_LINE_BREAKS.has(name)) return '\n';
|
|
24
|
+
const children = node.get(PM_CHILDREN) as Container | undefined;
|
|
25
|
+
if (!children || children.kind() !== 'List') return '';
|
|
26
|
+
const list = children as LoroList;
|
|
27
|
+
const parts: Array<{ text: boolean; value: string }> = [];
|
|
28
|
+
for (let i = 0; i < list.length; i++) {
|
|
29
|
+
const child = list.get(i) as Container | unknown;
|
|
30
|
+
if (!child || typeof child !== 'object' || !('kind' in child)) continue;
|
|
31
|
+
const c = child as Container;
|
|
32
|
+
if (c.kind() === 'Text') parts.push({ text: true, value: c.toString() });
|
|
33
|
+
else if (c.kind() === 'Map') parts.push({ text: false, value: nodeText(c as LoroMap) });
|
|
34
|
+
}
|
|
35
|
+
const inline = parts.some((p) => p.text);
|
|
36
|
+
return parts.map((p) => p.value).join(inline ? '' : '\n');
|
|
37
|
+
}
|
|
@@ -80,6 +80,22 @@ describe('drain', () => {
|
|
|
80
80
|
expect(out.dispatched.at(-1)).toEqual({ type: 'SyncOutcome', ok: true, error: undefined });
|
|
81
81
|
});
|
|
82
82
|
|
|
83
|
+
it('a CRDT edit-log CREATE that finds its row already there is acked, any other duplicate rolls back', async () => {
|
|
84
|
+
const logEnv = defaultEnv({ tables: [{ name: 'thing', columns: {} }, { name: 'doc__crdt_log', columns: {} }] } as any, { outboxBatchSize: 2 });
|
|
85
|
+
const out = await runPure(drain(logEnv), {
|
|
86
|
+
state: stateWith([1, 2]),
|
|
87
|
+
handlers: {
|
|
88
|
+
'local.query': (e: any) =>
|
|
89
|
+
e.sql === 'SELECT * FROM $ids'
|
|
90
|
+
? [[pendingRow(1, 'create', { tableName: 'doc__crdt_log', recordId: 'doc__crdt_log:1' }), pendingRow(2)]]
|
|
91
|
+
: [],
|
|
92
|
+
'remote.query': () => [err('Database record `doc__crdt_log:1` already exists'), err('Database record `thing:2` already exists')],
|
|
93
|
+
},
|
|
94
|
+
});
|
|
95
|
+
expect(out.state.outbox.find((i) => i.id === '_00_pending_mutations:m1')?.status).toBe('acked');
|
|
96
|
+
expect(out.state.outbox.some((i) => i.id === '_00_pending_mutations:m2')).toBe(false);
|
|
97
|
+
});
|
|
98
|
+
|
|
83
99
|
it('transport failures keep the tail queued behind a backoff timer (thrown, or a network ERR mid-batch, or a short result list)', async () => {
|
|
84
100
|
const thrown = await runPure(drain(env), {
|
|
85
101
|
state: stateWith([1, 2]),
|
|
@@ -67,6 +67,14 @@ export function* refreshFailedCount(): Saga<void> {
|
|
|
67
67
|
|
|
68
68
|
const ERR = (r: StatementResult | undefined): string | null => (r && r.status === 'ERR' ? r.error : null);
|
|
69
69
|
|
|
70
|
+
/**
|
|
71
|
+
* A CRDT edit-log row (`<table>__crdt_log`) is written once and never changes,
|
|
72
|
+
* so a CREATE retried after its first attempt committed (and only the answer
|
|
73
|
+
* was lost) finds it already there: that is a success, not a rejection.
|
|
74
|
+
*/
|
|
75
|
+
const isRepeatedLogCreate = (row: { mutationType: string; tableName: string }, err: string): boolean =>
|
|
76
|
+
row.mutationType === 'create' && row.tableName.endsWith('__crdt_log') && /already exists/i.test(err);
|
|
77
|
+
|
|
70
78
|
/**
|
|
71
79
|
* Drain the outbox: one serial lane, FIFO, up to `outboxBatchSize` statements
|
|
72
80
|
* per request, per-statement outcome. Accepted rows are acked (they stay in
|
|
@@ -127,7 +135,7 @@ export function* drain(env: SagaEnv): Saga<void> {
|
|
|
127
135
|
break;
|
|
128
136
|
}
|
|
129
137
|
const err = ERR(r);
|
|
130
|
-
if (err === null) {
|
|
138
|
+
if (err === null || isRepeatedLogCreate(row, err)) {
|
|
131
139
|
const del = deletePendingRow(row.id);
|
|
132
140
|
try {
|
|
133
141
|
yield fx.local.query(del.sql, del.vars);
|
|
@@ -449,6 +449,37 @@ describe('anonymous _00_query isolation (real SurrealDB)', () => {
|
|
|
449
449
|
expect(await heartbeatFound(g, qg)).toBe(true);
|
|
450
450
|
});
|
|
451
451
|
|
|
452
|
+
// Without `anonymousLiveQueries` the SSP stores a signed-out registration
|
|
453
|
+
// with no identity at all (`auth_id = ''`). The rule admitted only `'anon'`
|
|
454
|
+
// rows to their subscribers, so such a page could not read its own row back
|
|
455
|
+
// and its preload never settled.
|
|
456
|
+
it("a signed-out view stored with no identity (anonymous live queries off) answers to its own session, and to nobody else", async (ctx) => {
|
|
457
|
+
if (!reachable) ctx.skip();
|
|
458
|
+
const [a, sidA] = await guest();
|
|
459
|
+
const [b] = await guest();
|
|
460
|
+
const [u] = await member('alice');
|
|
461
|
+
const qa = viewId();
|
|
462
|
+
await register(qa, sidA, '');
|
|
463
|
+
expect((await row(qa))?.auth_id).toBe('');
|
|
464
|
+
|
|
465
|
+
expect(await readBack(a, qa)).toEqual({ rowCount: 3, state: 'ready' });
|
|
466
|
+
expect(await heartbeatFound(a, qa)).toBe(true);
|
|
467
|
+
|
|
468
|
+
const before = await row(qa);
|
|
469
|
+
for (const other of [b, u]) {
|
|
470
|
+
expect(await readBack(other, qa)).toBeNull();
|
|
471
|
+
expect(await heartbeatFound(other, qa)).toBe(false);
|
|
472
|
+
expect(await other.query('SELECT id FROM _00_query')).toEqual([[]]);
|
|
473
|
+
await other.query('DELETE $id', { id: qa });
|
|
474
|
+
}
|
|
475
|
+
expect(await row(qa)).toEqual(before);
|
|
476
|
+
|
|
477
|
+
expect(await a.query('RETURN fn::query::unsubscribe($id)', { id: qa })).toEqual([
|
|
478
|
+
{ released: true, remaining: 0 },
|
|
479
|
+
]);
|
|
480
|
+
expect(await row(qa)).toBeUndefined();
|
|
481
|
+
});
|
|
482
|
+
|
|
452
483
|
it('the owning guest can still release its view', async (ctx) => {
|
|
453
484
|
if (!reachable) ctx.skip();
|
|
454
485
|
const [a, sidA] = await guest();
|
package/src/query/env.ts
CHANGED
|
@@ -49,6 +49,15 @@ export function listRefTable(env: SagaEnv, s: ClientState): string {
|
|
|
49
49
|
return listRefTableFor(env.refMode, userId);
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
+
/**
|
|
53
|
+
* Whether this session can be sent a view's membership at all. A signed-out
|
|
54
|
+
* session reads edges only from `_00_list_ref_anon`, and only with
|
|
55
|
+
* `enableAnonymousLiveQueries`: without it the SSP files its views under no
|
|
56
|
+
* identity in `_00_list_ref`, whose rule shows those edges to no caller. Such
|
|
57
|
+
* a view can register and report its row count, but never its rows.
|
|
58
|
+
*/
|
|
59
|
+
export const receivesMembership = (env: SagaEnv, s: ClientState): boolean => s.userId !== null || env.anonLive;
|
|
60
|
+
|
|
52
61
|
export function columnsFor(env: SagaEnv, table: string): Record<string, unknown> | null {
|
|
53
62
|
const found = env.schema.tables.find((t) => t.name === table);
|
|
54
63
|
if (found) return found.columns as Record<string, unknown>;
|
|
@@ -151,6 +151,16 @@ export interface LocalStore extends LocalCacheEngine {
|
|
|
151
151
|
* without it boots the circuit empty, as before.
|
|
152
152
|
*/
|
|
153
153
|
scanVersions?(tables: string[]): Promise<Record<string, [string, number][]>>;
|
|
154
|
+
/**
|
|
155
|
+
* Reset the cached `_00_rv` of every table whose column set (`fingerprints`,
|
|
156
|
+
* see `columnFingerprints`) changed since the store last recorded it, except
|
|
157
|
+
* the rows in `keep`, and record the new sets. Returns the tables reset.
|
|
158
|
+
* For engines that keep bodies across schema changes: a body landed under an
|
|
159
|
+
* older schema lacks the columns it did not know, at the server's version,
|
|
160
|
+
* and would never be fetched again. OPTIONAL: an engine that rebuilds its
|
|
161
|
+
* store on a schema change (SurrealDB) does not need it.
|
|
162
|
+
*/
|
|
163
|
+
resetChangedTables?(fingerprints: Record<string, string>, keep: Set<string>): Promise<string[]>;
|
|
154
164
|
/**
|
|
155
165
|
* Circuit snapshot storage, keyed. Lives in the same durable store as the
|
|
156
166
|
* rows (OPFS SQLite), so it is per-bucket by construction, atomic, and
|
|
@@ -17,10 +17,11 @@ export const sha1 = async (str: string): Promise<string> => {
|
|
|
17
17
|
|
|
18
18
|
/**
|
|
19
19
|
* Local-only state a schema change must not throw away: the outbox, the
|
|
20
|
-
* failed-mutation tray
|
|
21
|
-
* rows refill from the server;
|
|
20
|
+
* failed-mutation tray, debounced patches not yet in the outbox and CRDT
|
|
21
|
+
* documents with edits not yet sent. Cached rows refill from the server;
|
|
22
|
+
* these exist nowhere else.
|
|
22
23
|
*/
|
|
23
|
-
const KEPT_TABLES = ['_00_pending_mutations', '_00_failed_mutations', '_00_pending_writes'] as const;
|
|
24
|
+
const KEPT_TABLES = ['_00_pending_mutations', '_00_failed_mutations', '_00_pending_writes', '_00_crdt_local'] as const;
|
|
24
25
|
|
|
25
26
|
export class LocalMigrator {
|
|
26
27
|
private logger: Logger;
|
|
@@ -59,6 +60,7 @@ export class LocalMigrator {
|
|
|
59
60
|
DEFINE TABLE IF NOT EXISTS _00_pending_mutations SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
|
|
60
61
|
DEFINE TABLE IF NOT EXISTS _00_pending_writes SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
|
|
61
62
|
DEFINE TABLE IF NOT EXISTS _00_blob SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
|
|
63
|
+
DEFINE TABLE IF NOT EXISTS _00_crdt_local SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
|
|
62
64
|
`;
|
|
63
65
|
const fullSchema = schemaSurql + '\n' + systemSchema;
|
|
64
66
|
|
|
@@ -30,6 +30,7 @@ import {
|
|
|
30
30
|
type SqliteTransport,
|
|
31
31
|
} from './sqlite-transport';
|
|
32
32
|
import { PROMOTION_OPEN_OPTIONS } from './sqlite-open';
|
|
33
|
+
import { COLUMNS_ROW_ID, planColumnReset } from './sqlite-column-reset';
|
|
33
34
|
import type { EngineTx, Id, LocalStore, OrderBy, RelationFetch, Row } from './cache-engine';
|
|
34
35
|
import type { EngineStorageDiagnostics } from '../../modules/devtools/storage-info';
|
|
35
36
|
|
|
@@ -72,6 +73,8 @@ const SYSTEM_TABLES = [
|
|
|
72
73
|
// asks for the ids it found in OPFS), so it has to exist up front like the
|
|
73
74
|
// rest — otherwise the very first reconcile throws and the cache runs cold.
|
|
74
75
|
'_00_blob',
|
|
76
|
+
// CRDT documents per open field, with edits not yet in the outbox.
|
|
77
|
+
'_00_crdt_local',
|
|
75
78
|
// Server-written, synced-down meta tables (see meta_tables_client.surql).
|
|
76
79
|
// DEFINE is a noop on this engine, so without seeding them here their synced
|
|
77
80
|
// rows have no local table to land in: feature flags silently fall back to
|
|
@@ -898,6 +901,31 @@ export class SqliteCacheEngine implements LocalStore {
|
|
|
898
901
|
return out;
|
|
899
902
|
}
|
|
900
903
|
|
|
904
|
+
/**
|
|
905
|
+
* Reset the cached `_00_rv` of every table whose column set changed since
|
|
906
|
+
* the last boot, so its rows are fetched again whole. See
|
|
907
|
+
* `sqlite-column-reset.ts`. Returns the tables reset.
|
|
908
|
+
*/
|
|
909
|
+
async resetChangedTables(fingerprints: Record<string, string>, keep: Set<string>): Promise<string[]> {
|
|
910
|
+
const { rows } = await this.call<{ rows: { data: unknown }[] }>('exec', {
|
|
911
|
+
sql: `SELECT data FROM "_00_schema" WHERE id = ?`,
|
|
912
|
+
bind: [COLUMNS_ROW_ID],
|
|
913
|
+
});
|
|
914
|
+
let stored: Record<string, string> | null = null;
|
|
915
|
+
try {
|
|
916
|
+
const raw = rows?.[0]?.data;
|
|
917
|
+
if (typeof raw === 'string') stored = JSON.parse(raw) as Record<string, string>;
|
|
918
|
+
} catch {
|
|
919
|
+
stored = null;
|
|
920
|
+
}
|
|
921
|
+
const { rows: tables } = await this.call<{ rows: { name: string }[] }>('exec', {
|
|
922
|
+
sql: `SELECT name FROM sqlite_master WHERE type = 'table'`,
|
|
923
|
+
});
|
|
924
|
+
const plan = planColumnReset(stored, fingerprints, new Set((tables ?? []).map((t) => t.name)), keep);
|
|
925
|
+
if (plan.statements.length > 0) await this.call('batch', plan.statements);
|
|
926
|
+
return plan.changed;
|
|
927
|
+
}
|
|
928
|
+
|
|
901
929
|
async getSnapshot(key: string): Promise<StoredSnapshot | null> {
|
|
902
930
|
const { rows } = await this.call<{ rows: { id: string; data: unknown }[] }>('exec', {
|
|
903
931
|
sql: `SELECT id, data FROM "${SNAPSHOT_TABLE}" WHERE id IN (?, ?)`,
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
2
|
+
import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
|
|
3
|
+
import { COLUMNS_ROW_ID, columnFingerprints, planColumnReset } from './sqlite-column-reset';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* A body landed under an older schema lacks the columns that schema did not
|
|
7
|
+
* know, at the server's version. Without a reset the next build never fetches
|
|
8
|
+
* it again (WhitePawn's puzzles stuck on "Preparing" for want of `layout`).
|
|
9
|
+
* These run the planned statements against a REAL in-memory SQLite, the same
|
|
10
|
+
* build the worker loads.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
let db: any;
|
|
14
|
+
|
|
15
|
+
const rows = (sql: string, bind?: unknown[]) =>
|
|
16
|
+
db.exec({ sql, bind, rowMode: 'object', returnValue: 'resultRows' }) as Record<string, unknown>[];
|
|
17
|
+
|
|
18
|
+
const versions = (table: string) =>
|
|
19
|
+
Object.fromEntries(
|
|
20
|
+
rows(`SELECT id, json_extract(data, '$._00_rv') AS rv FROM "${table}"`).map((r) => [r.id, r.rv])
|
|
21
|
+
);
|
|
22
|
+
|
|
23
|
+
const storedSets = () => {
|
|
24
|
+
const r = rows(`SELECT data FROM "_00_schema" WHERE id = ?`, [COLUMNS_ROW_ID]);
|
|
25
|
+
return r[0] ? (JSON.parse(r[0].data as string) as Record<string, string>) : null;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
const existing = () => new Set(rows(`SELECT name FROM sqlite_master WHERE type = 'table'`).map((r) => r.name as string));
|
|
29
|
+
|
|
30
|
+
function run(current: Record<string, string>, keep: string[] = []) {
|
|
31
|
+
const plan = planColumnReset(storedSets(), current, existing(), keep);
|
|
32
|
+
db.exec({ sql: 'BEGIN' });
|
|
33
|
+
for (const s of plan.statements) db.exec({ sql: s.sql, bind: s.bind });
|
|
34
|
+
db.exec({ sql: 'COMMIT' });
|
|
35
|
+
return plan;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
beforeEach(async () => {
|
|
39
|
+
const sqlite3: any = await sqlite3InitModule();
|
|
40
|
+
db = new sqlite3.oo1.DB(':memory:', 'c');
|
|
41
|
+
for (const t of ['_00_schema', 'puzzle', 'game']) {
|
|
42
|
+
db.exec({ sql: `CREATE TABLE "${t}" (id TEXT PRIMARY KEY, data TEXT NOT NULL)` });
|
|
43
|
+
}
|
|
44
|
+
const put = (t: string, id: string, body: object) =>
|
|
45
|
+
db.exec({ sql: `INSERT INTO "${t}"(id, data) VALUES(?, ?)`, bind: [id, JSON.stringify(body)] });
|
|
46
|
+
// Landed by the old build: no `layout`, at the server's version 6.
|
|
47
|
+
put('puzzle', 'puzzle:a', { title: 'A', _00_rv: 6 });
|
|
48
|
+
put('puzzle', 'puzzle:b', { title: 'B', _00_rv: 6 });
|
|
49
|
+
put('game', 'game:1', { white: 'x', _00_rv: 3 });
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
afterEach(() => db?.close());
|
|
53
|
+
|
|
54
|
+
const OLD = { puzzle: 'title', game: 'white' };
|
|
55
|
+
const NEW = { puzzle: 'layout,title', game: 'white' };
|
|
56
|
+
|
|
57
|
+
describe('column reset on the SQLite cache', () => {
|
|
58
|
+
it('fingerprints are the sorted declared columns', () => {
|
|
59
|
+
expect(columnFingerprints([{ name: 'puzzle', columns: { title: {}, layout: {} } }])).toEqual({
|
|
60
|
+
puzzle: 'layout,title',
|
|
61
|
+
});
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
it('first boot with no recorded sets resets every cached table once', () => {
|
|
65
|
+
const plan = run(NEW);
|
|
66
|
+
expect(plan.changed.sort()).toEqual(['game', 'puzzle']);
|
|
67
|
+
expect(versions('puzzle')).toEqual({ 'puzzle:a': 0, 'puzzle:b': 0 });
|
|
68
|
+
expect(versions('game')).toEqual({ 'game:1': 0 });
|
|
69
|
+
expect(storedSets()).toEqual(NEW);
|
|
70
|
+
// Bodies stay: they keep rendering until the refetch lands.
|
|
71
|
+
expect(rows(`SELECT data FROM puzzle WHERE id = 'puzzle:a'`)[0].data).toContain('"title":"A"');
|
|
72
|
+
|
|
73
|
+
// The next boot of the same build changes nothing.
|
|
74
|
+
const again = run(NEW);
|
|
75
|
+
expect(again.statements).toEqual([]);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it('a new column resets only that table', () => {
|
|
79
|
+
run(OLD);
|
|
80
|
+
db.exec({ sql: `UPDATE puzzle SET data = json_set(data, '$._00_rv', 6)` });
|
|
81
|
+
db.exec({ sql: `UPDATE game SET data = json_set(data, '$._00_rv', 3)` });
|
|
82
|
+
|
|
83
|
+
const plan = run(NEW);
|
|
84
|
+
expect(plan.changed).toEqual(['puzzle']);
|
|
85
|
+
expect(versions('puzzle')).toEqual({ 'puzzle:a': 0, 'puzzle:b': 0 });
|
|
86
|
+
expect(versions('game')).toEqual({ 'game:1': 3 });
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
it('keeps the version of a row with an unsettled local write', () => {
|
|
90
|
+
const plan = run(NEW, ['puzzle:b', 'game:9']);
|
|
91
|
+
expect(plan.changed).toContain('puzzle');
|
|
92
|
+
expect(versions('puzzle')).toEqual({ 'puzzle:a': 0, 'puzzle:b': 6 });
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
it('records a table the store has no rows for without touching it', () => {
|
|
96
|
+
const plan = run({ ...NEW, puzzle_chapter: 'puzzle,sort,title' });
|
|
97
|
+
expect(plan.changed).not.toContain('puzzle_chapter');
|
|
98
|
+
expect(storedSets()).toEqual({ ...NEW, puzzle_chapter: 'puzzle,sort,title' });
|
|
99
|
+
});
|
|
100
|
+
});
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cached bodies outlive a schema change on the SQLite engine, and that loses
|
|
3
|
+
* columns for good.
|
|
4
|
+
*
|
|
5
|
+
* A body is landed through `cleanRecord`, which keeps only the columns the
|
|
6
|
+
* running app's schema declares, and is stored at the server's `_00_rv`. The
|
|
7
|
+
* engine is schemaless (no migrator, nothing is recreated when the schema
|
|
8
|
+
* changes), so a tab on an older build that fetched a row after the server
|
|
9
|
+
* added and backfilled a column stored it WITHOUT that column at the new
|
|
10
|
+
* version. The next build knows the column, but its local version equals the
|
|
11
|
+
* server's, so the fetch plan never asks for the row again and the column
|
|
12
|
+
* stays missing until something else bumps the row. (WhitePawn, 2026-10-11:
|
|
13
|
+
* every puzzle read as unconverted because `layout` never arrived.)
|
|
14
|
+
*
|
|
15
|
+
* So the store remembers each table's column set (`_00_schema`, row
|
|
16
|
+
* `columns`). On boot, before the prime reads the versions, a table whose set
|
|
17
|
+
* changed, or every table when no set was recorded yet, gets its cached
|
|
18
|
+
* `_00_rv` reset to 0. The bodies stay and keep rendering; any view that names
|
|
19
|
+
* one of those rows sees it as stale and fetches it again, whole. Rows with an
|
|
20
|
+
* unsettled local write keep their version: their echo is a newer body anyway.
|
|
21
|
+
*/
|
|
22
|
+
|
|
23
|
+
/** `_00_schema` row holding the recorded column sets. */
|
|
24
|
+
export const COLUMNS_ROW_ID = 'columns';
|
|
25
|
+
|
|
26
|
+
export interface ColumnResetPlan {
|
|
27
|
+
/** Tables with cached rows whose versions are reset. */
|
|
28
|
+
changed: string[];
|
|
29
|
+
/** One transaction: the resets, then the new column sets. Empty when nothing changed. */
|
|
30
|
+
statements: { sql: string; bind?: unknown[] }[];
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* The column set of every table, as `cleanRecord` sees it: the sorted declared
|
|
35
|
+
* column names. Plain names rather than a hash, so a stored set says what it is.
|
|
36
|
+
*/
|
|
37
|
+
export function columnFingerprints(
|
|
38
|
+
tables: readonly { name: string; columns: Record<string, unknown> }[]
|
|
39
|
+
): Record<string, string> {
|
|
40
|
+
const out: Record<string, string> = {};
|
|
41
|
+
for (const t of tables) out[t.name] = Object.keys(t.columns).sort().join(',');
|
|
42
|
+
return out;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const sameSets = (a: Record<string, string> | null, b: Record<string, string>): boolean => {
|
|
46
|
+
if (!a) return false;
|
|
47
|
+
const ka = Object.keys(a);
|
|
48
|
+
return ka.length === Object.keys(b).length && ka.every((k) => a[k] === b[k]);
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* What to run for the column sets `current`, given the `stored` ones (null when
|
|
53
|
+
* none were recorded), the tables that exist in the store, and the encoded ids
|
|
54
|
+
* (`table:id`) with an unsettled local write.
|
|
55
|
+
*/
|
|
56
|
+
export function planColumnReset(
|
|
57
|
+
stored: Record<string, string> | null,
|
|
58
|
+
current: Record<string, string>,
|
|
59
|
+
existingTables: ReadonlySet<string>,
|
|
60
|
+
keep: Iterable<string>
|
|
61
|
+
): ColumnResetPlan {
|
|
62
|
+
if (sameSets(stored, current)) return { changed: [], statements: [] };
|
|
63
|
+
|
|
64
|
+
const keepByTable = new Map<string, string[]>();
|
|
65
|
+
for (const id of keep) {
|
|
66
|
+
const table = id.slice(0, id.indexOf(':'));
|
|
67
|
+
if (!keepByTable.has(table)) keepByTable.set(table, []);
|
|
68
|
+
keepByTable.get(table)!.push(id);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const changed: string[] = [];
|
|
72
|
+
const statements: ColumnResetPlan['statements'] = [];
|
|
73
|
+
for (const [table, set] of Object.entries(current)) {
|
|
74
|
+
if (stored && stored[table] === set) continue;
|
|
75
|
+
if (!existingTables.has(table)) continue;
|
|
76
|
+
changed.push(table);
|
|
77
|
+
const quoted = `"${table.replace(/"/g, '""')}"`;
|
|
78
|
+
const reset = `UPDATE ${quoted} SET data = json_set(data, '$._00_rv', 0) WHERE json_extract(data, '$._00_rv') != 0`;
|
|
79
|
+
const kept = keepByTable.get(table);
|
|
80
|
+
statements.push(
|
|
81
|
+
kept
|
|
82
|
+
? { sql: `${reset} AND id NOT IN (SELECT value FROM json_each(?))`, bind: [JSON.stringify(kept)] }
|
|
83
|
+
: { sql: reset }
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
statements.push({
|
|
87
|
+
sql: `INSERT OR REPLACE INTO "_00_schema"(id, data) VALUES (?, ?)`,
|
|
88
|
+
bind: [COLUMNS_ROW_ID, JSON.stringify(current)],
|
|
89
|
+
});
|
|
90
|
+
return { changed, statements };
|
|
91
|
+
}
|
|
@@ -87,6 +87,10 @@ export interface CircuitPrimeContext {
|
|
|
87
87
|
/** Encoded ids with an unsettled local mutation (their `_00_rv` was bumped
|
|
88
88
|
* locally and may exceed the server's next version). */
|
|
89
89
|
pendingIds: Set<string>;
|
|
90
|
+
/** Each table's client column set (`columnFingerprints`). An engine that
|
|
91
|
+
* keeps bodies across schema changes resets the versions of a table whose
|
|
92
|
+
* set changed before they are read, so its rows are fetched again whole. */
|
|
93
|
+
columnFingerprints?: Record<string, string>;
|
|
90
94
|
/** Receives every `(id, rv)` the prime put into the circuit, per table, so
|
|
91
95
|
* the sync layer can skip re-downloading bodies it already has. */
|
|
92
96
|
onVersions?: (table: string, entries: [string, number][]) => void;
|
|
@@ -507,6 +511,26 @@ export class StreamProcessorService {
|
|
|
507
511
|
this.processor === processor && generation === this.stateGeneration && epoch === this.db.epoch;
|
|
508
512
|
const t0 = performance.now();
|
|
509
513
|
|
|
514
|
+
// Before anything reads a version: a table whose columns changed since its
|
|
515
|
+
// bodies were cached reads as stale from here on (see sqlite-column-reset).
|
|
516
|
+
if (ctx.columnFingerprints && typeof this.db.resetChangedTables === 'function') {
|
|
517
|
+
try {
|
|
518
|
+
const reset = await this.db.resetChangedTables(ctx.columnFingerprints, ctx.pendingIds);
|
|
519
|
+
if (!alive()) return;
|
|
520
|
+
if (reset.length > 0) {
|
|
521
|
+
this.logger.info(
|
|
522
|
+
{ tables: reset, Category: 'sp00ky-client::StreamProcessorService::primeFromLocal' },
|
|
523
|
+
'Columns changed since these rows were cached; they will be fetched again'
|
|
524
|
+
);
|
|
525
|
+
}
|
|
526
|
+
} catch (e) {
|
|
527
|
+
this.logger.warn(
|
|
528
|
+
{ error: e, Category: 'sp00ky-client::StreamProcessorService::primeFromLocal' },
|
|
529
|
+
'Column change check failed; cached versions kept'
|
|
530
|
+
);
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
|
|
510
534
|
let restored = false;
|
|
511
535
|
if (
|
|
512
536
|
this.persistCircuit &&
|
|
@@ -106,6 +106,42 @@ describe('StreamProcessorService.primeFromLocal', () => {
|
|
|
106
106
|
await expect(service.whenPrimed()).resolves.toBeUndefined();
|
|
107
107
|
});
|
|
108
108
|
|
|
109
|
+
it('resets tables whose columns changed BEFORE reading the versions it reports', async () => {
|
|
110
|
+
const { service, db } = makeService();
|
|
111
|
+
const order: string[] = [];
|
|
112
|
+
// The engine zeroes `thing`'s cached versions (a column was added).
|
|
113
|
+
const reset = vi.fn(async () => {
|
|
114
|
+
order.push('reset');
|
|
115
|
+
db.scanVersions.mockImplementation(async (tables: string[]) => {
|
|
116
|
+
order.push('scan');
|
|
117
|
+
return Object.fromEntries(tables.map((t) => [t, t === 'thing' ? [['thing:a', 0], ['thing:b', 0]] : [['other:x', 5]]]));
|
|
118
|
+
});
|
|
119
|
+
return ['thing'];
|
|
120
|
+
});
|
|
121
|
+
(db as any).resetChangedTables = reset;
|
|
122
|
+
await service.init();
|
|
123
|
+
const c = { ...ctx(), pendingIds: new Set(['thing:b']), columnFingerprints: { thing: 'layout,title', other: 'title' } };
|
|
124
|
+
await service.primeFromLocal(c);
|
|
125
|
+
expect(reset).toHaveBeenCalledWith({ thing: 'layout,title', other: 'title' }, new Set(['thing:b']));
|
|
126
|
+
expect(order).toEqual(['reset', 'scan']);
|
|
127
|
+
// Reported as stale, so the fetch plan pulls the bodies again.
|
|
128
|
+
expect(c.onVersions).toHaveBeenCalledWith('thing', [['thing:a', 0]]);
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
it('keeps priming when the column check fails', async () => {
|
|
132
|
+
const { service, db } = makeService();
|
|
133
|
+
(db as any).resetChangedTables = vi.fn(async () => {
|
|
134
|
+
throw new Error('worker gone');
|
|
135
|
+
});
|
|
136
|
+
await service.init();
|
|
137
|
+
const c = { ...ctx(), columnFingerprints: { thing: 'title' } };
|
|
138
|
+
await service.primeFromLocal(c);
|
|
139
|
+
expect(c.onVersions).toHaveBeenCalledWith('thing', [
|
|
140
|
+
['thing:a', 1],
|
|
141
|
+
['thing:b', 2],
|
|
142
|
+
]);
|
|
143
|
+
});
|
|
144
|
+
|
|
109
145
|
it('with a snapshot, restores it under the registered views and only fetches what reconcile asks for', async () => {
|
|
110
146
|
const { service, db, received } = makeService({ snapshot: { bytes: new Uint8Array([7]), meta: goodMeta } });
|
|
111
147
|
await service.init();
|
|
@@ -37,7 +37,7 @@ export function fakeServiceBundle<S extends SchemaStructure>(over: Partial<Servi
|
|
|
37
37
|
persistence: { set: async () => undefined, get: async () => null, remove: async () => undefined },
|
|
38
38
|
streamProcessor: { addReceiver: (r: unknown) => void receivers.push(r), checkpoint: async () => undefined, dispose: noop } as any,
|
|
39
39
|
migrator: {} as any,
|
|
40
|
-
crdt: { closeAll: noop, dispose: noop, open: async () => 'field', close:
|
|
40
|
+
crdt: { attach: noop, setSessionId: noop, closeAll: noop, dispose: noop, open: async () => 'field', close: async () => {} } as any,
|
|
41
41
|
auth: { subscribe: (cb: (u: string | null) => void) => (authListeners.push(cb), noop), subscribeImpersonation: () => noop, onBeforeSignOut: () => noop, currentUser: null, isAuthenticated: false, token: null, impersonation: null, eventSystem: { subscribe: noop } } as any,
|
|
42
42
|
tabs: null,
|
|
43
43
|
tabsUnsupportedReason: 'test',
|
package/src/types.ts
CHANGED
|
@@ -259,11 +259,11 @@ export interface Sp00kyConfig<S extends SchemaStructure> {
|
|
|
259
259
|
streamDebounceTime?: number;
|
|
260
260
|
/**
|
|
261
261
|
* Debounce time in milliseconds for syncing collaborative (CRDT) field
|
|
262
|
-
*
|
|
263
|
-
*
|
|
264
|
-
*
|
|
265
|
-
* more
|
|
266
|
-
* collaborators. Defaults to
|
|
262
|
+
* edits to the server. Local state is kept on every change (so reload and
|
|
263
|
+
* offline work), but a field's own new ops go out as one edit-log row per
|
|
264
|
+
* pause of this length (at most 2 s apart while typing continues).
|
|
265
|
+
* Lower = snappier propagation and more rows; higher = fewer rows and more
|
|
266
|
+
* lag for other collaborators. Defaults to 300ms.
|
|
267
267
|
*/
|
|
268
268
|
crdtDebounceMs?: number;
|
|
269
269
|
/**
|
|
@@ -321,7 +321,8 @@ export interface Sp00kyConfig<S extends SchemaStructure> {
|
|
|
321
321
|
* page gets live `useQuery` updates over world-readable tables. Requires the
|
|
322
322
|
* server to be deployed with `anonymousLiveQueries: true` in `sp00ky.yml`
|
|
323
323
|
* (this flag must match it). Defaults to `false`: anonymous clients can read
|
|
324
|
-
* one-shot but never sync live
|
|
324
|
+
* one-shot but never sync live, and a `preload` made while signed out returns
|
|
325
|
+
* at once rather than wait for membership that never comes.
|
|
325
326
|
*/
|
|
326
327
|
enableAnonymousLiveQueries?: boolean;
|
|
327
328
|
/**
|
package/src/utils/surql.ts
CHANGED
|
@@ -120,10 +120,9 @@ export const surql: SurqlHelper = {
|
|
|
120
120
|
return `UPSERT ONLY $${idVar} REPLACE $${dataVar}`;
|
|
121
121
|
},
|
|
122
122
|
|
|
123
|
-
// Use this for sync-down ingestion: the remote payload
|
|
124
|
-
// fields injected by the CLI's local-schema setup
|
|
125
|
-
//
|
|
126
|
-
// break offline reload of CRDT state.
|
|
123
|
+
// Use this for sync-down ingestion: the remote payload can omit local-only
|
|
124
|
+
// fields injected by the CLI's local-schema setup, and REPLACE would wipe
|
|
125
|
+
// them on every round-trip.
|
|
127
126
|
upsertMerge(idVar: string, dataVar: string) {
|
|
128
127
|
return `UPSERT ONLY $${idVar} MERGE $${dataVar}`;
|
|
129
128
|
},
|