@spooky-sync/core 0.0.1-canary.322 → 0.0.1-canary.324

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/AGENTS.md +4 -3
  2. package/dist/index.d.ts +156 -161
  3. package/dist/index.js +910 -482
  4. package/dist/types.d.ts +17 -6
  5. package/package.json +4 -4
  6. package/src/boot/preload.saga.test.ts +29 -0
  7. package/src/boot/preload.saga.ts +11 -2
  8. package/src/client/services.ts +3 -1
  9. package/src/client/sp00ky-client.test.ts +4 -1
  10. package/src/client/sp00ky-client.ts +37 -4
  11. package/src/modules/crdt/acceptance.ts +75 -0
  12. package/src/modules/crdt/bytes.ts +50 -0
  13. package/src/modules/crdt/crdt-field.ts +48 -255
  14. package/src/modules/crdt/crdt.test.ts +365 -0
  15. package/src/modules/crdt/hash.ts +103 -0
  16. package/src/modules/crdt/index.ts +610 -363
  17. package/src/modules/crdt/mirror.ts +37 -0
  18. package/src/mutation/push.saga.test.ts +16 -0
  19. package/src/mutation/push.saga.ts +9 -1
  20. package/src/query/anon-query-isolation.integration.test.ts +31 -0
  21. package/src/query/env.ts +9 -0
  22. package/src/services/database/cache-engine.ts +10 -0
  23. package/src/services/database/local-migrator.ts +5 -3
  24. package/src/services/database/sqlite-cache-engine.ts +28 -0
  25. package/src/services/database/sqlite-column-reset.integration.test.ts +100 -0
  26. package/src/services/database/sqlite-column-reset.ts +91 -0
  27. package/src/services/stream-processor/index.ts +24 -0
  28. package/src/services/stream-processor/stream-processor.prime.test.ts +36 -0
  29. package/src/testing/fake-services.ts +1 -1
  30. package/src/types.ts +7 -6
  31. package/src/utils/surql.ts +3 -4
  32. package/src/modules/crdt/crdt-hydration.test.ts +0 -210
  33. package/src/modules/crdt/crdt-reconnect.test.ts +0 -195
@@ -0,0 +1,37 @@
1
+ import type { Container, LoroDoc, LoroList, LoroMap } from 'loro-crdt';
2
+
3
+ /**
4
+ * The plain text a `@crdt` field shows as its column value, rendered the same
5
+ * way the SSP fold renders it (`packages/ssp-node/src/crdt/mirror.rs`).
6
+ */
7
+
8
+ export type CrdtKind = 'text' | 'prosemirror';
9
+
10
+ const PM_ROOT = 'doc';
11
+ const PM_NODE_NAME = 'nodeName';
12
+ const PM_CHILDREN = 'children';
13
+ const PM_LINE_BREAKS = new Set(['hard_break', 'hardBreak']);
14
+
15
+ export function renderText(doc: LoroDoc, field: string, kind: CrdtKind): string {
16
+ return kind === 'text' ? doc.getText(field).toString() : nodeText(doc.getMap(PM_ROOT));
17
+ }
18
+
19
+ /** A node holding text runs is inline content (its children concatenate); a
20
+ * node holding only nodes is a block container (its children are lines). */
21
+ function nodeText(node: LoroMap): string {
22
+ const name = node.get(PM_NODE_NAME);
23
+ if (typeof name === 'string' && PM_LINE_BREAKS.has(name)) return '\n';
24
+ const children = node.get(PM_CHILDREN) as Container | undefined;
25
+ if (!children || children.kind() !== 'List') return '';
26
+ const list = children as LoroList;
27
+ const parts: Array<{ text: boolean; value: string }> = [];
28
+ for (let i = 0; i < list.length; i++) {
29
+ const child = list.get(i) as Container | unknown;
30
+ if (!child || typeof child !== 'object' || !('kind' in child)) continue;
31
+ const c = child as Container;
32
+ if (c.kind() === 'Text') parts.push({ text: true, value: c.toString() });
33
+ else if (c.kind() === 'Map') parts.push({ text: false, value: nodeText(c as LoroMap) });
34
+ }
35
+ const inline = parts.some((p) => p.text);
36
+ return parts.map((p) => p.value).join(inline ? '' : '\n');
37
+ }
@@ -80,6 +80,22 @@ describe('drain', () => {
80
80
  expect(out.dispatched.at(-1)).toEqual({ type: 'SyncOutcome', ok: true, error: undefined });
81
81
  });
82
82
 
83
+ it('a CRDT edit-log CREATE that finds its row already there is acked, any other duplicate rolls back', async () => {
84
+ const logEnv = defaultEnv({ tables: [{ name: 'thing', columns: {} }, { name: 'doc__crdt_log', columns: {} }] } as any, { outboxBatchSize: 2 });
85
+ const out = await runPure(drain(logEnv), {
86
+ state: stateWith([1, 2]),
87
+ handlers: {
88
+ 'local.query': (e: any) =>
89
+ e.sql === 'SELECT * FROM $ids'
90
+ ? [[pendingRow(1, 'create', { tableName: 'doc__crdt_log', recordId: 'doc__crdt_log:1' }), pendingRow(2)]]
91
+ : [],
92
+ 'remote.query': () => [err('Database record `doc__crdt_log:1` already exists'), err('Database record `thing:2` already exists')],
93
+ },
94
+ });
95
+ expect(out.state.outbox.find((i) => i.id === '_00_pending_mutations:m1')?.status).toBe('acked');
96
+ expect(out.state.outbox.some((i) => i.id === '_00_pending_mutations:m2')).toBe(false);
97
+ });
98
+
83
99
  it('transport failures keep the tail queued behind a backoff timer (thrown, or a network ERR mid-batch, or a short result list)', async () => {
84
100
  const thrown = await runPure(drain(env), {
85
101
  state: stateWith([1, 2]),
@@ -67,6 +67,14 @@ export function* refreshFailedCount(): Saga<void> {
67
67
 
68
68
  const ERR = (r: StatementResult | undefined): string | null => (r && r.status === 'ERR' ? r.error : null);
69
69
 
70
+ /**
71
+ * A CRDT edit-log row (`<table>__crdt_log`) is written once and never changes,
72
+ * so a CREATE retried after its first attempt committed (and only the answer
73
+ * was lost) finds it already there: that is a success, not a rejection.
74
+ */
75
+ const isRepeatedLogCreate = (row: { mutationType: string; tableName: string }, err: string): boolean =>
76
+ row.mutationType === 'create' && row.tableName.endsWith('__crdt_log') && /already exists/i.test(err);
77
+
70
78
  /**
71
79
  * Drain the outbox: one serial lane, FIFO, up to `outboxBatchSize` statements
72
80
  * per request, per-statement outcome. Accepted rows are acked (they stay in
@@ -127,7 +135,7 @@ export function* drain(env: SagaEnv): Saga<void> {
127
135
  break;
128
136
  }
129
137
  const err = ERR(r);
130
- if (err === null) {
138
+ if (err === null || isRepeatedLogCreate(row, err)) {
131
139
  const del = deletePendingRow(row.id);
132
140
  try {
133
141
  yield fx.local.query(del.sql, del.vars);
@@ -449,6 +449,37 @@ describe('anonymous _00_query isolation (real SurrealDB)', () => {
449
449
  expect(await heartbeatFound(g, qg)).toBe(true);
450
450
  });
451
451
 
452
+ // Without `anonymousLiveQueries` the SSP stores a signed-out registration
453
+ // with no identity at all (`auth_id = ''`). The rule admitted only `'anon'`
454
+ // rows to their subscribers, so such a page could not read its own row back
455
+ // and its preload never settled.
456
+ it("a signed-out view stored with no identity (anonymous live queries off) answers to its own session, and to nobody else", async (ctx) => {
457
+ if (!reachable) ctx.skip();
458
+ const [a, sidA] = await guest();
459
+ const [b] = await guest();
460
+ const [u] = await member('alice');
461
+ const qa = viewId();
462
+ await register(qa, sidA, '');
463
+ expect((await row(qa))?.auth_id).toBe('');
464
+
465
+ expect(await readBack(a, qa)).toEqual({ rowCount: 3, state: 'ready' });
466
+ expect(await heartbeatFound(a, qa)).toBe(true);
467
+
468
+ const before = await row(qa);
469
+ for (const other of [b, u]) {
470
+ expect(await readBack(other, qa)).toBeNull();
471
+ expect(await heartbeatFound(other, qa)).toBe(false);
472
+ expect(await other.query('SELECT id FROM _00_query')).toEqual([[]]);
473
+ await other.query('DELETE $id', { id: qa });
474
+ }
475
+ expect(await row(qa)).toEqual(before);
476
+
477
+ expect(await a.query('RETURN fn::query::unsubscribe($id)', { id: qa })).toEqual([
478
+ { released: true, remaining: 0 },
479
+ ]);
480
+ expect(await row(qa)).toBeUndefined();
481
+ });
482
+
452
483
  it('the owning guest can still release its view', async (ctx) => {
453
484
  if (!reachable) ctx.skip();
454
485
  const [a, sidA] = await guest();
package/src/query/env.ts CHANGED
@@ -49,6 +49,15 @@ export function listRefTable(env: SagaEnv, s: ClientState): string {
49
49
  return listRefTableFor(env.refMode, userId);
50
50
  }
51
51
 
52
+ /**
53
+ * Whether this session can be sent a view's membership at all. A signed-out
54
+ * session reads edges only from `_00_list_ref_anon`, and only with
55
+ * `enableAnonymousLiveQueries`: without it the SSP files its views under no
56
+ * identity in `_00_list_ref`, whose rule shows those edges to no caller. Such
57
+ * a view can register and report its row count, but never its rows.
58
+ */
59
+ export const receivesMembership = (env: SagaEnv, s: ClientState): boolean => s.userId !== null || env.anonLive;
60
+
52
61
  export function columnsFor(env: SagaEnv, table: string): Record<string, unknown> | null {
53
62
  const found = env.schema.tables.find((t) => t.name === table);
54
63
  if (found) return found.columns as Record<string, unknown>;
@@ -151,6 +151,16 @@ export interface LocalStore extends LocalCacheEngine {
151
151
  * without it boots the circuit empty, as before.
152
152
  */
153
153
  scanVersions?(tables: string[]): Promise<Record<string, [string, number][]>>;
154
+ /**
155
+ * Reset the cached `_00_rv` of every table whose column set (`fingerprints`,
156
+ * see `columnFingerprints`) changed since the store last recorded it, except
157
+ * the rows in `keep`, and record the new sets. Returns the tables reset.
158
+ * For engines that keep bodies across schema changes: a body landed under an
159
+ * older schema lacks the columns it did not know, at the server's version,
160
+ * and would never be fetched again. OPTIONAL: an engine that rebuilds its
161
+ * store on a schema change (SurrealDB) does not need it.
162
+ */
163
+ resetChangedTables?(fingerprints: Record<string, string>, keep: Set<string>): Promise<string[]>;
154
164
  /**
155
165
  * Circuit snapshot storage, keyed. Lives in the same durable store as the
156
166
  * rows (OPFS SQLite), so it is per-bucket by construction, atomic, and
@@ -17,10 +17,11 @@ export const sha1 = async (str: string): Promise<string> => {
17
17
 
18
18
  /**
19
19
  * Local-only state a schema change must not throw away: the outbox, the
20
- * failed-mutation tray and debounced patches not yet in the outbox. Cached
21
- * rows refill from the server; these exist nowhere else.
20
+ * failed-mutation tray, debounced patches not yet in the outbox and CRDT
21
+ * documents with edits not yet sent. Cached rows refill from the server;
22
+ * these exist nowhere else.
22
23
  */
23
- const KEPT_TABLES = ['_00_pending_mutations', '_00_failed_mutations', '_00_pending_writes'] as const;
24
+ const KEPT_TABLES = ['_00_pending_mutations', '_00_failed_mutations', '_00_pending_writes', '_00_crdt_local'] as const;
24
25
 
25
26
  export class LocalMigrator {
26
27
  private logger: Logger;
@@ -59,6 +60,7 @@ export class LocalMigrator {
59
60
  DEFINE TABLE IF NOT EXISTS _00_pending_mutations SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
60
61
  DEFINE TABLE IF NOT EXISTS _00_pending_writes SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
61
62
  DEFINE TABLE IF NOT EXISTS _00_blob SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
63
+ DEFINE TABLE IF NOT EXISTS _00_crdt_local SCHEMALESS PERMISSIONS FOR select, create, update, delete WHERE true;
62
64
  `;
63
65
  const fullSchema = schemaSurql + '\n' + systemSchema;
64
66
 
@@ -30,6 +30,7 @@ import {
30
30
  type SqliteTransport,
31
31
  } from './sqlite-transport';
32
32
  import { PROMOTION_OPEN_OPTIONS } from './sqlite-open';
33
+ import { COLUMNS_ROW_ID, planColumnReset } from './sqlite-column-reset';
33
34
  import type { EngineTx, Id, LocalStore, OrderBy, RelationFetch, Row } from './cache-engine';
34
35
  import type { EngineStorageDiagnostics } from '../../modules/devtools/storage-info';
35
36
 
@@ -72,6 +73,8 @@ const SYSTEM_TABLES = [
72
73
  // asks for the ids it found in OPFS), so it has to exist up front like the
73
74
  // rest — otherwise the very first reconcile throws and the cache runs cold.
74
75
  '_00_blob',
76
+ // CRDT documents per open field, with edits not yet in the outbox.
77
+ '_00_crdt_local',
75
78
  // Server-written, synced-down meta tables (see meta_tables_client.surql).
76
79
  // DEFINE is a noop on this engine, so without seeding them here their synced
77
80
  // rows have no local table to land in: feature flags silently fall back to
@@ -898,6 +901,31 @@ export class SqliteCacheEngine implements LocalStore {
898
901
  return out;
899
902
  }
900
903
 
904
+ /**
905
+ * Reset the cached `_00_rv` of every table whose column set changed since
906
+ * the last boot, so its rows are fetched again whole. See
907
+ * `sqlite-column-reset.ts`. Returns the tables reset.
908
+ */
909
+ async resetChangedTables(fingerprints: Record<string, string>, keep: Set<string>): Promise<string[]> {
910
+ const { rows } = await this.call<{ rows: { data: unknown }[] }>('exec', {
911
+ sql: `SELECT data FROM "_00_schema" WHERE id = ?`,
912
+ bind: [COLUMNS_ROW_ID],
913
+ });
914
+ let stored: Record<string, string> | null = null;
915
+ try {
916
+ const raw = rows?.[0]?.data;
917
+ if (typeof raw === 'string') stored = JSON.parse(raw) as Record<string, string>;
918
+ } catch {
919
+ stored = null;
920
+ }
921
+ const { rows: tables } = await this.call<{ rows: { name: string }[] }>('exec', {
922
+ sql: `SELECT name FROM sqlite_master WHERE type = 'table'`,
923
+ });
924
+ const plan = planColumnReset(stored, fingerprints, new Set((tables ?? []).map((t) => t.name)), keep);
925
+ if (plan.statements.length > 0) await this.call('batch', plan.statements);
926
+ return plan.changed;
927
+ }
928
+
901
929
  async getSnapshot(key: string): Promise<StoredSnapshot | null> {
902
930
  const { rows } = await this.call<{ rows: { id: string; data: unknown }[] }>('exec', {
903
931
  sql: `SELECT id, data FROM "${SNAPSHOT_TABLE}" WHERE id IN (?, ?)`,
@@ -0,0 +1,100 @@
1
+ import { describe, it, expect, beforeEach, afterEach } from 'vitest';
2
+ import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
3
+ import { COLUMNS_ROW_ID, columnFingerprints, planColumnReset } from './sqlite-column-reset';
4
+
5
+ /**
6
+ * A body landed under an older schema lacks the columns that schema did not
7
+ * know, at the server's version. Without a reset the next build never fetches
8
+ * it again (WhitePawn's puzzles stuck on "Preparing" for want of `layout`).
9
+ * These run the planned statements against a REAL in-memory SQLite, the same
10
+ * build the worker loads.
11
+ */
12
+
13
+ let db: any;
14
+
15
+ const rows = (sql: string, bind?: unknown[]) =>
16
+ db.exec({ sql, bind, rowMode: 'object', returnValue: 'resultRows' }) as Record<string, unknown>[];
17
+
18
+ const versions = (table: string) =>
19
+ Object.fromEntries(
20
+ rows(`SELECT id, json_extract(data, '$._00_rv') AS rv FROM "${table}"`).map((r) => [r.id, r.rv])
21
+ );
22
+
23
+ const storedSets = () => {
24
+ const r = rows(`SELECT data FROM "_00_schema" WHERE id = ?`, [COLUMNS_ROW_ID]);
25
+ return r[0] ? (JSON.parse(r[0].data as string) as Record<string, string>) : null;
26
+ };
27
+
28
+ const existing = () => new Set(rows(`SELECT name FROM sqlite_master WHERE type = 'table'`).map((r) => r.name as string));
29
+
30
+ function run(current: Record<string, string>, keep: string[] = []) {
31
+ const plan = planColumnReset(storedSets(), current, existing(), keep);
32
+ db.exec({ sql: 'BEGIN' });
33
+ for (const s of plan.statements) db.exec({ sql: s.sql, bind: s.bind });
34
+ db.exec({ sql: 'COMMIT' });
35
+ return plan;
36
+ }
37
+
38
+ beforeEach(async () => {
39
+ const sqlite3: any = await sqlite3InitModule();
40
+ db = new sqlite3.oo1.DB(':memory:', 'c');
41
+ for (const t of ['_00_schema', 'puzzle', 'game']) {
42
+ db.exec({ sql: `CREATE TABLE "${t}" (id TEXT PRIMARY KEY, data TEXT NOT NULL)` });
43
+ }
44
+ const put = (t: string, id: string, body: object) =>
45
+ db.exec({ sql: `INSERT INTO "${t}"(id, data) VALUES(?, ?)`, bind: [id, JSON.stringify(body)] });
46
+ // Landed by the old build: no `layout`, at the server's version 6.
47
+ put('puzzle', 'puzzle:a', { title: 'A', _00_rv: 6 });
48
+ put('puzzle', 'puzzle:b', { title: 'B', _00_rv: 6 });
49
+ put('game', 'game:1', { white: 'x', _00_rv: 3 });
50
+ });
51
+
52
+ afterEach(() => db?.close());
53
+
54
+ const OLD = { puzzle: 'title', game: 'white' };
55
+ const NEW = { puzzle: 'layout,title', game: 'white' };
56
+
57
+ describe('column reset on the SQLite cache', () => {
58
+ it('fingerprints are the sorted declared columns', () => {
59
+ expect(columnFingerprints([{ name: 'puzzle', columns: { title: {}, layout: {} } }])).toEqual({
60
+ puzzle: 'layout,title',
61
+ });
62
+ });
63
+
64
+ it('first boot with no recorded sets resets every cached table once', () => {
65
+ const plan = run(NEW);
66
+ expect(plan.changed.sort()).toEqual(['game', 'puzzle']);
67
+ expect(versions('puzzle')).toEqual({ 'puzzle:a': 0, 'puzzle:b': 0 });
68
+ expect(versions('game')).toEqual({ 'game:1': 0 });
69
+ expect(storedSets()).toEqual(NEW);
70
+ // Bodies stay: they keep rendering until the refetch lands.
71
+ expect(rows(`SELECT data FROM puzzle WHERE id = 'puzzle:a'`)[0].data).toContain('"title":"A"');
72
+
73
+ // The next boot of the same build changes nothing.
74
+ const again = run(NEW);
75
+ expect(again.statements).toEqual([]);
76
+ });
77
+
78
+ it('a new column resets only that table', () => {
79
+ run(OLD);
80
+ db.exec({ sql: `UPDATE puzzle SET data = json_set(data, '$._00_rv', 6)` });
81
+ db.exec({ sql: `UPDATE game SET data = json_set(data, '$._00_rv', 3)` });
82
+
83
+ const plan = run(NEW);
84
+ expect(plan.changed).toEqual(['puzzle']);
85
+ expect(versions('puzzle')).toEqual({ 'puzzle:a': 0, 'puzzle:b': 0 });
86
+ expect(versions('game')).toEqual({ 'game:1': 3 });
87
+ });
88
+
89
+ it('keeps the version of a row with an unsettled local write', () => {
90
+ const plan = run(NEW, ['puzzle:b', 'game:9']);
91
+ expect(plan.changed).toContain('puzzle');
92
+ expect(versions('puzzle')).toEqual({ 'puzzle:a': 0, 'puzzle:b': 6 });
93
+ });
94
+
95
+ it('records a table the store has no rows for without touching it', () => {
96
+ const plan = run({ ...NEW, puzzle_chapter: 'puzzle,sort,title' });
97
+ expect(plan.changed).not.toContain('puzzle_chapter');
98
+ expect(storedSets()).toEqual({ ...NEW, puzzle_chapter: 'puzzle,sort,title' });
99
+ });
100
+ });
@@ -0,0 +1,91 @@
1
+ /**
2
+ * Cached bodies outlive a schema change on the SQLite engine, and that loses
3
+ * columns for good.
4
+ *
5
+ * A body is landed through `cleanRecord`, which keeps only the columns the
6
+ * running app's schema declares, and is stored at the server's `_00_rv`. The
7
+ * engine is schemaless (no migrator, nothing is recreated when the schema
8
+ * changes), so a tab on an older build that fetched a row after the server
9
+ * added and backfilled a column stored it WITHOUT that column at the new
10
+ * version. The next build knows the column, but its local version equals the
11
+ * server's, so the fetch plan never asks for the row again and the column
12
+ * stays missing until something else bumps the row. (WhitePawn, 2026-10-11:
13
+ * every puzzle read as unconverted because `layout` never arrived.)
14
+ *
15
+ * So the store remembers each table's column set (`_00_schema`, row
16
+ * `columns`). On boot, before the prime reads the versions, a table whose set
17
+ * changed, or every table when no set was recorded yet, gets its cached
18
+ * `_00_rv` reset to 0. The bodies stay and keep rendering; any view that names
19
+ * one of those rows sees it as stale and fetches it again, whole. Rows with an
20
+ * unsettled local write keep their version: their echo is a newer body anyway.
21
+ */
22
+
23
+ /** `_00_schema` row holding the recorded column sets. */
24
+ export const COLUMNS_ROW_ID = 'columns';
25
+
26
+ export interface ColumnResetPlan {
27
+ /** Tables with cached rows whose versions are reset. */
28
+ changed: string[];
29
+ /** One transaction: the resets, then the new column sets. Empty when nothing changed. */
30
+ statements: { sql: string; bind?: unknown[] }[];
31
+ }
32
+
33
+ /**
34
+ * The column set of every table, as `cleanRecord` sees it: the sorted declared
35
+ * column names. Plain names rather than a hash, so a stored set says what it is.
36
+ */
37
+ export function columnFingerprints(
38
+ tables: readonly { name: string; columns: Record<string, unknown> }[]
39
+ ): Record<string, string> {
40
+ const out: Record<string, string> = {};
41
+ for (const t of tables) out[t.name] = Object.keys(t.columns).sort().join(',');
42
+ return out;
43
+ }
44
+
45
+ const sameSets = (a: Record<string, string> | null, b: Record<string, string>): boolean => {
46
+ if (!a) return false;
47
+ const ka = Object.keys(a);
48
+ return ka.length === Object.keys(b).length && ka.every((k) => a[k] === b[k]);
49
+ };
50
+
51
+ /**
52
+ * What to run for the column sets `current`, given the `stored` ones (null when
53
+ * none were recorded), the tables that exist in the store, and the encoded ids
54
+ * (`table:id`) with an unsettled local write.
55
+ */
56
+ export function planColumnReset(
57
+ stored: Record<string, string> | null,
58
+ current: Record<string, string>,
59
+ existingTables: ReadonlySet<string>,
60
+ keep: Iterable<string>
61
+ ): ColumnResetPlan {
62
+ if (sameSets(stored, current)) return { changed: [], statements: [] };
63
+
64
+ const keepByTable = new Map<string, string[]>();
65
+ for (const id of keep) {
66
+ const table = id.slice(0, id.indexOf(':'));
67
+ if (!keepByTable.has(table)) keepByTable.set(table, []);
68
+ keepByTable.get(table)!.push(id);
69
+ }
70
+
71
+ const changed: string[] = [];
72
+ const statements: ColumnResetPlan['statements'] = [];
73
+ for (const [table, set] of Object.entries(current)) {
74
+ if (stored && stored[table] === set) continue;
75
+ if (!existingTables.has(table)) continue;
76
+ changed.push(table);
77
+ const quoted = `"${table.replace(/"/g, '""')}"`;
78
+ const reset = `UPDATE ${quoted} SET data = json_set(data, '$._00_rv', 0) WHERE json_extract(data, '$._00_rv') != 0`;
79
+ const kept = keepByTable.get(table);
80
+ statements.push(
81
+ kept
82
+ ? { sql: `${reset} AND id NOT IN (SELECT value FROM json_each(?))`, bind: [JSON.stringify(kept)] }
83
+ : { sql: reset }
84
+ );
85
+ }
86
+ statements.push({
87
+ sql: `INSERT OR REPLACE INTO "_00_schema"(id, data) VALUES (?, ?)`,
88
+ bind: [COLUMNS_ROW_ID, JSON.stringify(current)],
89
+ });
90
+ return { changed, statements };
91
+ }
@@ -87,6 +87,10 @@ export interface CircuitPrimeContext {
87
87
  /** Encoded ids with an unsettled local mutation (their `_00_rv` was bumped
88
88
  * locally and may exceed the server's next version). */
89
89
  pendingIds: Set<string>;
90
+ /** Each table's client column set (`columnFingerprints`). An engine that
91
+ * keeps bodies across schema changes resets the versions of a table whose
92
+ * set changed before they are read, so its rows are fetched again whole. */
93
+ columnFingerprints?: Record<string, string>;
90
94
  /** Receives every `(id, rv)` the prime put into the circuit, per table, so
91
95
  * the sync layer can skip re-downloading bodies it already has. */
92
96
  onVersions?: (table: string, entries: [string, number][]) => void;
@@ -507,6 +511,26 @@ export class StreamProcessorService {
507
511
  this.processor === processor && generation === this.stateGeneration && epoch === this.db.epoch;
508
512
  const t0 = performance.now();
509
513
 
514
+ // Before anything reads a version: a table whose columns changed since its
515
+ // bodies were cached reads as stale from here on (see sqlite-column-reset).
516
+ if (ctx.columnFingerprints && typeof this.db.resetChangedTables === 'function') {
517
+ try {
518
+ const reset = await this.db.resetChangedTables(ctx.columnFingerprints, ctx.pendingIds);
519
+ if (!alive()) return;
520
+ if (reset.length > 0) {
521
+ this.logger.info(
522
+ { tables: reset, Category: 'sp00ky-client::StreamProcessorService::primeFromLocal' },
523
+ 'Columns changed since these rows were cached; they will be fetched again'
524
+ );
525
+ }
526
+ } catch (e) {
527
+ this.logger.warn(
528
+ { error: e, Category: 'sp00ky-client::StreamProcessorService::primeFromLocal' },
529
+ 'Column change check failed; cached versions kept'
530
+ );
531
+ }
532
+ }
533
+
510
534
  let restored = false;
511
535
  if (
512
536
  this.persistCircuit &&
@@ -106,6 +106,42 @@ describe('StreamProcessorService.primeFromLocal', () => {
106
106
  await expect(service.whenPrimed()).resolves.toBeUndefined();
107
107
  });
108
108
 
109
+ it('resets tables whose columns changed BEFORE reading the versions it reports', async () => {
110
+ const { service, db } = makeService();
111
+ const order: string[] = [];
112
+ // The engine zeroes `thing`'s cached versions (a column was added).
113
+ const reset = vi.fn(async () => {
114
+ order.push('reset');
115
+ db.scanVersions.mockImplementation(async (tables: string[]) => {
116
+ order.push('scan');
117
+ return Object.fromEntries(tables.map((t) => [t, t === 'thing' ? [['thing:a', 0], ['thing:b', 0]] : [['other:x', 5]]]));
118
+ });
119
+ return ['thing'];
120
+ });
121
+ (db as any).resetChangedTables = reset;
122
+ await service.init();
123
+ const c = { ...ctx(), pendingIds: new Set(['thing:b']), columnFingerprints: { thing: 'layout,title', other: 'title' } };
124
+ await service.primeFromLocal(c);
125
+ expect(reset).toHaveBeenCalledWith({ thing: 'layout,title', other: 'title' }, new Set(['thing:b']));
126
+ expect(order).toEqual(['reset', 'scan']);
127
+ // Reported as stale, so the fetch plan pulls the bodies again.
128
+ expect(c.onVersions).toHaveBeenCalledWith('thing', [['thing:a', 0]]);
129
+ });
130
+
131
+ it('keeps priming when the column check fails', async () => {
132
+ const { service, db } = makeService();
133
+ (db as any).resetChangedTables = vi.fn(async () => {
134
+ throw new Error('worker gone');
135
+ });
136
+ await service.init();
137
+ const c = { ...ctx(), columnFingerprints: { thing: 'title' } };
138
+ await service.primeFromLocal(c);
139
+ expect(c.onVersions).toHaveBeenCalledWith('thing', [
140
+ ['thing:a', 1],
141
+ ['thing:b', 2],
142
+ ]);
143
+ });
144
+
109
145
  it('with a snapshot, restores it under the registered views and only fetches what reconcile asks for', async () => {
110
146
  const { service, db, received } = makeService({ snapshot: { bytes: new Uint8Array([7]), meta: goodMeta } });
111
147
  await service.init();
@@ -37,7 +37,7 @@ export function fakeServiceBundle<S extends SchemaStructure>(over: Partial<Servi
37
37
  persistence: { set: async () => undefined, get: async () => null, remove: async () => undefined },
38
38
  streamProcessor: { addReceiver: (r: unknown) => void receivers.push(r), checkpoint: async () => undefined, dispose: noop } as any,
39
39
  migrator: {} as any,
40
- crdt: { closeAll: noop, dispose: noop, open: async () => 'field', close: noop } as any,
40
+ crdt: { attach: noop, setSessionId: noop, closeAll: noop, dispose: noop, open: async () => 'field', close: async () => {} } as any,
41
41
  auth: { subscribe: (cb: (u: string | null) => void) => (authListeners.push(cb), noop), subscribeImpersonation: () => noop, onBeforeSignOut: () => noop, currentUser: null, isAuthenticated: false, token: null, impersonation: null, eventSystem: { subscribe: noop } } as any,
42
42
  tabs: null,
43
43
  tabsUnsupportedReason: 'test',
package/src/types.ts CHANGED
@@ -259,11 +259,11 @@ export interface Sp00kyConfig<S extends SchemaStructure> {
259
259
  streamDebounceTime?: number;
260
260
  /**
261
261
  * Debounce time in milliseconds for syncing collaborative (CRDT) field
262
- * changes to the remote database. Local writes happen immediately on
263
- * every keystroke (so reload/offline works), but the remote UPSERT is
264
- * coalesced over this window. Lower = snappier remote propagation +
265
- * more network traffic; higher = less traffic + more lag for other
266
- * collaborators. Defaults to 500ms.
262
+ * edits to the server. Local state is kept on every change (so reload and
263
+ * offline work), but a field's own new ops go out as one edit-log row per
264
+ * pause of this length (at most 2 s apart while typing continues).
265
+ * Lower = snappier propagation and more rows; higher = fewer rows and more
266
+ * lag for other collaborators. Defaults to 300ms.
267
267
  */
268
268
  crdtDebounceMs?: number;
269
269
  /**
@@ -321,7 +321,8 @@ export interface Sp00kyConfig<S extends SchemaStructure> {
321
321
  * page gets live `useQuery` updates over world-readable tables. Requires the
322
322
  * server to be deployed with `anonymousLiveQueries: true` in `sp00ky.yml`
323
323
  * (this flag must match it). Defaults to `false`: anonymous clients can read
324
- * one-shot but never sync live.
324
+ * one-shot but never sync live, and a `preload` made while signed out returns
325
+ * at once rather than wait for membership that never comes.
325
326
  */
326
327
  enableAnonymousLiveQueries?: boolean;
327
328
  /**
@@ -120,10 +120,9 @@ export const surql: SurqlHelper = {
120
120
  return `UPSERT ONLY $${idVar} REPLACE $${dataVar}`;
121
121
  },
122
122
 
123
- // Use this for sync-down ingestion: the remote payload omits local-only
124
- // fields injected by the CLI's local-schema setup (`_00_crdt`,
125
- // `_00_cursor`), and REPLACE would wipe them on every round-trip and
126
- // break offline reload of CRDT state.
123
+ // Use this for sync-down ingestion: the remote payload can omit local-only
124
+ // fields injected by the CLI's local-schema setup, and REPLACE would wipe
125
+ // them on every round-trip.
127
126
  upsertMerge(idVar: string, dataVar: string) {
128
127
  return `UPSERT ONLY $${idVar} MERGE $${dataVar}`;
129
128
  },