@spooky-sync/core 0.0.1-canary.321 → 0.0.1-canary.323

package/src/services/database/sqlite-column-reset.integration.test.ts

@@ -0,0 +1,100 @@
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
+import { COLUMNS_ROW_ID, columnFingerprints, planColumnReset } from './sqlite-column-reset';
+
+/**
+ * A body landed under an older schema lacks the columns that schema did not
+ * know, at the server's version. Without a reset the next build never fetches
+ * it again (WhitePawn's puzzles stuck on "Preparing" for want of `layout`).
+ * These run the planned statements against a REAL in-memory SQLite, the same
+ * build the worker loads.
+ */
+
+let db: any;
+
+const rows = (sql: string, bind?: unknown[]) =>
+  db.exec({ sql, bind, rowMode: 'object', returnValue: 'resultRows' }) as Record<string, unknown>[];
+
+const versions = (table: string) =>
+  Object.fromEntries(
+    rows(`SELECT id, json_extract(data, '$._00_rv') AS rv FROM "${table}"`).map((r) => [r.id, r.rv])
+  );
+
+const storedSets = () => {
+  const r = rows(`SELECT data FROM "_00_schema" WHERE id = ?`, [COLUMNS_ROW_ID]);
+  return r[0] ? (JSON.parse(r[0].data as string) as Record<string, string>) : null;
+};
+
+const existing = () => new Set(rows(`SELECT name FROM sqlite_master WHERE type = 'table'`).map((r) => r.name as string));
+
+function run(current: Record<string, string>, keep: string[] = []) {
+  const plan = planColumnReset(storedSets(), current, existing(), keep);
+  db.exec({ sql: 'BEGIN' });
+  for (const s of plan.statements) db.exec({ sql: s.sql, bind: s.bind });
+  db.exec({ sql: 'COMMIT' });
+  return plan;
+}
+
+beforeEach(async () => {
+  const sqlite3: any = await sqlite3InitModule();
+  db = new sqlite3.oo1.DB(':memory:', 'c');
+  for (const t of ['_00_schema', 'puzzle', 'game']) {
+    db.exec({ sql: `CREATE TABLE "${t}" (id TEXT PRIMARY KEY, data TEXT NOT NULL)` });
+  }
+  const put = (t: string, id: string, body: object) =>
+    db.exec({ sql: `INSERT INTO "${t}"(id, data) VALUES(?, ?)`, bind: [id, JSON.stringify(body)] });
+  // Landed by the old build: no `layout`, at the server's version 6.
+  put('puzzle', 'puzzle:a', { title: 'A', _00_rv: 6 });
+  put('puzzle', 'puzzle:b', { title: 'B', _00_rv: 6 });
+  put('game', 'game:1', { white: 'x', _00_rv: 3 });
+});
+
+afterEach(() => db?.close());
+
+const OLD = { puzzle: 'title', game: 'white' };
+const NEW = { puzzle: 'layout,title', game: 'white' };
+
+describe('column reset on the SQLite cache', () => {
+  it('fingerprints are the sorted declared columns', () => {
+    expect(columnFingerprints([{ name: 'puzzle', columns: { title: {}, layout: {} } }])).toEqual({
+      puzzle: 'layout,title',
+    });
+  });
+
+  it('first boot with no recorded sets resets every cached table once', () => {
+    const plan = run(NEW);
+    expect(plan.changed.sort()).toEqual(['game', 'puzzle']);
+    expect(versions('puzzle')).toEqual({ 'puzzle:a': 0, 'puzzle:b': 0 });
+    expect(versions('game')).toEqual({ 'game:1': 0 });
+    expect(storedSets()).toEqual(NEW);
+    // Bodies stay: they keep rendering until the refetch lands.
+    expect(rows(`SELECT data FROM puzzle WHERE id = 'puzzle:a'`)[0].data).toContain('"title":"A"');
+
+    // The next boot of the same build changes nothing.
+    const again = run(NEW);
+    expect(again.statements).toEqual([]);
+  });
+
+  it('a new column resets only that table', () => {
+    run(OLD);
+    db.exec({ sql: `UPDATE puzzle SET data = json_set(data, '$._00_rv', 6)` });
+    db.exec({ sql: `UPDATE game SET data = json_set(data, '$._00_rv', 3)` });
+
+    const plan = run(NEW);
+    expect(plan.changed).toEqual(['puzzle']);
+    expect(versions('puzzle')).toEqual({ 'puzzle:a': 0, 'puzzle:b': 0 });
+    expect(versions('game')).toEqual({ 'game:1': 3 });
+  });
+
+  it('keeps the version of a row with an unsettled local write', () => {
+    const plan = run(NEW, ['puzzle:b', 'game:9']);
+    expect(plan.changed).toContain('puzzle');
+    expect(versions('puzzle')).toEqual({ 'puzzle:a': 0, 'puzzle:b': 6 });
+  });
+
+  it('records a table the store has no rows for without touching it', () => {
+    const plan = run({ ...NEW, puzzle_chapter: 'puzzle,sort,title' });
+    expect(plan.changed).not.toContain('puzzle_chapter');
+    expect(storedSets()).toEqual({ ...NEW, puzzle_chapter: 'puzzle,sort,title' });
+  });
+});

package/src/services/database/sqlite-column-reset.ts

@@ -0,0 +1,91 @@
+/**
+ * Cached bodies outlive a schema change on the SQLite engine, and that loses
+ * columns for good.
+ *
+ * A body is landed through `cleanRecord`, which keeps only the columns the
+ * running app's schema declares, and is stored at the server's `_00_rv`. The
+ * engine is schemaless (no migrator, nothing is recreated when the schema
+ * changes), so a tab on an older build that fetched a row after the server
+ * added and backfilled a column stored it WITHOUT that column at the new
+ * version. The next build knows the column, but its local version equals the
+ * server's, so the fetch plan never asks for the row again and the column
+ * stays missing until something else bumps the row. (WhitePawn, 2026-10-11:
+ * every puzzle read as unconverted because `layout` never arrived.)
+ *
+ * So the store remembers each table's column set (`_00_schema`, row
+ * `columns`). On boot, before the prime reads the versions, a table whose set
+ * changed, or every table when no set was recorded yet, gets its cached
+ * `_00_rv` reset to 0. The bodies stay and keep rendering; any view that names
+ * one of those rows sees it as stale and fetches it again, whole. Rows with an
+ * unsettled local write keep their version: their echo is a newer body anyway.
+ */
+
+/** `_00_schema` row holding the recorded column sets. */
+export const COLUMNS_ROW_ID = 'columns';
+
+export interface ColumnResetPlan {
+  /** Tables with cached rows whose versions are reset. */
+  changed: string[];
+  /** One transaction: the resets, then the new column sets. Empty when nothing changed. */
+  statements: { sql: string; bind?: unknown[] }[];
+}
+
+/**
+ * The column set of every table, as `cleanRecord` sees it: the sorted declared
+ * column names. Plain names rather than a hash, so a stored set says what it is.
+ */
+export function columnFingerprints(
+  tables: readonly { name: string; columns: Record<string, unknown> }[]
+): Record<string, string> {
+  const out: Record<string, string> = {};
+  for (const t of tables) out[t.name] = Object.keys(t.columns).sort().join(',');
+  return out;
+}
+
+const sameSets = (a: Record<string, string> | null, b: Record<string, string>): boolean => {
+  if (!a) return false;
+  const ka = Object.keys(a);
+  return ka.length === Object.keys(b).length && ka.every((k) => a[k] === b[k]);
+};
+
+/**
+ * What to run for the column sets `current`, given the `stored` ones (null when
+ * none were recorded), the tables that exist in the store, and the encoded ids
+ * (`table:id`) with an unsettled local write.
+ */
+export function planColumnReset(
+  stored: Record<string, string> | null,
+  current: Record<string, string>,
+  existingTables: ReadonlySet<string>,
+  keep: Iterable<string>
+): ColumnResetPlan {
+  if (sameSets(stored, current)) return { changed: [], statements: [] };
+
+  const keepByTable = new Map<string, string[]>();
+  for (const id of keep) {
+    const table = id.slice(0, id.indexOf(':'));
+    if (!keepByTable.has(table)) keepByTable.set(table, []);
+    keepByTable.get(table)!.push(id);
+  }
+
+  const changed: string[] = [];
+  const statements: ColumnResetPlan['statements'] = [];
+  for (const [table, set] of Object.entries(current)) {
+    if (stored && stored[table] === set) continue;
+    if (!existingTables.has(table)) continue;
+    changed.push(table);
+    const quoted = `"${table.replace(/"/g, '""')}"`;
+    const reset = `UPDATE ${quoted} SET data = json_set(data, '$._00_rv', 0) WHERE json_extract(data, '$._00_rv') != 0`;
+    const kept = keepByTable.get(table);
+    statements.push(
+      kept
+        ? { sql: `${reset} AND id NOT IN (SELECT value FROM json_each(?))`, bind: [JSON.stringify(kept)] }
+        : { sql: reset }
+    );
+  }
+  statements.push({
+    sql: `INSERT OR REPLACE INTO "_00_schema"(id, data) VALUES (?, ?)`,
+    bind: [COLUMNS_ROW_ID, JSON.stringify(current)],
+  });
+  return { changed, statements };
+}

package/src/services/stream-processor/index.ts

@@ -87,6 +87,10 @@ export interface CircuitPrimeContext {
   /** Encoded ids with an unsettled local mutation (their `_00_rv` was bumped
    *  locally and may exceed the server's next version). */
   pendingIds: Set<string>;
+  /** Each table's client column set (`columnFingerprints`). An engine that
+   *  keeps bodies across schema changes resets the versions of a table whose
+   *  set changed before they are read, so its rows are fetched again whole. */
+  columnFingerprints?: Record<string, string>;
   /** Receives every `(id, rv)` the prime put into the circuit, per table, so
    *  the sync layer can skip re-downloading bodies it already has. */
   onVersions?: (table: string, entries: [string, number][]) => void;
@@ -507,6 +511,26 @@ export class StreamProcessorService {
       this.processor === processor && generation === this.stateGeneration && epoch === this.db.epoch;
     const t0 = performance.now();
 
+    // Before anything reads a version: a table whose columns changed since its
+    // bodies were cached reads as stale from here on (see sqlite-column-reset).
+    if (ctx.columnFingerprints && typeof this.db.resetChangedTables === 'function') {
+      try {
+        const reset = await this.db.resetChangedTables(ctx.columnFingerprints, ctx.pendingIds);
+        if (!alive()) return;
+        if (reset.length > 0) {
+          this.logger.info(
+            { tables: reset, Category: 'sp00ky-client::StreamProcessorService::primeFromLocal' },
+            'Columns changed since these rows were cached; they will be fetched again'
+          );
+        }
+      } catch (e) {
+        this.logger.warn(
+          { error: e, Category: 'sp00ky-client::StreamProcessorService::primeFromLocal' },
+          'Column change check failed; cached versions kept'
+        );
+      }
+    }
+
     let restored = false;
     if (
       this.persistCircuit &&

package/src/services/stream-processor/stream-processor.prime.test.ts

@@ -106,6 +106,42 @@ describe('StreamProcessorService.primeFromLocal', () => {
     await expect(service.whenPrimed()).resolves.toBeUndefined();
   });
 
+  it('resets tables whose columns changed BEFORE reading the versions it reports', async () => {
+    const { service, db } = makeService();
+    const order: string[] = [];
+    // The engine zeroes `thing`'s cached versions (a column was added).
+    const reset = vi.fn(async () => {
+      order.push('reset');
+      db.scanVersions.mockImplementation(async (tables: string[]) => {
+        order.push('scan');
+        return Object.fromEntries(tables.map((t) => [t, t === 'thing' ? [['thing:a', 0], ['thing:b', 0]] : [['other:x', 5]]]));
+      });
+      return ['thing'];
+    });
+    (db as any).resetChangedTables = reset;
+    await service.init();
+    const c = { ...ctx(), pendingIds: new Set(['thing:b']), columnFingerprints: { thing: 'layout,title', other: 'title' } };
+    await service.primeFromLocal(c);
+    expect(reset).toHaveBeenCalledWith({ thing: 'layout,title', other: 'title' }, new Set(['thing:b']));
+    expect(order).toEqual(['reset', 'scan']);
+    // Reported as stale, so the fetch plan pulls the bodies again.
+    expect(c.onVersions).toHaveBeenCalledWith('thing', [['thing:a', 0]]);
+  });
+
+  it('keeps priming when the column check fails', async () => {
+    const { service, db } = makeService();
+    (db as any).resetChangedTables = vi.fn(async () => {
+      throw new Error('worker gone');
+    });
+    await service.init();
+    const c = { ...ctx(), columnFingerprints: { thing: 'title' } };
+    await service.primeFromLocal(c);
+    expect(c.onVersions).toHaveBeenCalledWith('thing', [
+      ['thing:a', 1],
+      ['thing:b', 2],
+    ]);
+  });
+
   it('with a snapshot, restores it under the registered views and only fetches what reconcile asks for', async () => {
     const { service, db, received } = makeService({ snapshot: { bytes: new Uint8Array([7]), meta: goodMeta } });
     await service.init();

package/src/state/client-state.ts

@@ -114,6 +114,13 @@ export interface ClientState {
   readonly userId: string | null;
   /** The principal the current session salt was minted for. */
   readonly saltUserId: string | null;
+  /**
+   * This page's secret, sent with every registration (`$config.owner`). An
+   * anonymous view answers only to the secret it was registered with, so it
+   * lives as long as the page's views: a reconnect keeps it, and a reload
+   * that keeps the salt keeps it too.
+   */
+  readonly ownerSecret: string | null;
   /** Latest bucket-switch target; an older switch that wakes up to a newer target skips. */
   readonly pendingBucket: string | null;
   readonly tabId: string;
@@ -167,6 +174,7 @@ export function emptyState(init: { tabId: string }): ClientState {
     sessionId: null,
     userId: null,
     saltUserId: null,
+    ownerSecret: null,
     pendingBucket: null,
     tabId: init.tabId,
     tabRole: 'solo',

package/src/state/reducers.ts

@@ -444,7 +444,10 @@ export const setFailedCount =
 export const setIdentity =
   (
     patch: Partial<
-      Pick<ClientState, 'sessionId' | 'userId' | 'saltUserId' | 'pendingBucket' | 'bucketId' | 'tabRole' | 'localReady' | 'primed'>
+      Pick<
+        ClientState,
+        'sessionId' | 'userId' | 'saltUserId' | 'ownerSecret' | 'pendingBucket' | 'bucketId' | 'tabRole' | 'localReady' | 'primed'
+      >
     >
   ): Reducer =>
   (s) => ({ ...s, ...patch });

package/src/testing/adapters.ts

@@ -78,7 +78,11 @@ export function fakeAdapters(over: {
       clear: (key) => void timers.pending.delete(key),
     },
     clock: { now: () => now },
-    ids: { mutation: () => `_00_pending_mutations:${String(++ids).padStart(13, '0')}_0001_tab`, salt: () => `salt-${++ids}` },
+    ids: {
+      mutation: () => `_00_pending_mutations:${String(++ids).padStart(13, '0')}_0001_tab`,
+      salt: () => `salt-${++ids}`,
+      owner: () => `owner-${++ids}`,
+    },
     hash: sha256Hex,
     services,
   };

package/src/utils/index.ts

@@ -51,6 +51,17 @@ export function generateId(): string {
   return Uuid.v4().toString().replace(/-/g, '');
 }
 
+/**
+ * 32 random bytes, hex: the secret a page proves it owns its anonymous views
+ * with (`$config.owner` on `fn::query::register`). `getRandomValues` rather
+ * than `randomUUID`, which a page served over plain http does not have.
+ */
+export function mintOwnerSecret(): string {
+  const bytes = new Uint8Array(32);
+  globalThis.crypto.getRandomValues(bytes);
+  return Array.from(bytes, (b) => b.toString(16).padStart(2, '0')).join('');
+}
+
 export function generateNewTableId<S extends SchemaStructure, T extends TableNames<S>>(
   tableName: T
 ): RecordId {