@spooky-sync/core 0.0.1-canary.21 → 0.0.1-canary.211

package/src/modules/data/data.settled-writes.test.ts

@@ -0,0 +1,206 @@
+import { describe, it, expect, vi, afterEach } from 'vitest';
+import { RecordId } from 'surrealdb';
+import { DataModule } from './index';
+import type { QueryPlan } from '@spooky-sync/query-builder';
+import type { QueryState, RecordVersionArray } from '../../types';
+
+/**
+ * A write is visible before the server confirms it because it sits in the
+ * outbox: the render set is
+ * `(membership ∪ (pendingWrites ∩ localArray)) − pendingDeletes`.
+ *
+ * The outbox row is deleted the instant the push succeeds, but the row does not
+ * enter membership until the SSP has ingested it, materialized the view, written
+ * the `_00_list_ref` edge, and this client has read that back. In between it is
+ * in NEITHER term, so it renders, disappears, and returns.
+ *
+ * Reported from production as "the comment shows up on every other client in
+ * realtime but not on the one that wrote it" — other clients don't blink because
+ * a client that never established membership renders from the predicate scan
+ * instead. The gap is invisible when the round trip is fast and glaring when the
+ * edge path is backed up.
+ */
+
+const noop = () => {};
+
+function makeLogger(): any {
+  const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
+  logger.child = () => logger;
+  return logger;
+}
+
+const schema = { tables: [{ name: 'comment', columns: {} }] } as any;
+const plan: QueryPlan = { table: 'comment', where: [['game', '=', 'game:g1']] } as any;
+
+function makeQueryState(
+  hash: string,
+  opts: { remoteArray?: RecordVersionArray; localArray?: RecordVersionArray } = {}
+): QueryState {
+  return {
+    config: {
+      id: new RecordId('_00_query', hash),
+      surql: 'SELECT * FROM comment WHERE game = $game;',
+      plan,
+      params: { game: 'game:g1' },
+      localArray: opts.localArray ?? [],
+      remoteArray: opts.remoteArray ?? [],
+      membershipKnown: true,
+      ttl: '10m',
+      lastActiveAt: new Date(),
+      tableName: 'comment',
+    },
+    records: [],
+    ttlTimer: null,
+    ttlDurationMs: 0,
+    updateCount: 0,
+    lastUpdatedAt: null,
+    materializationSamples: [],
+    lastIngestLatencyMs: null,
+    errorCount: 0,
+    status: 'idle',
+    phaseSamples: {},
+    phaseLast: {},
+    registrationTimings: { parseMs: null, planMs: null, snapshotMs: null, wallMs: null },
+  } as QueryState;
+}
+
+function setup(
+  stateOpts: Parameters<typeof makeQueryState>[1] = {},
+  pendingRows: Array<{ recordId: RecordId; mutationType: string }> = []
+) {
+  const bodies = new Map(
+    ['old', 'new'].map((k) => [`comment:${k}`, { id: new RecordId('comment', k), body: k }])
+  );
+  const local: any = {
+    epoch: 1,
+    select: vi.fn(async (p: QueryPlan) => {
+      if ((p as any).ids) {
+        return ((p as any).ids as RecordId[])
+          .map((id) => bodies.get(id.toString()))
+          .filter((r): r is NonNullable<typeof r> => r !== undefined);
+      }
+      return Array.from(bodies.values());
+    }),
+    query: vi.fn(async (sql: string) => {
+      if (sql.includes('_00_pending_mutations')) return [pendingRows];
+      if (sql.includes('FROM ONLY')) return [null];
+      return [Array.from(bodies.values())];
+    }),
+    getById: vi.fn(async () => null),
+    upsert: vi.fn(async () => {}),
+  };
+  const dm = new DataModule({ saveBatch: async () => {} } as any, local, schema, makeLogger(), 100);
+  const state = makeQueryState('h1', stateOpts);
+  (dm as any).activeQueries.set('h1', state);
+  return { dm, state };
+}
+
+const ids = (rows: Array<Record<string, any>>) => rows.map((r) => String(r.id));
+const materialize = (dm: DataModule<any>, state: QueryState, ssp?: RecordVersionArray) =>
+  (dm as any).materializeRecords(state, ssp) as Promise<Record<string, any>[]>;
+
+describe('settled-write grace window', () => {
+  afterEach(() => vi.useRealTimers());
+
+  it('keeps an accepted write rendered while membership has not caught up', async () => {
+    // Membership still only knows the old comment; the outbox row for the new
+    // one is already gone because the push succeeded.
+    const { dm, state } = setup({
+      remoteArray: [['comment:old', 1]],
+      localArray: [
+        ['comment:old', 1],
+        ['comment:new', 1],
+      ],
+    });
+
+    dm.noteWriteSettled('comment:new', 'create');
+
+    expect(ids(await materialize(dm, state))).toEqual(['comment:new', 'comment:old']);
+  });
+
+  it('renders nothing extra for a write the server never accepted', async () => {
+    // The rollback path never reports a mutation settled, so a rejected write
+    // has no grace at all. This is what keeps the window from showing rows the
+    // server refused.
+    const { dm, state } = setup({
+      remoteArray: [['comment:old', 1]],
+      localArray: [
+        ['comment:old', 1],
+        ['comment:new', 1],
+      ],
+    });
+
+    expect(ids(await materialize(dm, state))).toEqual(['comment:old']);
+  });
+
+  it('stops granting grace once membership names the row', async () => {
+    const { dm, state } = setup({
+      remoteArray: [['comment:old', 1]],
+      localArray: [
+        ['comment:old', 1],
+        ['comment:new', 1],
+      ],
+    });
+    dm.noteWriteSettled('comment:new', 'create');
+    await materialize(dm, state);
+
+    // Membership catches up; the id must not be double-counted…
+    state.config.remoteArray = [
+      ['comment:old', 1],
+      ['comment:new', 1],
+    ];
+    expect(ids(await materialize(dm, state))).toEqual(['comment:new', 'comment:old']);
+    expect((dm as any).settledWrites.size).toBe(0);
+
+    // …and once the server later drops it from the window, it goes away rather
+    // than being resurrected by a stale grace entry.
+    state.config.remoteArray = [['comment:old', 1]];
+    expect(ids(await materialize(dm, state))).toEqual(['comment:old']);
+  });
+
+  it('expires the grace so a write whose membership never arrives cannot linger', async () => {
+    vi.useFakeTimers();
+    const { dm, state } = setup({
+      remoteArray: [['comment:old', 1]],
+      localArray: [
+        ['comment:old', 1],
+        ['comment:new', 1],
+      ],
+    });
+    dm.noteWriteSettled('comment:new', 'create');
+    expect(ids(await materialize(dm, state))).toContain('comment:new');
+
+    vi.advanceTimersByTime(11_000);
+    expect(ids(await materialize(dm, state))).toEqual(['comment:old']);
+  });
+
+  it('keeps an accepted delete subtracted until membership drops the row', async () => {
+    // The mirror case: membership still lists the row until the SSP publishes
+    // its removal, so without this the deleted row flashes back.
+    const { dm, state } = setup({
+      remoteArray: [
+        ['comment:old', 1],
+        ['comment:new', 1],
+      ],
+      localArray: [['comment:old', 1]],
+    });
+
+    dm.noteWriteSettled('comment:new', 'delete');
+
+    expect(ids(await materialize(dm, state))).toEqual(['comment:old']);
+  });
+
+  it('only grants grace to rows the local view still says match', async () => {
+    // `localArray` is the local SSP's answer to "does this row match the
+    // predicate". A settled write that does NOT match must not be forced into
+    // the render set — same rule the pending-write union already follows.
+    const { dm, state } = setup({
+      remoteArray: [['comment:old', 1]],
+      localArray: [['comment:old', 1]],
+    });
+
+    dm.noteWriteSettled('comment:new', 'create');
+
+    expect(ids(await materialize(dm, state))).toEqual(['comment:old']);
+  });
+});

package/src/modules/data/data.status.test.ts

@@ -0,0 +1,249 @@
+import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest';
+import { RecordId } from 'surrealdb';
+import { DataModule } from './index';
+import type { QueryState, QueryStatus } from '../../types';
+
+/**
+ * Tests for the per-query fetch status (idle/fetching) added to DataModule:
+ * setQueryStatus updates the query object and notifies both the DevTools
+ * observer hook and subscribeStatus listeners; beginFetching/endFetching
+ * refcount overlapping fetch cycles; flushPendingStreamUpdate lands the
+ * debounced result before a query flips back to idle.
+ */
+
+function makeLogger(): any {
+  const noop = () => {};
+  const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
+  logger.child = () => logger;
+  return logger;
+}
+
+function makeDataModule(): DataModule<any> {
+  return new DataModule(
+    {} as any, // cache — unused by status methods
+    {} as any, // local — unused by status methods
+    { tables: [] } as any, // schema — unused by status methods
+    makeLogger(),
+    100
+  );
+}
+
+function makeQueryState(hash: string): QueryState {
+  return {
+    config: {
+      id: new RecordId('_00_query', hash),
+      surql: 'SELECT * FROM user',
+      params: {},
+      localArray: [],
+      remoteArray: [],
+      ttl: '10m',
+      lastActiveAt: new Date(),
+      tableName: 'user',
+    },
+    records: [],
+    ttlTimer: null,
+    ttlDurationMs: 0,
+    updateCount: 0,
+    lastUpdatedAt: null,
+    materializationSamples: [],
+    lastIngestLatencyMs: null,
+    errorCount: 0,
+    status: 'idle',
+    phaseSamples: {},
+    phaseLast: {},
+    registrationTimings: { parseMs: null, planMs: null, snapshotMs: null, wallMs: null },
+  };
+}
+
+describe('DataModule query fetch status', () => {
+  let dm: DataModule<any>;
+  const hash = 'h1';
+
+  beforeEach(() => {
+    dm = makeDataModule();
+    (dm as any).activeQueries.set(hash, makeQueryState(hash));
+  });
+
+  it('subscribeStatus with immediate reports the current status', () => {
+    const seen: QueryStatus[] = [];
+    dm.subscribeStatus(hash, (s) => seen.push(s), { immediate: true });
+    expect(seen).toEqual(['idle']);
+  });
+
+  it('setQueryStatus updates the query object and notifies subscribers + observer', () => {
+    const observed: Array<[string, QueryStatus]> = [];
+    dm.onQueryStatusChange = (h, s) => observed.push([h, s]);
+
+    const seen: QueryStatus[] = [];
+    dm.subscribeStatus(hash, (s) => seen.push(s));
+
+    dm.setQueryStatus(hash, 'fetching');
+    expect((dm as any).activeQueries.get(hash).status).toBe('fetching');
+    expect(seen).toEqual(['fetching']);
+    expect(observed).toEqual([[hash, 'fetching']]);
+
+    dm.setQueryStatus(hash, 'idle');
+    expect(seen).toEqual(['fetching', 'idle']);
+    expect(observed).toEqual([[hash, 'fetching'], [hash, 'idle']]);
+  });
+
+  it('setQueryStatus is a no-op when the status is unchanged', () => {
+    const seen: QueryStatus[] = [];
+    dm.subscribeStatus(hash, (s) => seen.push(s));
+    dm.setQueryStatus(hash, 'idle'); // already idle
+    expect(seen).toEqual([]);
+  });
+
+  it('setQueryStatus is a no-op for an unknown query', () => {
+    let called = false;
+    dm.onQueryStatusChange = () => {
+      called = true;
+    };
+    dm.setQueryStatus('does-not-exist', 'fetching');
+    expect(called).toBe(false);
+  });
+
+  it('unsubscribe stops further status notifications', () => {
+    const seen: QueryStatus[] = [];
+    const unsub = dm.subscribeStatus(hash, (s) => seen.push(s));
+    dm.setQueryStatus(hash, 'fetching');
+    unsub();
+    dm.setQueryStatus(hash, 'idle');
+    expect(seen).toEqual(['fetching']);
+  });
+});
+
+describe('DataModule beginFetching/endFetching refcount', () => {
+  let dm: DataModule<any>;
+  const hash = 'h1';
+  let seen: QueryStatus[];
+
+  beforeEach(() => {
+    dm = makeDataModule();
+    (dm as any).activeQueries.set(hash, makeQueryState(hash));
+    seen = [];
+    dm.subscribeStatus(hash, (s) => seen.push(s));
+  });
+
+  it('emits fetching on 0→1 and idle only on the last exit', () => {
+    dm.beginFetching(hash); // registration
+    dm.beginFetching(hash); // overlapping poll round
+    expect(seen).toEqual(['fetching']);
+
+    dm.endFetching(hash); // inner cycle finishes — must NOT emit idle
+    expect(seen).toEqual(['fetching']);
+    expect((dm as any).activeQueries.get(hash).status).toBe('fetching');
+
+    dm.endFetching(hash);
+    expect(seen).toEqual(['fetching', 'idle']);
+  });
+
+  it('unbalanced endFetching settles to idle without going negative', () => {
+    dm.endFetching(hash); // already idle → no-op notification
+    expect(seen).toEqual([]);
+    dm.beginFetching(hash);
+    dm.endFetching(hash);
+    expect(seen).toEqual(['fetching', 'idle']);
+    expect((dm as any).fetchDepth.has(hash)).toBe(false);
+  });
+});
+
+describe('DataModule pending stream-update flush', () => {
+  let dm: DataModule<any>;
+  const hash = 'h1';
+  let processed: any[];
+
+  const makeUpdate = (op: 'CREATE' | 'UPDATE' | 'DELETE') =>
+    ({ queryHash: hash, op, localArray: [] }) as any;
+
+  beforeEach(() => {
+    vi.useFakeTimers();
+    dm = makeDataModule();
+    (dm as any).activeQueries.set(hash, makeQueryState(hash));
+    processed = [];
+    (dm as any).processStreamUpdate = async (u: any) => {
+      processed.push(u);
+    };
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+  });
+
+  it('flushPendingStreamUpdate processes the debounced update exactly once', async () => {
+    const update = makeUpdate('CREATE');
+    await dm.onStreamUpdate(update);
+    expect(processed).toEqual([]); // still debounced
+
+    await dm.flushPendingStreamUpdate(hash);
+    expect(processed).toEqual([update]);
+    expect((dm as any).debounceTimers.has(hash)).toBe(false);
+    expect((dm as any).pendingStreamUpdates.has(hash)).toBe(false);
+
+    // The cancelled timer must not process it a second time.
+    await vi.runAllTimersAsync();
+    expect(processed).toEqual([update]);
+
+    // Nothing pending → flush is a no-op.
+    await dm.flushPendingStreamUpdate(hash);
+    expect(processed).toEqual([update]);
+  });
+
+  it('the trailing-edge timer clears the pending entry itself', async () => {
+    const update = makeUpdate('UPDATE');
+    await dm.onStreamUpdate(update);
+    await vi.runAllTimersAsync();
+    expect(processed).toEqual([update]);
+    expect((dm as any).pendingStreamUpdates.has(hash)).toBe(false);
+  });
+
+  it('a DELETE supersedes and drops the pending coalesced update', async () => {
+    const create = makeUpdate('CREATE');
+    const del = makeUpdate('DELETE');
+    await dm.onStreamUpdate(create);
+    await dm.onStreamUpdate(del); // immediate, drops the pending CREATE
+    expect(processed).toEqual([del]);
+    expect((dm as any).pendingStreamUpdates.has(hash)).toBe(false);
+    await vi.runAllTimersAsync();
+    expect(processed).toEqual([del]);
+  });
+
+  it('finalizeDeregister clears pending update and fetch depth', async () => {
+    (dm as any).cache = { unregisterQuery: () => {} };
+    await dm.onStreamUpdate(makeUpdate('CREATE'));
+    dm.beginFetching(hash);
+    dm.finalizeDeregister(hash);
+    expect((dm as any).pendingStreamUpdates.has(hash)).toBe(false);
+    expect((dm as any).debounceTimers.has(hash)).toBe(false);
+    expect((dm as any).fetchDepth.has(hash)).toBe(false);
+    await vi.runAllTimersAsync();
+    expect(processed).toEqual([]);
+  });
+});
+
+describe('DataModule notifyQuerySynced', () => {
+  const hash = 'h1';
+
+  function makeDmWithLocal(rows: any[]): DataModule<any> {
+    const local: any = { query: async () => [rows] };
+    return new DataModule({} as any, local, { tables: [] } as any, makeLogger(), 100);
+  }
+
+  it('notifies once per registration lifetime even when records are unchanged and updateCount > 0', async () => {
+    const dm = makeDmWithLocal([]);
+    const state = makeQueryState(hash);
+    state.updateCount = 7; // persisted from a previous registration
+    (dm as any).activeQueries.set(hash, state);
+
+    let notified = 0;
+    dm.subscribe(hash, () => {
+      notified++;
+    });
+
+    await dm.notifyQuerySynced(hash); // empty result, unchanged — must still emit
+    expect(notified).toBe(1);
+
+    await dm.notifyQuerySynced(hash); // already notified this lifetime → silent
+    expect(notified).toBe(1);
+  });
+});

package/src/modules/data/id-set-plan.test.ts

@@ -0,0 +1,122 @@
+import { describe, it, expect } from 'vitest';
+import type { QueryPlan } from '@spooky-sync/query-builder';
+import { buildIdSetPlan, buildIdSetSurql, buildWindowMaterializationPlan } from './window-query';
+import { executeSelect, type SelectDb } from '../../services/database/sqlite-select';
+import type { Row } from '../../services/database/cache-engine';
+
+// `buildIdSetPlan` is what makes membership-authoritative rendering possible on
+// any local engine: it turns a query's plan into "these exact ids, ordered,
+// relations intact". Dropping `where` is the point — re-applying the predicate
+// against stale local bodies is what let removed rows keep rendering.
+
+const BASE: QueryPlan = {
+  table: 'thread',
+  where: [{ field: 'done', op: '=', value: false }],
+  limit: 50,
+  offset: 0,
+  orderBy: [['title', 'asc']],
+  relations: [
+    { alias: 'comments', table: 'comment', cardinality: 'many', foreignKeyField: 'thread' },
+  ],
+} as any;
+
+describe('buildIdSetPlan', () => {
+  it('binds the id-set and drops where/limit/offset, keeping orderBy + relations', () => {
+    const out = buildIdSetPlan(BASE, ['thread:A']);
+    expect(out.ids).toEqual(['thread:A']);
+    expect(out.where).toBeUndefined();
+    expect(out.limit).toBeUndefined();
+    expect(out.offset).toBeUndefined();
+    expect(out.orderBy).toEqual([['title', 'asc']]);
+    expect(out.relations).toEqual(BASE.relations);
+    expect(out.table).toBe('thread');
+  });
+
+  it('applies to non-offset plans, unlike the windowed-only helper', () => {
+    // The windowed helper deliberately bails so the old caller kept its scan;
+    // membership rendering needs the same rewrite for every query.
+    expect(buildWindowMaterializationPlan(BASE, ['thread:A'])).toBeNull();
+    expect(buildIdSetPlan(BASE, ['thread:A']).ids).toEqual(['thread:A']);
+  });
+
+  it('does not mutate the input plan', () => {
+    buildIdSetPlan(BASE, ['thread:A']);
+    expect(BASE.where).toEqual([{ field: 'done', op: '=', value: false }]);
+    expect(BASE.limit).toBe(50);
+  });
+});
+
+describe('buildIdSetSurql', () => {
+  it('rewrites a non-offset query to the id-set source, keeping ORDER BY', () => {
+    const out = buildIdSetSurql('SELECT * FROM thread WHERE done = false ORDER BY title ASC LIMIT 50;');
+    expect(out?.query).toBe('SELECT * FROM $__win ORDER BY title ASC');
+  });
+
+  it('preserves the projection', () => {
+    const out = buildIdSetSurql('SELECT id, title FROM thread WHERE done = false;');
+    expect(out?.query).toBe('SELECT id, title FROM $__win');
+  });
+
+  it('returns null when there is no top-level FROM to replace', () => {
+    expect(buildIdSetSurql('RETURN true;')).toBeNull();
+  });
+});
+
+// ---- Engine parity -------------------------------------------------------
+//
+// Both local engines short-circuit to `selectByIds` when `plan.ids` is set. This
+// pins that the id-set plan really does filter AND still resolves `.related()`
+// on the SQLite engine (the `where`-dropping path the SurrealDB engine shares).
+
+const FIXTURE: Record<string, Row[]> = {
+  thread: [
+    { id: 'thread:A', title: 'a thread', done: false },
+    { id: 'thread:B', title: 'b thread', done: false },
+  ],
+  comment: [
+    { id: 'comment:1', thread: 'thread:A', body: 'on A' },
+    { id: 'comment:2', thread: 'thread:B', body: 'on B' },
+  ],
+};
+
+function makeSelectDb(): SelectDb {
+  const respond = (sql: string, bind: unknown[] = []): { data: string }[] => {
+    const table = /FROM "([^"]+)"/.exec(sql)?.[1];
+    let rows = table ? (FIXTURE[table] ?? []) : [];
+    const rel = /WHERE json_extract\(data, '\$\.([A-Za-z0-9_.]+)'\) IN/.exec(sql);
+    if (rel) rows = rows.filter((r) => bind.includes(r[rel[1]]));
+    else if (/WHERE id IN/.test(sql)) rows = rows.filter((r) => bind.includes(r.id));
+    return rows.map((r) => ({ data: JSON.stringify(r) }));
+  };
+  return {
+    exec: (sql, bind) => respond(sql, bind ?? []),
+    run: () => {},
+    knownTables: new Set<string>(),
+  };
+}
+
+describe('id-set plan on the SQLite engine', () => {
+  it('returns only the id-set even though both bodies match the predicate', async () => {
+    const { rows } = await executeSelect(
+      buildIdSetPlan(BASE, ['thread:A']),
+      {},
+      makeSelectDb()
+    );
+    expect(rows.map((r) => r.id)).toEqual(['thread:A']);
+  });
+
+  it('still resolves .related() children for the id-set rows', async () => {
+    const { rows, relationFetches } = await executeSelect(
+      buildIdSetPlan(BASE, ['thread:A']),
+      {},
+      makeSelectDb()
+    );
+    expect(relationFetches).toBe(1);
+    expect((rows[0] as any).comments.map((c: Row) => c.id)).toEqual(['comment:1']);
+  });
+
+  it('renders nothing for an empty id-set (known-and-empty membership)', async () => {
+    const { rows } = await executeSelect(buildIdSetPlan(BASE, []), {}, makeSelectDb());
+    expect(rows).toEqual([]);
+  });
+});