@spooky-sync/core 0.0.1-canary.20 → 0.0.1-canary.201

package/src/modules/data/data.membership.test.ts

@@ -0,0 +1,462 @@
+import { describe, it, expect, vi } from 'vitest';
+import { RecordId } from 'surrealdb';
+import { DataModule } from './index';
+import type { QueryPlan } from '@spooky-sync/query-builder';
+import type { QueryState, RecordVersionArray } from '../../types';
+
+/**
+ * A query's rows are its MEMBERSHIP — the id-set the server put in
+ * `_00_list_ref` — not "every locally cached body that matches the WHERE".
+ *
+ * Those two disagree whenever a row leaves a query's window while still existing
+ * upstream: `SyncEngine.handleRemovedRecords` keeps the local body and never
+ * re-fetches it, so a predicate re-scan finds that stale body still matching and
+ * keeps rendering the row. That is the "removed item comes back" bug, and it was
+ * permanent offline because membership used to live only on the session-salted
+ * `_00_query` row (wiped on reload) while bodies were durable.
+ */
+
+const noop = () => {};
+
+function makeLogger(): any {
+  const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
+  logger.child = () => logger;
+  return logger;
+}
+
+const schema = { tables: [{ name: 'thread', columns: {} }] } as any;
+
+const plan: QueryPlan = { table: 'thread', where: [['done', '=', false]] } as any;
+
+function makeQueryState(
+  hash: string,
+  opts: {
+    remoteArray?: RecordVersionArray;
+    localArray?: RecordVersionArray;
+    membershipKnown?: boolean;
+    membershipKey?: string;
+    withPlan?: boolean;
+  } = {}
+): QueryState {
+  return {
+    config: {
+      id: new RecordId('_00_query', hash),
+      surql: 'SELECT * FROM thread WHERE done = false;',
+      plan: opts.withPlan === false ? undefined : plan,
+      params: {},
+      localArray: opts.localArray ?? [],
+      remoteArray: opts.remoteArray ?? [],
+      membershipKnown: opts.membershipKnown,
+      membershipKey: opts.membershipKey,
+      ttl: '10m',
+      lastActiveAt: new Date(),
+      tableName: 'thread',
+    },
+    records: [],
+    ttlTimer: null,
+    ttlDurationMs: 0,
+    updateCount: 0,
+    lastUpdatedAt: null,
+    materializationSamples: [],
+    lastIngestLatencyMs: null,
+    errorCount: 0,
+    status: 'idle',
+    phaseSamples: {},
+    phaseLast: {},
+    registrationTimings: { parseMs: null, planMs: null, snapshotMs: null, wallMs: null },
+  };
+}
+
+/**
+ * Local store holding three thread bodies, ALL of which match the query's
+ * predicate. `select` mimics the real engines: when `plan.ids` is set it returns
+ * exactly those ids (the membership path); otherwise it returns everything (the
+ * predicate-scan path).
+ */
+function makeLocal(pendingRows: Array<{ recordId: RecordId; mutationType: string }> = []) {
+  const bodies = new Map(
+    ['a', 'b', 'c'].map((k) => [`thread:${k}`, { id: new RecordId('thread', k), title: k }])
+  );
+  const windowRows = new Map<string, { ids: RecordVersionArray }>();
+  const local: any = {
+    epoch: 1,
+    bodies,
+    windowRows,
+    select: vi.fn(async (p: QueryPlan) => {
+      if (p.ids) {
+        return (p.ids as RecordId[])
+          .map((id) => bodies.get(id.toString()))
+          .filter((r): r is NonNullable<typeof r> => r !== undefined);
+      }
+      return Array.from(bodies.values());
+    }),
+    query: vi.fn(async (sql: string, vars?: any) => {
+      if (sql.includes('_00_pending_mutations')) return [pendingRows];
+      // `createNewQuery` reads the `_00_query` row, then CREATEs it when absent.
+      // A reload always takes the absent path: the id is session-salted.
+      if (sql.includes('FROM ONLY')) return [null];
+      if (sql.startsWith('CREATE')) return [{ id: vars.id, ...vars.data }];
+      if (sql.startsWith('UPDATE')) return [null];
+      return [Array.from(bodies.values())];
+    }),
+    getById: vi.fn(async (_table: string, id: RecordId) => windowRows.get(String(id.id)) ?? null),
+    upsert: vi.fn(async (_t: string, id: RecordId, data: any) => {
+      windowRows.set(String(id.id), data);
+    }),
+  };
+  return local;
+}
+
+function setup(
+  stateOpts: Parameters<typeof makeQueryState>[1] = {},
+  pendingRows: Array<{ recordId: RecordId; mutationType: string }> = []
+) {
+  const hash = 'h1';
+  const local = makeLocal(pendingRows);
+  const dm = new DataModule({ saveBatch: async () => {} } as any, local, schema, makeLogger(), 100);
+  const state = makeQueryState(hash, stateOpts);
+  (dm as any).activeQueries.set(hash, state);
+  return { dm, state, local, hash };
+}
+
+const ids = (rows: Array<Record<string, any>>) => rows.map((r) => String(r.id));
+const materialize = (dm: DataModule<any>, state: QueryState, ssp?: RecordVersionArray) =>
+  (dm as any).materializeRecords(state, ssp) as Promise<Record<string, any>[]>;
+
+describe('membership-authoritative rendering', () => {
+  it('hides a locally cached body that is absent from membership', async () => {
+    // `thread:c` still matches the predicate and its body is still cached, but
+    // the server dropped it from the window. It must not render.
+    const { dm, state } = setup({
+      membershipKnown: true,
+      remoteArray: [
+        ['thread:a', 1],
+        ['thread:b', 1],
+      ],
+    });
+
+    expect(ids(await materialize(dm, state))).toEqual(['thread:a', 'thread:b']);
+  });
+
+  it('renders an empty list when membership is known and empty', async () => {
+    const { dm, state } = setup({ membershipKnown: true, remoteArray: [] });
+    expect(await materialize(dm, state)).toEqual([]);
+  });
+
+  it('falls back to the predicate scan when membership was never established', async () => {
+    // A query first run on this device: there is nothing authoritative to render
+    // from, so an offline first paint must still show the cached bodies.
+    const { dm, state, local } = setup({ membershipKnown: false });
+    expect(ids(await materialize(dm, state))).toEqual(['thread:a', 'thread:b', 'thread:c']);
+    expect(local.select).toHaveBeenCalledWith(
+      expect.objectContaining({ where: plan.where }),
+      expect.anything()
+    );
+  });
+
+  it('drops where/limit/offset and keeps orderBy on the membership path', async () => {
+    const { dm, state, local } = setup({ membershipKnown: true, remoteArray: [['thread:a', 1]] });
+    state.config.plan = { ...plan, limit: 50, offset: 0, orderBy: [['title', 'asc']] } as any;
+
+    await materialize(dm, state);
+
+    const passed = local.select.mock.calls.at(-1)![0] as QueryPlan;
+    expect(passed.where).toBeUndefined();
+    expect(passed.limit).toBeUndefined();
+    expect(passed.offset).toBeUndefined();
+    expect(passed.orderBy).toEqual([['title', 'asc']]);
+  });
+
+  describe('optimistic writes', () => {
+    it('renders a pending create the server has not acknowledged yet', async () => {
+      // Not in remoteArray, but the SSP (fed every local write) says it matches.
+      const { dm, state } = setup(
+        {
+          membershipKnown: true,
+          remoteArray: [['thread:a', 1]],
+          localArray: [
+            ['thread:a', 1],
+            ['thread:c', 1],
+          ],
+        },
+        [{ recordId: new RecordId('thread', 'c'), mutationType: 'create' }]
+      );
+
+      expect(ids(await materialize(dm, state))).toEqual(['thread:a', 'thread:c']);
+    });
+
+    it('hides a pending update that moved a row OUT of the window', async () => {
+      // Pending write, but the SSP no longer lists it: the local body stopped
+      // matching. Unioning pending ids blindly would wrongly re-admit it.
+      const { dm, state } = setup(
+        {
+          membershipKnown: true,
+          remoteArray: [['thread:a', 1]],
+          localArray: [['thread:a', 1]],
+        },
+        [{ recordId: new RecordId('thread', 'c'), mutationType: 'update' }]
+      );
+
+      expect(ids(await materialize(dm, state))).toEqual(['thread:a']);
+    });
+
+    it('hides a row the server still lists while our DELETE is queued', async () => {
+      const { dm, state } = setup(
+        {
+          membershipKnown: true,
+          remoteArray: [
+            ['thread:a', 1],
+            ['thread:b', 1],
+          ],
+        },
+        [{ recordId: new RecordId('thread', 'b'), mutationType: 'delete' }]
+      );
+
+      expect(ids(await materialize(dm, state))).toEqual(['thread:a']);
+    });
+
+    it('prefers the fresher sspArray over the persisted localArray', async () => {
+      const { dm, state } = setup(
+        { membershipKnown: true, remoteArray: [['thread:a', 1]], localArray: [] },
+        [{ recordId: new RecordId('thread', 'c'), mutationType: 'create' }]
+      );
+
+      const rows = await materialize(dm, state, [
+        ['thread:a', 1],
+        ['thread:c', 1],
+      ]);
+      expect(ids(rows)).toEqual(['thread:a', 'thread:c']);
+    });
+  });
+
+  describe('render order', () => {
+    it('sorts an unordered query so both paints agree', async () => {
+      // `_00_list_ref` is selected without an ORDER BY, so membership arrives
+      // shuffled. The first paint came from the local scan in id order, so
+      // rendering server order here is what made lists visibly reorder about a
+      // second after load.
+      const { dm, hash, state } = setup();
+      state.config.plan = { table: 'thread', where: [['done', '=', false]] } as any;
+
+      const ids = await (dm as any).buildRenderIds(state.config, [
+        ['thread:c', 1],
+        ['thread:a', 1],
+        ['thread:b', 1],
+      ]);
+
+      expect(ids.map(String)).toEqual(['thread:a', 'thread:b', 'thread:c']);
+    });
+
+    it('leaves an explicitly ordered query to the engine', async () => {
+      const { dm, state } = setup();
+      state.config.plan = { table: 'thread', orderBy: [['created', 'desc']] } as any;
+
+      const ids = await (dm as any).buildRenderIds(state.config, [
+        ['thread:c', 1],
+        ['thread:a', 1],
+      ]);
+
+      // Untouched: the engine applies the ORDER BY over the id set.
+      expect(ids.map(String)).toEqual(['thread:c', 'thread:a']);
+    });
+
+    it('preserves the slice order of a windowed query', async () => {
+      // For a window the id-set order IS the window: sorting it would reorder
+      // rows within the page.
+      const { dm, state } = setup();
+      state.config.surql = 'SELECT * FROM thread LIMIT 50 START 100;';
+      state.config.plan = { table: 'thread' } as any;
+
+      const ids = await (dm as any).buildRenderIds(state.config, [
+        ['thread:c', 1],
+        ['thread:a', 1],
+      ]);
+
+      expect(ids.map(String)).toEqual(['thread:c', 'thread:a']);
+    });
+  });
+
+  describe('durability across a reload', () => {
+    it('updateQueryRemoteArray latches membership and writes _00_window', async () => {
+      const { dm, state, local, hash } = setup({ membershipKey: 'stable-key' });
+
+      await dm.updateQueryRemoteArray(hash, [['thread:a', 1]]);
+
+      expect(state.config.membershipKnown).toBe(true);
+      expect(local.windowRows.get('stable-key')).toMatchObject({ ids: [['thread:a', 1]] });
+    });
+
+    it('seeds membership from _00_window under a different session salt', async () => {
+      // The reload case: `_00_query` is keyed by a `session::id()`-salted hash, so
+      // a reload always lands on a fresh row with empty arrays. Only the durable
+      // row, keyed session-independently, can carry the removal across.
+      const { dm, local } = setup({ membershipKey: 'stable-key' });
+      local.windowRows.set('stable-key', { ids: [['thread:a', 1]] });
+
+      const fresh = await (dm as any).createNewQuery({
+        recordId: new RecordId('_00_query', 'a-totally-different-salted-hash'),
+        surql: 'SELECT * FROM thread WHERE done = false;',
+        params: {},
+        ttl: '10m',
+        tableName: 'thread',
+        plan,
+        membershipKey: 'stable-key',
+      });
+
+      expect(fresh.config.membershipKnown).toBe(true);
+      expect(fresh.config.remoteArray).toEqual([['thread:a', 1]]);
+      // `thread:b`/`thread:c` bodies are still cached and still match, but are
+      // not in the persisted list — the reported bug, offline.
+      expect(ids(fresh.records)).toEqual(['thread:a']);
+    });
+
+    it('stays scan-based when no durable row exists', async () => {
+      const { dm } = setup({ membershipKey: 'never-written' });
+
+      const fresh = await (dm as any).createNewQuery({
+        recordId: new RecordId('_00_query', 'h-cold'),
+        surql: 'SELECT * FROM thread WHERE done = false;',
+        params: {},
+        ttl: '10m',
+        tableName: 'thread',
+        plan,
+        membershipKey: 'never-written',
+      });
+
+      expect(fresh.config.membershipKnown).toBeFalsy();
+      expect(ids(fresh.records)).toEqual(['thread:a', 'thread:b', 'thread:c']);
+    });
+
+    it('ignores an empty id-set until a real one has been seen', async () => {
+      // The registration race: the SSP queues a view's initial edges to a
+      // coalescing flusher and returns from `fn::query::register` before they
+      // land, so this read says nothing about the query being empty. Believing
+      // it rendered a blank list AND persisted the blankness.
+      const { dm, state, local, hash } = setup({
+        membershipKey: 'stable-key',
+        remoteArray: [['thread:a', 1]],
+        membershipKnown: true,
+      });
+
+      await dm.updateQueryRemoteArray(hash, []);
+
+      expect(state.config.remoteArray).toEqual([['thread:a', 1]]);
+      expect(local.windowRows.has('stable-key')).toBe(false);
+    });
+
+    it('believes an empty id-set once a non-empty one has arrived', async () => {
+      // The genuine transition — the last row left the window. Honouring it is
+      // what stops a removed row resurrecting from the local body cache.
+      const { dm, state, local, hash } = setup({ membershipKey: 'stable-key' });
+
+      await dm.updateQueryRemoteArray(hash, [['thread:a', 1]]);
+      await dm.updateQueryRemoteArray(hash, []);
+
+      expect(state.config.membershipKnown).toBe(true);
+      expect(state.config.remoteArray).toEqual([]);
+      expect(local.windowRows.get('stable-key')).toMatchObject({ ids: [] });
+    });
+
+    it('believes a repeated empty id-set even with nothing seen this session', async () => {
+      // A window that really did empty while this device was away: the durable
+      // seed must not render forever, so the second read is taken at face value.
+      const { dm, state, hash } = setup({
+        membershipKey: 'stable-key',
+        remoteArray: [['thread:a', 1]],
+        membershipKnown: true,
+      });
+
+      await dm.updateQueryRemoteArray(hash, []);
+      expect(state.config.remoteArray).toEqual([['thread:a', 1]]);
+
+      await dm.updateQueryRemoteArray(hash, []);
+      expect(state.config.remoteArray).toEqual([]);
+    });
+
+    it('ignores an empty id-set while the server still reports rows', async () => {
+      // The reported failure: registration returns before the SSP flushes the
+      // view's edges, and the poll 500ms later is still inside that window. A
+      // retry counter believes the second read and blanks the list ~2s after
+      // load; `rowCount` says the query has 26 rows, so no number of empty
+      // reads should ever be taken as "empty".
+      const { dm, state, local, hash } = setup({
+        membershipKey: 'stable-key',
+        remoteArray: [['thread:a', 1]],
+        membershipKnown: true,
+      });
+
+      for (let i = 0; i < 5; i++) {
+        await dm.updateQueryRemoteArray(hash, [], { serverRowCount: 26 });
+      }
+
+      expect(state.config.remoteArray).toEqual([['thread:a', 1]]);
+      expect(local.windowRows.has('stable-key')).toBe(false);
+    });
+
+    it('believes an empty id-set the moment the server reports zero rows', async () => {
+      // The other half: a genuinely empty query must not sit on a stale seed
+      // waiting for a retry budget to run out.
+      const { dm, state, hash } = setup({
+        membershipKey: 'stable-key',
+        remoteArray: [['thread:a', 1]],
+        membershipKnown: true,
+      });
+
+      await dm.updateQueryRemoteArray(hash, [], { serverRowCount: 0 });
+
+      expect(state.config.membershipKnown).toBe(true);
+      expect(state.config.remoteArray).toEqual([]);
+    });
+
+    it('falls back to the retry budget when the row count is unreadable', async () => {
+      // Older servers, or a row the client cannot select: unknown must not
+      // strand the device on its durable seed forever.
+      const { dm, state, hash } = setup({
+        membershipKey: 'stable-key',
+        remoteArray: [['thread:a', 1]],
+        membershipKnown: true,
+      });
+
+      await dm.updateQueryRemoteArray(hash, [], { serverRowCount: null });
+      expect(state.config.remoteArray).toEqual([['thread:a', 1]]);
+
+      await dm.updateQueryRemoteArray(hash, [], { serverRowCount: null });
+      expect(state.config.remoteArray).toEqual([]);
+    });
+
+    it('does not seed membership from an empty durable row', async () => {
+      // Self-heals devices poisoned before the guard existed: an empty durable
+      // row is indistinguishable from "never had membership", so it must fall
+      // back to the scan rather than paint an empty list.
+      const { dm, local } = setup({ membershipKey: 'stable-key' });
+      local.windowRows.set('stable-key', { ids: [] });
+
+      const fresh = await (dm as any).createNewQuery({
+        recordId: new RecordId('_00_query', 'h-poisoned'),
+        surql: 'SELECT * FROM thread WHERE done = false;',
+        params: {},
+        ttl: '10m',
+        tableName: 'thread',
+        plan,
+        membershipKey: 'stable-key',
+      });
+
+      expect(fresh.config.membershipKnown).toBeFalsy();
+      expect(ids(fresh.records)).toEqual(['thread:a', 'thread:b', 'thread:c']);
+    });
+
+    it('derives the membership key without the session salt', async () => {
+      const { dm } = setup();
+      const key = () => (dm as any).calculateMembershipKey({ surql: 'X', params: {} });
+
+      const before = await key();
+      dm.setSessionId('session-two');
+      expect(await key()).toBe(before);
+
+      // ...whereas the `_00_query` hash intentionally moves with the session.
+      const salted = await (dm as any).calculateHash({ surql: 'X', params: {} });
+      dm.setSessionId('session-three');
+      expect(await (dm as any).calculateHash({ surql: 'X', params: {} })).not.toBe(salted);
+    });
+  });
+});

package/src/modules/data/data.rebind.test.ts

@@ -0,0 +1,147 @@
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+import { RecordId } from 'surrealdb';
+import { DataModule } from './index';
+import type { QueryState } from '../../types';
+
+/**
+ * Tests for the local-bucket-switch surface of DataModule:
+ * - `quiesce()` disarms every debounce + TTL timer;
+ * - `rebindAfterBucketSwitch()` keeps query hashes (subscriptions stay
+ *   attached), resets sync state, notifies subscribers with the emptied
+ *   records so the previous user's rows leave the UI, re-registers the SSP
+ *   view, and returns the hashes for remote re-registration;
+ * - stale-epoch stream updates are dropped instead of applied.
+ */
+
+function makeLogger(): any {
+  const noop = () => {};
+  const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
+  logger.child = () => logger;
+  return logger;
+}
+
+function makeQueryState(hash: string, records: Record<string, any>[]): QueryState {
+  return {
+    config: {
+      id: new RecordId('_00_query', hash),
+      surql: 'SELECT * FROM user',
+      params: {},
+      localArray: [['user:a', 1]],
+      remoteArray: [['user:a', 1]],
+      ttl: '10m',
+      lastActiveAt: new Date(),
+      tableName: 'user',
+    },
+    records,
+    hydrated: true,
+    ttlTimer: null,
+    ttlDurationMs: 600_000,
+    updateCount: 3,
+    lastUpdatedAt: null,
+    materializationSamples: [],
+    lastIngestLatencyMs: null,
+    errorCount: 0,
+    status: 'idle',
+    phaseSamples: {},
+    phaseLast: {},
+    registrationTimings: { parseMs: null, planMs: null, snapshotMs: null, wallMs: null },
+  };
+}
+
+function makeHarness(epoch = 0) {
+  const localQueries: Array<{ query: unknown; vars: unknown; opts: unknown }> = [];
+  const local: any = {
+    epoch,
+    query: vi.fn(async (query: unknown, vars?: unknown, opts?: unknown) => {
+      localQueries.push({ query, vars, opts });
+      return [[]];
+    }),
+  };
+  const cache: any = {
+    registerQuery: vi.fn(() => ({ localArray: [] })),
+  };
+  const dm = new DataModule(cache, local, { tables: [] } as any, makeLogger(), 100);
+  return { dm, local, cache, localQueries };
+}
+
+describe('DataModule.quiesce', () => {
+  it('clears debounce and TTL timers', () => {
+    vi.useFakeTimers();
+    try {
+      const { dm } = makeHarness();
+      const qs = makeQueryState('h1', []);
+      qs.ttlTimer = setTimeout(() => {}, 60_000);
+      (dm as any).activeQueries.set('h1', qs);
+      (dm as any).debounceTimers.set('h1', setTimeout(() => {}, 60_000));
+
+      dm.quiesce();
+
+      expect(qs.ttlTimer).toBeNull();
+      expect((dm as any).debounceTimers.size).toBe(0);
+      expect(vi.getTimerCount()).toBe(0);
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+});
+
+describe('DataModule.rebindAfterBucketSwitch', () => {
+  let harness: ReturnType<typeof makeHarness>;
+
+  beforeEach(() => {
+    harness = makeHarness();
+    (harness.dm as any).activeQueries.set(
+      'h1',
+      makeQueryState('h1', [{ id: 'user:a', name: 'Previous User Row' }])
+    );
+  });
+
+  it('keeps the hash, resets sync state, and notifies subscribers with empty records', async () => {
+    const { dm, cache } = harness;
+    const emissions: unknown[][] = [];
+    dm.subscribe('h1', (records) => emissions.push(records as unknown[]));
+
+    const hashes = await dm.rebindAfterBucketSwitch();
+
+    expect(hashes).toEqual(['h1']);
+    const qs = (dm as any).activeQueries.get('h1') as QueryState;
+    expect(qs.records).toEqual([]);
+    expect(qs.config.localArray).toEqual([]);
+    expect(qs.config.remoteArray).toEqual([]);
+    expect(qs.hydrated).toBe(false);
+    expect(qs.status).toBe('fetching');
+    // Subscribers saw the previous user's rows drop out.
+    expect(emissions).toEqual([[]]);
+    // The SSP view was re-registered (on the fresh, post-reset processor).
+    expect(cache.registerQuery).toHaveBeenCalledWith(
+      expect.objectContaining({ queryHash: 'h1', surql: 'SELECT * FROM user' })
+    );
+    // The TTL heartbeat is re-armed.
+    expect(qs.ttlTimer).not.toBeNull();
+    if (qs.ttlTimer) clearTimeout(qs.ttlTimer);
+  });
+});
+
+describe('stale-epoch stream updates', () => {
+  it('drops an update whose chain started before a bucket switch', async () => {
+    const { dm, local } = makeHarness();
+    const qs = makeQueryState('h1', [{ id: 'user:a' }]);
+    (dm as any).activeQueries.set('h1', qs);
+
+    // The materialize read happens, then the epoch moves (bucket switched).
+    local.query.mockImplementation(async () => {
+      local.epoch = 1;
+      return [[{ id: 'user:b', name: 'other user row' }]];
+    });
+
+    await (dm as any).processStreamUpdate({
+      queryHash: 'h1',
+      localArray: [['user:b', 1]],
+      op: 'CREATE',
+    });
+
+    // Neither the records nor the persisted arrays moved.
+    expect(qs.records).toEqual([{ id: 'user:a' }]);
+    expect(qs.config.localArray).toEqual([['user:a', 1]]);
+  });
+});