@spooky-sync/core 0.0.1-canary.21 → 0.0.1-canary.211

package/src/modules/data/data.pending-ids.test.ts

@@ -0,0 +1,199 @@
+import { describe, it, expect, beforeEach, vi, afterEach } from 'vitest';
+import { RecordId } from 'surrealdb';
+import { DataModule } from './index';
+
+/**
+ * `getPendingRecordIds` used to run a full `SELECT ... FROM
+ * _00_pending_mutations` on EVERY materialization of EVERY query — a round trip
+ * down the local engine's single-flight op queue, paid tens of times per ingest
+ * against an outbox that is usually empty. It is now cached, invalidated when
+ * the outbox actually changes, with a short TTL as a backstop.
+ */
+
+function makeLogger(): any {
+  const noop = () => {};
+  const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
+  logger.child = () => logger;
+  return logger;
+}
+
+/** DataModule over a local store that counts outbox reads and can be told what
+ *  to return. */
+function makeModule(rows: { recordId: RecordId<string>; mutationType: string }[] = []) {
+  const state = { rows, reads: 0 };
+  const local = {
+    query: vi.fn(async () => {
+      state.reads++;
+      return [state.rows];
+    }),
+  };
+  const dm = new DataModule(
+    {} as any,
+    local as any,
+    { tables: [] } as any,
+    makeLogger(),
+    100
+  );
+  return { dm, state };
+}
+
+const pending = (id: string, mutationType = 'update') => ({
+  recordId: new RecordId('game', id),
+  mutationType,
+});
+
+describe('DataModule pending-id cache', () => {
+  beforeEach(() => vi.useFakeTimers());
+  afterEach(() => vi.useRealTimers());
+
+  it('reads the outbox once for repeated calls', async () => {
+    const { dm, state } = makeModule([pending('a')]);
+    await dm.getPendingRecordIds();
+    await dm.getPendingRecordIds();
+    await dm.getPendingRecordIds();
+    expect(state.reads).toBe(1);
+  });
+
+  it('collapses a burst of CONCURRENT calls into one read', async () => {
+    // The real shape: one ingest fans out to many queries materializing at once.
+    const { dm, state } = makeModule([pending('a')]);
+    await Promise.all([
+      dm.getPendingRecordIds(),
+      dm.getPendingRecordIds(),
+      dm.getPendingRecordIds(),
+      dm.getPendingRecordIds(),
+    ]);
+    expect(state.reads).toBe(1);
+  });
+
+  it('hands out COPIES, so a caller mutating the sets cannot poison the cache', async () => {
+    // buildRenderIds merges the settled-write ids into what it gets back.
+    const { dm } = makeModule([pending('a')]);
+    const first = await dm.getPendingRecordIds();
+    first.writes.add('game:injected');
+    const second = await dm.getPendingRecordIds();
+    expect(second.writes.has('game:injected')).toBe(false);
+    expect([...second.writes]).toEqual(['game:a']);
+  });
+
+  it('re-reads after a mutation settles (its outbox row is gone)', async () => {
+    const { dm, state } = makeModule([pending('a')]);
+    await dm.getPendingRecordIds();
+    expect(state.reads).toBe(1);
+
+    state.rows = [];
+    dm.noteWriteSettled('game:a', 'update');
+    const after = await dm.getPendingRecordIds();
+
+    expect(state.reads).toBe(2);
+    expect(after.writes.size).toBe(0);
+  });
+
+  it('re-reads once the TTL backstop lapses', async () => {
+    // Covers any path that removes an outbox row without telling us: staleness
+    // is bounded to a tick rather than lasting the session.
+    const { dm, state } = makeModule([pending('a')]);
+    await dm.getPendingRecordIds();
+    expect(state.reads).toBe(1);
+
+    vi.setSystemTime(Date.now() + 1_000);
+    await dm.getPendingRecordIds();
+    expect(state.reads).toBe(2);
+  });
+
+  it('does NOT cache a failed read', async () => {
+    // Empty sets are this call's fallback, not a claim that the outbox is empty.
+    const { dm, state } = makeModule([pending('a')]);
+    (dm as any).local.query = vi.fn(async () => {
+      state.reads++;
+      throw new Error('engine down');
+    });
+    const failed = await dm.getPendingRecordIds();
+    expect(failed.writes.size).toBe(0);
+    expect(state.reads).toBe(1);
+
+    await dm.getPendingRecordIds();
+    expect(state.reads).toBe(2);
+  });
+
+  it('splits writes from deletes', async () => {
+    const { dm } = makeModule([pending('a'), pending('b', 'delete')]);
+    const { writes, deletes } = await dm.getPendingRecordIds();
+    expect([...writes]).toEqual(['game:a']);
+    expect([...deletes]).toEqual(['game:b']);
+  });
+});
+
+describe('DataModule pending-id cache: generations', () => {
+  beforeEach(() => vi.useFakeTimers());
+  afterEach(() => vi.useRealTimers());
+
+  // The outbox read is a round trip down the local op queue, so it can be
+  // queued BEFORE a create's outbox row commits and resolve AFTER
+  // `invalidatePendingIds()` ran for it. Cached as-is, that pre-create set
+  // hid the new row from every materialization that joined the read - the one
+  // stream update a CREATE gets, so the row was invisible until reload.
+  function makeDeferredModule() {
+    const pendingReads: Array<(rows: any[]) => void> = [];
+    const state = { reads: 0 };
+    const local = {
+      query: vi.fn(
+        () =>
+          new Promise<any[]>((resolve) => {
+            state.reads++;
+            pendingReads.push((rows) => resolve([rows]));
+          })
+      ),
+    };
+    const dm = new DataModule({} as any, local as any, { tables: [] } as any, makeLogger(), 100);
+    return { dm, state, pendingReads };
+  }
+  // Let the awaiting loop observe a resolved read and issue the next one.
+  const settle = async () => {
+    for (let i = 0; i < 8; i++) await Promise.resolve();
+  };
+
+  it('does not cache a read that was in flight when the outbox changed', async () => {
+    const { dm, state, pendingReads } = makeDeferredModule();
+    const first = dm.getPendingRecordIds();
+    expect(state.reads).toBe(1);
+    // The create commits and invalidates while the read is still out.
+    (dm as any).invalidatePendingIds();
+    // The stale read answers with the pre-create outbox...
+    pendingReads[0]!([]);
+    await settle();
+    // ...so the joiner re-read under the new generation and gets the fresh one.
+    expect(state.reads).toBe(2);
+    pendingReads[1]!([pending('new', 'create')]);
+    expect([...(await first).writes]).toEqual(['game:new']);
+  });
+
+  it('a stale in-flight result never repopulates the cache', async () => {
+    const { dm, state, pendingReads } = makeDeferredModule();
+    const first = dm.getPendingRecordIds();
+    (dm as any).invalidatePendingIds();
+    pendingReads[0]!([]);
+    await settle();
+    pendingReads[1]!([pending('new', 'create')]);
+    await first;
+    // A fresh call must be served from the cache the CURRENT generation's
+    // read wrote, i.e. see the new row without another read.
+    const again = await dm.getPendingRecordIds();
+    expect(state.reads).toBe(2);
+    expect([...again.writes]).toEqual(['game:new']);
+  });
+
+  it('bounds the re-read when the outbox changes on every tick', async () => {
+    const { dm, state, pendingReads } = makeDeferredModule();
+    const call = dm.getPendingRecordIds();
+    for (let i = 0; i < 5; i++) {
+      (dm as any).invalidatePendingIds();
+      pendingReads[i]?.([pending(`r${i}`)]);
+      await settle();
+    }
+    const result = await call;
+    // Three reads at most, and the caller still gets an answer.
+    expect(state.reads).toBeLessThanOrEqual(3);
+    expect(result.writes.size).toBe(1);
+  });
+});

package/src/modules/data/data.rebind.test.ts

@@ -0,0 +1,170 @@
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+import { RecordId } from 'surrealdb';
+import { DataModule } from './index';
+import type { QueryState } from '../../types';
+
+/**
+ * Tests for the local-bucket-switch surface of DataModule:
+ * - `quiesce()` disarms every debounce + TTL timer;
+ * - `rebindAfterBucketSwitch()` keeps query hashes (subscriptions stay
+ *   attached), resets sync state, notifies subscribers with the emptied
+ *   records so the previous user's rows leave the UI, re-registers the SSP
+ *   view, and returns the hashes for remote re-registration;
+ * - stale-epoch stream updates are dropped instead of applied.
+ */
+
+function makeLogger(): any {
+  const noop = () => {};
+  const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
+  logger.child = () => logger;
+  return logger;
+}
+
+function makeQueryState(hash: string, records: Record<string, any>[]): QueryState {
+  return {
+    config: {
+      id: new RecordId('_00_query', hash),
+      surql: 'SELECT * FROM user',
+      params: {},
+      localArray: [['user:a', 1]],
+      remoteArray: [['user:a', 1]],
+      ttl: '10m',
+      lastActiveAt: new Date(),
+      tableName: 'user',
+    },
+    records,
+    hydrated: true,
+    ttlTimer: null,
+    ttlDurationMs: 600_000,
+    updateCount: 3,
+    lastUpdatedAt: null,
+    materializationSamples: [],
+    lastIngestLatencyMs: null,
+    errorCount: 0,
+    status: 'idle',
+    phaseSamples: {},
+    phaseLast: {},
+    registrationTimings: { parseMs: null, planMs: null, snapshotMs: null, wallMs: null },
+  };
+}
+
+function makeHarness(epoch = 0) {
+  const localQueries: Array<{ query: unknown; vars: unknown; opts: unknown }> = [];
+  const local: any = {
+    epoch,
+    query: vi.fn(async (query: unknown, vars?: unknown, opts?: unknown) => {
+      localQueries.push({ query, vars, opts });
+      return [[]];
+    }),
+  };
+  const cache: any = {
+    registerQuery: vi.fn(() => ({ localArray: [] })),
+  };
+  const dm = new DataModule(cache, local, { tables: [] } as any, makeLogger(), 100);
+  return { dm, local, cache, localQueries };
+}
+
+describe('DataModule.quiesce', () => {
+  it('clears debounce and TTL timers', () => {
+    vi.useFakeTimers();
+    try {
+      const { dm } = makeHarness();
+      const qs = makeQueryState('h1', []);
+      qs.ttlTimer = setTimeout(() => {}, 60_000);
+      (dm as any).activeQueries.set('h1', qs);
+      (dm as any).debounceTimers.set('h1', setTimeout(() => {}, 60_000));
+
+      dm.quiesce();
+
+      expect(qs.ttlTimer).toBeNull();
+      expect((dm as any).debounceTimers.size).toBe(0);
+      expect(vi.getTimerCount()).toBe(0);
+    } finally {
+      vi.useRealTimers();
+    }
+  });
+});
+
+describe('DataModule.rebindAfterBucketSwitch', () => {
+  let harness: ReturnType<typeof makeHarness>;
+
+  beforeEach(() => {
+    harness = makeHarness();
+    (harness.dm as any).activeQueries.set(
+      'h1',
+      makeQueryState('h1', [{ id: 'user:a', name: 'Previous User Row' }])
+    );
+  });
+
+  it('keeps the hash, resets sync state, and notifies subscribers with empty records', async () => {
+    const { dm, cache } = harness;
+    const emissions: unknown[][] = [];
+    dm.subscribe('h1', (records) => emissions.push(records as unknown[]));
+
+    const hashes = await dm.rebindAfterBucketSwitch();
+
+    expect(hashes).toEqual(['h1']);
+    const qs = (dm as any).activeQueries.get('h1') as QueryState;
+    expect(qs.records).toEqual([]);
+    expect(qs.config.localArray).toEqual([]);
+    expect(qs.config.remoteArray).toEqual([]);
+    expect(qs.hydrated).toBe(false);
+    expect(qs.status).toBe('fetching');
+    // Subscribers saw the previous user's rows drop out.
+    expect(emissions).toEqual([[]]);
+    // The SSP view was re-registered (on the fresh, post-reset processor).
+    expect(cache.registerQuery).toHaveBeenCalledWith(
+      expect.objectContaining({ queryHash: 'h1', surql: 'SELECT * FROM user' })
+    );
+    // The TTL heartbeat is re-armed.
+    expect(qs.ttlTimer).not.toBeNull();
+    if (qs.ttlTimer) clearTimeout(qs.ttlTimer);
+  });
+});
+
+describe('DataModule.rebindAfterBucketSwitch durable seed', () => {
+  it('seeds a confirmed-empty membership from the new bucket, and ignores an unconfirmed one', async () => {
+    const harness = makeHarness();
+    const { dm, local } = harness as any;
+    const state = makeQueryState('h1', [{ id: 'user:a', name: 'Previous User Row' }]);
+    state.config.membershipKey = 'stable-key';
+    (dm as any).activeQueries.set('h1', state);
+    local.getById = vi.fn(async () => ({ ids: [], confirmed: true }));
+
+    await dm.rebindAfterBucketSwitch();
+    let qs = (dm as any).activeQueries.get('h1') as QueryState;
+    expect(qs.config.membershipKnown).toBe(true);
+    expect(qs.config.remoteArray).toEqual([]);
+    if (qs.ttlTimer) clearTimeout(qs.ttlTimer);
+
+    local.getById = vi.fn(async () => ({ ids: [] }));
+    await dm.rebindAfterBucketSwitch();
+    qs = (dm as any).activeQueries.get('h1') as QueryState;
+    expect(qs.config.membershipKnown).toBe(false);
+    if (qs.ttlTimer) clearTimeout(qs.ttlTimer);
+  });
+});
+
+describe('stale-epoch stream updates', () => {
+  it('drops an update whose chain started before a bucket switch', async () => {
+    const { dm, local } = makeHarness();
+    const qs = makeQueryState('h1', [{ id: 'user:a' }]);
+    (dm as any).activeQueries.set('h1', qs);
+
+    // The materialize read happens, then the epoch moves (bucket switched).
+    local.query.mockImplementation(async () => {
+      local.epoch = 1;
+      return [[{ id: 'user:b', name: 'other user row' }]];
+    });
+
+    await (dm as any).processStreamUpdate({
+      queryHash: 'h1',
+      localArray: [['user:b', 1]],
+      op: 'CREATE',
+    });
+
+    // Neither the records nor the persisted arrays moved.
+    expect(qs.records).toEqual([{ id: 'user:a' }]);
+    expect(qs.config.localArray).toEqual([['user:a', 1]]);
+  });
+});

package/src/modules/data/data.rematerialize.test.ts

@@ -0,0 +1,114 @@
+import { describe, it, expect, vi, afterEach } from 'vitest';
+import { RecordId } from 'surrealdb';
+import { DataModule } from './index';
+import type { QueryPlan } from '@spooky-sync/query-builder';
+import type { QueryState, RecordVersionArray } from '../../types';
+
+/**
+ * A row this client wrote itself reaches server membership at the very
+ * version the local CREATE memoized, so the sync engine rightly fetches
+ * nothing when the membership lands - and then nothing re-materialized the
+ * query: `remoteArray` gained the id and no subscriber heard about it until a
+ * reload rebuilt the view from membership. `scheduleRematerialize` closes that
+ * gap through the same per-query debounce a real stream update uses.
+ */
+
+const noop = () => {};
+function makeLogger(): any {
+  const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
+  logger.child = () => logger;
+  return logger;
+}
+
+const schema = { tables: [{ name: 'message', columns: {} }] } as any;
+const plan: QueryPlan = { table: 'message', where: [['conversation', '=', 'conversation:c1']] } as any;
+
+function makeQueryState(hash: string, remoteArray: RecordVersionArray, localArray: RecordVersionArray): QueryState {
+  return {
+    config: {
+      id: new RecordId('_00_query', hash),
+      surql: 'SELECT * FROM message WHERE conversation = $conversation;',
+      plan,
+      params: { conversation: 'conversation:c1' },
+      localArray,
+      remoteArray,
+      membershipKnown: true,
+      ttl: '10m',
+      lastActiveAt: new Date(),
+      tableName: 'message',
+    },
+    records: [],
+    ttlTimer: null,
+    ttlDurationMs: 0,
+    updateCount: 0,
+    lastUpdatedAt: null,
+    materializationSamples: [],
+    lastIngestLatencyMs: null,
+    errorCount: 0,
+    status: 'idle',
+    phaseSamples: {},
+    phaseLast: {},
+    registrationTimings: { parseMs: null, planMs: null, snapshotMs: null, wallMs: null },
+  } as QueryState;
+}
+
+function setup(remoteArray: RecordVersionArray, localArray: RecordVersionArray) {
+  const body = { id: new RecordId('message', 'm1'), text: 'hi' };
+  const local: any = {
+    epoch: 1,
+    select: vi.fn(async (p: QueryPlan) => {
+      const ids = ((p as any).ids as RecordId[] | undefined) ?? [];
+      return ids.some((id) => id.toString() === 'message:m1') ? [body] : [];
+    }),
+    query: vi.fn(async (sql: string) => {
+      if (sql.includes('_00_pending_mutations')) return [[]];
+      return [[]];
+    }),
+  };
+  const dm = new DataModule({} as any, local, schema, makeLogger(), 10);
+  const hash = 'q1';
+  (dm as any).activeQueries.set(hash, makeQueryState(hash, remoteArray, localArray));
+  const subscriber = vi.fn();
+  (dm as any).subscriptions.set(hash, new Set([subscriber]));
+  return { dm, local, hash, subscriber };
+}
+
+describe('DataModule.scheduleRematerialize', () => {
+  afterEach(() => vi.useRealTimers());
+
+  it('notifies subscribers with the row once membership holds it', async () => {
+    vi.useFakeTimers();
+    const { dm, local, hash, subscriber } = setup([['message:m1', 1]], [['message:m1', 1]]);
+    dm.scheduleRematerialize(hash);
+    await vi.advanceTimersByTimeAsync(20);
+    expect(subscriber).toHaveBeenCalledTimes(1);
+    expect(subscriber.mock.calls[0]![0].map((r: any) => r.id.toString())).toEqual(['message:m1']);
+    // A synthetic update describes no ingest: nothing is persisted to _00_query.
+    expect(local.query.mock.calls.some(([sql]: [string]) => sql.includes('localArray'))).toBe(false);
+  });
+
+  it('is a no-op behind a pending real stream update', async () => {
+    vi.useFakeTimers();
+    const { dm, hash, subscriber } = setup([['message:m1', 1]], [['message:m1', 1]]);
+    await dm.onStreamUpdate({ queryHash: hash, localArray: [['message:m1', 1]], op: 'CREATE' });
+    dm.scheduleRematerialize(hash);
+    await vi.advanceTimersByTimeAsync(20);
+    // Exactly one notify: the real update's, not two.
+    expect(subscriber).toHaveBeenCalledTimes(1);
+  });
+
+  it('does not notify when the records are unchanged', async () => {
+    vi.useFakeTimers();
+    const { dm, hash, subscriber } = setup([['message:m1', 1]], [['message:m1', 1]]);
+    dm.scheduleRematerialize(hash);
+    await vi.advanceTimersByTimeAsync(20);
+    dm.scheduleRematerialize(hash);
+    await vi.advanceTimersByTimeAsync(20);
+    expect(subscriber).toHaveBeenCalledTimes(1);
+  });
+
+  it('ignores an unknown query', () => {
+    const { dm } = setup([], []);
+    expect(() => dm.scheduleRematerialize('nope')).not.toThrow();
+  });
+});

package/src/modules/data/data.run.test.ts

@@ -0,0 +1,113 @@
+import { describe, it, expect, vi } from 'vitest';
+import { DataModule } from './index';
+
+/**
+ * Tests for `DataModule.run`, the one-shot outbox API.
+ *
+ * Recurring jobs used to live here too (`runRecurring` / `pokeRecurring` /
+ * `cancelRecurring`, which wrote a single durable row the runner re-armed
+ * forever). They are gone: schedules are now declared server-side under
+ * `schedules:` in sp00ky.yml, and the scheduler creates a fresh job row per
+ * cycle — so every row this API writes is exactly one execution.
+ *
+ * The create/update pipeline is spied, so these assert the orchestration
+ * (table resolution, argument validation, field building) without a real engine.
+ */
+
+function makeLogger(): any {
+  const noop = () => {};
+  const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
+  logger.child = () => logger;
+  return logger;
+}
+
+const schema = {
+  tables: [{ name: 'job', columns: {} }],
+  backends: {
+    gamesync: {
+      outboxTable: 'job',
+      routes: {
+        '/syncGames': {
+          args: { connection: { optional: false }, since: { optional: true } },
+        },
+      },
+    },
+    noOutbox: {
+      routes: { '/whatever': { args: {} } },
+    },
+  },
+};
+
+const CONN = 'connection:CONN_abc';
+
+function makeDm() {
+  const local = { query: vi.fn().mockResolvedValue([]) };
+  const dm = new DataModule({} as any, local as any, schema as any, makeLogger(), 100);
+  const create = vi.spyOn(dm, 'create').mockResolvedValue(undefined as any);
+  return { dm, local, create };
+}
+
+describe('DataModule.run', () => {
+  it('creates the one-shot job with status pending so the optimistic row reads in-flight', async () => {
+    const { dm, create } = makeDm();
+    await dm.run('gamesync' as any, '/syncGames' as any, { connection: CONN } as any, {
+      assignedTo: CONN,
+    });
+
+    expect(create).toHaveBeenCalledTimes(1);
+    const [id, record] = create.mock.calls[0] as [string, any];
+    expect(id.startsWith('job:')).toBe(true);
+    // The schema's DEFAULT ALWAYS "pending" only runs server-side; without the
+    // explicit field the local optimistic row has status undefined and
+    // in-flight indicators miss it until the first server echo.
+    expect(record.status).toBe('pending');
+  });
+
+  it('gives each call its own row', async () => {
+    const { dm, create } = makeDm();
+    await dm.run('gamesync' as any, '/syncGames' as any, { connection: CONN } as any);
+    await dm.run('gamesync' as any, '/syncGames' as any, { connection: CONN } as any);
+
+    const [firstId] = create.mock.calls[0] as [string, any];
+    const [secondId] = create.mock.calls[1] as [string, any];
+    expect(firstId).not.toBe(secondId);
+  });
+
+  it('carries the retry, timeout and delay options onto the row', async () => {
+    const { dm, create } = makeDm();
+    await dm.run('gamesync' as any, '/syncGames' as any, { connection: CONN } as any, {
+      assignedTo: CONN,
+      max_retries: 5,
+      retry_strategy: 'exponential',
+      timeout: 30,
+      delay: 60_000,
+    });
+
+    const [, record] = create.mock.calls[0] as [string, any];
+    expect(record.max_retries).toBe(5);
+    expect(record.retry_strategy).toBe('exponential');
+    expect(record.timeout).toBe(30);
+    expect(record.delay).toBe(60_000);
+  });
+
+  it('serializes the payload and rejects a missing required argument', async () => {
+    const { dm, create } = makeDm();
+    await dm.run('gamesync' as any, '/syncGames' as any, { connection: CONN } as any);
+    const [, record] = create.mock.calls[0] as [string, any];
+    expect(JSON.parse(record.payload)).toEqual({ connection: CONN });
+
+    await expect(
+      dm.run('gamesync' as any, '/syncGames' as any, {} as any)
+    ).rejects.toThrow(/connection/);
+  });
+
+  it('rejects an unknown route and a backend with no outbox table', async () => {
+    const { dm } = makeDm();
+    await expect(
+      dm.run('gamesync' as any, '/nope' as any, {} as any)
+    ).rejects.toThrow(/not found/);
+    await expect(
+      dm.run('noOutbox' as any, '/whatever' as any, {} as any)
+    ).rejects.toThrow(/Outbox table/);
+  });
+});