@spooky-sync/core 0.0.1-canary.236 → 0.0.1-canary.238

package/src/modules/sync/sync.list-ref-meta.test.ts

@@ -0,0 +1,193 @@
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { RecordId } from 'surrealdb';
+import { Sp00kySync } from './sync';
+
+/**
+ * The list_ref readers (registration, poll, targeted re-read) read the
+ * `_00_query` row's `rowCount` and publish `state` next to the edges, dedupe
+ * duplicate edges, and hand the DataModule's verdict back to the poll so a
+ * rejected snapshot (SSP still publishing, view lost) leaves the query alone.
+ */
+
+function makeSync(states: Record<string, { held: Array<[string, number]> }>) {
+  const logger: any = {
+    child: () => logger,
+    debug: () => {},
+    info: () => {},
+    warn: () => {},
+    error: () => {},
+    trace: () => {},
+  };
+  const remote: any = { query: vi.fn() };
+  const queryStates: Record<string, any> = {};
+  for (const [hash, { held }] of Object.entries(states)) {
+    queryStates[hash] = {
+      config: {
+        id: new RecordId('_00_query', hash),
+        localArray: [],
+        remoteArray: held,
+        subqueryRemoteArray: undefined,
+        membershipKnown: held.length > 0,
+      },
+    };
+  }
+  const updateQueryRemoteArray = vi.fn().mockResolvedValue('applied');
+  const dataModule: any = {
+    getCurrentUserId: () => null,
+    getQueryByHash: (h: string) => queryStates[h],
+    getQueryById: (id: RecordId) => queryStates[String(id.id)],
+    getActiveQueryHashes: () => Object.keys(queryStates),
+    updateQueryRemoteArray,
+    scheduleRematerialize: vi.fn(),
+    notifyQuerySynced: vi.fn().mockResolvedValue(undefined),
+  };
+  const sync = new Sp00kySync({} as any, remote, {} as any, dataModule, {} as any, logger);
+  (sync as any).syncQuery = vi.fn().mockResolvedValue(undefined);
+  (sync as any).applySubqueryChildren = vi.fn().mockResolvedValue(undefined);
+  return { sync, remote, dataModule, queryStates, updateQueryRemoteArray };
+}
+
+const rid = (s: string) => {
+  const [table, id] = s.split(':');
+  return new RecordId(table, id);
+};
+
+describe('list_ref snapshot meta', () => {
+  beforeEach(() => vi.clearAllMocks());
+
+  it('reads the row state next to the edges and dedupes duplicate edges (single form)', async () => {
+    const { sync, remote } = makeSync({ h1: { held: [] } });
+    remote.query.mockResolvedValue([
+      [
+        { out: rid('thread:a'), version: 1 },
+        { out: rid('thread:a'), version: 2 },
+        { out: rid('thread:b'), version: 1 },
+      ],
+      { rowCount: 2, state: 'ready' },
+      [],
+    ]);
+
+    const snapshot = await (sync as any).fetchListRefSnapshot(rid('_00_query:h1'));
+
+    expect(snapshot.primary.sort()).toEqual([
+      ['thread:a', 2],
+      ['thread:b', 1],
+    ]);
+    expect(snapshot.meta).toEqual({ present: true, rowCount: 2, state: 'ready' });
+    expect(snapshot.rowCount).toBe(2);
+  });
+
+  it('reports a missing row as present: false', async () => {
+    const { sync, remote } = makeSync({ h1: { held: [] } });
+    remote.query.mockResolvedValue([[], null, []]);
+    const snapshot = await (sync as any).fetchListRefSnapshot(rid('_00_query:h1'));
+    expect(snapshot.meta).toEqual({ present: false, rowCount: null, state: null });
+  });
+
+  it('treats an unknown state value as no state (legacy server)', async () => {
+    const { sync, remote } = makeSync({ h1: { held: [] } });
+    remote.query.mockResolvedValue([[], { rowCount: 0 }, []]);
+    const snapshot = await (sync as any).fetchListRefSnapshot(rid('_00_query:h1'));
+    expect(snapshot.meta).toEqual({ present: true, rowCount: 0, state: null });
+  });
+
+  it('fills meta per query in the batched form and dedupes', async () => {
+    const { sync, remote } = makeSync({ h1: { held: [] }, h2: { held: [] } });
+    remote.query.mockResolvedValue([
+      [
+        { in: rid('_00_query:h1'), out: rid('thread:a'), version: 1, parent: null },
+        { in: rid('_00_query:h1'), out: rid('thread:a'), version: 3, parent: null },
+        { in: rid('_00_query:h2'), out: rid('thread:z'), version: 1, parent: null },
+      ],
+      [
+        { id: rid('_00_query:h1'), rowCount: 1, state: 'materializing' },
+        { id: rid('_00_query:h2'), rowCount: 1, state: 'ready' },
+      ],
+    ]);
+
+    const out = await (sync as any).fetchListRefSnapshots(['h1', 'h2']);
+
+    expect(out.get('h1').primary).toEqual([['thread:a', 3]]);
+    expect(out.get('h1').meta).toEqual({ present: true, rowCount: 1, state: 'materializing' });
+    expect(out.get('h2').meta).toEqual({ present: true, rowCount: 1, state: 'ready' });
+  });
+
+  it('re-reads a held query one at a time when its row is missing from the batch', async () => {
+    const { sync, remote } = makeSync({ h1: { held: [['thread:a', 1]] }, h2: { held: [] } });
+    // Batch: no edges and no row for h1, which we hold rows for.
+    remote.query.mockResolvedValue([[], [{ id: rid('_00_query:h2'), rowCount: 0, state: 'ready' }]]);
+    const single = vi.fn().mockResolvedValue({
+      primary: [],
+      subquery: [],
+      rowCount: null,
+      meta: { present: false, rowCount: null, state: null },
+    });
+    (sync as any).fetchListRefSnapshot = single;
+
+    const out = await (sync as any).fetchListRefSnapshots(['h1', 'h2']);
+
+    expect(single).toHaveBeenCalled();
+    expect(out.get('h1').meta.present).toBe(false);
+  });
+
+  describe('applyListRefSnapshot', () => {
+    const snapshot = (primary: Array<[string, number]>, meta: any) => ({
+      primary,
+      subquery: [],
+      rowCount: meta.rowCount,
+      meta,
+    });
+
+    it('hands the verdict back and touches nothing when the set is rejected', async () => {
+      const { sync, dataModule, updateQueryRemoteArray } = makeSync({ h1: { held: [['thread:a', 1]] } });
+      updateQueryRemoteArray.mockResolvedValue('ignored');
+
+      const result = await (sync as any).applyListRefSnapshot(
+        'h1',
+        snapshot([], { present: true, rowCount: 1, state: 'materializing' })
+      );
+
+      expect(result).toEqual({ changed: false, outcome: 'ignored' });
+      expect(updateQueryRemoteArray).toHaveBeenCalledWith('h1', [], {
+        meta: { present: true, rowCount: 1, state: 'materializing' },
+      });
+      expect(dataModule.scheduleRematerialize).not.toHaveBeenCalled();
+      expect(dataModule.notifyQuerySynced).not.toHaveBeenCalled();
+      expect((sync as any).applySubqueryChildren).not.toHaveBeenCalled();
+    });
+
+    it('does the same for a lost view', async () => {
+      const { sync, dataModule, updateQueryRemoteArray } = makeSync({ h1: { held: [['thread:a', 1]] } });
+      updateQueryRemoteArray.mockResolvedValue('view-lost');
+      const result = await (sync as any).applyListRefSnapshot(
+        'h1',
+        snapshot([], { present: false, rowCount: null, state: null })
+      );
+      expect(result).toEqual({ changed: false, outcome: 'view-lost' });
+      expect(dataModule.notifyQuerySynced).not.toHaveBeenCalled();
+    });
+
+    it('re-materializes and re-renders removals when the set is applied', async () => {
+      const { sync, dataModule } = makeSync({ h1: { held: [['thread:a', 1], ['thread:b', 1]] } });
+      const result = await (sync as any).applyListRefSnapshot(
+        'h1',
+        snapshot([['thread:a', 1]], { present: true, rowCount: 1, state: 'ready' })
+      );
+      expect(result).toEqual({ changed: true, outcome: 'applied' });
+      expect(dataModule.scheduleRematerialize).toHaveBeenCalledWith('h1');
+      expect(dataModule.notifyQuerySynced).toHaveBeenCalledWith('h1');
+      expect((sync as any).applySubqueryChildren).toHaveBeenCalled();
+    });
+
+    it('is quiet when the server set equals what is held', async () => {
+      const { sync, dataModule, updateQueryRemoteArray } = makeSync({ h1: { held: [['thread:a', 1]] } });
+      const result = await (sync as any).applyListRefSnapshot(
+        'h1',
+        snapshot([['thread:a', 1]], { present: true, rowCount: 1, state: 'ready' })
+      );
+      expect(result).toEqual({ changed: false, outcome: 'unchanged' });
+      expect(updateQueryRemoteArray).not.toHaveBeenCalled();
+      expect(dataModule.scheduleRematerialize).not.toHaveBeenCalled();
+    });
+  });
+});

package/src/modules/sync/sync.live-removal.test.ts

@@ -1,19 +1,28 @@
-import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
 import { RecordId } from 'surrealdb';
 import { Sp00kySync } from './sync';
 
 // A LIVE `_00_list_ref` DELETE is how one window learns another deleted a record.
 //
-// `remoteArray` — the authoritative membership rows are now rendered FROM — used
-// to be written only by registration and the poll, so a LIVE removal left the
-// departed id in the list (in memory AND persisted) until the next poll tick: up
-// to 5s of showing a deleted row, and on a page that then went offline, forever.
+// It is also how a view teardown arrives: the TTL sweep, an SSP reset, a
+// scheduler wipe and a full republish all delete a view's edges, one DELETE per
+// row, indistinguishable from a peer deleting rows. Obeying them on arrival
+// blanked painted lists n-1 rows at a time. So removals are held briefly,
+// coalesced per query, and VERIFIED upstream before membership moves: an id
+// that no longer exists is dropped (its body is deleted inside the same sync
+// round); anything else is settled by re-reading the server's whole set for the
+// query, where a missing `_00_query` row means "view lost, keep everything".
 //
-// A removal-only diff also gets no re-render for free: `runSyncForQuery` leaves
-// `fetching` false, and a removal needs no record fetch to trigger a stream
-// update, so the notify has to be forced.
+// CREATE/UPDATE are applied on arrival as before.
 
-function makeSync(opts: { membershipKnown?: boolean } = {}) {
+const COALESCE_MS = 100;
+
+function makeSync(
+  opts: {
+    membershipKnown?: boolean;
+    verify?: { confirmedRemovedIds?: string[]; stillRemoteIds?: string[] };
+  } = {}
+) {
   const logger: any = {
     child: () => logger,
     debug: () => {},
@@ -41,6 +50,7 @@ function makeSync(opts: { membershipKnown?: boolean } = {}) {
 
   const updateQueryRemoteArray = vi.fn(async (_h: string, next: any) => {
     queryState.config.remoteArray = next;
+    return 'applied';
   });
   const notifyQuerySynced = vi.fn().mockResolvedValue(undefined);
   const dataModule: any = {
@@ -62,7 +72,16 @@ function makeSync(opts: { membershipKnown?: boolean } = {}) {
     logger
   );
   // `runSyncForQuery` drives the engine + scheduler; both are out of scope here.
-  (sync as any).runSyncForQuery = vi.fn().mockResolvedValue(undefined);
+  // It reports which removed ids were verified gone upstream and which still
+  // exist there.
+  const runSyncForQuery = vi.fn().mockResolvedValue({
+    confirmedRemovedIds: opts.verify?.confirmedRemovedIds ?? [],
+    stillRemoteIds: opts.verify?.stillRemoteIds ?? [],
+  });
+  (sync as any).runSyncForQuery = runSyncForQuery;
+  // The targeted server re-read a non-confirmed removal falls back to.
+  const refetch = vi.fn().mockResolvedValue({ changed: false, outcome: 'unchanged' });
+  (sync as any).refetchListRefForQuery = refetch;
 
   const live = (action: 'CREATE' | 'UPDATE' | 'DELETE', id: string, version: number) =>
     (sync as any).handleRemoteListRefChange(
@@ -71,26 +90,111 @@ function makeSync(opts: { membershipKnown?: boolean } = {}) {
       new RecordId('thread', id),
       version
     ) as Promise<void>;
+  // Let the coalescing window and the zero-delay re-read timer fire.
+  const settle = () => vi.advanceTimersByTimeAsync(COALESCE_MS + 10);
 
-  return { sync, queryState, updateQueryRemoteArray, notifyQuerySynced, live };
+  return {
+    sync,
+    queryState,
+    updateQueryRemoteArray,
+    notifyQuerySynced,
+    runSyncForQuery,
+    refetch,
+    live,
+    settle,
+  };
 }
 
 describe('LIVE list_ref removal → membership', () => {
-  beforeEach(() => vi.clearAllMocks());
+  beforeEach(() => {
+    vi.useFakeTimers();
+    vi.clearAllMocks();
+  });
+  afterEach(() => vi.useRealTimers());
 
-  it('drops the removed id from remoteArray immediately', async () => {
-    const { queryState, updateQueryRemoteArray, live } = makeSync();
+  it('does not move membership on arrival', async () => {
+    const { queryState, updateQueryRemoteArray, runSyncForQuery, live } = makeSync({
+      verify: { confirmedRemovedIds: ['thread:b'] },
+    });
 
     await live('DELETE', 'b', 2);
 
-    expect(updateQueryRemoteArray).toHaveBeenCalledWith('h1', [['thread:a', 1]]);
-    expect(queryState.config.remoteArray).toEqual([['thread:a', 1]]);
+    expect(updateQueryRemoteArray).not.toHaveBeenCalled();
+    expect(runSyncForQuery).not.toHaveBeenCalled();
+    expect(queryState.config.remoteArray).toEqual([
+      ['thread:a', 1],
+      ['thread:b', 1],
+    ]);
   });
 
-  it('forces a re-render for a removal-only diff', async () => {
-    const { notifyQuerySynced, live } = makeSync();
+  it('drops a removed id once it is verified gone upstream', async () => {
+    const { queryState, updateQueryRemoteArray, notifyQuerySynced, runSyncForQuery, refetch, live, settle } =
+      makeSync({ verify: { confirmedRemovedIds: ['thread:b'] } });
+
     await live('DELETE', 'b', 2);
+    await settle();
+
+    expect(runSyncForQuery).toHaveBeenCalledTimes(1);
+    const [, diff] = runSyncForQuery.mock.calls[0];
+    expect(diff.added).toEqual([]);
+    expect(diff.updated).toEqual([]);
+    expect(diff.removed.map(String)).toEqual(['thread:b']);
+    expect(updateQueryRemoteArray).toHaveBeenCalledWith('h1', [['thread:a', 1]], {
+      verifiedRemoval: true,
+    });
+    expect(queryState.config.remoteArray).toEqual([['thread:a', 1]]);
+    // A removal-only diff gets no re-render for free: it has to be forced.
     expect(notifyQuerySynced).toHaveBeenCalledWith('h1');
+    expect(refetch).not.toHaveBeenCalled();
+  });
+
+  it('keeps a row that still exists upstream and re-reads the server set instead', async () => {
+    // The row left this view's edges but is still there: a membership change
+    // to be read back as a whole set, or a view teardown to be recognised by
+    // its missing `_00_query` row. Either way, not a blind removal.
+    const { queryState, updateQueryRemoteArray, notifyQuerySynced, refetch, live, settle } = makeSync({
+      verify: { stillRemoteIds: ['thread:b'] },
+    });
+
+    await live('DELETE', 'b', 2);
+    await settle();
+
+    expect(updateQueryRemoteArray).not.toHaveBeenCalled();
+    expect(notifyQuerySynced).not.toHaveBeenCalled();
+    expect(queryState.config.remoteArray).toEqual([
+      ['thread:a', 1],
+      ['thread:b', 1],
+    ]);
+    expect(refetch).toHaveBeenCalledWith('h1');
+  });
+
+  it('coalesces a burst of removals into one verification', async () => {
+    // A sweep or a republish delivers a view's whole set within a few ms.
+    const { runSyncForQuery, live, settle } = makeSync({
+      verify: { confirmedRemovedIds: ['thread:a', 'thread:b'] },
+    });
+
+    await live('DELETE', 'a', 2);
+    await vi.advanceTimersByTimeAsync(30);
+    await live('DELETE', 'b', 2);
+    await live('DELETE', 'b', 2);
+    await settle();
+
+    expect(runSyncForQuery).toHaveBeenCalledTimes(1);
+    const [, diff] = runSyncForQuery.mock.calls[0];
+    expect(diff.removed.map(String).sort()).toEqual(['thread:a', 'thread:b']);
+  });
+
+  it('falls back to a server re-read when the verification could not run', async () => {
+    // `handleRemovedRecords` answers with nothing on a failed existence check:
+    // neither confirmed nor still-remote. Nothing may be dropped on that.
+    const { updateQueryRemoteArray, refetch, live, settle } = makeSync({ verify: {} });
+
+    await live('DELETE', 'b', 2);
+    await settle();
+
+    expect(updateQueryRemoteArray).not.toHaveBeenCalled();
+    expect(refetch).toHaveBeenCalledWith('h1');
   });
 
   it('does not force a re-render when rows were added (the stream update covers it)', async () => {
@@ -116,7 +220,7 @@ describe('LIVE list_ref removal → membership', () => {
   // fetch diff is empty. Membership still has to be recorded, or the row lives
   // on the settled-write grace alone until the poll catches it.
   it('adds membership for a row the circuit already holds at the server version', async () => {
-    const { queryState, updateQueryRemoteArray, sync, live } = makeSync();
+    const { queryState, updateQueryRemoteArray, runSyncForQuery, live } = makeSync();
     queryState.config.localArray = [
       ['thread:a', 1],
       ['thread:b', 1],
@@ -131,7 +235,7 @@ describe('LIVE list_ref removal → membership', () => {
       ['thread:c', 1],
     ]);
     // …without a refetch: the diff handed to the sync is empty.
-    expect((sync as any).runSyncForQuery).toHaveBeenCalledWith('h1', {
+    expect(runSyncForQuery).toHaveBeenCalledWith('h1', {
       added: [],
       updated: [],
       removed: [],
@@ -153,14 +257,22 @@ describe('LIVE list_ref removal → membership', () => {
 
   it('leaves membership alone while it is still unknown', async () => {
     // Nothing authoritative has arrived yet, so there is no list to amend —
-    // registration will supply the whole thing shortly.
-    const { updateQueryRemoteArray, live } = makeSync({ membershipKnown: false });
+    // registration will supply the whole thing shortly. The body delete of a
+    // verified-gone row still happens inside the sync round.
+    const { updateQueryRemoteArray, runSyncForQuery, live, settle } = makeSync({
+      membershipKnown: false,
+      verify: { confirmedRemovedIds: ['thread:b'] },
+    });
+
     await live('DELETE', 'b', 2);
+    await settle();
+
+    expect(runSyncForQuery).toHaveBeenCalledTimes(1);
     expect(updateQueryRemoteArray).not.toHaveBeenCalled();
   });
 
   it('is a no-op for an unknown query', async () => {
-    const { sync, updateQueryRemoteArray } = makeSync();
+    const { sync, updateQueryRemoteArray, runSyncForQuery, settle } = makeSync();
     (sync as any).dataModule.getQueryById = vi.fn().mockReturnValue(undefined);
 
     await (sync as any).handleRemoteListRefChange(
@@ -169,7 +281,9 @@ describe('LIVE list_ref removal → membership', () => {
       new RecordId('thread', 'b'),
       2
     );
+    await settle();
 
     expect(updateQueryRemoteArray).not.toHaveBeenCalled();
+    expect(runSyncForQuery).not.toHaveBeenCalled();
   });
 });