@spooky-sync/core 0.0.1-canary.237 → 0.0.1-canary.239

package/dist/otel/index.d.ts

@@ -1,4 +1,4 @@
-import { d as PinoTransmit } from "../types.js";
+import { f as PinoTransmit } from "../types.js";
 import { Level } from "pino";
 
 //#region src/otel/index.d.ts

package/dist/types.d.ts

@@ -891,32 +891,6 @@ interface QueryConfig {
    * by an old client that mirrored unflushed reads still self-heals.
    */
   membershipKnown?: boolean;
-  /**
-   * Whether a NON-EMPTY id-set has arrived from the server for this query in
-   * this session. Gates whether an empty read may be believed.
-   *
-   * The server publishes `_00_list_ref` asynchronously — the SSP queues a
-   * view's initial edges to a coalescing flusher and returns from
-   * `fn::query::register` before they land — so an empty read right after
-   * registration says nothing about the query being empty. Believing it (and
-   * mirroring it to the durable `_00_window` row) blanked lists and kept them
-   * blank across reloads. Once a real set has been seen, a later empty one is a
-   * genuine transition and must be honoured, or removed rows resurrect.
-   *
-   * In-memory only: a fresh session must re-earn the right to believe empties.
-   * What does persist is the `confirmed` marker on the `_00_window` row, which
-   * an empty set earns when it arrives with a server row count of zero or after
-   * a non-empty set in the same session.
-   */
-  remoteSeen?: boolean;
-  /**
-   * Consecutive empty id-sets read from the server while `remoteSeen` is still
-   * false. Bounds how long an unconfirmed empty may be ignored, so a window
-   * that genuinely emptied while this device was away is believed on the second
-   * read instead of rendering stale rows forever. Reset by any non-empty set.
-   * In-memory only.
-   */
-  emptyReads?: number;
   /**
    * Key of this query's durable `_00_window` membership row: a hash of
    * `{surql, params}` WITHOUT the `session::id()` salt that `id` carries, so it
@@ -945,6 +919,28 @@ type QueryConfigRecord = QueryConfig & {
  *   flip back to `idle`, so idle status never races ahead of the rows.
  */
 type QueryStatus = 'idle' | 'fetching';
+/**
+ * What the server's `_00_query` row says about a view, read alongside its
+ * `_00_list_ref` edges (registration, the poll, a targeted re-read).
+ *
+ * - `present: false`: the row is not there (or not readable). The SSP no
+ *   longer has this view: a TTL sweep, an SSP reset, a scheduler wipe. An
+ *   empty edge set read at the same time is NOT "no rows", it is "no view".
+ * - `state: 'materializing'`: the SSP has (re)computed the set and its edges
+ *   are in flight; an empty read is "not landed yet".
+ * - `state: 'ready'`: the edges are committed (flipped in the same
+ *   transaction), so `rowCount === 0` with no edges is a real empty result.
+ * - `state: null`: a server that predates the marker; only `rowCount` can be
+ *   used, with its `> 0` = flush window / `0` = empty ambiguity.
+ */
+interface ServerViewMeta {
+  present: boolean;
+  /** `_00_query.rowCount`; `null` when unreadable. */
+  rowCount: number | null;
+  state: 'materializing' | 'ready' | null;
+}
+/** Outcome of handing a server id-set to `DataModule.updateQueryRemoteArray`. */
+type MembershipOutcome = 'applied' | 'ignored' | 'view-lost';
 /**
  * Internal state of a live query.
  */
@@ -961,6 +957,15 @@ interface QueryState {
    * the instant-hydrate, so a hydrate that finishes later must not prime
    * membership over the authoritative set. Ephemeral. */
   serverMembership?: boolean;
+  /**
+   * Set when the server's `_00_query` row for this query went missing while
+   * we held membership (TTL sweep, SSP reset, scheduler wipe). The view is
+   * being re-registered; until a readable row comes back, empty reads are
+   * NOT the server's answer and nothing is dropped. Ephemeral.
+   */
+  viewLost?: boolean;
+  /** Last `_00_query.state` read from the server, for DevTools. Ephemeral. */
+  serverState?: 'materializing' | 'ready' | null;
   /** Set once `notifyQuerySynced` has emitted for this registration lifetime.
    * Ephemeral (unlike the persisted `updateCount`), so a re-registered query
    * always emits at least once even when its records are unchanged — otherwise
@@ -970,6 +975,8 @@ interface QueryState {
   ttlTimer: NodeJS.Timeout | null;
   /** TTL duration in milliseconds. */
   ttlDurationMs: number;
+  /** When the last TTL heartbeat for this query was sent (ms epoch); `undefined` before the first. */
+  lastHeartbeatAt?: number;
   /** Number of times the query has been updated. */
   updateCount: number;
   /** Timestamp (ms) of the last user-visible update, or null before the first
@@ -1041,6 +1048,12 @@ interface QueryTimings {
 }
 type QueryUpdateCallback = (records: Record<string, any>[]) => void;
 type QueryStatusCallback = (status: QueryStatus) => void;
+/**
+ * Observer for a query's authority flips: `true` once server membership is
+ * known for it (registration, poll, durable seed), `false` when it is reset
+ * (bucket switch). What the bindings' `isAuthoritative()` mirrors.
+ */
+type QueryAuthorityCallback = (known: boolean) => void;
 type MutationCallback = (mutations: UpEvent[]) => void;
 type MutationEventType = 'create' | 'update' | 'delete';
 /**
@@ -1101,4 +1114,4 @@ interface DebounceOptions {
   delay?: number;
 }
 //#endregion
-export { Sp00kyQueryResultPromise as A, LocalStore as B, ReconnectConfig as C, RunOptions as D, RegistrationTimings as E, SyncHealthConfig as F, SyncEventSystem as G, DatabaseEventSystem as H, SyncHealthStatus as I, EventDefinition as K, TimingPhase as L, StorageHealthStatus as M, StoreType as N, Sp00kyConfig as O, SyncHealth as P, UpdateOptions as R, QueryUpdateCallback as S, RecordVersionDiff as T, DatabaseEventTypes as U, SealedQuery as V, Logger$1 as W, QueryState as _, MATERIALIZATION_SAMPLE_WINDOW as a, QueryTimeToLive as b, MutationEventType as c, PinoTransmit as d, PreloadOptions as f, QueryHash as g, QueryConfigRecord as h, Level$1 as i, StorageHealth as j, Sp00kyQueryResult as k, PersistenceClient as l, QueryConfig as m, DebounceOptions as n, MutationCallback as o, PreloadRefresh as p, EventSystem as q, EventSubscriptionOptions as r, MutationEvent as s, ConnectionState as t, PhaseStat as u, QueryStatus as v, RecordVersionArray as w, QueryTimings as x, QueryStatusCallback as y, UpEvent as z };
+export { ServerViewMeta as A, TimingPhase as B, QueryTimings as C, RecordVersionDiff as D, RecordVersionArray as E, StorageHealthStatus as F, DatabaseEventSystem as G, UpEvent as H, StoreType as I, SyncEventSystem as J, DatabaseEventTypes as K, SyncHealth as L, Sp00kyQueryResult as M, Sp00kyQueryResultPromise as N, RegistrationTimings as O, StorageHealth as P, SyncHealthConfig as R, QueryTimeToLive as S, ReconnectConfig as T, LocalStore as U, UpdateOptions as V, SealedQuery as W, EventSystem as X, EventDefinition as Y, QueryConfigRecord as _, MATERIALIZATION_SAMPLE_WINDOW as a, QueryStatus as b, MutationEvent as c, PhaseStat as d, PinoTransmit as f, QueryConfig as g, QueryAuthorityCallback as h, Level$1 as i, Sp00kyConfig as j, RunOptions as k, MutationEventType as l, PreloadRefresh as m, DebounceOptions as n, MembershipOutcome as o, PreloadOptions as p, Logger$1 as q, EventSubscriptionOptions as r, MutationCallback as s, ConnectionState as t, PersistenceClient as u, QueryHash as v, QueryUpdateCallback as w, QueryStatusCallback as x, QueryState as y, SyncHealthStatus as z };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@spooky-sync/core",
-  "version": "0.0.1-canary.237",
+  "version": "0.0.1-canary.239",
   "type": "module",
   "sideEffects": false,
   "main": "./dist/index.js",
@@ -60,8 +60,8 @@
     }
   },
   "dependencies": {
-    "@spooky-sync/query-builder": "0.0.1-canary.237",
-    "@spooky-sync/ssp-wasm": "0.0.1-canary.237",
+    "@spooky-sync/query-builder": "0.0.1-canary.239",
+    "@spooky-sync/ssp-wasm": "0.0.1-canary.239",
     "@sqlite.org/sqlite-wasm": "3.53.0-build1",
     "@surrealdb/wasm": "^3.0.3",
     "blurhash": "^2.0.5",

package/src/modules/data/data.activity.test.ts

@@ -0,0 +1,109 @@
+import { describe, it, expect, vi } from 'vitest';
+import { RecordId } from 'surrealdb';
+import { DataModule } from './index';
+import type { QueryState } from '../../types';
+
+/**
+ * The aggregate fetch-activity channel (`subscribeActivity`): one number, how
+ * many queries are inside their outermost fetch cycle, so an app-chrome
+ * "downloading" indicator needs a single subscription instead of one per query.
+ */
+
+function makeLogger(): any {
+  const noop = () => {};
+  const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
+  logger.child = () => logger;
+  return logger;
+}
+
+function makeDataModule(): DataModule<any> {
+  return new DataModule({} as any, {} as any, { tables: [] } as any, makeLogger(), 100);
+}
+
+function makeQueryState(hash: string): QueryState {
+  return {
+    config: {
+      id: new RecordId('_00_query', hash),
+      surql: 'SELECT * FROM user',
+      params: {},
+      localArray: [],
+      remoteArray: [],
+      ttl: '10m',
+      lastActiveAt: new Date(),
+      tableName: 'user',
+    },
+    records: [],
+    ttlTimer: null,
+    ttlDurationMs: 0,
+    updateCount: 0,
+    lastUpdatedAt: null,
+    materializationSamples: [],
+    lastIngestLatencyMs: null,
+    errorCount: 0,
+    status: 'idle',
+    phaseSamples: {},
+    phaseLast: {},
+    registrationTimings: { parseMs: null, planMs: null, snapshotMs: null, wallMs: null },
+  };
+}
+
+describe('DataModule fetch activity', () => {
+  it('emits the current count immediately and on every change', () => {
+    const dm = makeDataModule();
+    (dm as any).activeQueries.set('a', makeQueryState('a'));
+    (dm as any).activeQueries.set('b', makeQueryState('b'));
+    const seen: number[] = [];
+    dm.subscribeActivity((n) => seen.push(n), { immediate: true });
+    expect(seen).toEqual([0]);
+
+    dm.beginFetching('a');
+    dm.beginFetching('b');
+    expect(seen).toEqual([0, 1, 2]);
+    expect(dm.fetchingQueryCount).toBe(2);
+
+    dm.endFetching('a');
+    dm.endFetching('b');
+    expect(seen).toEqual([0, 1, 2, 1, 0]);
+  });
+
+  it('counts a query once however deep its fetch cycles nest', () => {
+    const dm = makeDataModule();
+    (dm as any).activeQueries.set('a', makeQueryState('a'));
+    const seen: number[] = [];
+    dm.subscribeActivity((n) => seen.push(n));
+    dm.beginFetching('a');
+    dm.beginFetching('a'); // overlapping poll round on the same hash
+    dm.endFetching('a');
+    expect(seen).toEqual([1], 'the inner cycle is silent');
+    dm.endFetching('a');
+    expect(seen).toEqual([1, 0]);
+  });
+
+  it('stops notifying after unsubscribe and survives a throwing subscriber', () => {
+    const dm = makeDataModule();
+    (dm as any).activeQueries.set('a', makeQueryState('a'));
+    const bad = vi.fn(() => {
+      throw new Error('boom');
+    });
+    const good = vi.fn();
+    dm.subscribeActivity(bad);
+    const off = dm.subscribeActivity(good);
+    dm.beginFetching('a');
+    expect(good).toHaveBeenCalledWith(1);
+    off();
+    dm.endFetching('a');
+    expect(good).toHaveBeenCalledTimes(1);
+    expect(bad).toHaveBeenCalledTimes(2);
+  });
+
+  it('drops to zero when a bucket switch quiesces in-flight fetches', () => {
+    const dm = makeDataModule();
+    (dm as any).activeQueries.set('a', makeQueryState('a'));
+    const seen: number[] = [];
+    dm.subscribeActivity((n) => seen.push(n));
+    dm.beginFetching('a');
+    dm.quiesce();
+    expect(seen).toEqual([1, 0]);
+    expect(dm.fetchingQueryCount).toBe(0);
+  });
+});

package/src/modules/data/data.authority.test.ts

@@ -0,0 +1,122 @@
+import { describe, it, expect, vi } from 'vitest';
+import { RecordId } from 'surrealdb';
+import { DataModule } from './index';
+import type { QueryState } from '../../types';
+
+/**
+ * A query's AUTHORITY is whether server membership is known for it. It is what
+ * the bindings' `isAuthoritative()` mirrors, and with `hasData` it is what
+ * separates "cold, still waiting for the server" from "the server answered and
+ * the answer is empty". `setMembershipKnown` is the one writer; it fans out to
+ * `subscribeAuthority` listeners and the DevTools hook only on a change.
+ */
+
+function makeLogger(): any {
+  const noop = () => {};
+  const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
+  logger.child = () => logger;
+  return logger;
+}
+
+function makeDataModule(): DataModule<any> {
+  return new DataModule(
+    { unregisterQuery: vi.fn() } as any,
+    {} as any,
+    { tables: [] } as any,
+    makeLogger(),
+    100
+  );
+}
+
+function makeQueryState(hash: string, membershipKnown?: boolean): QueryState {
+  return {
+    config: {
+      id: new RecordId('_00_query', hash),
+      surql: 'SELECT * FROM user',
+      params: {},
+      localArray: [],
+      remoteArray: [],
+      membershipKnown,
+      ttl: '10m',
+      lastActiveAt: new Date(),
+      tableName: 'user',
+    },
+    records: [],
+    ttlTimer: null,
+    ttlDurationMs: 0,
+    updateCount: 0,
+    lastUpdatedAt: null,
+    materializationSamples: [],
+    lastIngestLatencyMs: null,
+    errorCount: 0,
+    status: 'idle',
+    phaseSamples: {},
+    phaseLast: {},
+    registrationTimings: { parseMs: null, planMs: null, snapshotMs: null, wallMs: null },
+  };
+}
+
+describe('DataModule query authority', () => {
+  const hash = 'h1';
+
+  it('reports the current value immediately, false for an unknown query', () => {
+    const dm = makeDataModule();
+    const unknown = vi.fn();
+    dm.subscribeAuthority('nope', unknown, { immediate: true });
+    expect(unknown).toHaveBeenCalledWith(false);
+
+    (dm as any).activeQueries.set(hash, makeQueryState(hash, true));
+    const known = vi.fn();
+    dm.subscribeAuthority(hash, known, { immediate: true });
+    expect(known).toHaveBeenCalledWith(true);
+    expect(dm.isAuthoritative(hash)).toBe(true);
+    expect(dm.isAuthoritative('nope')).toBe(false);
+  });
+
+  it('emits once per transition through the single writer', () => {
+    const dm = makeDataModule();
+    (dm as any).activeQueries.set(hash, makeQueryState(hash));
+    const cb = vi.fn();
+    const hook = vi.fn();
+    dm.onQueryAuthorityChange = hook;
+    dm.subscribeAuthority(hash, cb);
+
+    (dm as any).setMembershipKnown(hash, true);
+    (dm as any).setMembershipKnown(hash, true);
+    expect(cb).toHaveBeenCalledTimes(1);
+    expect(cb).toHaveBeenLastCalledWith(true);
+    expect(hook).toHaveBeenCalledWith(hash, true);
+
+    (dm as any).setMembershipKnown(hash, false);
+    expect(cb).toHaveBeenCalledTimes(2);
+    expect(cb).toHaveBeenLastCalledWith(false);
+    expect(dm.isAuthoritative(hash)).toBe(false);
+  });
+
+  it('is a no-op for an unknown query', () => {
+    const dm = makeDataModule();
+    const hook = vi.fn();
+    dm.onQueryAuthorityChange = hook;
+    (dm as any).setMembershipKnown('nope', true);
+    expect(hook).not.toHaveBeenCalled();
+  });
+
+  it('stops delivering after unsubscribe', () => {
+    const dm = makeDataModule();
+    (dm as any).activeQueries.set(hash, makeQueryState(hash));
+    const cb = vi.fn();
+    const off = dm.subscribeAuthority(hash, cb);
+    off();
+    (dm as any).setMembershipKnown(hash, true);
+    expect(cb).not.toHaveBeenCalled();
+    expect((dm as any).authoritySubscriptions.has(hash)).toBe(false);
+  });
+
+  it('drops the authority subscribers with the query on finalizeDeregister', () => {
+    const dm = makeDataModule();
+    (dm as any).activeQueries.set(hash, makeQueryState(hash));
+    dm.subscribeAuthority(hash, vi.fn());
+    dm.finalizeDeregister(hash);
+    expect((dm as any).authoritySubscriptions.has(hash)).toBe(false);
+  });
+});

package/src/modules/data/data.heartbeat-wake.test.ts

@@ -0,0 +1,115 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { RecordId } from 'surrealdb';
+import { DataModule } from './index';
+import type { QueryState } from '../../types';
+
+/**
+ * The TTL heartbeat is the only thing keeping a view alive on the server, and
+ * browsers freeze timers in hidden tabs. `heartbeatOnWake` re-asserts liveness
+ * for every WATCHED query the moment the tab is back (visibilitychange /
+ * online) and re-anchors its timer. There is no client-side expiry anywhere:
+ * the TTL belongs to the server.
+ */
+
+function makeLogger(): any {
+  const noop = () => {};
+  const logger: any = { debug: noop, info: noop, warn: noop, error: noop, trace: noop };
+  logger.child = () => logger;
+  return logger;
+}
+
+function makeDataModule(): DataModule<any> {
+  return new DataModule({} as any, {} as any, { tables: [] } as any, makeLogger(), 100);
+}
+
+const TTL_MS = 600_000;
+
+function makeQueryState(hash: string): QueryState {
+  return {
+    config: {
+      id: new RecordId('_00_query', hash),
+      surql: 'SELECT * FROM user',
+      params: {},
+      localArray: [],
+      remoteArray: [['user:a', 1]],
+      membershipKnown: true,
+      ttl: '10m',
+      lastActiveAt: new Date(),
+      tableName: 'user',
+    },
+    records: [{ id: 'user:a' }],
+    ttlTimer: null,
+    ttlDurationMs: TTL_MS,
+    updateCount: 0,
+    lastUpdatedAt: null,
+    materializationSamples: [],
+    lastIngestLatencyMs: null,
+    errorCount: 0,
+    status: 'idle',
+    phaseSamples: {},
+    phaseLast: {},
+    registrationTimings: { parseMs: null, planMs: null, snapshotMs: null, wallMs: null },
+  };
+}
+
+function setup() {
+  const dm = makeDataModule();
+  const onHeartbeat = vi.fn();
+  dm.onHeartbeat = onHeartbeat;
+  const watched = makeQueryState('watched');
+  const idle = makeQueryState('idle');
+  (dm as any).activeQueries.set('watched', watched);
+  (dm as any).activeQueries.set('idle', idle);
+  (dm as any).subscriptions.set('watched', new Set([() => {}]));
+  (dm as any).startTTLHeartbeat(watched, 'watched');
+  (dm as any).startTTLHeartbeat(idle, 'idle');
+  return { dm, onHeartbeat, watched, idle };
+}
+
+describe('DataModule heartbeat on wake', () => {
+  beforeEach(() => vi.useFakeTimers());
+  afterEach(() => vi.useRealTimers());
+
+  it('beats every watched query now and re-anchors its timer', async () => {
+    const { dm, onHeartbeat, watched } = setup();
+    await vi.advanceTimersByTimeAsync(200_000);
+
+    expect(dm.heartbeatOnWake()).toBe(1);
+    expect(onHeartbeat).toHaveBeenCalledTimes(1);
+    expect(onHeartbeat).toHaveBeenCalledWith('watched');
+    expect(watched.ttlTimer).not.toBeNull();
+
+    // The regular beat would have been due at 300s; it is now due 300s after
+    // the wake beat instead.
+    await vi.advanceTimersByTimeAsync(TTL_MS / 2 - 1_000);
+    expect(onHeartbeat).toHaveBeenCalledTimes(1);
+    await vi.advanceTimersByTimeAsync(2_000);
+    expect(onHeartbeat).toHaveBeenCalledTimes(2);
+  });
+
+  it('skips queries nobody is watching', () => {
+    const { dm, onHeartbeat } = setup();
+    dm.heartbeatOnWake();
+    expect(onHeartbeat).not.toHaveBeenCalledWith('idle');
+  });
+
+  it('does not beat the same query twice within the gap', async () => {
+    const { dm, onHeartbeat } = setup();
+    expect(dm.heartbeatOnWake()).toBe(1);
+    expect(dm.heartbeatOnWake()).toBe(0);
+    await vi.advanceTimersByTimeAsync(31_000);
+    expect(dm.heartbeatOnWake()).toBe(1);
+    expect(onHeartbeat).toHaveBeenCalledTimes(2);
+  });
+
+  it('never expires local query state on its own', async () => {
+    const { dm, watched, idle } = setup();
+    // Well past the TTL, with no server in the picture.
+    await vi.advanceTimersByTimeAsync(TTL_MS * 3);
+    expect((dm as any).activeQueries.get('watched')).toBe(watched);
+    expect((dm as any).activeQueries.get('idle')).toBe(idle);
+    expect(watched.records).toEqual([{ id: 'user:a' }]);
+    expect(watched.config.remoteArray).toEqual([['user:a', 1]]);
+    expect(watched.config.membershipKnown).toBe(true);
+  });
+});