@spooky-sync/core 0.0.1-canary.33 → 0.0.1-canary.331

package/src/state/selectors.test.ts

@@ -0,0 +1,188 @@
+import { describe, expect, it } from 'vitest';
+import * as S from './selectors';
+import * as R from './reducers';
+import { buildEntry, buildOutboxItem, buildState } from '../testing/build';
+
+const e = (hash: string, over: Parameters<typeof buildEntry>[0] = {}) => buildEntry({ ...over, def: { hash, ...over.def } });
+
+describe('basic lookups', () => {
+  it('queryByHash / activeHashes / hashesForTable / queryStatus', () => {
+    const s = buildState([e('a', { def: { tableName: 't1' } }), e('b', { def: { tableName: 't2' } })]);
+    expect(S.queryByHash(s, 'a')?.def.hash).toBe('a');
+    expect(S.activeHashes(s)).toEqual(['a', 'b']);
+    expect(S.hashesForTable(s, 't2')).toEqual(['b']);
+    expect(S.queryStatus(s, 'a')).toBe('idle');
+    expect(S.queryStatus(s, 'zz')).toBeUndefined();
+  });
+});
+
+describe('overlay and outbox counts', () => {
+  it('derives writes/deletes from pending and acked items', () => {
+    const s = buildState(
+      [],
+      R.outboxReplace([
+        buildOutboxItem({ id: '1', type: 'create', recordId: 'thing:1' }),
+        buildOutboxItem({ id: '2', type: 'update', recordId: 'thing:2', status: 'acked', ackedAt: 1 }),
+        buildOutboxItem({ id: '3', type: 'delete', recordId: 'thing:3' }),
+      ])
+    );
+    const o = S.overlay(s);
+    expect([...o.writes].sort()).toEqual(['thing:1', 'thing:2']);
+    expect([...o.deletes]).toEqual(['thing:3']);
+    expect([...S.pendingDeleteIds(s)]).toEqual(['thing:3']);
+    expect(S.hasAckedWrites(s)).toBe(true);
+    expect(S.pendingMutationCount(s)).toBe(2);
+  });
+});
+
+describe('needed / planFetch / settled', () => {
+  const live = { phase: 'live' } as const;
+  it('needed compares membership versions against local versions and skips pending deletes', () => {
+    const s = buildState(
+      [e('a', { lifecycle: live, remoteArray: [['thing:1', 2], ['thing:2', 1], ['thing:3', 1]] })],
+      R.setVersions([['thing:1', 2], ['thing:2', 0]]),
+      R.outboxReplace([buildOutboxItem({ type: 'delete', recordId: 'thing:3' })])
+    );
+    expect(S.needed(s, 'a')).toEqual([['thing:2', 1]]);
+    expect(S.needed(s, 'missing')).toEqual([]);
+    const cached = buildState([e('c', { lifecycle: { phase: 'cached' }, remoteArray: [['thing:9', 1]] })]);
+    expect(S.needed(cached, 'c')).toEqual([]);
+  });
+  it('planFetch dedupes across queries (highest version wins), includes subquery children, chunks', () => {
+    const s = buildState([
+      e('a', { lifecycle: live, remoteArray: [['thing:1', 1], ['thing:2', 3]] }),
+      e('b', { lifecycle: live, remoteArray: [['thing:2', 1], ['thing:3', 1]], subqueryRemoteArray: [['child:1', 2]] }),
+      e('c', { lifecycle: live, subqueryRemoteArray: [['child:1', 1], ['child:2', 1]] }),
+      e('d', { lifecycle: { phase: 'cached' }, remoteArray: [['thing:9', 1]] }),
+    ]);
+    const plan = S.planFetch(s, 2);
+    expect(plan.hashes).toEqual(['a', 'b']);
+    expect(plan.chunks).toEqual([['thing:1', 'thing:2'], ['thing:3', 'child:1'], ['child:2']]);
+    expect(plan.versions.get('thing:2')).toBe(3);
+    expect(plan.versions.get('child:1')).toBe(2);
+    expect(S.planFetch(buildState()).chunks).toEqual([]);
+    expect(S.neededChildren(s, 'missing')).toEqual([]);
+    expect(S.neededChildren(R.setVersions([['child:1', 2]])(s), 'b')).toEqual([]);
+  });
+  it('settled requires live, complete, clean, notified', () => {
+    const base = e('a', { lifecycle: { ...live, notified: true }, remoteArray: [['thing:1', 1]] });
+    const s = { ...buildState([base], R.setVersions([['thing:1', 1]])), dirty: new Set<string>() };
+    expect(S.settled(s, 'a')).toBe(true);
+    expect(S.settled(R.markDirty(['a'])(s), 'a')).toBe(false);
+    expect(S.settled(R.setVersions([['thing:1', 0]])(s), 'a')).toBe(false);
+    expect(S.settled(buildState([e('a', { lifecycle: { phase: 'cached', notified: true } })]), 'a')).toBe(false);
+    expect(S.settled(buildState([e('a', { lifecycle: live })]), 'a')).toBe(false);
+    expect(S.settled(s, 'missing')).toBe(false);
+    expect(S.settleFailed(s, 'missing')).toBe(true);
+    expect(S.settleFailed(s, 'a')).toBe(false);
+    expect(S.settleFailed(R.applyLifecycle('a', { type: 'remote-failed' })(s), 'a')).toBe(true);
+  });
+  it('queryStatus stays fetching until a live query is settled', () => {
+    const base = e('a', { lifecycle: { ...live, notified: true }, remoteArray: [['thing:1', 1]] });
+    const s = { ...buildState([base], R.setVersions([['thing:1', 1]])), dirty: new Set<string>() };
+    expect(S.queryStatus(s, 'a')).toBe('idle');
+    // Membership in, body still downloading (registration already ended its cycle).
+    expect(S.queryStatus(R.setVersions([['thing:1', 0]])(s), 'a')).toBe('fetching');
+    // Body landed, not yet shown to subscribers.
+    expect(S.queryStatus(R.markDirty(['a'])(s), 'a')).toBe('fetching');
+    // A cold query with no fetch in flight has nothing to wait for.
+    expect(S.queryStatus(buildState([e('c', { lifecycle: { phase: 'cold' } })]), 'c')).toBe('idle');
+    expect(S.queryStatus(R.applyLifecycle('a', { type: 'fetch-begin' })(s), 'a')).toBe('fetching');
+    expect(S.queryStatus(s, 'missing')).toBeUndefined();
+  });
+  it('fetchingQueryCount counts entries with fetch depth', () => {
+    const s = buildState([e('a', { lifecycle: { fetchDepth: 2 } }), e('b')]);
+    expect(S.fetchingQueryCount(s)).toBe(1);
+  });
+  it('fetchingQueryCount leaves out background re-verification', () => {
+    const s = buildState([
+      e('verifying', { lifecycle: { phase: 'cached', fetchDepth: 1, backgroundDepth: 1 } }),
+      e('downloading', { lifecycle: { phase: 'live', fetchDepth: 2, backgroundDepth: 1 } }),
+    ]);
+    expect(S.fetchingQueryCount(s)).toBe(1);
+    expect(S.queryStatus(s, 'verifying')).toBe('fetching');
+  });
+});
+
+describe('registration / eviction / ttl', () => {
+  it('desiredRegistrations lists unregistered queries', () => {
+    const s = buildState([e('a'), e('b', { lifecycle: { remote: 'registered' } }), e('c', { lifecycle: { remote: 'registering' } })]);
+    expect(S.desiredRegistrations(s)).toEqual(['a']);
+  });
+  it('evictable respects subscribers and ttl', () => {
+    const s = buildState([
+      e('gone', { lastSubscriberLeftAt: 0, def: { ttlMs: 100 } }),
+      e('fresh', { lastSubscriberLeftAt: 950, def: { ttlMs: 100 } }),
+      e('watched', { subscribers: 1, lastSubscriberLeftAt: 0, def: { ttlMs: 100 } }),
+      e('never', { def: { ttlMs: 100 } }),
+    ]);
+    expect(S.evictable(s, 1000)).toEqual(['gone']);
+  });
+  it('shortestTtlMs', () => {
+    expect(S.shortestTtlMs(buildState())).toBeNull();
+    expect(S.shortestTtlMs(buildState([e('a', { def: { ttlMs: 500 } }), e('b', { def: { ttlMs: 100 } })]))).toBe(100);
+  });
+});
+
+describe('toQueryState', () => {
+  it('projects the legacy shape with derived flags', () => {
+    const entry = e('a', {
+      lifecycle: { phase: 'view-lost', fetchDepth: 1, notified: true },
+      remoteArray: [['thing:1', 1]],
+      subqueryRemoteArray: [['child:1', 1]],
+      records: [{ id: 'thing:1' }],
+      serverState: 'ready',
+      lastHeartbeatAt: 123,
+    });
+    const qs = S.toQueryState(entry);
+    expect(qs.config.membershipKnown).toBe(true);
+    expect(qs.config.membershipKey).toBe('view-a');
+    expect(qs.viewLost).toBe(true);
+    expect(qs.serverMembership).toBe(true);
+    expect(qs.status).toBe('fetching');
+    expect(qs.syncNotified).toBe(true);
+    expect(qs.config.subqueryRemoteArray).toEqual([['child:1', 1]]);
+    expect(qs.config.lastActiveAt.getTime()).toBe(123);
+    expect(qs.records).toEqual([{ id: 'thing:1' }]);
+    const cold = S.toQueryState(e('b'));
+    expect(cold.config.membershipKnown).toBe(false);
+    expect(cold.config.subqueryRemoteArray).toBeUndefined();
+    expect(cold.lastHeartbeatAt).toBeUndefined();
+    expect(cold.config.lastActiveAt.getTime()).toBe(1_700_000_000_000);
+  });
+});
+
+describe('phaseTimings', () => {
+  it('summarizes every phase, with nulls before the first sample', () => {
+    const cold = S.phaseTimings(e('a'));
+    expect(cold.ssp).toEqual({ lastMs: null, p50: null, p90: null, p99: null, count: 0 });
+    expect(cold.localFetch.count).toBe(0);
+    let s = buildState([e('a')]);
+    for (const ms of [5, 1, 9, 3]) s = R.recordIngest('a', ms)(s);
+    s = R.recordPhase('a', 'remoteFetch', 7)(s);
+    s = R.recordError('a')(s);
+    const t = S.phaseTimings(s.queries.get('a')!);
+    expect(t.ssp).toEqual({ lastMs: 3, p50: 5, p90: 9, p99: 9, count: 4 });
+    expect(t.remoteFetch).toEqual({ lastMs: 7, p50: 7, p90: 7, p99: 7, count: 1 });
+    expect(t.errorCount).toBe(1);
+    expect(t.registration.parseMs).toBeNull();
+  });
+});
+
+describe('view index selectors', () => {
+  it('vouched is the union of every view row, memoized on the index', () => {
+    const s = R.reloadViews(new Map([['v1', ['t:1', 't:2']], ['v2', ['t:2', 'c:1']]]))(buildState());
+    expect([...S.vouched(s)]).toEqual(['t:1', 't:2', 'c:1']);
+    expect(S.vouched({ ...s, dirty: new Set(['x']) })).toBe(S.vouched(s));
+  });
+  it('retained: vouched, held in memory (members, children), in the outbox or a debounced write, or bookkeeping', () => {
+    const s = R.compose(
+      R.reloadViews(new Map([['v', ['t:vouched']]])),
+      R.outboxReplace([buildOutboxItem({ recordId: 't:outbox' })]),
+      (st) => ({ ...st, pendingWrites: new Map([['k', { key: 'k', table: 't', recordId: 't:debounced', data: {}, before: null, firstAt: 0, mirrorId: null }]]) })
+    )(buildState([e('a', { remoteArray: [['t:member', 1]], subqueryRemoteArray: [['t:child', 1]] })]));
+    const keep = S.retained(s);
+    for (const id of ['t:vouched', 't:member', 't:child', 't:outbox', 't:debounced', '_00_view:x']) expect(keep(id)).toBe(true);
+    expect(keep('t:orphan')).toBe(false);
+  });
+});

package/src/state/selectors.ts

@@ -0,0 +1,258 @@
+import type { PhaseStat, QueryHash, QueryState, QueryStatus, QueryTimings, RecordVersionArray } from '../types';
+import type { ClientState, QueryEntry } from './client-state';
+import { deriveStatus, hasServerMembership, isAuthoritative, isForegroundFetching } from './lifecycle';
+import { FETCH_CHUNK } from '../kernel/constants';
+
+/** Pure reads over `ClientState`. Sagas use them through `state.read`. */
+
+export const queryByHash = (s: ClientState, hash: QueryHash): QueryEntry | undefined => s.queries.get(hash);
+
+export const activeHashes = (s: ClientState): QueryHash[] => [...s.queries.keys()];
+
+export const hashesForTable = (s: ClientState, table: string): QueryHash[] =>
+  [...s.queries].filter(([, e]) => e.def.tableName === table).map(([h]) => h);
+
+/**
+ * The status subscribers see. `fetching` while a fetch cycle is in flight,
+ * and also while a live query is not yet `settled`: the server's membership
+ * is in but its rows are still downloading or not yet shown. Registration
+ * ends its cycle as soon as membership is applied and the body fetch starts
+ * its own a moment later, so the fetch count alone read `idle` in between,
+ * and a binding that settles on "membership known and idle" told the app a
+ * row did not exist while it was on its way (a by-id screen showed "not
+ * found" for a moment, or for as long as the bodies took).
+ */
+export const queryStatus = (s: ClientState, hash: QueryHash): QueryStatus | undefined => {
+  const e = s.queries.get(hash);
+  if (!e) return undefined;
+  if (deriveStatus(e.lifecycle) === 'fetching') return 'fetching';
+  return e.lifecycle.phase === 'live' && !settled(s, hash) ? 'fetching' : 'idle';
+};
+
+export interface Overlay {
+  /** Ids with an unsynced or just-acked create/update. */
+  readonly writes: ReadonlySet<string>;
+  /** Ids with an unsynced or just-acked delete. */
+  readonly deletes: ReadonlySet<string>;
+}
+
+/** The optimistic overlay, derived from the outbox (pending AND acked items). */
+export function overlay(s: ClientState): Overlay {
+  const writes = new Set<string>();
+  const deletes = new Set<string>();
+  for (const item of s.outbox) {
+    if (item.type === 'delete') deletes.add(item.recordId);
+    else writes.add(item.recordId);
+  }
+  return { writes, deletes };
+}
+
+const vouchedCache = new WeakMap<ClientState['views'], ReadonlySet<string>>();
+
+/**
+ * Every id some durable `_00_view` row vouches for. Memoized on the index,
+ * which only changes when a view row does.
+ */
+export function vouched(s: ClientState): ReadonlySet<string> {
+  const hit = vouchedCache.get(s.views);
+  if (hit) return hit;
+  const out = new Set<string>();
+  for (const ids of s.views.values()) for (const id of ids) out.add(id);
+  vouchedCache.set(s.views, out);
+  return out;
+}
+
+/**
+ * Bodies the GC must keep: `_00_` bookkeeping, anything a durable view
+ * vouches for, anything a query in memory names (members or children), and
+ * anything a write is still in flight for.
+ */
+export function retained(s: ClientState): (id: string) => boolean {
+  const held = new Set<string>();
+  for (const e of s.queries.values()) {
+    for (const [id] of e.remoteArray) held.add(id);
+    for (const [id] of e.subqueryRemoteArray) held.add(id);
+  }
+  for (const item of s.outbox) held.add(item.recordId);
+  for (const write of s.pendingWrites.values()) held.add(write.recordId);
+  const known = vouched(s);
+  return (id) => id.startsWith('_00_') || known.has(id) || held.has(id);
+}
+
+/**
+ * Ids something live still needs: a mounted query's membership or subquery
+ * children, or a local write not yet settled. Unlike `retained`, durable view
+ * rows of queries nobody runs right now do not count: a row the server just
+ * took out of a live query must stop showing even if an old view named it.
+ */
+export function activelyHeld(s: ClientState): ReadonlySet<string> {
+  const held = new Set<string>();
+  for (const e of s.queries.values()) {
+    for (const [id] of e.remoteArray) held.add(id);
+    for (const [id] of e.subqueryRemoteArray) held.add(id);
+  }
+  for (const item of s.outbox) held.add(item.recordId);
+  for (const write of s.pendingWrites.values()) held.add(write.recordId);
+  return held;
+}
+
+/** Ids of `before` that `after` no longer names. */
+export function departedIds(before: RecordVersionArray, after: RecordVersionArray): string[] {
+  if (before.length === 0) return [];
+  const kept = new Set(after.map(([id]) => id));
+  return before.filter(([id]) => !kept.has(id)).map(([id]) => id);
+}
+
+export const pendingDeleteIds = (s: ClientState): ReadonlySet<string> =>
+  new Set(s.outbox.filter((i) => i.type === 'delete').map((i) => i.recordId));
+
+export const hasAckedWrites = (s: ClientState): boolean => s.outbox.some((i) => i.status === 'acked');
+
+export const pendingMutationCount = (s: ClientState): number =>
+  s.outbox.filter((i) => i.status === 'pending').length;
+
+/**
+ * Queries with traffic the user is waiting on. A re-registration of a query
+ * that already shows rows (every mounted query after a reload) is left out:
+ * it only confirms what the cache painted, and any newer body it finds is
+ * fetched by `fetchRows`, which counts.
+ */
+export const fetchingQueryCount = (s: ClientState): number =>
+  [...s.queries.values()].filter((e) => isForegroundFetching(e.lifecycle)).length;
+
+/** Membership entries whose body is missing or older locally. */
+export function needed(s: ClientState, hash: QueryHash): RecordVersionArray {
+  const e = s.queries.get(hash);
+  if (!e || !hasServerMembership(e.lifecycle)) return [];
+  const deletes = pendingDeleteIds(s);
+  return e.remoteArray.filter(([id, v]) => !deletes.has(id) && (s.versions.get(id) ?? -1) < v);
+}
+
+/** Subquery child bodies missing or stale locally (never part of `settled`). */
+export function neededChildren(s: ClientState, hash: QueryHash): RecordVersionArray {
+  const e = s.queries.get(hash);
+  if (!e || e.subqueryRemoteArray.length === 0) return [];
+  return e.subqueryRemoteArray.filter(([id, v]) => (s.versions.get(id) ?? -1) < v);
+}
+
+export interface FetchPlan {
+  /** Queries whose primary membership needs bodies (they flip to `fetching`). */
+  readonly hashes: QueryHash[];
+  readonly chunks: string[][];
+  /** Highest requested version per id, across every query naming it. */
+  readonly versions: ReadonlyMap<string, number>;
+}
+
+/** Cross-query, deduped, chunked list of ids to pull from the server. */
+export function planFetch(s: ClientState, chunkSize = FETCH_CHUNK): FetchPlan {
+  const versions = new Map<string, number>();
+  const hashes: QueryHash[] = [];
+  const add = (pairs: RecordVersionArray) => {
+    for (const [id, v] of pairs) versions.set(id, Math.max(v, versions.get(id) ?? -1));
+  };
+  for (const hash of s.queries.keys()) {
+    const missing = needed(s, hash);
+    if (missing.length > 0) hashes.push(hash);
+    add(missing);
+    add(neededChildren(s, hash));
+  }
+  const all = [...versions.keys()];
+  const chunks: string[][] = [];
+  for (let i = 0; i < all.length; i += chunkSize) chunks.push(all.slice(i, i + chunkSize));
+  return { hashes, chunks, versions };
+}
+
+/**
+ * "This query's rows are authoritative and complete": server membership
+ * accepted, every body present at its version, nothing waiting to be
+ * re-rendered, subscribers told at least once.
+ */
+export function settled(s: ClientState, hash: QueryHash): boolean {
+  const e = s.queries.get(hash);
+  if (!e || e.lifecycle.phase !== 'live') return false;
+  return needed(s, hash).length === 0 && !s.dirty.has(hash) && e.lifecycle.notified;
+}
+
+export const settleFailed = (s: ClientState, hash: QueryHash): boolean => {
+  const e = s.queries.get(hash);
+  return !e || e.lifecycle.remote === 'failed';
+};
+
+export const desiredRegistrations = (s: ClientState): QueryHash[] =>
+  [...s.queries].filter(([, e]) => e.lifecycle.remote === 'unregistered').map(([h]) => h);
+
+export const evictable = (s: ClientState, now: number): QueryHash[] =>
+  [...s.queries]
+    .filter(
+      ([, e]) => e.subscribers === 0 && e.lastSubscriberLeftAt !== null && now - e.lastSubscriberLeftAt >= e.def.ttlMs
+    )
+    .map(([h]) => h);
+
+export const shortestTtlMs = (s: ClientState): number | null => {
+  let min: number | null = null;
+  for (const e of s.queries.values()) min = min === null ? e.def.ttlMs : Math.min(min, e.def.ttlMs);
+  return min;
+};
+
+/** Legacy `QueryState` shape for DevTools and the public type surface. */
+export function toQueryState(e: QueryEntry): QueryState {
+  const l = e.lifecycle;
+  return {
+    config: {
+      id: e.def.id,
+      surql: e.def.surql,
+      plan: e.def.plan,
+      params: { ...e.def.params },
+      localArray: [...e.localArray],
+      remoteArray: [...e.remoteArray],
+      subqueryRemoteArray: e.subqueryRemoteArray.length ? [...e.subqueryRemoteArray] : undefined,
+      membershipKnown: isAuthoritative(l),
+      membershipKey: e.def.viewKey,
+      ttl: e.def.ttl,
+      lastActiveAt: new Date(e.lastHeartbeatAt ?? e.def.createdAt),
+      tableName: e.def.tableName,
+    },
+    records: [...e.records] as Record<string, any>[],
+    serverMembership: hasServerMembership(l),
+    viewLost: l.phase === 'view-lost',
+    serverState: e.serverState,
+    syncNotified: l.notified,
+    ttlTimer: null,
+    ttlDurationMs: e.def.ttlMs,
+    lastHeartbeatAt: e.lastHeartbeatAt ?? undefined,
+    updateCount: e.telemetry.updateCount,
+    lastUpdatedAt: e.telemetry.lastUpdatedAt,
+    materializationSamples: [...e.telemetry.materializationSamples],
+    lastIngestLatencyMs: e.telemetry.lastIngestLatencyMs,
+    errorCount: e.telemetry.errorCount,
+    status: deriveStatus(l),
+    phaseSamples: Object.fromEntries(Object.entries(e.telemetry.phaseSamples).map(([k, v]) => [k, [...v]])),
+    phaseLast: { ...e.telemetry.phaseLast },
+    registrationTimings: e.telemetry.registrationTimings,
+  };
+}
+
+function phaseStatOf(samples: ReadonlyArray<number>, lastMs: number | null): PhaseStat {
+  if (samples.length === 0) return { lastMs, p50: null, p90: null, p99: null, count: 0 };
+  const sorted = [...samples].sort((a, b) => a - b);
+  const pick = (q: number) => sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))]!;
+  return { lastMs, p50: pick(0.5), p90: pick(0.9), p99: pick(0.99), count: samples.length };
+}
+
+/** Per-query processing-time breakdown for DevTools and the MCP. */
+export function phaseTimings(e: QueryEntry): QueryTimings {
+  const t = e.telemetry;
+  const stat = (phase: string) => phaseStatOf(t.phaseSamples[phase] ?? [], t.phaseLast[phase] ?? null);
+  return {
+    ssp: phaseStatOf(t.materializationSamples, t.lastIngestLatencyMs),
+    sspStoreApply: stat('sspStoreApply'),
+    sspCircuitStep: stat('sspCircuitStep'),
+    sspTransform: stat('sspTransform'),
+    localFetch: stat('localFetch'),
+    remoteFetch: stat('remoteFetch'),
+    frontend: stat('frontend'),
+    registration: t.registrationTimings,
+    updateCount: t.updateCount,
+    errorCount: t.errorCount,
+  };
+}