@spooky-sync/core 0.0.1-canary.240 → 0.0.1-canary.241

package/src/state/selectors.ts

@@ -0,0 +1,184 @@
+import type { PhaseStat, QueryHash, QueryState, QueryStatus, QueryTimings, RecordVersionArray } from '../types';
+import type { ClientState, QueryEntry } from './client-state';
+import { deriveStatus, hasServerMembership, isAuthoritative } from './lifecycle';
+import { FETCH_CHUNK } from '../kernel/constants';
+
+/** Pure reads over `ClientState`. Sagas use them through `state.read`. */
+
+export const queryByHash = (s: ClientState, hash: QueryHash): QueryEntry | undefined => s.queries.get(hash);
+
+export const activeHashes = (s: ClientState): QueryHash[] => [...s.queries.keys()];
+
+export const hashesForTable = (s: ClientState, table: string): QueryHash[] =>
+  [...s.queries].filter(([, e]) => e.def.tableName === table).map(([h]) => h);
+
+export const queryStatus = (s: ClientState, hash: QueryHash): QueryStatus | undefined => {
+  const e = s.queries.get(hash);
+  return e ? deriveStatus(e.lifecycle) : undefined;
+};
+
+export interface Overlay {
+  /** Ids with an unsynced or just-acked create/update. */
+  readonly writes: ReadonlySet<string>;
+  /** Ids with an unsynced or just-acked delete. */
+  readonly deletes: ReadonlySet<string>;
+}
+
+/** The optimistic overlay, derived from the outbox (pending AND acked items). */
+export function overlay(s: ClientState): Overlay {
+  const writes = new Set<string>();
+  const deletes = new Set<string>();
+  for (const item of s.outbox) {
+    if (item.type === 'delete') deletes.add(item.recordId);
+    else writes.add(item.recordId);
+  }
+  return { writes, deletes };
+}
+
+export const pendingDeleteIds = (s: ClientState): ReadonlySet<string> =>
+  new Set(s.outbox.filter((i) => i.type === 'delete').map((i) => i.recordId));
+
+export const hasAckedWrites = (s: ClientState): boolean => s.outbox.some((i) => i.status === 'acked');
+
+export const pendingMutationCount = (s: ClientState): number =>
+  s.outbox.filter((i) => i.status === 'pending').length;
+
+export const fetchingQueryCount = (s: ClientState): number =>
+  [...s.queries.values()].filter((e) => e.lifecycle.fetchDepth > 0).length;
+
+/** Membership entries whose body is missing or older locally. */
+export function needed(s: ClientState, hash: QueryHash): RecordVersionArray {
+  const e = s.queries.get(hash);
+  if (!e || !hasServerMembership(e.lifecycle)) return [];
+  const deletes = pendingDeleteIds(s);
+  return e.remoteArray.filter(([id, v]) => !deletes.has(id) && (s.versions.get(id) ?? -1) < v);
+}
+
+/** Subquery child bodies missing or stale locally (never part of `settled`). */
+export function neededChildren(s: ClientState, hash: QueryHash): RecordVersionArray {
+  const e = s.queries.get(hash);
+  if (!e || e.subqueryRemoteArray.length === 0) return [];
+  return e.subqueryRemoteArray.filter(([id, v]) => (s.versions.get(id) ?? -1) < v);
+}
+
+export interface FetchPlan {
+  /** Queries whose primary membership needs bodies (they flip to `fetching`). */
+  readonly hashes: QueryHash[];
+  readonly chunks: string[][];
+  /** Highest requested version per id, across every query naming it. */
+  readonly versions: ReadonlyMap<string, number>;
+}
+
+/** Cross-query, deduped, chunked list of ids to pull from the server. */
+export function planFetch(s: ClientState, chunkSize = FETCH_CHUNK): FetchPlan {
+  const versions = new Map<string, number>();
+  const hashes: QueryHash[] = [];
+  const add = (pairs: RecordVersionArray) => {
+    for (const [id, v] of pairs) versions.set(id, Math.max(v, versions.get(id) ?? -1));
+  };
+  for (const hash of s.queries.keys()) {
+    const missing = needed(s, hash);
+    if (missing.length > 0) hashes.push(hash);
+    add(missing);
+    add(neededChildren(s, hash));
+  }
+  const all = [...versions.keys()];
+  const chunks: string[][] = [];
+  for (let i = 0; i < all.length; i += chunkSize) chunks.push(all.slice(i, i + chunkSize));
+  return { hashes, chunks, versions };
+}
+
+/**
+ * "This query's rows are authoritative and complete": server membership
+ * accepted, every body present at its version, nothing waiting to be
+ * re-rendered, subscribers told at least once.
+ */
+export function settled(s: ClientState, hash: QueryHash): boolean {
+  const e = s.queries.get(hash);
+  if (!e || e.lifecycle.phase !== 'live') return false;
+  return needed(s, hash).length === 0 && !s.dirty.has(hash) && e.lifecycle.notified;
+}
+
+export const settleFailed = (s: ClientState, hash: QueryHash): boolean => {
+  const e = s.queries.get(hash);
+  return !e || e.lifecycle.remote === 'failed';
+};
+
+export const desiredRegistrations = (s: ClientState): QueryHash[] =>
+  [...s.queries].filter(([, e]) => e.lifecycle.remote === 'unregistered').map(([h]) => h);
+
+export const evictable = (s: ClientState, now: number): QueryHash[] =>
+  [...s.queries]
+    .filter(
+      ([, e]) => e.subscribers === 0 && e.lastSubscriberLeftAt !== null && now - e.lastSubscriberLeftAt >= e.def.ttlMs
+    )
+    .map(([h]) => h);
+
+export const shortestTtlMs = (s: ClientState): number | null => {
+  let min: number | null = null;
+  for (const e of s.queries.values()) min = min === null ? e.def.ttlMs : Math.min(min, e.def.ttlMs);
+  return min;
+};
+
+/** Legacy `QueryState` shape for DevTools and the public type surface. */
+export function toQueryState(e: QueryEntry): QueryState {
+  const l = e.lifecycle;
+  return {
+    config: {
+      id: e.def.id,
+      surql: e.def.surql,
+      plan: e.def.plan,
+      params: { ...e.def.params },
+      localArray: [...e.localArray],
+      remoteArray: [...e.remoteArray],
+      subqueryRemoteArray: e.subqueryRemoteArray.length ? [...e.subqueryRemoteArray] : undefined,
+      membershipKnown: isAuthoritative(l),
+      membershipKey: e.def.viewKey,
+      ttl: e.def.ttl,
+      lastActiveAt: new Date(e.lastHeartbeatAt ?? e.def.createdAt),
+      tableName: e.def.tableName,
+    },
+    records: [...e.records] as Record<string, any>[],
+    serverMembership: hasServerMembership(l),
+    viewLost: l.phase === 'view-lost',
+    serverState: e.serverState,
+    syncNotified: l.notified,
+    ttlTimer: null,
+    ttlDurationMs: e.def.ttlMs,
+    lastHeartbeatAt: e.lastHeartbeatAt ?? undefined,
+    updateCount: e.telemetry.updateCount,
+    lastUpdatedAt: e.telemetry.lastUpdatedAt,
+    materializationSamples: [...e.telemetry.materializationSamples],
+    lastIngestLatencyMs: e.telemetry.lastIngestLatencyMs,
+    errorCount: e.telemetry.errorCount,
+    status: deriveStatus(l),
+    phaseSamples: Object.fromEntries(Object.entries(e.telemetry.phaseSamples).map(([k, v]) => [k, [...v]])),
+    phaseLast: { ...e.telemetry.phaseLast },
+    registrationTimings: e.telemetry.registrationTimings,
+  };
+}
+
+function phaseStatOf(samples: ReadonlyArray<number>, lastMs: number | null): PhaseStat {
+  if (samples.length === 0) return { lastMs, p50: null, p90: null, p99: null, count: 0 };
+  const sorted = [...samples].sort((a, b) => a - b);
+  const pick = (q: number) => sorted[Math.min(sorted.length - 1, Math.floor(q * sorted.length))]!;
+  return { lastMs, p50: pick(0.5), p90: pick(0.9), p99: pick(0.99), count: samples.length };
+}
+
+/** Per-query processing-time breakdown for DevTools and the MCP. */
+export function phaseTimings(e: QueryEntry): QueryTimings {
+  const t = e.telemetry;
+  const stat = (phase: string) => phaseStatOf(t.phaseSamples[phase] ?? [], t.phaseLast[phase] ?? null);
+  return {
+    ssp: phaseStatOf(t.materializationSamples, t.lastIngestLatencyMs),
+    sspStoreApply: stat('sspStoreApply'),
+    sspCircuitStep: stat('sspCircuitStep'),
+    sspTransform: stat('sspTransform'),
+    localFetch: stat('localFetch'),
+    remoteFetch: stat('remoteFetch'),
+    frontend: stat('frontend'),
+    registration: t.registrationTimings,
+    updateCount: t.updateCount,
+    errorCount: t.errorCount,
+  };
+}

package/src/sync/connection.saga.test.ts

@@ -0,0 +1,86 @@
+import { describe, expect, it } from 'vitest';
+import { runPure } from '../testing/run-pure';
+import { buildEntry, buildOutboxItem, buildState } from '../testing/build';
+import * as R from '../state/reducers';
+import { defaultEnv } from '../query/env';
+import { connectionChanged, selfHealTick, syncOutcome } from './connection.saga';
+
+const env = defaultEnv({ tables: [] } as any);
+
+describe('connectionChanged', () => {
+  it('a drop arms resubscribe and invalidates LIVE; the initial connect does nothing else', async () => {
+    const s = R.patchSync({ liveUuid: 'u', liveTable: 't' })(buildState());
+    const dropped = await runPure(connectionChanged(env, 'reconnecting'), { state: s });
+    expect(dropped.state.sync).toMatchObject({ needsResubscribe: true, liveUuid: null, health: { connection: 'reconnecting' } });
+    const first = await runPure(connectionChanged(env, 'connected'), { state: buildState() });
+    expect(first.dispatched).toEqual([]);
+    const connecting = await runPure(connectionChanged(env, 'connecting'), { state: dropped.state });
+    expect(connecting.dispatched).toEqual([]);
+  });
+  it('reconnect after a drop drops every registration and re-drives, once per cooldown window', async () => {
+    const s = R.patchSync({ needsResubscribe: true })(
+      buildState([
+        buildEntry({ def: { hash: 'a' }, lifecycle: { remote: 'registered' } }),
+        buildEntry({ def: { hash: 'b' }, lifecycle: { remote: 'failed' } }),
+        buildEntry({ def: { hash: 'c' } }),
+      ])
+    );
+    const out = await runPure(connectionChanged(env, 'connected'), { state: s, now: 1000 });
+    expect([...out.state.queries.values()].map((e) => e.lifecycle.remote)).toEqual(['unregistered', 'unregistered', 'unregistered']);
+    expect(out.state.sync).toMatchObject({ needsResubscribe: false, lastReconnectRefetchAt: 1000 });
+    expect(out.dispatched).toEqual([{ type: 'EnsureRegistered', requireAuth: true }, { type: 'LiveStart' }, { type: 'Drain' }]);
+    const burst = await runPure(connectionChanged(env, 'connected'), { state: R.patchSync({ needsResubscribe: true })(out.state), now: 5000 });
+    expect(burst.dispatched).toEqual([]);
+    expect(burst.state.sync.needsResubscribe).toBe(false);
+    const later = await runPure(connectionChanged(env, 'connected'), { state: R.patchSync({ needsResubscribe: true })(out.state), now: 20_000 });
+    expect(later.dispatched).toHaveLength(3);
+  });
+});
+
+describe('syncOutcome', () => {
+  it('degrades after the threshold (arming self-heal), recovers on success (disarming it)', async () => {
+    let state = buildState();
+    for (let i = 0; i < 2; i++) state = (await runPure(syncOutcome(env, false, new Error('socket')), { state })).state;
+    expect(state.sync.health.status).toBe('healthy');
+    const third = await runPure(syncOutcome(env, false, new Error('socket')), { state });
+    expect(third.state.sync.health.status).toBe('degraded');
+    // The saga emits nothing: the runtime notifies `health:changed` off the
+    // health object's identity, which `setHealth` only replaces on a real move.
+    expect(third.emitted).toEqual([]);
+    expect(third.timers.get('heal')).toEqual({ ms: 2000, event: { type: 'SelfHealTick' } });
+    expect(third.state.sync.selfHealAttempts).toBe(0);
+    const back = await runPure(syncOutcome(env, true), { state: third.state });
+    expect(back.state.sync.health.status).toBe('healthy');
+    expect(back.state.sync.health).not.toBe(third.state.sync.health);
+    expect(back.log.some((e) => e.kind === 'timer.clear' && (e as any).key === 'heal')).toBe(true);
+    const quiet = await runPure(syncOutcome(env, true), { state: back.state });
+    expect(quiet.state.sync.health).toBe(back.state.sync.health);
+  });
+});
+
+describe('selfHealTick', () => {
+  const degraded = (s: ReturnType<typeof buildState>) =>
+    R.compose(R.setHealth({ ...s.sync.health, status: 'degraded' }), R.patchSync({ selfHealAttempts: 1 }))(s);
+  it('does nothing when healthy; drains first, then re-registers (failed included), else probes; always re-arms', async () => {
+    expect((await runPure(selfHealTick(env), { state: buildState() })).timers.size).toBe(0);
+    const withOutbox = await runPure(selfHealTick(env), { state: degraded(buildState([], R.outboxReplace([buildOutboxItem()]))) });
+    expect(withOutbox.dispatched).toEqual([{ type: 'Drain' }]);
+    expect(withOutbox.timers.get('heal')).toEqual({ ms: 8000, event: { type: 'SelfHealTick' } });
+    const withFailed = await runPure(selfHealTick(env), { state: degraded(buildState([buildEntry({ def: { hash: 'a' }, lifecycle: { remote: 'failed' } })])) });
+    expect(withFailed.state.queries.get('a')!.lifecycle.remote).toBe('unregistered');
+    expect(withFailed.dispatched).toEqual([{ type: 'EnsureRegistered', requireAuth: true }]);
+    const withDesired = await runPure(selfHealTick(env), { state: degraded(buildState([buildEntry({ def: { hash: 'a' } })])) });
+    expect(withDesired.dispatched).toEqual([{ type: 'EnsureRegistered', requireAuth: true }]);
+    const probe = await runPure(selfHealTick(env), { state: degraded(buildState()), handlers: { 'remote.query': () => [{ status: 'OK', result: true }] } });
+    expect(probe.dispatched).toEqual([{ type: 'SyncOutcome', ok: true }]);
+    const probeFail = await runPure(selfHealTick(env), {
+      state: degraded(buildState()),
+      handlers: {
+        'remote.query': () => {
+          throw new Error('offline');
+        },
+      },
+    });
+    expect(probeFail.dispatched).toEqual([{ type: 'SyncOutcome', ok: false, error: new Error('offline') }]);
+  });
+});

package/src/sync/connection.saga.ts

@@ -0,0 +1,85 @@
+import type { ConnectionState } from '../types';
+import type { Saga } from '../kernel/saga';
+import { fx } from '../kernel/effects';
+import { RECONNECT_REFETCH_COOLDOWN_MS } from '../kernel/constants';
+import type { ClientState } from '../state/client-state';
+import * as R from '../state/reducers';
+import { desiredRegistrations, pendingMutationCount } from '../state/selectors';
+import type { SagaEnv } from '../query/env';
+import { liveInvalidate } from './live.saga';
+import { nextHealth, selfHealDelayMs } from './policy';
+
+/**
+ * Transport state changes. A drop invalidates the live query and arms a
+ * resubscribe; the next `connected` (outside the burst cooldown) drops every
+ * remote registration so `ensureRegistered` rebuilds them behind the
+ * `$auth.id` gate, and restarts LIVE.
+ */
+export function* connectionChanged(env: SagaEnv, connection: ConnectionState): Saga<void> {
+  const state = (yield fx.state.read((s) => s)) as ClientState;
+  yield fx.state.update(R.setConnection(connection));
+  if (connection === 'disconnected' || connection === 'reconnecting') {
+    yield fx.state.update(R.patchSync({ needsResubscribe: true }));
+    yield* liveInvalidate();
+    return;
+  }
+  if (connection !== 'connected' || !state.sync.needsResubscribe) return;
+  const now = (yield fx.now()) as number;
+  const last = state.sync.lastReconnectRefetchAt;
+  if (last !== null && now - last < RECONNECT_REFETCH_COOLDOWN_MS) {
+    yield fx.state.update(R.patchSync({ needsResubscribe: false }));
+    return;
+  }
+  yield fx.state.update(
+    R.compose(
+      R.patchSync({ needsResubscribe: false, lastReconnectRefetchAt: now }),
+      ...[...state.queries.values()]
+        .filter((e) => e.lifecycle.remote !== 'unregistered')
+        .map((e) => R.applyLifecycle(e.def.hash, { type: 'remote-dropped' }))
+    )
+  );
+  yield fx.dispatch({ type: 'EnsureRegistered', requireAuth: true });
+  yield fx.dispatch({ type: 'LiveStart' });
+  yield fx.dispatch({ type: 'Drain' });
+}
+
+/** Fold one sync round's outcome into health; start / stop self-heal. */
+export function* syncOutcome(env: SagaEnv, ok: boolean, error?: unknown): Saga<void> {
+  const sync = (yield fx.state.read((s) => s.sync)) as ClientState['sync'];
+  const next = nextHealth({ health: sync.health, consecutiveFailures: sync.consecutiveFailures, hasSyncedOnce: sync.hasSyncedOnce }, ok, error, env.degradeAfter);
+  yield fx.state.update(
+    R.compose(R.setHealth(next.health), R.patchSync({ consecutiveFailures: next.consecutiveFailures, hasSyncedOnce: next.hasSyncedOnce }))
+  );
+  if (next.degradedNow) {
+    yield fx.state.update(R.patchSync({ selfHealAttempts: 0 }));
+    yield fx.timer.set('heal', selfHealDelayMs(0), { type: 'SelfHealTick' });
+  }
+  if (next.recoveredNow) yield fx.timer.clear('heal');
+}
+
+/**
+ * While degraded: re-drive whatever can prove the server is back, on a
+ * growing backoff. Retry the outbox first, then the registrations (failed
+ * ones included, behind the identity gate), else a bare probe.
+ */
+export function* selfHealTick(env: SagaEnv): Saga<void> {
+  const state = (yield fx.state.read((s) => s)) as ClientState;
+  if (state.sync.health.status !== 'degraded') return;
+  const attempt = state.sync.selfHealAttempts + 1;
+  yield fx.state.update(R.patchSync({ selfHealAttempts: attempt }));
+  const failed = [...state.queries.values()].filter((e) => e.lifecycle.remote === 'failed').map((e) => e.def.hash);
+  if (pendingMutationCount(state) > 0) {
+    yield fx.dispatch({ type: 'Drain' });
+  } else if (desiredRegistrations(state).length > 0 || failed.length > 0) {
+    if (failed.length > 0) yield fx.state.update(R.compose(...failed.map((h) => R.applyLifecycle(h, { type: 'remote-dropped' }))));
+    yield fx.dispatch({ type: 'EnsureRegistered', requireAuth: true });
+  } else {
+    try {
+      yield fx.remote.query('RETURN true', undefined, env.remoteTimeoutMs);
+      yield fx.dispatch({ type: 'SyncOutcome', ok: true });
+    } catch (error) {
+      yield fx.dispatch({ type: 'SyncOutcome', ok: false, error });
+    }
+  }
+  yield fx.timer.set('heal', selfHealDelayMs(attempt), { type: 'SelfHealTick' });
+}

package/src/sync/live.saga.test.ts

@@ -0,0 +1,69 @@
+import { describe, expect, it } from 'vitest';
+import { runPure } from '../testing/run-pure';
+import { buildEntry, buildState } from '../testing/build';
+import * as R from '../state/reducers';
+import { defaultEnv } from '../query/env';
+import { liveChange, liveInvalidate, liveStart } from './live.saga';
+
+const env = defaultEnv({ tables: [] } as any);
+
+describe('liveStart', () => {
+  it('subscribes on the session table and records the uuid; no-op when already on it; followers never subscribe', async () => {
+    const out = await runPure(liveStart(env), { state: { ...buildState(), userId: 'user:abc' }, handlers: { 'remote.live': (e: any) => `uuid-${e.table}` } });
+    expect(out.state.sync).toMatchObject({ liveUuid: 'uuid-_00_list_ref_user_abc', liveTable: '_00_list_ref_user_abc' });
+    const same = await runPure(liveStart(env), { state: out.state });
+    expect(same.log.filter((e) => e.kind === 'remote.live')).toHaveLength(0);
+    const follower = await runPure(liveStart(env), { state: { ...buildState(), tabRole: 'follower' } });
+    expect(follower.log.filter((e) => e.kind === 'remote.live')).toHaveLength(0);
+  });
+  it('kills the previous subscription when connected (ignoring kill errors), tolerates a failing subscribe', async () => {
+    const prev = R.compose(R.patchSync({ liveUuid: 'old', liveTable: '_00_list_ref' }), R.setConnection('connected'))({ ...buildState(), userId: 'user:x' });
+    const killed: string[] = [];
+    const out = await runPure(liveStart(env), {
+      state: prev,
+      handlers: {
+        'remote.kill': (e: any) => {
+          killed.push(e.uuid);
+          throw new Error('gone');
+        },
+        'remote.live': () => 'new',
+      },
+    });
+    expect(killed).toEqual(['old']);
+    expect(out.state.sync.liveUuid).toBe('new');
+    const clean = await runPure(liveStart(env), { state: prev, handlers: { 'remote.kill': () => undefined, 'remote.live': () => 'new2' } });
+    expect(clean.state.sync.liveUuid).toBe('new2');
+    expect(clean.emitted).toEqual([]);
+    const offline = R.patchSync({ liveUuid: 'old', liveTable: '_00_list_ref' })({ ...buildState(), userId: 'user:x' });
+    const noKill = await runPure(liveStart(env), {
+      state: offline,
+      handlers: {
+        'remote.live': () => {
+          throw new Error('not connected');
+        },
+      },
+    });
+    expect(noKill.log.filter((e) => e.kind === 'remote.kill')).toHaveLength(0);
+    expect(noKill.state.sync.liveUuid).toBeNull();
+    expect(noKill.emitted).toEqual([expect.objectContaining({ level: 'warn' })]);
+  });
+  it('liveInvalidate clears the bookkeeping', async () => {
+    const out = await runPure(liveInvalidate(), { state: R.patchSync({ liveUuid: 'u', liveTable: 't' })(buildState()) });
+    expect(out.state.sync).toMatchObject({ liveUuid: null, liveTable: null });
+  });
+});
+
+describe('liveChange', () => {
+  it('marks known hashes dirty, resets the poll streak, relays as leader; ignores unknown hashes', async () => {
+    const s = R.patchSync({ pollIdleStreak: 3 })(buildState([buildEntry({ def: { hash: 'a' } })]));
+    const out = await runPure(liveChange(['a', 'zz']), { state: { ...s, tabRole: 'leader' } });
+    expect([...out.state.membershipDirty]).toEqual(['a']);
+    expect(out.state.sync.pollIdleStreak).toBe(0);
+    expect(out.timers.get('membership')).toEqual({ ms: 50, event: { type: 'ReadDirtyMembership' } });
+    expect(out.emitted).toEqual([{ type: 'tabs:broadcast', message: { type: 'membership-dirty', hashes: ['a'] } }]);
+    const solo = await runPure(liveChange(['a']), { state: s });
+    expect(solo.emitted).toEqual([]);
+    const none = await runPure(liveChange(['zz']), { state: s });
+    expect(none.timers.size).toBe(0);
+  });
+});

package/src/sync/live.saga.ts

@@ -0,0 +1,49 @@
+import type { QueryHash } from '../types';
+import type { Saga } from '../kernel/saga';
+import { fx } from '../kernel/effects';
+import type { ClientState } from '../state/client-state';
+import * as R from '../state/reducers';
+import type { SagaEnv } from '../query/env';
+import { listRefTable } from '../query/env';
+import { markMembershipDirty } from '../query/membership.saga';
+
+/**
+ * LIVE on the session's `_00_list_ref` table. Events never carry data into
+ * state: every change only marks its query's membership dirty, and the
+ * batched re-read decides. Followers get the same dirt relayed by the
+ * leader.
+ */
+export function* liveStart(env: SagaEnv): Saga<void> {
+  const state = (yield fx.state.read((s) => s)) as ClientState;
+  if (state.tabRole === 'follower') return;
+  const table = listRefTable(env, state);
+  if (state.sync.liveUuid && state.sync.liveTable === table) return;
+  if (state.sync.liveUuid && state.sync.health.connection === 'connected') {
+    try {
+      yield fx.remote.kill(state.sync.liveUuid);
+    } catch (error) {
+      yield fx.emit({ type: 'log', level: 'debug', message: 'kill of the previous live query failed', data: { error } });
+    }
+  }
+  yield fx.state.update(R.patchSync({ liveUuid: null, liveTable: null }));
+  try {
+    const uuid = (yield fx.remote.live(table)) as string;
+    yield fx.state.update(R.patchSync({ liveUuid: uuid, liveTable: table }));
+  } catch (error) {
+    yield fx.emit({ type: 'log', level: 'warn', message: 'live subscription failed; the poll covers membership', data: { table, error } });
+  }
+}
+
+/** The socket dropped: the server-side live query is gone with it. */
+export function* liveInvalidate(): Saga<void> {
+  yield fx.state.update(R.patchSync({ liveUuid: null, liveTable: null }));
+}
+
+/** One or more edges of these queries changed on the server. */
+export function* liveChange(hashes: QueryHash[]): Saga<void> {
+  const [known, role] = (yield fx.state.read((s) => [hashes.filter((h) => s.queries.has(h)), s.tabRole])) as [QueryHash[], ClientState['tabRole']];
+  if (known.length === 0) return;
+  yield fx.state.update(R.patchSync({ pollIdleStreak: 0 }));
+  yield* markMembershipDirty(known);
+  if (role === 'leader') yield fx.emit({ type: 'tabs:broadcast', message: { type: 'membership-dirty', hashes: known } });
+}

package/src/sync/policy.health.test.ts

@@ -0,0 +1,56 @@
+import { describe, expect, it } from 'vitest';
+import { nextHealth, selfHealDelayMs } from './policy';
+import { initialHealth } from '../state/client-state';
+import { sameHealth } from '../state/reducers';
+
+const start = { health: initialHealth('connected'), consecutiveFailures: 0, hasSyncedOnce: false };
+
+describe('nextHealth', () => {
+  it('disabled reporting never changes anything', () => {
+    const out = nextHealth(start, false, new Error('x'), 0);
+    expect(out).toMatchObject({ ...start, degradedNow: false, recoveredNow: false });
+  });
+  it('first success latches everConnected and hasSyncedOnce', () => {
+    const out = nextHealth(start, true, undefined, 3);
+    expect(out.health.everConnected).toBe(true);
+    expect(out.hasSyncedOnce).toBe(true);
+    // A second success moves nothing, so the runtime must not re-notify.
+    const again = nextHealth(out, true, undefined, 3);
+    expect(sameHealth(again.health, out.health)).toBe(true);
+  });
+  it('failures accumulate, degrade at the threshold once, recover on success', () => {
+    let s = nextHealth(start, false, new Error('socket closed'), 3);
+    expect(s.health.status).toBe('healthy');
+    expect(s.consecutiveFailures).toBe(1);
+    expect(s.health.kind).toBe('network');
+    s = nextHealth(s, false, 'permission denied', 3);
+    expect(s.health.kind).toBe('application');
+    s = nextHealth(s, false, new Error('timeout'), 3);
+    expect(s.health.status).toBe('degraded');
+    expect(s.degradedNow).toBe(true);
+    expect(s.health.consecutiveFailures).toBe(3);
+    s = nextHealth(s, false, new Error('timeout'), 3);
+    expect(s.degradedNow).toBe(false);
+    expect(s.consecutiveFailures).toBe(4);
+    const r = nextHealth(s, true, undefined, 3);
+    expect(r.health.status).toBe('healthy');
+    expect(r.recoveredNow).toBe(true);
+    expect(r.health.kind).toBeUndefined();
+    expect(r.consecutiveFailures).toBe(0);
+  });
+  it('a success after sub-threshold failures resets without a recovery flag', () => {
+    const s = nextHealth(start, false, new Error('x'), 3);
+    const r = nextHealth(s, true, undefined, 3);
+    expect(r.recoveredNow).toBe(false);
+    expect(r.consecutiveFailures).toBe(0);
+  });
+});
+
+describe('selfHealDelayMs', () => {
+  it('doubles from 2s and caps at 30s', () => {
+    expect(selfHealDelayMs(0)).toBe(2000);
+    expect(selfHealDelayMs(2)).toBe(8000);
+    expect(selfHealDelayMs(10)).toBe(30000);
+    expect(selfHealDelayMs(100)).toBe(30000);
+  });
+});

package/src/{modules/sync/utils.test.ts → sync/policy.test.ts}

@@ -16,9 +16,37 @@ import {
   listRefPollDelayMs,
   LIST_REF_POLL_MAX_INTERVAL_MS,
   recordVersionArraysEqual,
-} from './utils';
-import type { RecordVersionArray, RecordVersionDiff } from '../../types';
-import { encodeRecordId } from '../../utils/index';
+} from './policy';
+import type { RecordVersionArray, RecordVersionDiff } from '../types';
+import { encodeRecordId } from '../utils/index';
+
+describe('ArraySyncer.update', () => {
+  it('planListRefPollChunks keeps duplicate hashes with equal ages stable', () => {
+    const now = 1000;
+    const chunks = planListRefPollChunks(
+      [
+        { hash: 'x', rows: 1, lastPolledAt: 0 },
+        { hash: 'x', rows: 1, lastPolledAt: 0 },
+        { hash: 'a', rows: 1, lastPolledAt: 0 },
+      ],
+      { now }
+    );
+    expect(chunks).toEqual([['a', 'x', 'x']]);
+  });
+  it('nextPollDelayMs treats a future live event as no event', () => {
+    expect(nextPollDelayMs({ now: 100, lastLiveEventAt: 200, baseIntervalMs: 500 })).toBe(500);
+  });
+  it('bumps only the matching record and re-sorts on the next diff', () => {
+    const syncer = new ArraySyncer([['t:b', 1], ['t:a', 1]], [['t:a', 2], ['t:b', 1]]);
+    syncer.update('t:a', 2);
+    expect(syncer.nextSet()).toEqual({ added: [], updated: [], removed: [] });
+    syncer.insert('t:c', 1);
+    syncer.delete('t:b');
+    const diff = syncer.nextSet();
+    expect(diff!.added.map((a) => encodeRecordId(a.id))).toEqual(['t:b']);
+    expect(diff!.removed.map((r) => encodeRecordId(r))).toEqual(['t:c']);
+  });
+});
 
 function rid(table: string, id: string): RecordId<string> {
   return new RecordId(table, id);

package/src/{modules/sync/utils.ts → sync/policy.ts}

@@ -1,6 +1,9 @@
 import type { RecordId } from 'surrealdb';
-import type { RecordVersionArray, RecordVersionDiff } from '../../types';
-import { parseRecordIdString, encodeRecordId } from '../../utils/index';
+import type { RecordVersionArray, RecordVersionDiff } from '../types';
+import { parseRecordIdString, encodeRecordId } from '../utils/index';
+import type { SyncHealth } from '../types';
+import { classifySyncError } from '../utils/error-classification';
+import { SELF_HEAL_BASE_MS, SELF_HEAL_MAX_MS } from '../kernel/constants';
 
 export class ArraySyncer {
   private localArray: RecordVersionArray;
@@ -8,8 +11,8 @@ export class ArraySyncer {
   private needsSort = false;
 
   constructor(localArray: RecordVersionArray, remoteArray: RecordVersionArray) {
-    this.remoteArray = remoteArray.toSorted((a, b) => a[0].localeCompare(b[0]));
-    this.localArray = localArray.toSorted((a, b) => a[0].localeCompare(b[0]));
+    this.remoteArray = remoteArray.toSorted((a: [string, number], b: [string, number]) => a[0].localeCompare(b[0]));
+    this.localArray = localArray.toSorted((a: [string, number], b: [string, number]) => a[0].localeCompare(b[0]));
   }
 
   /**
@@ -128,7 +131,7 @@ export function applyRecordVersionDiff(
     currentMap.set(encodeRecordId(item.id), item.version);
   }
 
-  return Array.from(currentMap).toSorted((a, b) => a[0].localeCompare(b[0]));
+  return Array.from(currentMap).toSorted((a: [string, number], b: [string, number]) => a[0].localeCompare(b[0]));
 }
 
 export function createDiffFromDbOp(
@@ -438,3 +441,62 @@ export function recordVersionArraysEqual(
   }
   return true;
 }
+
+// ---- health ------------------------------------------------------------------
+
+
+export interface HealthInput {
+  health: SyncHealth;
+  consecutiveFailures: number;
+  hasSyncedOnce: boolean;
+}
+
+export interface HealthOutput extends HealthInput {
+  /** Crossed into degraded on this outcome: start self-heal. */
+  degradedNow: boolean;
+  /** Left degraded on this outcome: stop self-heal. */
+  recoveredNow: boolean;
+}
+
+/**
+ * Fold one sync round's outcome into health. A single failure is absorbed;
+ * `degradeAfter` consecutive ones flip to degraded, the next success flips
+ * back. `degradeAfter <= 0` disables reporting.
+ */
+export function nextHealth(input: HealthInput, ok: boolean, error: unknown, degradeAfter: number): HealthOutput {
+  const base = { ...input, degradedNow: false, recoveredNow: false };
+  if (degradeAfter <= 0) return base;
+  if (ok) {
+    const everConnected = true;
+    if (input.consecutiveFailures === 0) {
+      return { ...base, hasSyncedOnce: true, health: { ...input.health, everConnected } };
+    }
+    const recovered = input.health.status === 'degraded';
+    return {
+      ...base,
+      hasSyncedOnce: true,
+      consecutiveFailures: 0,
+      health: { ...input.health, status: 'healthy', kind: undefined, error: undefined, everConnected },
+      recoveredNow: recovered,
+    };
+  }
+  const consecutiveFailures = input.consecutiveFailures + 1;
+  const kind = classifySyncError(error);
+  const message = error instanceof Error ? error.message : String(error);
+  if (input.health.status !== 'degraded' && consecutiveFailures >= degradeAfter) {
+    return {
+      ...base,
+      consecutiveFailures,
+      health: { ...input.health, status: 'degraded', consecutiveFailures, kind, error: message },
+      degradedNow: true,
+    };
+  }
+  return {
+    ...base,
+    consecutiveFailures,
+    health: { ...input.health, consecutiveFailures, kind, error: message },
+  };
+}
+
+export const selfHealDelayMs = (attempt: number): number =>
+  Math.min(SELF_HEAL_MAX_MS, SELF_HEAL_BASE_MS * 2 ** Math.min(attempt, 30));