@north-light/api 0.3.580 → 0.3.588

package/dist/daemon/reconcilers/broker-supervision.js

@@ -0,0 +1,842 @@
+import { existsSync } from 'node:fs';
+import { LAUNCHER_SOCKET } from '../../launcher/spec.js';
+import { launcherRequest } from '../../launcher/client.js';
+import { execFile } from 'node:child_process';
+import { promisify } from 'node:util';
+import { getNode, isWarmSpare, subscribersOf } from '../../core/canvas/index.js';
+import { isSafeNodeId } from '../../core/canvas/paths.js';
+import { fullName } from '../../core/canvas/labels.js';
+import { recordedPidLiveness } from '../../core/canvas/pid.js';
+import { transition } from '../../core/runtime/lifecycle.js';
+import { hasAdmittedTurn, listUnnotifiedFatalFaultNodes, markFaultNotified, readFault, readLiveTurn, recordFault } from '../../core/canvas/recovery.js';
+import { errorClassFromFault } from '../../core/fault-classifier.js';
+import { nextFaultRetry } from '../../core/runtime/fault-recovery.js';
+import { publishNotice } from '../../core/runtime/close.js';
+import { openDb, withCanvasWrite } from '../../core/canvas/db.js';
+import { getById } from '../../core/graph/objects.js';
+import { watcherIdsOf } from '../../core/graph/watches.js';
+import { headlessBrokerHost } from '../../core/runtime/host.js';
+import { endExecution, liveExecutionForNode, requestExecutionStop } from '../../core/canvas/executions.js';
+import { runnerForExecution } from '../../core/runtime/runner/index.js';
+import { emitEvent } from '../../core/events/emit.js';
+import { operationIdContext } from '../../core/events/operation-id.js';
+import { hasUnseenWakeMail, resetStaleExecutionMail } from '../../core/canvas/deliveries.js';
+import { runningJobs } from '../../core/graph/jobs.js';
+import { focusOf } from '../../core/runtime/placement.js';
+import { hasAttachedBrokerViewers } from '../api/viewer-presence.js';
+import { deliverLive } from '../../core/runtime/interactive-deliver.js';
+import { BrokerUnreachableError } from '../../core/runtime/broker-request.js';
+import { formatCard, PARK_SUMMARY_PROMPT } from '../../shared/generated-context.js';
+import { brokerThresholdsForDaemon } from '../../core/runtime/fleet.js';
+import { readConfig } from '../../core/config.js';
+import { beginPendingPark, clearSupervisionMark, consumePendingPark, pendingParkIds, pendingParkStartedAt, readSupervisionMark, writeSupervisionMark } from '../park-pending.js';
+import { shouldParkAfterSummary } from '../park-activity.js';
+import { applyParkEvent, noticeUnfinishedIdle, parkVerdict } from './live-obligation.js';
+import { readGoal } from '../../core/runtime/kickoff.js';
+import { memoryDeniedRecently } from '../../core/runtime/memory-admission.js';
+import { envNoDaemonAutostart, envTestUnattendedParkMs, parkSummaryGraceMs, envUnattendedParkMs, } from '../../shared/env.js';
+const execFileAsync = promisify(execFile);
+/** A node born with a prompt whose first turn has not opened. The prompt rides the launch argv, so nothing in the mail or the turn rows marks the broker busy. */
+async function awaitsBirthTurn(nodeId) {
+    return !(await hasAdmittedTurn(nodeId)) && ((await readGoal(nodeId))?.trim() ?? '') !== '';
+}
+/** The node's live execution when its broker runs on a runner this daemon holds no process for (Blaxel). */
+async function remoteExecutionOf(nodeId) {
+    const execution = await liveExecutionForNode(nodeId);
+    return execution !== null && execution.runner !== 'process' && execution.state === 'running' ? execution : null;
+}
+async function liveBusy(nodeId) {
+    return (await readLiveTurn(nodeId))?.busy === true;
+}
+async function liveTurnProgressMs(nodeId) {
+    const progressAt = (await readLiveTurn(nodeId))?.progressAt;
+    if (progressAt === null || progressAt === undefined)
+        return null;
+    const progressMs = Date.parse(progressAt);
+    return Number.isNaN(progressMs) ? null : progressMs;
+}
+function incarnationOf(pid, identity) {
+    return identity === null ? null : `${pid}:${identity}`;
+}
+async function busySinceForIncarnation(row) {
+    const turn = await readLiveTurn(row.node_id);
+    const since = turn?.busy === true && turn.progressAt !== null ? Date.parse(turn.progressAt) : Number.NaN;
+    const launchedAt = row.launched_at === null || row.launched_at === undefined ? Number.NaN : Date.parse(row.launched_at);
+    if (!Number.isFinite(since) || !Number.isFinite(launchedAt) || since < launchedAt)
+        return null;
+    return since;
+}
+export const YIELD_STALL_GRACE_MS = 3 * 60_000;
+const KILL_ESCALATE_MS = 20_000;
+export const WEDGE_QUIET_MS = 20 * 60_000;
+export const WEDGE_CPU_MAX_PERCENT = 2;
+const OVER_CAP_WARN_THROTTLE_MS = 5 * 60_000;
+/** Whether a fatal provider fault ends the node: a parentless terminal node's run has failed,
+ *  and an app run's root (resident, `run_id` = its own id) settles once a model call is refused
+ *  `cap_exhausted` (D2-VOC-18), so the app sees the cap instead of a run left idle. Any other
+ *  node stays alive and recovers on its next wake. */
+export function settlesOnFatalProviderFault(meta, code) {
+    if (meta === null || meta.parent != null)
+        return false;
+    if (meta.lifecycle === 'terminal')
+        return true;
+    return code === 'cap_exhausted' && meta.run_id === meta.node_id;
+}
+export function unattendedParkMsForDaemon() {
+    // The `CRTR_UNATTENDED_PARK_MS` override wins whenever set, autostart or not.
+    const envOverride = envUnattendedParkMs();
+    if (envOverride !== undefined)
+        return envOverride;
+    // Otherwise the user-scope config value is the base (default 15 min), with the
+    // test-only override winning only when autostart is disabled.
+    const configured = readConfig('user').lifecycle.unattendedParkMs;
+    if (!envNoDaemonAutostart())
+        return configured;
+    return envTestUnattendedParkMs() ?? configured;
+}
+export function parkSummaryGraceMsForDaemon() {
+    return parkSummaryGraceMs();
+}
+export function yieldStallVerdict(piPidAlive, intent, busy, stalledFor) {
+    if (piPidAlive !== true)
+        return 'leave';
+    if (intent !== 'refresh')
+        return 'leave';
+    if (busy)
+        return 'leave';
+    if (stalledFor === null || stalledFor < YIELD_STALL_GRACE_MS)
+        return 'pending';
+    return 'kill';
+}
+export function wedgeVerdict(input) {
+    if (!input.busy)
+        return 'leave';
+    if (input.quietForMs === null || input.quietForMs < WEDGE_QUIET_MS)
+        return 'leave';
+    if (input.cpuPercent === null)
+        return 'pending';
+    if (input.hasDescendants !== false && input.cpuPercent > WEDGE_CPU_MAX_PERCENT)
+        return 'pending';
+    return 'wedged';
+}
+function parsePsTree(psOutput) {
+    const byPid = new Map();
+    const childrenOf = new Map();
+    for (const line of psOutput.split('\n')) {
+        const trimmed = line.trim();
+        if (trimmed === '')
+            continue;
+        const parts = trimmed.split(/\s+/);
+        if (parts.length < 3)
+            continue;
+        const pid = Number(parts[0]);
+        const ppid = Number(parts[1]);
+        const pcpu = Number(parts[2]);
+        if (!Number.isFinite(pid) || !Number.isFinite(ppid) || !Number.isFinite(pcpu))
+            continue;
+        byPid.set(pid, pcpu);
+        const siblings = childrenOf.get(ppid);
+        if (siblings === undefined)
+            childrenOf.set(ppid, [pid]);
+        else
+            siblings.push(pid);
+    }
+    return { byPid, childrenOf };
+}
+export function sumTreeCpu(psOutput, rootPid) {
+    const { byPid, childrenOf } = parsePsTree(psOutput);
+    if (!byPid.has(rootPid))
+        return null;
+    let total = 0;
+    const stack = [rootPid];
+    const seen = new Set();
+    while (stack.length > 0) {
+        const pid = stack.pop();
+        if (seen.has(pid))
+            continue;
+        seen.add(pid);
+        total += byPid.get(pid) ?? 0;
+        const kids = childrenOf.get(pid);
+        if (kids !== undefined)
+            stack.push(...kids);
+    }
+    return total;
+}
+export function treeHasDescendants(psOutput, rootPid) {
+    const { byPid, childrenOf } = parsePsTree(psOutput);
+    if (!byPid.has(rootPid))
+        return null;
+    return (childrenOf.get(rootPid) ?? []).length > 0;
+}
+export class BrokerSupervisionReconciler {
+    /** SIGTERM times of yield-stalled brokers this process signalled; the stall's start is `nodes.stall_noticed_at`. */
+    yieldTermedAt = new Map();
+    sampleTree;
+    signal;
+    lastOverCapWarnAt = Number.NEGATIVE_INFINITY;
+    constructor(options = {}) {
+        this.sampleTree = options.readTreeSample ?? ((pid) => this.readTreeSample(pid));
+        this.signal = options.signal ?? ((pid, signal) => {
+            if (process.platform === 'linux' && existsSync(LAUNCHER_SOCKET)) {
+                if (signal !== 'SIGTERM' && signal !== 'SIGKILL')
+                    throw new Error(`unsupported launcher signal ${signal}`);
+                return launcherRequest({ operation: 'stop', pid, signal }).then(() => { });
+            }
+            process.kill(pid, signal);
+        });
+    }
+    /** The fleet-wide preamble, run BEFORE the tick's row-list guard: the thresholds
+     *  it returns are read once for the whole tick. There is no credential-reload fan:
+     *  the daemon owns credentials and a broker's `reload_auth` handler only acks, so
+     *  fanning it on every provider-store revision was dead work. */
+    async runFleetPreamble(_fleet) {
+        return brokerThresholdsForDaemon();
+    }
+    /** The saturation notice, run LAST in the tick: the frozen split it reports
+     *  is only known once the freeze lane has spent this tick's free slots. */
+    warnIfSaturated(ctx) {
+        if (ctx.fleet.size() < ctx.thresholds.warning || !ctx.lifecycle.acceptsDetachedWork())
+            return;
+        const warning = operationIdContext.fresh(() => this.warnOverCapInTmux(ctx.fleet.size(), ctx.thresholds, ctx.frozen));
+        ctx.lifecycle.registerDetached(warning);
+    }
+    async run(now, ctx) {
+        // Resolve the unattended-park interval once per tick — it reads user
+        // config.json, so a per-broker read would add one file read per eligible
+        // broker and could apply different values within one tick after a config
+        // write (mirrors how `runFleetPreamble` snapshots the broker thresholds).
+        const unattendedParkMs = unattendedParkMsForDaemon();
+        // Read from the fault table, not the fleet: a node whose broker is gone still owes its subscribers the notice.
+        for (const id of await listUnnotifiedFatalFaultNodes()) {
+            await operationIdContext.fresh(async () => {
+                try {
+                    await this.handleFatalFault(id);
+                }
+                catch (err) {
+                    emitEvent({
+                        level: 'error',
+                        event: 'broker.supervision.failed',
+                        ...(isSafeNodeId(id) ? { node_id: id } : { fields: { affected_node_id: id } }),
+                        error: err,
+                    });
+                }
+            });
+        }
+        for (const row of ctx.rows) {
+            const id = row.node_id;
+            const entry = ctx.fleet.get(id);
+            // A broker this daemon does not hold is supervised from its execution row and stopped through its runner.
+            const remote = entry === undefined ? await remoteExecutionOf(id) : null;
+            if ((entry === undefined && remote === null) || (row.status !== 'active' && row.status !== 'idle')) {
+                await clearSupervisionMark(id, 'unattended_since');
+                continue;
+            }
+            await operationIdContext.fresh(async () => {
+                try {
+                    if (entry !== undefined) {
+                        await this.handleYieldStall(row, entry, ctx.fleet, now);
+                        await this.handleWedgeDetection(row, entry, ctx.fleet, now);
+                    }
+                    else {
+                        await this.handleRemoteYieldStall(row, remote, now);
+                        await this.handleRemoteWedge(row, remote, now);
+                    }
+                    await this.handleUnattendedParking(id, await getNode(id), entry?.pid ?? remote.pid, remote, now, ctx, unattendedParkMs);
+                }
+                catch (err) {
+                    await clearSupervisionMark(id, 'unattended_since');
+                    emitEvent({
+                        level: 'error',
+                        event: 'broker.supervision.failed',
+                        ...(isSafeNodeId(id) ? { node_id: id } : { fields: { affected_node_id: id } }),
+                        error: err,
+                    });
+                }
+            });
+        }
+        // A broker that died mid-summary leaves no pending park behind: its parking
+        // turn can never settle, and the node's next clock starts from scratch.
+        for (const id of await pendingParkIds()) {
+            if (!await ctx.fleet.has(id))
+                await consumePendingPark(id);
+        }
+        await resetStaleExecutionMail(now);
+    }
+    /** Every condition that resets the unattended clock EXCEPT `isBusy` — the
+     *  node is unavailable to park, attended, or has work waiting. Short-circuits
+     *  in cost order so the common tick does no file reads it can avoid. */
+    async parkResetCondition(id, row, livePid, now) {
+        if (row.pi_session_id == null
+            || row.pi_pid !== livePid
+            || row.status === 'done'
+            || row.status === 'canceled'
+            || row.status === 'dead'
+            || row.intent !== null
+            || (await runningJobs(await openDb(), id)).length > 0
+            || await focusOf(id) !== null
+            || await hasAttachedBrokerViewers(id))
+            return true;
+        const fault = await readFault(id);
+        const activeAutoProviderRetry = fault !== null
+            && fault.retry.disposition === 'auto'
+            && fault.retry.by === 'daemon'
+            && (fault.kind === 'connection' || fault.kind === 'rate-limit' || fault.kind === 'overloaded')
+            && (fault.retry.nextAt !== undefined || nextFaultRetry(fault, now) !== null);
+        if (activeAutoProviderRetry)
+            return true;
+        return await hasUnseenWakeMail(id);
+    }
+    async handleUnattendedParking(id, row, livePid, remote, now, ctx, unattendedParkMs) {
+        // An unclaimed warm-pool spare is not a conversation: the booted broker it
+        // holds IS its whole value, and nobody is meant to be attending it. Its
+        // lifetime belongs to `reapStaleSpares` (warm-pool.ts) and its own TTL, so
+        // parking one here would empty the pool every 15 minutes.
+        if (await isWarmSpare(id)) {
+            await clearSupervisionMark(id, 'unattended_since');
+            await consumePendingPark(id);
+            return;
+        }
+        if (row === null) {
+            await clearSupervisionMark(id, 'unattended_since');
+            await consumePendingPark(id);
+            return;
+        }
+        const reset = await this.parkResetCondition(id, row, livePid, now);
+        // A shadow parking turn owns this marker until it completes or runs out of
+        // grace. Both paths take the same main-engine activity verdict before
+        // terminalizing, so input during the summary only keeps the node live.
+        const parkingSince = await pendingParkStartedAt(id);
+        if (parkingSince !== undefined) {
+            if (now - parkingSince < parkSummaryGraceMsForDaemon())
+                return;
+            const pending = await consumePendingPark(id);
+            if (pending === undefined)
+                return;
+            await clearSupervisionMark(id, 'unattended_since');
+            if (!await shouldParkAfterSummary(id, row, pending))
+                return;
+            // The degraded path has no report. Status flips BEFORE the host dies (the
+            // crash-safety invariant).
+            emitEvent({
+                level: 'warn',
+                event: 'broker.park.summary_expired',
+                ...(isSafeNodeId(id) ? { node_id: id } : {}),
+                fields: {
+                    ...(isSafeNodeId(id) ? {} : { affected_node_id: id }),
+                    grace_ms: parkSummaryGraceMsForDaemon(),
+                },
+            });
+            await applyParkEvent(id, 'park');
+            void this.teardownBroker(id, remote);
+            return;
+        }
+        // A broker whose birth prompt has not reached a first turn is not idle: the prompt rides the launch argv, so no mail
+        // or turn row marks it busy, and releasing or parking it would strand the node with nothing to wake it.
+        if (reset || await liveBusy(id) || await awaitsBirthTurn(id)) {
+            await clearSupervisionMark(id, 'unattended_since');
+            return;
+        }
+        const since = await readSupervisionMark(id, 'unattended_since');
+        if (since === undefined) {
+            await writeSupervisionMark(id, 'unattended_since', now);
+            return;
+        }
+        const { event, heldOnlyBySubscriber } = await parkVerdict(id, row.lifecycle);
+        // An app short of memory cannot launch its frozen nodes until idle brokers let go: release or park now.
+        // `cancel` ends a node for good, so it keeps the full grace.
+        const grace = event !== 'cancel' && memoryDeniedRecently(row.grantee ?? 'app:terminal', now) ? 0 : unattendedParkMs;
+        if (now - since < grace)
+            return;
+        await clearSupervisionMark(id, 'unattended_since');
+        if (event === 'release') {
+            // A pause, not a terminal write: something can still wake this node, so
+            // its row stays wakeable while the idle broker is released. Skipping the
+            // teardown here would hold the broker process for as long as the
+            // obligation lasts — the exact leak this clock exists to close.
+            await applyParkEvent(id, event);
+            if (heldOnlyBySubscriber)
+                await noticeUnfinishedIdle(id);
+            void this.teardownBroker(id, remote);
+            return;
+        }
+        if (event === 'cancel') {
+            await applyParkEvent(id, event);
+            void this.teardownBroker(id, remote);
+            return;
+        }
+        // App runs park without spending a model turn to write an owner-style summary.
+        if (row.grantee !== undefined && row.grantee !== 'app:terminal') {
+            await applyParkEvent(id, 'park');
+            void this.teardownBroker(id, remote);
+            return;
+        }
+        // A resident gets one isolated turn to leave a report behind. Its completion
+        // re-evaluates this verdict before terminalizing the node.
+        if (!ctx.lifecycle.acceptsDetachedWork())
+            return;
+        await beginPendingPark(id, now);
+        ctx.lifecycle.registerDetached(this.requestParkSummary(id, remote));
+    }
+    /** Deliver the parking prompt over the live broker's serialized frame loop.
+     * An unreachable broker falls through to degraded parking; either prompt route
+     * starts the same isolated turn, so a concurrent main turn stays passive. */
+    async requestParkSummary(id, remote) {
+        try {
+            const route = await deliverLive(id, formatCard('park', {}, PARK_SUMMARY_PROMPT), {
+                origin: { type: 'runtime', kind: 'park' },
+            });
+            emitEvent({
+                level: 'info',
+                event: 'broker.park.summary_requested',
+                ...(isSafeNodeId(id) ? { node_id: id } : {}),
+                fields: { ...(isSafeNodeId(id) ? {} : { affected_node_id: id }), route },
+            });
+        }
+        catch (err) {
+            if (await consumePendingPark(id) === undefined)
+                return;
+            if (err instanceof BrokerUnreachableError) {
+                await applyParkEvent(id, 'park');
+                void this.teardownBroker(id, remote);
+                return;
+            }
+            emitEvent({
+                level: 'error',
+                event: 'broker.park.summary_failed',
+                ...(isSafeNodeId(id) ? { node_id: id } : { fields: { affected_node_id: id } }),
+                error: err,
+            });
+        }
+    }
+    async warnOverCapInTmux(count, thresholds, frozen) {
+        const now = Date.now();
+        if (now - this.lastOverCapWarnAt < OVER_CAP_WARN_THROTTLE_MS)
+            return;
+        this.lastOverCapWarnAt = now;
+        const { warning, automaticReviveCap: max } = thresholds;
+        const waiting = frozen.stalled + frozen.waitingOnMail;
+        emitEvent({
+            level: 'warn',
+            event: 'broker.capacity.warning',
+            fields: {
+                live_count: count,
+                warning_threshold: warning,
+                hard_cap: max,
+                frozen_stalled: frozen.stalled,
+                frozen_waiting_on_mail: frozen.waitingOnMail,
+            },
+        });
+        const frozenNote = waiting === 0
+            ? ''
+            : ` — ${waiting} frozen (${frozen.stalled} stalled, ${frozen.waitingOnMail} waiting on mail)`;
+        const msg = `⚠ crtrd: ${count} live brokers (warning ${warning}; hard cap ${max})${frozenNote} — possible broker leak. ps -ax | grep broker-cli.js`;
+        try {
+            const clients = await execFileAsync('tmux', ['list-clients', '-F', '#{client_name}'], {
+                encoding: 'utf8',
+                timeout: 2000,
+            });
+            if (typeof clients.stdout !== 'string')
+                return;
+            await Promise.allSettled(clients.stdout.split('\n').map(async (client) => {
+                if (client === '')
+                    return;
+                try {
+                    // `-C` keeps the client's panes painting while the warning sits in
+                    // its status line; without it every attached viewer freezes for the
+                    // full 10s. The plain retry below covers a tmux without the flag.
+                    await execFileAsync('tmux', ['display-message', '-C', '-d', '10000', '-c', client, msg], {
+                        encoding: 'utf8',
+                        timeout: 2000,
+                    });
+                }
+                catch {
+                    try {
+                        await execFileAsync('tmux', ['display-message', '-c', client, msg], {
+                            encoding: 'utf8',
+                            timeout: 2000,
+                        });
+                    }
+                    catch (err) {
+                        emitEvent({
+                            level: 'error',
+                            event: 'broker.capacity.tmux_display_failed',
+                            error: err,
+                            fields: { client },
+                        });
+                    }
+                }
+            }));
+        }
+        catch (err) {
+            emitEvent({ level: 'error', event: 'broker.capacity.tmux_list_clients_failed', error: err });
+        }
+    }
+    /** Stop a node's broker. A remote broker's stop is recorded on its row, enacted through its runner, and ends exactly that
+     *  execution once confirmed; any other broker is torn down by the local host. */
+    async teardownBroker(id, remote) {
+        if (remote === null)
+            return await headlessBrokerHost.teardown(id);
+        return await this.stopRemote(remote, 'teardown');
+    }
+    async stopRemote(execution, reason) {
+        await requestExecutionStop(execution.nodeId, { afterTurn: false, reason });
+        const result = await runnerForExecution(execution.runner).stop(execution, { reason });
+        if (result.confirmed)
+            await endExecution(execution.executionId, { reason });
+        else
+            emitEvent({ level: 'error', event: 'broker.teardown.unconfirmed', node_id: execution.nodeId, error: result.error });
+        return result;
+    }
+    /** A refresh that never reached its replacement: the same stall clock as a local broker, enacted through the runner. */
+    async handleRemoteYieldStall(row, execution, now) {
+        const id = row.node_id;
+        if (row.intent !== 'refresh' || await busySinceForIncarnation(row) !== null) {
+            await clearSupervisionMark(id, 'stall_noticed_at');
+            return;
+        }
+        const stalledSince = await readSupervisionMark(id, 'stall_noticed_at');
+        if (stalledSince === undefined) {
+            await writeSupervisionMark(id, 'stall_noticed_at', now);
+            return;
+        }
+        if (yieldStallVerdict(true, row.intent, false, now - stalledSince) !== 'kill')
+            return;
+        if ((await liveExecutionForNode(id))?.executionId !== execution.executionId)
+            return;
+        emitEvent({
+            level: 'warn',
+            event: 'broker.yield_stall.remote_stop',
+            ...(isSafeNodeId(id) ? { node_id: id } : {}),
+            fields: { ...(isSafeNodeId(id) ? {} : { affected_node_id: id }), execution_id: execution.executionId, elapsed_ms: now - stalledSince, grace_ms: YIELD_STALL_GRACE_MS },
+        });
+        await this.stopRemote(execution, 'yield_stall');
+    }
+    /** A remote broker has no process this daemon can sample, so its only evidence is the turn's progress clock. A quiet turn is
+     *  reported once and the broker is left running: with no view of its subprocesses the daemon cannot tell a runaway from a long command. */
+    async handleRemoteWedge(row, execution, now) {
+        const id = row.node_id;
+        const since = await busySinceForIncarnation(row);
+        if (since === null) {
+            await clearSupervisionMark(id, 'wedge_noticed_at');
+            return;
+        }
+        const quietForMs = now - since;
+        if (quietForMs < WEDGE_QUIET_MS)
+            return;
+        const current = await getNode(id);
+        if (current === null || (await liveExecutionForNode(id))?.executionId !== execution.executionId || await busySinceForIncarnation(current) !== since)
+            return;
+        if (await readSupervisionMark(id, 'wedge_noticed_at') === since)
+            return;
+        await writeSupervisionMark(id, 'wedge_noticed_at', since);
+        const minutes = Math.round(quietForMs / 60_000);
+        await this.reportWedge(id, {
+            quietForMs, cpuPercent: null, hasDescendants: null, kickAttempted: false, kicked: false, kickSuppressionReason: 'remote_no_process_evidence', pid: execution.pid,
+            label: `Child wedged — ${fullName(current)} (${id}) runs in a sandbox and is mid-turn but has shown NO engine progress for ${minutes}+ minutes. ` +
+                `The daemon cannot see its processes there, so it did not stop the broker; if it is not running a long command, interrupt or close it.`,
+        });
+    }
+    async handleYieldStall(row, entry, fleet, now) {
+        const id = row.node_id;
+        const incarnation = incarnationOf(entry.pid, row.pi_pid_identity);
+        if (row.intent !== 'refresh' || await busySinceForIncarnation(row) !== null || incarnation === null) {
+            await clearSupervisionMark(id, 'stall_noticed_at');
+            if (incarnation !== null)
+                this.yieldTermedAt.delete(incarnation);
+            return;
+        }
+        const stalledSince = await readSupervisionMark(id, 'stall_noticed_at');
+        if (stalledSince === undefined) {
+            await writeSupervisionMark(id, 'stall_noticed_at', now);
+            return;
+        }
+        const state = { since: stalledSince, termedAt: this.yieldTermedAt.get(incarnation) };
+        if (yieldStallVerdict(true, row.intent, false, now - state.since) !== 'kill')
+            return;
+        const current = await getNode(id);
+        if (fleet.get(id) !== entry || current?.pi_pid !== entry.pid || current.pi_pid_identity !== row.pi_pid_identity || recordedPidLiveness(entry.pid, row.pi_pid_identity) !== 'alive')
+            return;
+        if (state.termedAt === undefined) {
+            if (!await this.signalYieldStall(id, entry.pid, 'SIGTERM'))
+                return;
+            this.yieldTermedAt.set(incarnation, now);
+            state.termedAt = now;
+            emitEvent({
+                level: 'warn',
+                event: 'broker.yield_stall.sigterm',
+                ...(isSafeNodeId(id) ? { node_id: id } : {}),
+                fields: {
+                    ...(isSafeNodeId(id) ? {} : { affected_node_id: id }),
+                    pid: entry.pid,
+                    elapsed_ms: now - state.since,
+                    grace_ms: YIELD_STALL_GRACE_MS,
+                },
+            });
+        }
+        else if (now - state.termedAt >= KILL_ESCALATE_MS && await this.signalYieldStall(id, entry.pid, 'SIGKILL')) {
+            emitEvent({
+                level: 'warn',
+                event: 'broker.yield_stall.sigkill',
+                ...(isSafeNodeId(id) ? { node_id: id } : {}),
+                fields: {
+                    ...(isSafeNodeId(id) ? {} : { affected_node_id: id }),
+                    pid: entry.pid,
+                    elapsed_ms: now - state.termedAt,
+                    grace_ms: KILL_ESCALATE_MS,
+                    stall_elapsed_ms: now - state.since,
+                },
+            });
+        }
+    }
+    async signalYieldStall(id, pid, signal) {
+        try {
+            await this.signal(pid, signal);
+            return true;
+        }
+        catch (err) {
+            emitEvent({
+                level: 'error',
+                event: 'broker.yield_stall.signal_failed',
+                ...(isSafeNodeId(id) ? { node_id: id } : {}),
+                error: err,
+                fields: {
+                    ...(isSafeNodeId(id) ? {} : { affected_node_id: id }),
+                    pid,
+                    signal,
+                },
+            });
+            return false;
+        }
+    }
+    async readTreeSample(pid) {
+        try {
+            const { stdout } = await execFileAsync('ps', ['-ax', '-o', 'pid=,ppid=,pcpu='], {
+                encoding: 'utf8',
+                timeout: 2000,
+            });
+            if (typeof stdout !== 'string')
+                return null;
+            const cpuPercent = sumTreeCpu(stdout, pid);
+            const hasDescendants = treeHasDescendants(stdout, pid);
+            if (cpuPercent === null || hasDescendants === null)
+                return null;
+            return { cpuPercent, hasDescendants };
+        }
+        catch {
+            return null;
+        }
+    }
+    async handleWedgeDetection(row, entry, fleet, now) {
+        const id = row.node_id;
+        const incarnation = incarnationOf(entry.pid, row.pi_pid_identity);
+        const since = await busySinceForIncarnation(row);
+        if (since === null || incarnation === null) {
+            await clearSupervisionMark(id, 'wedge_noticed_at');
+            return;
+        }
+        const quietForMs = now - since;
+        const sample = quietForMs >= WEDGE_QUIET_MS ? await this.sampleTree(entry.pid) : null;
+        const current = await getNode(id);
+        const currentSince = current === null ? null : await busySinceForIncarnation(current);
+        if (fleet.get(id) !== entry || current === null || current.pi_pid !== entry.pid || current.pi_pid_identity !== row.pi_pid_identity || currentSince !== since)
+            return;
+        const cpuPercent = sample?.cpuPercent ?? null;
+        const sampleHasDescendants = sample?.hasDescendants ?? null;
+        if (wedgeVerdict({ busy: true, quietForMs, cpuPercent, hasDescendants: sampleHasDescendants }) !== 'wedged')
+            return;
+        if (await readSupervisionMark(id, 'wedge_noticed_at') === since)
+            return;
+        await writeSupervisionMark(id, 'wedge_noticed_at', since);
+        const name = fullName(current);
+        const minutes = Math.round((quietForMs ?? WEDGE_QUIET_MS) / 60_000);
+        const hasDescendants = sampleHasDescendants ?? true;
+        const kickSuppressionReason = !hasDescendants && recordedPidLiveness(entry.pid, row.pi_pid_identity) !== 'alive'
+            ? 'pid_identity_unproven'
+            : null;
+        let kickAttempted = false;
+        let kicked = false;
+        let label;
+        if (hasDescendants) {
+            label =
+                `Child wedged — ${name} (${id}) is alive and mid-turn but has shown NO engine progress for ` +
+                    `${minutes}+ minutes and its process is near-idle. It will NOT recover on its own — read ` +
+                    `\`crtr canvas read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this ` +
+                    `node) so its turn can resume with no lost context.`;
+        }
+        else if (kickSuppressionReason !== null) {
+            label =
+                `Child wedged — ${name} (${id}) is alive and mid-turn but has shown NO engine progress for ` +
+                    `${minutes}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did ` +
+                    `not SIGTERM the broker because its PID identity cannot be ` +
+                    `confirmed alive.`;
+        }
+        else {
+            kickAttempted = true;
+            try {
+                await this.signal(entry.pid, 'SIGTERM');
+                kicked = true;
+                label =
+                    `Child wedged — ${name} (${id}) is alive and mid-turn but has shown NO engine progress for ` +
+                        `${minutes}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon sent ` +
+                        `SIGTERM to the broker and will resume a cleanly aborted session with a continuation, or start a ` +
+                        `recovery cycle if the abort cannot settle.`;
+            }
+            catch (err) {
+                emitEvent({
+                    level: 'error',
+                    event: 'broker.wedge.signal_failed',
+                    ...(isSafeNodeId(id) ? { node_id: id } : {}),
+                    error: err,
+                    fields: {
+                        ...(isSafeNodeId(id) ? {} : { affected_node_id: id }),
+                        pid: entry.pid,
+                        signal: 'SIGTERM',
+                    },
+                });
+                label =
+                    `Child wedged — ${name} (${id}) is alive and mid-turn but has shown NO engine progress for ` +
+                        `${minutes}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon ` +
+                        `attempted SIGTERM to the broker, but the signal call failed.`;
+            }
+        }
+        await this.reportWedge(id, {
+            quietForMs, cpuPercent, hasDescendants, kickAttempted, kicked, kickSuppressionReason, pid: entry.pid, label,
+        });
+    }
+    /** Record and announce a wedged broker: the event, the fault, and the notice to its managers. */
+    async reportWedge(id, r) {
+        emitEvent({
+            level: 'error',
+            event: 'broker.wedge.detected',
+            ...(isSafeNodeId(id) ? { node_id: id } : {}),
+            fields: {
+                ...(isSafeNodeId(id) ? {} : { affected_node_id: id }),
+                pid: r.pid,
+                quiet_ms: r.quietForMs,
+                cpu_percent: r.cpuPercent,
+                has_descendants: r.hasDescendants,
+                kick_attempted: r.kickAttempted,
+                kicked: r.kicked,
+                ...(r.kickSuppressionReason === null ? {} : { kick_suppression_reason: r.kickSuppressionReason }),
+            },
+        });
+        try {
+            await recordFault(id, {
+                link: 'daemon→node',
+                op: 'engine wedge',
+                kind: 'wedged',
+                retry: { disposition: 'manual' },
+                message: r.label,
+            });
+        }
+        catch (err) {
+            emitEvent({
+                level: 'error',
+                event: 'broker.wedge.fault_record_failed',
+                ...(isSafeNodeId(id) ? { node_id: id } : { fields: { affected_node_id: id } }),
+                error: err,
+            });
+        }
+        await publishNotice(id, 'notice.crash', r.label, {
+            reason: 'child-wedged',
+            child: id,
+            quiet_ms: r.quietForMs,
+            cpu_percent: r.cpuPercent,
+            kick_attempted: r.kickAttempted,
+            kicked: r.kicked,
+            ...(r.kickSuppressionReason === null ? {} : { kick_suppression_reason: r.kickSuppressionReason }),
+        });
+    }
+    /** Publish a fatal provider fault's notice once per episode and, for a node that settles on it, end the node.
+     *  The notices, the `notified_since` mark and the terminal transition commit in one write, so a daemon exit
+     *  leaves the fault either fully announced or fully unannounced. The daemon registers this as the announcer
+     *  every clear of an unannounced fatal fault runs first with `settle` false: a clear ends the fault on purpose
+     *  (a revive, a completed turn), so it announces but does not end the node. */
+    async handleFatalFault(id, settle = true) {
+        const fault = await readFault(id);
+        if (fault === null || fault.retry.disposition !== 'fatal' || fault.link !== 'pi→provider')
+            return;
+        if (fault.notifiedSince === fault.since)
+            return;
+        const meta = await getNode(id);
+        const remote = await remoteExecutionOf(id);
+        const name = meta !== null ? fullName(meta) : id;
+        const isAuth = fault.kind === 'auth';
+        const firstLine = (fault.message.split('\n')[0] ?? '').slice(0, 200);
+        const providerError = fault.providerError;
+        const providerDetail = providerError === undefined
+            ? ''
+            : ` Provider: ${providerError.provider}/${providerError.model}.` +
+                (providerError.status === undefined ? '' : ` HTTP status: ${providerError.status}.`) +
+                (providerError.errno === undefined ? '' : ` Errno: ${providerError.errno}.`) +
+                (providerError.code === undefined ? '' : ` Code: ${providerError.code}.`) +
+                (providerError.body === undefined ? '' : ` Body: ${providerError.body.slice(0, 200)}.`) +
+                (providerError.ratelimit === undefined ? '' : ` Rate limit: ${providerError.ratelimit.scope ?? 'account'}${providerError.ratelimit.claim === undefined ? '' : ` (${providerError.ratelimit.claim})`}${providerError.ratelimit.model === undefined ? '' : ` on ${providerError.ratelimit.model}`}.`);
+        const terminalOverflow = fault.kind === 'context-overflow';
+        const terminalProviderFault = settlesOnFatalProviderFault(meta, typeof providerError?.code === 'string' ? providerError.code : undefined);
+        const terminalFault = terminalOverflow || terminalProviderFault;
+        const what = terminalOverflow
+            ? `exceeded its model context window and is now dead. ` +
+                `Shrink the task or loaded context, or choose a larger-context model before spawning it again. Error: ${firstLine}`
+            : `got a non-retryable ${fault.kind} error and will NOT recover on its own; ` +
+                (terminalProviderFault
+                    ? `its terminal run has failed. `
+                    : `it stays alive, and waking it clears the fault as soon as one turn completes — a wake that hits the same cause fails again. ` +
+                        (isAuth ? `Its provider credential is dead — re-authenticate (\`/login\`, or re-auth crtr); it recovers on its next wake once the auth reloads. ` : ``)) +
+                `Error: ${firstLine}.${providerDetail}`;
+        const headline = terminalOverflow ? 'failed' : 'hit a fatal provider fault';
+        // The faulted node's managers see a child; every other watcher (a child waiting on its parent, a reader) sees a node it watches.
+        const managers = new Set((await subscribersOf(id)).map((m) => m.node_id));
+        const watchers = await watcherIdsOf(await openDb(), (await getById(await openDb(), id)));
+        const notices = [
+            { relation: 'Child', recipients: watchers.filter((w) => managers.has(w)) },
+            { relation: 'Watched node', recipients: watchers.filter((w) => !managers.has(w)) },
+        ];
+        // Publish before the terminal transition: it ends message subscriptions, and a watcher that only holds one must still be told.
+        emitEvent({
+            level: 'error',
+            event: 'fault.fatal.fanout',
+            ...(isSafeNodeId(id) ? { node_id: id } : {}),
+            fields: {
+                ...(isSafeNodeId(id) ? {} : { affected_node_id: id }),
+                kind: fault.kind,
+                link: fault.link,
+                terminal_overflow: terminalOverflow,
+                terminal_provider_fault: terminalProviderFault,
+            },
+        });
+        await withCanvasWrite(async () => {
+            for (const { relation, recipients } of notices) {
+                if (recipients.length === 0)
+                    continue;
+                await publishNotice(id, 'notice.crash', `${relation} ${headline} — ${name} (${id}) ${what}`, {
+                    reason: terminalOverflow ? 'child-context-overflow' : 'child-fault-fatal',
+                    child: id,
+                    kind: fault.kind,
+                    message: firstLine,
+                    ...(providerError === undefined ? {} : { provider_error: providerError }),
+                }, 'normal', null, recipients);
+            }
+            await markFaultNotified(fault.faultId);
+            if (settle && terminalFault) {
+                await transition(id, 'crash', {
+                    reason: terminalOverflow ? 'context_overflow' : providerError?.code === 'cap_exhausted' ? 'cap_exhausted' : 'provider_fatal',
+                    outcome: {
+                        detail: {
+                            schema: 'crtr.node-outcome-detail/v1',
+                            message: fault.message,
+                            fault_kind: fault.kind,
+                            error_class: errorClassFromFault(fault).class,
+                        },
+                    },
+                });
+            }
+        });
+        if (settle && terminalFault) {
+            void this.teardownBroker(id, remote).then((result) => {
+                if (!result.confirmed)
+                    emitEvent({ level: 'error', event: 'fault.fatal.teardown_failed', node_id: id, error: result.error ?? new Error('Broker teardown not confirmed') });
+            }, (error) => emitEvent({ level: 'error', event: 'fault.fatal.teardown_failed', node_id: id, error }));
+        }
+    }
+}