@north-light/crouter 0.3.318 → 0.3.320

package/dist/daemon/fleet.js

@@ -1,475 +1,17 @@
-// src/daemon/fleet.ts — the daemon's broker fleet: the ChildProcess-handle
-// registry (the ONE normal liveness authority), the exit-policy classifier
-// (invariant-5 policy table), and durable respawn recovery.
-//
-// Constructed by runDaemon with the serial work lane's `enqueue` and this
-// daemon's epoch id, then injected into core via `bindFleet` BEFORE the API
-// server or any tick starts. Exit events enqueue observation only; the durable
-// row tick owns recovery policy.
-//
-// Row state (status/intent/busy marker/session identity), never exit codes,
-// is the policy input (design D-4): the stophook writes the exit's explanation
-// before every deliberate exit, and that durable state is robust to every kill
-// path. Codes/signals are logged for diagnostics only.
-import { getNode, getRow, subscribersOf } from '../core/canvas/index.js';
-import { isProfilePaused } from '../core/profiles/manifest.js';
-import { fullName } from '../core/canvas/labels.js';
-import { isSafeNodeId } from '../core/canvas/paths.js';
-import { transition } from '../core/runtime/lifecycle.js';
-import { fanDoctrineWake } from '../core/runtime/close.js';
-import { readInterruptedTurn, readTransportPoisonVerdict } from '../core/canvas/recovery.js';
-import { hasPendingWake } from './reconcilers/live-obligation.js';
-import { clearRespawnPolicy, deferRespawnWhileOffline, MAX_RESPAWN_ATTEMPTS, OFFLINE_RESPAWN_RETRY_MS, recordRespawnPolicy, respawnBackoffMs, } from './reconcilers/node-lifecycle/respawn-policy.js';
-import { probeOnline as probeOnlineReal } from '../core/runtime/connectivity.js';
-import { reviveNode } from '../core/runtime/revive.js';
-import { pushUrgent } from '../core/feed/feed.js';
-import { emitEvent } from '../core/events/emit.js';
-import { brokerThresholdsForDaemon } from '../core/runtime/fleet.js';
-// Shared crash helpers (moved from crtrd.ts — the fleet's policy paths are now
-// their primary consumers; crtrd imports them back for its surviving passes).
-/** Surface a vehicle that never booted.
- *
- *  A pi that dies before its first session_start (so `pi_session_id` was never
- *  recorded) is invisible to its parent — the spawn returned optimistically.
- *  When the daemon observes that death it errors LOUDLY up the spine: an
- *  urgent push so the parent learns the child failed to launch instead of just
- *  seeing a silent `dead`. */
-function bootFailureBody(meta, exitStatus) {
-    const status = exitStatus === undefined
-        ? 'The daemon did not observe an exit status.'
-        : `The observed exit status was code ${exitStatus.code ?? 'null'}, signal ${exitStatus.signal ?? 'null'}.`;
-    return (`⚠ Spawn failed — \`${meta.name}\` (${meta.kind}) never started.\n\n` +
-        `Its pi vehicle exited before the session came up (no pi_session_id was ever ` +
-        `recorded), so the node produced no output. ${status}\n\n` +
-        `Inspect canonical event diagnostics with \`crtr sys logs --node ${meta.node_id}\`.\n\n` +
-        `If the work still needs doing, re-spawn it.`);
-}
-export async function surfaceBootFailure(meta, exitStatus) {
-    await pushUrgent(meta.node_id, bootFailureBody(meta, exitStatus), { from: meta.node_id });
-}
-async function surfaceCrashLoop(meta) {
-    const transportPoison = readTransportPoisonVerdict(meta.node_id);
-    const observedCause = transportPoison === null
-        ? ''
-        : `\n\nProvider rotation observed ${transportPoison.consecutivePasses} consecutive route-queue passes with no HTTP response (tried ${transportPoison.routes.join(', ')}). Check network and provider transport before reviving.`;
-    const body = `⚠ Crash loop — \`${meta.name}\` (${meta.kind}) kept dying without settling a turn.\n\n` +
-        `Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying.` +
-        observedCause +
-        `\n\nFix the crash cause, then revive the node if the work still needs doing.`;
-    await pushUrgent(meta.node_id, body, { from: meta.node_id });
-}
-/** Surface a recovery refusal that happened before a broker could launch. */
-async function surfaceRecoveryFailure(meta, error) {
-    const message = error instanceof Error ? error.message : String(error);
-    const body = `⚠ Recovery failed — \`${meta.name}\` (${meta.kind}) could not be relaunched.\n\n` +
-        `The daemon refused this recovery before starting a broker: ${message}\n\n` +
-        `The node is now dead. Fix the recorded launch configuration, then revive it if the work still needs doing.`;
-    await pushUrgent(meta.node_id, body, { from: meta.node_id });
-}
-/** Whether a dead relaunch should resume the saved session strictly, or retry
- *  as a fresh cycle instead. A cycling attempt (cycle_pending) that dies before
- *  session_start must retry AS a cycle: the fresh branch it was opening is not
- *  confirmed, so resuming would silently continue the old leaf the refresh was
- *  meant to leave behind.
- *
- *  An interrupted turn is NOT a reason to cycle. A strict reopen does restore
- *  the transcript at an idle prompt without restarting the dead turn — but the
- *  fix for that is the continuation prompt recovery delivers (see `#respawn`),
- *  not throwing the conversation away. */
-export function retryResumeMode(meta) {
-    return meta?.cycle_pending !== true;
-}
-/** One pure function producing the action for a node known-dead. Used at
- *  exactly two call sites — an exit-policy job and the node-lifecycle tick
- *  (design D-12) — so exit recovery and tick recovery can never diverge.
- *  Evaluated in table order; `busy` is the on-disk busy marker.
- *
- *  `busy` no longer selects the ACTION. Whether the engine died mid-turn or
- *  parked at an idle prompt, the saved transcript is equally resumable — pi
- *  persists whole entries (`appendFileSync` per line) so a killed generation
- *  simply leaves the file ending at its last complete entry, and the loader
- *  skips a malformed line rather than rejecting the session. A mid-turn death
- *  needs the turn re-driven, which is a continuation prompt on top of the
- *  resume (see `#respawn`), not a fresh branch. `busy` therefore survives here
- *  only to route a node that died before session_start ever recorded a session
- *  to a fresh relaunch instead of terminalizing it as a boot failure.
- *
- *  `pendingWake` is the one non-row input (`hasPendingWake`): whether anything
- *  would drive a turn if the engine came back. It selects row 4b and defaults
- *  to `true`, so an uninformed caller keeps the conservative respawn. */
-export function classifyDeadNode(row, busy, pendingWake = true) {
-    if (row === null)
-        return 'forget'; // row 1: row deleted
-    if (row.status === 'done' || row.status === 'canceled' || row.status === 'dead')
-        return 'forget'; // row 2
-    if (row.intent === 'refresh')
-        return 'respawn-fresh'; // row 3
-    if (row.intent === 'idle-release')
-        return 'dormant'; // row 4
-    // Row 4b — a RESIDENT node is interactable and is never forced to finish
-    // anything (stop-guard's `dormant`), so one that died between turns with
-    // nothing waiting for it has no turn to come back to. Resuming it would boot
-    // an engine that sits at an idle prompt until the park clock releases it 15
-    // minutes later, and a daemon restart classifies EVERY such node at once —
-    // which is how a canvas with a hundred parked conversations stampedes its
-    // host to death at boot. Release instead: the row stays revivable and its
-    // next message wakes it with the same session resume.
-    // A terminal node is deliberately excluded: with nothing live and no final
-    // pushed it has STALLED, and coming back to be reprompted is the point.
-    if (row.lifecycle === 'resident' && !busy && !pendingWake && row.pi_session_id != null)
-        return 'release-idle';
-    if (row.pi_session_id != null)
-        return retryResumeMode(row) ? 'respawn-resume' : 'respawn-cycle'; // row 5: any saved session
-    if (busy)
-        return 'respawn-fresh'; // row 6: died mid-turn before a session was ever recorded
-    if (row.fork_from != null)
-        return 'respawn-fresh'; // row 7: bounded ordinary fork-birth retry
-    return 'boot-failure'; // row 8
-}
-/** How long a capacity denial stays quiet before it is worth logging again. */
-const CAPACITY_LOG_THROTTLE_MS = 30_000;
-export class DaemonFleet {
-    #epoch;
-    #enqueue;
-    #now;
-    #probeOnline;
-    #map = new Map();
-    /** Slots claimed by launches that have not registered a broker yet. */
-    #reservations = new Set();
-    /** Brokers deliberately interrupted by profile pause. This belongs to the
-     * exact child, so a resume cannot turn its later SIGTERM exit into a crash. */
-    #profilePauseInterruptions = new Set();
-    /** Exit status held only until dead-row policy surfaces this broker's boot failure. */
-    #exitStatuses = new Map();
-    #lastCapacityLogAt = Number.NEGATIVE_INFINITY;
-    constructor(opts) {
-        this.#epoch = opts.epoch;
-        this.#enqueue = opts.enqueue;
-        this.#now = opts.now ?? Date.now;
-        this.#probeOnline = opts.probeOnline ?? (() => probeOnlineReal());
-    }
-    epoch() {
-        return this.#epoch;
-    }
-    register(nodeId, handle) {
-        if (this.#map.has(nodeId)) {
-            throw new Error(`fleet: ${nodeId} already has a live broker entry — refusing a second launch (programming error)`);
-        }
-        if (handle.pid === null) {
-            throw new Error(`fleet: refusing to register ${nodeId} with no pid`);
-        }
-        this.#reservations.delete(nodeId);
-        this.#exitStatuses.delete(nodeId);
-        this.#map.set(nodeId, { pid: handle.pid, handle });
-    }
-    has(nodeId) {
-        return this.#map.has(nodeId);
-    }
-    isClaimed(nodeId) {
-        return this.#map.has(nodeId) || this.#reservations.has(nodeId);
-    }
-    get(nodeId) {
-        return this.#map.get(nodeId);
-    }
-    size() {
-        return this.#map.size;
-    }
-    occupancy() {
-        return this.#map.size + this.#reservations.size;
-    }
-    reserveSlot(nodeId) {
-        if (this.#reservations.has(nodeId) || this.#map.has(nodeId))
-            return true;
-        const { automaticReviveCap } = brokerThresholdsForDaemon();
-        if (this.occupancy() < automaticReviveCap) {
-            this.#reservations.add(nodeId);
-            return true;
-        }
-        const now = this.#now();
-        if (now - this.#lastCapacityLogAt >= CAPACITY_LOG_THROTTLE_MS) {
-            this.#lastCapacityLogAt = now;
-            emitEvent({
-                level: 'warn',
-                event: 'broker.capacity.revive_deferred',
-                fields: { live_count: this.occupancy(), hard_cap: automaticReviveCap },
-            });
-        }
-        return false;
-    }
-    releaseReservation(nodeId) {
-        this.#reservations.delete(nodeId);
-    }
-    entries() {
-        return this.#map.entries();
-    }
-    interruptForProfilePause(nodeId, pid = this.#map.get(nodeId)?.pid) {
-        if (pid === undefined)
-            return;
-        this.#profilePauseInterruptions.add(nodeId);
-        try {
-            process.kill(pid, 'SIGTERM');
-        }
-        catch {
-            // A live handle or identity-checked PID can exit between its observation
-            // and signal. The marker still makes that exact observed exit harmless.
-        }
-    }
-    hasProfilePauseInterruption(nodeId) {
-        return this.#profilePauseInterruptions.has(nodeId);
-    }
-    forget(nodeId) {
-        this.#map.delete(nodeId);
-        this.#reservations.delete(nodeId);
-        this.#exitStatuses.delete(nodeId);
-    }
-    onChildExit(nodeId, status) {
-        // The real child exit is the liveness boundary. Its queued observation only
-        // records diagnostics, so it cannot retain capacity after the process is
-        // gone and make an explicit replacement look already live.
-        const meta = getNode(nodeId);
-        const launchedMs = meta?.launched_at == null ? Number.NaN : Date.parse(meta.launched_at);
-        const uptimeMs = Number.isFinite(launchedMs) ? Math.max(0, this.#now() - launchedMs) : null;
-        const consecutiveFailures = meta?.respawn_failures ?? 0;
-        this.#map.delete(nodeId);
-        if (meta?.status === 'done' || meta?.status === 'canceled' || meta?.status === 'dead')
-            this.#exitStatuses.delete(nodeId);
-        else
-            this.#exitStatuses.set(nodeId, status);
-        this.#enqueue(() => this.#emitExitObserved(nodeId, status, uptimeMs, consecutiveFailures));
-    }
-    deliverExit(nodeId, status) {
-        this.onChildExit(nodeId, status);
-    }
-    /** The node-lifecycle tick's enactment entry. */
-    async applyDeadRowPolicy(nodeId) {
-        const exitStatus = this.#exitStatuses.get(nodeId);
-        try {
-            await this.#applyPolicy(nodeId, exitStatus);
-        }
-        finally {
-            if (this.#exitStatuses.get(nodeId) === exitStatus)
-                this.#exitStatuses.delete(nodeId);
-        }
-    }
-    /** A revive refusal before its launch try/catch leaves the row active but
-     *  with no fleet entry. Convert that otherwise-silent strand into the same
-     *  terminal outcome as a boot failure. The fresh read avoids duplicating
-     *  reviveNode's launch-failure crash path. */
-    async terminalizeIfStillNonTerminal(nodeId, event, error, reason, detail) {
-        const meta = getNode(nodeId);
-        if (meta === null || meta.status === 'done' || meta.status === 'canceled' || meta.status === 'dead')
-            return false;
-        await this.#terminalize(nodeId, meta, {
-            event,
-            reason,
-            detail,
-            wake: `Child crashed — ${fullName(meta)} (${nodeId}) could not be relaunched and is now dead. ` +
-                `It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`,
-            surface: () => surfaceRecoveryFailure(meta, error),
-        });
-        return true;
-    }
-    dispose() { }
-    #emitExitObserved(nodeId, status, uptimeMs, consecutiveFailures) {
-        emitEvent({
-            level: 'info',
-            event: 'broker.exit.observed',
-            ...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
-            fields: {
-                ...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
-                code: status.code,
-                signal: status.signal,
-                uptime_ms: uptimeMs,
-                consecutive_failures: consecutiveFailures,
-            },
-        });
-    }
-    async #applyPolicy(nodeId, exitStatus) {
-        return this.#decideAndEnact(nodeId, exitStatus);
-    }
-    async #decideAndEnact(nodeId, exitStatus) {
-        const meta = getNode(nodeId);
-        if (meta !== null && isProfilePaused(meta.profile_id)) {
-            clearRespawnPolicy(nodeId);
-            return 'forget';
-        }
-        // Consume one unpaused policy pass for the exact broker profile pause
-        // interrupted. The following tick owns ordinary recovery regardless of
-        // which dead-row verdict the still-recorded PID would otherwise select.
-        if (this.#profilePauseInterruptions.delete(nodeId)) {
-            clearRespawnPolicy(nodeId);
-            return 'forget';
-        }
-        // The execution-scoped turn witness survives either a dirty death or a
-        // broker-certified abort. Both need the same recovery: resume the saved
-        // transcript and re-drive the stopped turn.
-        const interruptedTurn = readInterruptedTurn(nodeId)?.interrupted === true;
-        const action = classifyDeadNode(meta, interruptedTurn, hasPendingWake(nodeId));
-        switch (action) {
-            case 'forget':
-                // Fanouts for terminal exits already happened at the transition site.
-                clearRespawnPolicy(nodeId);
-                break;
-            case 'dormant':
-                // Chosen dormancy is not a failure — the counter does not survive it.
-                // The pathological wake→fault→release→wake loop is bounded by the
-                // tick's pass-2 cursor-keyed retry cap (D-11), not this policy.
-                clearRespawnPolicy(nodeId);
-                break;
-            case 'release-idle':
-                // Dormancy the daemon chose ON the node's behalf, and the same
-                // non-failure: the row lands exactly where a stophook idle-release
-                // would have left it, so pass-2 inbox wakes own its next revive.
-                clearRespawnPolicy(nodeId);
-                transition(nodeId, 'release');
-                emitEvent({
-                    level: 'info',
-                    event: 'broker.recovery.released',
-                    ...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
-                    fields: {
-                        ...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
-                    },
-                });
-                break;
-            case 'boot-failure':
-                await this.#terminalize(nodeId, meta, {
-                    event: 'broker.boot.failed',
-                    reason: 'boot_failed',
-                    detail: { schema: 'crtr.node-outcome-detail/v1', message: bootFailureBody(meta, exitStatus) },
-                    wake: `Child crashed — ${fullName(meta)} (${nodeId}) exited before its session ever came up and is now dead. ` +
-                        `It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`,
-                    surface: () => surfaceBootFailure(meta, exitStatus),
-                });
-                break;
-            case 'respawn-fresh':
-            case 'respawn-cycle':
-            case 'respawn-resume':
-                await this.#respawn(nodeId, meta, action, interruptedTurn);
-                break;
-        }
-        return action;
-    }
-    async #respawn(nodeId, meta, action, interruptedTurn) {
-        const row = getRow(nodeId);
-        if (row === null)
-            return;
-        const priorFailures = row.respawn_failures ?? 0;
-        // A box with no network fails every relaunch for a reason that has nothing
-        // to do with this node: with no route reachable, the broker's provider queue
-        // exhausts with no HTTP response at all, it declares its own transport
-        // poisoned, and it exits for a fresh process that fails identically. Six of
-        // those and the node is dead a few minutes into an outage, holding an
-        // unfinished turn that only a human revive resumes. Park instead of counting
-        // — the outage is a condition to wait out, not a failure sequence to pace.
-        if (!(await this.#probeOnline())) {
-            deferRespawnWhileOffline(nodeId, priorFailures, this.#now());
-            emitEvent({
-                level: 'info',
-                event: 'broker.respawn.offline_deferred',
-                ...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
-                fields: {
-                    ...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
-                    action,
-                    retry_in_ms: OFFLINE_RESPAWN_RETRY_MS,
-                    consecutive_failures: priorFailures,
-                },
-            });
-            return;
-        }
-        const assess = row.respawn_not_before == null;
-        const attempt = assess
-            ? recordRespawnPolicy(row, this.#now())
-            : { failures: priorFailures, notBefore: row.respawn_not_before, exhausted: priorFailures >= MAX_RESPAWN_ATTEMPTS };
-        if (attempt.exhausted) {
-            const wake = `Child crashed — ${fullName(meta)} (${nodeId}) kept dying without settling a turn (${attempt.failures} consecutive ` +
-                `launches) and is now dead. It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`;
-            await this.#terminalize(nodeId, meta, {
-                event: 'broker.respawn.exhausted',
-                reason: 'crash_looped',
-                detail: { schema: 'crtr.node-outcome-detail/v1', message: wake, respawn_failures: attempt.failures },
-                wake,
-                ...(meta.pi_session_id != null ? { surface: () => surfaceCrashLoop(meta) } : {}),
-            });
-            return;
-        }
-        if (assess && attempt.notBefore !== null) {
-            emitEvent({
-                level: 'info',
-                event: 'broker.respawn.deferred',
-                ...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
-                fields: {
-                    ...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
-                    action,
-                    backoff_ms: respawnBackoffMs(attempt.failures),
-                    consecutive_failures: attempt.failures,
-                },
-            });
-            return;
-        }
-        emitEvent({
-            level: 'info',
-            event: 'broker.respawn.started',
-            ...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
-            fields: {
-                ...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
-                action,
-                consecutive_failures: attempt.failures,
-            },
-        });
-        try {
-            await reviveNode(nodeId, {
-                // A recovery that loses the race for the last slot freezes and waits
-                // for one — the tick thaws it — rather than erroring the node away.
-                capacity: 'freeze',
-                resume: action === 'respawn-resume',
-                // A resumed session whose turn was interrupted comes back parked at an
-                // idle prompt with the dead turn unfinished. The continuation re-drives
-                // it in place, so the conversation continues instead of being rebuilt
-                // from disk on a fresh branch. Only a true resume can carry it; a cycle
-                // or fresh relaunch gets its own kickoff.
-                wakeReason: action === 'respawn-resume' && interruptedTurn ? 'runtime-restart-abort' : undefined,
-                // Replacing a dead broker instance is recovery, not a wake: nothing
-                // external arrived for this node, so an armed deadline survives the
-                // relaunch instead of being consumed by it.
-                recovery: true,
-            });
-        }
-        catch (err) {
-            emitEvent({
-                level: 'error',
-                event: 'broker.respawn.failed',
-                ...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
-                error: err,
-                fields: {
-                    ...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
-                    action,
-                },
-            });
-            // Launch failures terminalize inside reviveNode, but its preflight
-            // refusals (for example cwd_missing) happen before that catch. Re-read
-            // before terminalizing so the launch path's settled terminal row wins.
-            await this.terminalizeIfStillNonTerminal(nodeId, 'broker.respawn.refused', err, 'launch_failed', { schema: 'crtr.node-outcome-detail/v1', message: err instanceof Error ? err.message : String(err) });
-        }
-    }
-    /** Terminal outcome shared by table row 8 and respawn exhaustion. */
-    async #terminalize(nodeId, meta, opts) {
-        emitEvent({
-            level: 'error',
-            event: opts.event,
-            ...(isSafeNodeId(nodeId) ? { node_id: nodeId } : { fields: { affected_node_id: nodeId } }),
-        });
-        transition(nodeId, 'crash', { reason: opts.reason, outcome: { detail: opts.detail } });
-        try {
-            await (opts.surface ?? (() => surfaceBootFailure(meta)))();
-        }
-        catch (err) {
-            emitEvent({
-                level: 'error',
-                event: 'broker.boot_failure_surface.failed',
-                ...(isSafeNodeId(nodeId) ? { node_id: nodeId } : { fields: { affected_node_id: nodeId } }),
-                error: err,
-            });
-        }
-        fanDoctrineWake(nodeId, subscribersOf(nodeId), opts.wake, { reason: 'child-crashed', child: nodeId });
-    }
-}
+var b=Object.defineProperty;var l=(s,e)=>b(s,"name",{value:e,configurable:!0});import{getNode as d,getRow as g,subscribersOf as k}from"../core/canvas/index.js";import{isProfilePaused as $}from"../core/profiles/manifest.js";import{fullName as p}from"../core/canvas/labels.js";import{isSafeNodeId as a}from"../core/canvas/paths.js";import{transition as m}from"../core/runtime/lifecycle.js";import{fanDoctrineWake as T}from"../core/runtime/close.js";import{readInterruptedTurn as P,readTransportPoisonVerdict as x}from"../core/canvas/recovery.js";import{hasPendingWake as E}from"./reconcilers/live-obligation.js";import{clearRespawnPolicy as h,deferRespawnWhileOffline as N,MAX_RESPAWN_ATTEMPTS as R,OFFLINE_RESPAWN_RETRY_MS as C,recordRespawnPolicy as F,respawnBackoffMs as S}from"./reconcilers/node-lifecycle/respawn-policy.js";import{probeOnline as M}from"../core/runtime/connectivity.js";import{reviveNode as O}from"../core/runtime/revive.js";import{pushUrgent as v}from"../core/feed/feed.js";import{emitEvent as c}from"../core/events/emit.js";import{brokerThresholdsForDaemon as A}from"../core/runtime/fleet.js";function w(s,e){const r=e===void 0?"The daemon did not observe an exit status.":`The observed exit status was code ${e.code??"null"}, signal ${e.signal??"null"}.`;return`\u26A0 Spawn failed \u2014 \`${s.name}\` (${s.kind}) never started.
+
+Its pi vehicle exited before the session came up (no pi_session_id was ever recorded), so the node produced no output. ${r}
+
+Inspect canonical event diagnostics with \`crtr sys logs --node ${s.node_id}\`.
+
+If the work still needs doing, re-spawn it.`}l(w,"bootFailureBody");async function y(s,e){await v(s.node_id,w(s,e),{from:s.node_id})}l(y,"surfaceBootFailure");async function z(s){const e=x(s.node_id),r=e===null?"":`
+
+Provider rotation observed ${e.consecutivePasses} consecutive route-queue passes with no HTTP response (tried ${e.routes.join(", ")}). Check network and provider transport before reviving.`,t=`\u26A0 Crash loop \u2014 \`${s.name}\` (${s.kind}) kept dying without settling a turn.
+
+Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying.`+r+`
+
+Fix the crash cause, then revive the node if the work still needs doing.`;await v(s.node_id,t,{from:s.node_id})}l(z,"surfaceCrashLoop");async function D(s,e){const r=e instanceof Error?e.message:String(e),t=`\u26A0 Recovery failed \u2014 \`${s.name}\` (${s.kind}) could not be relaunched.
+
+The daemon refused this recovery before starting a broker: ${r}
+
+The node is now dead. Fix the recorded launch configuration, then revive it if the work still needs doing.`;await v(s.node_id,t,{from:s.node_id})}l(D,"surfaceRecoveryFailure");function B(s){return s?.cycle_pending!==!0}l(B,"retryResumeMode");function L(s,e,r=!0){return s===null||s.status==="done"||s.status==="canceled"||s.status==="dead"?"forget":s.intent==="refresh"?"respawn-fresh":s.intent==="idle-release"?"dormant":s.lifecycle==="resident"&&!e&&!r&&s.pi_session_id!=null?"release-idle":s.pi_session_id!=null?B(s)?"respawn-resume":"respawn-cycle":e||s.fork_from!=null?"respawn-fresh":"boot-failure"}l(L,"classifyDeadNode");const W=3e4;class te{static{l(this,"DaemonFleet")}#n;#o;#s;#l;#e=new Map;#r=new Set;#i=new Set;#t=new Map;#c=Number.NEGATIVE_INFINITY;constructor(e){this.#n=e.epoch,this.#o=e.enqueue,this.#s=e.now??Date.now,this.#l=e.probeOnline??(()=>M())}epoch(){return this.#n}register(e,r){if(this.#e.has(e))throw new Error(`fleet: ${e} already has a live broker entry \u2014 refusing a second launch (programming error)`);if(r.pid===null)throw new Error(`fleet: refusing to register ${e} with no pid`);this.#r.delete(e),this.#t.delete(e),this.#e.set(e,{pid:r.pid,handle:r})}has(e){return this.#e.has(e)}isClaimed(e){return this.#e.has(e)||this.#r.has(e)}get(e){return this.#e.get(e)}size(){return this.#e.size}occupancy(){return this.#e.size+this.#r.size}reserveSlot(e){if(this.#r.has(e)||this.#e.has(e))return!0;const{automaticReviveCap:r}=A();if(this.occupancy()<r)return this.#r.add(e),!0;const t=this.#s();return t-this.#c>=W&&(this.#c=t,c({level:"warn",event:"broker.capacity.revive_deferred",fields:{live_count:this.occupancy(),hard_cap:r}})),!1}releaseReservation(e){this.#r.delete(e)}entries(){return this.#e.entries()}interruptForProfilePause(e,r=this.#e.get(e)?.pid){if(r!==void 0){this.#i.add(e);try{process.kill(r,"SIGTERM")}catch{}}}hasProfilePauseInterruption(e){return this.#i.has(e)}forget(e){this.#e.delete(e),this.#r.delete(e),this.#t.delete(e)}onChildExit(e,r){const t=d(e),n=t?.launched_at==null?Number.NaN:Date.parse(t.launched_at),o=Number.isFinite(n)?Math.max(0,this.#s()-n):null,i=t?.respawn_failures??0;this.#e.delete(e),t?.status==="done"||t?.status==="canceled"||t?.status==="dead"?this.#t.delete(e):this.#t.set(e,r),this.#o(()=>this.#u(e,r,o,i))}deliverExit(e,r){this.onChildExit(e,r)}async applyDeadRowPolicy(e){const r=this.#t.get(e);try{await this.#f(e,r)}finally{this.#t.get(e)===r&&this.#t.delete(e)}}async terminalizeIfStillNonTerminal(e,r,t,n,o){const i=d(e);return i===null||i.status==="done"||i.status==="canceled"||i.status==="dead"?!1:(await this.#a(e,i,{event:r,reason:n,detail:o,wake:`Child crashed \u2014 ${p(i)} (${e}) could not be relaunched and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>D(i,t),"surface")}),!0)}dispose(){}#u(e,r,t,n){c({level:"info",event:"broker.exit.observed",...a(e)?{node_id:e}:{},fields:{...a(e)?{}:{affected_node_id:e},code:r.code,signal:r.signal,uptime_ms:t,consecutive_failures:n}})}async#f(e,r){return this.#h(e,r)}async#h(e,r){const t=d(e);if(t!==null&&$(t.profile_id))return h(e),"forget";if(this.#i.delete(e))return h(e),"forget";const n=P(e)?.interrupted===!0,o=L(t,n,E(e));switch(o){case"forget":h(e);break;case"dormant":h(e);break;case"release-idle":h(e),m(e,"release"),c({level:"info",event:"broker.recovery.released",...a(e)?{node_id:e}:{},fields:{...a(e)?{}:{affected_node_id:e}}});break;case"boot-failure":await this.#a(e,t,{event:"broker.boot.failed",reason:"boot_failed",detail:{schema:"crtr.node-outcome-detail/v1",message:w(t,r)},wake:`Child crashed \u2014 ${p(t)} (${e}) exited before its session ever came up and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`,surface:l(()=>y(t,r),"surface")});break;case"respawn-fresh":case"respawn-cycle":case"respawn-resume":await this.#d(e,t,o,n);break}return o}async#d(e,r,t,n){const o=g(e);if(o===null)return;const i=o.respawn_failures??0;if(!await this.#l()){N(e,i,this.#s()),c({level:"info",event:"broker.respawn.offline_deferred",...a(e)?{node_id:e}:{},fields:{...a(e)?{}:{affected_node_id:e},action:t,retry_in_ms:C,consecutive_failures:i}});return}const _=o.respawn_not_before==null,f=_?F(o,this.#s()):{failures:i,notBefore:o.respawn_not_before,exhausted:i>=R};if(f.exhausted){const u=`Child crashed \u2014 ${p(r)} (${e}) kept dying without settling a turn (${f.failures} consecutive launches) and is now dead. It stays dead until you revive it \u2014 \`crtr node lifecycle revive ${e}\`.`;await this.#a(e,r,{event:"broker.respawn.exhausted",reason:"crash_looped",detail:{schema:"crtr.node-outcome-detail/v1",message:u,respawn_failures:f.failures},wake:u,...r.pi_session_id!=null?{surface:l(()=>z(r),"surface")}:{}});return}if(_&&f.notBefore!==null){c({level:"info",event:"broker.respawn.deferred",...a(e)?{node_id:e}:{},fields:{...a(e)?{}:{affected_node_id:e},action:t,backoff_ms:S(f.failures),consecutive_failures:f.failures}});return}c({level:"info",event:"broker.respawn.started",...a(e)?{node_id:e}:{},fields:{...a(e)?{}:{affected_node_id:e},action:t,consecutive_failures:f.failures}});try{await O(e,{capacity:"freeze",resume:t==="respawn-resume",wakeReason:t==="respawn-resume"&&n?"runtime-restart-abort":void 0,recovery:!0})}catch(u){c({level:"error",event:"broker.respawn.failed",...a(e)?{node_id:e}:{},error:u,fields:{...a(e)?{}:{affected_node_id:e},action:t}}),await this.terminalizeIfStillNonTerminal(e,"broker.respawn.refused",u,"launch_failed",{schema:"crtr.node-outcome-detail/v1",message:u instanceof Error?u.message:String(u)})}}async#a(e,r,t){c({level:"error",event:t.event,...a(e)?{node_id:e}:{fields:{affected_node_id:e}}}),m(e,"crash",{reason:t.reason,outcome:{detail:t.detail}});try{await(t.surface??(()=>y(r)))()}catch(n){c({level:"error",event:"broker.boot_failure_surface.failed",...a(e)?{node_id:e}:{fields:{affected_node_id:e}},error:n})}T(e,k(e),t.wake,{reason:"child-crashed",child:e})}}export{te as DaemonFleet,L as classifyDeadNode,B as retryResumeMode,y as surfaceBootFailure};