@north-light/crouter 0.3.173 → 0.3.174

package/dist/core/runtime/broker/daemon-ops.js

@@ -0,0 +1,49 @@
+// Broker-to-daemon operation client. A broker is launched by crtrd, so a cold
+// daemon socket must fail loud — it must never use the CLI factory that can
+// autostart another daemon.
+import { CrtrClient } from '../../../api/client.js';
+import { ApiError } from '../../../api/errors.js';
+function daemonClient() {
+    return CrtrClient.forLocalSocket({ autostart: false });
+}
+export function bindBrokerSession(nodeId, request) {
+    return daemonClient().bindBrokerSession(nodeId, request);
+}
+export function settleBroker(nodeId, request) {
+    return daemonClient().settleBroker(nodeId, request);
+}
+export function advanceBrokerInboxCursor(nodeId, request) {
+    return daemonClient().advanceBrokerInboxCursor(nodeId, request);
+}
+/** Fixed canvas projection for broker extension rendering. It deliberately
+ * exposes no generic node/database facade. */
+export function commitBrokerModel(nodeId, request) {
+    return daemonClient().commitBrokerModel(nodeId, request);
+}
+export function brokerExtensionState(nodeId) {
+    return daemonClient().brokerExtensionState(nodeId);
+}
+export function commitBrokerGeneratedName(nodeId, request) {
+    return daemonClient().commitBrokerGeneratedName(nodeId, request);
+}
+export function commitBrokerPersonaAck(nodeId, request) {
+    return daemonClient().commitBrokerPersonaAck(nodeId, request);
+}
+/** Reports still use the canonical daemon report handler; this broker adapter
+ * only preserves terminal structured output's idempotent finalization result. */
+export async function publishBrokerReport(nodeId, request) {
+    try {
+        await daemonClient().pushReport(nodeId, request);
+    }
+    catch (error) {
+        if (request.tier === 'final' && error instanceof ApiError && error.code === 'already_finalized')
+            return;
+        throw error;
+    }
+}
+/** Root `/new` remains on the dedicated relaunch operation because it launches
+ * a replacement broker and tears down this broker. Callers must not await it
+ * from Pi's session-start dispatch. */
+export function relaunchBrokerRoot(nodeId) {
+    return daemonClient().relaunchRoot(nodeId);
+}

package/dist/core/runtime/broker/event-projection.d.ts

@@ -0,0 +1,42 @@
+import type { BrokerToClient } from '../broker-protocol.js';
+import type { BrokerSession } from './read-ops.js';
+import type { BrokerClientRegistry } from './client-registry.js';
+import type { SubscriptionGeneration } from './rebind.js';
+import type { FaultRetry } from './fault-retry.js';
+type EventProjectionDeps = {
+    registry: BrokerClientRegistry;
+    faultRetry: FaultRetry;
+    installedGeneration: () => SubscriptionGeneration | undefined;
+    currentSession: () => BrokerSession;
+    notifyTurnAccepted: () => void;
+    emitStartupMilestone: (event: string, model: string | null) => void;
+    formatModelSpec: (model: BrokerSession['model'], thinkingLevel?: string) => string | null;
+    refreshIntent: () => Promise<boolean>;
+    onRefreshIntentError: (error: unknown) => void;
+};
+/** Projects one live engine event stream to broker viewers and owns its relay timers. */
+export declare class EventProjection {
+    private readonly deps;
+    private pendingUpdate;
+    private pendingUpdateTimer;
+    private workingActivity;
+    private readonly streamWatchdog;
+    private readonly streamWatchdogMs;
+    private pendingBroadcast;
+    private broadcastGeneration;
+    constructor(deps: EventProjectionDeps);
+    currentWorkingActivity(): string;
+    isCurrent(generation: SubscriptionGeneration): boolean;
+    flush(): void;
+    reset(): void;
+    dispose(): void;
+    knownStreamWait(deadlineMs: number): () => void;
+    relayEvent(event: BrokerToClient, generation: SubscriptionGeneration): void;
+    private clearPendingUpdate;
+    private flushPendingUpdate;
+    private invalidatePendingBroadcasts;
+    private broadcast;
+    private broadcastMessageEnd;
+    private setPendingBroadcast;
+}
+export {};

package/dist/core/runtime/broker/event-projection.js

@@ -0,0 +1,178 @@
+import { readConfig } from '../../config.js';
+import { emitEvent } from '../../events/emit.js';
+import { releaseKnownStreamWait, resolveStreamWatchdogMs, StreamWatchdog } from '../stream-watchdog.js';
+import { withoutYieldAbort } from '../session-cycles.js';
+import { DEFAULT_WORKING_ACTIVITY, randomWorkingActivity } from '../../../shared/working-activity.js';
+const MESSAGE_UPDATE_COALESCE_MS = 75;
+/** Projects one live engine event stream to broker viewers and owns its relay timers. */
+export class EventProjection {
+    deps;
+    pendingUpdate = null;
+    pendingUpdateTimer = null;
+    workingActivity = DEFAULT_WORKING_ACTIVITY;
+    streamWatchdog;
+    streamWatchdogMs = resolveStreamWatchdogMs();
+    pendingBroadcast = null;
+    broadcastGeneration = 0;
+    constructor(deps) {
+        this.deps = deps;
+        this.streamWatchdog = new StreamWatchdog((armedGeneration) => {
+            const generation = this.deps.installedGeneration();
+            if (generation === undefined || armedGeneration !== generation.id)
+                return;
+            emitEvent({
+                level: 'debug',
+                event: 'broker.runtime.diagnostic',
+                fields: { message: `stream watchdog: no engine event for ${this.streamWatchdogMs}ms while streaming — aborting stalled provider stream` },
+            });
+            generation.stagedWatchdogAbort = true;
+            void generation.session.abort().catch((err) => {
+                emitEvent({ level: 'debug', event: 'broker.runtime.diagnostic', fields: { message: `stream watchdog abort failed: ${String(err)}` } });
+            });
+        }, this.streamWatchdogMs);
+    }
+    currentWorkingActivity() {
+        return this.workingActivity;
+    }
+    isCurrent(generation) {
+        return this.deps.installedGeneration() === generation;
+    }
+    flush() {
+        this.flushPendingUpdate();
+    }
+    reset() {
+        this.clearPendingUpdate();
+        this.invalidatePendingBroadcasts();
+        this.streamWatchdog.reset();
+    }
+    dispose() {
+        this.clearPendingUpdate();
+        this.invalidatePendingBroadcasts();
+        this.streamWatchdog.reset();
+    }
+    knownStreamWait(deadlineMs) {
+        if (!Number.isFinite(deadlineMs) || deadlineMs <= Date.now())
+            return () => { };
+        const generation = this.deps.installedGeneration();
+        if (generation === undefined)
+            return () => { };
+        const waitingSession = generation.session;
+        this.streamWatchdog.pause();
+        return () => {
+            if (this.deps.installedGeneration() !== generation || this.deps.currentSession() !== waitingSession)
+                return;
+            releaseKnownStreamWait(this.streamWatchdog, generation.id, waitingSession.isStreaming, generation.executingTools.size > 0);
+        };
+    }
+    relayEvent(event, generation) {
+        this.deps.faultRetry.onEvent(event);
+        const type = event.type;
+        if (type === 'message_start' && event.message.role === 'user') {
+            try {
+                this.workingActivity = randomWorkingActivity(readConfig('user').working_gerunds);
+            }
+            catch {
+                this.workingActivity = DEFAULT_WORKING_ACTIVITY;
+            }
+            this.broadcast({ type: 'working_activity', activity: this.workingActivity });
+        }
+        if (type === 'agent_start') {
+            this.deps.notifyTurnAccepted();
+            this.deps.emitStartupMilestone('broker.startup.agent_start', this.deps.formatModelSpec(this.deps.currentSession().model, this.deps.currentSession().thinkingLevel));
+        }
+        if (type === 'message_update' && event.message.role === 'assistant') {
+            this.deps.emitStartupMilestone('broker.startup.provider_first_delta', this.deps.formatModelSpec(this.deps.currentSession().model, this.deps.currentSession().thinkingLevel));
+        }
+        if (type === 'message_end' && event.message.role === 'assistant') {
+            this.deps.emitStartupMilestone('broker.startup.assistant_message_end', this.deps.formatModelSpec(this.deps.currentSession().model, this.deps.currentSession().thinkingLevel));
+        }
+        if (type === 'agent_start' || type === 'turn_start')
+            this.deps.faultRetry.clearTimer();
+        if (type === 'agent_end') {
+            generation.executingTools.clear();
+            this.streamWatchdog.reset();
+        }
+        else if (type === 'tool_execution_start') {
+            generation.executingTools.add(event.toolCallId ?? '');
+            this.streamWatchdog.clear();
+        }
+        else if (type === 'tool_execution_end') {
+            generation.executingTools.delete(event.toolCallId ?? '');
+            if (generation.executingTools.size === 0 && generation.session.isStreaming)
+                this.streamWatchdog.poke(generation.id);
+        }
+        else if (generation.session.isStreaming && generation.executingTools.size === 0) {
+            this.streamWatchdog.poke(generation.id);
+        }
+        if (type === 'message_update') {
+            this.pendingUpdate = event;
+            if (this.pendingUpdateTimer === null) {
+                this.pendingUpdateTimer = setTimeout(() => this.flushPendingUpdate(), MESSAGE_UPDATE_COALESCE_MS);
+                if (typeof this.pendingUpdateTimer.unref === 'function')
+                    this.pendingUpdateTimer.unref();
+            }
+            return;
+        }
+        this.flushPendingUpdate();
+        if (type === 'message_end') {
+            this.broadcastMessageEnd(event);
+            return;
+        }
+        this.broadcast(event);
+    }
+    clearPendingUpdate() {
+        if (this.pendingUpdateTimer !== null) {
+            clearTimeout(this.pendingUpdateTimer);
+            this.pendingUpdateTimer = null;
+        }
+        this.pendingUpdate = null;
+    }
+    flushPendingUpdate() {
+        if (this.pendingUpdateTimer !== null) {
+            clearTimeout(this.pendingUpdateTimer);
+            this.pendingUpdateTimer = null;
+        }
+        if (this.pendingUpdate !== null) {
+            const frame = this.pendingUpdate;
+            this.pendingUpdate = null;
+            this.broadcast(frame);
+        }
+    }
+    invalidatePendingBroadcasts() {
+        this.broadcastGeneration++;
+        this.pendingBroadcast = null;
+    }
+    broadcast(frame) {
+        const pending = this.pendingBroadcast;
+        if (pending === null) {
+            this.deps.registry.broadcast(frame);
+            return;
+        }
+        const generation = this.broadcastGeneration;
+        this.setPendingBroadcast(pending.then(() => {
+            if (this.broadcastGeneration !== generation)
+                return;
+            this.deps.registry.broadcast(frame);
+        }));
+    }
+    broadcastMessageEnd(event) {
+        const preceding = this.pendingBroadcast ?? Promise.resolve();
+        const generation = this.broadcastGeneration;
+        this.setPendingBroadcast(preceding.then(async () => {
+            const refresh = await this.deps.refreshIntent();
+            if (this.broadcastGeneration !== generation)
+                return;
+            this.deps.registry.broadcast(refresh ? { ...event, message: withoutYieldAbort(event.message) } : event);
+        }).catch((error) => {
+            if (this.broadcastGeneration === generation)
+                this.deps.onRefreshIntentError(error);
+        }));
+    }
+    setPendingBroadcast(work) {
+        this.pendingBroadcast = work;
+        void work.then(() => {
+            if (this.pendingBroadcast === work)
+                this.pendingBroadcast = null;
+        });
+    }
+}

package/dist/core/runtime/broker/fault-retry.d.ts

@@ -0,0 +1,39 @@
+import type { AgentSessionServices, ModelRegistry } from '@earendil-works/pi-coding-agent';
+import { readFault } from '../fault.js';
+import type { BrokerSdkConfig } from '../launch.js';
+import type { BrokerToClient } from '../broker-protocol.js';
+import type { BrokerSession } from './read-ops.js';
+import type { SubscriptionGeneration } from './rebind.js';
+type FaultRetryDeps = {
+    nodeId: string;
+    cfg: BrokerSdkConfig;
+    retryProbe: () => Promise<boolean>;
+    currentSession: () => BrokerSession;
+    currentServices: () => AgentSessionServices;
+    installedGeneration: () => SubscriptionGeneration | undefined;
+    registryOf: (services: AgentSessionServices) => ModelRegistry;
+    formatModelSpec: (model: BrokerSession['model'], thinkingLevel?: string) => string | null;
+    broadcastModelChanged: (persist?: boolean) => Promise<void>;
+};
+/** Owns provider fallback and daemon-driven fault retry state for one broker session. */
+export declare class FaultRetry {
+    private readonly deps;
+    private timer;
+    private readonly failedRetryRoutes;
+    private readonly notFoundModels;
+    private providerFallbackInFlight;
+    private notFoundFallbackInFlight;
+    constructor(deps: FaultRetryDeps);
+    clearTimer(): void;
+    reset(): void;
+    onEvent(event: BrokerToClient): void;
+    isActiveAutoFault(fault: ReturnType<typeof readFault>): fault is NonNullable<ReturnType<typeof readFault>>;
+    publishSettledProviderFault(generation: SubscriptionGeneration): void;
+    schedule(): void;
+    private maybeSwitchProviderForRetry;
+    private maybeFallbackOnUnusableModel;
+    private faultRetryDelayMs;
+    private clearAndArmFaultRetry;
+    private runFaultRetry;
+}
+export {};

package/dist/core/runtime/broker/fault-retry.js

@@ -0,0 +1,277 @@
+import { isContextOverflow } from '@earendil-works/pi-ai';
+import { classify } from '../../fault-classifier.js';
+import { clearFault, isModelNotFoundError, isProviderUnconfiguredError, readFault, recordFault } from '../fault.js';
+import { extractCoolingDeadline } from '../managed-provider-cooling.js';
+import { expandModelCandidates, modelRequestFromConfig, probeRouteAvailability } from '../../model-routes.js';
+import { operationIdContext } from '../../events/operation-id.js';
+import { emitEvent } from '../../events/emit.js';
+import { CONNECTION_FAULT_RECOVERY_BODY, PROVIDER_FAULT_RECOVERY_BODY, formatModelFallbackRecovery } from '../../../shared/generated-context.js';
+import { markFatalOnExhaust, nextFaultRetry, policyFor } from '../fault-recovery.js';
+import { parseModelSpec } from '../model-selection.js';
+import { resolveProviderCandidates } from '../../subscription-state.js';
+const MAX_TIMEOUT_MS = 2_147_483_647;
+/** Owns provider fallback and daemon-driven fault retry state for one broker session. */
+export class FaultRetry {
+    deps;
+    timer = null;
+    failedRetryRoutes = new Set();
+    notFoundModels = new Set();
+    providerFallbackInFlight = null;
+    notFoundFallbackInFlight = null;
+    constructor(deps) {
+        this.deps = deps;
+    }
+    clearTimer() {
+        if (this.timer === null)
+            return;
+        clearTimeout(this.timer);
+        this.timer = null;
+    }
+    reset() {
+        this.clearTimer();
+        this.providerFallbackInFlight = null;
+        this.notFoundFallbackInFlight = null;
+    }
+    onEvent(event) {
+        this.maybeSwitchProviderForRetry(event);
+        this.maybeFallbackOnUnusableModel(event);
+    }
+    isActiveAutoFault(fault) {
+        if (fault === null)
+            return false;
+        if (fault.retry.disposition !== 'auto')
+            return false;
+        if (fault.retry.by !== 'daemon')
+            return false;
+        return fault.kind === 'connection' || fault.kind === 'rate-limit' || fault.kind === 'overloaded';
+    }
+    publishSettledProviderFault(generation) {
+        const agentEnd = generation.stagedProviderAgentEnd;
+        const watchdogAbort = generation.stagedWatchdogAbort;
+        const overflowFailure = generation.stagedOverflowFailure;
+        generation.stagedProviderAgentEnd = null;
+        generation.stagedWatchdogAbort = false;
+        generation.stagedOverflowFailure = null;
+        const settledSession = generation.session;
+        const messages = Array.isArray(agentEnd?.messages) ? agentEnd.messages : [];
+        const last = [...messages].reverse().find((message) => typeof message === 'object' && message !== null && message.role === 'assistant');
+        if (overflowFailure !== null) {
+            clearFault(this.deps.nodeId, { link: 'pi→provider' });
+            recordFault(this.deps.nodeId, {
+                link: 'pi→provider', op: 'context overflow recovery', kind: 'context-overflow', retry: { disposition: 'fatal' },
+                message: overflowFailure.errorMessage, anchorEntryId: settledSession.sessionManager.getLeafId?.() ?? undefined,
+            });
+            return;
+        }
+        if (watchdogAbort && last?.stopReason === 'aborted') {
+            recordFault(this.deps.nodeId, {
+                link: 'pi→provider', op: 'generation turn', kind: 'connection', retry: { disposition: 'auto', by: 'daemon' },
+                message: typeof last.errorMessage === 'string' ? last.errorMessage : 'engine error (no errorMessage recorded)',
+                anchorEntryId: settledSession.sessionManager.getLeafId?.() ?? undefined,
+            });
+            return;
+        }
+        if (last?.stopReason !== 'error')
+            return;
+        const raw = typeof last.errorMessage === 'string' ? last.errorMessage : 'engine error (no errorMessage recorded)';
+        const uncompactedOverflow = isContextOverflow(last, settledSession.model?.contextWindow);
+        const classified = classify('pi→provider', last);
+        const coolingDeadline = extractCoolingDeadline(last);
+        if (uncompactedOverflow)
+            clearFault(this.deps.nodeId, { link: 'pi→provider' });
+        recordFault(this.deps.nodeId, {
+            link: 'pi→provider', op: 'generation turn', kind: uncompactedOverflow ? 'context-overflow' : classified.kind,
+            retry: uncompactedOverflow
+                ? { disposition: 'fatal' }
+                : classified.disposition === 'auto'
+                    ? { disposition: 'auto', by: 'daemon', ...(coolingDeadline !== undefined ? { nextAt: new Date(coolingDeadline).toISOString() } : {}) }
+                    : { disposition: classified.disposition },
+            message: raw, anchorEntryId: settledSession.sessionManager.getLeafId?.() ?? undefined,
+        });
+    }
+    schedule() {
+        const generation = this.deps.installedGeneration();
+        const session = this.deps.currentSession();
+        if (generation === undefined || session.isStreaming || this.timer !== null)
+            return;
+        const fault = readFault(this.deps.nodeId);
+        if (!this.isActiveAutoFault(fault)) {
+            this.clearTimer();
+            return;
+        }
+        const delayMs = this.faultRetryDelayMs(fault, Date.now());
+        if (delayMs !== null)
+            this.clearAndArmFaultRetry(delayMs, generation);
+    }
+    maybeSwitchProviderForRetry(event) {
+        if (event.type !== 'auto_retry_start')
+            return;
+        const kind = classify('pi→provider', event.errorMessage ?? '').kind;
+        if ((kind !== 'rate-limit' && kind !== 'connection') || this.providerFallbackInFlight !== null || this.deps.cfg.modelExact === true)
+            return;
+        const generation = this.deps.installedGeneration();
+        if (generation === undefined)
+            return;
+        const candidateSession = this.deps.currentSession();
+        const candidateServices = this.deps.currentServices();
+        const currentSpec = this.deps.formatModelSpec(candidateSession.model, candidateSession.thinkingLevel);
+        if (currentSpec === null)
+            return;
+        const routeRequest = modelRequestFromConfig(this.deps.cfg.model, process.env['CRTR_MODEL_INTENT'], false);
+        if (routeRequest === undefined)
+            return;
+        const routes = expandModelCandidates(routeRequest, process.env['CRTR_NODE_CWD'] ?? this.deps.cfg.cwd, process.env['CRTR_PROFILE_ID']);
+        const current = parseModelSpec(currentSpec);
+        const liveRank = routes.find((candidate) => `${candidate.providerId}/${candidate.modelId}` === current.modelSpec && candidate.thinkingLevel === current.thinkingLevel)?.rank;
+        const nextRoute = liveRank === undefined ? undefined : routes.slice(liveRank + 1).find((candidate) => !this.failedRetryRoutes.has(candidate.routeId) && resolveProviderCandidates([{ ...candidate, ...probeRouteAvailability(candidate, this.deps.registryOf(candidateServices)), automaticFallback: true }]).candidates.length > 0);
+        if (nextRoute === undefined)
+            return;
+        const fallbackPromise = (async () => {
+            const target = this.deps.registryOf(candidateServices).find(nextRoute.providerId, nextRoute.modelId);
+            if (!target)
+                return;
+            await candidateSession.setModel(target);
+            if (this.deps.installedGeneration() !== generation || this.deps.currentSession() !== candidateSession)
+                return;
+            candidateSession.setThinkingLevel((nextRoute.thinkingLevel ?? 'off'));
+            this.failedRetryRoutes.add(nextRoute.routeId);
+            await this.deps.broadcastModelChanged(false);
+        })().catch((err) => {
+            emitEvent({ level: 'debug', event: 'broker.runtime.diagnostic', fields: { message: `provider route fallback failed: ${err instanceof Error ? err.message : String(err)}` } });
+        }).finally(() => {
+            if (this.deps.installedGeneration() === generation)
+                this.providerFallbackInFlight = null;
+        });
+        this.providerFallbackInFlight = fallbackPromise;
+    }
+    maybeFallbackOnUnusableModel(event) {
+        if (event.type !== 'message_end' || event.message.role !== 'assistant' || event.message.stopReason !== 'error')
+            return;
+        const unconfigured = isProviderUnconfiguredError(event.message.errorMessage ?? '');
+        if ((!unconfigured && !isModelNotFoundError(event.message.errorMessage ?? '')) || this.deps.cfg.modelExact === true || this.notFoundFallbackInFlight !== null)
+            return;
+        const generation = this.deps.installedGeneration();
+        if (generation === undefined)
+            return;
+        const candidateSession = this.deps.currentSession();
+        const candidateServices = this.deps.currentServices();
+        const currentSpec = this.deps.formatModelSpec(candidateSession.model, candidateSession.thinkingLevel);
+        if (currentSpec === null)
+            return;
+        const current = parseModelSpec(currentSpec);
+        this.notFoundModels.add(current.modelSpec);
+        const routeRequest = modelRequestFromConfig(this.deps.cfg.model, process.env['CRTR_MODEL_INTENT'], false);
+        const routes = routeRequest ? expandModelCandidates(routeRequest, process.env['CRTR_NODE_CWD'] ?? this.deps.cfg.cwd, process.env['CRTR_PROFILE_ID']) : [];
+        const liveRank = routes.find((candidate) => `${candidate.providerId}/${candidate.modelId}` === current.modelSpec && candidate.thinkingLevel === current.thinkingLevel)?.rank;
+        const nextRoute = liveRank === undefined ? undefined : routes.slice(liveRank + 1).find((candidate) => !this.notFoundModels.has(`${candidate.providerId}/${candidate.modelId}`) && resolveProviderCandidates([{ ...candidate, ...probeRouteAvailability(candidate, this.deps.registryOf(candidateServices)), automaticFallback: true }]).candidates.length > 0);
+        if (!nextRoute)
+            return;
+        const target = `${nextRoute.providerId}/${nextRoute.modelId}${nextRoute.thinkingLevel ? `:${nextRoute.thinkingLevel}` : ''}`;
+        const fallbackPromise = (async () => {
+            const model = this.deps.registryOf(candidateServices).find(nextRoute.providerId, nextRoute.modelId);
+            if (model === undefined) {
+                emitEvent({ level: 'debug', event: 'broker.runtime.diagnostic', fields: { message: `${unconfigured ? 'unconfigured-provider' : 'not_found'} fallback skipped: ${target} is not registered` } });
+                return;
+            }
+            await candidateSession.setModel(model);
+            if (this.deps.installedGeneration() !== generation || this.deps.currentSession() !== candidateSession)
+                return;
+            candidateSession.setThinkingLevel((nextRoute.thinkingLevel ?? 'off'));
+            await this.deps.broadcastModelChanged();
+            const resolved = this.deps.formatModelSpec(candidateSession.model, candidateSession.thinkingLevel) ?? target;
+            emitEvent({ level: 'debug', event: 'broker.runtime.diagnostic', fields: { message: `${unconfigured ? 'unconfigured-provider' : 'not_found'} fallback: ${currentSpec} unavailable → ${resolved}; re-driving turn` } });
+            await candidateSession.agent.waitForIdle();
+            if (this.deps.installedGeneration() !== generation || this.deps.currentSession() !== candidateSession)
+                return;
+            if (unconfigured)
+                clearFault(this.deps.nodeId, { link: 'pi→provider' });
+            await operationIdContext.fresh(() => candidateSession.prompt(formatModelFallbackRecovery(currentSpec, resolved, unconfigured ? 'credential' : 'not-found')));
+        })().catch((err) => {
+            emitEvent({ level: 'debug', event: 'broker.runtime.diagnostic', fields: { message: `${unconfigured ? 'unconfigured-provider' : 'not_found'} fallback failed: ${err instanceof Error ? err.message : String(err)}` } });
+        }).finally(() => {
+            if (this.deps.installedGeneration() === generation)
+                this.notFoundFallbackInFlight = null;
+        });
+        this.notFoundFallbackInFlight = fallbackPromise;
+    }
+    faultRetryDelayMs(fault, now) {
+        if (fault.retry.nextAt !== undefined) {
+            const ms = Date.parse(fault.retry.nextAt) - now;
+            return Number.isFinite(ms) ? Math.max(0, ms) : null;
+        }
+        return nextFaultRetry(fault, now)?.delayMs ?? null;
+    }
+    clearAndArmFaultRetry(delayMs, generation) {
+        if (this.deps.installedGeneration() !== generation)
+            return;
+        this.clearTimer();
+        const chunkMs = Math.min(Math.max(0, delayMs), MAX_TIMEOUT_MS);
+        const remainingMs = Math.max(0, delayMs - chunkMs);
+        this.timer = setTimeout(() => {
+            this.timer = null;
+            if (this.deps.installedGeneration() !== generation)
+                return;
+            if (remainingMs > 0) {
+                this.clearAndArmFaultRetry(remainingMs, generation);
+                return;
+            }
+            void this.runFaultRetry().catch((err) => {
+                emitEvent({ level: 'debug', event: 'broker.runtime.diagnostic', fields: { message: `fault retry fire failed: ${err instanceof Error ? err.message : String(err)}` } });
+            });
+        }, chunkMs);
+        if (typeof this.timer.unref === 'function')
+            this.timer.unref();
+    }
+    async runFaultRetry() {
+        const generation = this.deps.installedGeneration();
+        if (generation === undefined)
+            return;
+        const candidateSession = generation.session;
+        if (candidateSession.isStreaming)
+            return;
+        const initial = readFault(this.deps.nodeId);
+        if (!this.isActiveAutoFault(initial))
+            return;
+        let fault = initial;
+        if (initial.kind === 'connection') {
+            const online = await this.deps.retryProbe();
+            if (this.deps.installedGeneration() !== generation || this.deps.currentSession() !== candidateSession || candidateSession.isStreaming)
+                return;
+            const fresh = readFault(this.deps.nodeId);
+            if (!this.isActiveAutoFault(fresh) || fresh.since !== initial.since)
+                return;
+            if (!online) {
+                this.clearAndArmFaultRetry(policyFor(fresh.kind).baseDelayMs, generation);
+                return;
+            }
+            fault = fresh;
+        }
+        const attempt = fault.retry.attempt ?? 0;
+        const policy = policyFor(fault.kind);
+        if (attempt >= policy.max) {
+            recordFault(this.deps.nodeId, markFatalOnExhaust(fault));
+            return;
+        }
+        const plan = nextFaultRetry(fault, Date.now());
+        if (plan === null) {
+            recordFault(this.deps.nodeId, markFatalOnExhaust(fault));
+            return;
+        }
+        if (fault.anchorEntryId !== undefined) {
+            try {
+                if ((candidateSession.sessionManager.getLeafId?.() ?? null) !== fault.anchorEntryId) {
+                    await candidateSession.navigateTree(fault.anchorEntryId);
+                    if (this.deps.installedGeneration() !== generation || this.deps.currentSession() !== candidateSession)
+                        return;
+                }
+            }
+            catch (err) {
+                emitEvent({ level: 'debug', event: 'broker.runtime.diagnostic', fields: { message: `fault retry rewind skipped: ${err instanceof Error ? err.message : String(err)}` } });
+            }
+        }
+        if (this.deps.installedGeneration() !== generation || this.deps.currentSession() !== candidateSession)
+            return;
+        recordFault(this.deps.nodeId, plan.fault);
+        await operationIdContext.fresh(() => candidateSession.prompt(fault.kind === 'connection' ? CONNECTION_FAULT_RECOVERY_BODY : PROVIDER_FAULT_RECOVERY_BODY));
+    }
+}