@north-light/crouter 0.3.318 → 0.3.319

package/dist/core/runtime/broker/fault-retry.js

@@ -1,588 +1 @@
-import { envModelIntent, envNodeCwd, envProfileId } from '../../../shared/env.js';
-import { isContextOverflow } from '@earendil-works/pi-ai';
-import { classify } from '../../fault-classifier.js';
-import { isModelNotFoundError, isProviderUnconfiguredError } from '../fault.js';
-import { brokerRecovery, mutateBrokerFault, mutateBrokerProviderRetry } from './daemon-ops.js';
-import { extractCoolingDeadline } from '../managed-provider-cooling.js';
-import { extractProviderFailureDetails } from '../../provider-failure.js';
-import { expandModelCandidates, modelRequestFromConfig, probeRouteAvailability } from '../../model-routes.js';
-import { operationIdContext } from '../../events/operation-id.js';
-import { emitEvent } from '../../events/emit.js';
-import { CONNECTION_FAULT_RECOVERY_BODY, PROVIDER_FAULT_RECOVERY_BODY, formatCard, formatModelFallbackRecovery } from '../../../shared/generated-context.js';
-import { markFatalOnExhaust, nextFaultRetry, policyFor } from '../fault-recovery.js';
-import { parseModelSpec } from '../model-selection.js';
-import { endedByAbort } from '../session-cycles.js';
-import { resolveProviderCandidates } from '../../subscription-state.js';
-import { announceMessageOrigin } from '../stamp/channel.js';
-const MAX_TIMEOUT_MS = 2_147_483_647;
-function policyFault(fault) {
-    return {
-        link: fault.link,
-        op: fault.op,
-        kind: fault.kind,
-        retry: fault.retry,
-        message: fault.message,
-        since: fault.since,
-        operation_id: fault.operation_id,
-        ...(fault.anchor_entry_id === undefined ? {} : { anchorEntryId: fault.anchor_entry_id }),
-        ...(fault.provider_error === undefined ? {} : { providerError: fault.provider_error }),
-    };
-}
-function faultInput(fault) {
-    return {
-        link: fault.link,
-        op: fault.op,
-        kind: fault.kind,
-        retry: fault.retry,
-        message: fault.message,
-        ...(fault.operation_id === undefined ? {} : { operation_id: fault.operation_id }),
-        ...(fault.since === undefined ? {} : { since: fault.since }),
-        ...(fault.anchorEntryId === undefined ? {} : { anchor_entry_id: fault.anchorEntryId }),
-        ...(fault.providerError === undefined ? {} : { provider_error: fault.providerError }),
-    };
-}
-/** Owns provider fallback and daemon-driven fault retry state for one broker session. */
-export class FaultRetry {
-    deps;
-    timer = null;
-    failedRetryRoutes = new Set();
-    notFoundModels = new Set();
-    providerFallbackInFlight = null;
-    notFoundFallbackInFlight = null;
-    // A fallback can learn its target cannot hold this session before the
-    // current turn settles. Keep that outcome here rather than on the engine's
-    // agent_end staging field: message_end precedes agent_end, which resets that
-    // field for the ordinary provider result.
-    stagedCapacityFailure = null;
-    personaGateFailed = false;
-    constructor(deps) {
-        this.deps = deps;
-    }
-    clearTimer() {
-        if (this.timer === null)
-            return;
-        clearTimeout(this.timer);
-        this.timer = null;
-    }
-    reset() {
-        this.clearTimer();
-        this.providerFallbackInFlight = null;
-        this.notFoundFallbackInFlight = null;
-        this.stagedCapacityFailure = null;
-        this.personaGateFailed = false;
-    }
-    /** A failed daemon-backed persona gate is local evidence, not a provider fault. */
-    notePersonaGateFailure() {
-        this.personaGateFailed = true;
-    }
-    state() {
-        return brokerRecovery(this.deps.nodeId);
-    }
-    async recordFault(input) {
-        return (await mutateBrokerFault(this.deps.nodeId, { op: 'record', input })).fault;
-    }
-    async clearFault(link, preserveEpisode = false) {
-        await mutateBrokerFault(this.deps.nodeId, { op: 'clear', ...(link === undefined ? {} : { link }), ...(preserveEpisode ? { preserve_episode: true } : {}) });
-    }
-    sessionId(session) {
-        const id = session.sessionManager.getSessionId?.();
-        return typeof id === 'string' && id !== '' ? id : null;
-    }
-    async clearEpisode(session) {
-        const sessionId = this.sessionId(session);
-        if (sessionId !== null)
-            await mutateBrokerProviderRetry(this.deps.nodeId, { op: 'clear', session_id: sessionId });
-    }
-    onEvent(event) {
-        this.maybeSwitchProviderForRetry(event);
-        this.maybeFallbackOnUnusableModel(event);
-    }
-    isActiveAutoFault(fault) {
-        if (fault === null)
-            return false;
-        if (fault.retry.disposition !== 'auto')
-            return false;
-        if (fault.retry.by !== 'daemon')
-            return false;
-        return fault.kind === 'connection' || fault.kind === 'rate-limit' || fault.kind === 'overloaded';
-    }
-    async publishSettledProviderFault(generation) {
-        const agentEnd = generation.stagedProviderAgentEnd;
-        const watchdogAbort = generation.stagedWatchdogAbort;
-        const refreshAbort = generation.stagedRefreshAbort;
-        const overflowFailure = generation.stagedOverflowFailure;
-        const capacityFailure = this.stagedCapacityFailure?.generation === generation ? this.stagedCapacityFailure : null;
-        const personaGateFailed = this.personaGateFailed;
-        this.stagedCapacityFailure = null;
-        this.personaGateFailed = false;
-        generation.stagedProviderAgentEnd = null;
-        generation.stagedWatchdogAbort = false;
-        generation.stagedRefreshAbort = false;
-        generation.stagedOverflowFailure = null;
-        const settledSession = generation.session;
-        // turn_end clears the ordinary marker before this settlement. A failed
-        // admitted retry therefore carries from its durable episode, while an
-        // initial provider failure still reads from the ordinary marker.
-        const initialState = await this.state();
-        const prior = initialState.fault;
-        const durableEpisode = initialState.episode;
-        const sessionId = this.sessionId(settledSession);
-        const episodeFault = this.isActiveAutoFault(prior) && prior.link === 'pi→provider'
-            ? prior
-            : durableEpisode?.state === 'admitted' &&
-                durableEpisode.session_id === sessionId &&
-                this.isActiveAutoFault(durableEpisode.fault)
-                ? durableEpisode.fault
-                : null;
-        const episode = episodeFault === null
-            ? null
-            : { since: episodeFault.since, anchorEntryId: episodeFault.anchor_entry_id, attempt: episodeFault.retry.attempt };
-        const messages = Array.isArray(agentEnd?.messages) ? agentEnd.messages : [];
-        const last = [...messages].reverse().find((message) => typeof message === 'object' && message !== null && message.role === 'assistant');
-        if (personaGateFailed) {
-            await this.clearEpisode(settledSession);
-            return;
-        }
-        if (capacityFailure !== null || overflowFailure !== null) {
-            await this.clearFault('pi→provider');
-            await this.clearEpisode(settledSession);
-            await this.recordFault({
-                link: 'pi→provider', op: 'context overflow recovery', kind: 'context-overflow', retry: { disposition: 'fatal' },
-                message: capacityFailure?.errorMessage ?? overflowFailure.errorMessage, anchor_entry_id: settledSession.sessionManager.getLeafId?.() ?? undefined,
-            });
-            return;
-        }
-        if (watchdogAbort && last?.stopReason === 'aborted') {
-            await this.recordPendingProviderFault(settledSession, {
-                link: 'pi→provider', op: 'generation turn', kind: 'connection',
-                retry: { disposition: 'auto', by: 'daemon', ...(episode?.attempt !== undefined ? { attempt: episode.attempt } : {}) },
-                message: typeof last.errorMessage === 'string' ? last.errorMessage : 'engine error (no errorMessage recorded)',
-                ...(episode !== null ? { since: episode.since } : {}),
-                anchor_entry_id: episode?.anchorEntryId ?? settledSession.sessionManager.getLeafId?.() ?? undefined,
-            });
-            return;
-        }
-        if (last?.stopReason !== 'error') {
-            if (last !== undefined)
-                await this.clearEpisode(settledSession);
-            return;
-        }
-        if (refreshAbort) {
-            await this.clearEpisode(settledSession);
-            return;
-        }
-        const raw = typeof last.errorMessage === 'string' ? last.errorMessage : 'engine error (no errorMessage recorded)';
-        // An abort landing between requests reaches pi's catch as a bare AbortError
-        // and settles as an error rather than `aborted`. Something asked that turn
-        // to stop, so there is no provider fault to record.
-        if (endedByAbort({ stopReason: 'error', errorMessage: raw })) {
-            await this.clearEpisode(settledSession);
-            return;
-        }
-        const uncompactedOverflow = isContextOverflow(last, settledSession.model?.contextWindow);
-        const classified = classify('pi→provider', last);
-        const coolingDeadline = extractCoolingDeadline(last);
-        const failure = extractProviderFailureDetails(last);
-        const provider = typeof last.provider === 'string' && last.provider !== '' ? last.provider : settledSession.model?.provider;
-        const model = typeof last.model === 'string' && last.model !== '' ? last.model : settledSession.model?.id;
-        const providerError = failure === undefined || provider === undefined || model === undefined
-            ? undefined
-            : { provider, model, ...failure };
-        if (uncompactedOverflow) {
-            await this.clearFault('pi→provider');
-            await this.clearEpisode(settledSession);
-        }
-        const faultInput = {
-            link: 'pi→provider', op: 'generation turn', kind: uncompactedOverflow ? 'context-overflow' : classified.kind,
-            retry: uncompactedOverflow
-                ? { disposition: 'fatal' }
-                : classified.disposition === 'auto'
-                    ? {
-                        disposition: 'auto', by: 'daemon',
-                        ...(coolingDeadline !== undefined ? { nextAt: new Date(coolingDeadline).toISOString() } : {}),
-                        ...(episode?.attempt !== undefined ? { attempt: episode.attempt } : {}),
-                    }
-                    : { disposition: classified.disposition },
-            message: raw,
-            ...(!uncompactedOverflow && classified.disposition === 'auto' && episode !== null ? { since: episode.since } : {}),
-            anchor_entry_id: !uncompactedOverflow && classified.disposition === 'auto' && episode?.anchorEntryId !== undefined
-                ? episode.anchorEntryId
-                : settledSession.sessionManager.getLeafId?.() ?? undefined,
-            ...(providerError === undefined ? {} : { provider_error: providerError }),
-        };
-        if (faultInput.retry.disposition === 'auto')
-            await this.recordPendingProviderFault(settledSession, faultInput);
-        else {
-            await this.clearEpisode(settledSession);
-            await this.recordFault(faultInput);
-        }
-    }
-    /** Startup re-drive reads the durable episode surviving from the process this
-     *  replaces. Revive deliberately clears the ordinary fault marker before this
-     *  boundary, so the episode is the recovery authority. A `pending` episode
-     *  re-arms unconditionally, exactly as before. An `admitted` episode re-arms
-     *  too, but only under a no-progress proof: reopened to `pending` when the
-     *  branch segment from its anchor to the current leaf carries no assistant
-     *  content and no tool activity — the dead attempt did nothing that rolling
-     *  back would destroy. An anchor absent from the loaded branch cannot happen
-     *  in production (session-file identity and the new-cycle guard already rule
-     *  it out below), so it is treated the same as "no progress" rather than
-     *  blocking the ladder. Returns 'retry-armed' when the timer now owns the
-     *  first turn, 'none' when the caller's default kickoff should proceed. */
-    async recoverOnStartup() {
-        const generation = this.deps.installedGeneration();
-        if (generation === undefined)
-            return 'none';
-        const episode = (await this.state()).episode;
-        const sessionId = this.sessionId(generation.session);
-        // A new-cycle launch deliberately roots a new transcript branch even though
-        // it reuses the same .jsonl. Its old anchor is no longer a valid recovery
-        // coordinate, so mark the episode invalid rather than inferring that from
-        // the shared path or letting startup re-drive into the fresh cycle.
-        if (this.deps.cfg.newCycle === true) {
-            if (episode !== null && sessionId !== null && episode.session_id === sessionId) {
-                await mutateBrokerProviderRetry(this.deps.nodeId, { op: 'invalidate', session_id: sessionId });
-            }
-            return 'none';
-        }
-        if (episode === null || sessionId === null || episode.session_id !== sessionId)
-            return 'none';
-        if (episode.state === 'admitted') {
-            if (episode.fault.anchor_entry_id !== undefined && !this.hasProgressPastAnchor(generation.session, episode.fault.anchor_entry_id)) {
-                if (!(await mutateBrokerProviderRetry(this.deps.nodeId, { op: 'reopen', session_id: sessionId })).applied)
-                    return 'none';
-            }
-            else {
-                // Progress detected past the anchor (or no anchor to prove against) — the
-                // dead attempt did real work. Never roll that back; clear the episode and
-                // let the caller's default kickoff continue where it actually got to.
-                await this.clearEpisode(generation.session);
-                return 'none';
-            }
-        }
-        else if (episode.state !== 'pending') {
-            return 'none';
-        }
-        await this.recordFault(faultInput(policyFault(episode.fault)));
-        await this.schedule();
-        return 'retry-armed';
-    }
-    /** True iff the branch segment strictly after `anchorEntryId` (root→leaf,
-     *  anchor exclusive) contains an assistant message with non-empty content or
-     *  a tool result — the two shapes a runtime card and a `content: []`
-     *  error-settled assistant message are not. An anchor missing from the loaded
-     *  branch cannot be distinguished from "no progress" here; callers treat it
-     *  as no progress rather than as a positive finding. */
-    hasProgressPastAnchor(session, anchorEntryId) {
-        const branch = session.sessionManager.getBranch();
-        const anchorIndex = branch.findIndex((entry) => entry.id === anchorEntryId);
-        const segment = anchorIndex === -1 ? branch : branch.slice(anchorIndex + 1);
-        return segment.some((entry) => {
-            if (entry.type !== 'message')
-                return false;
-            const { message } = entry;
-            if (message.role === 'toolResult')
-                return true;
-            return message.role === 'assistant' && Array.isArray(message.content) && message.content.length > 0;
-        });
-    }
-    async schedule() {
-        const generation = this.deps.installedGeneration();
-        const session = this.deps.currentSession();
-        if (generation === undefined || session.isStreaming || this.timer !== null)
-            return;
-        const state = await this.state();
-        const fault = state.fault;
-        // A live session without a durable session binding still owns its ordinary
-        // in-process timer; once one exists, require its pending episode so
-        // admitted/invalidated retries stay inert before replacement startup.
-        if (!this.isActiveAutoFault(fault) || (this.sessionId(session) !== null && !this.pendingEpisodeMatches(session, fault, state))) {
-            this.clearTimer();
-            return;
-        }
-        const delayMs = this.faultRetryDelayMs(fault, Date.now());
-        if (delayMs !== null)
-            this.clearAndArmFaultRetry(delayMs, generation);
-    }
-    maybeSwitchProviderForRetry(event) {
-        if (event.type !== 'auto_retry_start')
-            return;
-        const kind = classify('pi→provider', event.errorMessage ?? '').kind;
-        if ((kind !== 'rate-limit' && kind !== 'connection') || this.providerFallbackInFlight !== null || this.deps.cfg.modelExact === true)
-            return;
-        const generation = this.deps.installedGeneration();
-        if (generation === undefined)
-            return;
-        const candidateSession = this.deps.currentSession();
-        const candidateServices = this.deps.currentServices();
-        const currentSpec = this.deps.formatModelSpec(candidateSession.model, candidateSession.thinkingLevel);
-        if (currentSpec === null)
-            return;
-        const routeRequest = modelRequestFromConfig(this.deps.cfg.model, envModelIntent(), false);
-        if (routeRequest === undefined)
-            return;
-        const routes = expandModelCandidates(routeRequest, envNodeCwd() ?? this.deps.cfg.cwd, envProfileId());
-        const current = parseModelSpec(currentSpec);
-        const liveRank = routes.find((candidate) => `${candidate.providerId}/${candidate.modelId}` === current.modelSpec && candidate.thinkingLevel === current.thinkingLevel)?.rank;
-        const fallback = liveRank === undefined
-            ? { route: undefined, capacityExhausted: false }
-            : this.nextEligibleFallbackRoute(routes.slice(liveRank + 1), this.deps.registryOf(candidateServices), candidateSession.getContextUsage()?.tokens, (candidate) => !this.failedRetryRoutes.has(candidate.routeId));
-        if (fallback.route === undefined) {
-            if (fallback.capacityExhausted) {
-                this.stageCapacityFailure(generation, candidateSession);
-                // Pi creates its retry AbortController immediately after it emits this
-                // event. Defer the abort so it cancels that controller, then lets the
-                // normal agent_settled path publish the staged fatal outcome.
-                queueMicrotask(() => {
-                    if (this.deps.installedGeneration() === generation && this.deps.currentSession() === candidateSession)
-                        candidateSession.abortRetry();
-                });
-            }
-            return;
-        }
-        const nextRoute = fallback.route;
-        const fallbackPromise = (async () => {
-            const target = this.deps.registryOf(candidateServices).find(nextRoute.providerId, nextRoute.modelId);
-            if (!target)
-                return;
-            await candidateSession.setModel(target);
-            if (this.deps.installedGeneration() !== generation || this.deps.currentSession() !== candidateSession)
-                return;
-            candidateSession.setThinkingLevel((nextRoute.thinkingLevel ?? 'off'));
-            this.failedRetryRoutes.add(nextRoute.routeId);
-            await this.deps.broadcastModelChanged(false);
-        })().catch((err) => {
-            emitEvent({ level: 'debug', event: 'broker.runtime.diagnostic', fields: { message: `provider route fallback failed: ${err instanceof Error ? err.message : String(err)}` } });
-        }).finally(() => {
-            if (this.deps.installedGeneration() === generation)
-                this.providerFallbackInFlight = null;
-        });
-        this.providerFallbackInFlight = fallbackPromise;
-    }
-    maybeFallbackOnUnusableModel(event) {
-        if (event.type !== 'message_end' || event.message.role !== 'assistant' || event.message.stopReason !== 'error')
-            return;
-        const unconfigured = isProviderUnconfiguredError(event.message.errorMessage ?? '');
-        if ((!unconfigured && !isModelNotFoundError(event.message.errorMessage ?? '')) || this.deps.cfg.modelExact === true || this.notFoundFallbackInFlight !== null)
-            return;
-        const generation = this.deps.installedGeneration();
-        if (generation === undefined)
-            return;
-        const candidateSession = this.deps.currentSession();
-        const candidateServices = this.deps.currentServices();
-        const currentSpec = this.deps.formatModelSpec(candidateSession.model, candidateSession.thinkingLevel);
-        if (currentSpec === null)
-            return;
-        const current = parseModelSpec(currentSpec);
-        this.notFoundModels.add(current.modelSpec);
-        const routeRequest = modelRequestFromConfig(this.deps.cfg.model, envModelIntent(), false);
-        const routes = routeRequest ? expandModelCandidates(routeRequest, envNodeCwd() ?? this.deps.cfg.cwd, envProfileId()) : [];
-        const liveRank = routes.find((candidate) => `${candidate.providerId}/${candidate.modelId}` === current.modelSpec && candidate.thinkingLevel === current.thinkingLevel)?.rank;
-        const fallback = liveRank === undefined
-            ? { route: undefined, capacityExhausted: false }
-            : this.nextEligibleFallbackRoute(routes.slice(liveRank + 1), this.deps.registryOf(candidateServices), candidateSession.getContextUsage()?.tokens, (candidate) => !this.notFoundModels.has(`${candidate.providerId}/${candidate.modelId}`));
-        if (fallback.route === undefined) {
-            if (fallback.capacityExhausted)
-                this.stageCapacityFailure(generation, candidateSession);
-            return;
-        }
-        const nextRoute = fallback.route;
-        const target = `${nextRoute.providerId}/${nextRoute.modelId}${nextRoute.thinkingLevel ? `:${nextRoute.thinkingLevel}` : ''}`;
-        const fallbackPromise = (async () => {
-            const model = this.deps.registryOf(candidateServices).find(nextRoute.providerId, nextRoute.modelId);
-            if (model === undefined) {
-                emitEvent({ level: 'debug', event: 'broker.runtime.diagnostic', fields: { message: `${unconfigured ? 'unconfigured-provider' : 'not_found'} fallback skipped: ${target} is not registered` } });
-                return;
-            }
-            await candidateSession.setModel(model);
-            if (this.deps.installedGeneration() !== generation || this.deps.currentSession() !== candidateSession)
-                return;
-            candidateSession.setThinkingLevel((nextRoute.thinkingLevel ?? 'off'));
-            await this.deps.broadcastModelChanged();
-            const resolved = this.deps.formatModelSpec(candidateSession.model, candidateSession.thinkingLevel) ?? target;
-            emitEvent({ level: 'debug', event: 'broker.runtime.diagnostic', fields: { message: `${unconfigured ? 'unconfigured-provider' : 'not_found'} fallback: ${currentSpec} unavailable → ${resolved}; re-driving turn` } });
-            await candidateSession.agent.waitForIdle();
-            if (this.deps.installedGeneration() !== generation || this.deps.currentSession() !== candidateSession)
-                return;
-            if (unconfigured)
-                await this.clearFault('pi→provider');
-            const fallbackCard = formatCard('recovery', {}, formatModelFallbackRecovery(currentSpec, resolved, unconfigured ? 'credential' : 'not-found'));
-            announceMessageOrigin(fallbackCard, { type: 'runtime', kind: 'recovery' });
-            await this.deps.turnAdmission.admit((release) => operationIdContext.fresh(() => candidateSession.prompt(fallbackCard, { preflightResult: release })));
-        })().catch((err) => {
-            emitEvent({ level: 'debug', event: 'broker.runtime.diagnostic', fields: { message: `${unconfigured ? 'unconfigured-provider' : 'not_found'} fallback failed: ${err instanceof Error ? err.message : String(err)}` } });
-        }).finally(() => {
-            if (this.deps.installedGeneration() === generation)
-                this.notFoundFallbackInFlight = null;
-        });
-        this.notFoundFallbackInFlight = fallbackPromise;
-    }
-    nextEligibleFallbackRoute(routes, registry, contextTokens, include) {
-        let available = 0;
-        let tooSmall = 0;
-        for (const candidate of routes) {
-            if (!include(candidate))
-                continue;
-            if (resolveProviderCandidates([{ ...candidate, ...probeRouteAvailability(candidate, registry), automaticFallback: true }]).candidates.length === 0)
-                continue;
-            const target = registry.find(candidate.providerId, candidate.modelId);
-            if (target === undefined)
-                continue;
-            available++;
-            if (this.targetCannotFitContext(target.contextWindow, contextTokens)) {
-                tooSmall++;
-                continue;
-            }
-            return { route: candidate, capacityExhausted: false };
-        }
-        return { route: undefined, capacityExhausted: available > 0 && available === tooSmall };
-    }
-    targetCannotFitContext(contextWindow, contextTokens) {
-        return Number.isFinite(contextWindow) && Number.isFinite(contextTokens) && contextWindow < contextTokens;
-    }
-    stageCapacityFailure(generation, session) {
-        const tokens = session.getContextUsage()?.tokens;
-        this.stagedCapacityFailure = {
-            generation,
-            errorMessage: Number.isFinite(tokens)
-                ? `All available fallback models have context windows smaller than the current conversation (${tokens} tokens).`
-                : 'All available fallback models have context windows smaller than the current conversation.',
-        };
-    }
-    async recordPendingProviderFault(session, input) {
-        const sessionId = this.sessionId(session);
-        // A session without a daemon binding cannot crash-recover, but its live
-        // broker still records its ordinary auto-retry fault.
-        if (sessionId === null) {
-            await this.recordFault(input);
-            return;
-        }
-        await mutateBrokerProviderRetry(this.deps.nodeId, { op: 'record-pending', session_id: sessionId, input });
-    }
-    pendingEpisodeMatches(session, fault, state) {
-        const sessionId = this.sessionId(session);
-        const episode = state.episode;
-        return sessionId !== null &&
-            episode !== null &&
-            episode.state === 'pending' &&
-            episode.session_id === sessionId &&
-            episode.fault.fault_id === fault.fault_id;
-    }
-    faultRetryDelayMs(fault, now) {
-        if (fault.retry.nextAt !== undefined) {
-            const ms = Date.parse(fault.retry.nextAt) - now;
-            return Number.isFinite(ms) ? Math.max(0, ms) : null;
-        }
-        return nextFaultRetry(policyFault(fault), now)?.delayMs ?? null;
-    }
-    clearAndArmFaultRetry(delayMs, generation) {
-        if (this.deps.installedGeneration() !== generation)
-            return;
-        this.clearTimer();
-        const chunkMs = Math.min(Math.max(0, delayMs), MAX_TIMEOUT_MS);
-        const remainingMs = Math.max(0, delayMs - chunkMs);
-        this.timer = setTimeout(() => {
-            this.timer = null;
-            if (this.deps.installedGeneration() !== generation)
-                return;
-            if (remainingMs > 0) {
-                this.clearAndArmFaultRetry(remainingMs, generation);
-                return;
-            }
-            void this.runFaultRetry().catch((err) => {
-                emitEvent({ level: 'debug', event: 'broker.runtime.diagnostic', fields: { message: `fault retry fire failed: ${err instanceof Error ? err.message : String(err)}` } });
-            });
-        }, chunkMs);
-        if (typeof this.timer.unref === 'function')
-            this.timer.unref();
-    }
-    async runFaultRetry() {
-        const generation = this.deps.installedGeneration();
-        if (generation === undefined)
-            return;
-        const candidateSession = generation.session;
-        if (candidateSession.isStreaming)
-            return;
-        const initial = (await this.state()).fault;
-        if (!this.isActiveAutoFault(initial))
-            return;
-        let fault = initial;
-        if (initial.kind === 'connection') {
-            const online = await this.deps.retryProbe();
-            if (this.deps.installedGeneration() !== generation || this.deps.currentSession() !== candidateSession || candidateSession.isStreaming)
-                return;
-            const fresh = (await this.state()).fault;
-            if (!this.isActiveAutoFault(fresh) || fresh.since !== initial.since)
-                return;
-            if (!online) {
-                this.clearAndArmFaultRetry(policyFor(fresh.kind).baseDelayMs, generation);
-                return;
-            }
-            fault = fresh;
-        }
-        const attempt = fault.retry.attempt ?? 0;
-        const policy = policyFor(fault.kind);
-        if (attempt >= policy.max) {
-            await this.recordFault(faultInput(markFatalOnExhaust(policyFault(fault))));
-            await this.clearEpisode(candidateSession);
-            return;
-        }
-        const plan = nextFaultRetry(policyFault(fault), Date.now());
-        if (plan === null) {
-            await this.recordFault(faultInput(markFatalOnExhaust(policyFault(fault))));
-            await this.clearEpisode(candidateSession);
-            return;
-        }
-        if (fault.anchor_entry_id !== undefined) {
-            try {
-                if ((candidateSession.sessionManager.getLeafId?.() ?? null) !== fault.anchor_entry_id) {
-                    await candidateSession.navigateTree(fault.anchor_entry_id);
-                    if (this.deps.installedGeneration() !== generation || this.deps.currentSession() !== candidateSession)
-                        return;
-                }
-            }
-            catch (err) {
-                emitEvent({ level: 'debug', event: 'broker.runtime.diagnostic', fields: { message: `fault retry rewind skipped: ${err instanceof Error ? err.message : String(err)}` } });
-            }
-        }
-        if (this.deps.installedGeneration() !== generation || this.deps.currentSession() !== candidateSession)
-            return;
-        await this.deps.turnAdmission.admit(async (release) => {
-            const current = (await this.state()).fault;
-            if (this.deps.installedGeneration() !== generation ||
-                this.deps.currentSession() !== candidateSession ||
-                candidateSession.isStreaming ||
-                !this.isActiveAutoFault(current) ||
-                current.since !== fault.since) {
-                release();
-                return;
-            }
-            // This is the admission point: all generation/session/streaming/fault
-            // revalidation above has passed, and only now may the durable episode
-            // become admitted. A rejected or failed mutation releases without
-            // calling prompt, so stale brokers never run a recovery turn.
-            const sessionId = this.sessionId(candidateSession);
-            if (sessionId === null) {
-                release();
-                return;
-            }
-            const admitted = await mutateBrokerProviderRetry(this.deps.nodeId, {
-                op: 'admit',
-                session_id: sessionId,
-                fault_id: current.fault_id,
-                retry: plan.fault.retry,
-            });
-            if (!admitted.applied) {
-                release();
-                return;
-            }
-            // The attempt facts are what makes a rolled-back retry legible: every
-            // superseded attempt falls off the active branch, so this one card is the
-            // whole visible record of the sequence, and without them a reader cannot
-            // tell the third attempt from the first.
-            const retryCard = formatCard('recovery', { attempt: plan.attempt, max: plan.max }, fault.kind === 'connection' ? CONNECTION_FAULT_RECOVERY_BODY : PROVIDER_FAULT_RECOVERY_BODY);
-            announceMessageOrigin(retryCard, { type: 'runtime', kind: 'recovery' });
-            return operationIdContext.fresh(() => candidateSession.prompt(retryCard, { preflightResult: release }));
-        });
-    }
-}
+var j=Object.defineProperty;var w=(s,e)=>j(s,"name",{value:e,configurable:!0});import{envModelIntent as P,envNodeCwd as G,envProfileId as O}from"../../../shared/env.js";import{isContextOverflow as X}from"@earendil-works/pi-ai";import{classify as $}from"../../fault-classifier.js";import{isModelNotFoundError as z,isProviderUnconfiguredError as H}from"../fault.js";import{brokerRecovery as J,mutateBrokerFault as L,mutateBrokerProviderRetry as b}from"./daemon-ops.js";import{extractCoolingDeadline as K}from"../managed-provider-cooling.js";import{extractProviderFailureDetails as Q}from"../../provider-failure.js";import{expandModelCandidates as D,modelRequestFromConfig as T,probeRouteAvailability as Z}from"../../model-routes.js";import{operationIdContext as N}from"../../events/operation-id.js";import{emitEvent as v}from"../../events/emit.js";import{CONNECTION_FAULT_RECOVERY_BODY as ee,PROVIDER_FAULT_RECOVERY_BODY as te,formatCard as U,formatModelFallbackRecovery as ie}from"../../../shared/generated-context.js";import{markFatalOnExhaust as B,nextFaultRetry as q,policyFor as Y}from"../fault-recovery.js";import{parseModelSpec as V}from"../model-selection.js";import{endedByAbort as ne}from"../session-cycles.js";import{resolveProviderCandidates as re}from"../../subscription-state.js";import{announceMessageOrigin as W}from"../stamp/channel.js";const se=2147483647;function I(s){return{link:s.link,op:s.op,kind:s.kind,retry:s.retry,message:s.message,since:s.since,operation_id:s.operation_id,...s.anchor_entry_id===void 0?{}:{anchorEntryId:s.anchor_entry_id},...s.provider_error===void 0?{}:{providerError:s.provider_error}}}w(I,"policyFault");function M(s){return{link:s.link,op:s.op,kind:s.kind,retry:s.retry,message:s.message,...s.operation_id===void 0?{}:{operation_id:s.operation_id},...s.since===void 0?{}:{since:s.since},...s.anchorEntryId===void 0?{}:{anchor_entry_id:s.anchorEntryId},...s.providerError===void 0?{}:{provider_error:s.providerError}}}w(M,"faultInput");class we{static{w(this,"FaultRetry")}deps;timer=null;failedRetryRoutes=new Set;notFoundModels=new Set;providerFallbackInFlight=null;notFoundFallbackInFlight=null;stagedCapacityFailure=null;personaGateFailed=!1;constructor(e){this.deps=e}clearTimer(){this.timer!==null&&(clearTimeout(this.timer),this.timer=null)}reset(){this.clearTimer(),this.providerFallbackInFlight=null,this.notFoundFallbackInFlight=null,this.stagedCapacityFailure=null,this.personaGateFailed=!1}notePersonaGateFailure(){this.personaGateFailed=!0}state(){return J(this.deps.nodeId)}async recordFault(e){return(await L(this.deps.nodeId,{op:"record",input:e})).fault}async clearFault(e,t=!1){await L(this.deps.nodeId,{op:"clear",...e===void 0?{}:{link:e},...t?{preserve_episode:!0}:{}})}sessionId(e){const t=e.sessionManager.getSessionId?.();return typeof t=="string"&&t!==""?t:null}async clearEpisode(e){const t=this.sessionId(e);t!==null&&await b(this.deps.nodeId,{op:"clear",session_id:t})}onEvent(e){this.maybeSwitchProviderForRetry(e),this.maybeFallbackOnUnusableModel(e)}isActiveAutoFault(e){return e===null||e.retry.disposition!=="auto"||e.retry.by!=="daemon"?!1:e.kind==="connection"||e.kind==="rate-limit"||e.kind==="overloaded"}async publishSettledProviderFault(e){const t=e.stagedProviderAgentEnd,n=e.stagedWatchdogAbort,i=e.stagedRefreshAbort,o=e.stagedOverflowFailure,l=this.stagedCapacityFailure?.generation===e?this.stagedCapacityFailure:null,d=this.personaGateFailed;this.stagedCapacityFailure=null,this.personaGateFailed=!1,e.stagedProviderAgentEnd=null,e.stagedWatchdogAbort=!1,e.stagedRefreshAbort=!1,e.stagedOverflowFailure=null;const r=e.session,u=await this.state(),m=u.fault,h=u.episode,p=this.sessionId(r),g=this.isActiveAutoFault(m)&&m.link==="pi\u2192provider"?m:h?.state==="admitted"&&h.session_id===p&&this.isActiveAutoFault(h.fault)?h.fault:null,a=g===null?null:{since:g.since,anchorEntryId:g.anchor_entry_id,attempt:g.retry.attempt},c=[...Array.isArray(t?.messages)?t.messages:[]].reverse().find(E=>typeof E=="object"&&E!==null&&E.role==="assistant");if(d){await this.clearEpisode(r);return}if(l!==null||o!==null){await this.clearFault("pi\u2192provider"),await this.clearEpisode(r),await this.recordFault({link:"pi\u2192provider",op:"context overflow recovery",kind:"context-overflow",retry:{disposition:"fatal"},message:l?.errorMessage??o.errorMessage,anchor_entry_id:r.sessionManager.getLeafId?.()??void 0});return}if(n&&c?.stopReason==="aborted"){await this.recordPendingProviderFault(r,{link:"pi\u2192provider",op:"generation turn",kind:"connection",retry:{disposition:"auto",by:"daemon",...a?.attempt!==void 0?{attempt:a.attempt}:{}},message:typeof c.errorMessage=="string"?c.errorMessage:"engine error (no errorMessage recorded)",...a!==null?{since:a.since}:{},anchor_entry_id:a?.anchorEntryId??r.sessionManager.getLeafId?.()??void 0});return}if(c?.stopReason!=="error"){c!==void 0&&await this.clearEpisode(r);return}if(i){await this.clearEpisode(r);return}const F=typeof c.errorMessage=="string"?c.errorMessage:"engine error (no errorMessage recorded)";if(ne({stopReason:"error",errorMessage:F})){await this.clearEpisode(r);return}const y=X(c,r.model?.contextWindow),k=$("pi\u2192provider",c),A=K(c),R=Q(c),S=typeof c.provider=="string"&&c.provider!==""?c.provider:r.model?.provider,x=typeof c.model=="string"&&c.model!==""?c.model:r.model?.id,C=R===void 0||S===void 0||x===void 0?void 0:{provider:S,model:x,...R};y&&(await this.clearFault("pi\u2192provider"),await this.clearEpisode(r));const _={link:"pi\u2192provider",op:"generation turn",kind:y?"context-overflow":k.kind,retry:y?{disposition:"fatal"}:k.disposition==="auto"?{disposition:"auto",by:"daemon",...A!==void 0?{nextAt:new Date(A).toISOString()}:{},...a?.attempt!==void 0?{attempt:a.attempt}:{}}:{disposition:k.disposition},message:F,...!y&&k.disposition==="auto"&&a!==null?{since:a.since}:{},anchor_entry_id:!y&&k.disposition==="auto"&&a?.anchorEntryId!==void 0?a.anchorEntryId:r.sessionManager.getLeafId?.()??void 0,...C===void 0?{}:{provider_error:C}};_.retry.disposition==="auto"?await this.recordPendingProviderFault(r,_):(await this.clearEpisode(r),await this.recordFault(_))}async recoverOnStartup(){const e=this.deps.installedGeneration();if(e===void 0)return"none";const t=(await this.state()).episode,n=this.sessionId(e.session);if(this.deps.cfg.newCycle===!0)return t!==null&&n!==null&&t.session_id===n&&await b(this.deps.nodeId,{op:"invalidate",session_id:n}),"none";if(t===null||n===null||t.session_id!==n)return"none";if(t.state==="admitted")if(t.fault.anchor_entry_id!==void 0&&!this.hasProgressPastAnchor(e.session,t.fault.anchor_entry_id)){if(!(await b(this.deps.nodeId,{op:"reopen",session_id:n})).applied)return"none"}else return await this.clearEpisode(e.session),"none";else if(t.state!=="pending")return"none";return await this.recordFault(M(I(t.fault))),await this.schedule(),"retry-armed"}hasProgressPastAnchor(e,t){const n=e.sessionManager.getBranch(),i=n.findIndex(l=>l.id===t);return(i===-1?n:n.slice(i+1)).some(l=>{if(l.type!=="message")return!1;const{message:d}=l;return d.role==="toolResult"?!0:d.role==="assistant"&&Array.isArray(d.content)&&d.content.length>0})}async schedule(){const e=this.deps.installedGeneration(),t=this.deps.currentSession();if(e===void 0||t.isStreaming||this.timer!==null)return;const n=await this.state(),i=n.fault;if(!this.isActiveAutoFault(i)||this.sessionId(t)!==null&&!this.pendingEpisodeMatches(t,i,n)){this.clearTimer();return}const o=this.faultRetryDelayMs(i,Date.now());o!==null&&this.clearAndArmFaultRetry(o,e)}maybeSwitchProviderForRetry(e){if(e.type!=="auto_retry_start")return;const t=$("pi\u2192provider",e.errorMessage??"").kind;if(t!=="rate-limit"&&t!=="connection"||this.providerFallbackInFlight!==null||this.deps.cfg.modelExact===!0)return;const n=this.deps.installedGeneration();if(n===void 0)return;const i=this.deps.currentSession(),o=this.deps.currentServices(),l=this.deps.formatModelSpec(i.model,i.thinkingLevel);if(l===null)return;const d=T(this.deps.cfg.model,P(),!1);if(d===void 0)return;const r=D(d,G()??this.deps.cfg.cwd,O()),u=V(l),m=r.find(a=>`${a.providerId}/${a.modelId}`===u.modelSpec&&a.thinkingLevel===u.thinkingLevel)?.rank,h=m===void 0?{route:void 0,capacityExhausted:!1}:this.nextEligibleFallbackRoute(r.slice(m+1),this.deps.registryOf(o),i.getContextUsage()?.tokens,a=>!this.failedRetryRoutes.has(a.routeId));if(h.route===void 0){h.capacityExhausted&&(this.stageCapacityFailure(n,i),queueMicrotask(()=>{this.deps.installedGeneration()===n&&this.deps.currentSession()===i&&i.abortRetry()}));return}const p=h.route,g=(async()=>{const a=this.deps.registryOf(o).find(p.providerId,p.modelId);a&&(await i.setModel(a),!(this.deps.installedGeneration()!==n||this.deps.currentSession()!==i)&&(i.setThinkingLevel(p.thinkingLevel??"off"),this.failedRetryRoutes.add(p.routeId),await this.deps.broadcastModelChanged(!1)))})().catch(a=>{v({level:"debug",event:"broker.runtime.diagnostic",fields:{message:`provider route fallback failed: ${a instanceof Error?a.message:String(a)}`}})}).finally(()=>{this.deps.installedGeneration()===n&&(this.providerFallbackInFlight=null)});this.providerFallbackInFlight=g}maybeFallbackOnUnusableModel(e){if(e.type!=="message_end"||e.message.role!=="assistant"||e.message.stopReason!=="error")return;const t=H(e.message.errorMessage??"");if(!t&&!z(e.message.errorMessage??"")||this.deps.cfg.modelExact===!0||this.notFoundFallbackInFlight!==null)return;const n=this.deps.installedGeneration();if(n===void 0)return;const i=this.deps.currentSession(),o=this.deps.currentServices(),l=this.deps.formatModelSpec(i.model,i.thinkingLevel);if(l===null)return;const d=V(l);this.notFoundModels.add(d.modelSpec);const r=T(this.deps.cfg.model,P(),!1),u=r?D(r,G()??this.deps.cfg.cwd,O()):[],m=u.find(f=>`${f.providerId}/${f.modelId}`===d.modelSpec&&f.thinkingLevel===d.thinkingLevel)?.rank,h=m===void 0?{route:void 0,capacityExhausted:!1}:this.nextEligibleFallbackRoute(u.slice(m+1),this.deps.registryOf(o),i.getContextUsage()?.tokens,f=>!this.notFoundModels.has(`${f.providerId}/${f.modelId}`));if(h.route===void 0){h.capacityExhausted&&this.stageCapacityFailure(n,i);return}const p=h.route,g=`${p.providerId}/${p.modelId}${p.thinkingLevel?`:${p.thinkingLevel}`:""}`,a=(async()=>{const f=this.deps.registryOf(o).find(p.providerId,p.modelId);if(f===void 0){v({level:"debug",event:"broker.runtime.diagnostic",fields:{message:`${t?"unconfigured-provider":"not_found"} fallback skipped: ${g} is not registered`}});return}if(await i.setModel(f),this.deps.installedGeneration()!==n||this.deps.currentSession()!==i)return;i.setThinkingLevel(p.thinkingLevel??"off"),await this.deps.broadcastModelChanged();const c=this.deps.formatModelSpec(i.model,i.thinkingLevel)??g;if(v({level:"debug",event:"broker.runtime.diagnostic",fields:{message:`${t?"unconfigured-provider":"not_found"} fallback: ${l} unavailable \u2192 ${c}; re-driving turn`}}),await i.agent.waitForIdle(),this.deps.installedGeneration()!==n||this.deps.currentSession()!==i)return;t&&await this.clearFault("pi\u2192provider");const F=U("recovery",{},ie(l,c,t?"credential":"not-found"));W(F,{type:"runtime",kind:"recovery"}),await this.deps.turnAdmission.admit(y=>N.fresh(()=>i.prompt(F,{preflightResult:y})))})().catch(f=>{v({level:"debug",event:"broker.runtime.diagnostic",fields:{message:`${t?"unconfigured-provider":"not_found"} fallback failed: ${f instanceof Error?f.message:String(f)}`}})}).finally(()=>{this.deps.installedGeneration()===n&&(this.notFoundFallbackInFlight=null)});this.notFoundFallbackInFlight=a}nextEligibleFallbackRoute(e,t,n,i){let o=0,l=0;for(const d of e){if(!i(d)||re([{...d,...Z(d,t),automaticFallback:!0}]).candidates.length===0)continue;const r=t.find(d.providerId,d.modelId);if(r!==void 0){if(o++,this.targetCannotFitContext(r.contextWindow,n)){l++;continue}return{route:d,capacityExhausted:!1}}}return{route:void 0,capacityExhausted:o>0&&o===l}}targetCannotFitContext(e,t){return Number.isFinite(e)&&Number.isFinite(t)&&e<t}stageCapacityFailure(e,t){const n=t.getContextUsage()?.tokens;this.stagedCapacityFailure={generation:e,errorMessage:Number.isFinite(n)?`All available fallback models have context windows smaller than the current conversation (${n} tokens).`:"All available fallback models have context windows smaller than the current conversation."}}async recordPendingProviderFault(e,t){const n=this.sessionId(e);if(n===null){await this.recordFault(t);return}await b(this.deps.nodeId,{op:"record-pending",session_id:n,input:t})}pendingEpisodeMatches(e,t,n){const i=this.sessionId(e),o=n.episode;return i!==null&&o!==null&&o.state==="pending"&&o.session_id===i&&o.fault.fault_id===t.fault_id}faultRetryDelayMs(e,t){if(e.retry.nextAt!==void 0){const n=Date.parse(e.retry.nextAt)-t;return Number.isFinite(n)?Math.max(0,n):null}return q(I(e),t)?.delayMs??null}clearAndArmFaultRetry(e,t){if(this.deps.installedGeneration()!==t)return;this.clearTimer();const n=Math.min(Math.max(0,e),se),i=Math.max(0,e-n);this.timer=setTimeout(()=>{if(this.timer=null,this.deps.installedGeneration()===t){if(i>0){this.clearAndArmFaultRetry(i,t);return}this.runFaultRetry().catch(o=>{v({level:"debug",event:"broker.runtime.diagnostic",fields:{message:`fault retry fire failed: ${o instanceof Error?o.message:String(o)}`}})})}},n),typeof this.timer.unref=="function"&&this.timer.unref()}async runFaultRetry(){const e=this.deps.installedGeneration();if(e===void 0)return;const t=e.session;if(t.isStreaming)return;const n=(await this.state()).fault;if(!this.isActiveAutoFault(n))return;let i=n;if(n.kind==="connection"){const r=await this.deps.retryProbe();if(this.deps.installedGeneration()!==e||this.deps.currentSession()!==t||t.isStreaming)return;const u=(await this.state()).fault;if(!this.isActiveAutoFault(u)||u.since!==n.since)return;if(!r){this.clearAndArmFaultRetry(Y(u.kind).baseDelayMs,e);return}i=u}const o=i.retry.attempt??0,l=Y(i.kind);if(o>=l.max){await this.recordFault(M(B(I(i)))),await this.clearEpisode(t);return}const d=q(I(i),Date.now());if(d===null){await this.recordFault(M(B(I(i)))),await this.clearEpisode(t);return}if(i.anchor_entry_id!==void 0)try{if((t.sessionManager.getLeafId?.()??null)!==i.anchor_entry_id&&(await t.navigateTree(i.anchor_entry_id),this.deps.installedGeneration()!==e||this.deps.currentSession()!==t))return}catch(r){v({level:"debug",event:"broker.runtime.diagnostic",fields:{message:`fault retry rewind skipped: ${r instanceof Error?r.message:String(r)}`}})}this.deps.installedGeneration()!==e||this.deps.currentSession()!==t||await this.deps.turnAdmission.admit(async r=>{const u=(await this.state()).fault;if(this.deps.installedGeneration()!==e||this.deps.currentSession()!==t||t.isStreaming||!this.isActiveAutoFault(u)||u.since!==i.since){r();return}const m=this.sessionId(t);if(m===null){r();return}if(!(await b(this.deps.nodeId,{op:"admit",session_id:m,fault_id:u.fault_id,retry:d.fault.retry})).applied){r();return}const p=U("recovery",{attempt:d.attempt,max:d.max},i.kind==="connection"?ee:te);return W(p,{type:"runtime",kind:"recovery"}),N.fresh(()=>t.prompt(p,{preflightResult:r}))})}}export{we as FaultRetry};