@north-light/crouter 0.3.170 → 0.3.172

package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js

@@ -0,0 +1,920 @@
+import { spawn } from "node:child_process";
+import { createAssistantMessageEventStream, } from "@earendil-works/pi-ai";
+import { streamSimpleAnthropic, streamSimpleOpenAICodexResponses } from "@earendil-works/pi-ai/compat";
+import { ANTHROPIC_PROVIDER_ID, OPENAI_CODEX_PROVIDER_ID, INVALID_REFRESH_TOKEN_BACKOFF_MS, formatStatusLine, getProviderLabel, isManagedProvider, isInvalidRefreshTokenError, addSubscription, enrichManagedLoginIdentity, markSubscriptionAuthenticationFailed, markSubscriptionRateLimited, markSubscriptionSuccess, parseRetryAfterHeader, promoteSubscription, readRotationConfig, readSubscriptionPool, refreshSubscriptionCredential, removeManagedAccount, resolveProviderCandidates, upsertSubscriptionWithUniqueAccount, } from "../../../core/subscription-state.js";
+import { readMergedLaunchConfig } from "../../../core/config.js";
+import { expandModelCandidates, modelRequestFromConfig, probeRouteAvailability } from "../../../core/model-routes.js";
+import { ManagedProviderCoolingError } from "../../../core/runtime/managed-provider-cooling.js";
+// A response without Retry-After proves only that THIS attempt was throttled; it
+// does not authorize crouter to invent a five-minute provider-wide outage. Keep
+// a short turn-local probe backoff to avoid an immediate retry loop, then let a
+// fresh turn prove whether the credential is usable again. Real
+// server-provided Retry-After deadlines are still honored exactly.
+const DEFAULT_RATE_LIMIT_BACKOFF_MS = 15 * 1000;
+// Short in-place retry budget for a transient network/5xx blip on the same credential.
+// Once this budget is exhausted, surface the error and rotate to another candidate.
+const TRANSIENT_RETRY_BACKOFFS_MS = [250, 750];
+// After the in-place transient retry budget is exhausted, rotate instead of failing: the
+// credential itself is probably healthy (this is a provider-side blip like "overloaded"),
+// so cool it down only briefly -- long enough to rotate to another credential / fallback
+// provider, short enough that it comes back quickly if it was the only option.
+const TRANSIENT_ROTATE_COOLDOWN_MS = 60 * 1000;
+// A few short outer retries absorb a normal multi-broker collision; if the metadata file
+// remains busy, the provider call still proceeds. Subscription bookkeeping must never turn a
+// healthy model response into a fatal generation fault.
+const SUBSCRIPTION_LOCK_RETRY_BACKOFFS_MS = [25, 75, 150];
+function createProviderRotationState(options) {
+    return { delegatedProviderStreams: new Map(), ...(options === undefined ? {} : { options: { ...options } }) };
+}
+function configuredValue(state, key, envName) {
+    return state.options === undefined ? process.env[envName] : state.options[key];
+}
+const TRANSIENT_MESSAGE_PATTERN = /connection|network|fetch failed|socket|econnreset|econnrefused|enotfound|etimedout|timed? out|timeout|overloaded|service.?unavailable|temporarily unavailable/i;
+const RATE_LIMIT_MESSAGE_PATTERN = /\b429\b|rate.?limit|too many requests/i;
+// Precise terminal usage-limit signals (as opposed to the bare `usage`/`quota` substrings
+// dropped above): pi-ai's Codex adapter surfaces `usage_limit_reached` / `usage_not_included`
+// as the friendly "You have hit your ChatGPT usage limit..." message, often on a non-429
+// status, so the status-code check above misses it. These are specific enough not to false-
+// positive on ordinary assistant text mentioning "usage".
+const USAGE_LIMIT_MESSAGE_PATTERN = /usage[ _]limit|usage_not_included|usagelimiterror|insufficient_quota|quota exceeded/i;
+function classifyProviderError(error, observed) {
+    const value = error;
+    const errorStatus = typeof value?.status === "number" ? value.status : undefined;
+    const status = observed.status ?? errorStatus;
+    const message = [value?.errorMessage, value?.message, error instanceof Error ? error.message : ""]
+        .filter((part) => typeof part === "string" && part.length > 0)
+        .join(" ");
+    const hadRetryAfterHeader = observed.retryAfterMs !== undefined;
+    if (status === 429)
+        return { kind: "rate_limit", status, reason: "http 429", hadRetryAfterHeader };
+    if (status === 503 && hadRetryAfterHeader)
+        return { kind: "rate_limit", status, reason: "http 503 with retry-after", hadRetryAfterHeader };
+    if (status === 500 || status === 502 || status === 503 || status === 504) {
+        return { kind: "transient", status, reason: `http ${status}`, hadRetryAfterHeader };
+    }
+    const transientMatch = message.match(TRANSIENT_MESSAGE_PATTERN);
+    if (transientMatch)
+        return { kind: "transient", status, reason: transientMatch[0].toLowerCase(), hadRetryAfterHeader };
+    const rateLimitMatch = message.match(RATE_LIMIT_MESSAGE_PATTERN);
+    if (rateLimitMatch)
+        return { kind: "rate_limit", status, reason: rateLimitMatch[0].toLowerCase(), hadRetryAfterHeader };
+    const usageLimitMatch = message.match(USAGE_LIMIT_MESSAGE_PATTERN);
+    if (usageLimitMatch)
+        return { kind: "rate_limit", status, reason: usageLimitMatch[0].toLowerCase(), hadRetryAfterHeader };
+    return { kind: "fatal", status, reason: "unclassified", hadRetryAfterHeader };
+}
+function providerErrorMessage(error) {
+    const value = error;
+    const message = typeof value?.errorMessage === "string" && value.errorMessage.trim()
+        ? value.errorMessage.trim()
+        : typeof value?.message === "string" && value.message.trim()
+            ? value.message.trim()
+            : error instanceof Error && error.message.trim()
+                ? error.message.trim()
+                : undefined;
+    return message;
+}
+async function defaultSleep(ms) {
+    await new Promise((resolve) => setTimeout(resolve, ms));
+}
+// Indirection so tests can inject deterministic waits without real timers.
+let activeSleep = defaultSleep;
+/** Test-only seam: override the sleep implementation used for in-place/wait-out retries. Pass undefined to restore the default. */
+export function __setSleepForTest(fn) {
+    activeSleep = fn ?? defaultSleep;
+}
+function isSubscriptionLockContention(error) {
+    const value = error;
+    return value?.code === "ELOCKED" || (typeof value?.message === "string" && /lock file is already being held/i.test(value.message));
+}
+async function recordSubscriptionMutation(description, mutate) {
+    for (let attempt = 0;; attempt++) {
+        try {
+            await mutate();
+            return true;
+        }
+        catch (error) {
+            if (!isSubscriptionLockContention(error))
+                throw error;
+            const backoffMs = SUBSCRIPTION_LOCK_RETRY_BACKOFFS_MS[attempt];
+            if (backoffMs === undefined) {
+                logRotation(`subscription metadata lock remained busy while ${description}; continuing without failing the provider turn`);
+                return false;
+            }
+            await activeSleep(backoffMs);
+        }
+    }
+}
+/** Test-only seam for the bounded, non-fatal subscription bookkeeping path. */
+export async function __recordSubscriptionMutationForTest(description, mutate) {
+    return recordSubscriptionMutation(description, mutate);
+}
+// A user/host abort (ESC, a critical-tier interrupt, a torn-down window) is NOT a provider
+// failure — it must never cool down a credential or rotate providers. The upstream pi-ai
+// providers set `reason: "aborted"` / `stopReason: "aborted"` (and throw "Request was aborted")
+// ONLY when the request's AbortSignal fired, so those markers alone are a reliable abort
+// signal; we do NOT require the signal to be threaded back through our own options (it may not
+// be), which is what made an ESC drain both provider pools.
+function isUserAbortEvent(event, signal) {
+    if (signal?.aborted)
+        return true;
+    if (event.type !== "error")
+        return false;
+    return event.reason === "aborted" || event.error.stopReason === "aborted" || /request was aborted/i.test(event.error.errorMessage ?? "");
+}
+function isUserAbortError(error, signal) {
+    if (signal?.aborted)
+        return true;
+    const value = error;
+    const message = [value?.errorMessage, value?.message, error instanceof Error ? error.message : String(error ?? "")]
+        .filter((part) => typeof part === "string" && part.length > 0)
+        .join(" ");
+    return (error instanceof Error && /abort/i.test(error.name)) || /request was aborted|operation was aborted|the user aborted/i.test(message);
+}
+function setStatus(state, text) {
+    state.runtimeContext?.ui.setStatus("provider-rotation", text);
+}
+function logRotation(message) {
+    process.stderr.write(`[provider-rotation] ${message}\n`);
+}
+function getCurrentModelRef(model) {
+    return isManagedProvider(model?.provider) ? { providerId: model.provider, modelId: model.id } : undefined;
+}
+function toAssistantError(model, errorMessage) {
+    return {
+        role: "assistant",
+        content: [],
+        api: model.api,
+        provider: model.provider,
+        model: model.id,
+        usage: {
+            input: 0,
+            output: 0,
+            cacheRead: 0,
+            cacheWrite: 0,
+            totalTokens: 0,
+            cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
+        },
+        stopReason: "error",
+        errorMessage,
+        timestamp: Date.now(),
+    };
+}
+function retryAfterMs(headers) {
+    const retryAfterMsHeader = headers["retry-after-ms"];
+    if (retryAfterMsHeader) {
+        const millis = Number(retryAfterMsHeader);
+        if (Number.isFinite(millis))
+            return Math.max(0, millis);
+    }
+    return parseRetryAfterHeader(headers["retry-after"]);
+}
+// Extract an account identity supplied directly by the OAuth credential.
+function loginAccountId(cred) {
+    return typeof cred.accountId === "string" && cred.accountId.trim() ? cred.accountId.trim() : undefined;
+}
+// Short, display-only rendering of a Codex account id for `/provider-sub list`, so distinct
+// accounts are observable without dumping the full opaque id.
+function shortAccountId(accountId) {
+    return accountId.length > 12 ? `${accountId.slice(0, 8)}\u2026${accountId.slice(-4)}` : accountId;
+}
+// The registered provider exposes its OAuth flow through `provider.auth.oauth`. Composed managed
+// providers retain that flow, so the session registry supplies the Anthropic and Codex flows.
+function oauthFlow(providerId, ctx) {
+    const oauth = ctx?.modelRegistry.getProvider(providerId)?.auth.oauth;
+    if (!oauth)
+        throw new Error(`${getProviderLabel(providerId)} has no OAuth flow registered`);
+    return oauth;
+}
+// `refresh` is a pure token exchange that reads only `credential.refresh` and persists
+// nothing; crouter records the rotated token in the selected pool row.
+async function defaultRefreshForProvider(state, providerId, refreshToken) {
+    return oauthFlow(providerId, state.runtimeContext).refresh({ type: "oauth", refresh: refreshToken, access: "", expires: 0 });
+}
+// Indirection so tests can inject refresh failures without real network calls.
+let activeRefreshForProvider;
+/** Test-only seam: override the per-provider refresh function. Pass undefined to restore the default. */
+export function __setRefreshForProviderForTest(fn) {
+    activeRefreshForProvider = fn;
+}
+function openBrowser(url) {
+    const [cmd, args] = process.platform === "darwin"
+        ? ["open", [url]]
+        : process.platform === "win32"
+            ? ["cmd", ["/c", "start", "", url]]
+            : ["xdg-open", [url]];
+    try {
+        const child = spawn(cmd, args, { stdio: "ignore", detached: true });
+        child.on("error", () => {
+            /* best effort -- the manual-code dialog's title carries the URL too */
+        });
+        child.unref();
+    }
+    catch {
+        /* best effort */
+    }
+}
+// Every account refreshes the same way: the pool holds its value, so an expired token is a
+// token exchange serialized on the pool file's lock. A still-fresh access token takes the
+// side-effect-free fast path and never joins the lock. We do NOT route through pi's public
+// `getApiKey`, which wraps pi-ai's `getOAuthApiKey` and collapses a refresh failure into a
+// generic message; calling `activeRefreshForProvider` directly keeps the real error intact so
+// `isInvalidRefreshTokenError` can classify it (cooldown -> reauth -> cross-provider-fallback).
+async function refreshCredentialIfNeeded(state, providerId, credential) {
+    if (!credential.refresh || !credential.access || credential.expires === undefined) {
+        throw new Error(`${getProviderLabel(providerId)} subscription "${credential.label}" is missing its credential value`);
+    }
+    if (credential.expires > Date.now())
+        return credential;
+    const refreshed = await refreshSubscriptionCredential(providerId, credential.label, (token) => activeRefreshForProvider?.(providerId, token) ?? defaultRefreshForProvider(state, providerId, token));
+    return { ...credential, ...refreshed };
+}
+/** Test-only seam: exercises the real `refreshCredentialIfNeeded` (including its locked refresh
+ * transaction) without duplicating its logic. */
+export async function __refreshCredentialIfNeededForTest(providerId, credential) {
+    return refreshCredentialIfNeeded(createProviderRotationState(), providerId, credential);
+}
+async function defaultLoginForProvider(providerId, ctx) {
+    return loginProvider(providerId, ctx);
+}
+let activeLoginForProvider = defaultLoginForProvider;
+/** Test-only seam: override the per-provider login function. Pass undefined to restore the default. */
+export function __setLoginForProviderForTest(fn) {
+    activeLoginForProvider = fn ?? defaultLoginForProvider;
+}
+async function reauthenticateCredential(state, providerId, credential) {
+    const ctx = state.runtimeContext;
+    if (!ctx?.hasUI)
+        return undefined;
+    ctx.ui.notify(`${getProviderLabel(providerId)} subscription "${credential.label}" needs re-authentication. Opening OAuth login...`, "warning");
+    const authenticated = await activeLoginForProvider(providerId, ctx);
+    const identity = await enrichManagedLoginIdentity(providerId, authenticated);
+    // A failed Anthropic profile lookup leaves identity unknown. Do not carry the old row's identity
+    // onto newly issued tokens: it could describe a different account and bypass that account's
+    // duplicate check. Codex supplies its identity directly, so it can retain its existing claim
+    // only when that provider's login result omits one.
+    const reauthedAccountId = identity.accountId ?? (providerId === ANTHROPIC_PROVIDER_ID ? undefined : loginAccountId(authenticated) ?? credential.accountId);
+    const { authFailure: _authFailure, ...healthyCredential } = credential;
+    const next = {
+        ...healthyCredential,
+        refresh: authenticated.refresh,
+        access: authenticated.access,
+        expires: authenticated.expires,
+        ...(reauthedAccountId ? { accountId: reauthedAccountId } : {}),
+        rateLimitedUntil: 0,
+        lastAttemptAt: Date.now(),
+        lastRateLimitedAt: credential.lastRateLimitedAt,
+    };
+    // Identity-checked upsert: reauthenticating this explicit slot into an account already held by
+    // ANOTHER slot throws DuplicateSubscriptionError rather than creating a duplicate quota entry.
+    return ((await upsertSubscriptionWithUniqueAccount(providerId, next)).find((entry) => entry.label === next.label) ?? next);
+}
+function defaultStreamForProvider(model, context, options) {
+    if (model.provider === ANTHROPIC_PROVIDER_ID) {
+        return streamSimpleAnthropic(model, context, options);
+    }
+    return streamSimpleOpenAICodexResponses(model, context, options);
+}
+// Indirection so tests can inject a fake per-provider stream without real network calls.
+let activeStreamForProvider = defaultStreamForProvider;
+function streamForProvider(model, context, options) {
+    return activeStreamForProvider(model, context, options);
+}
+/** Test-only seam: override the per-provider stream factory. Pass undefined to restore the default. */
+export function __setStreamForProviderForTest(fn) {
+    activeStreamForProvider = fn ?? defaultStreamForProvider;
+}
+/** Test-only seam: exercise `fallbackRejectionHint` for one rejection discriminant
+ *  (with an optional resolved-but-rejected candidate) without staging a full turn. */
+// Drives one credential, retrying transient (non-rate-limit) failures in place on the
+// same credential and reporting a classified result for rotation logging.
+// A genuine rate limit ends the attempt immediately (the caller cools the credential down
+// and rotates); a transient blip gets a short in-place retry budget before giving up.
+async function streamProviderAttempt(state, providerId, model, context, options, credential, outer) {
+    for (let attempt = 1;; attempt++) {
+        const attemptAt = Date.now();
+        let observedStatus;
+        let requestedRetryAfterMs;
+        const streamOptions = {
+            ...options,
+            ...(credential ? { apiKey: credential.access } : {}),
+            onResponse: async (response, responseModel) => {
+                observedStatus = response.status;
+                if (response.status === 429 || response.status === 503) {
+                    requestedRetryAfterMs = retryAfterMs(response.headers);
+                }
+                await options?.onResponse?.(response, responseModel);
+            },
+        };
+        const provider = state.runtimeContext?.modelRegistry.getProvider?.(model.provider);
+        const inner = credential
+            ? streamForProvider(model, context, streamOptions)
+            : state.delegatedProviderStreams.get(providerId)?.(model, context, streamOptions)
+                ?? provider?.streamSimple(model, context, streamOptions)
+                ?? streamForProvider(model, context, streamOptions);
+        const bufferedEvents = [];
+        let sawContent = false;
+        let committed = false;
+        let classification;
+        let failureMessage;
+        // Events are buffered ONLY until this attempt commits to the provider -- i.e. until the
+        // first real content event (thinking/text/toolcall) arrives, after which rotating away is
+        // no longer possible. Before commit the buffer lets a clean pre-content rate-limit rotate
+        // to another credential with nothing yet shown. Once committed we flush and forward every
+        // subsequent event LIVE, so viewers stream token-by-token instead of receiving the whole
+        // message at once when the turn finally completes.
+        const emit = (event) => {
+            if (committed)
+                outer.push(event);
+            else
+                bufferedEvents.push(event);
+        };
+        const flush = () => {
+            for (const event of bufferedEvents)
+                outer.push(event);
+            bufferedEvents.length = 0;
+            committed = true;
+        };
+        try {
+            for await (const event of inner) {
+                if (event.type === "error") {
+                    if (isUserAbortEvent(event, options?.signal)) {
+                        emit(event);
+                        flush();
+                        return { result: "fatal", attemptAt };
+                    }
+                    failureMessage = providerErrorMessage(event.error);
+                    classification = classifyProviderError(event.error, { status: observedStatus, retryAfterMs: requestedRetryAfterMs });
+                    if (classification.kind === "fatal" || sawContent) {
+                        emit(event);
+                        flush();
+                        return { result: "fatal", attemptAt, classification };
+                    }
+                    break;
+                }
+                if (event.type !== "start" && event.type !== "done")
+                    sawContent = true;
+                emit(event);
+                // First content event: commit to this provider -- flush the buffer and switch to live
+                // forwarding so the rest of the turn streams incrementally.
+                if (sawContent && !committed)
+                    flush();
+            }
+        }
+        catch (error) {
+            if (isUserAbortError(error, options?.signal)) {
+                emit({
+                    type: "error",
+                    reason: "aborted",
+                    error: { ...toAssistantError(model, error instanceof Error ? error.message : String(error)), stopReason: "aborted" },
+                });
+                flush();
+                return { result: "fatal", attemptAt };
+            }
+            failureMessage = providerErrorMessage(error);
+            classification = classifyProviderError(error, { status: observedStatus, retryAfterMs: requestedRetryAfterMs });
+            if (classification.kind === "fatal" || sawContent) {
+                emit({
+                    type: "error",
+                    reason: "error",
+                    error: toAssistantError(model, error instanceof Error ? error.message : String(error)),
+                });
+                flush();
+                return { result: "fatal", attemptAt, classification };
+            }
+        }
+        if (!classification && requestedRetryAfterMs !== undefined && !sawContent) {
+            // Some providers signal a 429/503 purely via the response header without ever raising
+            // a stream "error" event; treat that the same as an explicit rate-limit classification.
+            classification = { kind: "rate_limit", status: observedStatus, reason: "retry-after header on success path", hadRetryAfterHeader: true };
+        }
+        if (!classification) {
+            flush();
+            return { result: "success", attemptAt };
+        }
+        if (classification.kind === "rate_limit") {
+            return { result: "rate_limited", retryAfterMs: requestedRetryAfterMs, attemptAt, classification };
+        }
+        // Transient: retry the SAME credential/provider in place first -- no cooldown for the
+        // quick in-place budget.
+        const backoffMs = TRANSIENT_RETRY_BACKOFFS_MS[attempt - 1];
+        if (backoffMs === undefined) {
+            // In-place retry budget exhausted. Rather than hard-failing the turn, hand back a
+            // transient-exhausted result so the caller surfaces the error to the user AND rotates
+            // to another credential / fallback provider. Nothing is emitted into the stream here:
+            // the buffered (pre-content) events are discarded exactly like the rate-limit path, so
+            // the rotation target streams a clean turn.
+            return { result: "transient_exhausted", attemptAt, classification, attempts: attempt, failureMessage };
+        }
+        logRotation(`${credential ? `${getProviderLabel(providerId)} subscription "${credential.label}"` : providerId} hit a transient error (${classification.reason}, status=${classification.status ?? "n/a"}); retrying same route in ${backoffMs}ms (attempt ${attempt + 1})`);
+        await activeSleep(backoffMs);
+    }
+}
+function emitTerminalError(stream, model, message) {
+    stream.push({ type: "error", reason: "error", error: toAssistantError(model, message) });
+    stream.end();
+}
+function enqueueFreshRouteCandidates(candidates, queuedRouteIds, knownRoutes, seenReadyLabels, cooldownAttempts, localCoolingDeadlines) {
+    const now = Date.now();
+    for (const [routeId, candidate] of knownRoutes) {
+        if (queuedRouteIds.has(routeId) || (localCoolingDeadlines.get(routeId) ?? 0) > now || !isManagedProvider(candidate.providerId))
+            continue;
+        const seenLabels = seenReadyLabels.get(routeId) ?? new Set();
+        const hasFreshCredential = readSubscriptionPool(candidate.providerId).some((credential) => {
+            if (credential.authFailure !== undefined || (credential.rateLimitedUntil || 0) > now)
+                return false;
+            const cooldownAttempt = cooldownAttempts.get(`${routeId}/${credential.label}`);
+            return !seenLabels.has(credential.label) || (cooldownAttempt !== undefined && credential.lastAttemptAt > cooldownAttempt);
+        });
+        if (hasFreshCredential) {
+            candidates.push(candidate);
+            queuedRouteIds.add(routeId);
+        }
+    }
+}
+async function switchToRoute(pi, ctx, candidate) {
+    if (ctx.model?.provider !== candidate.providerId || ctx.model.id !== candidate.model.id) {
+        if (!await pi.setModel(candidate.model))
+            return false;
+    }
+    pi.setThinkingLevel((candidate.thinkingLevel ?? 'off'));
+    return ctx.model?.provider === candidate.providerId && ctx.model.id === candidate.model.id;
+}
+function logRouteQueueAdvance(from, to, result) {
+    logRotation(`from=${from.providerId}/${from.model.id} from_route_id=${from.routeId} from_credential_source=${from.credentialSource} from_rank=${from.rank} to=${to ? `${to.providerId}/${to.model.id}` : "none"} to_route_id=${to?.routeId ?? "none"} to_credential_source=${to?.credentialSource ?? "none"} to_rank=${to?.rank ?? "none"} reason=${result.classification?.reason ?? result.result}; advancing route queue`);
+}
+// Drive the deterministic route queue from the live rank. Managed routes walk
+// their pool rows; every other declared route is one configured credential.
+async function runManagedProvider(state, model, context, options, stream) {
+    const liveThinking = state.runtimeContext?.thinkingLevel === undefined || state.runtimeContext?.thinkingLevel === 'off'
+        ? undefined
+        : state.runtimeContext.thinkingLevel;
+    const request = modelRequestFromConfig(`${model.provider}/${model.id}${liveThinking ? `:${liveThinking}` : ''}`, configuredValue(state, 'modelIntent', 'CRTR_MODEL_INTENT'), false);
+    const expanded = request ? expandModelCandidates(request, configuredValue(state, 'nodeCwd', 'CRTR_NODE_CWD'), configuredValue(state, 'profileId', 'CRTR_PROFILE_ID')) : [];
+    const liveRank = expanded.find((candidate) => candidate.providerId === model.provider && candidate.modelId === model.id && candidate.thinkingLevel === liveThinking)?.rank ?? 0;
+    const routeCandidates = expanded.slice(liveRank)
+        .map((route) => {
+        const registry = state.runtimeContext?.modelRegistry;
+        const routeModel = registry?.find(route.providerId, route.modelId);
+        if (!registry || !routeModel)
+            return undefined;
+        const availability = probeRouteAvailability(route, registry);
+        if (!availability.registered || (!isManagedProvider(route.providerId) && !availability.authenticated))
+            return undefined;
+        return { routeId: route.routeId, credentialSource: route.credentialSource, rank: route.rank, providerId: route.providerId, model: routeModel, thinkingLevel: route.thinkingLevel };
+    })
+        .filter((candidate) => candidate !== undefined);
+    const isLiveRoute = (candidate) => candidate.providerId === model.provider && candidate.model.id === model.id && candidate.thinkingLevel === liveThinking;
+    // A cooling or unauthenticated managed route contributes its diagnostic and may
+    // later re-enter when a credential becomes ready, but it is never selected now.
+    const candidates = routeCandidates.filter((candidate) => isLiveRoute(candidate) || !isManagedProvider(candidate.providerId) || readSubscriptionPool(candidate.providerId).some((entry) => entry.authFailure === undefined && (entry.rateLimitedUntil || 0) <= Date.now()));
+    if (!candidates.some(isLiveRoute)) {
+        candidates.unshift({ routeId: `live:${model.provider}/${model.id}`, credentialSource: 'live', rank: liveRank, providerId: model.provider, model, thinkingLevel: liveThinking });
+    }
+    const queuedRouteIds = new Set(candidates.map((candidate) => candidate.routeId));
+    const knownRoutes = new Map([...routeCandidates, ...candidates].map((candidate) => [candidate.routeId, candidate]));
+    const attemptedRoutes = new Set();
+    const authFailures = new Set();
+    const localCoolingDeadlines = new Map();
+    const seenReadyLabels = new Map();
+    const cooldownAttempts = new Map();
+    let terminalModel = model;
+    let lastTransientFailure;
+    for (let candidate = candidates.shift(); candidate !== undefined; candidate = candidates.shift()) {
+        const { routeId, providerId, model: candidateModel } = candidate;
+        queuedRouteIds.delete(routeId);
+        knownRoutes.set(routeId, candidate);
+        attemptedRoutes.add(routeId);
+        terminalModel = candidateModel;
+        if (!await switchToRoute(state.extensionAPI, state.runtimeContext, candidate)) {
+            emitTerminalError(stream, candidateModel, `Provider route ${routeId} is unavailable: failed to select ${providerId}/${candidateModel.id}`);
+            return;
+        }
+        if (!isManagedProvider(providerId)) {
+            const result = await streamProviderAttempt(state, providerId, candidateModel, context, options, undefined, stream);
+            if (result.result === 'success') {
+                stream.end();
+                return;
+            }
+            if (result.result === 'fatal') {
+                stream.end();
+                return;
+            }
+            const cooldownMs = result.result === 'rate_limited' ? result.retryAfterMs ?? DEFAULT_RATE_LIMIT_BACKOFF_MS : TRANSIENT_ROTATE_COOLDOWN_MS;
+            if (result.result === 'transient_exhausted')
+                lastTransientFailure = result;
+            localCoolingDeadlines.set(routeId, Math.max(localCoolingDeadlines.get(routeId) ?? 0, Date.now() + cooldownMs));
+            logRouteQueueAdvance(candidate, candidates[0], result);
+            continue;
+        }
+        const rawPool = readSubscriptionPool(providerId);
+        for (const entry of rawPool)
+            if (entry.authFailure === 'invalid_grant')
+                authFailures.add(`${providerId}/${entry.label}`);
+        const available = rawPool.filter((entry) => entry.authFailure === undefined && (!entry.rateLimitedUntil || entry.rateLimitedUntil <= Date.now()));
+        const seenLabels = seenReadyLabels.get(routeId) ?? new Set();
+        for (const credential of available)
+            seenLabels.add(credential.label);
+        seenReadyLabels.set(routeId, seenLabels);
+        let advanceResult;
+        for (const credential of available) {
+            let freshCredential;
+            try {
+                freshCredential = await refreshCredentialIfNeeded(state, providerId, credential);
+            }
+            catch (error) {
+                if (!isInvalidRefreshTokenError(error))
+                    throw error;
+                let reauthenticated;
+                try {
+                    reauthenticated = await reauthenticateCredential(state, providerId, credential);
+                }
+                catch (loginError) {
+                    state.runtimeContext?.ui.notify(`${getProviderLabel(providerId)} re-authentication failed: ${loginError instanceof Error ? loginError.message : String(loginError)}`, 'error');
+                }
+                if (!reauthenticated) {
+                    const attemptAt = Date.now();
+                    await recordSubscriptionMutation(`recording ${getProviderLabel(providerId)} authentication failure`, () => markSubscriptionAuthenticationFailed(providerId, credential.label, INVALID_REFRESH_TOKEN_BACKOFF_MS, attemptAt));
+                    authFailures.add(`${providerId}/${credential.label}`);
+                    continue;
+                }
+                freshCredential = reauthenticated;
+            }
+            const result = await streamProviderAttempt(state, providerId, candidateModel, context, options, freshCredential, stream);
+            if (result.result === 'success') {
+                if (freshCredential.rateLimitedUntil || freshCredential.authFailure)
+                    await recordSubscriptionMutation(`recording ${getProviderLabel(providerId)} success`, () => markSubscriptionSuccess(providerId, freshCredential.label, result.attemptAt));
+                stream.end();
+                return;
+            }
+            if (result.result === 'fatal') {
+                stream.end();
+                return;
+            }
+            const cooldownMs = result.result === 'rate_limited' ? result.retryAfterMs ?? DEFAULT_RATE_LIMIT_BACKOFF_MS : TRANSIENT_ROTATE_COOLDOWN_MS;
+            if (result.result === 'transient_exhausted')
+                lastTransientFailure = result;
+            if (result.result === 'rate_limited' && result.retryAfterMs !== undefined) {
+                const persisted = await recordSubscriptionMutation(`recording ${getProviderLabel(providerId)} server cooldown`, () => markSubscriptionRateLimited(providerId, freshCredential.label, cooldownMs, result.attemptAt));
+                if (persisted)
+                    cooldownAttempts.set(`${routeId}/${freshCredential.label}`, result.attemptAt);
+                else
+                    localCoolingDeadlines.set(routeId, Math.max(localCoolingDeadlines.get(routeId) ?? 0, Date.now() + cooldownMs));
+            }
+            else {
+                localCoolingDeadlines.set(routeId, Math.max(localCoolingDeadlines.get(routeId) ?? 0, Date.now() + cooldownMs));
+            }
+            advanceResult = result;
+        }
+        enqueueFreshRouteCandidates(candidates, queuedRouteIds, knownRoutes, seenReadyLabels, cooldownAttempts, localCoolingDeadlines);
+        if (advanceResult !== undefined)
+            logRouteQueueAdvance(candidate, candidates[0], advanceResult);
+    }
+    const coolingDeadline = earliestCoolingDeadline([...knownRoutes.values()], localCoolingDeadlines);
+    const tried = [...attemptedRoutes].join(', ');
+    if (lastTransientFailure !== undefined) {
+        const detail = lastTransientFailure.failureMessage ?? lastTransientFailure.classification?.reason ?? 'unknown connection failure';
+        emitTerminalError(stream, terminalModel, `Provider routes failed after transient retries: ${detail} (tried ${tried})`);
+        return;
+    }
+    if (coolingDeadline !== undefined) {
+        emitCoolingError(stream, terminalModel, new ManagedProviderCoolingError(`All provider routes are rate limited (cooling down ~${Math.ceil((coolingDeadline - Date.now()) / 1000)}s; tried ${tried})`, coolingDeadline));
+        return;
+    }
+    const authFailure = [...authFailures][0];
+    if (authFailure !== undefined) {
+        emitTerminalError(stream, terminalModel, `Provider authentication failed (invalid_grant) for ${authFailure}; re-authenticate the credential and retry (tried ${tried})`);
+        return;
+    }
+    emitTerminalError(stream, terminalModel, `All provider routes exhausted (tried ${tried})`);
+}
+// The earliest strictly-future deadline only when every attempted route is cooling.
+function earliestCoolingDeadline(attempted, localCoolingDeadlines) {
+    const now = Date.now();
+    let earliest;
+    let sawRoute = false;
+    for (const route of attempted) {
+        const localUntil = localCoolingDeadlines.get(route.routeId) ?? 0;
+        if (localUntil > now) {
+            sawRoute = true;
+            earliest = earliest === undefined ? localUntil : Math.min(earliest, localUntil);
+            continue;
+        }
+        if (!isManagedProvider(route.providerId))
+            return undefined;
+        const pool = readSubscriptionPool(route.providerId).filter((entry) => entry.authFailure === undefined);
+        // Routes with no usable credential were never runnable this turn, so they do
+        // not defeat the cooling outcome of another route. An auth-failed-only pool
+        // is likewise unauthenticated; a valid cooling row still contributes its
+        // deadline even when a sibling row carries invalid_grant.
+        if (pool.length === 0)
+            continue;
+        sawRoute = true;
+        if (pool.some((entry) => (entry.rateLimitedUntil || 0) <= now))
+            return undefined;
+        const routeEarliest = Math.min(...pool.map((entry) => entry.rateLimitedUntil || 0));
+        earliest = earliest === undefined ? routeEarliest : Math.min(earliest, routeEarliest);
+    }
+    return sawRoute ? earliest : undefined;
+}
+// Emit the typed cooling terminal error, attaching its diagnostic (the earliest
+// deadline) to the assistant error message so the deadline survives verbatim into
+// agent_end.messages for the stophook to read.
+function emitCoolingError(stream, model, error) {
+    const message = toAssistantError(model, error.message);
+    message.diagnostics = [error.toDiagnostic()];
+    stream.push({ type: "error", reason: "error", error: message });
+    stream.end();
+}
+function streamManagedProvider(state, model, context, options) {
+    const stream = createAssistantMessageEventStream();
+    runManagedProvider(state, model, context, options, stream).catch((error) => {
+        stream.push({
+            type: "error",
+            reason: "error",
+            error: toAssistantError(model, error instanceof Error ? error.message : String(error)),
+        });
+        stream.end();
+    });
+    return stream;
+}
+function parseCommand(args) {
+    return args
+        .trim()
+        .split(/\s+/)
+        .map((part) => part.trim())
+        .filter(Boolean);
+}
+function parseProvider(value) {
+    return isManagedProvider(value) ? value : undefined;
+}
+function installDeclaredRouteProviders(state, pi, ctx) {
+    const config = readMergedLaunchConfig(ctx.cwd, configuredValue(state, 'profileId', 'CRTR_PROFILE_ID') || null);
+    const providerIds = new Set(Object.values(config.modelRoutes ?? {}).map((route) => route.providerId));
+    for (const providerId of providerIds) {
+        if (isManagedProvider(providerId) || state.delegatedProviderStreams.has(providerId))
+            continue;
+        const provider = ctx.modelRegistry.getProvider?.(providerId);
+        const model = ctx.modelRegistry.getAll?.().find((entry) => entry.provider === providerId);
+        if (!provider || !model)
+            continue;
+        state.delegatedProviderStreams.set(providerId, provider.streamSimple.bind(provider));
+        // `api` is required by pi whenever a registration supplies streamSimple.
+        // Re-registration merges this wrapper over the existing declaration, so the
+        // provider's models, auth, and captured composed transport remain intact.
+        pi.registerProvider(providerId, { api: model.api, streamSimple: (model, context, options) => streamManagedProvider(state, model, context, options) });
+    }
+}
+function updateRotationStatus(state, ctx) {
+    const current = getCurrentModelRef(ctx.model);
+    if (current && isManagedProvider(current.providerId))
+        setStatus(state, formatStatusLine(current.providerId));
+}
+/** Restore only when the deterministic route rank improves; no shared-home
+ * preferred-model state is needed or consulted. */
+async function restoreHigherRankIfAvailable(state, pi, ctx) {
+    if (!readRotationConfig().revertWhenAvailable)
+        return false;
+    const liveThinking = ctx.thinkingLevel === undefined || ctx.thinkingLevel === 'off' ? undefined : ctx.thinkingLevel;
+    const request = modelRequestFromConfig(`${ctx.model?.provider ?? ''}/${ctx.model?.id ?? ''}${liveThinking ? `:${liveThinking}` : ''}`, configuredValue(state, 'modelIntent', 'CRTR_MODEL_INTENT'), false);
+    if (!request || !ctx.model)
+        return false;
+    const candidates = expandModelCandidates(request, configuredValue(state, 'nodeCwd', 'CRTR_NODE_CWD'), configuredValue(state, 'profileId', 'CRTR_PROFILE_ID'));
+    const liveRank = candidates.find((candidate) => candidate.providerId === ctx.model?.provider && candidate.modelId === ctx.model?.id && candidate.thinkingLevel === liveThinking)?.rank;
+    if (liveRank === undefined)
+        return false;
+    const preferred = candidates.find((candidate) => candidate.rank < liveRank && resolveProviderCandidates([{
+            ...candidate,
+            ...probeRouteAvailability(candidate, ctx.modelRegistry),
+            automaticFallback: true,
+        }]).candidates.length > 0);
+    if (!preferred)
+        return false;
+    const model = ctx.modelRegistry.find(preferred.providerId, preferred.modelId);
+    if (!model || !await pi.setModel(model))
+        return false;
+    pi.setThinkingLevel((preferred.thinkingLevel ?? 'off'));
+    return true;
+}
+// Manual-code input is a human-in-the-loop OAuth round-trip (open link, log in on
+// whatever device, copy the localhost redirect URL back) that routinely exceeds
+// DEFAULT_DIALOG_TIMEOUT_MS -- give it a human-scale bound instead of the ordinary
+// confirm/select default.
+const OAUTH_MANUAL_INPUT_TIMEOUT_MS = 600_000;
+/**
+ * pi-ai races the manual-code-paste dialog against its loopback callback server. When the
+ * callback supplies a code, the manual-input promise can remain pending, so `run` receives:
+ *  - `readCode`, a `ctx.ui.input` wrapper with an AbortSignal. Aborting it removes the pending
+ *    broker entry and emits an `extension_ui_dismiss` frame for exactly that dialog.
+ *  - `dismissManual`, which aborts the signal at the post-code `onProgress` boundary before
+ *    the token exchange completes. `onProgress` is emitted only after a code is available, so
+ *    dismissing then cannot close a dialog the user still needs; `finally` covers paths without
+ *    progress, and repeated AbortController calls are inert.
+ *
+ * `undefined` from `readCode` means cancellation or timeout. It throws so OAuth ends directly
+ * instead of opening a second paste-only dialog.
+ */
+export async function loginWithManualInputCleanup(ctx, run) {
+    const controller = new AbortController();
+    const readCode = async (title, placeholder) => {
+        const result = await ctx.ui.input(title, placeholder, { timeout: OAUTH_MANUAL_INPUT_TIMEOUT_MS, signal: controller.signal });
+        if (result === undefined)
+            throw new Error("OAuth login cancelled");
+        return result;
+    };
+    try {
+        return await run(readCode, () => controller.abort());
+    }
+    finally {
+        controller.abort();
+    }
+}
+// The manual-code dialog carries the authorize URL in its title (rendered as a tappable link
+// on web), so a headless remote box where `openBrowser` opens nothing still has a real path
+// through login. pi's own `manual_code` message is URL-less, hence these per-provider titles.
+const MANUAL_CODE_DIALOG = {
+    [ANTHROPIC_PROVIDER_ID]: {
+        title: (url) => `Open this URL to authenticate Claude:\n${url}\n\nPaste the redirect URL or code`,
+        placeholder: "http://localhost:53692/callback?...",
+    },
+    [OPENAI_CODEX_PROVIDER_ID]: {
+        title: (url) => `Open this URL to authenticate OpenAI Codex:\n${url}\n\nPaste the redirect URL or code`,
+        placeholder: "http://localhost:1455/auth/callback?...",
+    },
+};
+// This extension adapts the provider OAuth interaction onto `ctx.ui`. The attach viewer and
+// `crtr sys setup` use LoginDialogComponent with `core/auth-interaction.ts`; the broker uses
+// the dialog primitives below.
+async function loginProvider(providerId, ctx) {
+    const oauth = oauthFlow(providerId, ctx);
+    const manual = MANUAL_CODE_DIALOG[providerId];
+    let authorizeUrl;
+    return loginWithManualInputCleanup(ctx, (readCode, dismissManual) => oauth.login({
+        notify: (event) => {
+            switch (event.type) {
+                case "auth_url":
+                    authorizeUrl = event.url;
+                    openBrowser(event.url);
+                    if (event.instructions)
+                        ctx.ui.notify(event.instructions, "info");
+                    break;
+                case "device_code":
+                    openBrowser(event.verificationUri);
+                    ctx.ui.notify(`Enter code ${event.userCode} at ${event.verificationUri}`, "info");
+                    break;
+                case "progress":
+                    // Progress follows receipt of an OAuth code, so dismiss the now-unneeded manual
+                    // dialog before token exchange. The browser path may emit no progress; `finally`
+                    // closes that dialog when the OAuth attempt settles.
+                    dismissManual();
+                    ctx.ui.notify(event.message, "info");
+                    break;
+                case "info":
+                    ctx.ui.notify(event.message, "info");
+                    break;
+            }
+        },
+        prompt: async (prompt) => {
+            if (prompt.type === "manual_code")
+                return readCode(manual.title(authorizeUrl), manual.placeholder);
+            if (prompt.type === "select") {
+                // Codex now opens its login with a browser-vs-device-code choice.
+                const labels = prompt.options.map((option) => option.label);
+                const chosen = await ctx.ui.select(prompt.message, labels);
+                const selected = chosen === undefined ? undefined : prompt.options[labels.indexOf(chosen)];
+                if (!selected)
+                    throw new Error("OAuth login cancelled");
+                return selected.id;
+            }
+            return (await ctx.ui.input(prompt.message, prompt.placeholder ?? "")) ?? "";
+        },
+    }));
+}
+export function createProviderRotationExtension(options) {
+    const state = createProviderRotationState(options);
+    return async (pi) => {
+        state.extensionAPI = pi;
+        pi.registerProvider(ANTHROPIC_PROVIDER_ID, {
+            api: "anthropic-messages",
+            streamSimple: (model, context, streamOptions) => streamManagedProvider(state, model, context, streamOptions),
+        });
+        pi.registerProvider(OPENAI_CODEX_PROVIDER_ID, {
+            api: "openai-codex-responses",
+            streamSimple: (model, context, streamOptions) => streamManagedProvider(state, model, context, streamOptions),
+        });
+        pi.on("session_start", async (_event, ctx) => {
+            state.runtimeContext = ctx;
+            installDeclaredRouteProviders(state, pi, ctx);
+            updateRotationStatus(state, ctx);
+        });
+        pi.on("before_agent_start", async (_event, ctx) => {
+            state.runtimeContext = ctx;
+            const restored = await restoreHigherRankIfAvailable(state, pi, ctx);
+            if (restored && ctx.model && isManagedProvider(ctx.model.provider)) {
+                logRotation(`restored higher-ranked route ${ctx.model.provider}/${ctx.model.id}`);
+                setStatus(state, formatStatusLine(ctx.model.provider));
+            }
+        });
+        pi.on("model_select", async (_event, ctx) => {
+            state.runtimeContext = ctx;
+            updateRotationStatus(state, ctx);
+        });
+        pi.registerCommand("provider-sub", {
+            description: "Manage subscription credential pools for provider rotation",
+            handler: async (args, ctx) => {
+                const [providerArg, verb, ...rest] = parseCommand(args);
+                const providerId = parseProvider(providerArg);
+                if (!providerId) {
+                    ctx.ui.notify("Usage: /provider-sub <anthropic|openai-codex> <list|add|select|rm>", "error");
+                    return;
+                }
+                const action = verb === "rm" ? "remove" : verb || "list";
+                if (action === "status" || action === "list") {
+                    const pool = readSubscriptionPool(providerId);
+                    const lines = [formatStatusLine(providerId), ""];
+                    for (const [index, entry] of pool.entries()) {
+                        const cooldown = entry.rateLimitedUntil > Date.now() ? ` cooldown=${Math.ceil((entry.rateLimitedUntil - Date.now()) / 1000)}s` : "";
+                        const account = entry.accountId ? ` account=${shortAccountId(entry.accountId)}` : "";
+                        lines.push(`${index + 1}. ${entry.label}${account}${cooldown}`);
+                    }
+                    ctx.ui.notify(lines.join("\n"), "info");
+                    return;
+                }
+                if (action === "add") {
+                    if (!ctx.hasUI) {
+                        ctx.ui.notify(`${getProviderLabel(providerId)} login requires an interactive UI.`, "error");
+                        return;
+                    }
+                    const pool = readSubscriptionPool(providerId);
+                    const label = (rest.join(" ") || (await ctx.ui.input(`${getProviderLabel(providerId)} subscription label`, "")) || "").trim();
+                    if (!label)
+                        return;
+                    if (pool.some((entry) => entry.label.toLowerCase() === label.toLowerCase())) {
+                        ctx.ui.notify(`${getProviderLabel(providerId)} subscription already exists: ${label}`, "error");
+                        return;
+                    }
+                    try {
+                        const cred = await activeLoginForProvider(providerId, ctx);
+                        const identity = await enrichManagedLoginIdentity(providerId, cred);
+                        // Explicit add always creates a new label, so a matching real account is rejected by
+                        // the locked uniqueness gate rather than creating a second quota slot.
+                        await addSubscription(providerId, {
+                            label,
+                            refresh: cred.refresh,
+                            access: cred.access,
+                            expires: cred.expires,
+                            ...(identity.accountId ? { accountId: identity.accountId } : {}),
+                            rateLimitedUntil: 0,
+                            lastAttemptAt: 0,
+                            lastRateLimitedAt: 0,
+                        });
+                        logRotation(`added ${getProviderLabel(providerId)} subscription "${label}"`);
+                        ctx.ui.notify(`Added ${getProviderLabel(providerId)} subscription: ${label}`, "info");
+                        updateRotationStatus(state, ctx);
+                    }
+                    catch (error) {
+                        ctx.ui.notify(error instanceof Error ? error.message : String(error), "error");
+                    }
+                    return;
+                }
+                if (action === "select") {
+                    const ref = rest.join(" ");
+                    if (!ref) {
+                        ctx.ui.notify(`Usage: /provider-sub ${providerId} select <label|index>`, "error");
+                        return;
+                    }
+                    try {
+                        await promoteSubscription(providerId, ref);
+                        logRotation(`selected ${getProviderLabel(providerId)} subscription "${ref}"`);
+                        ctx.ui.notify(`Preferred ${getProviderLabel(providerId)} subscription: ${ref}`, "info");
+                        updateRotationStatus(state, ctx);
+                    }
+                    catch (error) {
+                        ctx.ui.notify(error instanceof Error ? error.message : String(error), "error");
+                    }
+                    return;
+                }
+                if (action === "remove") {
+                    const ref = rest.join(" ");
+                    if (!ref) {
+                        ctx.ui.notify(`Usage: /provider-sub ${providerId} rm <label|index>`, "error");
+                        return;
+                    }
+                    try {
+                        await removeManagedAccount(providerId, ref);
+                        logRotation(`removed ${getProviderLabel(providerId)} subscription "${ref}"`);
+                        ctx.ui.notify(`Removed ${getProviderLabel(providerId)} subscription: ${ref}`, "info");
+                        updateRotationStatus(state, ctx);
+                    }
+                    catch (error) {
+                        ctx.ui.notify(error instanceof Error ? error.message : String(error), "error");
+                    }
+                    return;
+                }
+                ctx.ui.notify(`Unknown /provider-sub verb: ${verb}`, "error");
+            },
+        });
+    };
+}
+const defaultProviderRotationExtension = createProviderRotationExtension();
+export default defaultProviderRotationExtension;