@north-light/crouter 0.3.154

package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts

@@ -0,0 +1,1232 @@
+import { spawn } from "node:child_process";
+import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import {
+  createAssistantMessageEventStream,
+  type AssistantMessage,
+  type AssistantMessageEventStream,
+  type AuthEvent,
+  type AuthPrompt,
+  type Context,
+  type Model,
+  type OAuthAuth,
+  type SimpleStreamOptions,
+} from "@earendil-works/pi-ai";
+import { streamSimpleAnthropic, streamSimpleOpenAICodexResponses } from "@earendil-works/pi-ai/compat";
+import {
+  ANTHROPIC_PROVIDER_ID,
+  OPENAI_CODEX_PROVIDER_ID,
+  INVALID_REFRESH_TOKEN_BACKOFF_MS,
+  formatStatusLine,
+  getLadderProviderId,
+  getProviderLabel,
+  isManagedProvider,
+  isInvalidRefreshTokenError,
+  addSubscription,
+  enrichManagedLoginIdentity,
+  markSubscriptionAuthenticationFailed,
+  markSubscriptionRateLimited,
+  markSubscriptionSuccess,
+  parseRetryAfterHeader,
+  promoteSubscription,
+  readRotationConfig,
+  readSubscriptionPool,
+  refreshSubscriptionCredential,
+  removeManagedAccount,
+  resolveFallbackTarget,
+  resolveProviderCandidates,
+  restorePreferredModelIfPossible,
+  switchToFallbackIfPossible,
+  upsertSubscriptionWithUniqueAccount,
+  type FallbackTarget,
+  type ManagedProviderId,
+  type SubscriptionCredential,
+} from "../../../core/subscription-state.js";
+import {
+  ManagedProviderCoolingError,
+  readManagedProviderRotationPolicy,
+} from "../../../core/runtime/managed-provider-cooling.js";
+
+// A response without Retry-After proves only that THIS attempt was throttled; it
+// does not authorize crouter to invent a five-minute provider-wide outage. Keep
+// a short turn-local probe backoff to avoid an immediate retry loop, then let a
+// fresh turn prove whether the credential is usable again. Real
+// server-provided Retry-After deadlines are still honored exactly.
+const DEFAULT_RATE_LIMIT_BACKOFF_MS = 15 * 1000;
+// Short in-place retry budget for a transient network/5xx blip on the same credential.
+// Once this budget is exhausted, surface the error and rotate to another candidate.
+const TRANSIENT_RETRY_BACKOFFS_MS = [250, 750];
+// After the in-place transient retry budget is exhausted, rotate instead of failing: the
+// credential itself is probably healthy (this is a provider-side blip like "overloaded"),
+// so cool it down only briefly -- long enough to rotate to another credential / fallback
+// provider, short enough that it comes back quickly if it was the only option.
+const TRANSIENT_ROTATE_COOLDOWN_MS = 60 * 1000;
+// A few short outer retries absorb a normal multi-broker collision; if the metadata file
+// remains busy, the provider call still proceeds. Subscription bookkeeping must never turn a
+// healthy model response into a fatal generation fault.
+const SUBSCRIPTION_LOCK_RETRY_BACKOFFS_MS = [25, 75, 150];
+
+let runtimeContext: ExtensionContext | undefined;
+let extensionAPI: ExtensionAPI | undefined;
+
+type AttemptResult = {
+  result: "success" | "rate_limited" | "transient_exhausted" | "fatal";
+  retryAfterMs?: number;
+  attemptAt: number;
+  attempts?: number;
+  classification?: ProviderErrorClassification;
+};
+
+// A genuine rate limit (HTTP 429, or 503 with a real Retry-After header) must
+// cool the credential down and rotate to the next one; a transient network/5xx blip
+// (connection reset, socket errors, timeouts, plain 500/502/504, or 503 with NO
+// retry-after) is not evidence the credential itself is bad -- it gets a short in-place
+// retry on the same credential, never a cooldown. Everything else is fatal. Bare
+// `usage`/`quota` substrings stay out of the message match: they are false-positive
+// magnets (e.g. "usage" appears in normal assistant text), while the precise
+// terminal usage-limit signals above still cool the credential down and rotate to the
+// next one.
+type ProviderErrorClassification = {
+  kind: "rate_limit" | "transient" | "fatal";
+  status?: number;
+  reason: string;
+  hadRetryAfterHeader: boolean;
+};
+
+const TRANSIENT_MESSAGE_PATTERN =
+  /connection|network|fetch failed|socket|econnreset|econnrefused|enotfound|etimedout|timed? out|timeout|overloaded|service.?unavailable|temporarily unavailable/i;
+const RATE_LIMIT_MESSAGE_PATTERN = /\b429\b|rate.?limit|too many requests/i;
+// Precise terminal usage-limit signals (as opposed to the bare `usage`/`quota` substrings
+// dropped above): pi-ai's Codex adapter surfaces `usage_limit_reached` / `usage_not_included`
+// as the friendly "You have hit your ChatGPT usage limit..." message, often on a non-429
+// status, so the status-code check above misses it. These are specific enough not to false-
+// positive on ordinary assistant text mentioning "usage".
+const USAGE_LIMIT_MESSAGE_PATTERN = /usage[ _]limit|usage_not_included|usagelimiterror|insufficient_quota|quota exceeded/i;
+
+function classifyProviderError(error: unknown, observed: { status?: number; retryAfterMs?: number }): ProviderErrorClassification {
+  const value = error as { status?: unknown; errorMessage?: unknown; message?: unknown } | undefined;
+  const errorStatus = typeof value?.status === "number" ? value.status : undefined;
+  const status = observed.status ?? errorStatus;
+  const message = [value?.errorMessage, value?.message, error instanceof Error ? error.message : ""]
+    .filter((part): part is string => typeof part === "string" && part.length > 0)
+    .join(" ");
+  const hadRetryAfterHeader = observed.retryAfterMs !== undefined;
+
+  if (status === 429) return { kind: "rate_limit", status, reason: "http 429", hadRetryAfterHeader };
+  if (status === 503 && hadRetryAfterHeader) return { kind: "rate_limit", status, reason: "http 503 with retry-after", hadRetryAfterHeader };
+  if (status === 500 || status === 502 || status === 503 || status === 504) {
+    return { kind: "transient", status, reason: `http ${status}`, hadRetryAfterHeader };
+  }
+  const transientMatch = message.match(TRANSIENT_MESSAGE_PATTERN);
+  if (transientMatch) return { kind: "transient", status, reason: transientMatch[0].toLowerCase(), hadRetryAfterHeader };
+  const rateLimitMatch = message.match(RATE_LIMIT_MESSAGE_PATTERN);
+  if (rateLimitMatch) return { kind: "rate_limit", status, reason: rateLimitMatch[0].toLowerCase(), hadRetryAfterHeader };
+  const usageLimitMatch = message.match(USAGE_LIMIT_MESSAGE_PATTERN);
+  if (usageLimitMatch) return { kind: "rate_limit", status, reason: usageLimitMatch[0].toLowerCase(), hadRetryAfterHeader };
+  return { kind: "fatal", status, reason: "unclassified", hadRetryAfterHeader };
+}
+
+async function defaultSleep(ms: number): Promise<void> {
+  await new Promise<void>((resolve) => setTimeout(resolve, ms));
+}
+
+// Indirection so tests can inject deterministic waits without real timers.
+let activeSleep: typeof defaultSleep = defaultSleep;
+
+/** Test-only seam: override the sleep implementation used for in-place/wait-out retries. Pass undefined to restore the default. */
+export function __setSleepForTest(fn: typeof defaultSleep | undefined): void {
+  activeSleep = fn ?? defaultSleep;
+}
+
+function isSubscriptionLockContention(error: unknown): boolean {
+  const value = error as { code?: unknown; message?: unknown } | undefined;
+  return value?.code === "ELOCKED" || (typeof value?.message === "string" && /lock file is already being held/i.test(value.message));
+}
+
+async function recordSubscriptionMutation(description: string, mutate: () => Promise<unknown>): Promise<boolean> {
+  for (let attempt = 0; ; attempt++) {
+    try {
+      await mutate();
+      return true;
+    } catch (error) {
+      if (!isSubscriptionLockContention(error)) throw error;
+      const backoffMs = SUBSCRIPTION_LOCK_RETRY_BACKOFFS_MS[attempt];
+      if (backoffMs === undefined) {
+        logRotation(`subscription metadata lock remained busy while ${description}; continuing without failing the provider turn`);
+        return false;
+      }
+      await activeSleep(backoffMs);
+    }
+  }
+}
+
+/** Test-only seam for the bounded, non-fatal subscription bookkeeping path. */
+export async function __recordSubscriptionMutationForTest(description: string, mutate: () => Promise<unknown>): Promise<boolean> {
+  return recordSubscriptionMutation(description, mutate);
+}
+
+// A user/host abort (ESC, a critical-tier interrupt, a torn-down window) is NOT a provider
+// failure — it must never cool down a credential or rotate providers. The upstream pi-ai
+// providers set `reason: "aborted"` / `stopReason: "aborted"` (and throw "Request was aborted")
+// ONLY when the request's AbortSignal fired, so those markers alone are a reliable abort
+// signal; we do NOT require the signal to be threaded back through our own options (it may not
+// be), which is what made an ESC drain both provider pools.
+function isUserAbortEvent(event: Parameters<AssistantMessageEventStream["push"]>[0], signal: AbortSignal | undefined): boolean {
+  if (signal?.aborted) return true;
+  if (event.type !== "error") return false;
+  return event.reason === "aborted" || event.error.stopReason === "aborted" || /request was aborted/i.test(event.error.errorMessage ?? "");
+}
+
+function isUserAbortError(error: unknown, signal: AbortSignal | undefined): boolean {
+  if (signal?.aborted) return true;
+  const value = error as { name?: unknown; errorMessage?: unknown; message?: unknown } | undefined;
+  const message = [value?.errorMessage, value?.message, error instanceof Error ? error.message : String(error ?? "")]
+    .filter((part): part is string => typeof part === "string" && part.length > 0)
+    .join(" ");
+  return (error instanceof Error && /abort/i.test(error.name)) || /request was aborted|operation was aborted|the user aborted/i.test(message);
+}
+
+function setStatus(text: string): void {
+  runtimeContext?.ui.setStatus("provider-rotation", text);
+}
+
+function logRotation(message: string): void {
+  process.stderr.write(`[provider-rotation] ${message}\n`);
+}
+
+function getCurrentModelRef(model?: Model<any>): { providerId: ManagedProviderId; modelId: string } | undefined {
+  return isManagedProvider(model?.provider) ? { providerId: model.provider, modelId: model.id } : undefined;
+}
+
+function toAssistantError(model: Model<any>, errorMessage: string): AssistantMessage {
+  return {
+    role: "assistant",
+    content: [],
+    api: model.api,
+    provider: model.provider,
+    model: model.id,
+    usage: {
+      input: 0,
+      output: 0,
+      cacheRead: 0,
+      cacheWrite: 0,
+      totalTokens: 0,
+      cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
+    },
+    stopReason: "error",
+    errorMessage,
+    timestamp: Date.now(),
+  };
+}
+
+function retryAfterMs(headers: Record<string, string>): number | undefined {
+  const retryAfterMsHeader = headers["retry-after-ms"];
+  if (retryAfterMsHeader) {
+    const millis = Number(retryAfterMsHeader);
+    if (Number.isFinite(millis)) return Math.max(0, millis);
+  }
+  return parseRetryAfterHeader(headers["retry-after"]);
+}
+
+type RefreshedCredential = { refresh: string; access: string; expires: number };
+// Codex login carries the ChatGPT `accountId`; Anthropic identity is enriched from its OAuth
+// profile after login. Optional so both providers and the test seam fit.
+type LoginCredential = { refresh: string; access: string; expires: number; accountId?: string };
+
+// Extract an account identity supplied directly by the OAuth credential.
+function loginAccountId(cred: { accountId?: unknown }): string | undefined {
+  return typeof cred.accountId === "string" && cred.accountId.trim() ? cred.accountId.trim() : undefined;
+}
+
+// Short, display-only rendering of a Codex account id for `/provider-sub list`, so distinct
+// accounts are observable without dumping the full opaque id.
+function shortAccountId(accountId: string): string {
+  return accountId.length > 12 ? `${accountId.slice(0, 8)}\u2026${accountId.slice(-4)}` : accountId;
+}
+
+// A credential that has been through refresh/reauth for this turn: `.access` is guaranteed
+// populated. Every account stores its own value in the pool, so downstream stream-attempt code
+// can rely on `.access` directly.
+type ResolvedCredential = SubscriptionCredential & { access: string };
+
+// The registered provider exposes its OAuth flow through `provider.auth.oauth`. Composed managed
+// providers retain that flow, so the session registry supplies the Anthropic and Codex flows.
+function oauthFlow(providerId: ManagedProviderId, ctx?: ExtensionContext): OAuthAuth {
+  const oauth = (ctx ?? runtimeContext)?.modelRegistry.getProvider(providerId)?.auth.oauth;
+  if (!oauth) throw new Error(`${getProviderLabel(providerId)} has no OAuth flow registered`);
+  return oauth;
+}
+
+// `refresh` is a pure token exchange that reads only `credential.refresh` and persists
+// nothing; crouter records the rotated token in the selected pool row.
+async function defaultRefreshForProvider(providerId: ManagedProviderId, refreshToken: string): Promise<RefreshedCredential> {
+  return oauthFlow(providerId).refresh({ type: "oauth", refresh: refreshToken, access: "", expires: 0 });
+}
+
+// Indirection so tests can inject refresh failures without real network calls.
+let activeRefreshForProvider: typeof defaultRefreshForProvider = defaultRefreshForProvider;
+
+/** Test-only seam: override the per-provider refresh function. Pass undefined to restore the default. */
+export function __setRefreshForProviderForTest(fn: typeof defaultRefreshForProvider | undefined): void {
+  activeRefreshForProvider = fn ?? defaultRefreshForProvider;
+}
+
+function openBrowser(url: string): void {
+  const [cmd, args] =
+    process.platform === "darwin"
+      ? ["open", [url]]
+      : process.platform === "win32"
+        ? ["cmd", ["/c", "start", "", url]]
+        : ["xdg-open", [url]];
+  try {
+    const child = spawn(cmd, args, { stdio: "ignore", detached: true });
+    child.on("error", () => {
+      /* best effort -- the manual-code dialog's title carries the URL too */
+    });
+    child.unref();
+  } catch {
+    /* best effort */
+  }
+}
+
+// Every account refreshes the same way: the pool holds its value, so an expired token is a
+// token exchange serialized on the pool file's lock. A still-fresh access token takes the
+// side-effect-free fast path and never joins the lock. We do NOT route through pi's public
+// `getApiKey`, which wraps pi-ai's `getOAuthApiKey` and collapses a refresh failure into a
+// generic message; calling `activeRefreshForProvider` directly keeps the real error intact so
+// `isInvalidRefreshTokenError` can classify it (cooldown -> reauth -> cross-provider-fallback).
+async function refreshCredentialIfNeeded(providerId: ManagedProviderId, credential: SubscriptionCredential): Promise<ResolvedCredential> {
+  if (!credential.refresh || !credential.access || credential.expires === undefined) {
+    throw new Error(`${getProviderLabel(providerId)} subscription "${credential.label}" is missing its credential value`);
+  }
+  if (credential.expires > Date.now()) return credential as ResolvedCredential;
+  const refreshed = await refreshSubscriptionCredential(providerId, credential.label, (token) => activeRefreshForProvider(providerId, token));
+  return { ...credential, ...refreshed } as ResolvedCredential;
+}
+
+/** Test-only seam: exercises the real `refreshCredentialIfNeeded` (including its locked refresh
+ * transaction) without duplicating its logic. */
+export async function __refreshCredentialIfNeededForTest(providerId: ManagedProviderId, credential: SubscriptionCredential): Promise<ResolvedCredential> {
+  return refreshCredentialIfNeeded(providerId, credential);
+}
+
+async function defaultLoginForProvider(providerId: ManagedProviderId, ctx: ExtensionContext): Promise<LoginCredential> {
+  return loginProvider(providerId, ctx);
+}
+
+let activeLoginForProvider: typeof defaultLoginForProvider = defaultLoginForProvider;
+
+/** Test-only seam: override the per-provider login function. Pass undefined to restore the default. */
+export function __setLoginForProviderForTest(fn: typeof defaultLoginForProvider | undefined): void {
+  activeLoginForProvider = fn ?? defaultLoginForProvider;
+}
+
+async function reauthenticateCredential(providerId: ManagedProviderId, credential: SubscriptionCredential): Promise<ResolvedCredential | undefined> {
+  const ctx = runtimeContext;
+  if (!ctx?.hasUI) return undefined;
+  ctx.ui.notify(`${getProviderLabel(providerId)} subscription "${credential.label}" needs re-authentication. Opening OAuth login...`, "warning");
+  const authenticated = await activeLoginForProvider(providerId, ctx);
+  const identity = await enrichManagedLoginIdentity(providerId, authenticated);
+  // A failed Anthropic profile lookup leaves identity unknown. Do not carry the old row's identity
+  // onto newly issued tokens: it could describe a different account and bypass that account's
+  // duplicate check. Codex supplies its identity directly, so it can retain its existing claim
+  // only when that provider's login result omits one.
+  const reauthedAccountId =
+    identity.accountId ?? (providerId === ANTHROPIC_PROVIDER_ID ? undefined : loginAccountId(authenticated) ?? credential.accountId);
+  const { authFailure: _authFailure, ...healthyCredential } = credential;
+  const next: SubscriptionCredential = {
+    ...healthyCredential,
+    refresh: authenticated.refresh,
+    access: authenticated.access,
+    expires: authenticated.expires,
+    ...(reauthedAccountId ? { accountId: reauthedAccountId } : {}),
+    rateLimitedUntil: 0,
+    lastAttemptAt: Date.now(),
+    lastRateLimitedAt: credential.lastRateLimitedAt,
+  };
+  // Identity-checked upsert: reauthenticating this explicit slot into an account already held by
+  // ANOTHER slot throws DuplicateSubscriptionError rather than creating a duplicate quota entry.
+  return ((await upsertSubscriptionWithUniqueAccount(providerId, next)).find((entry) => entry.label === next.label) ?? next) as ResolvedCredential;
+}
+
+function defaultStreamForProvider(model: Model<any>, context: Context, options: SimpleStreamOptions | undefined): AssistantMessageEventStream {
+  if (model.provider === ANTHROPIC_PROVIDER_ID) {
+    return streamSimpleAnthropic(model as Model<"anthropic-messages">, context, options);
+  }
+  return streamSimpleOpenAICodexResponses(model as Model<"openai-codex-responses">, context, options);
+}
+
+// Indirection so tests can inject a fake per-provider stream without real network calls.
+let activeStreamForProvider: typeof defaultStreamForProvider = defaultStreamForProvider;
+
+function streamForProvider(model: Model<any>, context: Context, options: SimpleStreamOptions | undefined): AssistantMessageEventStream {
+  return activeStreamForProvider(model, context, options);
+}
+
+/** Test-only seam: override the per-provider stream factory. Pass undefined to restore the default. */
+export function __setStreamForProviderForTest(fn: typeof defaultStreamForProvider | undefined): void {
+  activeStreamForProvider = fn ?? defaultStreamForProvider;
+}
+
+/** Test-only seam: exercise `fallbackRejectionHint` for one rejection discriminant
+ *  (with an optional resolved-but-rejected candidate) without staging a full turn. */
+export function __fallbackRejectionHintForTest(reason: FallbackRejectionReason, candidate?: FallbackTarget): string {
+  return fallbackRejectionHint({ reason, candidate });
+}
+
+// Drives one credential, retrying transient (non-rate-limit) failures in place on the
+// same credential and reporting a classified result for rotation logging.
+// A genuine rate limit ends the attempt immediately (the caller cools the credential down
+// and rotates); a transient blip gets a short in-place retry budget before giving up.
+async function streamProviderAttempt(
+  providerId: ManagedProviderId,
+  model: Model<any>,
+  context: Context,
+  options: SimpleStreamOptions | undefined,
+  credential: ResolvedCredential,
+  outer: AssistantMessageEventStream,
+): Promise<AttemptResult> {
+  for (let attempt = 1; ; attempt++) {
+    const attemptAt = Date.now();
+    let observedStatus: number | undefined;
+    let requestedRetryAfterMs: number | undefined;
+    const inner = streamForProvider(model, context, {
+      ...options,
+      apiKey: credential.access,
+      onResponse: async (response, responseModel) => {
+        observedStatus = response.status;
+        if (response.status === 429 || response.status === 503) {
+          requestedRetryAfterMs = retryAfterMs(response.headers);
+        }
+        await options?.onResponse?.(response, responseModel);
+      },
+    });
+
+    const bufferedEvents: Parameters<AssistantMessageEventStream["push"]>[0][] = [];
+    let sawContent = false;
+    let committed = false;
+    let classification: ProviderErrorClassification | undefined;
+
+    // Events are buffered ONLY until this attempt commits to the provider -- i.e. until the
+    // first real content event (thinking/text/toolcall) arrives, after which rotating away is
+    // no longer possible. Before commit the buffer lets a clean pre-content rate-limit rotate
+    // to another credential with nothing yet shown. Once committed we flush and forward every
+    // subsequent event LIVE, so viewers stream token-by-token instead of receiving the whole
+    // message at once when the turn finally completes.
+    const emit = (event: Parameters<AssistantMessageEventStream["push"]>[0]): void => {
+      if (committed) outer.push(event);
+      else bufferedEvents.push(event);
+    };
+    const flush = (): void => {
+      for (const event of bufferedEvents) outer.push(event);
+      bufferedEvents.length = 0;
+      committed = true;
+    };
+
+    try {
+      for await (const event of inner) {
+        if (event.type === "error") {
+          if (isUserAbortEvent(event, options?.signal)) {
+            emit(event);
+            flush();
+            return { result: "fatal", attemptAt };
+          }
+          classification = classifyProviderError(event.error, { status: observedStatus, retryAfterMs: requestedRetryAfterMs });
+          if (classification.kind === "fatal" || sawContent) {
+            emit(event);
+            flush();
+            return { result: "fatal", attemptAt, classification };
+          }
+          break;
+        }
+        if (event.type !== "start" && event.type !== "done") sawContent = true;
+        emit(event);
+        // First content event: commit to this provider -- flush the buffer and switch to live
+        // forwarding so the rest of the turn streams incrementally.
+        if (sawContent && !committed) flush();
+      }
+    } catch (error) {
+      if (isUserAbortError(error, options?.signal)) {
+        emit({
+          type: "error",
+          reason: "aborted",
+          error: { ...toAssistantError(model, error instanceof Error ? error.message : String(error)), stopReason: "aborted" },
+        });
+        flush();
+        return { result: "fatal", attemptAt };
+      }
+      classification = classifyProviderError(error, { status: observedStatus, retryAfterMs: requestedRetryAfterMs });
+      if (classification.kind === "fatal" || sawContent) {
+        emit({
+          type: "error",
+          reason: "error",
+          error: toAssistantError(model, error instanceof Error ? error.message : String(error)),
+        });
+        flush();
+        return { result: "fatal", attemptAt, classification };
+      }
+    }
+
+    if (!classification && requestedRetryAfterMs !== undefined && !sawContent) {
+      // Some providers signal a 429/503 purely via the response header without ever raising
+      // a stream "error" event; treat that the same as an explicit rate-limit classification.
+      classification = { kind: "rate_limit", status: observedStatus, reason: "retry-after header on success path", hadRetryAfterHeader: true };
+    }
+
+    if (!classification) {
+      flush();
+      return { result: "success", attemptAt };
+    }
+
+    if (classification.kind === "rate_limit") {
+      return { result: "rate_limited", retryAfterMs: requestedRetryAfterMs, attemptAt, classification };
+    }
+
+    // Transient: retry the SAME credential/provider in place first -- no cooldown for the
+    // quick in-place budget.
+    const backoffMs = TRANSIENT_RETRY_BACKOFFS_MS[attempt - 1];
+    if (backoffMs === undefined) {
+      // In-place retry budget exhausted. Rather than hard-failing the turn, hand back a
+      // transient-exhausted result so the caller surfaces the error to the user AND rotates
+      // to another credential / fallback provider. Nothing is emitted into the stream here:
+      // the buffered (pre-content) events are discarded exactly like the rate-limit path, so
+      // the rotation target streams a clean turn.
+      return { result: "transient_exhausted", attemptAt, classification, attempts: attempt };
+    }
+    logRotation(
+      `${getProviderLabel(providerId)} subscription "${credential.label}" hit a transient error (${classification.reason}, status=${classification.status ?? "n/a"}); retrying same credential in ${backoffMs}ms (attempt ${attempt + 1})`,
+    );
+    await activeSleep(backoffMs);
+  }
+}
+
+function emitTerminalError(stream: AssistantMessageEventStream, model: Model<any>, message: string): void {
+  stream.push({ type: "error", reason: "error", error: toAssistantError(model, message) });
+  stream.end();
+}
+
+// Why a candidate fallback was rejected -- each reason gets its OWN accurate hint
+// (`fallbackRejectionHint`) so the terminal error never gives advice that can't fix the
+// actual blocker. Only `no_credentials` is a genuine "go authenticate" case; the others
+// are config/exhaustion states where telling the user to authenticate would be misleading.
+type FallbackRejectionReason =
+  | "no_ladder_entry"
+  | "already_attempted"
+  | "no_credentials"
+  | "model_unregistered"
+  | "cooling"
+  | "provider_pinned";
+
+// On rejection we carry the resolved-but-rejected `candidate` FallbackTarget whenever one
+// exists (every reason except `no_ladder_entry`, where no target resolved at all), so the
+// hint can name the exact provider/model and build an executable repair command from the
+// candidate's config ladder key + strength.
+type FallbackResolution =
+  | { target: FallbackTarget }
+  | { target?: undefined; reason: FallbackRejectionReason; candidate?: FallbackTarget };
+
+// Cross-provider fallback requires a target that is usable right now: authenticated
+// (its subscription pool has at least one credential, even if currently
+// cooling down) AND registered in the model registry -- rather than blindly switching the
+// session model to a provider nobody logged into. Returns the rejection reason (config has
+// no ladder rung, the target was already tried this turn, the target has zero credentials,
+// or its resolved model isn't in the registry) rather than just `undefined`, so callers can
+// surface an accurate terminal error instead of always defaulting to "authenticate a
+// fallback provider" -- which is actively wrong when the real blocker is a stale ladder.
+function resolveViableFallback(
+  providerId: ManagedProviderId,
+  modelId: string | undefined,
+  attempted: Set<ManagedProviderId>,
+): FallbackResolution {
+  const target = resolveFallbackTarget(providerId, modelId, readRotationConfig(), extensionAPI?.getThinkingLevel?.());
+  if (!target) return { reason: "no_ladder_entry" };
+  if (attempted.has(target.providerId)) return { reason: "already_attempted", candidate: target };
+  // Pool occupancy IS authentication: every stored row carries a token pair, so an empty pool
+  // means switching the session model onto this provider would make pi fail the NEXT turn with a
+  // non-retryable "Provider is not configured" that no agent can clear from inside its own turn.
+  const pool = readSubscriptionPool(target.providerId);
+  const resolution = resolveProviderCandidates([{
+    providerId: target.providerId,
+    modelId: target.modelId,
+    thinkingLevel: target.thinkingLevel,
+    registered: Boolean(runtimeContext?.modelRegistry.find(target.providerId, target.modelId)),
+    authenticated: pool.length > 0,
+    coolingUntil: pool.length > 0 && pool.every((entry) => (entry.rateLimitedUntil || 0) > Date.now())
+      ? Math.min(...pool.map((entry) => entry.rateLimitedUntil || 0))
+      : undefined,
+  }]);
+  if (resolution.candidates.length > 0) return { target };
+  switch (resolution.rejected[0]?.reason) {
+    case "unauthenticated": return { reason: "no_credentials", candidate: target };
+    case "unregistered": return { reason: "model_unregistered", candidate: target };
+    // Do not churn the session model merely to wait on an unavailable target.
+    // The caller includes this candidate's pool in the typed cooling deadline,
+    // so the broker can re-drive when either provider becomes ready.
+    case "cooling": return { reason: "cooling", candidate: target };
+    default: return { reason: "no_credentials", candidate: target };
+  }
+}
+
+// Actionable hint appended to a terminal error when no fallback is viable. Each rejection
+// reason gets advice that can resolve it; only `no_credentials` advises authentication because
+// it is the sole case where a login supplies the missing capability.
+function fallbackRejectionHint(resolution: FallbackResolution): string {
+  if (resolution.target) return "try again later";
+  const candidate = resolution.candidate;
+  switch (resolution.reason) {
+    case "no_credentials": {
+      const who = candidate ? ` (${getProviderLabel(candidate.providerId)}, e.g. /provider-sub ${candidate.providerId} add)` : " with /provider-sub";
+      return `authenticate a fallback provider${who}, or try again later`;
+    }
+    case "model_unregistered": {
+      // Repair command must be EXECUTABLE as printed: `--value`, and the CONFIG ladder
+      // provider key (`openai`, not the runtime `openai-codex`) plus the candidate's own
+      // strength -- so the user pastes it, swaps in an installed model id, and it works.
+      const ladderKey = candidate ? getLadderProviderId(candidate.providerId) : "<provider>";
+      const strength = candidate?.strength ?? "<strength>";
+      const named = candidate ? candidate.label : "the configured fallback";
+      return `configured fallback ${named} is not in this build's model registry -- the modelLadders config is out of sync with the installed model catalog. Point it at an installed model with \`crtr sys config set modelLadders.${ladderKey}.${strength} --value <installed-model-id>\`, or upgrade/downgrade to a build whose ladder matches`;
+    }
+    case "already_attempted":
+      return "every managed provider was already tried this turn and is exhausted or cooling down -- try again later";
+    case "cooling":
+      return "the configured fallback provider is also cooling down -- retrying when the first provider becomes ready";
+    case "provider_pinned":
+      return "the selected provider is explicitly pinned -- retrying it when its first credential becomes ready";
+    case "no_ladder_entry":
+      return "no cross-provider fallback is configured for this model -- set one with `crtr sys config set modelLadders.<provider>.<strength> --value <model-id>`, or try again later";
+  }
+}
+
+// Switches the session model to an already-viability-checked fallback target. On the rare
+// race where the switch itself still fails (e.g. `setModel` rejects it), emits a graceful,
+// actionable terminal error on `stream` instead of throwing a generic "failed to switch".
+async function switchToFallbackModel(
+  stream: AssistantMessageEventStream,
+  model: Model<any>,
+  originalModel: { providerId: ManagedProviderId; modelId: string },
+): Promise<FallbackTarget | undefined> {
+  const ctx = runtimeContext;
+  if (!extensionAPI || !ctx) {
+    emitTerminalError(stream, model, `Provider fallback unavailable: runtime not initialized for ${originalModel.providerId}/${originalModel.modelId}`);
+    return undefined;
+  }
+  const target = await switchToFallbackIfPossible(extensionAPI, ctx, originalModel);
+  if (!target) {
+    emitTerminalError(stream, model, `Provider fallback unavailable: failed to switch models for ${getProviderLabel(originalModel.providerId)}`);
+    return undefined;
+  }
+  return target;
+}
+
+// Drives the ordered recovery candidates for one managed turn. Each candidate is a
+// provider/model whose ready credentials are tried in order. A provider returns to
+// the list only for a newly ready label or a success that followed this turn's
+// cooldown, so bookkeeping updates never retry an exhausted pool.
+type ManagedProviderCandidate = {
+  model: Model<any>;
+  providerId: ManagedProviderId;
+};
+
+type ManagedProviderTerminalState = {
+  model: Model<any>;
+  providerId: ManagedProviderId;
+  currentPool: SubscriptionCredential[];
+  fallbackResolution: FallbackResolution;
+};
+
+function credentialKey(providerId: ManagedProviderId, label: string): string {
+  return `${providerId}/${label}`;
+}
+
+function enqueueFreshProviderCandidates(
+  candidates: ManagedProviderCandidate[],
+  queuedProviderIds: Set<ManagedProviderId>,
+  knownModels: Map<ManagedProviderId, Model<any>>,
+  seenReadyLabels: Map<ManagedProviderId, Set<string>>,
+  cooldownAttempts: Map<string, number>,
+  localCoolingDeadlines: Map<ManagedProviderId, number>,
+): void {
+  const now = Date.now();
+  for (const [providerId, model] of knownModels) {
+    if (queuedProviderIds.has(providerId) || (localCoolingDeadlines.get(providerId) ?? 0) > now) continue;
+    const seenLabels = seenReadyLabels.get(providerId) ?? new Set<string>();
+    const hasFreshCredential = readSubscriptionPool(providerId).some((credential) => {
+      if (credential.rateLimitedUntil && credential.rateLimitedUntil > now) return false;
+      const cooldownAttempt = cooldownAttempts.get(credentialKey(providerId, credential.label));
+      return !seenLabels.has(credential.label) || (cooldownAttempt !== undefined && credential.lastAttemptAt > cooldownAttempt);
+    });
+    if (hasFreshCredential) {
+      candidates.push({ model, providerId });
+      queuedProviderIds.add(providerId);
+    }
+  }
+}
+
+// Drives managed providers through an explicit candidate list. The list is exhausted
+// only when no provider has a newly-ready credential or recovered cooldown and no viable fallback remains.
+// The broker owns delayed retry through typed cooling errors; this loop never sleeps
+// while waiting for a provider pool to recover.
+async function runManagedProvider(
+  model: Model<any>,
+  context: Context,
+  options: SimpleStreamOptions | undefined,
+  stream: AssistantMessageEventStream,
+): Promise<void> {
+  const initialProviderId = model.provider as ManagedProviderId;
+  const candidates: ManagedProviderCandidate[] = [{ model, providerId: initialProviderId }];
+  const queuedProviderIds = new Set<ManagedProviderId>([initialProviderId]);
+  const knownModels = new Map<ManagedProviderId, Model<any>>([[initialProviderId, model]]);
+  const attempted = new Set<ManagedProviderId>();
+  const authFailures = new Set<string>();
+  const localCoolingDeadlines = new Map<ManagedProviderId, number>();
+  const seenReadyLabels = new Map<ManagedProviderId, Set<string>>();
+  const cooldownAttempts = new Map<string, number>();
+  let terminal: ManagedProviderTerminalState | undefined;
+
+  for (let candidate = candidates.shift(); candidate !== undefined; candidate = candidates.shift()) {
+    const { model: candidateModel, providerId } = candidate;
+    queuedProviderIds.delete(providerId);
+    knownModels.set(providerId, candidateModel);
+    attempted.add(providerId);
+
+    const liveModel = getCurrentModelRef(runtimeContext?.model);
+    if (liveModel && (liveModel.providerId !== providerId || liveModel.modelId !== candidateModel.id)) {
+      const target = await switchToFallbackModel(stream, candidateModel, liveModel);
+      if (!target) return; // terminal error already emitted
+      if (target.providerId !== providerId || target.modelId !== candidateModel.id) {
+        emitTerminalError(stream, candidateModel, `Provider re-entry unavailable: expected ${providerId}/${candidateModel.id} but switched to ${target.label}`);
+        return;
+      }
+      logRotation(`${getProviderLabel(liveModel.providerId)} pool recovered -> ${target.label}`);
+      setStatus(`${getProviderLabel(liveModel.providerId)} -> ${target.label}`);
+    }
+
+    const rawPool = readSubscriptionPool(providerId);
+    for (const entry of rawPool) {
+      if (entry.authFailure === "invalid_grant") authFailures.add(`${providerId}/${entry.label}`);
+    }
+    const available = rawPool.filter((entry) => !entry.rateLimitedUntil || entry.rateLimitedUntil <= Date.now());
+    const seenLabels = seenReadyLabels.get(providerId) ?? new Set<string>();
+    for (const credential of available) seenLabels.add(credential.label);
+    seenReadyLabels.set(providerId, seenLabels);
+
+    for (const credential of available) {
+      let freshCredential: ResolvedCredential;
+      try {
+        freshCredential = await refreshCredentialIfNeeded(providerId, credential);
+      } catch (error) {
+        const attemptAt = Date.now();
+        if (isInvalidRefreshTokenError(error)) {
+          let reauthenticated: ResolvedCredential | undefined;
+          try {
+            reauthenticated = await reauthenticateCredential(providerId, credential);
+          } catch (loginError) {
+            runtimeContext?.ui.notify(
+              `${getProviderLabel(providerId)} re-authentication failed: ${loginError instanceof Error ? loginError.message : String(loginError)}`,
+              "error",
+            );
+          }
+          if (reauthenticated) {
+            freshCredential = reauthenticated;
+          } else {
+            // Keep the unusable credential out of ordinary rotation, but retain
+            // its auth cause for this turn. A 30-day exclusion is not a transient
+            // rate limit and must never author an auto rate-limit recovery fault.
+            await recordSubscriptionMutation(
+              `recording ${getProviderLabel(providerId)} authentication failure`,
+              () => markSubscriptionAuthenticationFailed(providerId, credential.label, INVALID_REFRESH_TOKEN_BACKOFF_MS, attemptAt),
+            );
+            authFailures.add(`${providerId}/${credential.label}`);
+            setStatus(formatStatusLine(providerId));
+            continue;
+          }
+        } else {
+          throw error;
+        }
+      }
+      const result = await streamProviderAttempt(providerId, candidateModel, context, options, freshCredential, stream);
+
+      if (result.result === "success") {
+        // The common healthy state already has nothing to clear. Avoid two global
+        // file locks just to advance display-only lastAttemptAt telemetry.
+        if (freshCredential.rateLimitedUntil || freshCredential.authFailure) {
+          await recordSubscriptionMutation(
+            `recording ${getProviderLabel(providerId)} success`,
+            () => markSubscriptionSuccess(providerId, freshCredential.label, result.attemptAt),
+          );
+        }
+        logRotation(`${getProviderLabel(providerId)} subscription "${freshCredential.label}" succeeded; cooldown cleared`);
+        setStatus(formatStatusLine(providerId));
+        stream.end();
+        return;
+      }
+
+      if (result.result === "rate_limited") {
+        const cooldownMs = result.retryAfterMs ?? DEFAULT_RATE_LIMIT_BACKOFF_MS;
+        if (result.retryAfterMs !== undefined) {
+          // A server-declared deadline is authoritative for every broker sharing
+          // this credential, so persist it in the cross-process pool.
+          const persisted = await recordSubscriptionMutation(
+            `recording ${getProviderLabel(providerId)} server cooldown`,
+            () => markSubscriptionRateLimited(providerId, freshCredential.label, cooldownMs, result.attemptAt),
+          );
+          if (persisted) {
+            // A zero server deadline is immediately eligible, but it still needs
+            // a later causal success before it becomes a new recovery candidate.
+            cooldownAttempts.set(credentialKey(providerId, freshCredential.label), result.attemptAt);
+          } else {
+            localCoolingDeadlines.set(
+              providerId,
+              Math.max(localCoolingDeadlines.get(providerId) ?? 0, Date.now() + cooldownMs),
+            );
+          }
+        } else {
+          // A headerless 429/usage-limit response proves only that this request
+          // failed. Persisting an invented deadline globally let one busy node
+          // gate unrelated nodes that were succeeding concurrently. Keep the
+          // short probe deadline turn-local instead.
+          localCoolingDeadlines.set(
+            providerId,
+            Math.max(localCoolingDeadlines.get(providerId) ?? 0, Date.now() + cooldownMs),
+          );
+        }
+        const retryAfterPart = result.retryAfterMs !== undefined ? `retry-after=${Math.ceil(result.retryAfterMs / 1000)}s` : "no retry-after header (default backoff)";
+        logRotation(
+          `${getProviderLabel(providerId)} subscription "${freshCredential.label}" rate-limited (status=${result.classification?.status ?? "n/a"}, reason=${result.classification?.reason ?? "unclassified"}, ${retryAfterPart}); ${result.retryAfterMs !== undefined ? "shared cooldown" : "turn-local retry backoff"} ${Math.ceil(cooldownMs / 1000)}s, trying next credential`,
+        );
+        runtimeContext?.ui.notify?.(
+          `${getProviderLabel(providerId)} "${freshCredential.label}" rate-limited (${result.retryAfterMs !== undefined ? "cooling down" : "retrying this turn in"} ${Math.ceil(cooldownMs / 1000)}s); rotating to another credential/provider.`,
+          "warning",
+        );
+        continue;
+      }
+
+      if (result.result === "transient_exhausted") {
+        // The credential is likely healthy -- this was a provider-side transient blip (e.g.
+        // "overloaded") that survived the in-place retries. Surface it to the user and rotate:
+        // cool the credential down briefly so this loop moves on to the next credential or fallback.
+        const reason = result.classification?.reason ?? "transient error";
+        const persisted = await recordSubscriptionMutation(
+          `recording ${getProviderLabel(providerId)} transient cooldown`,
+          () => markSubscriptionRateLimited(providerId, freshCredential.label, TRANSIENT_ROTATE_COOLDOWN_MS, result.attemptAt),
+        );
+        if (persisted) {
+          cooldownAttempts.set(credentialKey(providerId, freshCredential.label), result.attemptAt);
+        } else {
+          localCoolingDeadlines.set(
+            providerId,
+            Math.max(localCoolingDeadlines.get(providerId) ?? 0, Date.now() + TRANSIENT_ROTATE_COOLDOWN_MS),
+          );
+        }
+        runtimeContext?.ui.notify?.(
+          `${getProviderLabel(providerId)} "${freshCredential.label}" hit a transient error after ${result.attempts ?? 1} attempts (${reason}); rotating to another credential/provider.`,
+          "warning",
+        );
+        logRotation(
+          `${getProviderLabel(providerId)} subscription "${freshCredential.label}" transient error exhausted (reason=${reason}, status=${result.classification?.status ?? "n/a"}); cooling down ${Math.ceil(TRANSIENT_ROTATE_COOLDOWN_MS / 1000)}s and rotating`,
+        );
+        setStatus(formatStatusLine(providerId));
+        continue;
+      }
+
+      // Fatal (or partial content already streamed): cannot safely switch providers mid-turn.
+      if (result.classification) {
+        logRotation(
+          `${getProviderLabel(providerId)} subscription "${freshCredential.label}" failed fatally (status=${result.classification.status ?? "n/a"}, reason=${result.classification.reason})`,
+        );
+      }
+      stream.end();
+      return;
+    }
+
+    // Read a fresh pool after the attempts above so terminal cooling state reflects
+    // mutations made by this candidate, not its initial lock-free snapshot.
+    const currentPool = readSubscriptionPool(providerId);
+    enqueueFreshProviderCandidates(candidates, queuedProviderIds, knownModels, seenReadyLabels, cooldownAttempts, localCoolingDeadlines);
+    if (candidates.length > 0) continue;
+
+    const originalModel = getCurrentModelRef(candidateModel);
+    if (!originalModel) throw new Error(`Provider fallback unavailable: missing current model`);
+
+    const rotationPolicy = readManagedProviderRotationPolicy(runtimeContext?.ui);
+    const fallbackResolution: FallbackResolution = rotationPolicy.allowCrossProviderFallback
+      ? resolveViableFallback(originalModel.providerId, originalModel.modelId, attempted)
+      : { reason: "provider_pinned" };
+    if (fallbackResolution.target) {
+      const target = await switchToFallbackModel(stream, candidateModel, originalModel);
+      if (!target) return; // terminal error already emitted
+      logRotation(`${getProviderLabel(providerId)} pool exhausted -> ${target.label}`);
+      setStatus(`${getProviderLabel(providerId)} -> ${target.label}`);
+
+      const fallbackModel = runtimeContext?.modelRegistry.find(target.providerId, target.modelId);
+      if (!fallbackModel) {
+        emitTerminalError(stream, candidateModel, `Provider fallback unavailable: ${target.label} not in model registry`);
+        return;
+      }
+
+      candidates.push({ model: fallbackModel, providerId: target.providerId });
+      queuedProviderIds.add(target.providerId);
+      knownModels.set(target.providerId, fallbackModel);
+      continue;
+    }
+
+    terminal = { model: candidateModel, providerId, currentPool, fallbackResolution };
+  }
+
+  if (!terminal) throw new Error("Managed provider recovery exhausted without a terminal state");
+
+  const { model: terminalModel, providerId, currentPool, fallbackResolution } = terminal;
+  const label = getProviderLabel(providerId);
+  const triedList = [...attempted].join(", ");
+
+  if (currentPool.length === 0) {
+    emitTerminalError(stream, terminalModel, `${label} not authenticated — run \`crtr sys setup\` (or /provider-sub ${providerId} add) (tried ${triedList})`);
+    return;
+  }
+
+  const fallbackHint = fallbackRejectionHint(fallbackResolution);
+  const coolingProviders = new Set(attempted);
+  if (!fallbackResolution.target && fallbackResolution.reason === "cooling" && fallbackResolution.candidate) {
+    coolingProviders.add(fallbackResolution.candidate.providerId);
+  }
+  const coolingDeadline = earliestCoolingDeadline(coolingProviders, localCoolingDeadlines);
+  if (coolingDeadline !== undefined) {
+    const coolingSecs = Math.ceil((coolingDeadline - Date.now()) / 1000);
+    emitCoolingError(
+      stream,
+      terminalModel,
+      new ManagedProviderCoolingError(
+        `All managed providers are rate limited (cooling down ~${coolingSecs}s); ${fallbackHint} (tried ${triedList})`,
+        coolingDeadline,
+      ),
+    );
+    return;
+  }
+
+  const authFailure = [...authFailures][0];
+  if (authFailure !== undefined) {
+    emitTerminalError(stream, terminalModel, `Provider authentication failed (invalid_grant) for ${authFailure}; re-authenticate the credential and retry (tried ${triedList})`);
+    return;
+  }
+
+  emitTerminalError(
+    stream,
+    terminalModel,
+    `All managed provider pools exhausted; ${fallbackHint} (tried ${triedList})`,
+  );
+}
+
+// The earliest strictly-future cooldown deadline across every managed pool tried
+// this turn, or undefined unless EVERY existing credential in those pools is
+// cooling (a single ready credential, or an empty attempted set, means this is not
+// an all-cooling condition and must not be typed as such). Read fresh under each
+// pool's lock-free snapshot, mirroring the wait-out math above.
+function earliestCoolingDeadline(
+  attempted: Set<ManagedProviderId>,
+  localCoolingDeadlines = new Map<ManagedProviderId, number>(),
+): number | undefined {
+  const now = Date.now();
+  let earliest: number | undefined;
+  let sawCredential = false;
+  for (const attemptedProvider of attempted) {
+    const localUntil = localCoolingDeadlines.get(attemptedProvider) ?? 0;
+    if (localUntil > now) {
+      sawCredential = true;
+      if (earliest === undefined || localUntil < earliest) earliest = localUntil;
+      continue;
+    }
+    for (const entry of readSubscriptionPool(attemptedProvider)) {
+      if (entry.authFailure !== undefined) continue;
+      sawCredential = true;
+      const until = entry.rateLimitedUntil || 0;
+      if (until <= now) return undefined; // a ready credential exists — not all cooling
+      if (earliest === undefined || until < earliest) earliest = until;
+    }
+  }
+  return sawCredential ? earliest : undefined;
+}
+
+// Emit the typed cooling terminal error, attaching its diagnostic (the earliest
+// deadline) to the assistant error message so the deadline survives verbatim into
+// agent_end.messages for the stophook to read.
+function emitCoolingError(stream: AssistantMessageEventStream, model: Model<any>, error: ManagedProviderCoolingError): void {
+  const message = toAssistantError(model, error.message);
+  message.diagnostics = [error.toDiagnostic()];
+  stream.push({ type: "error", reason: "error", error: message });
+  stream.end();
+}
+
+function streamManagedProvider(model: Model<any>, context: Context, options?: SimpleStreamOptions): AssistantMessageEventStream {
+  if (!isManagedProvider(model.provider)) {
+    return streamForProvider(model, context, options);
+  }
+
+  const stream = createAssistantMessageEventStream();
+
+  runManagedProvider(model, context, options, stream).catch((error) => {
+    stream.push({
+      type: "error",
+      reason: "error",
+      error: toAssistantError(model, error instanceof Error ? error.message : String(error)),
+    });
+    stream.end();
+  });
+
+  return stream;
+}
+
+function parseCommand(args: string): string[] {
+  return args
+    .trim()
+    .split(/\s+/)
+    .map((part) => part.trim())
+    .filter(Boolean);
+}
+
+function parseProvider(value: string | undefined): ManagedProviderId | undefined {
+  return isManagedProvider(value) ? value : undefined;
+}
+
+function updateRotationStatus(ctx: ExtensionContext): void {
+  const current = getCurrentModelRef(ctx.model);
+  if (current) setStatus(formatStatusLine(current.providerId));
+}
+
+// Manual-code input is a human-in-the-loop OAuth round-trip (open link, log in on
+// whatever device, copy the localhost redirect URL back) that routinely exceeds
+// DEFAULT_DIALOG_TIMEOUT_MS -- give it a human-scale bound instead of the ordinary
+// confirm/select default.
+const OAUTH_MANUAL_INPUT_TIMEOUT_MS = 600_000;
+
+/**
+ * pi-ai races the manual-code-paste dialog against its loopback callback server. When the
+ * callback supplies a code, the manual-input promise can remain pending, so `run` receives:
+ *  - `readCode`, a `ctx.ui.input` wrapper with an AbortSignal. Aborting it removes the pending
+ *    broker entry and emits an `extension_ui_dismiss` frame for exactly that dialog.
+ *  - `dismissManual`, which aborts the signal at the post-code `onProgress` boundary before
+ *    the token exchange completes. `onProgress` is emitted only after a code is available, so
+ *    dismissing then cannot close a dialog the user still needs; `finally` covers paths without
+ *    progress, and repeated AbortController calls are inert.
+ *
+ * `undefined` from `readCode` means cancellation or timeout. It throws so OAuth ends directly
+ * instead of opening a second paste-only dialog.
+ */
+export async function loginWithManualInputCleanup<T>(
+  ctx: ExtensionContext,
+  run: (
+    readCode: (title: string, placeholder: string) => Promise<string>,
+    dismissManual: () => void,
+  ) => Promise<T>,
+): Promise<T> {
+  const controller = new AbortController();
+  const readCode = async (title: string, placeholder: string): Promise<string> => {
+    const result = await ctx.ui.input(title, placeholder, { timeout: OAUTH_MANUAL_INPUT_TIMEOUT_MS, signal: controller.signal });
+    if (result === undefined) throw new Error("OAuth login cancelled");
+    return result;
+  };
+  try {
+    return await run(readCode, () => controller.abort());
+  } finally {
+    controller.abort();
+  }
+}
+
+// The manual-code dialog carries the authorize URL in its title (rendered as a tappable link
+// on web), so a headless remote box where `openBrowser` opens nothing still has a real path
+// through login. pi's own `manual_code` message is URL-less, hence these per-provider titles.
+const MANUAL_CODE_DIALOG: Record<ManagedProviderId, { title: (url: string | undefined) => string; placeholder: string }> = {
+  [ANTHROPIC_PROVIDER_ID]: {
+    title: (url) => `Open this URL to authenticate Claude:\n${url}\n\nPaste the redirect URL or code`,
+    placeholder: "http://localhost:53692/callback?...",
+  },
+  [OPENAI_CODEX_PROVIDER_ID]: {
+    title: (url) => `Open this URL to authenticate OpenAI Codex:\n${url}\n\nPaste the redirect URL or code`,
+    placeholder: "http://localhost:1455/auth/callback?...",
+  },
+};
+
+// This extension adapts the provider OAuth interaction onto `ctx.ui`. The attach viewer and
+// `crtr sys setup` use LoginDialogComponent with `core/auth-interaction.ts`; the broker uses
+// the dialog primitives below.
+async function loginProvider(providerId: ManagedProviderId, ctx: ExtensionContext): Promise<LoginCredential> {
+  const oauth = oauthFlow(providerId, ctx);
+  const manual = MANUAL_CODE_DIALOG[providerId];
+  let authorizeUrl: string | undefined;
+  return loginWithManualInputCleanup(ctx, (readCode, dismissManual) =>
+    oauth.login({
+      notify: (event: AuthEvent): void => {
+        switch (event.type) {
+          case "auth_url":
+            authorizeUrl = event.url;
+            openBrowser(event.url);
+            if (event.instructions) ctx.ui.notify(event.instructions, "info");
+            break;
+          case "device_code":
+            openBrowser(event.verificationUri);
+            ctx.ui.notify(`Enter code ${event.userCode} at ${event.verificationUri}`, "info");
+            break;
+          case "progress":
+            // Progress follows receipt of an OAuth code, so dismiss the now-unneeded manual
+            // dialog before token exchange. The browser path may emit no progress; `finally`
+            // closes that dialog when the OAuth attempt settles.
+            dismissManual();
+            ctx.ui.notify(event.message, "info");
+            break;
+          case "info":
+            ctx.ui.notify(event.message, "info");
+            break;
+        }
+      },
+      prompt: async (prompt: AuthPrompt): Promise<string> => {
+        if (prompt.type === "manual_code") return readCode(manual.title(authorizeUrl), manual.placeholder);
+        if (prompt.type === "select") {
+          // Codex now opens its login with a browser-vs-device-code choice.
+          const labels = prompt.options.map((option) => option.label);
+          const chosen = await ctx.ui.select(prompt.message, labels);
+          const selected = chosen === undefined ? undefined : prompt.options[labels.indexOf(chosen)];
+          if (!selected) throw new Error("OAuth login cancelled");
+          return selected.id;
+        }
+        return (await ctx.ui.input(prompt.message, prompt.placeholder ?? "")) ?? "";
+      },
+    }),
+  );
+}
+
+export default async function (pi: ExtensionAPI): Promise<void> {
+  extensionAPI = pi;
+
+  pi.registerProvider(ANTHROPIC_PROVIDER_ID, {
+    api: "anthropic-messages",
+    streamSimple: streamManagedProvider,
+  });
+
+  pi.registerProvider(OPENAI_CODEX_PROVIDER_ID, {
+    api: "openai-codex-responses",
+    streamSimple: streamManagedProvider,
+  });
+
+  pi.on("session_start", async (_event, ctx) => {
+    runtimeContext = ctx;
+    const restored = readManagedProviderRotationPolicy(ctx.ui).allowCrossProviderFallback
+      && await restorePreferredModelIfPossible(pi, ctx);
+    if (restored && ctx.model) {
+      logRotation(`restored preferred ${ctx.model.provider}/${ctx.model.id}`);
+      setStatus(formatStatusLine(ctx.model.provider as ManagedProviderId));
+    }
+    updateRotationStatus(ctx);
+  });
+
+  pi.on("before_agent_start", async (_event, ctx) => {
+    runtimeContext = ctx;
+    const restored = readManagedProviderRotationPolicy(ctx.ui).allowCrossProviderFallback
+      && await restorePreferredModelIfPossible(pi, ctx);
+    if (restored && ctx.model) {
+      logRotation(`restored preferred ${ctx.model.provider}/${ctx.model.id}`);
+      setStatus(formatStatusLine(ctx.model.provider as ManagedProviderId));
+    }
+  });
+
+  pi.on("model_select", async (_event, ctx) => {
+    runtimeContext = ctx;
+    updateRotationStatus(ctx);
+  });
+
+  pi.registerCommand("provider-sub", {
+    description: "Manage subscription credential pools for provider rotation",
+    handler: async (args, ctx) => {
+      const [providerArg, verb, ...rest] = parseCommand(args);
+      const providerId = parseProvider(providerArg);
+      if (!providerId) {
+        ctx.ui.notify("Usage: /provider-sub <anthropic|openai-codex> <list|add|select|rm>", "error");
+        return;
+      }
+
+      const action = verb === "rm" ? "remove" : verb || "list";
+
+      if (action === "status" || action === "list") {
+        const pool = readSubscriptionPool(providerId);
+        const lines = [formatStatusLine(providerId), ""];
+        for (const [index, entry] of pool.entries()) {
+          const cooldown = entry.rateLimitedUntil > Date.now() ? ` cooldown=${Math.ceil((entry.rateLimitedUntil - Date.now()) / 1000)}s` : "";
+          const account = entry.accountId ? ` account=${shortAccountId(entry.accountId)}` : "";
+          lines.push(`${index + 1}. ${entry.label}${account}${cooldown}`);
+        }
+        ctx.ui.notify(lines.join("\n"), "info");
+        return;
+      }
+
+      if (action === "add") {
+        if (!ctx.hasUI) {
+          ctx.ui.notify(`${getProviderLabel(providerId)} login requires an interactive UI.`, "error");
+          return;
+        }
+        const pool = readSubscriptionPool(providerId);
+        const label = (rest.join(" ") || (await ctx.ui.input(`${getProviderLabel(providerId)} subscription label`, "")) || "").trim();
+        if (!label) return;
+        if (pool.some((entry) => entry.label.toLowerCase() === label.toLowerCase())) {
+          ctx.ui.notify(`${getProviderLabel(providerId)} subscription already exists: ${label}`, "error");
+          return;
+        }
+        try {
+          const cred = await activeLoginForProvider(providerId, ctx);
+          const identity = await enrichManagedLoginIdentity(providerId, cred);
+          // Explicit add always creates a new label, so a matching real account is rejected by
+          // the locked uniqueness gate rather than creating a second quota slot.
+          await addSubscription(providerId, {
+            label,
+            refresh: cred.refresh,
+            access: cred.access,
+            expires: cred.expires,
+            ...(identity.accountId ? { accountId: identity.accountId } : {}),
+            rateLimitedUntil: 0,
+            lastAttemptAt: 0,
+            lastRateLimitedAt: 0,
+          });
+          logRotation(`added ${getProviderLabel(providerId)} subscription "${label}"`);
+          ctx.ui.notify(`Added ${getProviderLabel(providerId)} subscription: ${label}`, "info");
+          updateRotationStatus(ctx);
+        } catch (error) {
+          ctx.ui.notify(error instanceof Error ? error.message : String(error), "error");
+        }
+        return;
+      }
+
+      if (action === "select") {
+        const ref = rest.join(" ");
+        if (!ref) {
+          ctx.ui.notify(`Usage: /provider-sub ${providerId} select <label|index>`, "error");
+          return;
+        }
+        try {
+          await promoteSubscription(providerId, ref);
+          logRotation(`selected ${getProviderLabel(providerId)} subscription "${ref}"`);
+          ctx.ui.notify(`Preferred ${getProviderLabel(providerId)} subscription: ${ref}`, "info");
+          updateRotationStatus(ctx);
+        } catch (error) {
+          ctx.ui.notify(error instanceof Error ? error.message : String(error), "error");
+        }
+        return;
+      }
+
+      if (action === "remove") {
+        const ref = rest.join(" ");
+        if (!ref) {
+          ctx.ui.notify(`Usage: /provider-sub ${providerId} rm <label|index>`, "error");
+          return;
+        }
+        try {
+          await removeManagedAccount(providerId, ref);
+          logRotation(`removed ${getProviderLabel(providerId)} subscription "${ref}"`);
+          ctx.ui.notify(`Removed ${getProviderLabel(providerId)} subscription: ${ref}`, "info");
+          updateRotationStatus(ctx);
+        } catch (error) {
+          ctx.ui.notify(error instanceof Error ? error.message : String(error), "error");
+        }
+        return;
+      }
+
+      ctx.ui.notify(`Unknown /provider-sub verb: ${verb}`, "error");
+    },
+  });
+}