codex-multi-auth 2.1.13-beta.1 → 2.2.0

package/dist/lib/runtime-rotation-proxy.js

@@ -1,21 +1,74 @@
-import { createHash, timingSafeEqual } from "node:crypto";
+import { createHash, randomUUID, timingSafeEqual } from "node:crypto";
 import { existsSync, readFileSync } from "node:fs";
 import { createServer } from "node:http";
 import { AccountManager, extractAccountId, } from "./accounts.js";
 import { getStoragePath } from "./storage.js";
-import { getFetchTimeoutMs, getNetworkErrorCooldownMs, getRetryAllAccountsMaxRetries, getServerErrorCooldownMs, getSessionAffinity, getSessionAffinityMaxEntries, getSessionAffinityTtlMs, getStreamStallTimeoutMs, getTokenRefreshSkewMs, loadPluginConfig, } from "./config.js";
+import { getFetchTimeoutMs, getNetworkErrorCooldownMs, getRetryAllAccountsMaxRetries, getServerErrorCooldownMs, getSessionAffinity, getSessionAffinityMaxEntries, getSessionAffinityTtlMs, getStreamStallTimeoutMs, getMinRotationIntervalMs, getTokenInvalidationCooldownMs, getTokenRefreshSkewMs, getPidOffsetEnabled, getRoutingMutexMode, loadPluginConfig, } from "./config.js";
 import { CODEX_BASE_URL, HTTP_STATUS, OPENAI_HEADERS, OPENAI_HEADER_VALUES, URL_PATHS, } from "./constants.js";
 import { getModelFamily } from "./prompts/codex.js";
 import { queuedRefresh } from "./refresh-queue.js";
 import { mutateRuntimeObservabilitySnapshot, recordRuntimePoolExhaustion, recordRuntimeReload, recordRuntimeReset, } from "./runtime/runtime-observability.js";
 import { createRuntimeUsageRecorder, evaluateRuntimePolicy, loadRuntimePolicyState, } from "./policy/runtime-policy.js";
 import { isWorkspaceDisabledError } from "./request/fetch-helpers.js";
+import { createLogger, maskString, runWithCorrelationId } from "./logger.js";
 import { SessionAffinityStore } from "./session-affinity.js";
 import { isRecord } from "./utils.js";
 const DEFAULT_HOST = "127.0.0.1";
+function isLoopbackHost(host) {
+    const normalized = host.trim().toLowerCase();
+    return (normalized === "127.0.0.1" ||
+        normalized === "localhost" ||
+        normalized === "::1" ||
+        normalized === "[::1]");
+}
+// IPv6 literals must be presented in two distinct forms and the proxy
+// previously conflated them (runtime-proxy IPv6 bug). Node's
+// net.Server.listen(port, host) requires the RAW literal ("::1"); a bracketed
+// literal ("[::1]") makes the bind fail or behave wrong. Conversely a URL
+// authority requires the BRACKETED literal ("[::1]") so "http://[::1]:port"
+// parses unambiguously — the raw form yields the unparseable "http://::1:port".
+// Normalize each form ONCE at startup so concurrent rotation paths never race
+// on inconsistent host string representations.
+function stripIpv6Brackets(host) {
+    const trimmed = host.trim();
+    if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
+        return trimmed.slice(1, -1);
+    }
+    return trimmed;
+}
+// Raw literal suitable for server.listen: "[::1]" -> "::1", others unchanged.
+function toBindHost(host) {
+    return stripIpv6Brackets(host);
+}
+// URL authority host: IPv6 literals are bracketed ("::1" -> "[::1]") while
+// IPv4 addresses and hostnames (no embedded colon) pass through unchanged.
+function toUrlHost(host) {
+    const bare = stripIpv6Brackets(host);
+    return bare.includes(":") ? `[${bare}]` : bare;
+}
+// Structured logger for the default-on runtime proxy (errors-logging-01,
+// runtime-proxy-04). Previously the 1900-LOC proxy had zero logger integration;
+// failures surfaced only as a last-write-wins status.lastError string. Logs are
+// level-gated and carry the per-request correlation id set in handleRequest.
+const proxyLog = createLogger("runtime-proxy");
 const DEFAULT_QUOTA_REMAINING_THRESHOLD = 10;
 const DEFAULT_AUTH_FAILURE_COOLDOWN_MS = 30_000;
 const DEFAULT_MAX_RUNTIME_ACCOUNT_ATTEMPTS = 4;
+// Phrases observed in upstream 401 response bodies when OpenAI/Microsoft has
+// explicitly revoked an OAuth token (as opposed to a generic expired-token 401
+// that can be retried after a refresh). Matching is case-insensitive substring.
+// If anti-abuse detection triggers different wording in production, add the new
+// phrase here and record the source provider and date. See issue #495.
+const TOKEN_INVALIDATION_PHRASES = [
+    "invalidated oauth token",
+    "authentication token has been invalidated",
+    "oauth token has been invalidated",
+    "token has been invalidated",
+];
+function isTokenInvalidationError(bodyText) {
+    const lower = bodyText.toLowerCase();
+    return TOKEN_INVALIDATION_PHRASES.some((phrase) => lower.includes(phrase));
+}
 const MAX_REQUEST_BODY_BYTES = 64 * 1024 * 1024;
 const MAX_THREAD_GOAL_FALLBACKS = 512;
 const HOP_BY_HOP_HEADERS = new Set([
@@ -90,6 +143,10 @@ function createOutboundHeaders(incoming, account, accessToken, accountId) {
     }
     headers.delete("host");
     headers.delete("x-api-key");
+    // Never forward inbound client credentials upstream: a Cookie / proxy-auth
+    // header would ride along with the managed OAuth Bearer to OpenAI.
+    headers.delete("cookie");
+    headers.delete("proxy-authorization");
     headers.set("authorization", `Bearer ${accessToken}`);
     headers.set(OPENAI_HEADERS.ACCOUNT_ID, accountId);
     headers.set(OPENAI_HEADERS.BETA, OPENAI_HEADER_VALUES.BETA_RESPONSES);
@@ -249,7 +306,12 @@ async function persistRuntimeActiveAccount(accountManager, account, family, isPi
         return;
     }
     try {
-        accountManager.markSwitched(account, "rotation", family);
+        // accounts-01/08: serialize the cursor mutation through the routing mutex
+        // (when routingMutex="enabled") because this commit spans an await
+        // (syncCodexCliActiveSelectionForIndex), which is the lost-update window the
+        // mutex exists to close. In legacy mode markSwitchedLocked runs inline, so
+        // behavior is unchanged by default.
+        await accountManager.markSwitchedLocked(account, "rotation", family);
         accountManager.saveToDiskDebounced();
         await accountManager.syncCodexCliActiveSelectionForIndex(account.index);
     }
@@ -424,6 +486,21 @@ function buildUpstreamUrl(req, upstreamBaseUrl, upstreamPath) {
     upstream.search = incomingUrl.search;
     return upstream.toString();
 }
+// Monotonic auth-failure cooldown: only extend, never shorten. Two concurrent
+// requests on the same account can race so that an invalidation path sets a
+// long cooldown (5 min) and a subsequent generic 401 truncates it (30 s).
+// Reading the live coolingDownUntil before writing prevents that race.
+function applyMonotonicAuthCooldown(accountManager, account, cooldownMs) {
+    const existing = accountManager.getAccountByIndex(account.index)?.coolingDownUntil ?? 0;
+    // Intentionally Date.now(), not the proxy's injected now(): coolingDownUntil is
+    // written by markAccountCoolingDown via nowMs() (== Date.now()), so both sides of
+    // this comparison must live in the same real-wall-clock domain. Switching to the
+    // injected now() would mis-compare an injected-clock value against a real-clock
+    // deadline and silently defeat the monotonic guard.
+    if (Date.now() + cooldownMs > existing) {
+        accountManager.markAccountCoolingDown(account, cooldownMs, "auth-failure");
+    }
+}
 function hasUsableAccessToken(account, now, skewMs) {
     return (typeof account.access === "string" &&
         account.access.trim().length > 0 &&
@@ -465,7 +542,7 @@ async function commitRefreshedAuthOnce(accountManager, account, auth) {
     return pending;
 }
 async function ensureFreshAccessToken(params) {
-    const { accountManager, account, family, model, now, tokenRefreshSkewMs } = params;
+    const { accountManager, account, family, model, now, tokenRefreshSkewMs, tokenInvalidationCooldownMs } = params;
     if (hasUsableAccessToken(account, now, tokenRefreshSkewMs)) {
         return { ok: true, accessToken: account.access ?? "", account };
     }
@@ -473,9 +550,14 @@ async function ensureFreshAccessToken(params) {
     if (refreshResult.type === "failed") {
         accountManager.recordFailure(account, family, model);
         accountManager.incrementAuthFailures(account);
-        accountManager.markAccountCoolingDown(account, DEFAULT_AUTH_FAILURE_COOLDOWN_MS, "auth-failure");
+        // If the refresh endpoint itself returns an explicit invalidation message
+        // (e.g. Microsoft/Outlook SSO revokes the refresh token server-side), apply
+        // the long cooldown and signal to the caller to stop rotating rather than
+        // presenting other accounts' tokens from the same IP.
+        const invalidated = isTokenInvalidationError(refreshResult.message ?? "");
+        applyMonotonicAuthCooldown(accountManager, account, invalidated ? tokenInvalidationCooldownMs : DEFAULT_AUTH_FAILURE_COOLDOWN_MS);
         accountManager.saveToDiskDebounced();
-        return { ok: false, retryable: isTokenRefreshRetryable(refreshResult) };
+        return { ok: false, retryable: isTokenRefreshRetryable(refreshResult), invalidated };
     }
     const auth = {
         type: "oauth",
@@ -548,14 +630,96 @@ function parseRetryAfterBodyMs(bodyText, now) {
     }
     return null;
 }
-async function readErrorBody(response) {
+async function readErrorBody(response, timeoutMs, maxBytes = 1024 * 1024) {
+    // The outbound fetch's abort timer is cleared once headers arrive, so a
+    // stalled error body would otherwise hang this handler forever (the success
+    // path is per-chunk stall-bounded; the error path was not). Read the body via
+    // a reader, bound it by an idle timeout AND a size cap, and cancel the stream
+    // on timeout/overflow so the socket is released.
+    const body = response.body;
+    if (!body || typeof body.getReader !== "function") {
+        // Fallback for impls without a streamable body: race text() against a timer.
+        try {
+            return await withTimeout(response.text(), timeoutMs, () => undefined, "error body stalled");
+        }
+        catch {
+            return "";
+        }
+    }
+    const reader = body.getReader();
+    const chunks = [];
+    let total = 0;
     try {
-        return await response.text();
+        for (;;) {
+            let idleTimer;
+            const idle = new Promise((_resolve, reject) => {
+                idleTimer = setTimeout(() => reject(new Error("error body stalled")), Math.max(1, timeoutMs));
+            });
+            let result;
+            try {
+                result = await Promise.race([reader.read(), idle]);
+            }
+            finally {
+                if (idleTimer)
+                    clearTimeout(idleTimer);
+            }
+            if (result.done)
+                break;
+            if (result.value) {
+                total += result.value.byteLength;
+                if (total > maxBytes)
+                    break; // cap: enough for diagnostics, no OOM
+                chunks.push(result.value);
+            }
+        }
+    }
+    catch {
+        // stalled or errored — fall through with whatever we have
+    }
+    finally {
+        await reader.cancel().catch(() => undefined);
+    }
+    try {
+        return Buffer.concat(chunks).toString("utf8");
     }
     catch {
         return "";
     }
 }
+const TOKEN_INVALIDATED_CODE = "token_invalidated";
+const TOKEN_INVALIDATED_FALLBACK_MESSAGE = "OAuth token has been invalidated. Please re-login.";
+// Both invalidation exit paths (refresh-failure and upstream-401) must hand the
+// client the same machine-readable shape — { error: { message, code:
+// "token_invalidated" } } — so a consumer keying off error.code behaves
+// identically regardless of which vector fired. The upstream forwards a raw body
+// with no guaranteed code, so we wrap it here while preserving its human-readable
+// message when one is present.
+export function buildTokenInvalidationBody(upstreamBodyText) {
+    let message = TOKEN_INVALIDATED_FALLBACK_MESSAGE;
+    const trimmed = upstreamBodyText.trim();
+    if (trimmed) {
+        try {
+            const parsed = JSON.parse(trimmed);
+            if (isRecord(parsed)) {
+                const direct = parsed.message;
+                if (typeof direct === "string" && direct.trim()) {
+                    message = direct.trim();
+                }
+                else if (isRecord(parsed.error)) {
+                    const nested = parsed.error.message;
+                    if (typeof nested === "string" && nested.trim()) {
+                        message = nested.trim();
+                    }
+                }
+            }
+        }
+        catch {
+            // Non-JSON upstream body (e.g. HTML error page): keep the stable fallback
+            // message rather than echoing markup back to the client.
+        }
+    }
+    return JSON.stringify({ error: { message, code: TOKEN_INVALIDATED_CODE } });
+}
 function extractErrorCodeFromBody(bodyText) {
     if (!bodyText.trim())
         return null;
@@ -616,7 +780,7 @@ function getQuotaNearExhaustionWaitMs(headers, remainingThreshold, now) {
     return candidates.length > 0 ? Math.max(...candidates) : 0;
 }
 export function chooseAccount(params) {
-    const { accountManager, sessionAffinityStore, sessionKey, family, model, attemptedIndexes, now, policy, pinnedIndex, skipReasons, } = params;
+    const { accountManager, sessionAffinityStore, sessionKey, family, model, attemptedIndexes, now, policy, pinnedIndex, skipReasons, stickyBoostByAccount, pidOffsetEnabled, } = params;
     // Manual pin (from `codex-multi-auth switch <n>`) overrides every other
     // selection signal. We do NOT call markSwitched here — the proxy must not
     // clobber the pin set by the CLI. See issue #474.
@@ -662,7 +826,14 @@ export function chooseAccount(params) {
         }
     }
     const selected = accountManager.getCurrentOrNextForFamilyHybrid(family, model, {
-        scoreBoostByAccount: policy?.scoreBoostByAccount,
+        scoreBoostByAccount: {
+            ...(policy?.scoreBoostByAccount ?? {}),
+            ...(stickyBoostByAccount ?? {}),
+        },
+        // accounts-05: carry the PID-offset distribution into the default-on proxy
+        // path too (index.ts already does). Without it, parallel proxy processes can
+        // stampede the same account instead of spreading across the pool.
+        pidOffsetEnabled,
     });
     if (selected &&
         !attemptedIndexes.has(selected.index) &&
@@ -816,8 +987,27 @@ export async function startRuntimeRotationProxy(options) {
     const pluginConfig = loadPluginConfig();
     let activeAccountManager = options.accountManager ?? (await AccountManager.loadFromDisk());
     const knownAccountManagers = new Set([activeAccountManager]);
+    // accounts-01/08: apply the configured routing-mutex mode so the proxy's
+    // async select->commit path (persistRuntimeActiveAccount) can serialize cursor
+    // mutations when routingMutex="enabled". Legacy mode keeps the inline fast path.
+    const routingMutexMode = getRoutingMutexMode(pluginConfig);
+    activeAccountManager.setRoutingMutexMode(routingMutexMode);
     const fetchImpl = options.fetchImpl ?? fetch;
     const host = options.host ?? DEFAULT_HOST;
+    // Defense in depth (runtime-proxy-01): the proxy presents managed OAuth tokens
+    // and must never be reachable off-box. It is loopback-only with NO opt-out —
+    // binding a non-loopback host would expose every managed account to the
+    // network, so it is refused unconditionally.
+    if (!isLoopbackHost(host)) {
+        throw new Error(`Runtime rotation proxy refuses to bind non-loopback host "${host}". ` +
+            "It forwards managed OAuth tokens and is loopback-only.");
+    }
+    // Normalize the validated host into its two representations exactly once so the
+    // listen() bind and the emitted baseUrl can never disagree under concurrent
+    // rotation: bindHost is the raw literal Node's listen() expects ("[::1]"->"::1"),
+    // urlHost is the bracketed form a URL authority requires ("::1"->"[::1]").
+    const bindHost = toBindHost(host);
+    const urlHost = toUrlHost(host);
     const port = options.port ?? 0;
     const upstreamBaseUrl = options.upstreamBaseUrl ?? CODEX_BASE_URL;
     const clientApiKey = typeof options.clientApiKey === "string" &&
@@ -831,6 +1021,11 @@ export async function startRuntimeRotationProxy(options) {
     const tokenRefreshSkewMs = getTokenRefreshSkewMs(pluginConfig);
     const networkErrorCooldownMs = getNetworkErrorCooldownMs(pluginConfig);
     const serverErrorCooldownMs = getServerErrorCooldownMs(pluginConfig);
+    const tokenInvalidationCooldownMs = getTokenInvalidationCooldownMs(pluginConfig);
+    const minRotationIntervalMs = getMinRotationIntervalMs(pluginConfig);
+    const pidOffsetEnabled = getPidOffsetEnabled(pluginConfig);
+    let lastGlobalAccountIndex = null;
+    let lastGlobalSwitchAt = 0;
     const fetchTimeoutMs = options.fetchTimeoutMs ?? getFetchTimeoutMs(pluginConfig);
     const streamStallTimeoutMs = options.streamStallTimeoutMs ?? getStreamStallTimeoutMs(pluginConfig);
     const configuredMaxRetries = getRetryAllAccountsMaxRetries(pluginConfig);
@@ -877,6 +1072,7 @@ export async function startRuntimeRotationProxy(options) {
             AccountManager.resetVolatileRuntimeState();
             recordRuntimeReset("pool-exhausted-no-account");
             const reloaded = await AccountManager.loadFromDisk();
+            reloaded.setRoutingMutexMode(routingMutexMode);
             activeAccountManager = reloaded;
             knownAccountManagers.add(reloaded);
             lastStaleRuntimeReloadAt = Date.now();
@@ -893,6 +1089,13 @@ export async function startRuntimeRotationProxy(options) {
         return staleRuntimeReloadPromise;
     };
     const handleRequest = async (req, res) => {
+        // Per-request trace id (errors-logging-03): distinct from sessionKey, which
+        // is shared across a thread's requests. Bound to this request's async context
+        // so every proxyLog line and usage row can be correlated to one request.
+        const traceId = randomUUID();
+        return runWithCorrelationId(traceId, () => handleRequestInner(req, res, traceId));
+    };
+    const handleRequestInner = async (req, res, traceId) => {
         let usageRecorder = null;
         let accountManager = activeAccountManager;
         try {
@@ -952,7 +1155,7 @@ export async function startRuntimeRotationProxy(options) {
                         : "responses",
                 model: context.model,
                 projectKey,
-                requestId: context.sessionKey,
+                requestId: traceId,
                 startedAt: requestStartedAt,
             });
             if (policyError) {
@@ -1010,6 +1213,11 @@ export async function startRuntimeRotationProxy(options) {
             }
             while (attemptedIndexes.size < accountCount &&
                 transientAttempts < transientAttemptLimit) {
+                const rotationStickyBoost = minRotationIntervalMs > 0 &&
+                    lastGlobalAccountIndex !== null &&
+                    now() - lastGlobalSwitchAt < minRotationIntervalMs
+                    ? { [lastGlobalAccountIndex]: 1000 }
+                    : {};
                 const selected = chooseAccount({
                     accountManager,
                     sessionAffinityStore,
@@ -1021,6 +1229,8 @@ export async function startRuntimeRotationProxy(options) {
                     policy: policyDecision,
                     pinnedIndex,
                     skipReasons: accountSkipReasons,
+                    stickyBoostByAccount: rotationStickyBoost,
+                    pidOffsetEnabled,
                 });
                 if (!selected) {
                     if (!reloadedAfterNoAccount &&
@@ -1057,10 +1267,28 @@ export async function startRuntimeRotationProxy(options) {
                     model: context.model,
                     now: now(),
                     tokenRefreshSkewMs,
+                    tokenInvalidationCooldownMs,
                 });
                 if (!refreshed.ok) {
                     accountManager.refundToken(selected, context.family, context.model);
                     exhaustionReason = "auth-failure";
+                    if (refreshed.invalidated) {
+                        // Refresh endpoint explicitly revoked the token. Stop cascade:
+                        // return auth error to client instead of rotating to the next account.
+                        sessionAffinityStore?.forgetSession(context.sessionKey);
+                        res.writeHead(HTTP_STATUS.UNAUTHORIZED, { "content-type": "application/json" });
+                        // Route through the shared builder so both invalidation exit paths stay
+                        // in lockstep — empty input yields { error: { message: <fallback>,
+                        // code: "token_invalidated" } }.
+                        res.end(buildTokenInvalidationBody(""));
+                        await usageRecorder.record({
+                            outcome: "failure",
+                            statusCode: HTTP_STATUS.UNAUTHORIZED,
+                            errorCode: "token_invalidated",
+                            account: selected,
+                        });
+                        return;
+                    }
                     if (!refreshed.retryable)
                         continue;
                     transientAttempts += 1;
@@ -1112,7 +1340,7 @@ export async function startRuntimeRotationProxy(options) {
                     continue;
                 }
                 if (upstream.status === HTTP_STATUS.TOO_MANY_REQUESTS) {
-                    const bodyText = await readErrorBody(upstream);
+                    const bodyText = await readErrorBody(upstream, streamStallTimeoutMs);
                     const retryAfterMs = parseRetryAfterHeaderMs(upstream.headers, now()) ??
                         parseRetryAfterBodyMs(bodyText, now()) ??
                         60_000;
@@ -1129,7 +1357,7 @@ export async function startRuntimeRotationProxy(options) {
                     continue;
                 }
                 if (upstream.status === 402 || upstream.status === HTTP_STATUS.FORBIDDEN) {
-                    const bodyText = await readErrorBody(upstream);
+                    const bodyText = await readErrorBody(upstream, streamStallTimeoutMs);
                     const errorCode = extractErrorCodeFromBody(bodyText);
                     if (isWorkspaceDisabledError(upstream.status, errorCode, bodyText)) {
                         const accountWasEnabled = accountManager.getAccountByIndex(refreshed.account.index)?.enabled !==
@@ -1212,10 +1440,34 @@ export async function startRuntimeRotationProxy(options) {
                     return;
                 }
                 if (upstream.status === HTTP_STATUS.UNAUTHORIZED) {
-                    await readErrorBody(upstream);
+                    const bodyText = await readErrorBody(upstream, streamStallTimeoutMs);
                     accountManager.refundToken(refreshed.account, context.family, context.model);
                     accountManager.recordFailure(refreshed.account, context.family, context.model);
-                    accountManager.markAccountCoolingDown(refreshed.account, DEFAULT_AUTH_FAILURE_COOLDOWN_MS, "auth-failure");
+                    if (isTokenInvalidationError(bodyText)) {
+                        // The upstream explicitly revoked this OAuth token. Applying a long
+                        // cooldown prevents cascade invalidation: rapidly presenting each
+                        // account's token from the same IP triggers OpenAI's anti-abuse
+                        // detection and invalidates them in sequence. Return the 401 directly
+                        // rather than rotating so the client can prompt for re-login.
+                        applyMonotonicAuthCooldown(accountManager, refreshed.account, tokenInvalidationCooldownMs);
+                        sessionAffinityStore?.forgetSession(context.sessionKey);
+                        accountManager.saveToDiskDebounced();
+                        // Emit the same machine-readable shape as the refresh-failure path
+                        // (code: "token_invalidated") instead of forwarding the raw upstream
+                        // body, so the client contract is consistent across both vectors.
+                        const clientHeaders = responseHeadersForClient(upstream.headers);
+                        clientHeaders["content-type"] = "application/json";
+                        res.writeHead(upstream.status, clientHeaders);
+                        res.end(buildTokenInvalidationBody(bodyText));
+                        await usageRecorder.record({
+                            outcome: "failure",
+                            statusCode: upstream.status,
+                            errorCode: "token_invalidated",
+                            account: refreshed.account,
+                        });
+                        return;
+                    }
+                    applyMonotonicAuthCooldown(accountManager, refreshed.account, DEFAULT_AUTH_FAILURE_COOLDOWN_MS);
                     accountManager.saveToDiskDebounced();
                     exhaustionReason = "auth-failure";
                     transientAttempts += 1;
@@ -1225,7 +1477,7 @@ export async function startRuntimeRotationProxy(options) {
                     continue;
                 }
                 if (upstream.status >= 500) {
-                    await readErrorBody(upstream);
+                    await readErrorBody(upstream, streamStallTimeoutMs);
                     accountManager.refundToken(refreshed.account, context.family, context.model);
                     accountManager.recordFailure(refreshed.account, context.family, context.model);
                     accountManager.markAccountCoolingDown(refreshed.account, serverErrorCooldownMs, "server-error");
@@ -1266,6 +1518,10 @@ export async function startRuntimeRotationProxy(options) {
                 }
                 else {
                     sessionAffinityStore?.remember(context.sessionKey, refreshed.account.index, now());
+                    if (refreshed.account.index !== lastGlobalAccountIndex) {
+                        lastGlobalAccountIndex = refreshed.account.index;
+                    }
+                    lastGlobalSwitchAt = now();
                 }
                 await persistRuntimeActiveAccount(accountManager, refreshed.account, context.family, isPinned && refreshed.account.index === pinnedIndex);
                 const forwarded = await forwardStreamingResponse(upstream, res, status, () => {
@@ -1334,7 +1590,20 @@ export async function startRuntimeRotationProxy(options) {
             }
         }
         catch (error) {
-            status.lastError = error instanceof Error ? error.message : String(error);
+            const rawErrorMessage = error instanceof Error ? error.message : String(error);
+            // errors-logging-08: redact any email/token material that leaked into a
+            // raw upstream or refresh error string before it reaches status consumers
+            // or the structured log. maskString is a no-op for clean diagnostic text.
+            const maskedErrorMessage = maskString(rawErrorMessage);
+            status.lastError = maskedErrorMessage;
+            // errors-logging-01: surface the failure through the structured logger
+            // (redaction-safe) with the request trace id, instead of only stashing a
+            // last-write-wins status string.
+            proxyLog.error("runtime proxy request failed", {
+                traceId,
+                code: isRuntimeProxyHttpError(error) ? error.code : "codex_runtime_rotation_proxy_error",
+                error: maskedErrorMessage,
+            });
             if (!res.headersSent) {
                 if (isRuntimeProxyHttpError(error)) {
                     await usageRecorder?.record({
@@ -1391,20 +1660,26 @@ export async function startRuntimeRotationProxy(options) {
         };
         server.once("error", onError);
         server.once("listening", onListening);
-        server.listen(port, host);
+        server.listen(port, bindHost);
     });
     server.on("error", onPostStartupServerError);
     const address = server.address();
     const resolvedPort = typeof address === "object" && address ? address.port : port;
     return {
-        host,
+        host: bindHost,
         port: resolvedPort,
-        baseUrl: `http://${host}:${resolvedPort}`,
+        baseUrl: `http://${urlHost}:${resolvedPort}`,
         close: async () => {
             await closeServer(server, sockets);
             await activeAccountManager.flushPendingSave();
         },
-        getStatus: () => ({ ...status }),
+        getStatus: () => ({
+            ...status,
+            // Redact any email/token material that leaked into a raw upstream or
+            // refresh error string before exposing it to status/report consumers
+            // (errors-logging-08). maskString is a no-op for clean diagnostic text.
+            lastError: status.lastError === null ? null : maskString(status.lastError),
+        }),
     };
 }
 async function closeServer(server, sockets) {