@botcord/daemon 0.2.75 → 0.2.76

package/dist/gateway/dispatcher.js

@@ -1,8 +1,11 @@
 import { randomUUID } from "node:crypto";
+import { looksLikeRuntimeAuthFailure } from "./runtime-errors.js";
 import { resolveRoute } from "./router.js";
 import { sessionKey } from "./session-store.js";
 import { truncateTextField, } from "./transcript.js";
 const DEFAULT_TURN_TIMEOUT_MS = 30 * 60 * 1000;
+const DEFAULT_RUNTIME_AUTH_FAILURE_THRESHOLD = 3;
+const DEFAULT_RUNTIME_AUTH_FAILURE_COOLDOWN_MS = 10 * 60 * 1000;
 /**
  * Owner-chat room prefix. Reply-text gating: only rooms with this prefix get
  * `result.text` forwarded to the channel; in every other room the runtime's
@@ -101,6 +104,26 @@ function redactSecretString(value) {
         .replace(/\b(token=)[^\s"']+/gi, "$1[REDACTED]")
         .replace(/\b(drt_|dit_|gho_)[A-Za-z0-9_-]+/g, "$1[REDACTED]");
 }
+function extractCloudRunBudget(msg) {
+    const envelope = msg.raw?.envelope;
+    if (envelope?.type !== "cloud_run")
+        return undefined;
+    const budget = envelope.payload?.cloud_run?.budget;
+    if (!budget)
+        return undefined;
+    const out = {};
+    if (typeof budget.max_wall_time_seconds === "number" &&
+        Number.isFinite(budget.max_wall_time_seconds) &&
+        budget.max_wall_time_seconds > 0) {
+        out.maxWallTimeMs = Math.floor(budget.max_wall_time_seconds * 1000);
+    }
+    if (typeof budget.max_tool_calls === "number" &&
+        Number.isFinite(budget.max_tool_calls) &&
+        budget.max_tool_calls > 0) {
+        out.maxToolCalls = Math.floor(budget.max_tool_calls);
+    }
+    return out.maxWallTimeMs !== undefined || out.maxToolCalls !== undefined ? out : undefined;
+}
 /**
  * Reason carried on `AbortController.abort()` when a cancel-previous wave
  * is taking over the slot. Distinguishing this from a timeout abort lets
@@ -137,10 +160,14 @@ export class Dispatcher {
     sessionStore;
     log;
     turnTimeoutMs;
+    runtimeAuthFailureThreshold;
+    runtimeAuthFailureCooldownMs;
     buildSystemContext;
     buildMemoryContext;
     onInbound;
     onOutbound;
+    onTurnComplete;
+    onRuntimeCircuitBreakerChange;
     composeUserTurn;
     managedRoutes;
     attentionGate;
@@ -148,6 +175,7 @@ export class Dispatcher {
     transcript;
     queues = new Map();
     deferredMultimodal = new Map();
+    runtimeAuthFailures = new Map();
     /**
      * Last `/hub/typing` ping timestamp per (accountId, conversationId).
      * Used to debounce cancel-previous bursts so we don't trip Hub's 20/min
@@ -161,10 +189,16 @@ export class Dispatcher {
         this.sessionStore = opts.sessionStore;
         this.log = opts.log;
         this.turnTimeoutMs = opts.turnTimeoutMs ?? DEFAULT_TURN_TIMEOUT_MS;
+        this.runtimeAuthFailureThreshold =
+            opts.runtimeAuthFailureThreshold ?? DEFAULT_RUNTIME_AUTH_FAILURE_THRESHOLD;
+        this.runtimeAuthFailureCooldownMs =
+            opts.runtimeAuthFailureCooldownMs ?? DEFAULT_RUNTIME_AUTH_FAILURE_COOLDOWN_MS;
         this.buildSystemContext = opts.buildSystemContext;
         this.buildMemoryContext = opts.buildMemoryContext;
         this.onInbound = opts.onInbound;
         this.onOutbound = opts.onOutbound;
+        this.onTurnComplete = opts.onTurnComplete;
+        this.onRuntimeCircuitBreakerChange = opts.onRuntimeCircuitBreakerChange;
         this.composeUserTurn = opts.composeUserTurn;
         this.managedRoutes = opts.managedRoutes;
         this.attentionGate = opts.attentionGate;
@@ -368,6 +402,11 @@ export class Dispatcher {
                 fallback: "raw_text",
             });
         }
+        const openAuthBreaker = this.openRuntimeAuthBreaker(dispatchRoute, dispatchMsg);
+        if (openAuthBreaker) {
+            await this.skipRuntimeForAuthBreaker(openAuthBreaker, dispatchRoute, dispatchMsg, dispatchChannel, dispatchTurnId);
+            return;
+        }
         if (mode === "cancel-previous") {
             await this.runCancelPrevious(queueKey, dispatchRoute, text, dispatchMsg, dispatchChannel, dispatchTurnId, mergedFromDeferredTurnIds);
         }
@@ -384,6 +423,15 @@ export class Dispatcher {
         }
         return out;
     }
+    runtimeCircuitBreakers() {
+        this.pruneExpiredRuntimeAuthBreakers();
+        const out = {};
+        for (const [key, state] of this.runtimeAuthFailures) {
+            if (state.blockedUntil > Date.now())
+                out[key] = { ...state };
+        }
+        return out;
+    }
     // ---------------------------------------------------------------------------
     // Internals
     // ---------------------------------------------------------------------------
@@ -444,6 +492,147 @@ export class Dispatcher {
         this.deferredMultimodal.delete(queueKey);
         return list;
     }
+    runtimeAuthBreakerKey(route, msg) {
+        const thread = msg.conversation.threadId ?? "";
+        return `${route.runtime}:${msg.channel}:${msg.accountId}:${msg.conversation.id}:${thread}`;
+    }
+    openRuntimeAuthBreaker(route, msg) {
+        const key = this.runtimeAuthBreakerKey(route, msg);
+        const state = this.runtimeAuthFailures.get(key);
+        if (!state)
+            return null;
+        if (state.blockedUntil > 0 && state.blockedUntil <= Date.now()) {
+            this.runtimeAuthFailures.delete(key);
+            return null;
+        }
+        return state.blockedUntil > Date.now() ? state : null;
+    }
+    pruneExpiredRuntimeAuthBreakers() {
+        const now = Date.now();
+        for (const [key, state] of this.runtimeAuthFailures) {
+            if (state.blockedUntil > 0 && state.blockedUntil <= now)
+                this.runtimeAuthFailures.delete(key);
+        }
+    }
+    recordRuntimeAuthFailure(route, msg, error) {
+        const now = Date.now();
+        const key = this.runtimeAuthBreakerKey(route, msg);
+        const prev = this.runtimeAuthFailures.get(key);
+        const failures = (prev?.failures ?? 0) + 1;
+        const openedAt = prev?.openedAt ?? now;
+        const state = {
+            key,
+            runtime: route.runtime,
+            channel: msg.channel,
+            accountId: msg.accountId,
+            conversationId: msg.conversation.id,
+            threadId: msg.conversation.threadId ?? null,
+            failures,
+            openedAt,
+            blockedUntil: failures >= this.runtimeAuthFailureThreshold
+                ? now + this.runtimeAuthFailureCooldownMs
+                : 0,
+            lastFailureAt: now,
+            lastError: error,
+        };
+        this.runtimeAuthFailures.set(key, state);
+        if (state.blockedUntil > now) {
+            this.log.error("dispatcher: runtime auth circuit breaker opened", {
+                key,
+                runtime: route.runtime,
+                agentId: msg.accountId,
+                roomId: msg.conversation.id,
+                topicId: msg.conversation.threadId ?? null,
+                failures,
+                blockedUntil: state.blockedUntil,
+                error,
+            });
+            this.notifyRuntimeCircuitBreakerChange();
+            return state;
+        }
+        this.log.warn("dispatcher: runtime authentication failure recorded", {
+            key,
+            runtime: route.runtime,
+            agentId: msg.accountId,
+            roomId: msg.conversation.id,
+            topicId: msg.conversation.threadId ?? null,
+            failures,
+            threshold: this.runtimeAuthFailureThreshold,
+            error,
+        });
+        return null;
+    }
+    clearRuntimeAuthFailures(route, msg) {
+        const key = this.runtimeAuthBreakerKey(route, msg);
+        if (!this.runtimeAuthFailures.delete(key))
+            return;
+        this.log.info("dispatcher: runtime auth circuit breaker cleared", {
+            key,
+            runtime: route.runtime,
+            agentId: msg.accountId,
+            roomId: msg.conversation.id,
+            topicId: msg.conversation.threadId ?? null,
+        });
+        this.notifyRuntimeCircuitBreakerChange();
+    }
+    notifyRuntimeCircuitBreakerChange() {
+        try {
+            this.onRuntimeCircuitBreakerChange?.();
+        }
+        catch (err) {
+            this.log.warn("dispatcher: onRuntimeCircuitBreakerChange threw", {
+                error: err instanceof Error ? err.message : String(err),
+            });
+        }
+    }
+    async skipRuntimeForAuthBreaker(state, route, msg, channel, turnId) {
+        const error = `runtime authentication failed repeatedly; dispatch paused until ${new Date(state.blockedUntil).toISOString()}`;
+        this.log.warn("dispatcher: runtime auth circuit breaker blocking turn", {
+            key: state.key,
+            runtime: route.runtime,
+            agentId: msg.accountId,
+            roomId: msg.conversation.id,
+            topicId: msg.conversation.threadId ?? null,
+            turnId,
+            blockedUntil: state.blockedUntil,
+        });
+        this.transcript.write({
+            ts: nowIso(),
+            kind: "turn_error",
+            turnId,
+            agentId: msg.accountId,
+            roomId: msg.conversation.id,
+            topicId: msg.conversation.threadId ?? null,
+            phase: "runtime",
+            error,
+            durationMs: 0,
+        });
+        const canDeliverRuntimeText = isOwnerChatRoom(msg) || !isBotCordChannel(channel);
+        const canDeliverRuntimeDiagnostics = canDeliverRuntimeText || isBotCordChannel(channel);
+        if (canDeliverRuntimeDiagnostics) {
+            const sendResult = await this.sendReply(channel, {
+                channel: msg.channel,
+                accountId: msg.accountId,
+                conversationId: msg.conversation.id,
+                threadId: msg.conversation.threadId ?? null,
+                type: "error",
+                text: `⚠️ Runtime error: ${truncate(error, 500)}`,
+                replyTo: this.providerReplyTo(msg),
+                traceId: msg.trace?.id ?? null,
+            }, turnId);
+            this.emitOutbound({
+                turnId,
+                msg,
+                runtime: route.runtime,
+                runtimeSessionId: null,
+                startedAt: Date.now(),
+                finalText: truncateTextField(""),
+                deliveryStatus: sendResult.ok ? "delivered" : "send_failed",
+                deliveryReason: sendResult.ok ? null : sendResult.error,
+                blocks: [],
+            });
+        }
+    }
     async runCancelPrevious(queueKey, route, text, msg, channel, turnId, mergedFromTurnIds = []) {
         const q = this.getQueue(queueKey);
         // Bump the generation on every arrival. Older arrivals still awaiting
@@ -703,6 +892,7 @@ export class Dispatcher {
             turnId,
             controller,
             timedOut: false,
+            budgetExceeded: null,
             snapshot,
             done,
             dispatchedAt: startedAt,
@@ -738,6 +928,9 @@ export class Dispatcher {
             ...(mergedFromTurnIds.length > 0 ? { mergedFromTurns: mergedFromTurnIds.length } : {}),
             composedPreview: logPreview(text),
         });
+        const cloudRunBudget = extractCloudRunBudget(msg);
+        const effectiveTurnTimeoutMs = Math.min(this.turnTimeoutMs, cloudRunBudget?.maxWallTimeMs ?? this.turnTimeoutMs);
+        let observedToolCalls = 0;
         // Hard-cap turn with a timeout.
         const timer = setTimeout(() => {
             slot.timedOut = true;
@@ -747,10 +940,10 @@ export class Dispatcher {
                 topicId: msg.conversation.threadId ?? null,
                 turnId,
                 queueKey,
-                timeoutMs: this.turnTimeoutMs,
+                timeoutMs: effectiveTurnTimeoutMs,
             });
             controller.abort();
-        }, this.turnTimeoutMs);
+        }, effectiveTurnTimeoutMs);
         if (typeof timer.unref === "function")
             timer.unref();
         const key = sessionKey({
@@ -773,6 +966,22 @@ export class Dispatcher {
             (streamable || !isBotCordChannel(channel));
         const canStream = streamable && typeof traceId === "string" && typeof channel.streamBlock === "function";
         const recordBlock = (block) => {
+            if (block.kind === "tool_use" && cloudRunBudget?.maxToolCalls !== undefined) {
+                observedToolCalls += 1;
+                if (observedToolCalls > cloudRunBudget.maxToolCalls && !controller.signal.aborted) {
+                    slot.budgetExceeded = `tool call budget exceeded after ${observedToolCalls} tool call(s)`;
+                    this.log.warn("dispatcher: cloud_run tool budget exceeded", {
+                        agentId: msg.accountId,
+                        roomId: msg.conversation.id,
+                        topicId: msg.conversation.threadId ?? null,
+                        turnId,
+                        queueKey,
+                        maxToolCalls: cloudRunBudget.maxToolCalls,
+                        observedToolCalls,
+                    });
+                    controller.abort(new Error(slot.budgetExceeded));
+                }
+            }
             const summary = summarizeStreamBlock(block);
             slot.blocks.push(summary);
             if (this.transcript.enabled) {
@@ -957,7 +1166,8 @@ export class Dispatcher {
                 sendThinkingMarker(event.phase, event.label, "runtime");
             }
             : undefined;
-        const onBlock = (canStream || this.transcript.enabled)
+        const shouldObserveBlocks = canStream || this.transcript.enabled || cloudRunBudget?.maxToolCalls !== undefined;
+        const onBlock = shouldObserveBlocks
             ? (block) => {
                 // Always record adapter-emitted blocks for transcript fidelity, even
                 // after abort — the transcript reflects what the runtime emitted,
@@ -1059,6 +1269,7 @@ export class Dispatcher {
         const runtime = this.runtimeFactory(route.runtime, route.extraArgs);
         let result;
         let threw;
+        const turnStartedAt = Date.now();
         try {
             try {
                 result = await runtime.run({
@@ -1081,6 +1292,7 @@ export class Dispatcher {
                         channel: msg.channel,
                         conversationKind: msg.conversation.kind,
                     },
+                    ...(cloudRunBudget ? { budget: cloudRunBudget } : {}),
                     gateway: route.gateway,
                     ...(route.hermesProfile ? { hermesProfile: route.hermesProfile } : {}),
                 });
@@ -1091,6 +1303,26 @@ export class Dispatcher {
             finally {
                 clearTimeout(timer);
             }
+            // Fire onTurnComplete observer. Cloud daemon hooks this to settle
+            // ``cloud_run`` envelopes against the Hub usage ledger. Errors are
+            // swallowed so settle failures never break the reply path.
+            if (this.onTurnComplete) {
+                const wallTimeMs = Date.now() - turnStartedAt;
+                try {
+                    await this.onTurnComplete({
+                        message: msg,
+                        result,
+                        wallTimeMs,
+                        ...(threw !== undefined ? { error: threw } : {}),
+                    });
+                }
+                catch (hookErr) {
+                    this.log.warn("dispatcher: onTurnComplete threw — continuing", {
+                        error: hookErr instanceof Error ? hookErr.message : String(hookErr),
+                        messageId: msg.id,
+                    });
+                }
+            }
             // Re-check the abort signal AFTER runtime.run resolves but BEFORE any
             // side effects (session write, reply send). This closes the race where
             // a cancel-previous arrives between runtime.run resolving and the
@@ -1103,7 +1335,7 @@ export class Dispatcher {
             // record from `runCancelPrevious` BEFORE aborting, so we MUST NOT also
             // emit a `turn_error` here — that would violate the "exactly one
             // terminal record per turnId" invariant.
-            if (controller.signal.aborted && !slot.timedOut) {
+            if (controller.signal.aborted && !slot.timedOut && !slot.budgetExceeded) {
                 return;
             }
             // Reply gating: BotCord network rooms only accept the runtime's plain
@@ -1126,7 +1358,9 @@ export class Dispatcher {
             const isOwnerChat = isOwnerChatRoom(msg);
             const canDeliverRuntimeText = isOwnerChat || !isBotCordChannel(channel);
             const canDeliverRuntimeDiagnostics = canDeliverRuntimeText || isBotCordChannel(channel);
-            if (slot.timedOut) {
+            if (slot.timedOut || slot.budgetExceeded) {
+                const phase = slot.budgetExceeded ? "budget" : "timeout";
+                const error = slot.budgetExceeded ?? `runtime timeout after ${effectiveTurnTimeoutMs}ms`;
                 this.transcript.write({
                     ts: nowIso(),
                     kind: "turn_error",
@@ -1134,8 +1368,8 @@ export class Dispatcher {
                     agentId: msg.accountId,
                     roomId: msg.conversation.id,
                     topicId: msg.conversation.threadId ?? null,
-                    phase: "timeout",
-                    error: `runtime timeout after ${this.turnTimeoutMs}ms`,
+                    phase,
+                    error,
                     durationMs: Date.now() - slot.dispatchedAt,
                 });
                 if (canDeliverRuntimeDiagnostics) {
@@ -1145,7 +1379,9 @@ export class Dispatcher {
                         conversationId: msg.conversation.id,
                         threadId: msg.conversation.threadId ?? null,
                         type: "error",
-                        text: `⚠️ Runtime timeout after ${Math.round(this.turnTimeoutMs / 60000)} minute(s); aborted`,
+                        text: slot.budgetExceeded
+                            ? `Cloud run budget exceeded: ${slot.budgetExceeded}`
+                            : `Runtime timeout after ${Math.round(effectiveTurnTimeoutMs / 60000)} minute(s); aborted`,
                         replyTo: this.providerReplyTo(msg),
                         traceId: msg.trace?.id ?? null,
                     }, turnId);
@@ -1157,7 +1393,8 @@ export class Dispatcher {
                         topicId: msg.conversation.threadId ?? null,
                         turnId,
                         queueKey,
-                        timeoutMs: this.turnTimeoutMs,
+                        timeoutMs: effectiveTurnTimeoutMs,
+                        budgetExceeded: slot.budgetExceeded,
                     });
                 }
                 return;
@@ -1209,8 +1446,28 @@ export class Dispatcher {
             }
             if (!result)
                 return;
-            const replyText = (result.text || "").trim();
-            const finalTextField = truncateTextField(result.text || "");
+            const rawReplyText = (result.text || "").trim();
+            const replyLooksLikeAuthFailure = looksLikeRuntimeAuthFailure(rawReplyText);
+            const replyText = replyLooksLikeAuthFailure ? "" : rawReplyText;
+            const effectiveError = result.error ?? (replyLooksLikeAuthFailure ? rawReplyText : undefined);
+            const authFailureError = effectiveError && looksLikeRuntimeAuthFailure(effectiveError) ? effectiveError : undefined;
+            const finalTextField = truncateTextField(replyLooksLikeAuthFailure ? "" : result.text || "");
+            if (replyLooksLikeAuthFailure) {
+                this.log.error("dispatcher: runtime text looked like authentication failure; treating as error", {
+                    agentId: msg.accountId,
+                    roomId: msg.conversation.id,
+                    topicId: msg.conversation.threadId ?? null,
+                    turnId,
+                    runtime: route.runtime,
+                    error: rawReplyText,
+                });
+            }
+            if (authFailureError) {
+                this.recordRuntimeAuthFailure(route, msg, authFailureError);
+            }
+            else if (!effectiveError) {
+                this.clearRuntimeAuthFailures(route, msg);
+            }
             // Persist session before reply so next turn sees the new id even if send fails.
             //
             // Adapter contract:
@@ -1220,14 +1477,14 @@ export class Dispatcher {
             //                                 even when the adapter echoes that id back
             //   result.newSessionId truthy  → upsert the entry
             //   otherwise                   → no-op (e.g. codex intentionally never persists)
-            if (sessionId && result.error && !replyText) {
+            if (sessionId && effectiveError && !replyText) {
                 try {
                     await this.sessionStore.delete(key);
                     this.log.info("dispatcher: dropped stale runtime session", {
                         key,
                         prevRuntimeSessionId: sessionId,
                         nextRuntimeSessionId: result.newSessionId || null,
-                        error: result.error,
+                        error: effectiveError,
                     });
                 }
                 catch (err) {
@@ -1237,7 +1494,7 @@ export class Dispatcher {
                     });
                 }
             }
-            else if (result.newSessionId) {
+            else if (result.newSessionId && !authFailureError) {
                 const session = {
                     key,
                     runtime: route.runtime,
@@ -1267,13 +1524,13 @@ export class Dispatcher {
                     });
                 }
             }
-            else if (sessionId && result.error) {
+            else if (sessionId && effectiveError) {
                 try {
                     await this.sessionStore.delete(key);
                     this.log.info("dispatcher: dropped stale runtime session", {
                         key,
                         prevRuntimeSessionId: sessionId,
-                        error: result.error,
+                        error: effectiveError,
                     });
                 }
                 catch (err) {
@@ -1284,14 +1541,14 @@ export class Dispatcher {
                 }
             }
             if (!replyText) {
-                if (result.error) {
+                if (effectiveError) {
                     this.log.warn("dispatcher: runtime returned error without reply text", {
                         agentId: msg.accountId,
                         roomId: msg.conversation.id,
                         topicId: msg.conversation.threadId ?? null,
                         turnId,
                         runtime: route.runtime,
-                        error: result.error,
+                        error: effectiveError,
                     });
                     if (canDeliverRuntimeDiagnostics) {
                         const sendResult = await this.sendReply(channel, {
@@ -1300,7 +1557,7 @@ export class Dispatcher {
                             conversationId: msg.conversation.id,
                             threadId: msg.conversation.threadId ?? null,
                             type: "error",
-                            text: `⚠️ Runtime error: ${truncate(result.error, 500)}`,
+                            text: `⚠️ Runtime error: ${truncate(effectiveError, 500)}`,
                             replyTo: this.providerReplyTo(msg),
                             traceId: msg.trace?.id ?? null,
                         }, turnId);
@@ -1328,7 +1585,7 @@ export class Dispatcher {
                     costUsd: result.costUsd,
                     finalText: finalTextField,
                     deliveryStatus: "empty_text",
-                    deliveryReason: result.error ?? null,
+                    deliveryReason: effectiveError ?? null,
                     blocks: slot.blocks,
                 });
                 return;

package/dist/gateway/gateway.d.ts

@@ -1,5 +1,5 @@
 import { type ChannelBackoffOptions } from "./channel-manager.js";
-import { type RuntimeFactory } from "./dispatcher.js";
+import { type DispatcherOptions, type RuntimeFactory } from "./dispatcher.js";
 import { type GatewayLogger } from "./log.js";
 import { type TranscriptWriter } from "./transcript.js";
 import type { ChannelAdapter, GatewayChannelConfig, GatewayConfig, GatewayInboundMessage, GatewayOutboundMessage, GatewayRoute, GatewayRuntimeSnapshot, InboundObserver, MemoryContextBuilder, OutboundObserver, SystemContextBuilder, UserTurnBuilder } from "./types.js";
@@ -41,6 +41,14 @@ export interface GatewayBootOptions {
      * bookkeeping like loop-risk tracking.
      */
     onOutbound?: OutboundObserver;
+    onRuntimeCircuitBreakerChange?: () => void;
+    /**
+     * Optional observer fired after each runtime turn resolves. Forwarded
+     * to the dispatcher verbatim — see {@link Dispatcher} for semantics.
+     * Cloud daemon hooks this to settle ``cloud_run`` envelopes against
+     * the Hub usage ledger.
+     */
+    onTurnComplete?: DispatcherOptions["onTurnComplete"];
     /**
      * Optional attention gate (PR3, design §4.2). Forwarded to the dispatcher
      * verbatim — see {@link Dispatcher} for semantics. Returning `false` skips

package/dist/gateway/gateway.js

@@ -1,5 +1,5 @@
 import { ChannelManager } from "./channel-manager.js";
-import { Dispatcher } from "./dispatcher.js";
+import { Dispatcher, } from "./dispatcher.js";
 import { consoleLogger } from "./log.js";
 import { createRuntime } from "./runtimes/registry.js";
 import { DEFAULT_SESSION_STORE_MAX_ENTRY_AGE_MS, SessionStore } from "./session-store.js";
@@ -72,6 +72,8 @@ export class Gateway {
             onInbound: opts.onInbound,
             composeUserTurn: opts.composeUserTurn,
             onOutbound: opts.onOutbound,
+            onTurnComplete: opts.onTurnComplete,
+            onRuntimeCircuitBreakerChange: opts.onRuntimeCircuitBreakerChange,
             managedRoutes: this.managedRoutes,
             attentionGate: opts.attentionGate,
             resolveHubUrl: opts.resolveHubUrl,
@@ -105,6 +107,7 @@ export class Gateway {
         return {
             channels: this.channelManager.status(),
             turns: this.dispatcher.turns(),
+            runtimeCircuitBreakers: this.dispatcher.runtimeCircuitBreakers(),
         };
     }
     /**

package/dist/gateway/runtime-errors.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Runtime CLIs sometimes report authentication failures as ordinary final
+ * text. Keep this intentionally narrow so normal model replies about auth do
+ * not get reclassified unless they look like a top-level CLI/API failure.
+ */
+export declare function looksLikeRuntimeAuthFailure(text: string): boolean;

package/dist/gateway/runtime-errors.js

@@ -0,0 +1,14 @@
+/**
+ * Runtime CLIs sometimes report authentication failures as ordinary final
+ * text. Keep this intentionally narrow so normal model replies about auth do
+ * not get reclassified unless they look like a top-level CLI/API failure.
+ */
+export function looksLikeRuntimeAuthFailure(text) {
+    const s = text.trim();
+    if (!s)
+        return false;
+    return (/^(Failed to authenticate|Authentication failed|Invalid API key|Invalid Anthropic API key)\b/i.test(s) ||
+        /^API Error:\s*4\d\d\b/i.test(s) ||
+        /\b(API Error:\s*4\d\d|Request not allowed|invalid x-api-key)\b/i.test(s) ||
+        /^(Unauthorized|Forbidden)(?:\b|:)/i.test(s));
+}

package/dist/gateway/runtimes/claude-code.d.ts

@@ -1,10 +1,17 @@
 import { NdjsonStreamAdapter, type NdjsonEventCtx } from "./ndjson-stream.js";
 import { type ProbeDeps } from "./probe.js";
 import type { RuntimeProbeResult, RuntimeRunOptions } from "../types.js";
+export declare function scrubClaudeCodeAuthEnv(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv;
 /** Resolve the Claude Code CLI path on PATH or the macOS desktop bundle fallback. */
 export declare function resolveClaudeCommand(deps?: ProbeDeps): string | null;
 /** Probe whether the Claude Code CLI is installed and report its version. */
 export declare function probeClaude(deps?: ProbeDeps): RuntimeProbeResult;
+export interface ClaudeAuthProbeResult {
+    checked: boolean;
+    ok: boolean;
+    message: string;
+}
+export declare function probeClaudeAuth(deps?: ProbeDeps): ClaudeAuthProbeResult;
 /**
  * Claude Code adapter — spawns `claude -p "<text>" --output-format stream-json`
  * (with `--resume <sid>` when available) and parses the ndjson stream.
@@ -26,5 +33,6 @@ export declare class ClaudeCodeAdapter extends NdjsonStreamAdapter {
     run(opts: RuntimeRunOptions): Promise<import("../types.js").RuntimeRunResult>;
     protected resolveBinary(): string;
     protected buildArgs(opts: RuntimeRunOptions): string[];
+    protected spawnEnv(opts: RuntimeRunOptions): NodeJS.ProcessEnv;
     protected handleEvent(raw: unknown, ctx: NdjsonEventCtx): void;
 }