@botcord/daemon 0.2.75 → 0.2.76

package/src/gateway/dispatcher.ts

@@ -1,6 +1,7 @@
 import { randomUUID } from "node:crypto";
 
 import type { GatewayLogger } from "./log.js";
+import { looksLikeRuntimeAuthFailure } from "./runtime-errors.js";
 import { resolveRoute } from "./router.js";
 import { sessionKey, type SessionStore } from "./session-store.js";
 import {
@@ -22,6 +23,8 @@ import type {
   OutboundObserver,
   QueueMode,
   RuntimeAdapter,
+  RuntimeRunResult,
+  RuntimeCircuitBreakerSnapshot,
   RuntimeStatusEvent,
   StreamBlock,
   SystemContextBuilder,
@@ -30,6 +33,8 @@ import type {
 } from "./types.js";
 
 const DEFAULT_TURN_TIMEOUT_MS = 30 * 60 * 1000;
+const DEFAULT_RUNTIME_AUTH_FAILURE_THRESHOLD = 3;
+const DEFAULT_RUNTIME_AUTH_FAILURE_COOLDOWN_MS = 10 * 60 * 1000;
 
 /**
  * Owner-chat room prefix. Reply-text gating: only rooms with this prefix get
@@ -142,6 +147,41 @@ function redactSecretString(value: string): string {
     .replace(/\b(drt_|dit_|gho_)[A-Za-z0-9_-]+/g, "$1[REDACTED]");
 }
 
+function extractCloudRunBudget(msg: GatewayInboundMessage): CloudRunBudgetCaps | undefined {
+  const envelope = (msg.raw as { envelope?: unknown } | undefined)?.envelope as
+    | {
+        type?: unknown;
+        payload?: {
+          cloud_run?: {
+            budget?: {
+              max_wall_time_seconds?: unknown;
+              max_tool_calls?: unknown;
+            } | null;
+          } | null;
+        } | null;
+      }
+    | undefined;
+  if (envelope?.type !== "cloud_run") return undefined;
+  const budget = envelope.payload?.cloud_run?.budget;
+  if (!budget) return undefined;
+  const out: CloudRunBudgetCaps = {};
+  if (
+    typeof budget.max_wall_time_seconds === "number" &&
+    Number.isFinite(budget.max_wall_time_seconds) &&
+    budget.max_wall_time_seconds > 0
+  ) {
+    out.maxWallTimeMs = Math.floor(budget.max_wall_time_seconds * 1000);
+  }
+  if (
+    typeof budget.max_tool_calls === "number" &&
+    Number.isFinite(budget.max_tool_calls) &&
+    budget.max_tool_calls > 0
+  ) {
+    out.maxToolCalls = Math.floor(budget.max_tool_calls);
+  }
+  return out.maxWallTimeMs !== undefined || out.maxToolCalls !== undefined ? out : undefined;
+}
+
 /** Factory signature for building a runtime adapter at turn dispatch time. */
 export type RuntimeFactory = (
   runtimeId: string,
@@ -156,6 +196,8 @@ export interface DispatcherOptions {
   sessionStore: SessionStore;
   log: GatewayLogger;
   turnTimeoutMs?: number;
+  runtimeAuthFailureThreshold?: number;
+  runtimeAuthFailureCooldownMs?: number;
   /**
    * Live reference to the Gateway's managed-route map. Dispatcher reads
    * `values()` on every `resolveRoute` call so hot-add/remove take effect
@@ -194,6 +236,24 @@ export interface DispatcherOptions {
    * and suppressed so observer failures never break the turn.
    */
   onOutbound?: OutboundObserver;
+  onRuntimeCircuitBreakerChange?: () => void;
+  /**
+   * Optional observer fired exactly once per turn after ``runtime.run``
+   * resolves (or throws / times out). Receives the inbound message, the
+   * raw runtime result (may be undefined on throw), the elapsed wall
+   * time in milliseconds, and any thrown error. The cloud daemon hooks
+   * this to settle ``cloud_run`` envelopes against the Hub's usage
+   * ledger; local daemons leave it unset.
+   *
+   * Errors thrown by the observer are logged and swallowed — settle
+   * failures must never break the agent reply path.
+   */
+  onTurnComplete?: (event: {
+    message: GatewayInboundMessage;
+    result?: RuntimeRunResult;
+    wallTimeMs: number;
+    error?: unknown;
+  }) => Promise<void> | void;
   /**
    * Optional attention gate (PR3, design §4.2). Resolved AFTER `onInbound`
    * runs and BEFORE the runtime turn enqueues, so working memory / activity
@@ -245,6 +305,7 @@ interface TurnSlot {
   turnId: string;
   controller: AbortController;
   timedOut: boolean;
+  budgetExceeded: string | null;
   snapshot: TurnStatusSnapshot;
   done: Promise<void>;
   dispatchedAt: number;
@@ -288,10 +349,17 @@ interface QueueState {
   serialWorkerActive: boolean;
 }
 
+interface CloudRunBudgetCaps {
+  maxWallTimeMs?: number;
+  maxToolCalls?: number;
+}
+
 interface DeferredMultimodalEntry extends BufferedSerialEntry {
   queuedAt: number;
 }
 
+interface RuntimeAuthFailureState extends RuntimeCircuitBreakerSnapshot {}
+
 /**
  * Gateway dispatcher: consumes `GatewayInboundEnvelope` and drives a runtime
  * turn per message, respecting queue mode, trust level, streaming, and
@@ -309,10 +377,14 @@ export class Dispatcher {
   private readonly sessionStore: SessionStore;
   private readonly log: GatewayLogger;
   private readonly turnTimeoutMs: number;
+  private readonly runtimeAuthFailureThreshold: number;
+  private readonly runtimeAuthFailureCooldownMs: number;
   private readonly buildSystemContext?: SystemContextBuilder;
   private readonly buildMemoryContext?: MemoryContextBuilder;
   private readonly onInbound?: InboundObserver;
   private readonly onOutbound?: OutboundObserver;
+  private readonly onTurnComplete?: DispatcherOptions["onTurnComplete"];
+  private readonly onRuntimeCircuitBreakerChange?: () => void;
   private readonly composeUserTurn?: UserTurnBuilder;
   private readonly managedRoutes?: Map<string, GatewayRoute>;
   private readonly attentionGate?: (
@@ -322,6 +394,7 @@ export class Dispatcher {
   private readonly transcript: TranscriptWriter;
   private readonly queues: Map<string, QueueState> = new Map();
   private readonly deferredMultimodal: Map<string, DeferredMultimodalEntry[]> = new Map();
+  private readonly runtimeAuthFailures: Map<string, RuntimeAuthFailureState> = new Map();
   /**
    * Last `/hub/typing` ping timestamp per (accountId, conversationId).
    * Used to debounce cancel-previous bursts so we don't trip Hub's 20/min
@@ -336,10 +409,16 @@ export class Dispatcher {
     this.sessionStore = opts.sessionStore;
     this.log = opts.log;
     this.turnTimeoutMs = opts.turnTimeoutMs ?? DEFAULT_TURN_TIMEOUT_MS;
+    this.runtimeAuthFailureThreshold =
+      opts.runtimeAuthFailureThreshold ?? DEFAULT_RUNTIME_AUTH_FAILURE_THRESHOLD;
+    this.runtimeAuthFailureCooldownMs =
+      opts.runtimeAuthFailureCooldownMs ?? DEFAULT_RUNTIME_AUTH_FAILURE_COOLDOWN_MS;
     this.buildSystemContext = opts.buildSystemContext;
     this.buildMemoryContext = opts.buildMemoryContext;
     this.onInbound = opts.onInbound;
     this.onOutbound = opts.onOutbound;
+    this.onTurnComplete = opts.onTurnComplete;
+    this.onRuntimeCircuitBreakerChange = opts.onRuntimeCircuitBreakerChange;
     this.composeUserTurn = opts.composeUserTurn;
     this.managedRoutes = opts.managedRoutes;
     this.attentionGate = opts.attentionGate;
@@ -559,6 +638,18 @@ export class Dispatcher {
       });
     }
 
+    const openAuthBreaker = this.openRuntimeAuthBreaker(dispatchRoute, dispatchMsg);
+    if (openAuthBreaker) {
+      await this.skipRuntimeForAuthBreaker(
+        openAuthBreaker,
+        dispatchRoute,
+        dispatchMsg,
+        dispatchChannel,
+        dispatchTurnId,
+      );
+      return;
+    }
+
     if (mode === "cancel-previous") {
       await this.runCancelPrevious(
         queueKey,
@@ -591,6 +682,15 @@ export class Dispatcher {
     return out;
   }
 
+  runtimeCircuitBreakers(): Record<string, RuntimeCircuitBreakerSnapshot> {
+    this.pruneExpiredRuntimeAuthBreakers();
+    const out: Record<string, RuntimeCircuitBreakerSnapshot> = {};
+    for (const [key, state] of this.runtimeAuthFailures) {
+      if (state.blockedUntil > Date.now()) out[key] = { ...state };
+    }
+    return out;
+  }
+
   // ---------------------------------------------------------------------------
   // Internals
   // ---------------------------------------------------------------------------
@@ -653,6 +753,166 @@ export class Dispatcher {
     return list;
   }
 
+  private runtimeAuthBreakerKey(route: GatewayRoute, msg: GatewayInboundMessage): string {
+    const thread = msg.conversation.threadId ?? "";
+    return `${route.runtime}:${msg.channel}:${msg.accountId}:${msg.conversation.id}:${thread}`;
+  }
+
+  private openRuntimeAuthBreaker(
+    route: GatewayRoute,
+    msg: GatewayInboundMessage,
+  ): RuntimeAuthFailureState | null {
+    const key = this.runtimeAuthBreakerKey(route, msg);
+    const state = this.runtimeAuthFailures.get(key);
+    if (!state) return null;
+    if (state.blockedUntil > 0 && state.blockedUntil <= Date.now()) {
+      this.runtimeAuthFailures.delete(key);
+      return null;
+    }
+    return state.blockedUntil > Date.now() ? state : null;
+  }
+
+  private pruneExpiredRuntimeAuthBreakers(): void {
+    const now = Date.now();
+    for (const [key, state] of this.runtimeAuthFailures) {
+      if (state.blockedUntil > 0 && state.blockedUntil <= now) this.runtimeAuthFailures.delete(key);
+    }
+  }
+
+  private recordRuntimeAuthFailure(
+    route: GatewayRoute,
+    msg: GatewayInboundMessage,
+    error: string,
+  ): RuntimeAuthFailureState | null {
+    const now = Date.now();
+    const key = this.runtimeAuthBreakerKey(route, msg);
+    const prev = this.runtimeAuthFailures.get(key);
+    const failures = (prev?.failures ?? 0) + 1;
+    const openedAt = prev?.openedAt ?? now;
+    const state: RuntimeAuthFailureState = {
+      key,
+      runtime: route.runtime,
+      channel: msg.channel,
+      accountId: msg.accountId,
+      conversationId: msg.conversation.id,
+      threadId: msg.conversation.threadId ?? null,
+      failures,
+      openedAt,
+      blockedUntil:
+        failures >= this.runtimeAuthFailureThreshold
+          ? now + this.runtimeAuthFailureCooldownMs
+          : 0,
+      lastFailureAt: now,
+      lastError: error,
+    };
+    this.runtimeAuthFailures.set(key, state);
+    if (state.blockedUntil > now) {
+      this.log.error("dispatcher: runtime auth circuit breaker opened", {
+        key,
+        runtime: route.runtime,
+        agentId: msg.accountId,
+        roomId: msg.conversation.id,
+        topicId: msg.conversation.threadId ?? null,
+        failures,
+        blockedUntil: state.blockedUntil,
+        error,
+      });
+      this.notifyRuntimeCircuitBreakerChange();
+      return state;
+    }
+    this.log.warn("dispatcher: runtime authentication failure recorded", {
+      key,
+      runtime: route.runtime,
+      agentId: msg.accountId,
+      roomId: msg.conversation.id,
+      topicId: msg.conversation.threadId ?? null,
+      failures,
+      threshold: this.runtimeAuthFailureThreshold,
+      error,
+    });
+    return null;
+  }
+
+  private clearRuntimeAuthFailures(route: GatewayRoute, msg: GatewayInboundMessage): void {
+    const key = this.runtimeAuthBreakerKey(route, msg);
+    if (!this.runtimeAuthFailures.delete(key)) return;
+    this.log.info("dispatcher: runtime auth circuit breaker cleared", {
+      key,
+      runtime: route.runtime,
+      agentId: msg.accountId,
+      roomId: msg.conversation.id,
+      topicId: msg.conversation.threadId ?? null,
+    });
+    this.notifyRuntimeCircuitBreakerChange();
+  }
+
+  private notifyRuntimeCircuitBreakerChange(): void {
+    try {
+      this.onRuntimeCircuitBreakerChange?.();
+    } catch (err) {
+      this.log.warn("dispatcher: onRuntimeCircuitBreakerChange threw", {
+        error: err instanceof Error ? err.message : String(err),
+      });
+    }
+  }
+
+  private async skipRuntimeForAuthBreaker(
+    state: RuntimeAuthFailureState,
+    route: GatewayRoute,
+    msg: GatewayInboundMessage,
+    channel: ChannelAdapter,
+    turnId: string,
+  ): Promise<void> {
+    const error =
+      `runtime authentication failed repeatedly; dispatch paused until ${new Date(state.blockedUntil).toISOString()}`;
+    this.log.warn("dispatcher: runtime auth circuit breaker blocking turn", {
+      key: state.key,
+      runtime: route.runtime,
+      agentId: msg.accountId,
+      roomId: msg.conversation.id,
+      topicId: msg.conversation.threadId ?? null,
+      turnId,
+      blockedUntil: state.blockedUntil,
+    });
+    this.transcript.write({
+      ts: nowIso(),
+      kind: "turn_error",
+      turnId,
+      agentId: msg.accountId,
+      roomId: msg.conversation.id,
+      topicId: msg.conversation.threadId ?? null,
+      phase: "runtime",
+      error,
+      durationMs: 0,
+    });
+
+    const canDeliverRuntimeText = isOwnerChatRoom(msg) || !isBotCordChannel(channel);
+    const canDeliverRuntimeDiagnostics = canDeliverRuntimeText || isBotCordChannel(channel);
+    if (canDeliverRuntimeDiagnostics) {
+      const sendResult = await this.sendReply(channel, {
+        channel: msg.channel,
+        accountId: msg.accountId,
+        conversationId: msg.conversation.id,
+        threadId: msg.conversation.threadId ?? null,
+        type: "error",
+        text: `⚠️ Runtime error: ${truncate(error, 500)}`,
+        replyTo: this.providerReplyTo(msg),
+        traceId: msg.trace?.id ?? null,
+      }, turnId);
+      this.emitOutbound({
+        turnId,
+        msg,
+        runtime: route.runtime,
+        runtimeSessionId: null,
+        startedAt: Date.now(),
+        finalText: truncateTextField(""),
+        deliveryStatus: sendResult.ok ? "delivered" : "send_failed",
+        deliveryReason: sendResult.ok ? null : sendResult.error,
+        blocks: [],
+      });
+    }
+  }
+
   private async runCancelPrevious(
     queueKey: string,
     route: GatewayRoute,
@@ -956,6 +1216,7 @@ export class Dispatcher {
       turnId,
       controller,
       timedOut: false,
+      budgetExceeded: null,
       snapshot,
       done,
       dispatchedAt: startedAt,
@@ -992,6 +1253,13 @@ export class Dispatcher {
       composedPreview: logPreview(text),
     });
 
+    const cloudRunBudget = extractCloudRunBudget(msg);
+    const effectiveTurnTimeoutMs = Math.min(
+      this.turnTimeoutMs,
+      cloudRunBudget?.maxWallTimeMs ?? this.turnTimeoutMs,
+    );
+    let observedToolCalls = 0;
+
     // Hard-cap turn with a timeout.
     const timer = setTimeout(() => {
       slot.timedOut = true;
@@ -1001,10 +1269,10 @@ export class Dispatcher {
         topicId: msg.conversation.threadId ?? null,
         turnId,
         queueKey,
-        timeoutMs: this.turnTimeoutMs,
+        timeoutMs: effectiveTurnTimeoutMs,
       });
       controller.abort();
-    }, this.turnTimeoutMs);
+    }, effectiveTurnTimeoutMs);
     if (typeof timer.unref === "function") timer.unref();
 
     const key = sessionKey({
@@ -1030,6 +1298,22 @@ export class Dispatcher {
     const canStream =
       streamable && typeof traceId === "string" && typeof channel.streamBlock === "function";
     const recordBlock = (block: StreamBlock): void => {
+      if (block.kind === "tool_use" && cloudRunBudget?.maxToolCalls !== undefined) {
+        observedToolCalls += 1;
+        if (observedToolCalls > cloudRunBudget.maxToolCalls && !controller.signal.aborted) {
+          slot.budgetExceeded = `tool call budget exceeded after ${observedToolCalls} tool call(s)`;
+          this.log.warn("dispatcher: cloud_run tool budget exceeded", {
+            agentId: msg.accountId,
+            roomId: msg.conversation.id,
+            topicId: msg.conversation.threadId ?? null,
+            turnId,
+            queueKey,
+            maxToolCalls: cloudRunBudget.maxToolCalls,
+            observedToolCalls,
+          });
+          controller.abort(new Error(slot.budgetExceeded));
+        }
+      }
       const summary = summarizeStreamBlock(block);
       slot.blocks.push(summary);
       if (this.transcript.enabled) {
@@ -1215,7 +1499,9 @@ export class Dispatcher {
         }
       : undefined;
 
-    const onBlock = (canStream || this.transcript.enabled)
+    const shouldObserveBlocks =
+      canStream || this.transcript.enabled || cloudRunBudget?.maxToolCalls !== undefined;
+    const onBlock = shouldObserveBlocks
       ? (block: StreamBlock) => {
           // Always record adapter-emitted blocks for transcript fidelity, even
           // after abort — the transcript reflects what the runtime emitted,
@@ -1316,8 +1602,9 @@ export class Dispatcher {
     }
 
     const runtime = this.runtimeFactory(route.runtime, route.extraArgs);
-    let result: { text: string; newSessionId: string; costUsd?: number; error?: string } | undefined;
+    let result: RuntimeRunResult | undefined;
     let threw: unknown;
+    const turnStartedAt = Date.now();
     try {
       try {
         result = await runtime.run({
@@ -1340,6 +1627,7 @@ export class Dispatcher {
             channel: msg.channel,
             conversationKind: msg.conversation.kind,
           },
+          ...(cloudRunBudget ? { budget: cloudRunBudget } : {}),
           gateway: route.gateway,
           ...(route.hermesProfile ? { hermesProfile: route.hermesProfile } : {}),
         });
@@ -1349,6 +1637,26 @@ export class Dispatcher {
         clearTimeout(timer);
       }
 
+      // Fire onTurnComplete observer. Cloud daemon hooks this to settle
+      // ``cloud_run`` envelopes against the Hub usage ledger. Errors are
+      // swallowed so settle failures never break the reply path.
+      if (this.onTurnComplete) {
+        const wallTimeMs = Date.now() - turnStartedAt;
+        try {
+          await this.onTurnComplete({
+            message: msg,
+            result,
+            wallTimeMs,
+            ...(threw !== undefined ? { error: threw } : {}),
+          });
+        } catch (hookErr) {
+          this.log.warn("dispatcher: onTurnComplete threw — continuing", {
+            error: hookErr instanceof Error ? hookErr.message : String(hookErr),
+            messageId: msg.id,
+          });
+        }
+      }
+
       // Re-check the abort signal AFTER runtime.run resolves but BEFORE any
       // side effects (session write, reply send). This closes the race where
       // a cancel-previous arrives between runtime.run resolving and the
@@ -1361,7 +1669,7 @@ export class Dispatcher {
       // record from `runCancelPrevious` BEFORE aborting, so we MUST NOT also
       // emit a `turn_error` here — that would violate the "exactly one
       // terminal record per turnId" invariant.
-      if (controller.signal.aborted && !slot.timedOut) {
+      if (controller.signal.aborted && !slot.timedOut && !slot.budgetExceeded) {
         return;
       }
 
@@ -1386,7 +1694,9 @@ export class Dispatcher {
       const canDeliverRuntimeText = isOwnerChat || !isBotCordChannel(channel);
       const canDeliverRuntimeDiagnostics = canDeliverRuntimeText || isBotCordChannel(channel);
 
-      if (slot.timedOut) {
+      if (slot.timedOut || slot.budgetExceeded) {
+        const phase = slot.budgetExceeded ? "budget" : "timeout";
+        const error = slot.budgetExceeded ?? `runtime timeout after ${effectiveTurnTimeoutMs}ms`;
         this.transcript.write({
           ts: nowIso(),
           kind: "turn_error",
@@ -1394,8 +1704,8 @@ export class Dispatcher {
           agentId: msg.accountId,
           roomId: msg.conversation.id,
           topicId: msg.conversation.threadId ?? null,
-          phase: "timeout",
-          error: `runtime timeout after ${this.turnTimeoutMs}ms`,
+          phase,
+          error,
           durationMs: Date.now() - slot.dispatchedAt,
         });
         if (canDeliverRuntimeDiagnostics) {
@@ -1405,7 +1715,9 @@ export class Dispatcher {
             conversationId: msg.conversation.id,
             threadId: msg.conversation.threadId ?? null,
             type: "error",
-            text: `⚠️ Runtime timeout after ${Math.round(this.turnTimeoutMs / 60000)} minute(s); aborted`,
+            text: slot.budgetExceeded
+              ? `Cloud run budget exceeded: ${slot.budgetExceeded}`
+              : `Runtime timeout after ${Math.round(effectiveTurnTimeoutMs / 60000)} minute(s); aborted`,
             replyTo: this.providerReplyTo(msg),
             traceId: msg.trace?.id ?? null,
           }, turnId);
@@ -1416,7 +1728,8 @@ export class Dispatcher {
             topicId: msg.conversation.threadId ?? null,
             turnId,
             queueKey,
-            timeoutMs: this.turnTimeoutMs,
+            timeoutMs: effectiveTurnTimeoutMs,
+            budgetExceeded: slot.budgetExceeded,
           });
         }
         return;
@@ -1469,8 +1782,28 @@ export class Dispatcher {
 
       if (!result) return;
 
-      const replyText = (result.text || "").trim();
-      const finalTextField = truncateTextField(result.text || "");
+      const rawReplyText = (result.text || "").trim();
+      const replyLooksLikeAuthFailure = looksLikeRuntimeAuthFailure(rawReplyText);
+      const replyText = replyLooksLikeAuthFailure ? "" : rawReplyText;
+      const effectiveError = result.error ?? (replyLooksLikeAuthFailure ? rawReplyText : undefined);
+      const authFailureError =
+        effectiveError && looksLikeRuntimeAuthFailure(effectiveError) ? effectiveError : undefined;
+      const finalTextField = truncateTextField(replyLooksLikeAuthFailure ? "" : result.text || "");
+      if (replyLooksLikeAuthFailure) {
+        this.log.error("dispatcher: runtime text looked like authentication failure; treating as error", {
+          agentId: msg.accountId,
+          roomId: msg.conversation.id,
+          topicId: msg.conversation.threadId ?? null,
+          turnId,
+          runtime: route.runtime,
+          error: rawReplyText,
+        });
+      }
+      if (authFailureError) {
+        this.recordRuntimeAuthFailure(route, msg, authFailureError);
+      } else if (!effectiveError) {
+        this.clearRuntimeAuthFailures(route, msg);
+      }
 
       // Persist session before reply so next turn sees the new id even if send fails.
       //
@@ -1481,14 +1814,14 @@ export class Dispatcher {
       //                                 even when the adapter echoes that id back
       //   result.newSessionId truthy  → upsert the entry
       //   otherwise                   → no-op (e.g. codex intentionally never persists)
-      if (sessionId && result.error && !replyText) {
+      if (sessionId && effectiveError && !replyText) {
         try {
           await this.sessionStore.delete(key);
           this.log.info("dispatcher: dropped stale runtime session", {
             key,
             prevRuntimeSessionId: sessionId,
             nextRuntimeSessionId: result.newSessionId || null,
-            error: result.error,
+            error: effectiveError,
           });
         } catch (err) {
           this.log.warn("dispatcher: session-store.delete failed", {
@@ -1496,7 +1829,7 @@ export class Dispatcher {
             error: err instanceof Error ? err.message : String(err),
           });
         }
-      } else if (result.newSessionId) {
+      } else if (result.newSessionId && !authFailureError) {
         const session: GatewaySessionEntry = {
           key,
           runtime: route.runtime,
@@ -1524,13 +1857,13 @@ export class Dispatcher {
             error: err instanceof Error ? err.message : String(err),
           });
         }
-      } else if (sessionId && result.error) {
+      } else if (sessionId && effectiveError) {
         try {
           await this.sessionStore.delete(key);
           this.log.info("dispatcher: dropped stale runtime session", {
             key,
             prevRuntimeSessionId: sessionId,
-            error: result.error,
+            error: effectiveError,
           });
         } catch (err) {
           this.log.warn("dispatcher: session-store.delete failed", {
@@ -1541,14 +1874,14 @@ export class Dispatcher {
       }
 
       if (!replyText) {
-        if (result.error) {
+        if (effectiveError) {
           this.log.warn("dispatcher: runtime returned error without reply text", {
             agentId: msg.accountId,
             roomId: msg.conversation.id,
             topicId: msg.conversation.threadId ?? null,
             turnId,
             runtime: route.runtime,
-            error: result.error,
+            error: effectiveError,
           });
           if (canDeliverRuntimeDiagnostics) {
             const sendResult = await this.sendReply(channel, {
@@ -1557,7 +1890,7 @@ export class Dispatcher {
               conversationId: msg.conversation.id,
               threadId: msg.conversation.threadId ?? null,
               type: "error",
-              text: `⚠️ Runtime error: ${truncate(result.error, 500)}`,
+              text: `⚠️ Runtime error: ${truncate(effectiveError, 500)}`,
               replyTo: this.providerReplyTo(msg),
               traceId: msg.trace?.id ?? null,
             }, turnId);
@@ -1585,7 +1918,7 @@ export class Dispatcher {
           costUsd: result.costUsd,
           finalText: finalTextField,
           deliveryStatus: "empty_text",
-          deliveryReason: result.error ?? null,
+          deliveryReason: effectiveError ?? null,
           blocks: slot.blocks,
         });
         return;

package/src/gateway/gateway.ts

@@ -1,5 +1,9 @@
 import { ChannelManager, type ChannelBackoffOptions } from "./channel-manager.js";
-import { Dispatcher, type RuntimeFactory } from "./dispatcher.js";
+import {
+  Dispatcher,
+  type DispatcherOptions,
+  type RuntimeFactory,
+} from "./dispatcher.js";
 import { consoleLogger, type GatewayLogger } from "./log.js";
 import { createRuntime } from "./runtimes/registry.js";
 import { DEFAULT_SESSION_STORE_MAX_ENTRY_AGE_MS, SessionStore } from "./session-store.js";
@@ -61,6 +65,14 @@ export interface GatewayBootOptions {
    * bookkeeping like loop-risk tracking.
    */
   onOutbound?: OutboundObserver;
+  onRuntimeCircuitBreakerChange?: () => void;
+  /**
+   * Optional observer fired after each runtime turn resolves. Forwarded
+   * to the dispatcher verbatim — see {@link Dispatcher} for semantics.
+   * Cloud daemon hooks this to settle ``cloud_run`` envelopes against
+   * the Hub usage ledger.
+   */
+  onTurnComplete?: DispatcherOptions["onTurnComplete"];
   /**
    * Optional attention gate (PR3, design §4.2). Forwarded to the dispatcher
    * verbatim — see {@link Dispatcher} for semantics. Returning `false` skips
@@ -169,6 +181,8 @@ export class Gateway {
       onInbound: opts.onInbound,
       composeUserTurn: opts.composeUserTurn,
       onOutbound: opts.onOutbound,
+      onTurnComplete: opts.onTurnComplete,
+      onRuntimeCircuitBreakerChange: opts.onRuntimeCircuitBreakerChange,
       managedRoutes: this.managedRoutes,
       attentionGate: opts.attentionGate,
       resolveHubUrl: opts.resolveHubUrl,
@@ -204,6 +218,7 @@ export class Gateway {
     return {
       channels: this.channelManager.status(),
       turns: this.dispatcher.turns(),
+      runtimeCircuitBreakers: this.dispatcher.runtimeCircuitBreakers(),
     };
   }
 

package/src/gateway/runtime-errors.ts

@@ -0,0 +1,15 @@
+/**
+ * Runtime CLIs sometimes report authentication failures as ordinary final
+ * text. Keep this intentionally narrow so normal model replies about auth do
+ * not get reclassified unless they look like a top-level CLI/API failure.
+ */
+export function looksLikeRuntimeAuthFailure(text: string): boolean {
+  const s = text.trim();
+  if (!s) return false;
+  return (
+    /^(Failed to authenticate|Authentication failed|Invalid API key|Invalid Anthropic API key)\b/i.test(s) ||
+    /^API Error:\s*4\d\d\b/i.test(s) ||
+    /\b(API Error:\s*4\d\d|Request not allowed|invalid x-api-key)\b/i.test(s) ||
+    /^(Unauthorized|Forbidden)(?:\b|:)/i.test(s)
+  );
+}