@humwick/server 0.0.0-stage → 0.130.0

package/runner/src/relay-mcp-proxy.ts

@@ -0,0 +1,704 @@
+import { registryToolName, mcpServerInfo, parseToolGeneration } from "@humwick/sdk";
+import { relayHeader, relayHeaderConflict, errMessage, isRecord, resendOnRotatedToken } from "@humwick/sdk";
+import { forwardSharedMcp, type WebToolLogOptions } from "./web-tool-call-proxy";
+import { materializeArtifactPathUpload } from "./artifact-path-materialize";
+import { logger } from "./logger";
+import { classifyTransportError, type TransportErrorClass } from "./transport-errors";
+import { RELAY_MCP_ONDEMAND_SURFACE_VALUE } from "./relay-mcp";
+
+// Loose fetch signature so tests can inject a plain async stub without Bun's `preconnect`
+// member; the real global `fetch` satisfies it.
+export type FetchLike = (input: string | URL | Request, init?: RequestInit) => Promise<Response>;
+
+// #2429 — Bun accepts a non-standard `timeout` member on the fetch init (ms, or `false` to disable
+// its native HTTP idle watchdog), but bun-types@1.3.14 don't declare it, so a bare literal fails
+// typecheck. Widen locally (mirrors the #2428 Phase-1 pattern) instead of casting to `any`.
+export type RelayForwardRequestInit = RequestInit & { timeout?: number | false };
+
+// Stage 2 of #213/#215 — the Runner becomes the MCP endpoint the agent connects to,
+// fronting the relay. The agent's MCP client points at this localhost server instead of
+// directly at the relay, so the Runner owns the relay connection, reconnect/backoff, and a
+// durable buffer. A relay restart/crash becomes invisible to the agent.
+//
+// This is a TRANSPARENT JSON-RPC pass-through to the relay's `/api/mcp` (which stays the
+// sole enforcement authority). It intervenes in exactly four places:
+//
+//   1. `initialize`   — forward, then advertise `capabilities.tools.listChanged: true`,
+//                       the live-tool-set capability the relay's static endpoint can't.
+//   2. `tools/list`   — forward (the relay already scope-filters per token), then context-
+//                       NARROW: hide tools that don't apply to this agent's workspace mode/
+//                       state (e.g. `relay_workspace_*` for a non-worktree agent). Strictly
+//                       a subset — never widens. Served from a last-known cache during a blip.
+//   3. `tools/call`   — forward with the runner's LIVE token; for bufferable writes during a
+//                       relay outage, enqueue durably + return a synthetic "queued" ok so the
+//                       call is never lost. Mutating spawn/shutdown are forwarded, never local.
+//   4. `GET` (SSE)    — the server→client notification channel; emits
+//                       `notifications/tools/list_changed` when a transition changes the
+//                       narrowed set, so the agent's tool menu updates mid-session.
+//
+// Narrow-never-widen is the safety invariant: filtering `tools/list` is UX/token-efficiency,
+// NEVER enforcement. The proxy can only ever surface a subset of what the relay's scope filter
+// already returned, so a proxy bug can never become an auth bypass — the relay owns the lock at
+// `tools/call`, the proxy owns the menu.
+//
+// #1450 (the #1570 unify) adds a SECOND narrowing dimension for Codex lean workers: the on-demand
+// relay set. The agent's DIRECT relay client has its on-demand tools hidden here so only the eager
+// hot-path set boots; the SAME tools are simultaneously fronted by the per-agent metaOnly callmux
+// (a second relay downstream over THIS proxy), reachable via callmux_call. callmux's requests carry
+// RELAY_MCP_ONDEMAND_SURFACE_HEADER, which opts OUT of this hiding so callmux sees the full set the
+// relay returned and can route to any of it. This is still strictly narrow-never-widen (the direct
+// client loses exactly what callmux gains; the header only restores the relay's own set, never the
+// call — tools/call stays scope-gated), and it is coupled at launch: a worker is only narrowed when
+// callmux is simultaneously fronting the on-demand set, so no capability is ever lost.
+
+const PROXY_PATH = "/mcp";
+// Relay-side failures we treat as "relay down" (buffer/serve-cache), as opposed to a real 4xx
+// rejection that must be surfaced to the agent verbatim.
+const GATEWAY_STATUSES = new Set([502, 503, 504]);
+// #2089 — gateway sub-classes for the connect-retry safety boundary. A 502/503 means the reverse
+// proxy could not reach the relay at all → the request never executed → safe to retry any tool
+// (connect-phase). A 504 means the proxy DID forward and the relay's response was lost → the outcome
+// is ambiguous → only safe to retry an idempotent call (same rule as a mid-flight socket reset).
+const CONNECT_PHASE_GATEWAY_STATUSES = new Set([502, 503]);
+const AMBIGUOUS_GATEWAY_STATUSES = new Set([504]);
+const SSE_KEEPALIVE_MS = 25_000;
+
+// #1790/#2089 — relay_spawn_agent is IDEMPOTENT by spawnRequestId: the server dedupes a repeat of the
+// same id to the existing spawn instead of creating a second agent (see src/services/spawn-agent.ts).
+// The proxy stabilizes that id up-front (below) so a retry across a restart can't double-create. This
+// name is used only for that id-stabilization; the connect-retry itself (#2089) is GENERIC across every
+// tool and keys "safe to resend an ambiguous failure" off the presence of an idempotency key, never a
+// tool name.
+const SPAWN_TOOL = "relay_spawn_agent";
+
+// #2089 — bounded connect-retry defaults. A CD deploy restarts the fleet relay on every green
+// origin/main SHA (systemctl restart, no drain/quiesce); tool calls in that ~1-2 min window otherwise
+// fail hard with "-32002 relay unreachable" and require a manual retry. The window (~75s) rides out
+// the restart; backoff climbs 0.5→1→2→4s (capped) with jitter to avoid a thundering-herd reconnect.
+// `attempts` is a hard safety cap so a pathological fast-failing relay can't spin unbounded — the
+// WINDOW is the real bound in production (base backoff > 0). With base backoff 0 (tests) the attempt
+// cap governs and the window never advances.
+const DEFAULT_CONNECT_RETRY_ATTEMPTS = 40;
+const DEFAULT_CONNECT_RETRY_BASE_BACKOFF_MS = 500;
+const DEFAULT_CONNECT_RETRY_MAX_BACKOFF_MS = 4_000;
+const DEFAULT_CONNECT_RETRY_WINDOW_MS = 75_000;
+const RELAY_MCP_METADATA_FORWARD_TIMEOUT_MS = 10_000;
+// A tool call can legitimately block in Relay: workspace status alone permits a
+// 600-second wait, and spawn/artifact operations can also take time. relay_run
+// separately caps its wait at 10s (#2790): this proxy budget cannot extend a
+// provider client's shorter timeout. Keep the deadline for the other tools.
+const RELAY_MCP_TOOL_CALL_FORWARD_TIMEOUT_MS = 610_000;
+
+// #2089 — the connect-retry safety boundary rests on distinguishing WHEN in a request's lifecycle the
+// transport failed. A failure is one of three classes:
+//
+//   "connect"   — the request provably NEVER reached the relay app: connection refused, DNS failure,
+//                 host/net unreachable, or the pre-send "Unable to connect"/"fetch failed" surface. The
+//                 call did not execute, so it is ALWAYS safe to retry, for EVERY tool (reads included).
+//   "ambiguous" — sent-then-response-lost: a socket reset/hang-up/broken-pipe or a client-side abort
+//                 AFTER the bytes went out. The relay MAY have executed the call, so it is safe to retry
+//                 ONLY for an idempotent call (one carrying spawnRequestId / idempotencyKey); a blind
+//                 resend of a non-idempotent mutation could double-apply.
+//   "fatal"     — not a transport failure at all (a genuine relay response, or an unrelated error). Never
+//                 retried — surfaces immediately so a real error is never masked by a pointless retry.
+//
+// Both predicates walk the `cause` chain and inspect `code`, `name`, and message text (Bun surfaces
+// "socket connection was closed unexpectedly"; Node uses ECONNRESET/ECONNREFUSED/EPIPE; fetch wraps
+// connect failures as "fetch failed"). "ambiguous" is checked FIRST: if any level shows a reset/abort
+// signal we treat the whole failure as sent-then-lost (the conservative choice), even if an outer
+// "fetch failed" also looks connect-ish.
+// A gateway status maps to the same connect/ambiguous/fatal classes as a thrown transport error, so the
+// retry decision is identical whether the relay was unreachable (thrown) or a reverse proxy answered for
+// it (5xx). 502/503 = upstream never served the request; 504 = forwarded but response lost (ambiguous);
+// anything else is a genuine relay answer (fatal — never retried).
+function classifyGatewayStatus(status: number): TransportErrorClass {
+  if (CONNECT_PHASE_GATEWAY_STATUSES.has(status)) return "connect";
+  if (AMBIGUOUS_GATEWAY_STATUSES.has(status)) return "ambiguous";
+  return "fatal";
+}
+
+// #2089 — a tool call is idempotent (safe to resend after an ambiguous failure) iff it carries a key the
+// server dedupes on: spawnRequestId (relay_spawn_agent) or idempotencyKey (buffered writes, task deliver).
+// Generic — no tool names — so any tool gaining such a key qualifies automatically.
+function isIdempotentCall(args: Record<string, unknown>): boolean {
+  const spawnRequestId = typeof args.spawnRequestId === "string" ? args.spawnRequestId.trim() : "";
+  const idempotencyKey = typeof args.idempotencyKey === "string" ? args.idempotencyKey.trim() : "";
+  return spawnRequestId.length > 0 || idempotencyKey.length > 0;
+}
+
+// The write tools whose loss during a relay outage is unacceptable and whose result the agent
+// does not need synchronously — safe to queue durably and replay on reconnect. Reads, claims
+// (409 contention), spawn/shutdown (need a real ack) are deliberately NOT bufferable.
+const DEFAULT_BUFFERABLE_TOOLS = new Set<string>([
+  "relay_send_message",
+  "relay_reply",
+  "relay_task_deliver",
+]);
+
+// Tools surfaced ONLY to an agent that owns a live isolated worktree. For any other agent the
+// proxy narrows them out of `tools/list` even though the token scope permits them (the relay
+// would return them). This is the context the coarse token scope can't express (#214/#215).
+const WORKTREE_ONLY_TOOLS = new Set<string>([
+  "relay_workspace_status",
+  "relay_workspace_deps",
+  "relay_workspace_list",
+  "relay_workspace_claim",
+  "relay_workspace_release",
+]);
+
+interface ProxyContext {
+  // The agent owns a live (non-terminal) isolated git worktree → workspace tools apply.
+  isolatedWorktree: boolean;
+  // #1502 — this worktree agent's own workspace descriptor (from AGENT_RELAY_WORKSPACE_JSON). The
+  // relay writes the workspace ROW lazily via the orchestrator's managed-agents report, so a worker
+  // that delivers before that write (or whose registration was lost under concurrent-spawn load)
+  // would 404. We inject this descriptor into `relay_task_deliver` so the relay can self-heal the
+  // missing registration from the worker's own truth instead of stranding the delivery.
+  workspaceDescriptor?: Record<string, unknown>;
+}
+
+// #1502 — tool calls the proxy augments with the agent's workspace descriptor for self-heal.
+const WORKSPACE_SELF_HEAL_TOOLS = new Set(["relay_task_deliver"]);
+
+// #1502 — the minimal descriptor the relay needs to (re)register a dropped workspace row, distilled
+// from the agent's WorkspaceMetadata. Undefined unless this agent owns a live isolated worktree with
+// the required id/repo/worktree present, so a shared/non-worktree agent never sends one.
+export function isolatedWorkspaceDescriptor(
+  workspace: { id?: string; repoRoot?: string; worktreePath?: string; sourceCwd?: string; branch?: string; baseRef?: string; baseSha?: string } | undefined,
+  ownsIsolatedWorktree: boolean,
+): Record<string, unknown> | undefined {
+  if (!ownsIsolatedWorktree || !workspace?.id || !workspace.repoRoot || !workspace.worktreePath) return undefined;
+  return {
+    id: workspace.id,
+    repoRoot: workspace.repoRoot,
+    worktreePath: workspace.worktreePath,
+    mode: "isolated",
+    ...(workspace.sourceCwd ? { sourceCwd: workspace.sourceCwd } : {}),
+    ...(workspace.branch ? { branch: workspace.branch } : {}),
+    ...(workspace.baseRef ? { baseRef: workspace.baseRef } : {}),
+    ...(workspace.baseSha ? { baseSha: workspace.baseSha } : {}),
+  };
+}
+
+export interface BufferedToolCall {
+  tool: string;
+  arguments: Record<string, unknown>;
+  idempotencyKey: string;
+}
+
+export interface RelayMcpProxyOptions {
+  webToolLog?: WebToolLogOptions;
+  redactToolResponse?: (response: Response) => Promise<Response>;
+  // The relay's MCP endpoint, e.g. http://localhost:4850/api/mcp.
+  relayMcpEndpoint: string;
+  // The runner's LIVE relay token (read on every forward so rotation is invisible to the agent).
+  getToken(): string | undefined;
+  // The bearer the agent must present to this localhost proxy (a per-session secret the runner
+  // mints and injects into the agent env). Decouples the agent from the rotating relay token.
+  authSecret: string;
+  // Persist a bufferable write durably for replay on reconnect (wired to the runner outbox).
+  enqueueBuffered(call: BufferedToolCall): void;
+  initialContext?: ProxyContext;
+  bufferableTools?: Set<string>;
+  // #1450 — the Codex-worker on-demand relay set to hide from the agent's DIRECT relay client so
+  // only the eager hot-path set boots. Undefined for every agent that isn't a lean Codex worker with
+  // callmux simultaneously fronting these tools (Claude, coordinators, reviewers, proxy-off) → no
+  // hiding, full surface. callmux's own relay downstream opts out via RELAY_MCP_ONDEMAND_SURFACE_HEADER.
+  onDemandTools?: Set<string>;
+  // Test seam.
+  fetchImpl?: FetchLike;
+  // Test seam for inspecting the deadline selected for each relay forward.
+  abortSignal?: (timeoutMs: number) => AbortSignal;
+  // #2089 — bounded connect-retry across a relay restart, GENERIC over every tool. Overridable for tests
+  // (fast/deterministic backoff + attempt cap); defaults ride out a CD-deploy's brief unreachable window.
+  //   attempts     — hard cap on forward attempts (safety bound; the window is the real bound in prod)
+  //   backoffMs    — base (first-retry) delay; doubles each retry up to maxBackoffMs
+  //   maxBackoffMs — per-delay ceiling
+  //   maxWindowMs  — total retry budget (sum of backoffs) before the original error is surfaced
+  connectRetry?: { attempts?: number; backoffMs?: number; maxBackoffMs?: number; maxWindowMs?: number };
+  // #1790 — deprecated alias for connectRetry (attempts/backoffMs only). `connectRetry` wins if both set.
+  spawnRetry?: { attempts?: number; backoffMs?: number };
+  // Test seam so the retry backoff doesn't sleep real wall-clock time in unit tests.
+  sleep?: (ms: number) => Promise<void>;
+  // Test seam for deterministic jitter (defaults to Math.random). Returns [0,1).
+  random?: () => number;
+}
+
+interface JsonRpcMessage {
+  jsonrpc?: string;
+  id?: string | number | null;
+  method?: string;
+  params?: unknown;
+}
+
+interface SseClient {
+  controller: ReadableStreamDefaultController<Uint8Array>;
+  keepalive: ReturnType<typeof setInterval>;
+}
+
+export class RelayMcpProxy {
+  private readonly redactToolResponse?: RelayMcpProxyOptions["redactToolResponse"];
+  private readonly relayMcpEndpoint: string;
+  private readonly getToken: () => string | undefined;
+  private readonly authSecret: string;
+  private readonly enqueueBuffered: (call: BufferedToolCall) => void;
+  private readonly bufferableTools: Set<string>;
+  private readonly onDemandTools?: Set<string>;
+  private readonly fetchImpl: FetchLike;
+  private readonly abortSignal: (timeoutMs: number) => AbortSignal;
+  private readonly retryAttempts: number;
+  private readonly retryBaseBackoffMs: number;
+  private readonly retryMaxBackoffMs: number;
+  private readonly retryWindowMs: number;
+  private readonly sleep: (ms: number) => Promise<void>;
+  private readonly random: () => number;
+  private readonly encoder = new TextEncoder();
+
+  private context: ProxyContext;
+  private server?: ReturnType<typeof Bun.serve>;
+  private readonly sseClients = new Set<SseClient>();
+  // Last successful relay tools/list — narrowed and served when the relay is briefly down so a
+  // read still works (reads serve from last-known where safe).
+  private lastRelayTools: Array<Record<string, unknown>> = [];
+  // Fingerprint of the definitions each consumer last saw (the agent's direct client, and callmux's
+  // on-demand front); a view stays undefined until that consumer first lists.
+  private readonly lastFingerprints: { direct?: string; front?: string } = {};
+
+  constructor(private readonly options: RelayMcpProxyOptions) {
+    this.redactToolResponse = options.redactToolResponse;
+    this.relayMcpEndpoint = options.relayMcpEndpoint;
+    this.getToken = options.getToken;
+    this.authSecret = options.authSecret;
+    this.enqueueBuffered = options.enqueueBuffered;
+    this.bufferableTools = options.bufferableTools ?? DEFAULT_BUFFERABLE_TOOLS;
+    this.onDemandTools = options.onDemandTools;
+    this.fetchImpl = options.fetchImpl ?? fetch;
+    this.abortSignal = options.abortSignal ?? AbortSignal.timeout;
+    const retry = options.connectRetry ?? options.spawnRetry;
+    this.retryAttempts = Math.max(1, retry?.attempts ?? DEFAULT_CONNECT_RETRY_ATTEMPTS);
+    this.retryBaseBackoffMs = Math.max(0, retry?.backoffMs ?? DEFAULT_CONNECT_RETRY_BASE_BACKOFF_MS);
+    this.retryMaxBackoffMs = Math.max(this.retryBaseBackoffMs, options.connectRetry?.maxBackoffMs ?? DEFAULT_CONNECT_RETRY_MAX_BACKOFF_MS);
+    this.retryWindowMs = Math.max(0, options.connectRetry?.maxWindowMs ?? DEFAULT_CONNECT_RETRY_WINDOW_MS);
+    this.sleep = options.sleep ?? ((ms: number) => new Promise<void>((resolve) => setTimeout(resolve, ms)));
+    this.random = options.random ?? Math.random;
+    this.context = options.initialContext ?? { isolatedWorktree: false };
+  }
+
+  get webToolsUrl(): string | undefined {
+    return this.options.webToolLog && this.server ? `http://127.0.0.1:${this.server.port}/web/${this.options.authSecret}` : undefined;
+  }
+
+  start(): { url: string; port: number } {
+    const self = this;
+    this.server = Bun.serve({
+      hostname: "127.0.0.1",
+      port: 0,
+      // SSE streams are long-lived; disable Bun's idle timeout and keep them alive with pings.
+      idleTimeout: 0,
+      fetch(req) {
+        return self.handle(req);
+      },
+    });
+    const port = this.server.port;
+    if (port === undefined) throw new Error("relay MCP proxy did not bind a port");
+    return { url: `http://127.0.0.1:${port}${PROXY_PATH}`, port };
+  }
+
+  stop(): void {
+    for (const client of this.sseClients) {
+      clearInterval(client.keepalive);
+      try { client.controller.close(); } catch { /* already closed */ }
+    }
+    this.sseClients.clear();
+    this.server?.stop(true);
+    this.server = undefined;
+  }
+
+  // The runner calls this on a workspace mode/state transition (active→ready→merged→terminal,
+  // or shared↔worktree). If it changes which tools the agent can see, emit list_changed so the
+  // agent's menu updates mid-session instead of staying frozen until reconnect.
+  setContext(context: ProxyContext): void {
+    this.context = context;
+    this.maybeEmitListChanged();
+  }
+
+  // The runner calls this after a token re-mint (scope may change, e.g. command:spawn) and on every
+  // bus re-register, since a deploy restarts the relay with new tools or schemas (#2973). Re-fetch
+  // the tool list with the live token and emit list_changed if the visible set changed; setContext
+  // covers the workspace mode/state path. Best-effort: a failed refresh keeps the last-known list.
+  async refreshTools(): Promise<void> {
+    const relay = await this.forward({ method: "tools/list", id: 0 }).catch(() => null);
+    const tools = relay && isRecord(relay.body?.result) && Array.isArray((relay.body!.result as Record<string, unknown>).tools)
+      ? ((relay.body!.result as Record<string, unknown>).tools as Array<Record<string, unknown>>)
+      : null;
+    if (!tools) return;
+    this.lastRelayTools = tools;
+    this.maybeEmitListChanged();
+  }
+
+  private async handle(req: Request): Promise<Response> {
+    const conflict = relayHeaderConflict(req.headers);
+    if (conflict) return Response.json({ error: conflict }, { status: 400 });
+    const url = new URL(req.url);
+    if (this.options.webToolLog && url.pathname === `/web/${this.options.authSecret}`) return forwardSharedMcp(req, this.options.webToolLog);
+    if (url.pathname !== PROXY_PATH) return new Response("not found", { status: 404 });
+    if (!this.authorized(req)) {
+      return Response.json(jsonRpcError(null, -32001, "proxy auth required"), { status: 401 });
+    }
+    // GET → open the server→client SSE notification channel (streamable-HTTP transport).
+    if (req.method === "GET") return this.openSse();
+    if (req.method === "DELETE") return new Response(null, { status: 204 });
+    if (req.method !== "POST") return new Response("method not allowed", { status: 405 });
+    return this.handleRpc(req);
+  }
+
+  private authorized(req: Request): boolean {
+    const header = req.headers.get("authorization") ?? "";
+    const bearer = header.startsWith("Bearer ") ? header.slice(7) : "";
+    return bearer === this.authSecret;
+  }
+
+  private async handleRpc(req: Request): Promise<Response> {
+    // #1450 — read the on-demand-front header BEFORE consuming the body. The per-agent metaOnly
+    // callmux's relay downstream sets it to opt OUT of the on-demand hiding (it fronts those tools);
+    // the agent's direct relay client never sets it, so it gets the eager hot-path set only.
+    const frontsOnDemand = relayHeader(req.headers, "ondemand-front") === RELAY_MCP_ONDEMAND_SURFACE_VALUE;
+    let message: JsonRpcMessage;
+    try {
+      const body = await req.json();
+      if (!body || typeof body !== "object" || Array.isArray(body)) {
+        return Response.json(jsonRpcError(null, -32600, "JSON-RPC body must be an object"));
+      }
+      message = body as JsonRpcMessage;
+    } catch {
+      return Response.json(jsonRpcError(null, -32700, "invalid JSON-RPC body"));
+    }
+    const id = message.id ?? null;
+    const method = message.method;
+
+    if (method === "initialize") return this.handleInitialize(id, message);
+    if (method === "tools/list") return this.handleToolsList(id, message, frontsOnDemand);
+    if (method === "tools/call") {
+      const response = await this.handleToolsCall(id, message);
+      return this.redactToolResponse ? this.redactToolResponse(response) : response;
+    }
+    // Everything else (notifications/initialized, ping, …) forwards verbatim.
+    return this.forwardRaw(message);
+  }
+
+  private async handleInitialize(id: string | number | null, message: JsonRpcMessage): Promise<Response> {
+    const relay = await this.forward({ ...message, method: "initialize", id }).catch((error) => {
+      logger.warn("mcp-proxy", `initialize forward failed: ${errMessage(error)}`);
+      return null;
+    });
+    // Degrade gracefully if the relay is momentarily down at connect: still hand the agent a
+    // usable initialize result advertising the proxy's capabilities.
+    const result: Record<string, unknown> = isRecord(relay?.body?.result) ? { ...(relay!.body!.result as Record<string, unknown>) } : {
+      protocolVersion: "2024-11-05",
+      serverInfo: { ...mcpServerInfo(parseToolGeneration(new URL(this.relayMcpEndpoint).searchParams.get("tools"))), version: "proxy" },
+    };
+    const caps = isRecord(result.capabilities) ? { ...result.capabilities } : {};
+    // The capability the relay's static endpoint doesn't advertise: live tool sets.
+    caps.tools = { ...(isRecord(caps.tools) ? caps.tools : {}), listChanged: true };
+    result.capabilities = caps;
+    return Response.json(jsonRpcResult(id, result));
+  }
+
+  private async handleToolsList(id: string | number | null, message: JsonRpcMessage, frontsOnDemand = false): Promise<Response> {
+    const relay = await this.forward({ ...message, method: "tools/list", id }).catch(() => null);
+    const tools = relay && isRecord(relay.body?.result) && Array.isArray((relay.body!.result as Record<string, unknown>).tools)
+      ? ((relay.body!.result as Record<string, unknown>).tools as Array<Record<string, unknown>>)
+      : null;
+    if (tools) this.lastRelayTools = tools; // refresh the last-known base list
+    // If the relay is down and we have no cache yet, surface an empty list rather than erroring —
+    // the agent can still operate (writes buffer; tools/list refreshes on the next call).
+    const base = tools ?? this.lastRelayTools;
+    // #1450 — callmux's relay downstream (frontsOnDemand) sees the full set so it can route
+    // callmux_call to any of it; the direct client has the on-demand set hidden.
+    const narrowed = this.narrow(base, frontsOnDemand);
+    this.lastFingerprints[frontsOnDemand ? "front" : "direct"] = toolsFingerprint(narrowed);
+    return Response.json(jsonRpcResult(id, { tools: narrowed }));
+  }
+
+  // Strict subset of the relay's already-scope-filtered list. Only removes — never adds. Narrowing
+  // rules: (1) workspace tools apply only to an agent that owns a live worktree; (2) #1450 — the
+  // on-demand relay set is hidden from the agent's DIRECT client (only when `hideOnDemand`), since
+  // the per-agent metaOnly callmux fronts those tools on demand. `frontsOnDemand` (callmux's own
+  // request) skips rule 2 so callmux sees the full relay set.
+  private narrow(tools: Array<Record<string, unknown>>, frontsOnDemand = false): Array<Record<string, unknown>> {
+    const hideOnDemand = !frontsOnDemand && this.onDemandTools !== undefined && this.onDemandTools.size > 0;
+    return tools.filter((tool) => {
+      const name = registryToolName(typeof tool.name === "string" ? tool.name : "");
+      if (WORKTREE_ONLY_TOOLS.has(name) && !this.context.isolatedWorktree) return false;
+      if (hideOnDemand && this.onDemandTools!.has(name)) return false;
+      return true;
+    });
+  }
+
+  private async handleToolsCall(id: string | number | null, message: JsonRpcMessage): Promise<Response> {
+    const params = isRecord(message.params) ? message.params : {};
+    const toolName = registryToolName(typeof params.name === "string" ? params.name : "");
+    let args = isRecord(params.arguments) ? params.arguments : {};
+
+    // #1502 — inject this agent's workspace descriptor so the relay can self-heal a missing/dropped
+    // workspace registration during delivery. Only for worktree agents, only when the agent didn't
+    // already supply one, and the augmented args are forwarded (not just used locally).
+    if (
+      WORKSPACE_SELF_HEAL_TOOLS.has(toolName) &&
+      this.context.isolatedWorktree &&
+      this.context.workspaceDescriptor &&
+      args.workspace === undefined
+    ) {
+      args = { ...args, workspace: this.context.workspaceDescriptor };
+      message = { ...message, params: { ...params, arguments: args } };
+    }
+
+    // #1790 — stabilize the spawn correlation id BEFORE the first attempt so a retry across a relay
+    // restart reuses it and the server dedupes to the SAME spawn (never a second agent). The caller's
+    // own id wins if supplied; otherwise mint an fs-safe `sp_<uuid>` here, because the relay would
+    // otherwise mint a FRESH id on each attempt and a retry would double-create. This is id-provisioning,
+    // not per-tool retry: it makes spawn qualify for the GENERIC ambiguous-resend rule below.
+    if (toolName === SPAWN_TOOL) {
+      const supplied = typeof args.spawnRequestId === "string" ? args.spawnRequestId.trim() : "";
+      if (!supplied) {
+        args = { ...args, spawnRequestId: `sp_${crypto.randomUUID()}` };
+        message = { ...message, params: { ...(isRecord(message.params) ? message.params : {}), arguments: args } };
+      }
+    }
+
+    // #2857 — a `path` upload names a file on THIS host, but the relay reads it from
+    // its own filesystem, so remote agents always got "path not found". Materialize the
+    // file here and forward bytes; rejections mirror the relay's own messages verbatim.
+    if (toolName === "relay_upload_artifact") {
+      const materialized = await materializeArtifactPathUpload(args, this.getToken);
+      if (materialized.error !== undefined) return Response.json(jsonRpcError(id, -32602, materialized.error));
+      if (materialized.args !== undefined) {
+        args = materialized.args;
+        message = { ...message, params: { ...(isRecord(message.params) ? message.params : {}), arguments: args } };
+      }
+    }
+
+    // #2089 — the call is safe to resend after an AMBIGUOUS (sent-then-response-lost) failure only if it
+    // carries an idempotency key the server dedupes on. Read generically off the args (spawnRequestId or
+    // idempotencyKey) — never off the tool name — so every idempotent tool qualifies and no other does.
+    const idempotent = isIdempotentCall(args);
+
+    const relay = await this.forwardWithConnectionRetry(message, toolName, idempotent).catch((error) => {
+      // A thrown fetch = transport failure (relay unreachable / DNS / connection refused / socket closed).
+      // The generic connect-retry (#2089) has already exhausted its bounded window by here.
+      return { ok: false, status: 0, body: null, transportError: errMessage(error) } as ForwardResult;
+    });
+
+    const relayDown = !relay.ok && (relay.status === 0 || GATEWAY_STATUSES.has(relay.status));
+    if (relayDown && this.bufferableTools.has(toolName)) {
+      // Durably queue the write and tell the agent it's safely accepted. It replays on reconnect.
+      const idempotencyKey = typeof args.idempotencyKey === "string" && args.idempotencyKey
+        ? args.idempotencyKey
+        : `mcp-${toolName}-${crypto.randomUUID()}`;
+      this.enqueueBuffered({ tool: toolName, arguments: { ...args, idempotencyKey }, idempotencyKey });
+      logger.info("mcp-proxy", `relay unreachable — buffered ${toolName} (idempotencyKey=${idempotencyKey})`);
+      return Response.json(jsonRpcResult(id, toolResult({
+        queued: true,
+        tool: toolName,
+        idempotencyKey,
+        note: "Relay was unreachable; your call was queued durably and will be delivered automatically when the relay comes back. It is not lost.",
+      })));
+    }
+
+    // #1418: propagate the relay's actual status (e.g. 413 payload-too-large, 400 validation,
+    // 403 auth) instead of the default 200 `Response.json` gives an untouched status option —
+    // a hard rejection silently reads as a normal 200 response to anything downstream that
+    // branches on HTTP status rather than parsing the JSON-RPC `error` field, masking a large
+    // delivery's outright rejection as an ambiguous "success". `forwardRaw` below already gets
+    // this right; `tools/call` was the one path that didn't.
+    if (relay.body) return Response.json(relay.body, { status: relay.status || 200 });
+    // Relay down and not bufferable: a real error the agent must see (and can retry).
+    return Response.json(jsonRpcError(id, -32002, `relay unreachable: ${relay.transportError ?? `status ${relay.status}`}`));
+  }
+
+  // Verbatim forward for methods with no proxy-specific handling. Returns the relay's response
+  // body+status unchanged (or a JSON-RPC error if the relay is down).
+  private async forwardRaw(message: JsonRpcMessage): Promise<Response> {
+    const relay = await this.forward(message).catch((error) => {
+      return { ok: false, status: 0, body: null, transportError: errMessage(error) } as ForwardResult;
+    });
+    if (relay.body) return Response.json(relay.body, { status: relay.status || 200 });
+    // Notifications (no id) tolerate a down relay — ack locally; requests get an error.
+    if (message.id === undefined || message.id === null) return new Response(null, { status: 202 });
+    return Response.json(jsonRpcError(message.id, -32002, `relay unreachable: ${relay.transportError ?? `status ${relay.status}`}`));
+  }
+
+  // POST the JSON-RPC message to the relay with the runner's LIVE token. The agent's incoming
+  // bearer is the proxy secret; we substitute the real relay credential here so the agent never
+  // holds it and token rotation is invisible. The body is re-serialized from the already-parsed
+  // message (the request stream was consumed during dispatch and can't be re-read).
+  private async forward(message: JsonRpcMessage): Promise<ForwardResult> {
+    return resendOnRotatedToken(this.getToken, (token) => this.forwardWithToken(message, token));
+  }
+
+  private async forwardWithToken(message: JsonRpcMessage, token: string | undefined): Promise<ForwardResult> {
+    const headers: Record<string, string> = { "content-type": "application/json" };
+    if (token) headers.authorization = `Bearer ${token}`;
+    const payload: Record<string, unknown> = { jsonrpc: "2.0", method: message.method };
+    if (message.id !== undefined) payload.id = message.id;
+    if (message.params !== undefined) payload.params = message.params;
+    // #2429 — Bun 1.3.x's native HTTP idle watchdog aborts any fetch that sees no response bytes
+    // for ~300s, and an AbortSignal does NOT govern it: the server awaits the WHOLE tool call
+    // before its first response byte (src/mcp/index.ts `tools/call` → Response.json, no streaming),
+    // so a legitimate >~300s tool call was killed client-side at ~300s despite the 610s budget.
+    // `timeout: false` disables the watchdog; the AbortSignal below remains the real deadline.
+    const init: RelayForwardRequestInit = {
+      method: "POST",
+      headers,
+      body: JSON.stringify(payload),
+      signal: this.abortSignal(relayMcpForwardTimeoutMs(message)),
+      timeout: false,
+    };
+    const response = await this.fetchImpl(this.relayMcpEndpoint, init);
+    const text = await response.text();
+    let parsed: Record<string, unknown> | null = null;
+    if (text) { try { parsed = JSON.parse(text); } catch { parsed = null; } }
+    return { ok: response.ok, status: response.status, body: parsed };
+  }
+
+  // #2089 — GENERIC bounded connect-retry so a relay restart (CD deploy, no drain) is invisible to the
+  // agent for EVERY tool, not just spawn. One chokepoint, uniform across the tool suite. The retry
+  // decision is driven purely by the failure CLASS (connect / ambiguous / fatal) and the call's
+  // idempotency — never by tool name:
+  //   - connect-phase (request never executed): retry unconditionally — covers reads, spawns, everything.
+  //   - ambiguous (sent-then-response-lost): retry ONLY when `idempotent` (the call carries a key the
+  //     server dedupes on), else surface — a blind resend of a non-idempotent mutation could double-apply.
+  //   - fatal (a genuine relay response, incl. any 4xx/5xx the relay actually produced, or an unrelated
+  //     error): returned/thrown as-is immediately — a real error is never masked, so non-transient
+  //     failures add ZERO latency.
+  // Backoff climbs exponentially (jittered, capped) and the total is bounded by both an attempt cap and
+  // a wall-budget window; on exhaustion the original failure is rethrown/returned so the caller's
+  // existing relay-down handling (buffer / surface `-32002 relay unreachable`) runs unchanged.
+  private async forwardWithConnectionRetry(message: JsonRpcMessage, toolName: string, idempotent: boolean): Promise<ForwardResult> {
+    let attempt = 0;
+    let spentMs = 0;
+    for (;;) {
+      attempt += 1;
+      try {
+        const result = await this.forward(message);
+        if (!result.ok) {
+          const cls = classifyGatewayStatus(result.status);
+          if (this.shouldRetry(cls, idempotent) && this.canRetryAgain(attempt, spentMs)) {
+            const delay = this.backoffFor(attempt);
+            logger.info("mcp-proxy", `relay ${result.status} on ${toolName} — relay restarting, retrying in ${delay}ms (attempt ${attempt})`);
+            await this.sleep(delay);
+            spentMs += delay;
+            continue;
+          }
+        }
+        return result; // ok, a genuine relay answer, or retry exhausted → caller handles it
+      } catch (error) {
+        const cls = classifyTransportError(error);
+        if (!this.shouldRetry(cls, idempotent) || !this.canRetryAgain(attempt, spentMs)) throw error;
+        const delay = this.backoffFor(attempt);
+        logger.info("mcp-proxy", `relay unreachable on ${toolName} — relay restarting, retrying in ${delay}ms (attempt ${attempt}: ${errMessage(error)})`);
+        await this.sleep(delay);
+        spentMs += delay;
+      }
+    }
+  }
+
+  // Whether a failure of the given class is safe to retry for this call. Connect-phase is always safe
+  // (never executed); ambiguous only when idempotent; fatal never.
+  private shouldRetry(cls: TransportErrorClass, idempotent: boolean): boolean {
+    if (cls === "connect") return true;
+    if (cls === "ambiguous") return idempotent;
+    return false;
+  }
+
+  // The retry window is bounded by BOTH a hard attempt cap and a wall-budget (sum of backoffs). Checked
+  // before sleeping, so the Nth (final) attempt's failure surfaces instead of sleeping pointlessly.
+  private canRetryAgain(attempt: number, spentMs: number): boolean {
+    return attempt < this.retryAttempts && spentMs < this.retryWindowMs;
+  }
+
+  // Exponential backoff (base·2^(n-1)) capped at maxBackoffMs, then jittered to 0.5×–1.5× to spread a
+  // fleet's reconnects. With base 0 (tests) every delay is 0 and the attempt cap governs.
+  private backoffFor(attempt: number): number {
+    const raw = Math.min(this.retryBaseBackoffMs * 2 ** (attempt - 1), this.retryMaxBackoffMs);
+    return Math.round(raw * (0.5 + this.random()));
+  }
+
+  private openSse(): Response {
+    const self = this;
+    let client: SseClient;
+    const stream = new ReadableStream<Uint8Array>({
+      start(controller) {
+        controller.enqueue(self.encoder.encode(": connected\n\n"));
+        const keepalive = setInterval(() => {
+          try { controller.enqueue(self.encoder.encode(": keepalive\n\n")); } catch { /* closed */ }
+        }, SSE_KEEPALIVE_MS);
+        keepalive.unref?.();
+        client = { controller, keepalive };
+        self.sseClients.add(client);
+      },
+      cancel() {
+        if (client) {
+          clearInterval(client.keepalive);
+          self.sseClients.delete(client);
+        }
+      },
+    });
+    return new Response(stream, {
+      headers: { "content-type": "text/event-stream", "cache-control": "no-cache", connection: "keep-alive" },
+    });
+  }
+
+  private maybeEmitListChanged(): void {
+    // Definitions, not names: a new parameter on an existing tool is a change too (#2973). A view
+    // nobody has listed yet needs no notification; its first list is current.
+    let changed = false;
+    for (const view of ["direct", "front"] as const) {
+      const previous = this.lastFingerprints[view];
+      if (previous === undefined) continue;
+      const fingerprint = toolsFingerprint(this.narrow(this.lastRelayTools, view === "front"));
+      if (fingerprint !== previous) changed = true;
+      this.lastFingerprints[view] = fingerprint;
+    }
+    if (!changed) return;
+    const frame = this.encoder.encode(`event: message\ndata: ${JSON.stringify({ jsonrpc: "2.0", method: "notifications/tools/list_changed" })}\n\n`);
+    for (const client of this.sseClients) {
+      try { client.controller.enqueue(frame); } catch { /* dropped client; cancel() cleans up */ }
+    }
+    logger.debug("mcp-proxy", `tools/list_changed emitted to ${this.sseClients.size} client(s)`);
+  }
+
+  // Test/observability hooks.
+  sseClientCount(): number { return this.sseClients.size; }
+  narrowedToolNames(): string[] { return this.narrow(this.lastRelayTools).map((t) => String(t.name)); }
+}
+
+function relayMcpForwardTimeoutMs(message: JsonRpcMessage): number {
+  return message.method === "tools/call"
+    ? RELAY_MCP_TOOL_CALL_FORWARD_TIMEOUT_MS
+    : RELAY_MCP_METADATA_FORWARD_TIMEOUT_MS;
+}
+
+interface ForwardResult {
+  ok: boolean;
+  status: number;
+  body: Record<string, unknown> | null;
+  transportError?: string;
+}
+
+function toolsFingerprint(tools: Array<Record<string, unknown>>): string {
+  const sorted = [...tools].sort((a, b) => String(a.name).localeCompare(String(b.name)));
+  return String(Bun.hash(JSON.stringify(sorted)));
+}
+
+function jsonRpcResult(id: string | number | null, result: unknown): Record<string, unknown> {
+  return { jsonrpc: "2.0", id, result };
+}
+
+function jsonRpcError(id: string | number | null, code: number, message: string): Record<string, unknown> {
+  return { jsonrpc: "2.0", id, error: { code, message } };
+}
+
+function toolResult(result: unknown): Record<string, unknown> {
+  return { content: [{ type: "text", text: JSON.stringify(result, null, 2) }], structuredContent: result };
+}