mulmoclaude 0.6.2 → 0.6.3

package/server/agent/backend/fake-echo.ts

@@ -0,0 +1,370 @@
+// Test-only LLM backend. Loaded by `getActiveBackend()` only when
+// `MULMOCLAUDE_FAKE_AGENT=1` (CI workflow boot wiring), and re-usable
+// from unit tests via `setFakeResponse()` / `resetFakeResponse()`.
+//
+// Default behavior:
+//   - emits a synthesized `claudeSessionId` so the orchestrator's
+//     resume bookkeeping sees the same shape as a real run
+//   - short-circuits `/<slug>` slash-command turns by reading the
+//     seeded SKILL.md and echoing the canary marker line
+//   - emits the concatenated per-session message history as the
+//     assistant text reply, so context-recall tests (session L-12)
+//     see prior turn content
+//
+// Tool dispatch: when the user prompt matches a known shape (see
+// detectToolCalls), fake-echo emits the corresponding tool_call
+// AND posts the args to the same internal plugin endpoint the MCP
+// bridge would use under real Claude (see PLUGIN_ENDPOINTS). The
+// handler runs unmodified, the artifact lands on disk, and the
+// canvas mounts the plugin View — fake at the LLM seam only, real
+// from the tool dispatch downward. Tests that need an LLM that
+// actually reasons (presentForm field design, agent-driven slug
+// choice in skill creation, etc.) still stay gated on
+// `E2E_LIVE_NO_LLM=1`.
+
+import { randomUUID } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+
+import { getCurrentToken } from "../../api/auth/token.js";
+import { makeUuid } from "../../utils/id.js";
+import { API_ROUTES } from "../../../src/config/apiRoutes.js";
+import { EVENT_TYPES } from "../../../src/types/events.js";
+import { WORKSPACE_DIRS } from "../../workspace/paths.js";
+import type { AgentEvent } from "../stream.js";
+import type { AgentInput, LLMBackend } from "./types.js";
+
+interface PluginEnvelope {
+  data?: unknown;
+  message?: unknown;
+  instructions?: unknown;
+  [key: string]: unknown;
+}
+
+export interface FakeToolCall {
+  toolName: string;
+  args: unknown;
+  /** Result string emitted in the matching `tool_call_result`.
+   *  Defaults to `{ ok: true }` JSON. */
+  result?: string;
+}
+
+export interface FakeResponse {
+  /** Tool calls emitted before the text block. Default generator
+   *  never emits any — tests that want tool events drive them
+   *  through `setFakeResponse()`. */
+  toolCalls?: readonly FakeToolCall[];
+  /** Assistant text. Omit to skip the text event entirely. */
+  text?: string;
+  /** When set, emit a single `error` AgentEvent with this message
+   *  and stop — mirrors what the claude-code backend does when the
+   *  CLI exits non-zero (`readAgentEvents`). Tool calls / text that
+   *  would otherwise follow are suppressed. */
+  error?: string;
+  /** Emit the `tool_call` for each `toolCalls` entry but NOT the
+   *  paired `tool_call_result` — simulates a truncated / partial
+   *  stream where the model died mid tool round-trip. */
+  omitToolResult?: boolean;
+}
+
+export type FakeResponseFn = (input: AgentInput) => FakeResponse | Promise<FakeResponse>;
+
+// Per-session conversation memory so context-recall tests see prior
+// turn content in the reply. Cleared by `resetFakeResponse()`.
+const sessionTurns = new Map<string, string[]>();
+
+async function defaultResponse(input: AgentInput): Promise<FakeResponse> {
+  // Slash-command turn shape: the SPA's "Run" button on a skill row
+  // (e2e-live L-22) starts a new chat with `/<slug>` as the only
+  // user message. Real Claude resolves this through its skill
+  // pipeline and uses the SKILL.md body as system prompt; here we
+  // short-circuit to read the seeded body and apply the
+  // "respond with this exact line" heuristic the e2e-live canaries
+  // rely on. Falls through to default echo on no match.
+  // Prompt-driven error trigger for e2e-live. The in-process
+  // `setFakeResponse()` knob is unreachable from a browser-driven
+  // spec (separate process), so the error-banner UI canary opts in
+  // by sending a message containing this exact marker. Prod never
+  // reaches fake-echo (real Claude backend) so this is inert there.
+  if (input.message.includes("__FAKE_ERROR__")) {
+    // Message text is rendered through marked() in the chat card,
+    // so keep it free of markdown-significant characters (no `__`,
+    // `*`, backticks) — the e2e-live canary asserts on a literal
+    // substring of this string.
+    return { error: "fake-echo forced error for the e2e-live error-banner canary" };
+  }
+
+  const slashMatch = input.message.trim().match(/^\/([a-z0-9][a-z0-9-]*)$/i);
+  if (slashMatch) {
+    const skillReply = await replyFromSeededSkill(input.workspacePath, slashMatch[1]);
+    if (skillReply !== null) return { text: skillReply };
+  }
+
+  const history = sessionTurns.get(input.sessionId) ?? [];
+  history.push(input.message);
+  sessionTurns.set(input.sessionId, history);
+
+  const toolCalls = detectToolCalls(input.message);
+  return {
+    toolCalls,
+    text: history.join("\n\n"),
+  };
+}
+
+// ── Tool-call pattern detectors ───────────────────────────────────
+//
+// Each detector matches one e2e-live prompt shape. The fake-echo
+// loop below dispatches each detected call to the matching server-
+// side plugin endpoint (see PLUGIN_ENDPOINTS) so the real handler
+// runs, the artifact lands on disk, and the canvas mounts the View.
+// Production never reaches this code path — gated by
+// MULMOCLAUDE_FAKE_AGENT=1 at server boot.
+
+function detectPresentMulmoScript(message: string): FakeToolCall | null {
+  if (!/presentMulmoScript/i.test(message)) return null;
+  const filePathMatch = message.match(/filePath:\s*["']([^"']+)["']/);
+  if (!filePathMatch) return null;
+  return { toolName: "presentMulmoScript", args: { filePath: filePathMatch[1] } };
+}
+
+function detectPresentHtml(message: string): FakeToolCall | null {
+  if (!/presentHtml/i.test(message)) return null;
+  const idx = message.indexOf("<");
+  if (idx < 0) return null;
+  // The handler expects a self-contained document; wrap if the
+  // prompt only supplies fragments (the spec's prompt does).
+  const fragment = message.slice(idx).trim();
+  const html = /^<!DOCTYPE/i.test(fragment) ? fragment : `<!DOCTYPE html><html><body>${fragment}</body></html>`;
+  return { toolName: "presentHtml", args: { html } };
+}
+
+function detectPresentForm(message: string): FakeToolCall | null {
+  if (!/presentForm/i.test(message)) return null;
+  const titleMatch = message.match(/titled\s+['"]([^'"]+)['"]/i);
+  const idMatch = message.match(/id\s*=\s*['"]([^'"]+)['"]/i);
+  const labelMatch = message.match(/label\s*=\s*['"]([^'"]+)['"]/i);
+  return {
+    toolName: "presentForm",
+    args: {
+      title: titleMatch?.[1] ?? "Quick check",
+      fields: [
+        {
+          id: idMatch?.[1] ?? "field1",
+          type: "text",
+          label: labelMatch?.[1] ?? "Field",
+          required: /required/i.test(message),
+          description: "auto-generated by fake-echo",
+        },
+      ],
+    },
+  };
+}
+
+function detectPresentChart(message: string): FakeToolCall | null {
+  if (!/presentChart/i.test(message)) return null;
+  const titleMatch = message.match(/titled\s+['"]([^'"]+)['"]/i);
+  const pairs = Array.from(message.matchAll(/\b([A-Za-z]{3,})\s+(\d{1,6})\b/g)).map(([, label, value]) => ({ label, value: Number(value) }));
+  const labels = pairs.length > 0 ? pairs.map((pair) => pair.label) : ["A", "B", "C"];
+  const values = pairs.length > 0 ? pairs.map((pair) => pair.value) : [1, 2, 3];
+  const title = titleMatch?.[1] ?? "Untitled";
+  return {
+    toolName: "presentChart",
+    args: {
+      document: {
+        title,
+        charts: [
+          {
+            title,
+            type: "bar",
+            option: {
+              xAxis: { type: "category", data: labels },
+              yAxis: { type: "value" },
+              series: [{ type: "bar", data: values }],
+            },
+          },
+        ],
+      },
+    },
+  };
+}
+
+function detectToolCalls(message: string): FakeToolCall[] | undefined {
+  const calls: FakeToolCall[] = [];
+  for (const detector of [detectPresentMulmoScript, detectPresentHtml, detectPresentForm, detectPresentChart]) {
+    const call = detector(message);
+    if (call) calls.push(call);
+  }
+  return calls.length > 0 ? calls : undefined;
+}
+
+// ── Plugin dispatch ───────────────────────────────────────────────
+//
+// Maps each fake-detected tool to the same internal API the MCP
+// bridge would post to in a real run, so the actual server-side
+// handler runs end-to-end: artifact saved, canvas slug returned.
+// Anything not in this table falls back to a synthesized success
+// envelope (caller can override via FakeToolCall.result).
+const PLUGIN_ENDPOINTS: Readonly<Record<string, string>> = {
+  presentForm: "/api/form",
+  presentHtml: "/api/html",
+  presentChart: "/api/chart",
+  presentMulmoScript: "/api/mulmoScript/save",
+};
+
+// Mirrors what server/agent/mcp-server.ts#handleToolCall does for
+// the real MCP bridge:
+//   1. POST to the plugin endpoint to get the envelope back
+//   2. If envelope.data is set, PUSH the envelope to
+//      /api/internal/tool-result — this is what surfaces the result
+//      to the canvas as a ToolResultComplete (toolName + uuid
+//      stamped by the bridge so the plugin can't impersonate).
+//   3. Return the text representation (message + instructions) so
+//      the matching `tool_call_result` event carries something
+//      meaningful for the tool-call history pane.
+async function dispatchToPlugin(call: FakeToolCall, port: number, chatSessionId: string): Promise<string> {
+  if (call.result !== undefined) return call.result;
+  const endpoint = PLUGIN_ENDPOINTS[call.toolName];
+  if (!endpoint) return '{"ok":true}';
+  const token = getCurrentToken();
+  const authHeaders: Record<string, string> = token ? { Authorization: `Bearer ${token}` } : {};
+  try {
+    const response = await fetch(`http://localhost:${port}${endpoint}`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json", ...authHeaders },
+      body: JSON.stringify(call.args),
+    });
+    if (!response.ok) {
+      const errBody = await response.text();
+      return JSON.stringify({ error: `plugin ${call.toolName} returned ${response.status}: ${errBody.slice(0, 200)}` });
+    }
+    const envelope = ((await response.json()) ?? {}) as PluginEnvelope;
+    if (envelope.data !== undefined) {
+      // Query key is `session`, not `chatSessionId` — matches the
+      // `getSessionQuery(req)` reader and what the MCP bridge's
+      // postJson(...) helper passes (`?session=${SESSION_ID}`).
+      const toolResultUrl = `http://localhost:${port}${API_ROUTES.agent.internal.toolResult}?session=${encodeURIComponent(chatSessionId)}`;
+      const pushRes = await fetch(toolResultUrl, {
+        method: "POST",
+        headers: { "Content-Type": "application/json", ...authHeaders },
+        body: JSON.stringify({ ...envelope, toolName: call.toolName, uuid: makeUuid() }),
+      });
+      if (!pushRes.ok) {
+        // Fail loudly per codex review — a swallowed publish would
+        // leave the canvas blank while the chat reads "Done", which
+        // masks a real wiring break. Surface the failure as the
+        // tool result so the test fails loud instead of timing out
+        // on an absent View.
+        const errBody = await pushRes.text();
+        return JSON.stringify({
+          error: `tool-result push failed for ${call.toolName}: ${pushRes.status} ${errBody.slice(0, 200)}`,
+        });
+      }
+    }
+    const text: string[] = [];
+    if (typeof envelope.message === "string") text.push(envelope.message);
+    if (typeof envelope.instructions === "string") text.push(envelope.instructions);
+    return text.length > 0 ? text.join("\n") : "Done";
+  } catch (err) {
+    // Don't tear down the chat turn on plugin-dispatch failure —
+    // surface the error in the tool_result so the test sees it.
+    return JSON.stringify({ error: err instanceof Error ? err.message : String(err) });
+  }
+}
+
+// Look up a project-scope skill seeded by `placeProjectSkill` and
+// extract the canary line the seeded body asks the model to echo
+// back ("respond with this exact line and nothing else: X").
+// Returns null when the file is missing or the marker shape is
+// absent — caller falls through to default echo.
+async function replyFromSeededSkill(workspacePath: string, slug: string): Promise<string | null> {
+  const skillFile = path.join(workspacePath, WORKSPACE_DIRS.claudeSkills, slug, "SKILL.md");
+  let body: string;
+  try {
+    body = await readFile(skillFile, "utf8");
+  } catch {
+    return null;
+  }
+  // Line-by-line scan to avoid backtracking surprises.
+  for (const line of body.split(/\r?\n/)) {
+    const match = line.match(/respond with this exact line(?: and nothing else)?:\s*(.+)/i);
+    if (match) return match[1].trim();
+  }
+  return null;
+}
+
+// ── Backend wiring ────────────────────────────────────────────────
+
+let responseFn: FakeResponseFn = defaultResponse;
+
+/** Replace the default echo + slash-command generator. Useful for
+ *  unit tests that want full control over what the fake backend
+ *  emits. Pair with `resetFakeResponse()` in teardown so the next
+ *  test sees a clean state. */
+export function setFakeResponse(generator: FakeResponseFn): void {
+  responseFn = generator;
+}
+
+/** Restore the default generator AND clear per-session history. */
+export function resetFakeResponse(): void {
+  responseFn = defaultResponse;
+  sessionTurns.clear();
+}
+
+// Abort is checked between every yield. Real claude-code kills the
+// subprocess on abort; the echo stub has no subprocess, so the
+// faithful equivalent is "stop emitting immediately".
+function aborted(input: AgentInput): boolean {
+  return input.abortSignal?.aborted === true;
+}
+
+async function* runFakeEchoAgent(input: AgentInput): AsyncGenerator<AgentEvent> {
+  if (aborted(input)) return;
+  yield { type: EVENT_TYPES.claudeSessionId, id: randomUUID() };
+
+  const response = await responseFn(input);
+
+  // Error short-circuit: surface the error and stop, exactly like
+  // the claude-code backend on a non-zero CLI exit.
+  if (response.error !== undefined) {
+    if (aborted(input)) return;
+    yield { type: EVENT_TYPES.error, message: response.error };
+    return;
+  }
+
+  for (const call of response.toolCalls ?? []) {
+    if (aborted(input)) return;
+    const toolUseId = `fake-${randomUUID()}`;
+    yield {
+      type: EVENT_TYPES.toolCall,
+      toolUseId,
+      toolName: call.toolName,
+      args: call.args,
+    };
+    // Partial-stream simulation: skip the result half.
+    if (response.omitToolResult) continue;
+    // Run the actual plugin handler AND push the envelope to
+    // /api/internal/tool-result so the canvas mounts the View — same
+    // two-step the MCP bridge does for real Claude.
+    const content = await dispatchToPlugin(call, input.port, input.sessionId);
+    if (aborted(input)) return;
+    yield {
+      type: EVENT_TYPES.toolCallResult,
+      toolUseId,
+      content,
+    };
+  }
+
+  if (response.text !== undefined && !aborted(input)) {
+    yield { type: EVENT_TYPES.text, message: response.text };
+  }
+}
+
+export const fakeEchoBackend: LLMBackend = {
+  id: "fake-echo",
+  // Resume-by-token / MCP aren't meaningfully replayable from a
+  // stub. Flag them unsupported so callers that depend on the real
+  // Claude semantics opt out instead of getting silently wrong
+  // behavior.
+  capabilities: { sessionResume: false, mcp: false },
+  runAgent: runFakeEchoAgent,
+};

package/server/agent/backend/index.ts

@@ -3,12 +3,27 @@
 // env / settings. Callers go through getActiveBackend() rather than
 // importing a concrete adapter so adding a backend doesn't require
 // touching every call site.
+//
+// Tests / CI swap in `fakeEchoBackend` via setActiveBackend() at
+// server bootstrap; the decision is made once and read with zero
+// per-call overhead by the agent orchestrator.
 
 import { claudeCodeBackend } from "./claude-code.js";
 import type { LLMBackend } from "./types.js";
 
 export type { AgentInput, BackendCapabilities, LLMBackend } from "./types.js";
 
+let activeBackend: LLMBackend = claudeCodeBackend;
+
+/** Replace the active backend. Intended for server-bootstrap wiring
+ *  (e.g. CI sets `MULMOCLAUDE_FAKE_AGENT=1`, the boot script then
+ *  passes `fakeEchoBackend` here). Not safe to call mid-flight — the
+ *  in-flight agent generators have already captured the previous
+ *  backend reference, and swapping under them would race. */
+export function setActiveBackend(backend: LLMBackend): void {
+  activeBackend = backend;
+}
+
 export function getActiveBackend(): LLMBackend {
-  return claudeCodeBackend;
+  return activeBackend;
 }

package/server/agent/config.ts

@@ -10,6 +10,7 @@ import type { Attachment } from "@mulmobridge/protocol";
 import { isImageMime, isNativeAttachmentMime } from "@mulmobridge/client";
 import { convertAttachment } from "./attachmentConverter.js";
 import { log } from "../system/logger/index.js";
+import { preflightUserServers, logPreflightResult } from "./mcpPreflight.js";
 
 export const CONTAINER_WORKSPACE_PATH = "/home/node/mulmoclaude";
 
@@ -73,8 +74,14 @@ function prepareUserStdioServer(spec: Extract<McpServerSpec, { type: "stdio" }>,
 }
 
 export function prepareUserServers(userServers: Record<string, McpServerSpec>, useDocker: boolean, hostWorkspacePath: string): Record<string, McpServerSpec> {
+  // Drop catalog-known entries that are missing required config (#1352).
+  // The dedup cache inside `logPreflightResult` keeps per-agent-run
+  // calls quiet so a Settings UI fix only logs once when it transitions
+  // missing → ok.
+  const preflight = preflightUserServers(userServers);
+  logPreflightResult(preflight, "agent-run");
   const out: Record<string, McpServerSpec> = {};
-  for (const [serverId, spec] of Object.entries(userServers)) {
+  for (const [serverId, spec] of Object.entries(preflight.ready)) {
     if (spec.enabled === false) continue;
     if (spec.type === "http") {
       out[serverId] = prepareUserHttpServer(spec, useDocker);

package/server/agent/mcpFailureMonitor.ts

@@ -0,0 +1,167 @@
+// Runtime failure monitor for external MCP servers (#1353).
+//
+// `mcpPreflight.ts` (#1352) handles the static case: servers that
+// can't even start because of missing required config. This module
+// handles the dynamic case: servers that start OK but fail to answer
+// tool calls (API key rotated, upstream down, OAuth scope wrong …).
+//
+// Signal source is `tool_result.is_error: true` from Claude Code's
+// stream-json. `stream.ts` now forwards that flag as
+// `AgentEvent.toolCallResult.isError`; this monitor consumes those
+// events, attributes errors to the originating MCP server (parsed
+// from `mcp__<server>__<tool>` names cached at toolCall time), and
+// fires a single warn + bell notification per server once a
+// consecutive-failure threshold is crossed.
+//
+// What the monitor intentionally does NOT do:
+//   - Restart the MCP server (operator's call).
+//   - Capture MCP subprocess stderr (Claude Agent SDK holds that —
+//     out of scope for #1353).
+//   - Auto-dismiss the bell entry when calls recover. The notification
+//     engine has no dismissal API exposed yet; future work can wire
+//     it once that lands. For now: bell stays until user dismisses.
+
+import { EVENT_TYPES } from "../../src/types/events.js";
+import { NOTIFICATION_ACTION_TYPES, NOTIFICATION_PRIORITIES } from "../../src/types/notification.js";
+import { publishNotification } from "../events/notifications.js";
+import { log } from "../system/logger/index.js";
+
+export const MCP_FAILURE_THRESHOLD = 3;
+
+// Server-id contract — mirrors `isMcpServerId` in
+// `server/system/config.ts`. Single `_` is allowed; consecutive
+// `__` is forbidden because it would collide with the
+// `mcp__<server>__<tool>` delimiter and make the parser ambiguous
+// (Codex iter-2 on #1356). Both ends agreeing on this shape is
+// what lets the monitor attribute failures back to the right
+// server entry in `mcp.json`.
+const MCP_SERVER_ID_PATTERN = /^[a-z][a-z0-9_-]{0,63}$/;
+const MCP_PREFIX = "mcp__";
+const MCP_DELIM = "__";
+
+function isValidServerId(value: string): boolean {
+  return MCP_SERVER_ID_PATTERN.test(value) && !value.includes("__");
+}
+
+/** Minimal AgentEvent surface the monitor needs. Defined locally to
+ *  avoid a circular import; structurally matches the relevant fields
+ *  on the real `AgentEvent` union. */
+interface TrackableEvent {
+  type: string;
+  toolUseId?: string;
+  toolName?: string;
+  content?: string;
+  isError?: boolean;
+}
+
+interface ServerStats {
+  consecutiveFailures: number;
+  totalFailures: number;
+  totalCalls: number;
+}
+
+interface NotificationSink {
+  publish: typeof publishNotification;
+  warn: (event: string, message: string, data?: Record<string, unknown>) => void;
+}
+
+const defaultSink: NotificationSink = {
+  publish: publishNotification,
+  warn: (event, message, data) => log.warn(event, message, data),
+};
+
+/** Pure helper: returns the server id encoded in an MCP tool name,
+ *  or `null` for non-MCP tools.
+ *
+ *  Parses by string-split rather than a single regex so:
+ *    - server ids containing `_` (allowed by `isMcpServerId`, e.g.
+ *      `a1_b2`) attribute correctly. The first `__` after the
+ *      `mcp__` prefix is treated as the server↔tool delimiter, so
+ *      `mcp__a1_b2__do_thing` resolves to server `"a1_b2"`,
+ *      tool-part `"do_thing"`.
+ *    - no regex backtracking surface (Codex flagged ReDoS on the
+ *      previous `[^_]+(?:_[^_]+)*` form; this fix uses split + a
+ *      simple per-character validator instead). */
+export function mcpServerFromToolName(toolName: string): string | null {
+  if (!toolName.startsWith(MCP_PREFIX)) return null;
+  const rest = toolName.slice(MCP_PREFIX.length);
+  const delim = rest.indexOf(MCP_DELIM);
+  if (delim <= 0) return null;
+  const serverId = rest.slice(0, delim);
+  // The tool-part is everything after the delimiter; it can carry
+  // `__` of its own (some MCP authors use `__` in tool names) — we
+  // only care that something is there.
+  if (rest.length <= delim + MCP_DELIM.length) return null;
+  return isValidServerId(serverId) ? serverId : null;
+}
+
+/** Build a session-scoped monitor. Returns the same shape as
+ *  `createMcpTracker` so the backend wires both in the same loop.
+ *
+ *  `sink` is injectable for tests so we don't need to mock the
+ *  notification engine / logger globally. */
+export function createMcpFailureMonitor(opts: { sink?: NotificationSink; threshold?: number } = {}): {
+  track: (event: TrackableEvent) => void;
+} {
+  const sink = opts.sink ?? defaultSink;
+  const threshold = opts.threshold ?? MCP_FAILURE_THRESHOLD;
+  const toolUseIdToServer = new Map<string, string>();
+  const stats = new Map<string, ServerStats>();
+  const notified = new Set<string>();
+
+  function recordCall(toolUseId: string, toolName: string): void {
+    const server = mcpServerFromToolName(toolName);
+    if (server === null) return;
+    toolUseIdToServer.set(toolUseId, server);
+  }
+
+  function recordResult(toolUseId: string, isError: boolean): void {
+    const server = toolUseIdToServer.get(toolUseId);
+    if (server === undefined) return; // not an MCP call (or orphan result)
+    toolUseIdToServer.delete(toolUseId);
+    const entry = stats.get(server) ?? { consecutiveFailures: 0, totalFailures: 0, totalCalls: 0 };
+    entry.totalCalls += 1;
+    if (isError) {
+      entry.consecutiveFailures += 1;
+      entry.totalFailures += 1;
+      if (entry.consecutiveFailures >= threshold && !notified.has(server)) {
+        notified.add(server);
+        emitFailureNotice(server, entry, sink);
+      }
+    } else {
+      entry.consecutiveFailures = 0;
+    }
+    stats.set(server, entry);
+  }
+
+  return {
+    track(event: TrackableEvent): void {
+      if (event.type === EVENT_TYPES.toolCall && typeof event.toolUseId === "string" && typeof event.toolName === "string") {
+        recordCall(event.toolUseId, event.toolName);
+      } else if (event.type === EVENT_TYPES.toolCallResult && typeof event.toolUseId === "string") {
+        recordResult(event.toolUseId, event.isError === true);
+      }
+    },
+  };
+}
+
+function emitFailureNotice(server: string, entry: ServerStats, sink: NotificationSink): void {
+  const message = `MCP server ${server} returned errors on ${String(entry.consecutiveFailures)} consecutive tool calls; check API key, network, or upstream service health.`;
+  sink.warn("mcp", "subprocess appears broken — consecutive tool errors crossed threshold", {
+    server,
+    consecutiveFailures: entry.consecutiveFailures,
+    totalFailures: entry.totalFailures,
+    totalCalls: entry.totalCalls,
+  });
+  sink.publish({
+    // Deterministic id so the notification engine's legacyId dedup
+    // matches across restarts — bell entries from previous boots
+    // for the same broken server don't pile up.
+    id: `mcp-failure-${server}`,
+    kind: "system",
+    title: "MCP server failing",
+    body: message,
+    action: { type: NOTIFICATION_ACTION_TYPES.none },
+    priority: NOTIFICATION_PRIORITIES.high,
+  });
+}