@tonyclaw/agent-inspector 2.0.26 → 2.0.28

package/src/mcp/server.ts

@@ -16,13 +16,31 @@
  */
 
 import { z } from "zod";
-import { McpServer, WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/server";
+import {
+  type GetPromptResult,
+  type ListResourcesResult,
+  McpServer,
+  type ReadResourceResult,
+  ResourceTemplate,
+  type Variables,
+  WebStandardStreamableHTTPServerTransport,
+} from "@modelcontextprotocol/server";
+import {
+  InspectorRunSchema,
+  InspectorRunStatusSchema,
+  UpdateInspectorRunInputSchema,
+} from "../lib/runContract";
 import { setCurrentPort } from "../lib/serverPort";
+import { JsonValueSchema } from "../proxy/schemas";
 import { callApi } from "./loopback";
 import {
   addProviderImpl,
+  createRunImpl,
   createSessionKnowledgeImpl,
+  exportEvidenceImpl,
+  getRecentFailuresImpl,
   getProjectContextImpl,
+  getRunImpl,
   getSessionImpl,
   getLogChunksImpl,
   getLogImpl,
@@ -35,8 +53,10 @@ import {
   promoteKnowledgeCandidateImpl,
   replayLogImpl,
   safeCall,
+  searchLogsImpl,
   searchKnowledgeImpl,
   testProviderImpl,
+  updateRunImpl,
   updateProviderImpl,
 } from "./toolHandlers";
 
@@ -52,9 +72,11 @@ let initPromise: Promise<ServerPair> | null = null;
 function buildServer(): ServerPair {
   const server = new McpServer(
     { name: "agent-inspector", version: "1.0.0" },
-    { capabilities: { tools: {} } },
+    { capabilities: { tools: {}, resources: {}, prompts: {} } },
   );
   registerTools(server);
+  registerResources(server);
+  registerPrompts(server);
   const transport = new WebStandardStreamableHTTPServerTransport({
     sessionIdGenerator: undefined, // stateless — see module docstring
   });
@@ -138,6 +160,152 @@ Either preview field is null when the body is unknown format, unparseable, or th
 const PROVIDER_WRITE_WARNING =
   "⚠ This tool mutates provider configuration. Confirm with the user before invoking.";
 
+const RunsListResponseSchema = z.object({
+  runs: z.array(InspectorRunSchema),
+});
+
+const LogResourceListResponseSchema = z.object({
+  logs: z.array(z.object({ id: z.number().int().positive() })),
+});
+
+const EvidenceReadResponseSchema = z.object({
+  markdown: z.string(),
+});
+
+type ResourceDescriptor = {
+  uri: string;
+  name: string;
+  title?: string;
+  mimeType?: string;
+  description?: string;
+};
+
+function textResource(uri: URL, mimeType: string, text: string): ReadResourceResult {
+  return {
+    contents: [{ uri: uri.href, mimeType, text }],
+  };
+}
+
+function jsonResource(uri: URL, data: unknown): ReadResourceResult {
+  return textResource(uri, "application/json", JSON.stringify(data, null, 2));
+}
+
+function variableString(variables: Variables, name: string): string | null {
+  const value = variables[name];
+  if (Array.isArray(value)) return value[0] ?? null;
+  return value ?? null;
+}
+
+function listResult(resources: ResourceDescriptor[]): ListResourcesResult {
+  return { resources };
+}
+
+async function readApiJsonResource(uri: URL, path: string): Promise<ReadResourceResult> {
+  try {
+    const response = await callApi(path);
+    if (!response.ok) {
+      return textResource(uri, "text/plain", `GET ${path} returned ${response.status}`);
+    }
+    return jsonResource(uri, await response.json());
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    return textResource(uri, "text/plain", `Resource read failed: ${message}`);
+  }
+}
+
+async function listSessionResources(): Promise<ListResourcesResult> {
+  try {
+    const response = await callApi("/api/sessions");
+    if (!response.ok) return listResult([]);
+    const parsed = z.array(z.string()).safeParse(await response.json());
+    if (!parsed.success) return listResult([]);
+    return listResult(
+      parsed.data.map((sessionId) => ({
+        uri: `inspector://sessions/${encodeURIComponent(sessionId)}`,
+        name: `session:${sessionId}`,
+        title: `Inspector session ${sessionId}`,
+        mimeType: "application/json",
+      })),
+    );
+  } catch {
+    return listResult([]);
+  }
+}
+
+async function listLogResources(): Promise<ListResourcesResult> {
+  try {
+    const response = await callApi("/api/logs?offset=0&limit=5&compact=1");
+    if (!response.ok) return listResult([]);
+    const parsed = LogResourceListResponseSchema.safeParse(await response.json());
+    if (!parsed.success) return listResult([]);
+    return listResult(
+      parsed.data.logs.map((log) => ({
+        uri: `inspector://logs/${String(log.id)}`,
+        name: `log:${String(log.id)}`,
+        title: `Inspector log ${String(log.id)}`,
+        mimeType: "application/json",
+      })),
+    );
+  } catch {
+    return listResult([]);
+  }
+}
+
+async function listRunResources(): Promise<ListResourcesResult> {
+  try {
+    const response = await callApi("/api/runs");
+    if (!response.ok) return listResult([]);
+    const parsed = RunsListResponseSchema.safeParse(await response.json());
+    if (!parsed.success) return listResult([]);
+    return listResult(
+      parsed.data.runs.map((run) => ({
+        uri: `inspector://runs/${encodeURIComponent(run.id)}`,
+        name: `run:${run.id}`,
+        title: run.title,
+        mimeType: "application/json",
+      })),
+    );
+  } catch {
+    return listResult([]);
+  }
+}
+
+async function listEvidenceResources(): Promise<ListResourcesResult> {
+  try {
+    const response = await callApi("/api/runs");
+    if (!response.ok) return listResult([]);
+    const parsed = RunsListResponseSchema.safeParse(await response.json());
+    if (!parsed.success) return listResult([]);
+    return listResult(
+      parsed.data.runs
+        .filter((run) => run.evidence !== null)
+        .map((run) => ({
+          uri: `inspector://evidence/${encodeURIComponent(run.id)}`,
+          name: `evidence:${run.id}`,
+          title: `${run.title} evidence pack`,
+          mimeType: "text/markdown",
+          description: run.evidence?.markdownPath,
+        })),
+    );
+  } catch {
+    return listResult([]);
+  }
+}
+
+function userPrompt(text: string): GetPromptResult {
+  return {
+    messages: [
+      {
+        role: "user",
+        content: {
+          type: "text",
+          text,
+        },
+      },
+    ],
+  };
+}
+
 // ---------------------------------------------------------------------------
 // Tool registration (thin wiring of impls to the SDK)
 // ---------------------------------------------------------------------------
@@ -165,6 +333,38 @@ function registerTools(server: McpServer): void {
     (args) => safeCall(() => listLogsImpl(callApi, args)),
   );
 
+  server.registerTool(
+    "inspector_search_logs",
+    {
+      title: "Search captured logs",
+      description:
+        "Searches recent compact proxy logs by id, session, model, provider, API path, status, error text, and client context. Results are bounded summaries; call inspector_get_log for full details after locating a candidate id.",
+      inputSchema: z.object({
+        query: z
+          .string()
+          .optional()
+          .describe("Search terms. Empty or omitted returns the newest scanned logs."),
+        offset: z.number().int().nonnegative().optional().describe("Skip this many matches."),
+        limit: z
+          .number()
+          .int()
+          .positive()
+          .optional()
+          .describe("How many summaries to return. Hard-capped at 5 by the MCP handler."),
+        scanLimit: z
+          .number()
+          .int()
+          .positive()
+          .max(1000)
+          .optional()
+          .describe("How many recent compact logs to scan before filtering. Default 200."),
+        sessionId: z.string().optional().describe("Optional session/affinity id filter."),
+        model: z.string().optional().describe("Optional model name filter."),
+      }),
+    },
+    (args) => safeCall(() => searchLogsImpl(callApi, args)),
+  );
+
   server.registerTool(
     "inspector_get_log",
     {
@@ -226,6 +426,100 @@ function registerTools(server: McpServer): void {
     (args) => safeCall(() => getSessionImpl(callApi, args)),
   );
 
+  server.registerTool(
+    "inspector_create_run",
+    {
+      title: "Create or declare an Inspector run",
+      description:
+        "Creates an evaluation-friendly run record before a coding-agent task starts. Use this when a benchmark or Jenkins job wants a stable runId/sessionId pair instead of inferring a task after the fact from logs.",
+      inputSchema: z.object({
+        runId: z.string().min(1).optional().describe("Stable run id. Generated when omitted."),
+        sessionId: z
+          .string()
+          .min(1)
+          .optional()
+          .describe("Inspector session id to attach. Defaults to the run id."),
+        title: z.string().min(1).optional().describe("Human-readable run title."),
+        task: z.string().nullable().optional().describe("Task statement or benchmark case."),
+        project: z.string().nullable().optional().describe("Project or repository under test."),
+        agent: z.string().nullable().optional().describe("Coding agent name, e.g. Codex/OpenCode."),
+        status: InspectorRunStatusSchema.optional().describe("Initial run status."),
+        tags: z.array(z.string()).optional().describe("Free-form run labels."),
+        metadata: z.record(z.string(), JsonValueSchema).optional().describe("JSON metadata."),
+      }),
+    },
+    (args) => safeCall(() => createRunImpl(callApi, args)),
+  );
+
+  server.registerTool(
+    "inspector_get_run",
+    {
+      title: "Get an Inspector run",
+      description:
+        "Returns one declared Inspector run, including its session binding and exported evidence paths when available.",
+      inputSchema: z.object({
+        runId: z.string().min(1).describe("The Inspector run id."),
+      }),
+    },
+    (args) => safeCall(() => getRunImpl(callApi, args)),
+  );
+
+  server.registerTool(
+    "inspector_update_run",
+    {
+      title: "Update an Inspector run",
+      description:
+        "PATCH-style update for a declared run. Use this during a coding-agent task to mark status transitions such as running, failed, completed, or to attach refined task metadata. Supplied metadata keys are merged into existing metadata.",
+      inputSchema: UpdateInspectorRunInputSchema.extend({
+        runId: z.string().min(1).describe("The Inspector run id."),
+      }),
+    },
+    (args) => safeCall(() => updateRunImpl(callApi, args)),
+  );
+
+  server.registerTool(
+    "inspector_export_evidence",
+    {
+      title: "Export an Inspector evidence pack",
+      description:
+        "Exports one run's evidence pack to local JSON, Markdown, and HTML files under <dataDir>/evidence/<runId> and returns the paths plus session summary.",
+      inputSchema: z.object({
+        runId: z.string().min(1).describe("The Inspector run id."),
+        includeHistory: z
+          .boolean()
+          .optional()
+          .describe("Scan persisted jsonl history for the bound session. Slower; default false."),
+        latestLogLimit: z
+          .number()
+          .int()
+          .positive()
+          .max(50)
+          .optional()
+          .describe("How many compact log summaries to include in the evidence pack. Max 50."),
+      }),
+    },
+    (args) => safeCall(() => exportEvidenceImpl(callApi, args)),
+  );
+
+  server.registerTool(
+    "inspector_get_recent_failures",
+    {
+      title: "Get recent Inspector failures",
+      description:
+        "Returns recent declared runs that are failed/cancelled or have non-success exported evidence, including classification, Jenkins summary, Inspector link, and evidence paths when available.",
+      inputSchema: z.object({
+        limit: z
+          .number()
+          .int()
+          .positive()
+          .max(20)
+          .optional()
+          .describe("Maximum failures to return. Default 5, max 20."),
+      }),
+    },
+    (args) => safeCall(() => getRecentFailuresImpl(callApi, args)),
+  );
+
   server.registerTool(
     "inspector_list_models",
     {
@@ -408,13 +702,254 @@ PATCH-style update: only the fields you supply are changed. Returns the updated
   );
 }
 
+function registerResources(server: McpServer): void {
+  server.registerResource(
+    "inspector_sessions",
+    "inspector://sessions",
+    {
+      title: "Inspector sessions",
+      description: "Observed Inspector session ids.",
+      mimeType: "application/json",
+    },
+    (uri) => readApiJsonResource(uri, "/api/sessions"),
+  );
+
+  server.registerResource(
+    "inspector_session",
+    new ResourceTemplate("inspector://sessions/{sessionId}", { list: listSessionResources }),
+    {
+      title: "Inspector session",
+      description:
+        "Evidence-friendly session summary with Inspector URL, compact logs URL, counts, models, providers, token totals, and recent compact logs.",
+      mimeType: "application/json",
+    },
+    (uri, variables) => {
+      const sessionId = variableString(variables, "sessionId");
+      if (sessionId === null) return textResource(uri, "text/plain", "Missing sessionId");
+      const params = new URLSearchParams({ sessionId, limit: "10" });
+      return readApiJsonResource(uri, `/api/sessions?${params.toString()}`);
+    },
+  );
+
+  server.registerResource(
+    "inspector_log",
+    new ResourceTemplate("inspector://logs/{id}", { list: listLogResources }),
+    {
+      title: "Inspector log",
+      description: "Single captured proxy log by numeric id.",
+      mimeType: "application/json",
+    },
+    async (uri, variables) => {
+      const id = variableString(variables, "id");
+      const parsed = id === null ? Number.NaN : Number(id);
+      if (!Number.isInteger(parsed) || parsed <= 0) {
+        return textResource(uri, "text/plain", "Invalid log id");
+      }
+      const result = await getLogImpl(callApi, parsed);
+      const first = result.content[0];
+      const text = first?.text ?? "";
+      return textResource(uri, result.isError === true ? "text/plain" : "application/json", text);
+    },
+  );
+
+  server.registerResource(
+    "inspector_runs",
+    "inspector://runs",
+    {
+      title: "Inspector runs",
+      description: "Declared Inspector evaluation runs.",
+      mimeType: "application/json",
+    },
+    (uri) => readApiJsonResource(uri, "/api/runs"),
+  );
+
+  server.registerResource(
+    "inspector_recent_failures",
+    "inspector://failures/recent",
+    {
+      title: "Recent Inspector failures",
+      description:
+        "Recent declared runs with failed/cancelled status or non-success exported evidence.",
+      mimeType: "application/json",
+    },
+    (uri) => readApiJsonResource(uri, "/api/runs?failures=1&limit=10"),
+  );
+
+  server.registerResource(
+    "inspector_run",
+    new ResourceTemplate("inspector://runs/{runId}", { list: listRunResources }),
+    {
+      title: "Inspector run",
+      description: "One declared Inspector run, including session binding and evidence paths.",
+      mimeType: "application/json",
+    },
+    (uri, variables) => {
+      const runId = variableString(variables, "runId");
+      if (runId === null) return textResource(uri, "text/plain", "Missing runId");
+      return readApiJsonResource(uri, `/api/runs/${encodeURIComponent(runId)}`);
+    },
+  );
+
+  server.registerResource(
+    "inspector_evidence",
+    new ResourceTemplate("inspector://evidence/{runId}", { list: listEvidenceResources }),
+    {
+      title: "Inspector evidence pack",
+      description:
+        "Markdown evidence pack for a declared Inspector run. Generate it first with inspector_export_evidence.",
+      mimeType: "text/markdown",
+    },
+    async (uri, variables) => {
+      const runId = variableString(variables, "runId");
+      if (runId === null) return textResource(uri, "text/plain", "Missing runId");
+      const path = `/api/runs/${encodeURIComponent(runId)}/evidence`;
+      try {
+        const response = await callApi(path);
+        if (!response.ok) {
+          return textResource(uri, "text/plain", `GET ${path} returned ${response.status}`);
+        }
+        const parsed = EvidenceReadResponseSchema.safeParse(await response.json());
+        if (!parsed.success) {
+          return textResource(uri, "text/plain", "Evidence endpoint returned an unparseable body");
+        }
+        return textResource(uri, "text/markdown", parsed.data.markdown);
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return textResource(uri, "text/plain", `Resource read failed: ${message}`);
+      }
+    },
+  );
+}
+
+function registerPrompts(server: McpServer): void {
+  server.registerPrompt(
+    "inspector_analyze_session_failure",
+    {
+      title: "Analyze session failure",
+      description:
+        "Use Inspector session/log resources to explain why a coding-agent session failed.",
+      argsSchema: z.object({
+        sessionId: z.string().min(1).describe("The Inspector session id."),
+        includeHistory: z
+          .boolean()
+          .optional()
+          .describe("When true, call inspector_get_session with includeHistory=true if needed."),
+      }),
+    },
+    ({ sessionId, includeHistory }) =>
+      userPrompt(`Analyze this Agent Inspector session and explain why it failed.
+
+Evidence to read:
+- Session resource: inspector://sessions/${encodeURIComponent(sessionId)}
+- Relevant log resources: inspector://logs/{id} for the failing or suspicious log ids
+
+If includeHistory is ${includeHistory === true ? "true" : "false"} or the session summary looks incomplete, call inspector_get_session with includeHistory=true and latestLogLimit=50.
+
+Return:
+1. Outcome and severity
+2. Most likely root cause
+3. Evidence bullets with log ids, models, providers, status codes, and error text
+4. Concrete fix or next diagnostic step
+5. Confidence and what evidence is still missing`),
+  );
+
+  server.registerPrompt(
+    "inspector_triage_recent_failures",
+    {
+      title: "Triage recent failures",
+      description:
+        "Use the recent failures resource and log search tool to prioritize coding-agent failures.",
+      argsSchema: z.object({
+        limit: z
+          .number()
+          .int()
+          .positive()
+          .max(20)
+          .optional()
+          .describe("How many failures to triage."),
+      }),
+    },
+    ({ limit }) =>
+      userPrompt(`Triage recent Agent Inspector failures.
+
+Evidence to read:
+- Recent failures resource: inspector://failures/recent
+- For missing details, call inspector_get_recent_failures with limit=${String(limit ?? 5)}
+- For suspicious sessions/models/providers, call inspector_search_logs and then inspector_get_log for cited ids
+
+Return:
+1. Highest-priority failure first
+2. Run id, session id, category, severity, and evidence links
+3. Likely root cause and concrete next action
+4. Whether evidence is sufficient or inspector_export_evidence should be run again`),
+  );
+
+  server.registerPrompt(
+    "inspector_generate_jenkins_report",
+    {
+      title: "Generate Jenkins report",
+      description:
+        "Generate a Jenkins-friendly Markdown report from an Inspector run and evidence pack.",
+      argsSchema: z.object({
+        runId: z.string().min(1).describe("The Inspector run id."),
+      }),
+    },
+    ({ runId }) =>
+      userPrompt(`Generate a Jenkins-ready Markdown report for this Inspector run.
+
+Evidence to read:
+- Run resource: inspector://runs/${encodeURIComponent(runId)}
+- Evidence pack resource: inspector://evidence/${encodeURIComponent(runId)}
+
+If the evidence pack resource says it has not been exported, call inspector_export_evidence for this run first.
+
+The report should include:
+1. Build/run status summary
+2. Run id, session id, Inspector URLs, and artifact paths
+3. Models/providers used, token totals, request counts, and error counts
+4. Failure analysis or success evidence
+5. Reproducible next action for a developer`),
+  );
+
+  server.registerPrompt(
+    "inspector_extract_repro_steps",
+    {
+      title: "Extract reproducible steps",
+      description:
+        "Turn an Inspector session into concise repro steps for a developer or evaluator.",
+      argsSchema: z.object({
+        sessionId: z.string().min(1).describe("The Inspector session id."),
+      }),
+    },
+    ({ sessionId }) =>
+      userPrompt(`Extract reproducible steps from this Agent Inspector session.
+
+Evidence to read:
+- Session resource: inspector://sessions/${encodeURIComponent(sessionId)}
+- Relevant log resources: inspector://logs/{id}
+
+Return:
+1. Preconditions and environment assumptions
+2. Ordered user/agent actions that led to the behavior
+3. Exact model/provider/API path details needed to reproduce
+4. Expected vs actual result
+5. Minimal verification command or UI check`),
+  );
+}
+
 // Surface the tool catalog for tests that assert on the tool name set.
 export const TOOL_NAMES = [
   "inspector_list_logs",
+  "inspector_search_logs",
   "inspector_get_log",
   "inspector_get_log_chunks",
   "inspector_list_sessions",
   "inspector_get_session",
+  "inspector_create_run",
+  "inspector_get_run",
+  "inspector_update_run",
+  "inspector_export_evidence",
+  "inspector_get_recent_failures",
   "inspector_list_models",
   "inspector_list_providers",
   "inspector_get_provider",
@@ -428,3 +963,20 @@ export const TOOL_NAMES = [
   "inspector_search_knowledge",
   "inspector_get_project_context",
 ] as const;
+
+export const RESOURCE_NAMES = [
+  "inspector_sessions",
+  "inspector_session",
+  "inspector_log",
+  "inspector_runs",
+  "inspector_recent_failures",
+  "inspector_run",
+  "inspector_evidence",
+] as const;
+
+export const PROMPT_NAMES = [
+  "inspector_analyze_session_failure",
+  "inspector_triage_recent_failures",
+  "inspector_generate_jenkins_report",
+  "inspector_extract_repro_steps",
+] as const;