@tonyclaw/agent-inspector 2.0.28 → 2.0.30

package/src/mcp/server.ts

@@ -25,6 +25,14 @@ import {
   type Variables,
   WebStandardStreamableHTTPServerTransport,
 } from "@modelcontextprotocol/server";
+import {
+  AddInspectorGroupSessionInputSchema,
+  GroupEvidenceExportOptionsSchema,
+  InspectorGroupSchema,
+  InspectorGroupStatusSchema,
+  InspectorGroupsListResponseSchema,
+  UpdateInspectorGroupInputSchema,
+} from "../lib/groupContract";
 import {
   InspectorRunSchema,
   InspectorRunStatusSchema,
@@ -34,10 +42,14 @@ import { setCurrentPort } from "../lib/serverPort";
 import { JsonValueSchema } from "../proxy/schemas";
 import { callApi } from "./loopback";
 import {
+  addGroupSessionImpl,
   addProviderImpl,
+  createGroupImpl,
   createRunImpl,
   createSessionKnowledgeImpl,
   exportEvidenceImpl,
+  exportGroupEvidenceImpl,
+  getGroupImpl,
   getRecentFailuresImpl,
   getProjectContextImpl,
   getRunImpl,
@@ -45,6 +57,7 @@ import {
   getLogChunksImpl,
   getLogImpl,
   getProviderImpl,
+  listGroupsImpl,
   listKnowledgeCandidatesImpl,
   listLogsImpl,
   listModelsImpl,
@@ -56,6 +69,7 @@ import {
   searchLogsImpl,
   searchKnowledgeImpl,
   testProviderImpl,
+  updateGroupImpl,
   updateRunImpl,
   updateProviderImpl,
 } from "./toolHandlers";
@@ -172,6 +186,10 @@ const EvidenceReadResponseSchema = z.object({
   markdown: z.string(),
 });
 
+const GroupEvidenceReadResponseSchema = z.object({
+  markdown: z.string(),
+});
+
 type ResourceDescriptor = {
   uri: string;
   name: string;
@@ -270,6 +288,66 @@ async function listRunResources(): Promise<ListResourcesResult> {
   }
 }
 
+async function listGroupResources(): Promise<ListResourcesResult> {
+  try {
+    const response = await callApi("/api/groups");
+    if (!response.ok) return listResult([]);
+    const parsed = InspectorGroupsListResponseSchema.safeParse(await response.json());
+    if (!parsed.success) return listResult([]);
+    return listResult(
+      parsed.data.groups.map((group) => ({
+        uri: `inspector://groups/${encodeURIComponent(group.id)}`,
+        name: `group:${group.id}`,
+        title: group.title,
+        mimeType: "application/json",
+      })),
+    );
+  } catch {
+    return listResult([]);
+  }
+}
+
+async function listGroupSessionResources(): Promise<ListResourcesResult> {
+  try {
+    const response = await callApi("/api/groups");
+    if (!response.ok) return listResult([]);
+    const parsed = InspectorGroupsListResponseSchema.safeParse(await response.json());
+    if (!parsed.success) return listResult([]);
+    return listResult(
+      parsed.data.groups.map((group) => ({
+        uri: `inspector://groups/${encodeURIComponent(group.id)}/sessions`,
+        name: `group-sessions:${group.id}`,
+        title: `${group.title} sessions`,
+        mimeType: "application/json",
+      })),
+    );
+  } catch {
+    return listResult([]);
+  }
+}
+
+async function listGroupEvidenceResources(): Promise<ListResourcesResult> {
+  try {
+    const response = await callApi("/api/groups");
+    if (!response.ok) return listResult([]);
+    const parsed = InspectorGroupsListResponseSchema.safeParse(await response.json());
+    if (!parsed.success) return listResult([]);
+    return listResult(
+      parsed.data.groups
+        .filter((group) => group.evidence !== null)
+        .map((group) => ({
+          uri: `inspector://groups/${encodeURIComponent(group.id)}/evidence`,
+          name: `group-evidence:${group.id}`,
+          title: `${group.title} evidence pack`,
+          mimeType: "text/markdown",
+          description: group.evidence?.markdownPath,
+        })),
+    );
+  } catch {
+    return listResult([]);
+  }
+}
+
 async function listEvidenceResources(): Promise<ListResourcesResult> {
   try {
     const response = await callApi("/api/runs");
@@ -520,6 +598,92 @@ function registerTools(server: McpServer): void {
     (args) => safeCall(() => getRecentFailuresImpl(callApi, args)),
   );
 
+  server.registerTool(
+    "inspector_list_groups",
+    {
+      title: "List Inspector groups",
+      description:
+        "Lists declared Inspector groups. Groups aggregate multiple sessions/runs, commonly one evaluation matrix spanning several providers or models.",
+      inputSchema: z.object({}),
+    },
+    () => safeCall(() => listGroupsImpl(callApi)),
+  );
+
+  server.registerTool(
+    "inspector_create_group",
+    {
+      title: "Create or declare an Inspector group",
+      description:
+        "Creates an evaluation/batch group that external systems can use to aggregate multiple OpenCode/Codex sessions across models such as MiniMax, DeepSeek, or GLM.",
+      inputSchema: z.object({
+        groupId: z.string().min(1).optional().describe("Stable group id. Generated when omitted."),
+        title: z.string().min(1).optional().describe("Human-readable group title."),
+        kind: z
+          .enum(["evaluation", "batch", "manual"])
+          .optional()
+          .describe("Group kind. Defaults to evaluation."),
+        task: z.string().nullable().optional().describe("Task, benchmark, or evaluation goal."),
+        project: z.string().nullable().optional().describe("Project or repository under test."),
+        status: InspectorGroupStatusSchema.optional().describe("Initial group status."),
+        tags: z.array(z.string()).optional().describe("Free-form group labels."),
+        metadata: z.record(z.string(), JsonValueSchema).optional().describe("JSON metadata."),
+      }),
+    },
+    (args) => safeCall(() => createGroupImpl(callApi, args)),
+  );
+
+  server.registerTool(
+    "inspector_get_group",
+    {
+      title: "Get an Inspector group",
+      description:
+        "Returns one declared Inspector group with its attached session/run members and exported evidence paths when available.",
+      inputSchema: z.object({
+        groupId: z.string().min(1).describe("The Inspector group id."),
+      }),
+    },
+    (args) => safeCall(() => getGroupImpl(callApi, args)),
+  );
+
+  server.registerTool(
+    "inspector_update_group",
+    {
+      title: "Update an Inspector group",
+      description:
+        "PATCH-style update for a declared group. Use this to mark a whole evaluation running, failed, completed, or to merge CI metadata.",
+      inputSchema: UpdateInspectorGroupInputSchema.extend({
+        groupId: z.string().min(1).describe("The Inspector group id."),
+      }),
+    },
+    (args) => safeCall(() => updateGroupImpl(callApi, args)),
+  );
+
+  server.registerTool(
+    "inspector_add_group_session",
+    {
+      title: "Attach a session to an Inspector group",
+      description:
+        "Adds or updates one group member. Use it after each OpenCode process discovers its session id, or while an evaluation matrix is running to attach model/provider labels.",
+      inputSchema: AddInspectorGroupSessionInputSchema.extend({
+        groupId: z.string().min(1).describe("The Inspector group id."),
+      }),
+    },
+    (args) => safeCall(() => addGroupSessionImpl(callApi, args)),
+  );
+
+  server.registerTool(
+    "inspector_export_group_evidence",
+    {
+      title: "Export an Inspector group evidence pack",
+      description:
+        "Exports a whole group's evidence pack to local JSON, Markdown, and HTML files under <dataDir>/evidence/groups/<groupId>. The export contains a provider/model matrix and compact session summaries.",
+      inputSchema: GroupEvidenceExportOptionsSchema.unwrap().extend({
+        groupId: z.string().min(1).describe("The Inspector group id."),
+      }),
+    },
+    (args) => safeCall(() => exportGroupEvidenceImpl(callApi, args)),
+  );
+
   server.registerTool(
     "inspector_list_models",
     {
@@ -763,6 +927,49 @@ function registerResources(server: McpServer): void {
     (uri) => readApiJsonResource(uri, "/api/runs"),
   );
 
+  server.registerResource(
+    "inspector_groups",
+    "inspector://groups",
+    {
+      title: "Inspector groups",
+      description: "Declared Inspector groups aggregating multiple sessions or runs.",
+      mimeType: "application/json",
+    },
+    (uri) => readApiJsonResource(uri, "/api/groups"),
+  );
+
+  server.registerResource(
+    "inspector_group",
+    new ResourceTemplate("inspector://groups/{groupId}", { list: listGroupResources }),
+    {
+      title: "Inspector group",
+      description: "One declared Inspector group with attached session/run members.",
+      mimeType: "application/json",
+    },
+    (uri, variables) => {
+      const groupId = variableString(variables, "groupId");
+      if (groupId === null) return textResource(uri, "text/plain", "Missing groupId");
+      return readApiJsonResource(uri, `/api/groups/${encodeURIComponent(groupId)}`);
+    },
+  );
+
+  server.registerResource(
+    "inspector_group_sessions",
+    new ResourceTemplate("inspector://groups/{groupId}/sessions", {
+      list: listGroupSessionResources,
+    }),
+    {
+      title: "Inspector group sessions",
+      description: "Attached session/run members for one Inspector group.",
+      mimeType: "application/json",
+    },
+    (uri, variables) => {
+      const groupId = variableString(variables, "groupId");
+      if (groupId === null) return textResource(uri, "text/plain", "Missing groupId");
+      return readApiJsonResource(uri, `/api/groups/${encodeURIComponent(groupId)}/sessions`);
+    },
+  );
+
   server.registerResource(
     "inspector_recent_failures",
     "inspector://failures/recent",
@@ -819,6 +1026,38 @@ function registerResources(server: McpServer): void {
       }
     },
   );
+
+  server.registerResource(
+    "inspector_group_evidence",
+    new ResourceTemplate("inspector://groups/{groupId}/evidence", {
+      list: listGroupEvidenceResources,
+    }),
+    {
+      title: "Inspector group evidence pack",
+      description:
+        "Markdown evidence pack for an Inspector group. Generate it first with inspector_export_group_evidence.",
+      mimeType: "text/markdown",
+    },
+    async (uri, variables) => {
+      const groupId = variableString(variables, "groupId");
+      if (groupId === null) return textResource(uri, "text/plain", "Missing groupId");
+      const path = `/api/groups/${encodeURIComponent(groupId)}/evidence`;
+      try {
+        const response = await callApi(path);
+        if (!response.ok) {
+          return textResource(uri, "text/plain", `GET ${path} returned ${response.status}`);
+        }
+        const parsed = GroupEvidenceReadResponseSchema.safeParse(await response.json());
+        if (!parsed.success) {
+          return textResource(uri, "text/plain", "Evidence endpoint returned an unparseable body");
+        }
+        return textResource(uri, "text/markdown", parsed.data.markdown);
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return textResource(uri, "text/plain", `Resource read failed: ${message}`);
+      }
+    },
+  );
 }
 
 function registerPrompts(server: McpServer): void {
@@ -911,6 +1150,34 @@ The report should include:
 5. Reproducible next action for a developer`),
   );
 
+  server.registerPrompt(
+    "inspector_generate_group_report",
+    {
+      title: "Generate group report",
+      description:
+        "Generate a model/provider evaluation report from an Inspector group and group evidence pack.",
+      argsSchema: z.object({
+        groupId: z.string().min(1).describe("The Inspector group id."),
+      }),
+    },
+    ({ groupId }) =>
+      userPrompt(`Generate a model/provider evaluation report for this Inspector group.
+
+Evidence to read:
+- Group resource: inspector://groups/${encodeURIComponent(groupId)}
+- Group session members: inspector://groups/${encodeURIComponent(groupId)}/sessions
+- Group evidence pack: inspector://groups/${encodeURIComponent(groupId)}/evidence
+
+If the evidence pack resource says it has not been exported, call inspector_export_group_evidence for this group first.
+
+The report should include:
+1. Evaluation status summary
+2. Provider/model matrix with session links
+3. Request counts, token totals, streaming timing, and error counts
+4. Winner/loser observations when the evidence supports them
+5. Concrete next actions for failed or missing sessions`),
+  );
+
   server.registerPrompt(
     "inspector_extract_repro_steps",
     {
@@ -950,6 +1217,12 @@ export const TOOL_NAMES = [
   "inspector_update_run",
   "inspector_export_evidence",
   "inspector_get_recent_failures",
+  "inspector_list_groups",
+  "inspector_create_group",
+  "inspector_get_group",
+  "inspector_update_group",
+  "inspector_add_group_session",
+  "inspector_export_group_evidence",
   "inspector_list_models",
   "inspector_list_providers",
   "inspector_get_provider",
@@ -972,11 +1245,16 @@ export const RESOURCE_NAMES = [
   "inspector_recent_failures",
   "inspector_run",
   "inspector_evidence",
+  "inspector_groups",
+  "inspector_group",
+  "inspector_group_sessions",
+  "inspector_group_evidence",
 ] as const;
 
 export const PROMPT_NAMES = [
   "inspector_analyze_session_failure",
   "inspector_triage_recent_failures",
   "inspector_generate_jenkins_report",
+  "inspector_generate_group_report",
   "inspector_extract_repro_steps",
 ] as const;

package/src/mcp/toolHandlers.ts

@@ -17,6 +17,14 @@ import { Buffer } from "node:buffer";
 import { z } from "zod";
 import { LoopbackTimeoutError, type CallApiOptions } from "./loopback";
 import { extractLastUserMessagePreview, extractResponsePreview } from "./previewExtractor";
+import {
+  GroupEvidenceExportResultSchema,
+  InspectorGroupSchema,
+  InspectorGroupsListResponseSchema,
+  type AddInspectorGroupSessionInput,
+  type CreateInspectorGroupInput,
+  type UpdateInspectorGroupInput,
+} from "../lib/groupContract";
 import {
   EvidenceExportResultSchema,
   InspectorRunSchema,
@@ -132,6 +140,9 @@ export function buildLogSummary(log: CapturedLog) {
     isStreaming: log.streaming,
     hasError,
     latencyMs: log.elapsedMs,
+    firstChunkMs: log.firstChunkMs ?? null,
+    totalStreamMs: log.totalStreamMs ?? null,
+    tokensPerSecond: log.tokensPerSecond ?? null,
     tokens: {
       input: log.inputTokens,
       output: log.outputTokens,
@@ -372,6 +383,121 @@ export async function exportEvidenceImpl(
   return textJson(parsed.data);
 }
 
+export async function listGroupsImpl(callApi: CallApiFn): Promise<ToolResult> {
+  const res = await callApi("/api/groups");
+  if (!res.ok) return toolError(`GET /api/groups returned ${res.status}`);
+  const rawBody: unknown = await res.json();
+  const parsed = InspectorGroupsListResponseSchema.safeParse(rawBody);
+  if (!parsed.success) return toolError("GET /api/groups returned an unparseable group list");
+  return textJson(parsed.data);
+}
+
+export async function createGroupImpl(
+  callApi: CallApiFn,
+  args: CreateInspectorGroupInput,
+): Promise<ToolResult> {
+  const res = await callApi("/api/groups", {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify(args ?? {}),
+  });
+  if (!res.ok) return toolError(`POST /api/groups returned ${res.status}`);
+  const rawBody: unknown = await res.json();
+  const parsed = InspectorGroupSchema.safeParse(rawBody);
+  if (!parsed.success) return toolError("POST /api/groups returned an unparseable group");
+  return textJson(parsed.data);
+}
+
+export type GetGroupArgs = {
+  groupId: string;
+};
+
+export async function getGroupImpl(callApi: CallApiFn, args: GetGroupArgs): Promise<ToolResult> {
+  const path = `/api/groups/${encodeURIComponent(args.groupId)}`;
+  const res = await callApi(path);
+  if (!res.ok) return toolError(`GET ${path} returned ${res.status}`);
+  const rawBody: unknown = await res.json();
+  const parsed = InspectorGroupSchema.safeParse(rawBody);
+  if (!parsed.success) return toolError("GET /api/groups/{groupId} returned an unparseable group");
+  return textJson(parsed.data);
+}
+
+export type UpdateGroupArgs = {
+  groupId: string;
+} & UpdateInspectorGroupInput;
+
+export async function updateGroupImpl(
+  callApi: CallApiFn,
+  args: UpdateGroupArgs,
+): Promise<ToolResult> {
+  const { groupId, ...patch } = args;
+  const path = `/api/groups/${encodeURIComponent(groupId)}`;
+  const res = await callApi(path, {
+    method: "PATCH",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify(patch),
+  });
+  if (!res.ok) return toolError(`PATCH ${path} returned ${res.status}`);
+  const rawBody: unknown = await res.json();
+  const parsed = InspectorGroupSchema.safeParse(rawBody);
+  if (!parsed.success) {
+    return toolError("PATCH /api/groups/{groupId} returned an unparseable group");
+  }
+  return textJson(parsed.data);
+}
+
+export type AddGroupSessionArgs = {
+  groupId: string;
+} & AddInspectorGroupSessionInput;
+
+export async function addGroupSessionImpl(
+  callApi: CallApiFn,
+  args: AddGroupSessionArgs,
+): Promise<ToolResult> {
+  const { groupId, ...session } = args;
+  const path = `/api/groups/${encodeURIComponent(groupId)}/sessions`;
+  const res = await callApi(path, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify(session),
+  });
+  if (!res.ok) return toolError(`POST ${path} returned ${res.status}`);
+  const rawBody: unknown = await res.json();
+  const parsed = InspectorGroupSchema.safeParse(rawBody);
+  if (!parsed.success) {
+    return toolError("POST /api/groups/{groupId}/sessions returned an unparseable group");
+  }
+  return textJson(parsed.data);
+}
+
+export type ExportGroupEvidenceArgs = {
+  groupId: string;
+  includeHistory?: boolean;
+  latestLogLimit?: number;
+};
+
+export async function exportGroupEvidenceImpl(
+  callApi: CallApiFn,
+  args: ExportGroupEvidenceArgs,
+): Promise<ToolResult> {
+  const path = `/api/groups/${encodeURIComponent(args.groupId)}/evidence`;
+  const res = await callApi(path, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({
+      includeHistory: args.includeHistory,
+      latestLogLimit: args.latestLogLimit,
+    }),
+  });
+  if (!res.ok) return toolError(`POST ${path} returned ${res.status}`);
+  const rawBody: unknown = await res.json();
+  const parsed = GroupEvidenceExportResultSchema.safeParse(rawBody);
+  if (!parsed.success) {
+    return toolError("POST /api/groups/{groupId}/evidence returned an unparseable evidence result");
+  }
+  return textJson(parsed.data);
+}
+
 export async function listModelsImpl(callApi: CallApiFn): Promise<ToolResult> {
   const res = await callApi("/api/models");
   if (!res.ok) return toolError(`GET /api/models returned ${res.status}`);

package/src/proxy/evidenceAnalysis.ts

@@ -318,6 +318,11 @@ function logDetails(log: SessionLogSummary): string {
     `streaming=${log.isStreaming ? "yes" : "no"}`,
     `latencyMs=${formatNullable(log.latencyMs)}`,
   ];
+  if (log.isStreaming) {
+    parts.push(`firstChunkMs=${formatNullable(log.firstChunkMs ?? null)}`);
+    parts.push(`totalStreamMs=${formatNullable(log.totalStreamMs ?? null)}`);
+    parts.push(`tokensPerSecond=${formatNullable(log.tokensPerSecond ?? null)}`);
+  }
   if (log.error !== null) {
     parts.push(`error=${log.error}`);
   }

package/src/proxy/evidenceExporter.ts

@@ -112,14 +112,19 @@ function buildMarkdown(document: EvidenceDocument): string {
       lines.push("No compact logs were available.", "");
     } else {
       lines.push(
-        "| ID | Status | Model | Latency | Path | Error |",
-        "| --- | --- | --- | --- | --- | --- |",
+        "| ID | Status | Model | Latency | Stream | Path | Error |",
+        "| --- | --- | --- | --- | --- | --- | --- |",
       );
       for (const log of session.latestLogs) {
+        const streamTiming = log.isStreaming
+          ? `first=${log.firstChunkMs ?? "n/a"}ms total=${log.totalStreamMs ?? "n/a"}ms tps=${
+              log.tokensPerSecond ?? "n/a"
+            }`
+          : "n/a";
         lines.push(
           `| ${log.id} | ${log.status ?? "n/a"} | ${log.model ?? "n/a"} | ${
             log.latencyMs ?? "n/a"
-          } | ${log.path} | ${log.hasError ? "yes" : "no"} |`,
+          } | ${streamTiming} | ${log.path} | ${log.hasError ? "yes" : "no"} |`,
         );
       }
       lines.push("");