@tonyclaw/agent-inspector 2.0.29 → 2.0.31

package/src/lib/groupContract.ts

@@ -0,0 +1,148 @@
+import { z } from "zod";
+import { JsonValueSchema } from "../proxy/schemas";
+import { EvidenceReportSchema, InspectorRunSchema, InspectorRunStatusSchema } from "./runContract";
+import { SessionInfoSchema, SessionTokenUsageSchema } from "./sessionInfoContract";
+
+export const InspectorGroupStatusSchema = z.enum([
+  "created",
+  "running",
+  "completed",
+  "failed",
+  "cancelled",
+]);
+
+export const InspectorGroupKindSchema = z.enum(["evaluation", "batch", "manual"]);
+
+export const InspectorGroupEvidenceSchema = z.object({
+  jsonPath: z.string(),
+  markdownPath: z.string(),
+  htmlPath: z.string(),
+  exportedAt: z.string(),
+});
+
+export const InspectorGroupMemberSchema = z.object({
+  id: z.string(),
+  groupId: z.string(),
+  sessionId: z.string(),
+  runId: z.string().nullable(),
+  label: z.string().nullable(),
+  agent: z.string().nullable(),
+  provider: z.string().nullable(),
+  model: z.string().nullable(),
+  status: InspectorRunStatusSchema.nullable(),
+  metadata: z.record(z.string(), JsonValueSchema),
+  addedAt: z.string(),
+  updatedAt: z.string(),
+});
+
+export const InspectorGroupSchema = z.object({
+  id: z.string(),
+  title: z.string(),
+  kind: InspectorGroupKindSchema,
+  task: z.string().nullable(),
+  project: z.string().nullable(),
+  status: InspectorGroupStatusSchema,
+  tags: z.array(z.string()),
+  metadata: z.record(z.string(), JsonValueSchema),
+  evidence: InspectorGroupEvidenceSchema.nullable(),
+  members: z.array(InspectorGroupMemberSchema),
+  createdAt: z.string(),
+  updatedAt: z.string(),
+});
+
+export const CreateInspectorGroupInputSchema = z
+  .object({
+    groupId: z.string().min(1).optional(),
+    title: z.string().min(1).optional(),
+    kind: InspectorGroupKindSchema.optional(),
+    task: z.string().nullable().optional(),
+    project: z.string().nullable().optional(),
+    status: InspectorGroupStatusSchema.optional(),
+    tags: z.array(z.string()).optional(),
+    metadata: z.record(z.string(), JsonValueSchema).optional(),
+  })
+  .optional();
+
+export const UpdateInspectorGroupInputSchema = z.object({
+  title: z.string().min(1).optional(),
+  kind: InspectorGroupKindSchema.optional(),
+  task: z.string().nullable().optional(),
+  project: z.string().nullable().optional(),
+  status: InspectorGroupStatusSchema.optional(),
+  tags: z.array(z.string()).optional(),
+  metadata: z.record(z.string(), JsonValueSchema).optional(),
+});
+
+export const AddInspectorGroupSessionInputSchema = z.object({
+  memberId: z.string().min(1).optional(),
+  sessionId: z.string().min(1),
+  runId: z.string().min(1).nullable().optional(),
+  label: z.string().min(1).nullable().optional(),
+  agent: z.string().min(1).nullable().optional(),
+  provider: z.string().min(1).nullable().optional(),
+  model: z.string().min(1).nullable().optional(),
+  status: InspectorRunStatusSchema.nullable().optional(),
+  metadata: z.record(z.string(), JsonValueSchema).optional(),
+});
+
+export const GroupEvidenceExportOptionsSchema = z
+  .object({
+    includeHistory: z.boolean().optional(),
+    latestLogLimit: z.number().int().positive().max(50).optional(),
+  })
+  .optional();
+
+export const InspectorGroupExportMemberSchema = z.object({
+  member: InspectorGroupMemberSchema,
+  run: InspectorRunSchema.nullable(),
+  session: SessionInfoSchema.nullable(),
+});
+
+export const InspectorGroupEvidenceSummarySchema = z.object({
+  memberCount: z.number().int().nonnegative(),
+  sessionsWithData: z.number().int().nonnegative(),
+  missingSessions: z.number().int().nonnegative(),
+  requestCount: z.number().int().nonnegative(),
+  errorCount: z.number().int().nonnegative(),
+  tokenUsage: SessionTokenUsageSchema,
+  models: z.array(z.string()),
+  providers: z.array(z.string()),
+  completedMembers: z.number().int().nonnegative(),
+  failedMembers: z.number().int().nonnegative(),
+  runningMembers: z.number().int().nonnegative(),
+});
+
+export const GroupEvidenceReadResponseSchema = z.object({
+  group: InspectorGroupSchema,
+  evidence: InspectorGroupEvidenceSchema,
+  markdown: z.string(),
+  summary: InspectorGroupEvidenceSummarySchema.nullable(),
+  report: EvidenceReportSchema.nullable(),
+});
+
+export const GroupEvidenceExportResultSchema = z.object({
+  group: InspectorGroupSchema,
+  evidence: InspectorGroupEvidenceSchema,
+  members: z.array(InspectorGroupExportMemberSchema),
+  summary: InspectorGroupEvidenceSummarySchema,
+  jenkinsReport: EvidenceReportSchema,
+});
+
+export const InspectorGroupsListResponseSchema = z.object({
+  groups: z.array(InspectorGroupSchema),
+});
+
+export type InspectorGroupStatus = z.infer<typeof InspectorGroupStatusSchema>;
+export type InspectorGroupKind = z.infer<typeof InspectorGroupKindSchema>;
+export type InspectorGroupEvidence = z.infer<typeof InspectorGroupEvidenceSchema>;
+export type InspectorGroupMember = z.infer<typeof InspectorGroupMemberSchema>;
+export type InspectorGroup = z.infer<typeof InspectorGroupSchema>;
+export type CreateInspectorGroupInput = z.infer<typeof CreateInspectorGroupInputSchema>;
+export type UpdateInspectorGroupInput = z.infer<typeof UpdateInspectorGroupInputSchema>;
+export type AddInspectorGroupSessionInput = z.infer<typeof AddInspectorGroupSessionInputSchema>;
+export type GroupEvidenceExportOptions = z.infer<typeof GroupEvidenceExportOptionsSchema>;
+export type InspectorGroupExportMember = z.infer<typeof InspectorGroupExportMemberSchema>;
+export type InspectorGroupEvidenceSummary = z.infer<typeof InspectorGroupEvidenceSummarySchema>;
+export type GroupEvidenceReadResponse = z.infer<typeof GroupEvidenceReadResponseSchema>;
+export type GroupEvidenceExportResult = z.infer<typeof GroupEvidenceExportResultSchema>;
+export type InspectorGroupsListResponse = z.infer<typeof InspectorGroupsListResponseSchema>;

package/src/lib/runContract.ts

@@ -71,6 +71,7 @@ export const JenkinsReportSchema = z.object({
   summary: z.string(),
   markdown: z.string(),
 });
+export const EvidenceReportSchema = JenkinsReportSchema;
 
 export const InspectorRunSchema = z.object({
   id: z.string(),
@@ -153,6 +154,7 @@ export type RunTimelineEvent = z.infer<typeof RunTimelineEventSchema>;
 export type FailureCategory = z.infer<typeof FailureCategorySchema>;
 export type FailureClassification = z.infer<typeof FailureClassificationSchema>;
 export type JenkinsReport = z.infer<typeof JenkinsReportSchema>;
+export type EvidenceReport = JenkinsReport;
 export type InspectorRun = z.infer<typeof InspectorRunSchema>;
 export type CreateInspectorRunInput = z.infer<typeof CreateInspectorRunInputSchema>;
 export type UpdateInspectorRunInput = z.infer<typeof UpdateInspectorRunInputSchema>;

package/src/lib/useGroupEvidence.ts

@@ -0,0 +1,33 @@
+import useSWR, { type SWRResponse } from "swr";
+import {
+  GroupEvidenceReadResponseSchema,
+  type GroupEvidenceReadResponse,
+  type InspectorGroup,
+} from "./groupContract";
+import { fetchJson } from "./apiClient";
+
+type UseGroupEvidenceResult = {
+  evidenceResponse: GroupEvidenceReadResponse | null;
+  isLoading: boolean;
+  error: Error | undefined;
+  mutate: SWRResponse<GroupEvidenceReadResponse, Error>["mutate"];
+};
+
+function evidenceKey(group: InspectorGroup | null): string | null {
+  if (group === null || group.evidence === null) return null;
+  return `/api/groups/${encodeURIComponent(group.id)}/evidence`;
+}
+
+export function useGroupEvidence(group: InspectorGroup | null): UseGroupEvidenceResult {
+  const response: SWRResponse<GroupEvidenceReadResponse, Error> = useSWR<
+    GroupEvidenceReadResponse,
+    Error
+  >(evidenceKey(group), (url: string) => fetchJson(url, GroupEvidenceReadResponseSchema));
+
+  return {
+    evidenceResponse: response.data ?? null,
+    isLoading: response.isLoading,
+    error: response.error,
+    mutate: response.mutate,
+  };
+}

package/src/lib/useGroups.ts

@@ -0,0 +1,28 @@
+import useSWR, { type SWRResponse } from "swr";
+import {
+  InspectorGroupsListResponseSchema,
+  type InspectorGroup,
+  type InspectorGroupsListResponse,
+} from "./groupContract";
+import { fetchJson } from "./apiClient";
+
+type UseGroupsResult = {
+  groups: InspectorGroup[];
+  isLoading: boolean;
+  error: Error | undefined;
+  mutate: SWRResponse<InspectorGroupsListResponse, Error>["mutate"];
+};
+
+export function useGroups(): UseGroupsResult {
+  const response: SWRResponse<InspectorGroupsListResponse, Error> = useSWR<
+    InspectorGroupsListResponse,
+    Error
+  >("/api/groups", (url: string) => fetchJson(url, InspectorGroupsListResponseSchema));
+
+  return {
+    groups: response.data?.groups ?? [],
+    isLoading: response.isLoading,
+    error: response.error,
+    mutate: response.mutate,
+  };
+}

package/src/mcp/server.ts

@@ -16,6 +16,7 @@
  */
 
 import { z } from "zod";
+import packageJson from "../../package.json";
 import {
   type GetPromptResult,
   type ListResourcesResult,
@@ -25,6 +26,14 @@ import {
   type Variables,
   WebStandardStreamableHTTPServerTransport,
 } from "@modelcontextprotocol/server";
+import {
+  AddInspectorGroupSessionInputSchema,
+  GroupEvidenceExportOptionsSchema,
+  InspectorGroupSchema,
+  InspectorGroupStatusSchema,
+  InspectorGroupsListResponseSchema,
+  UpdateInspectorGroupInputSchema,
+} from "../lib/groupContract";
 import {
   InspectorRunSchema,
   InspectorRunStatusSchema,
@@ -34,10 +43,14 @@ import { setCurrentPort } from "../lib/serverPort";
 import { JsonValueSchema } from "../proxy/schemas";
 import { callApi } from "./loopback";
 import {
+  addGroupSessionImpl,
   addProviderImpl,
+  createGroupImpl,
   createRunImpl,
   createSessionKnowledgeImpl,
   exportEvidenceImpl,
+  exportGroupEvidenceImpl,
+  getGroupImpl,
   getRecentFailuresImpl,
   getProjectContextImpl,
   getRunImpl,
@@ -45,6 +58,7 @@ import {
   getLogChunksImpl,
   getLogImpl,
   getProviderImpl,
+  listGroupsImpl,
   listKnowledgeCandidatesImpl,
   listLogsImpl,
   listModelsImpl,
@@ -56,6 +70,7 @@ import {
   searchLogsImpl,
   searchKnowledgeImpl,
   testProviderImpl,
+  updateGroupImpl,
   updateRunImpl,
   updateProviderImpl,
 } from "./toolHandlers";
@@ -66,12 +81,14 @@ import {
 
 type ServerPair = { server: McpServer; transport: WebStandardStreamableHTTPServerTransport };
 
+export const MCP_SERVER_VERSION = packageJson.version;
+
 let initialized: ServerPair | null = null;
 let initPromise: Promise<ServerPair> | null = null;
 
 function buildServer(): ServerPair {
   const server = new McpServer(
-    { name: "agent-inspector", version: "1.0.0" },
+    { name: "agent-inspector", version: MCP_SERVER_VERSION },
     { capabilities: { tools: {}, resources: {}, prompts: {} } },
   );
   registerTools(server);
@@ -172,6 +189,10 @@ const EvidenceReadResponseSchema = z.object({
   markdown: z.string(),
 });
 
+const GroupEvidenceReadResponseSchema = z.object({
+  markdown: z.string(),
+});
+
 type ResourceDescriptor = {
   uri: string;
   name: string;
@@ -270,6 +291,66 @@ async function listRunResources(): Promise<ListResourcesResult> {
   }
 }
 
+async function listGroupResources(): Promise<ListResourcesResult> {
+  try {
+    const response = await callApi("/api/groups");
+    if (!response.ok) return listResult([]);
+    const parsed = InspectorGroupsListResponseSchema.safeParse(await response.json());
+    if (!parsed.success) return listResult([]);
+    return listResult(
+      parsed.data.groups.map((group) => ({
+        uri: `inspector://groups/${encodeURIComponent(group.id)}`,
+        name: `group:${group.id}`,
+        title: group.title,
+        mimeType: "application/json",
+      })),
+    );
+  } catch {
+    return listResult([]);
+  }
+}
+
+async function listGroupSessionResources(): Promise<ListResourcesResult> {
+  try {
+    const response = await callApi("/api/groups");
+    if (!response.ok) return listResult([]);
+    const parsed = InspectorGroupsListResponseSchema.safeParse(await response.json());
+    if (!parsed.success) return listResult([]);
+    return listResult(
+      parsed.data.groups.map((group) => ({
+        uri: `inspector://groups/${encodeURIComponent(group.id)}/sessions`,
+        name: `group-sessions:${group.id}`,
+        title: `${group.title} sessions`,
+        mimeType: "application/json",
+      })),
+    );
+  } catch {
+    return listResult([]);
+  }
+}
+
+async function listGroupEvidenceResources(): Promise<ListResourcesResult> {
+  try {
+    const response = await callApi("/api/groups");
+    if (!response.ok) return listResult([]);
+    const parsed = InspectorGroupsListResponseSchema.safeParse(await response.json());
+    if (!parsed.success) return listResult([]);
+    return listResult(
+      parsed.data.groups
+        .filter((group) => group.evidence !== null)
+        .map((group) => ({
+          uri: `inspector://groups/${encodeURIComponent(group.id)}/evidence`,
+          name: `group-evidence:${group.id}`,
+          title: `${group.title} evidence pack`,
+          mimeType: "text/markdown",
+          description: group.evidence?.markdownPath,
+        })),
+    );
+  } catch {
+    return listResult([]);
+  }
+}
+
 async function listEvidenceResources(): Promise<ListResourcesResult> {
   try {
     const response = await callApi("/api/runs");
@@ -520,6 +601,92 @@ function registerTools(server: McpServer): void {
     (args) => safeCall(() => getRecentFailuresImpl(callApi, args)),
   );
 
+  server.registerTool(
+    "inspector_list_groups",
+    {
+      title: "List Inspector groups",
+      description:
+        "Lists declared Inspector groups. Groups aggregate multiple sessions/runs, commonly one evaluation matrix spanning several providers or models.",
+      inputSchema: z.object({}),
+    },
+    () => safeCall(() => listGroupsImpl(callApi)),
+  );
+
+  server.registerTool(
+    "inspector_create_group",
+    {
+      title: "Create or declare an Inspector group",
+      description:
+        "Creates an evaluation/batch group that external systems can use to aggregate multiple OpenCode/Codex sessions across models such as MiniMax, DeepSeek, or GLM.",
+      inputSchema: z.object({
+        groupId: z.string().min(1).optional().describe("Stable group id. Generated when omitted."),
+        title: z.string().min(1).optional().describe("Human-readable group title."),
+        kind: z
+          .enum(["evaluation", "batch", "manual"])
+          .optional()
+          .describe("Group kind. Defaults to evaluation."),
+        task: z.string().nullable().optional().describe("Task, benchmark, or evaluation goal."),
+        project: z.string().nullable().optional().describe("Project or repository under test."),
+        status: InspectorGroupStatusSchema.optional().describe("Initial group status."),
+        tags: z.array(z.string()).optional().describe("Free-form group labels."),
+        metadata: z.record(z.string(), JsonValueSchema).optional().describe("JSON metadata."),
+      }),
+    },
+    (args) => safeCall(() => createGroupImpl(callApi, args)),
+  );
+
+  server.registerTool(
+    "inspector_get_group",
+    {
+      title: "Get an Inspector group",
+      description:
+        "Returns one declared Inspector group with its attached session/run members and exported evidence paths when available.",
+      inputSchema: z.object({
+        groupId: z.string().min(1).describe("The Inspector group id."),
+      }),
+    },
+    (args) => safeCall(() => getGroupImpl(callApi, args)),
+  );
+
+  server.registerTool(
+    "inspector_update_group",
+    {
+      title: "Update an Inspector group",
+      description:
+        "PATCH-style update for a declared group. Use this to mark a whole evaluation running, failed, completed, or to merge CI metadata.",
+      inputSchema: UpdateInspectorGroupInputSchema.extend({
+        groupId: z.string().min(1).describe("The Inspector group id."),
+      }),
+    },
+    (args) => safeCall(() => updateGroupImpl(callApi, args)),
+  );
+
+  server.registerTool(
+    "inspector_add_group_session",
+    {
+      title: "Attach a session to an Inspector group",
+      description:
+        "Adds or updates one group member. Use it after each OpenCode process discovers its session id, or while an evaluation matrix is running to attach model/provider labels.",
+      inputSchema: AddInspectorGroupSessionInputSchema.extend({
+        groupId: z.string().min(1).describe("The Inspector group id."),
+      }),
+    },
+    (args) => safeCall(() => addGroupSessionImpl(callApi, args)),
+  );
+
+  server.registerTool(
+    "inspector_export_group_evidence",
+    {
+      title: "Export an Inspector group evidence pack",
+      description:
+        "Exports a whole group's evidence pack to local JSON, Markdown, and HTML files under <dataDir>/evidence/groups/<groupId>. The export contains a provider/model matrix and compact session summaries.",
+      inputSchema: GroupEvidenceExportOptionsSchema.unwrap().extend({
+        groupId: z.string().min(1).describe("The Inspector group id."),
+      }),
+    },
+    (args) => safeCall(() => exportGroupEvidenceImpl(callApi, args)),
+  );
+
   server.registerTool(
     "inspector_list_models",
     {
@@ -763,6 +930,49 @@ function registerResources(server: McpServer): void {
     (uri) => readApiJsonResource(uri, "/api/runs"),
   );
 
+  server.registerResource(
+    "inspector_groups",
+    "inspector://groups",
+    {
+      title: "Inspector groups",
+      description: "Declared Inspector groups aggregating multiple sessions or runs.",
+      mimeType: "application/json",
+    },
+    (uri) => readApiJsonResource(uri, "/api/groups"),
+  );
+
+  server.registerResource(
+    "inspector_group",
+    new ResourceTemplate("inspector://groups/{groupId}", { list: listGroupResources }),
+    {
+      title: "Inspector group",
+      description: "One declared Inspector group with attached session/run members.",
+      mimeType: "application/json",
+    },
+    (uri, variables) => {
+      const groupId = variableString(variables, "groupId");
+      if (groupId === null) return textResource(uri, "text/plain", "Missing groupId");
+      return readApiJsonResource(uri, `/api/groups/${encodeURIComponent(groupId)}`);
+    },
+  );
+
+  server.registerResource(
+    "inspector_group_sessions",
+    new ResourceTemplate("inspector://groups/{groupId}/sessions", {
+      list: listGroupSessionResources,
+    }),
+    {
+      title: "Inspector group sessions",
+      description: "Attached session/run members for one Inspector group.",
+      mimeType: "application/json",
+    },
+    (uri, variables) => {
+      const groupId = variableString(variables, "groupId");
+      if (groupId === null) return textResource(uri, "text/plain", "Missing groupId");
+      return readApiJsonResource(uri, `/api/groups/${encodeURIComponent(groupId)}/sessions`);
+    },
+  );
+
   server.registerResource(
     "inspector_recent_failures",
     "inspector://failures/recent",
@@ -819,6 +1029,38 @@ function registerResources(server: McpServer): void {
       }
     },
   );
+
+  server.registerResource(
+    "inspector_group_evidence",
+    new ResourceTemplate("inspector://groups/{groupId}/evidence", {
+      list: listGroupEvidenceResources,
+    }),
+    {
+      title: "Inspector group evidence pack",
+      description:
+        "Markdown evidence pack for an Inspector group. Generate it first with inspector_export_group_evidence.",
+      mimeType: "text/markdown",
+    },
+    async (uri, variables) => {
+      const groupId = variableString(variables, "groupId");
+      if (groupId === null) return textResource(uri, "text/plain", "Missing groupId");
+      const path = `/api/groups/${encodeURIComponent(groupId)}/evidence`;
+      try {
+        const response = await callApi(path);
+        if (!response.ok) {
+          return textResource(uri, "text/plain", `GET ${path} returned ${response.status}`);
+        }
+        const parsed = GroupEvidenceReadResponseSchema.safeParse(await response.json());
+        if (!parsed.success) {
+          return textResource(uri, "text/plain", "Evidence endpoint returned an unparseable body");
+        }
+        return textResource(uri, "text/markdown", parsed.data.markdown);
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        return textResource(uri, "text/plain", `Resource read failed: ${message}`);
+      }
+    },
+  );
 }
 
 function registerPrompts(server: McpServer): void {
@@ -911,6 +1153,34 @@ The report should include:
 5. Reproducible next action for a developer`),
   );
 
+  server.registerPrompt(
+    "inspector_generate_group_report",
+    {
+      title: "Generate group report",
+      description:
+        "Generate a model/provider evaluation report from an Inspector group and group evidence pack.",
+      argsSchema: z.object({
+        groupId: z.string().min(1).describe("The Inspector group id."),
+      }),
+    },
+    ({ groupId }) =>
+      userPrompt(`Generate a model/provider evaluation report for this Inspector group.
+
+Evidence to read:
+- Group resource: inspector://groups/${encodeURIComponent(groupId)}
+- Group session members: inspector://groups/${encodeURIComponent(groupId)}/sessions
+- Group evidence pack: inspector://groups/${encodeURIComponent(groupId)}/evidence
+
+If the evidence pack resource says it has not been exported, call inspector_export_group_evidence for this group first.
+
+The report should include:
+1. Evaluation status summary
+2. Provider/model matrix with session links
+3. Request counts, token totals, streaming timing, and error counts
+4. Winner/loser observations when the evidence supports them
+5. Concrete next actions for failed or missing sessions`),
+  );
+
   server.registerPrompt(
     "inspector_extract_repro_steps",
     {
@@ -950,6 +1220,12 @@ export const TOOL_NAMES = [
   "inspector_update_run",
   "inspector_export_evidence",
   "inspector_get_recent_failures",
+  "inspector_list_groups",
+  "inspector_create_group",
+  "inspector_get_group",
+  "inspector_update_group",
+  "inspector_add_group_session",
+  "inspector_export_group_evidence",
   "inspector_list_models",
   "inspector_list_providers",
   "inspector_get_provider",
@@ -972,11 +1248,16 @@ export const RESOURCE_NAMES = [
   "inspector_recent_failures",
   "inspector_run",
   "inspector_evidence",
+  "inspector_groups",
+  "inspector_group",
+  "inspector_group_sessions",
+  "inspector_group_evidence",
 ] as const;
 
 export const PROMPT_NAMES = [
   "inspector_analyze_session_failure",
   "inspector_triage_recent_failures",
   "inspector_generate_jenkins_report",
+  "inspector_generate_group_report",
   "inspector_extract_repro_steps",
 ] as const;