@tonyclaw/agent-inspector 2.0.29 → 2.0.31

package/src/mcp/toolHandlers.ts

@@ -17,6 +17,14 @@ import { Buffer } from "node:buffer";
 import { z } from "zod";
 import { LoopbackTimeoutError, type CallApiOptions } from "./loopback";
 import { extractLastUserMessagePreview, extractResponsePreview } from "./previewExtractor";
+import {
+  GroupEvidenceExportResultSchema,
+  InspectorGroupSchema,
+  InspectorGroupsListResponseSchema,
+  type AddInspectorGroupSessionInput,
+  type CreateInspectorGroupInput,
+  type UpdateInspectorGroupInput,
+} from "../lib/groupContract";
 import {
   EvidenceExportResultSchema,
   InspectorRunSchema,
@@ -375,6 +383,121 @@ export async function exportEvidenceImpl(
   return textJson(parsed.data);
 }
 
+export async function listGroupsImpl(callApi: CallApiFn): Promise<ToolResult> {
+  const res = await callApi("/api/groups");
+  if (!res.ok) return toolError(`GET /api/groups returned ${res.status}`);
+  const rawBody: unknown = await res.json();
+  const parsed = InspectorGroupsListResponseSchema.safeParse(rawBody);
+  if (!parsed.success) return toolError("GET /api/groups returned an unparseable group list");
+  return textJson(parsed.data);
+}
+
+export async function createGroupImpl(
+  callApi: CallApiFn,
+  args: CreateInspectorGroupInput,
+): Promise<ToolResult> {
+  const res = await callApi("/api/groups", {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify(args ?? {}),
+  });
+  if (!res.ok) return toolError(`POST /api/groups returned ${res.status}`);
+  const rawBody: unknown = await res.json();
+  const parsed = InspectorGroupSchema.safeParse(rawBody);
+  if (!parsed.success) return toolError("POST /api/groups returned an unparseable group");
+  return textJson(parsed.data);
+}
+
+export type GetGroupArgs = {
+  groupId: string;
+};
+
+export async function getGroupImpl(callApi: CallApiFn, args: GetGroupArgs): Promise<ToolResult> {
+  const path = `/api/groups/${encodeURIComponent(args.groupId)}`;
+  const res = await callApi(path);
+  if (!res.ok) return toolError(`GET ${path} returned ${res.status}`);
+  const rawBody: unknown = await res.json();
+  const parsed = InspectorGroupSchema.safeParse(rawBody);
+  if (!parsed.success) return toolError("GET /api/groups/{groupId} returned an unparseable group");
+  return textJson(parsed.data);
+}
+
+export type UpdateGroupArgs = {
+  groupId: string;
+} & UpdateInspectorGroupInput;
+
+export async function updateGroupImpl(
+  callApi: CallApiFn,
+  args: UpdateGroupArgs,
+): Promise<ToolResult> {
+  const { groupId, ...patch } = args;
+  const path = `/api/groups/${encodeURIComponent(groupId)}`;
+  const res = await callApi(path, {
+    method: "PATCH",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify(patch),
+  });
+  if (!res.ok) return toolError(`PATCH ${path} returned ${res.status}`);
+  const rawBody: unknown = await res.json();
+  const parsed = InspectorGroupSchema.safeParse(rawBody);
+  if (!parsed.success) {
+    return toolError("PATCH /api/groups/{groupId} returned an unparseable group");
+  }
+  return textJson(parsed.data);
+}
+
+export type AddGroupSessionArgs = {
+  groupId: string;
+} & AddInspectorGroupSessionInput;
+
+export async function addGroupSessionImpl(
+  callApi: CallApiFn,
+  args: AddGroupSessionArgs,
+): Promise<ToolResult> {
+  const { groupId, ...session } = args;
+  const path = `/api/groups/${encodeURIComponent(groupId)}/sessions`;
+  const res = await callApi(path, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify(session),
+  });
+  if (!res.ok) return toolError(`POST ${path} returned ${res.status}`);
+  const rawBody: unknown = await res.json();
+  const parsed = InspectorGroupSchema.safeParse(rawBody);
+  if (!parsed.success) {
+    return toolError("POST /api/groups/{groupId}/sessions returned an unparseable group");
+  }
+  return textJson(parsed.data);
+}
+
+export type ExportGroupEvidenceArgs = {
+  groupId: string;
+  includeHistory?: boolean;
+  latestLogLimit?: number;
+};
+
+export async function exportGroupEvidenceImpl(
+  callApi: CallApiFn,
+  args: ExportGroupEvidenceArgs,
+): Promise<ToolResult> {
+  const path = `/api/groups/${encodeURIComponent(args.groupId)}/evidence`;
+  const res = await callApi(path, {
+    method: "POST",
+    headers: { "content-type": "application/json" },
+    body: JSON.stringify({
+      includeHistory: args.includeHistory,
+      latestLogLimit: args.latestLogLimit,
+    }),
+  });
+  if (!res.ok) return toolError(`POST ${path} returned ${res.status}`);
+  const rawBody: unknown = await res.json();
+  const parsed = GroupEvidenceExportResultSchema.safeParse(rawBody);
+  if (!parsed.success) {
+    return toolError("POST /api/groups/{groupId}/evidence returned an unparseable evidence result");
+  }
+  return textJson(parsed.data);
+}
+
 export async function listModelsImpl(callApi: CallApiFn): Promise<ToolResult> {
   const res = await callApi("/api/models");
   if (!res.ok) return toolError(`GET /api/models returned ${res.status}`);

package/src/proxy/groupEvidenceExporter.ts

@@ -0,0 +1,354 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import {
+  GroupEvidenceExportResultSchema,
+  type GroupEvidenceExportOptions,
+  type GroupEvidenceExportResult,
+  type InspectorGroup,
+  type InspectorGroupEvidence,
+  type InspectorGroupEvidenceSummary,
+  type InspectorGroupExportMember,
+} from "../lib/groupContract";
+import type { JenkinsReport } from "../lib/runContract";
+import type { SessionInfo, SessionTokenUsage } from "../lib/sessionInfoContract";
+import { getDataDir } from "./dataDir";
+import { getRun } from "./runStore";
+import { getSessionInfo } from "./store";
+import { updateGroupEvidence } from "./groupStore";
+
+type GroupEvidenceDocument = GroupEvidenceExportResult;
+
+function groupEvidenceDir(groupId: string): string {
+  return join(getDataDir(), "evidence", "groups", groupId);
+}
+
+function jsonPath(groupId: string): string {
+  return join(groupEvidenceDir(groupId), "evidence.json");
+}
+
+function markdownPath(groupId: string): string {
+  return join(groupEvidenceDir(groupId), "evidence.md");
+}
+
+function htmlPath(groupId: string): string {
+  return join(groupEvidenceDir(groupId), "evidence.html");
+}
+
+function escapeMarkdownCell(value: string): string {
+  return value.replace(/\|/g, "\\|").replace(/\r?\n/g, " ");
+}
+
+function escapeHtml(value: string): string {
+  return value
+    .replace(/&/g, "&")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;");
+}
+
+function formatText(value: string | null | undefined): string {
+  if (value === null || value === undefined || value.length === 0) return "n/a";
+  return value;
+}
+
+function formatNumber(value: number | null | undefined): string {
+  if (value === null || value === undefined) return "n/a";
+  return String(value);
+}
+
+function collectSortedValues(values: readonly (string | null | undefined)[]): string[] {
+  const set = new Set<string>();
+  for (const value of values) {
+    if (value !== null && value !== undefined && value.length > 0) {
+      set.add(value);
+    }
+  }
+  return [...set].sort();
+}
+
+function addTokenUsage(total: SessionTokenUsage, session: SessionInfo | null): SessionTokenUsage {
+  if (session === null) return total;
+  return {
+    input: total.input + session.tokenUsage.input,
+    output: total.output + session.tokenUsage.output,
+    cacheCreate: total.cacheCreate + session.tokenUsage.cacheCreate,
+    cacheRead: total.cacheRead + session.tokenUsage.cacheRead,
+    total: total.total + session.tokenUsage.total,
+  };
+}
+
+function memberEffectiveStatus(member: InspectorGroupExportMember): string {
+  if (member.member.status !== null) return member.member.status;
+  if (member.run !== null) return member.run.status;
+  return member.session?.status ?? "unknown";
+}
+
+function isCompletedMember(member: InspectorGroupExportMember): boolean {
+  const status = memberEffectiveStatus(member);
+  return status === "completed";
+}
+
+function isFailedMember(member: InspectorGroupExportMember): boolean {
+  const status = memberEffectiveStatus(member);
+  return status === "failed" || status === "cancelled";
+}
+
+function isRunningMember(member: InspectorGroupExportMember): boolean {
+  const status = memberEffectiveStatus(member);
+  return status === "running" || status === "active";
+}
+
+function latestLogMetric(
+  session: SessionInfo | null,
+  field: "firstChunkMs" | "totalStreamMs" | "tokensPerSecond",
+): number | null {
+  if (session === null) return null;
+  for (const log of session.latestLogs) {
+    const value = log[field];
+    if (value !== undefined && value !== null) return value;
+  }
+  return null;
+}
+
+function buildSummary(
+  members: readonly InspectorGroupExportMember[],
+): InspectorGroupEvidenceSummary {
+  const tokenUsage = members.reduce<SessionTokenUsage>(
+    (total, member) => addTokenUsage(total, member.session),
+    { input: 0, output: 0, cacheCreate: 0, cacheRead: 0, total: 0 },
+  );
+  const sessionsWithData = members.filter((member) => member.session !== null).length;
+  const requestCount = members.reduce(
+    (total, member) => total + (member.session?.requestCount ?? 0),
+    0,
+  );
+  const errorCount = members.reduce(
+    (total, member) => total + (member.session?.errorCount ?? 0),
+    0,
+  );
+
+  return {
+    memberCount: members.length,
+    sessionsWithData,
+    missingSessions: members.length - sessionsWithData,
+    requestCount,
+    errorCount,
+    tokenUsage,
+    models: collectSortedValues(
+      members.flatMap((member) => [member.member.model, ...(member.session?.models ?? [])]),
+    ),
+    providers: collectSortedValues(
+      members.flatMap((member) => [member.member.provider, ...(member.session?.providers ?? [])]),
+    ),
+    completedMembers: members.filter(isCompletedMember).length,
+    failedMembers: members.filter(isFailedMember).length,
+    runningMembers: members.filter(isRunningMember).length,
+  };
+}
+
+function buildJenkinsReport(
+  group: InspectorGroup,
+  summary: InspectorGroupEvidenceSummary,
+): JenkinsReport {
+  if (summary.failedMembers > 0 || summary.errorCount > 0 || group.status === "failed") {
+    return {
+      status: "FAIL",
+      summary: `[FAIL] ${summary.failedMembers} failed members, ${summary.errorCount} captured errors.`,
+      markdown: `# Jenkins Report: ${group.title}\n\nStatus: FAIL\n\n${summary.failedMembers} failed members and ${summary.errorCount} captured errors need triage.`,
+    };
+  }
+
+  if (summary.runningMembers > 0 || group.status === "running") {
+    return {
+      status: "RUNNING",
+      summary: `[RUNNING] ${summary.runningMembers} group members are still active.`,
+      markdown: `# Jenkins Report: ${group.title}\n\nStatus: RUNNING\n\n${summary.runningMembers} group members are still active.`,
+    };
+  }
+
+  if (
+    summary.memberCount > 0 &&
+    summary.sessionsWithData > 0 &&
+    summary.errorCount === 0 &&
+    group.status === "completed"
+  ) {
+    return {
+      status: "PASS",
+      summary: `[PASS] ${summary.memberCount} group members completed with no captured errors.`,
+      markdown: `# Jenkins Report: ${group.title}\n\nStatus: PASS\n\n${summary.memberCount} group members completed with no captured errors.`,
+    };
+  }
+
+  return {
+    status: "UNKNOWN",
+    summary: `[UNKNOWN] Group has ${summary.memberCount} members and ${summary.missingSessions} missing sessions.`,
+    markdown: `# Jenkins Report: ${group.title}\n\nStatus: UNKNOWN\n\nExported group evidence, but the outcome is not conclusive yet.`,
+  };
+}
+
+function buildMemberRows(members: readonly InspectorGroupExportMember[]): string[] {
+  if (members.length === 0) return ["No sessions have been attached to this group.", ""];
+
+  const rows = [
+    "| Label | Provider | Model | Session | Run | Status | Requests | Errors | Tokens | First Chunk | Stream Total | TPS | Inspector |",
+    "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |",
+  ];
+
+  for (const member of members) {
+    const session = member.session;
+    rows.push(
+      [
+        formatText(member.member.label),
+        formatText(member.member.provider ?? session?.providers[0]),
+        formatText(member.member.model ?? session?.lastModel),
+        member.member.sessionId,
+        formatText(member.member.runId),
+        memberEffectiveStatus(member),
+        formatNumber(session?.requestCount),
+        formatNumber(session?.errorCount),
+        formatNumber(session?.tokenUsage.total),
+        formatNumber(latestLogMetric(session, "firstChunkMs")),
+        formatNumber(latestLogMetric(session, "totalStreamMs")),
+        formatNumber(latestLogMetric(session, "tokensPerSecond")),
+        formatText(session?.inspectorUrl),
+      ]
+        .map(escapeMarkdownCell)
+        .join(" | ")
+        .replace(/^/, "| ")
+        .replace(/$/, " |"),
+    );
+  }
+
+  rows.push("");
+  return rows;
+}
+
+function buildMarkdown(document: GroupEvidenceDocument): string {
+  const group = document.group;
+  const summary = document.summary;
+  const lines = [
+    `# ${group.title}`,
+    "",
+    `- Group ID: ${group.id}`,
+    `- Kind: ${group.kind}`,
+    `- Status: ${group.status}`,
+    `- Project: ${group.project ?? "n/a"}`,
+    `- Created: ${group.createdAt}`,
+    `- Members: ${summary.memberCount}`,
+    `- Sessions With Data: ${summary.sessionsWithData}`,
+    `- Missing Sessions: ${summary.missingSessions}`,
+    `- Requests: ${summary.requestCount}`,
+    `- Errors: ${summary.errorCount}`,
+    `- Tokens: ${summary.tokenUsage.total}`,
+    `- Models: ${summary.models.length > 0 ? summary.models.join(", ") : "n/a"}`,
+    `- Providers: ${summary.providers.length > 0 ? summary.providers.join(", ") : "n/a"}`,
+    `- Evidence JSON: ${document.evidence.jsonPath}`,
+    `- Evidence HTML: ${document.evidence.htmlPath}`,
+    "",
+  ];
+
+  if (group.task !== null) {
+    lines.push("## Task", "", group.task, "");
+  }
+
+  lines.push("## Evaluation Matrix", "", ...buildMemberRows(document.members));
+  lines.push("## Jenkins Report", "", document.jenkinsReport.markdown, "");
+
+  return `${lines.join("\n")}\n`;
+}
+
+function buildHtml(markdown: string, title: string): string {
+  return `<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <title>${escapeHtml(title)}</title>
+  <style>
+    body { font-family: system-ui, sans-serif; max-width: 1120px; margin: 40px auto; line-height: 1.5; }
+    pre { white-space: pre-wrap; background: #f6f8fa; padding: 16px; border-radius: 8px; }
+  </style>
+</head>
+<body>
+  <pre>${escapeHtml(markdown)}</pre>
+</body>
+</html>
+`;
+}
+
+async function buildExportMembers(
+  group: InspectorGroup,
+  options: GroupEvidenceExportOptions,
+  baseUrl: string,
+): Promise<InspectorGroupExportMember[]> {
+  const members: InspectorGroupExportMember[] = [];
+  for (const member of group.members) {
+    const run = member.runId === null ? null : getRun(member.runId);
+    const session = await getSessionInfo(member.sessionId, {
+      baseUrl,
+      includeHistory: options?.includeHistory,
+      latestLogLimit: options?.latestLogLimit,
+    });
+    members.push({ member, run, session });
+  }
+  return members;
+}
+
+export async function exportGroupEvidence(
+  group: InspectorGroup,
+  options: GroupEvidenceExportOptions,
+  baseUrl: string,
+): Promise<GroupEvidenceExportResult> {
+  const members = await buildExportMembers(group, options, baseUrl);
+  const exportedAt = new Date().toISOString();
+  const evidence: InspectorGroupEvidence = {
+    jsonPath: jsonPath(group.id),
+    markdownPath: markdownPath(group.id),
+    htmlPath: htmlPath(group.id),
+    exportedAt,
+  };
+  const documentGroup: InspectorGroup = {
+    ...group,
+    evidence,
+    updatedAt: exportedAt,
+  };
+  const summary = buildSummary(members);
+  const jenkinsReport = buildJenkinsReport(documentGroup, summary);
+  const document: GroupEvidenceDocument = {
+    group: documentGroup,
+    evidence,
+    members,
+    summary,
+    jenkinsReport,
+  };
+  const markdown = buildMarkdown(document);
+
+  mkdirSync(groupEvidenceDir(group.id), { recursive: true });
+  writeFileSync(evidence.jsonPath, `${JSON.stringify(document, null, 2)}\n`, "utf8");
+  writeFileSync(evidence.markdownPath, markdown, "utf8");
+  writeFileSync(evidence.htmlPath, buildHtml(markdown, group.title), "utf8");
+
+  const updatedGroup = updateGroupEvidence(group.id, evidence) ?? documentGroup;
+  return { ...document, group: updatedGroup };
+}
+
+export function readGroupEvidenceMarkdown(group: InspectorGroup): string | null {
+  const path = group.evidence?.markdownPath;
+  if (path === undefined || !existsSync(path)) return null;
+  try {
+    return readFileSync(path, "utf8");
+  } catch {
+    return null;
+  }
+}
+
+export function readGroupEvidenceDocument(group: InspectorGroup): GroupEvidenceExportResult | null {
+  const path = group.evidence?.jsonPath;
+  if (path === undefined || !existsSync(path)) return null;
+  try {
+    const raw: unknown = JSON.parse(readFileSync(path, "utf8"));
+    const parsed = GroupEvidenceExportResultSchema.safeParse(raw);
+    return parsed.success ? parsed.data : null;
+  } catch {
+    return null;
+  }
+}