@tonyclaw/agent-inspector 2.0.29 → 2.0.30

package/src/proxy/groupEvidenceExporter.ts

@@ -0,0 +1,341 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import {
+  type GroupEvidenceExportOptions,
+  type GroupEvidenceExportResult,
+  type InspectorGroup,
+  type InspectorGroupEvidence,
+  type InspectorGroupEvidenceSummary,
+  type InspectorGroupExportMember,
+} from "../lib/groupContract";
+import type { JenkinsReport } from "../lib/runContract";
+import type { SessionInfo, SessionTokenUsage } from "../lib/sessionInfoContract";
+import { getDataDir } from "./dataDir";
+import { getRun } from "./runStore";
+import { getSessionInfo } from "./store";
+import { updateGroupEvidence } from "./groupStore";
+
+type GroupEvidenceDocument = GroupEvidenceExportResult;
+
+function groupEvidenceDir(groupId: string): string {
+  return join(getDataDir(), "evidence", "groups", groupId);
+}
+
+function jsonPath(groupId: string): string {
+  return join(groupEvidenceDir(groupId), "evidence.json");
+}
+
+function markdownPath(groupId: string): string {
+  return join(groupEvidenceDir(groupId), "evidence.md");
+}
+
+function htmlPath(groupId: string): string {
+  return join(groupEvidenceDir(groupId), "evidence.html");
+}
+
+function escapeMarkdownCell(value: string): string {
+  return value.replace(/\|/g, "\\|").replace(/\r?\n/g, " ");
+}
+
+function escapeHtml(value: string): string {
+  return value
+    .replace(/&/g, "&")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;");
+}
+
+function formatText(value: string | null | undefined): string {
+  if (value === null || value === undefined || value.length === 0) return "n/a";
+  return value;
+}
+
+function formatNumber(value: number | null | undefined): string {
+  if (value === null || value === undefined) return "n/a";
+  return String(value);
+}
+
+function collectSortedValues(values: readonly (string | null | undefined)[]): string[] {
+  const set = new Set<string>();
+  for (const value of values) {
+    if (value !== null && value !== undefined && value.length > 0) {
+      set.add(value);
+    }
+  }
+  return [...set].sort();
+}
+
+function addTokenUsage(total: SessionTokenUsage, session: SessionInfo | null): SessionTokenUsage {
+  if (session === null) return total;
+  return {
+    input: total.input + session.tokenUsage.input,
+    output: total.output + session.tokenUsage.output,
+    cacheCreate: total.cacheCreate + session.tokenUsage.cacheCreate,
+    cacheRead: total.cacheRead + session.tokenUsage.cacheRead,
+    total: total.total + session.tokenUsage.total,
+  };
+}
+
+function memberEffectiveStatus(member: InspectorGroupExportMember): string {
+  if (member.member.status !== null) return member.member.status;
+  if (member.run !== null) return member.run.status;
+  return member.session?.status ?? "unknown";
+}
+
+function isCompletedMember(member: InspectorGroupExportMember): boolean {
+  const status = memberEffectiveStatus(member);
+  return status === "completed";
+}
+
+function isFailedMember(member: InspectorGroupExportMember): boolean {
+  const status = memberEffectiveStatus(member);
+  return status === "failed" || status === "cancelled";
+}
+
+function isRunningMember(member: InspectorGroupExportMember): boolean {
+  const status = memberEffectiveStatus(member);
+  return status === "running" || status === "active";
+}
+
+function latestLogMetric(
+  session: SessionInfo | null,
+  field: "firstChunkMs" | "totalStreamMs" | "tokensPerSecond",
+): number | null {
+  if (session === null) return null;
+  for (const log of session.latestLogs) {
+    const value = log[field];
+    if (value !== undefined && value !== null) return value;
+  }
+  return null;
+}
+
+function buildSummary(
+  members: readonly InspectorGroupExportMember[],
+): InspectorGroupEvidenceSummary {
+  const tokenUsage = members.reduce<SessionTokenUsage>(
+    (total, member) => addTokenUsage(total, member.session),
+    { input: 0, output: 0, cacheCreate: 0, cacheRead: 0, total: 0 },
+  );
+  const sessionsWithData = members.filter((member) => member.session !== null).length;
+  const requestCount = members.reduce(
+    (total, member) => total + (member.session?.requestCount ?? 0),
+    0,
+  );
+  const errorCount = members.reduce(
+    (total, member) => total + (member.session?.errorCount ?? 0),
+    0,
+  );
+
+  return {
+    memberCount: members.length,
+    sessionsWithData,
+    missingSessions: members.length - sessionsWithData,
+    requestCount,
+    errorCount,
+    tokenUsage,
+    models: collectSortedValues(
+      members.flatMap((member) => [member.member.model, ...(member.session?.models ?? [])]),
+    ),
+    providers: collectSortedValues(
+      members.flatMap((member) => [member.member.provider, ...(member.session?.providers ?? [])]),
+    ),
+    completedMembers: members.filter(isCompletedMember).length,
+    failedMembers: members.filter(isFailedMember).length,
+    runningMembers: members.filter(isRunningMember).length,
+  };
+}
+
+function buildJenkinsReport(
+  group: InspectorGroup,
+  summary: InspectorGroupEvidenceSummary,
+): JenkinsReport {
+  if (summary.failedMembers > 0 || summary.errorCount > 0 || group.status === "failed") {
+    return {
+      status: "FAIL",
+      summary: `[FAIL] ${summary.failedMembers} failed members, ${summary.errorCount} captured errors.`,
+      markdown: `# Jenkins Report: ${group.title}\n\nStatus: FAIL\n\n${summary.failedMembers} failed members and ${summary.errorCount} captured errors need triage.`,
+    };
+  }
+
+  if (summary.runningMembers > 0 || group.status === "running") {
+    return {
+      status: "RUNNING",
+      summary: `[RUNNING] ${summary.runningMembers} group members are still active.`,
+      markdown: `# Jenkins Report: ${group.title}\n\nStatus: RUNNING\n\n${summary.runningMembers} group members are still active.`,
+    };
+  }
+
+  if (
+    summary.memberCount > 0 &&
+    summary.sessionsWithData > 0 &&
+    summary.errorCount === 0 &&
+    group.status === "completed"
+  ) {
+    return {
+      status: "PASS",
+      summary: `[PASS] ${summary.memberCount} group members completed with no captured errors.`,
+      markdown: `# Jenkins Report: ${group.title}\n\nStatus: PASS\n\n${summary.memberCount} group members completed with no captured errors.`,
+    };
+  }
+
+  return {
+    status: "UNKNOWN",
+    summary: `[UNKNOWN] Group has ${summary.memberCount} members and ${summary.missingSessions} missing sessions.`,
+    markdown: `# Jenkins Report: ${group.title}\n\nStatus: UNKNOWN\n\nExported group evidence, but the outcome is not conclusive yet.`,
+  };
+}
+
+function buildMemberRows(members: readonly InspectorGroupExportMember[]): string[] {
+  if (members.length === 0) return ["No sessions have been attached to this group.", ""];
+
+  const rows = [
+    "| Label | Provider | Model | Session | Run | Status | Requests | Errors | Tokens | First Chunk | Stream Total | TPS | Inspector |",
+    "| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |",
+  ];
+
+  for (const member of members) {
+    const session = member.session;
+    rows.push(
+      [
+        formatText(member.member.label),
+        formatText(member.member.provider ?? session?.providers[0]),
+        formatText(member.member.model ?? session?.lastModel),
+        member.member.sessionId,
+        formatText(member.member.runId),
+        memberEffectiveStatus(member),
+        formatNumber(session?.requestCount),
+        formatNumber(session?.errorCount),
+        formatNumber(session?.tokenUsage.total),
+        formatNumber(latestLogMetric(session, "firstChunkMs")),
+        formatNumber(latestLogMetric(session, "totalStreamMs")),
+        formatNumber(latestLogMetric(session, "tokensPerSecond")),
+        formatText(session?.inspectorUrl),
+      ]
+        .map(escapeMarkdownCell)
+        .join(" | ")
+        .replace(/^/, "| ")
+        .replace(/$/, " |"),
+    );
+  }
+
+  rows.push("");
+  return rows;
+}
+
+function buildMarkdown(document: GroupEvidenceDocument): string {
+  const group = document.group;
+  const summary = document.summary;
+  const lines = [
+    `# ${group.title}`,
+    "",
+    `- Group ID: ${group.id}`,
+    `- Kind: ${group.kind}`,
+    `- Status: ${group.status}`,
+    `- Project: ${group.project ?? "n/a"}`,
+    `- Created: ${group.createdAt}`,
+    `- Members: ${summary.memberCount}`,
+    `- Sessions With Data: ${summary.sessionsWithData}`,
+    `- Missing Sessions: ${summary.missingSessions}`,
+    `- Requests: ${summary.requestCount}`,
+    `- Errors: ${summary.errorCount}`,
+    `- Tokens: ${summary.tokenUsage.total}`,
+    `- Models: ${summary.models.length > 0 ? summary.models.join(", ") : "n/a"}`,
+    `- Providers: ${summary.providers.length > 0 ? summary.providers.join(", ") : "n/a"}`,
+    `- Evidence JSON: ${document.evidence.jsonPath}`,
+    `- Evidence HTML: ${document.evidence.htmlPath}`,
+    "",
+  ];
+
+  if (group.task !== null) {
+    lines.push("## Task", "", group.task, "");
+  }
+
+  lines.push("## Evaluation Matrix", "", ...buildMemberRows(document.members));
+  lines.push("## Jenkins Report", "", document.jenkinsReport.markdown, "");
+
+  return `${lines.join("\n")}\n`;
+}
+
+function buildHtml(markdown: string, title: string): string {
+  return `<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <title>${escapeHtml(title)}</title>
+  <style>
+    body { font-family: system-ui, sans-serif; max-width: 1120px; margin: 40px auto; line-height: 1.5; }
+    pre { white-space: pre-wrap; background: #f6f8fa; padding: 16px; border-radius: 8px; }
+  </style>
+</head>
+<body>
+  <pre>${escapeHtml(markdown)}</pre>
+</body>
+</html>
+`;
+}
+
+async function buildExportMembers(
+  group: InspectorGroup,
+  options: GroupEvidenceExportOptions,
+  baseUrl: string,
+): Promise<InspectorGroupExportMember[]> {
+  const members: InspectorGroupExportMember[] = [];
+  for (const member of group.members) {
+    const run = member.runId === null ? null : getRun(member.runId);
+    const session = await getSessionInfo(member.sessionId, {
+      baseUrl,
+      includeHistory: options?.includeHistory,
+      latestLogLimit: options?.latestLogLimit,
+    });
+    members.push({ member, run, session });
+  }
+  return members;
+}
+
+export async function exportGroupEvidence(
+  group: InspectorGroup,
+  options: GroupEvidenceExportOptions,
+  baseUrl: string,
+): Promise<GroupEvidenceExportResult> {
+  const members = await buildExportMembers(group, options, baseUrl);
+  const exportedAt = new Date().toISOString();
+  const evidence: InspectorGroupEvidence = {
+    jsonPath: jsonPath(group.id),
+    markdownPath: markdownPath(group.id),
+    htmlPath: htmlPath(group.id),
+    exportedAt,
+  };
+  const documentGroup: InspectorGroup = {
+    ...group,
+    evidence,
+    updatedAt: exportedAt,
+  };
+  const summary = buildSummary(members);
+  const jenkinsReport = buildJenkinsReport(documentGroup, summary);
+  const document: GroupEvidenceDocument = {
+    group: documentGroup,
+    evidence,
+    members,
+    summary,
+    jenkinsReport,
+  };
+  const markdown = buildMarkdown(document);
+
+  mkdirSync(groupEvidenceDir(group.id), { recursive: true });
+  writeFileSync(evidence.jsonPath, `${JSON.stringify(document, null, 2)}\n`, "utf8");
+  writeFileSync(evidence.markdownPath, markdown, "utf8");
+  writeFileSync(evidence.htmlPath, buildHtml(markdown, group.title), "utf8");
+
+  const updatedGroup = updateGroupEvidence(group.id, evidence) ?? documentGroup;
+  return { ...document, group: updatedGroup };
+}
+
+export function readGroupEvidenceMarkdown(group: InspectorGroup): string | null {
+  const path = group.evidence?.markdownPath;
+  if (path === undefined || !existsSync(path)) return null;
+  try {
+    return readFileSync(path, "utf8");
+  } catch {
+    return null;
+  }
+}

package/src/proxy/groupStore.ts

@@ -0,0 +1,259 @@
+import { randomUUID } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { z } from "zod";
+import {
+  AddInspectorGroupSessionInputSchema,
+  CreateInspectorGroupInputSchema,
+  InspectorGroupSchema,
+  UpdateInspectorGroupInputSchema,
+  type AddInspectorGroupSessionInput,
+  type CreateInspectorGroupInput,
+  type InspectorGroup,
+  type InspectorGroupEvidence,
+  type InspectorGroupMember,
+  type UpdateInspectorGroupInput,
+} from "../lib/groupContract";
+import { getDataDir } from "./dataDir";
+
+const GroupsFileSchema = z.object({
+  groups: z.array(InspectorGroupSchema),
+});
+
+function groupsFilePath(): string {
+  return join(getDataDir(), "groups.json");
+}
+
+function nowIso(): string {
+  return new Date().toISOString();
+}
+
+function stableId(input: string): string {
+  return input
+    .trim()
+    .replace(/[^A-Za-z0-9_.-]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .slice(0, 80);
+}
+
+function createGroupId(): string {
+  return `grp_${Date.now().toString(36)}_${randomUUID().slice(0, 8)}`;
+}
+
+function createMemberId(): string {
+  return `mem_${Date.now().toString(36)}_${randomUUID().slice(0, 8)}`;
+}
+
+function normalizeGroupId(input: string | undefined): string {
+  if (input === undefined) return createGroupId();
+  const normalized = stableId(input);
+  return normalized.length > 0 ? normalized : createGroupId();
+}
+
+function normalizeMemberId(input: string | undefined, sessionId: string): string {
+  if (input === undefined) {
+    const normalizedSession = stableId(sessionId);
+    return normalizedSession.length > 0 ? `mem_${normalizedSession}` : createMemberId();
+  }
+  const normalized = stableId(input);
+  return normalized.length > 0 ? normalized : createMemberId();
+}
+
+function nullableText(value: string | null | undefined): string | null {
+  if (value === undefined || value === null) return null;
+  const trimmed = value.trim();
+  return trimmed.length > 0 ? trimmed : null;
+}
+
+function readGroups(): InspectorGroup[] {
+  const filePath = groupsFilePath();
+  if (!existsSync(filePath)) return [];
+
+  try {
+    const parsed: unknown = JSON.parse(readFileSync(filePath, "utf8"));
+    const result = GroupsFileSchema.safeParse(parsed);
+    return result.success ? result.data.groups : [];
+  } catch {
+    return [];
+  }
+}
+
+function writeGroups(groups: readonly InspectorGroup[]): void {
+  const filePath = groupsFilePath();
+  mkdirSync(dirname(filePath), { recursive: true });
+  writeFileSync(filePath, `${JSON.stringify({ groups }, null, 2)}\n`, "utf8");
+}
+
+function sortedGroups(groups: readonly InspectorGroup[]): InspectorGroup[] {
+  return [...groups].sort((left, right) => right.updatedAt.localeCompare(left.updatedAt));
+}
+
+export function listGroups(): InspectorGroup[] {
+  return sortedGroups(readGroups());
+}
+
+export function getGroup(groupId: string): InspectorGroup | null {
+  return readGroups().find((group) => group.id === groupId) ?? null;
+}
+
+export function createGroup(input: CreateInspectorGroupInput): InspectorGroup {
+  const parsed = CreateInspectorGroupInputSchema.parse(input);
+  const groupId = normalizeGroupId(parsed?.groupId);
+  const existing = getGroup(groupId);
+  if (existing !== null) return existing;
+
+  const timestamp = nowIso();
+  const group: InspectorGroup = {
+    id: groupId,
+    title: parsed?.title ?? `Inspector Group ${groupId}`,
+    kind: parsed?.kind ?? "evaluation",
+    task: parsed?.task ?? null,
+    project: parsed?.project ?? null,
+    status: parsed?.status ?? "created",
+    tags: parsed?.tags ?? [],
+    metadata: parsed?.metadata ?? {},
+    evidence: null,
+    members: [],
+    createdAt: timestamp,
+    updatedAt: timestamp,
+  };
+
+  writeGroups([...readGroups(), group]);
+  return group;
+}
+
+export function updateGroup(
+  groupId: string,
+  input: UpdateInspectorGroupInput,
+): InspectorGroup | null {
+  const parsed = UpdateInspectorGroupInputSchema.parse(input);
+  const groups = readGroups();
+  let updatedGroup: InspectorGroup | null = null;
+  const updatedAt = nowIso();
+
+  const updatedGroups = groups.map((group) => {
+    if (group.id !== groupId) return group;
+    updatedGroup = {
+      ...group,
+      title: parsed.title ?? group.title,
+      kind: parsed.kind ?? group.kind,
+      task: parsed.task === undefined ? group.task : parsed.task,
+      project: parsed.project === undefined ? group.project : parsed.project,
+      status: parsed.status ?? group.status,
+      tags: parsed.tags ?? group.tags,
+      metadata:
+        parsed.metadata === undefined ? group.metadata : { ...group.metadata, ...parsed.metadata },
+      updatedAt,
+    };
+    return updatedGroup;
+  });
+
+  if (updatedGroup === null) return null;
+  writeGroups(updatedGroups);
+  return updatedGroup;
+}
+
+function matchesExistingMember(
+  member: InspectorGroupMember,
+  memberId: string,
+  sessionId: string,
+  runId: string | null,
+): boolean {
+  if (member.id === memberId) return true;
+  if (member.sessionId !== sessionId) return false;
+  return member.runId === runId;
+}
+
+function upsertMember(
+  group: InspectorGroup,
+  input: AddInspectorGroupSessionInput,
+  updatedAt: string,
+): InspectorGroup {
+  const parsed = AddInspectorGroupSessionInputSchema.parse(input);
+  const sessionId = parsed.sessionId.trim();
+  const runId = nullableText(parsed.runId);
+  const memberId = normalizeMemberId(parsed.memberId, sessionId);
+  let matched = false;
+
+  const members = group.members.map((member) => {
+    if (!matchesExistingMember(member, memberId, sessionId, runId)) return member;
+    matched = true;
+    return {
+      ...member,
+      sessionId,
+      runId,
+      label: parsed.label === undefined ? member.label : nullableText(parsed.label),
+      agent: parsed.agent === undefined ? member.agent : nullableText(parsed.agent),
+      provider: parsed.provider === undefined ? member.provider : nullableText(parsed.provider),
+      model: parsed.model === undefined ? member.model : nullableText(parsed.model),
+      status: parsed.status === undefined ? member.status : parsed.status,
+      metadata:
+        parsed.metadata === undefined
+          ? member.metadata
+          : { ...member.metadata, ...parsed.metadata },
+      updatedAt,
+    };
+  });
+
+  if (!matched) {
+    members.push({
+      id: memberId,
+      groupId: group.id,
+      sessionId,
+      runId,
+      label: nullableText(parsed.label),
+      agent: nullableText(parsed.agent),
+      provider: nullableText(parsed.provider),
+      model: nullableText(parsed.model),
+      status: parsed.status ?? null,
+      metadata: parsed.metadata ?? {},
+      addedAt: updatedAt,
+      updatedAt,
+    });
+  }
+
+  return { ...group, members, updatedAt };
+}
+
+export function addGroupSession(
+  groupId: string,
+  input: AddInspectorGroupSessionInput,
+): InspectorGroup | null {
+  const groups = readGroups();
+  let updatedGroup: InspectorGroup | null = null;
+  const updatedAt = nowIso();
+
+  const updatedGroups = groups.map((group) => {
+    if (group.id !== groupId) return group;
+    updatedGroup = upsertMember(group, input, updatedAt);
+    return updatedGroup;
+  });
+
+  if (updatedGroup === null) return null;
+  writeGroups(updatedGroups);
+  return updatedGroup;
+}
+
+export function updateGroupEvidence(
+  groupId: string,
+  evidence: InspectorGroupEvidence,
+): InspectorGroup | null {
+  const groups = readGroups();
+  let updatedGroup: InspectorGroup | null = null;
+  const updatedGroups = groups.map((group) => {
+    if (group.id !== groupId) return group;
+    updatedGroup = {
+      ...group,
+      evidence,
+      updatedAt: evidence.exportedAt,
+    };
+    return updatedGroup;
+  });
+  if (updatedGroup === null) return null;
+  writeGroups(updatedGroups);
+  return updatedGroup;
+}
+
+export function _resetGroupsForTests(): void {
+  writeGroups([]);
+}

package/src/routes/api/groups.$groupId.evidence.ts

@@ -0,0 +1,62 @@
+import { createFileRoute } from "@tanstack/react-router";
+import { GroupEvidenceExportOptionsSchema } from "../../lib/groupContract";
+import { exportGroupEvidence, readGroupEvidenceMarkdown } from "../../proxy/groupEvidenceExporter";
+import { getGroup } from "../../proxy/groupStore";
+
+type JsonBodyResult = { ok: true; value: unknown } | { ok: false };
+
+async function readJsonBody(request: Request): Promise<JsonBodyResult> {
+  try {
+    const raw = await request.text();
+    if (raw.trim().length === 0) return { ok: true, value: undefined };
+    return { ok: true, value: JSON.parse(raw) };
+  } catch {
+    return { ok: false };
+  }
+}
+
+export const Route = createFileRoute("/api/groups/$groupId/evidence")({
+  server: {
+    handlers: {
+      GET: ({ params }: { params: { groupId: string } }) => {
+        const group = getGroup(params.groupId);
+        if (group === null) {
+          return Response.json({ error: "Group not found" }, { status: 404 });
+        }
+
+        const markdown = readGroupEvidenceMarkdown(group);
+        if (group.evidence === null || markdown === null) {
+          return Response.json(
+            { error: "Evidence has not been exported for this group" },
+            { status: 404 },
+          );
+        }
+
+        return Response.json({ group, evidence: group.evidence, markdown });
+      },
+      POST: async ({ params, request }: { params: { groupId: string }; request: Request }) => {
+        const group = getGroup(params.groupId);
+        if (group === null) {
+          return Response.json({ error: "Group not found" }, { status: 404 });
+        }
+
+        const body = await readJsonBody(request);
+        if (!body.ok) {
+          return Response.json({ error: "Invalid JSON body" }, { status: 400 });
+        }
+
+        const parsed = GroupEvidenceExportOptionsSchema.safeParse(body.value);
+        if (!parsed.success) {
+          return Response.json(
+            { error: "Invalid request body", details: parsed.error.format() },
+            { status: 400 },
+          );
+        }
+
+        return Response.json(
+          await exportGroupEvidence(group, parsed.data, new URL(request.url).origin),
+        );
+      },
+    },
+  },
+});