@tonyclaw/agent-inspector 2.0.26 → 2.0.27

package/src/proxy/evidenceExporter.ts

@@ -0,0 +1,215 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import {
+  type EvidenceExportOptions,
+  type EvidenceExportResult,
+  type InspectorRun,
+  type InspectorRunEvidence,
+} from "../lib/runContract";
+import { getDataDir } from "./dataDir";
+import { analyzeEvidence } from "./evidenceAnalysis";
+import { getSessionInfo } from "./store";
+import { updateRunEvidence } from "./runStore";
+
+type EvidenceDocument = EvidenceExportResult;
+
+function evidenceDir(runId: string): string {
+  return join(getDataDir(), "evidence", runId);
+}
+
+function jsonPath(runId: string): string {
+  return join(evidenceDir(runId), "evidence.json");
+}
+
+function markdownPath(runId: string): string {
+  return join(evidenceDir(runId), "evidence.md");
+}
+
+function htmlPath(runId: string): string {
+  return join(evidenceDir(runId), "evidence.html");
+}
+
+function formatList(values: readonly string[]): string {
+  return values.length > 0 ? values.join(", ") : "n/a";
+}
+
+function escapeHtml(value: string): string {
+  return value
+    .replace(/&/g, "&")
+    .replace(/</g, "&lt;")
+    .replace(/>/g, "&gt;")
+    .replace(/"/g, "&quot;");
+}
+
+function buildMarkdown(document: EvidenceDocument): string {
+  const session = document.session;
+  const lines = [
+    `# ${document.run.title}`,
+    "",
+    `- Run ID: ${document.run.id}`,
+    `- Session ID: ${document.run.sessionId}`,
+    `- Status: ${document.run.status}`,
+    `- Project: ${document.run.project ?? "n/a"}`,
+    `- Agent: ${document.run.agent ?? "n/a"}`,
+    `- Created: ${document.run.createdAt}`,
+    "",
+    `- Outcome: ${document.classification.outcome}`,
+    `- Failure Category: ${document.classification.category}`,
+    `- Severity: ${document.classification.severity}`,
+    `- Evidence JSON: ${document.evidence.jsonPath}`,
+    `- Evidence HTML: ${document.evidence.htmlPath}`,
+    "",
+  ];
+
+  if (document.run.task !== null) {
+    lines.push("## Task", "", document.run.task, "");
+  }
+
+  if (session === null) {
+    lines.push("## Session", "", "No Inspector session data was available for this run.", "");
+  } else {
+    lines.push(
+      "## Session",
+      "",
+      `- Inspector URL: ${session.inspectorUrl}`,
+      `- Compact Logs URL: ${session.compactLogsUrl}`,
+      `- Status: ${session.status}`,
+      `- Coverage: ${session.logCoverage}`,
+      `- Requests: ${session.requestCount}`,
+      `- Errors: ${session.errorCount}`,
+      `- Models: ${formatList(session.models)}`,
+      `- Providers: ${formatList(session.providers)}`,
+      `- Tokens: ${session.tokenUsage.total}`,
+      "",
+      "## Failure Classification",
+      "",
+      document.classification.summary,
+      "",
+      "### Evidence Logs",
+      "",
+      document.classification.evidenceLogIds.length > 0
+        ? document.classification.evidenceLogIds.map((id) => `- #${id}`).join("\n")
+        : "No failing log ids were identified.",
+      "",
+      "### Suggested Next Actions",
+      "",
+      ...document.classification.hints.map((hint) => `- ${hint}`),
+      "",
+      "## Run Timeline",
+      "",
+      "| Time | Event | Severity | Evidence |",
+      "| --- | --- | --- | --- |",
+      ...document.timeline.map((event) => {
+        const evidence = event.logId === null ? "n/a" : `log ${event.logId}`;
+        return `| ${event.timestamp} | ${event.title} | ${event.severity} | ${evidence} |`;
+      }),
+      "",
+      "## Latest Logs",
+      "",
+    );
+
+    if (session.latestLogs.length === 0) {
+      lines.push("No compact logs were available.", "");
+    } else {
+      lines.push(
+        "| ID | Status | Model | Latency | Path | Error |",
+        "| --- | --- | --- | --- | --- | --- |",
+      );
+      for (const log of session.latestLogs) {
+        lines.push(
+          `| ${log.id} | ${log.status ?? "n/a"} | ${log.model ?? "n/a"} | ${
+            log.latencyMs ?? "n/a"
+          } | ${log.path} | ${log.hasError ? "yes" : "no"} |`,
+        );
+      }
+      lines.push("");
+    }
+  }
+
+  lines.push("## Jenkins Report", "", document.jenkinsReport.markdown, "");
+
+  return `${lines.join("\n")}\n`;
+}
+
+function buildHtml(markdown: string, title: string): string {
+  return `<!doctype html>
+<html lang="en">
+<head>
+  <meta charset="utf-8">
+  <title>${escapeHtml(title)}</title>
+  <style>
+    body { font-family: system-ui, sans-serif; max-width: 960px; margin: 40px auto; line-height: 1.5; }
+    pre { white-space: pre-wrap; background: #f6f8fa; padding: 16px; border-radius: 8px; }
+  </style>
+</head>
+<body>
+  <pre>${escapeHtml(markdown)}</pre>
+</body>
+</html>
+`;
+}
+
+export async function exportRunEvidence(
+  run: InspectorRun,
+  options: EvidenceExportOptions,
+  baseUrl: string,
+): Promise<EvidenceExportResult> {
+  const session = await getSessionInfo(run.sessionId, {
+    baseUrl,
+    includeHistory: options?.includeHistory,
+    latestLogLimit: options?.latestLogLimit,
+  });
+  const exportedAt = new Date().toISOString();
+  const evidence: InspectorRunEvidence = {
+    jsonPath: jsonPath(run.id),
+    markdownPath: markdownPath(run.id),
+    htmlPath: htmlPath(run.id),
+    exportedAt,
+  };
+  const documentRun: InspectorRun = {
+    ...run,
+    evidence,
+    updatedAt: exportedAt,
+  };
+  const analysis = analyzeEvidence({
+    run: documentRun,
+    session,
+    exportedAt,
+    evidencePaths: {
+      jsonPath: evidence.jsonPath,
+      markdownPath: evidence.markdownPath,
+      htmlPath: evidence.htmlPath,
+    },
+  });
+  const document: EvidenceDocument = {
+    run: documentRun,
+    evidence,
+    session,
+    timeline: analysis.timeline,
+    classification: analysis.classification,
+    jenkinsReport: analysis.jenkinsReport,
+  };
+  const markdown = buildMarkdown(document);
+
+  mkdirSync(evidenceDir(run.id), { recursive: true });
+  writeFileSync(
+    evidence.jsonPath,
+    `${JSON.stringify({ ...document, evidence }, null, 2)}\n`,
+    "utf8",
+  );
+  writeFileSync(evidence.markdownPath, markdown, "utf8");
+  writeFileSync(evidence.htmlPath, buildHtml(markdown, run.title), "utf8");
+
+  const updatedRun = updateRunEvidence(run.id, evidence) ?? documentRun;
+  return { ...document, run: updatedRun };
+}
+
+export function readEvidenceMarkdown(run: InspectorRun): string | null {
+  const path = run.evidence?.markdownPath;
+  if (path === undefined || !existsSync(path)) return null;
+  try {
+    return readFileSync(path, "utf8");
+  } catch {
+    return null;
+  }
+}

package/src/proxy/logSearch.ts

@@ -0,0 +1,118 @@
+import type { CapturedLog } from "./schemas";
+import { listLogsPage } from "./store";
+
+export const LOG_SEARCH_DEFAULT_LIMIT = 10;
+export const LOG_SEARCH_MAX_LIMIT = 50;
+export const LOG_SEARCH_DEFAULT_SCAN_LIMIT = 200;
+export const LOG_SEARCH_MAX_SCAN_LIMIT = 1_000;
+
+export type LogSearchOptions = {
+  query: string;
+  sessionId?: string;
+  model?: string;
+  offset: number;
+  limit: number;
+  scanLimit: number;
+};
+
+export type LogSearchResult = {
+  query: string;
+  logs: CapturedLog[];
+  total: number;
+  offset: number;
+  limit: number;
+  scanned: number;
+};
+
+function clampPositiveInt(value: number, fallback: number, max: number): number {
+  if (!Number.isInteger(value) || value < 1) return fallback;
+  return Math.min(value, max);
+}
+
+export function clampLogSearchLimit(value: number): number {
+  return clampPositiveInt(value, LOG_SEARCH_DEFAULT_LIMIT, LOG_SEARCH_MAX_LIMIT);
+}
+
+export function clampLogSearchScanLimit(value: number): number {
+  return clampPositiveInt(value, LOG_SEARCH_DEFAULT_SCAN_LIMIT, LOG_SEARCH_MAX_SCAN_LIMIT);
+}
+
+function queryTerms(query: string): string[] {
+  return query
+    .trim()
+    .toLowerCase()
+    .split(/\s+/)
+    .filter((term) => term.length > 0);
+}
+
+function optionalText(value: string | number | boolean | null | undefined): string | null {
+  if (value === null || value === undefined) return null;
+  return String(value);
+}
+
+function searchableFields(log: CapturedLog): string[] {
+  return [
+    String(log.id),
+    log.timestamp,
+    log.method,
+    log.path,
+    optionalText(log.model),
+    optionalText(log.sessionId),
+    optionalText(log.responseStatus),
+    optionalText(log.elapsedMs),
+    optionalText(log.streaming),
+    log.streaming ? "streaming" : "non-streaming",
+    log.apiFormat,
+    optionalText(log.providerName),
+    optionalText(log.userAgent),
+    optionalText(log.origin),
+    optionalText(log.clientPid),
+    optionalText(log.clientCwd),
+    optionalText(log.clientProjectFolder),
+    optionalText(log.error),
+  ].filter((value): value is string => value !== null && value.length > 0);
+}
+
+function matchesTerms(log: CapturedLog, terms: readonly string[]): boolean {
+  if (terms.length === 0) return true;
+  const haystack = searchableFields(log).join("\n").toLowerCase();
+  return terms.every((term) => haystack.includes(term));
+}
+
+function sortNewestFirst(logs: CapturedLog[]): CapturedLog[] {
+  return logs.sort((left, right) => {
+    const timestampCompare = right.timestamp.localeCompare(left.timestamp);
+    return timestampCompare !== 0 ? timestampCompare : right.id - left.id;
+  });
+}
+
+export async function searchLogsPage(options: LogSearchOptions): Promise<LogSearchResult> {
+  const limit = clampLogSearchLimit(options.limit);
+  const scanLimit = clampLogSearchScanLimit(Math.max(options.scanLimit, options.offset + limit));
+  const firstPage = await listLogsPage({
+    sessionId: options.sessionId,
+    model: options.model,
+    offset: 0,
+    limit: 1,
+    includeBodies: false,
+  });
+  const scanOffset = Math.max(firstPage.total - scanLimit, 0);
+  const scanPage = await listLogsPage({
+    sessionId: options.sessionId,
+    model: options.model,
+    offset: scanOffset,
+    limit: scanLimit,
+    includeBodies: false,
+  });
+  const terms = queryTerms(options.query);
+  const matches = sortNewestFirst(scanPage.logs.filter((log) => matchesTerms(log, terms)));
+
+  return {
+    query: options.query,
+    logs: matches.slice(options.offset, options.offset + limit),
+    total: matches.length,
+    offset: options.offset,
+    limit,
+    scanned: scanPage.logs.length,
+  };
+}

package/src/proxy/runFailures.ts

@@ -0,0 +1,100 @@
+import { existsSync, readFileSync } from "node:fs";
+import {
+  EvidenceExportResultSchema,
+  type EvidenceExportResult,
+  type FailureClassification,
+  type InspectorRun,
+  type InspectorRunEvidence,
+  type JenkinsReport,
+  type RecentFailure,
+  type RecentFailuresResponse,
+} from "../lib/runContract";
+import { listRuns } from "./runStore";
+
+export const RECENT_FAILURES_DEFAULT_LIMIT = 5;
+export const RECENT_FAILURES_MAX_LIMIT = 20;
+
+export function clampRecentFailuresLimit(value: number): number {
+  if (!Number.isInteger(value) || value < 1) return RECENT_FAILURES_DEFAULT_LIMIT;
+  return Math.min(value, RECENT_FAILURES_MAX_LIMIT);
+}
+
+function readEvidenceDocument(run: InspectorRun): EvidenceExportResult | null {
+  const path = run.evidence?.jsonPath;
+  if (path === undefined || !existsSync(path)) return null;
+
+  try {
+    const parsed: unknown = JSON.parse(readFileSync(path, "utf8"));
+    const result = EvidenceExportResultSchema.safeParse(parsed);
+    return result.success ? result.data : null;
+  } catch {
+    return null;
+  }
+}
+
+function syntheticFailedClassification(run: InspectorRun): FailureClassification {
+  return {
+    outcome: "failure",
+    severity: "high",
+    category: "unknown-failure",
+    summary: `Run ${run.id} is marked ${run.status}, but no exported evidence document was available.`,
+    evidenceLogIds: [],
+    hints: [
+      "Call inspector_export_evidence for this run to collect session metrics, failing log ids, and artifact paths.",
+      "Confirm the run's sessionId matches the coding agent traffic captured by Inspector.",
+    ],
+  };
+}
+
+function shouldIncludeRun(
+  run: InspectorRun,
+  document: EvidenceExportResult | null,
+  classification: FailureClassification,
+): boolean {
+  if (run.status === "failed" || run.status === "cancelled") return true;
+  if (document === null) return false;
+  return classification.outcome !== "success";
+}
+
+function evidenceFor(
+  run: InspectorRun,
+  document: EvidenceExportResult | null,
+): InspectorRunEvidence | null {
+  return document?.evidence ?? run.evidence;
+}
+
+function jenkinsReportFor(document: EvidenceExportResult | null): JenkinsReport | null {
+  return document?.jenkinsReport ?? null;
+}
+
+function failureFromRun(run: InspectorRun): RecentFailure | null {
+  const document = readEvidenceDocument(run);
+  const classification =
+    document === null ? syntheticFailedClassification(run) : document.classification;
+  if (!shouldIncludeRun(run, document, classification)) return null;
+
+  const evidence = evidenceFor(run, document);
+  return {
+    run,
+    classification,
+    evidence,
+    jenkinsReport: jenkinsReportFor(document),
+    sessionId: run.sessionId,
+    inspectorUrl: document?.session?.inspectorUrl ?? null,
+    evidenceMarkdownPath: evidence?.markdownPath ?? null,
+    updatedAt: evidence?.exportedAt ?? run.updatedAt,
+  };
+}
+
+export function listRecentFailures(limit: number): RecentFailuresResponse {
+  const failures = listRuns()
+    .map(failureFromRun)
+    .filter((failure): failure is RecentFailure => failure !== null)
+    .sort((left, right) => right.updatedAt.localeCompare(left.updatedAt));
+  const clampedLimit = clampRecentFailuresLimit(limit);
+  return {
+    failures: failures.slice(0, clampedLimit),
+    total: failures.length,
+    limit: clampedLimit,
+  };
+}

package/src/proxy/runStore.ts

@@ -0,0 +1,159 @@
+import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
+import { randomUUID } from "node:crypto";
+import { dirname, join } from "node:path";
+import { z } from "zod";
+import {
+  CreateInspectorRunInputSchema,
+  InspectorRunSchema,
+  UpdateInspectorRunInputSchema,
+  type CreateInspectorRunInput,
+  type InspectorRun,
+  type InspectorRunEvidence,
+  type UpdateInspectorRunInput,
+} from "../lib/runContract";
+import { getDataDir } from "./dataDir";
+
+const RunsFileSchema = z.object({
+  runs: z.array(InspectorRunSchema),
+});
+
+function runsFilePath(): string {
+  return join(getDataDir(), "runs.json");
+}
+
+function nowIso(): string {
+  return new Date().toISOString();
+}
+
+function stableId(input: string): string {
+  return input
+    .trim()
+    .replace(/[^A-Za-z0-9_.-]+/g, "-")
+    .replace(/^-+|-+$/g, "")
+    .slice(0, 80);
+}
+
+function createRunId(): string {
+  return `run_${Date.now().toString(36)}_${randomUUID().slice(0, 8)}`;
+}
+
+function normalizeRunId(input: string | undefined): string {
+  if (input === undefined) return createRunId();
+  const normalized = stableId(input);
+  return normalized.length > 0 ? normalized : createRunId();
+}
+
+function normalizeSessionId(input: string | undefined, runId: string): string {
+  if (input === undefined || input.trim().length === 0) return runId;
+  return input.trim();
+}
+
+function readRuns(): InspectorRun[] {
+  const filePath = runsFilePath();
+  if (!existsSync(filePath)) return [];
+
+  try {
+    const parsed: unknown = JSON.parse(readFileSync(filePath, "utf8"));
+    const result = RunsFileSchema.safeParse(parsed);
+    return result.success ? result.data.runs : [];
+  } catch {
+    return [];
+  }
+}
+
+function writeRuns(runs: readonly InspectorRun[]): void {
+  const filePath = runsFilePath();
+  mkdirSync(dirname(filePath), { recursive: true });
+  writeFileSync(filePath, `${JSON.stringify({ runs }, null, 2)}\n`, "utf8");
+}
+
+export function listRuns(): InspectorRun[] {
+  return readRuns().sort((left, right) => right.updatedAt.localeCompare(left.updatedAt));
+}
+
+export function getRun(runId: string): InspectorRun | null {
+  return readRuns().find((run) => run.id === runId) ?? null;
+}
+
+export function createRun(input: CreateInspectorRunInput): InspectorRun {
+  const parsed = CreateInspectorRunInputSchema.parse(input);
+  const runId = normalizeRunId(parsed?.runId);
+  const existing = getRun(runId);
+  if (existing !== null) return existing;
+
+  const timestamp = nowIso();
+  const sessionId = normalizeSessionId(parsed?.sessionId, runId);
+  const run: InspectorRun = {
+    id: runId,
+    sessionId,
+    title: parsed?.title ?? `Inspector Run ${runId}`,
+    task: parsed?.task ?? null,
+    project: parsed?.project ?? null,
+    agent: parsed?.agent ?? null,
+    status: parsed?.status ?? "created",
+    tags: parsed?.tags ?? [],
+    metadata: parsed?.metadata ?? {},
+    evidence: null,
+    createdAt: timestamp,
+    updatedAt: timestamp,
+  };
+
+  writeRuns([...readRuns(), run]);
+  return run;
+}
+
+export function updateRunEvidence(
+  runId: string,
+  evidence: InspectorRunEvidence,
+): InspectorRun | null {
+  const runs = readRuns();
+  let updatedRun: InspectorRun | null = null;
+  const updatedRuns = runs.map((run) => {
+    if (run.id !== runId) return run;
+    updatedRun = {
+      ...run,
+      evidence,
+      updatedAt: evidence.exportedAt,
+    };
+    return updatedRun;
+  });
+  if (updatedRun === null) return null;
+  writeRuns(updatedRuns);
+  return updatedRun;
+}
+
+export function updateRun(runId: string, input: UpdateInspectorRunInput): InspectorRun | null {
+  const parsed = UpdateInspectorRunInputSchema.parse(input);
+  const runs = readRuns();
+  let updatedRun: InspectorRun | null = null;
+  const updatedAt = nowIso();
+
+  const updatedRuns = runs.map((run) => {
+    if (run.id !== runId) return run;
+    updatedRun = {
+      ...run,
+      sessionId:
+        parsed.sessionId === undefined
+          ? run.sessionId
+          : normalizeSessionId(parsed.sessionId, run.id),
+      title: parsed.title ?? run.title,
+      task: parsed.task === undefined ? run.task : parsed.task,
+      project: parsed.project === undefined ? run.project : parsed.project,
+      agent: parsed.agent === undefined ? run.agent : parsed.agent,
+      status: parsed.status ?? run.status,
+      tags: parsed.tags ?? run.tags,
+      metadata:
+        parsed.metadata === undefined ? run.metadata : { ...run.metadata, ...parsed.metadata },
+      updatedAt,
+    };
+    return updatedRun;
+  });
+
+  if (updatedRun === null) return null;
+  writeRuns(updatedRuns);
+  return updatedRun;
+}
+
+export function _resetRunsForTests(): void {
+  writeRuns([]);
+}

package/src/routes/api/logs.ts

@@ -7,6 +7,11 @@ import {
   getReplayLogsForSource,
   listLogsPage,
 } from "../../proxy/store";
+import {
+  LOG_SEARCH_DEFAULT_LIMIT,
+  LOG_SEARCH_DEFAULT_SCAN_LIMIT,
+  searchLogsPage,
+} from "../../proxy/logSearch";
 
 const DeleteBodySchema = z
   .object({
@@ -47,7 +52,18 @@ export const Route = createFileRoute("/api/logs")({
         }
         const sessionId = url.searchParams.get("sessionId") ?? undefined;
         const model = url.searchParams.get("model") ?? undefined;
+        const query = url.searchParams.get("q") ?? url.searchParams.get("search");
         const offset = parseNonNegativeInt(url.searchParams.get("offset"), 0);
+        if (query !== null) {
+          const limit = parsePositiveInt(url.searchParams.get("limit"), LOG_SEARCH_DEFAULT_LIMIT);
+          const scanLimit = parsePositiveInt(
+            url.searchParams.get("scanLimit"),
+            LOG_SEARCH_DEFAULT_SCAN_LIMIT,
+          );
+          return Response.json(
+            await searchLogsPage({ query, sessionId, model, offset, limit, scanLimit }),
+          );
+        }
         const limit = parsePositiveInt(url.searchParams.get("limit"), 50);
         const includeBodies = url.searchParams.get("compact") !== "1";
 

package/src/routes/api/runs.$runId.evidence.ts

@@ -0,0 +1,62 @@
+import { createFileRoute } from "@tanstack/react-router";
+import { EvidenceExportOptionsSchema } from "../../lib/runContract";
+import { exportRunEvidence, readEvidenceMarkdown } from "../../proxy/evidenceExporter";
+import { getRun } from "../../proxy/runStore";
+
+type JsonBodyResult = { ok: true; value: unknown } | { ok: false };
+
+async function readJsonBody(request: Request): Promise<JsonBodyResult> {
+  try {
+    const raw = await request.text();
+    if (raw.trim().length === 0) return { ok: true, value: undefined };
+    return { ok: true, value: JSON.parse(raw) };
+  } catch {
+    return { ok: false };
+  }
+}
+
+export const Route = createFileRoute("/api/runs/$runId/evidence")({
+  server: {
+    handlers: {
+      GET: ({ params }: { params: { runId: string } }) => {
+        const run = getRun(params.runId);
+        if (run === null) {
+          return Response.json({ error: "Run not found" }, { status: 404 });
+        }
+
+        const markdown = readEvidenceMarkdown(run);
+        if (run.evidence === null || markdown === null) {
+          return Response.json(
+            { error: "Evidence has not been exported for this run" },
+            { status: 404 },
+          );
+        }
+
+        return Response.json({ run, evidence: run.evidence, markdown });
+      },
+      POST: async ({ params, request }: { params: { runId: string }; request: Request }) => {
+        const run = getRun(params.runId);
+        if (run === null) {
+          return Response.json({ error: "Run not found" }, { status: 404 });
+        }
+
+        const body = await readJsonBody(request);
+        if (!body.ok) {
+          return Response.json({ error: "Invalid JSON body" }, { status: 400 });
+        }
+
+        const parsed = EvidenceExportOptionsSchema.safeParse(body.value);
+        if (!parsed.success) {
+          return Response.json(
+            { error: "Invalid request body", details: parsed.error.format() },
+            { status: 400 },
+          );
+        }
+
+        return Response.json(
+          await exportRunEvidence(run, parsed.data, new URL(request.url).origin),
+        );
+      },
+    },
+  },
+});