@tonyclaw/agent-inspector 4.0.1 → 4.0.2

package/src/proxy/evidenceAnalysis.ts

@@ -1,526 +0,0 @@
-import type { SessionInfo, SessionLogSummary } from "../lib/sessionInfoContract";
-import type {
-  FailureCategory,
-  FailureClassification,
-  InspectorRun,
-  JenkinsReport,
-  RunTimelineEvent,
-} from "../lib/runContract";
-
-type AnalyzeEvidenceInput = {
-  run: InspectorRun;
-  session: SessionInfo | null;
-  exportedAt: string;
-  evidencePaths: {
-    jsonPath: string;
-    markdownPath: string;
-    htmlPath: string;
-  };
-};
-
-type ErrorCandidate = {
-  log: SessionLogSummary | null;
-  text: string;
-};
-
-function formatNullable(value: string | number | null): string {
-  return value === null ? "n/a" : String(value);
-}
-
-function formatList(values: readonly string[]): string {
-  return values.length > 0 ? values.join(", ") : "n/a";
-}
-
-function lower(value: string): string {
-  return value.toLowerCase();
-}
-
-function logEvidenceText(log: SessionLogSummary): string {
-  return lower(
-    [
-      log.error,
-      log.path,
-      log.model,
-      log.provider,
-      log.status === null ? null : String(log.status),
-      log.isStreaming ? "streaming" : "non-streaming",
-    ]
-      .filter((value) => value !== null)
-      .join(" "),
-  );
-}
-
-function collectErrorCandidates(session: SessionInfo): ErrorCandidate[] {
-  const candidates: ErrorCandidate[] = session.latestLogs
-    .filter((log) => log.hasError)
-    .map((log) => ({ log, text: logEvidenceText(log) }));
-
-  if (session.lastTaskError !== null) {
-    candidates.push({ log: null, text: lower(session.lastTaskError) });
-  }
-
-  return candidates;
-}
-
-function hasText(candidates: readonly ErrorCandidate[], patterns: readonly string[]): boolean {
-  return candidates.some((candidate) =>
-    patterns.some((pattern) => candidate.text.includes(pattern)),
-  );
-}
-
-function hasStatus(session: SessionInfo, predicate: (status: number) => boolean): boolean {
-  return session.latestLogs.some((log) => log.status !== null && predicate(log.status));
-}
-
-function evidenceIds(candidates: readonly ErrorCandidate[]): number[] {
-  const ids = new Set<number>();
-  for (const candidate of candidates) {
-    if (candidate.log !== null) {
-      ids.add(candidate.log.id);
-    }
-  }
-  return [...ids].sort((left, right) => left - right);
-}
-
-function hasStreamingMismatch(session: SessionInfo): boolean {
-  for (const failed of session.latestLogs.filter((log) => log.hasError)) {
-    const matchingSuccess = session.latestLogs.some(
-      (log) =>
-        !log.hasError &&
-        log.model === failed.model &&
-        log.provider === failed.provider &&
-        log.isStreaming !== failed.isStreaming,
-    );
-    if (matchingSuccess) return true;
-  }
-  return false;
-}
-
-/**
- * Per-category summary, hints, severity, and outcome. Keyed by every
- * `FailureCategory` so TypeScript's `Record` enforces exhaustiveness —
- * adding a new category is a single-place compile error instead of four
- * parallel switch statements missing a case.
- */
-const FAILURE_CATEGORY_INFO: Record<
-  FailureCategory,
-  {
-    summary: string;
-    hints: string[];
-    severity: FailureClassification["severity"];
-    outcome: FailureClassification["outcome"];
-  }
-> = {
-  none: {
-    summary: "Session completed without captured errors.",
-    hints: ["Keep the evidence pack as the success proof for CI or release notes."],
-    severity: "none",
-    outcome: "success",
-  },
-  active: {
-    summary: "Session still has active requests or running tasks.",
-    hints: ["Wait for the session to settle, then export evidence again."],
-    severity: "low",
-    outcome: "active",
-  },
-  "provider-timeout": {
-    summary: "Provider call appears to have timed out.",
-    hints: [
-      "Increase the provider test timeout for slow models.",
-      "Compare streaming and non-streaming probes for the same model.",
-    ],
-    severity: "high",
-    outcome: "failure",
-  },
-  "provider-http-error": {
-    summary: "Provider returned an HTTP error.",
-    hints: ["Open the cited log ids and inspect upstream status, response body, and provider URL."],
-    severity: "high",
-    outcome: "failure",
-  },
-  "provider-auth-error": {
-    summary: "Provider authentication or API key configuration appears invalid.",
-    hints: ["Verify the provider API key, auth header mode, and configured base URL."],
-    severity: "high",
-    outcome: "failure",
-  },
-  "provider-rate-limit": {
-    summary: "Provider rejected the request because of rate limit or quota pressure.",
-    hints: ["Retry after provider quota recovers or switch to a less constrained provider/model."],
-    severity: "high",
-    outcome: "failure",
-  },
-  "network-or-container": {
-    summary: "Network connectivity looks suspicious; check container boundaries and localhost use.",
-    hints: [
-      "If the coding agent and Inspector are in different containers, replace localhost with a reachable host.",
-      "Check DNS, proxy, firewall, and container network routes.",
-    ],
-    severity: "high",
-    outcome: "failure",
-  },
-  "streaming-mismatch": {
-    summary: "Streaming and non-streaming behavior differ for the same provider/model.",
-    hints: [
-      "Compare the streaming and non-streaming captured logs side by side.",
-      "Check whether the provider's OpenAI-compatible endpoint handles stream=false differently.",
-    ],
-    severity: "high",
-    outcome: "failure",
-  },
-  "context-pressure": {
-    summary: "The failure looks related to context window or token pressure.",
-    hints: [
-      "Reduce prompt/tool schema/history size or configure the model context window metadata.",
-    ],
-    severity: "medium",
-    outcome: "failure",
-  },
-  "runtime-task-error": {
-    summary: "Inspector session runtime reported a task error.",
-    hints: ["Inspect the session runtime error and retry after clearing stuck session tasks."],
-    severity: "high",
-    outcome: "failure",
-  },
-  "model-output-error": {
-    summary: "The model output or upstream payload shape appears invalid.",
-    hints: ["Inspect the raw response and schema parsing details for the cited log ids."],
-    severity: "medium",
-    outcome: "failure",
-  },
-  "unknown-failure": {
-    summary: "Inspector captured errors, but the root cause is not yet classified.",
-    hints: ["Open the cited logs and inspect error/status/latency fields to refine the diagnosis."],
-    severity: "medium",
-    outcome: "failure",
-  },
-  "no-session-data": {
-    summary: "No Inspector session data was available for this run.",
-    hints: [
-      "Confirm the run's sessionId matches captured traffic and that history scanning is enabled when needed.",
-    ],
-    severity: "medium",
-    outcome: "unknown",
-  },
-};
-
-function classifyCategory(
-  session: SessionInfo,
-  candidates: readonly ErrorCandidate[],
-): FailureCategory {
-  if (session.status === "active") return "active";
-  if (session.errorCount === 0 && session.failedRequests === 0 && session.lastTaskError === null) {
-    return "none";
-  }
-  if (hasStreamingMismatch(session)) return "streaming-mismatch";
-  if (
-    hasText(candidates, [
-      "timeout",
-      "timed out",
-      "etimedout",
-      "deadline",
-      "aborterror",
-      "econnaborted",
-    ])
-  ) {
-    return "provider-timeout";
-  }
-  if (hasText(candidates, ["localhost", "econnrefused", "enotfound", "network", "fetch failed"])) {
-    return "network-or-container";
-  }
-  if (hasStatus(session, (status) => status === 401 || status === 403)) {
-    return "provider-auth-error";
-  }
-  if (hasText(candidates, ["unauthorized", "forbidden", "invalid api key", "api key"])) {
-    return "provider-auth-error";
-  }
-  if (hasStatus(session, (status) => status === 429)) return "provider-rate-limit";
-  if (hasText(candidates, ["rate limit", "quota"])) return "provider-rate-limit";
-  if (hasText(candidates, ["context", "too many tokens", "token limit", "maximum context"])) {
-    return "context-pressure";
-  }
-  if (session.lastTaskError !== null) return "runtime-task-error";
-  if (hasText(candidates, ["invalid json", "schema", "parse", "malformed"])) {
-    return "model-output-error";
-  }
-  if (hasStatus(session, (status) => status >= 400)) return "provider-http-error";
-  return "unknown-failure";
-}
-
-function severityFor(category: FailureCategory): FailureClassification["severity"] {
-  return FAILURE_CATEGORY_INFO[category].severity;
-}
-
-function outcomeFor(category: FailureCategory): FailureClassification["outcome"] {
-  return FAILURE_CATEGORY_INFO[category].outcome;
-}
-
-function categorySummary(category: FailureCategory): string {
-  return FAILURE_CATEGORY_INFO[category].summary;
-}
-
-function categoryHints(category: FailureCategory): string[] {
-  return FAILURE_CATEGORY_INFO[category].hints;
-}
-
-export function classifyRunFailure(
-  run: InspectorRun,
-  session: SessionInfo | null,
-): FailureClassification {
-  if (session === null) {
-    const category: FailureCategory = "no-session-data";
-    return {
-      outcome: outcomeFor(category),
-      severity: severityFor(category),
-      category,
-      summary: `${categorySummary(category)} Run ${run.id} is bound to session ${run.sessionId}.`,
-      evidenceLogIds: [],
-      hints: categoryHints(category),
-    };
-  }
-
-  const candidates = collectErrorCandidates(session);
-  const category = classifyCategory(session, candidates);
-  const summary =
-    category === "none"
-      ? `${categorySummary(category)} ${String(session.requestCount)} request(s), ${String(
-          session.tokenUsage.total,
-        )} token(s).`
-      : `${categorySummary(category)} ${String(session.errorCount)} error log(s), ${String(
-          session.failedRequests,
-        )} failed request(s).`;
-
-  return {
-    outcome: outcomeFor(category),
-    severity: severityFor(category),
-    category,
-    summary,
-    evidenceLogIds: evidenceIds(candidates),
-    hints: categoryHints(category),
-  };
-}
-
-function logTitle(log: SessionLogSummary): string {
-  if (log.hasError) return `Request failed: log ${String(log.id)}`;
-  if (log.status !== null && log.status >= 200 && log.status < 400) {
-    return `Request succeeded: log ${String(log.id)}`;
-  }
-  return `Request captured: log ${String(log.id)}`;
-}
-
-function logDetails(log: SessionLogSummary): string {
-  const parts = [
-    `path=${log.path}`,
-    `status=${formatNullable(log.status)}`,
-    `model=${formatNullable(log.model)}`,
-    `provider=${formatNullable(log.provider)}`,
-    `streaming=${log.isStreaming ? "yes" : "no"}`,
-    `latencyMs=${formatNullable(log.latencyMs)}`,
-  ];
-  if (log.isStreaming) {
-    parts.push(`firstChunkMs=${formatNullable(log.firstChunkMs ?? null)}`);
-    parts.push(`totalStreamMs=${formatNullable(log.totalStreamMs ?? null)}`);
-    parts.push(`tokensPerSecond=${formatNullable(log.tokensPerSecond ?? null)}`);
-  }
-  if (log.error !== null) {
-    parts.push(`error=${log.error}`);
-  }
-  return parts.join(", ");
-}
-
-export function buildRunTimeline(
-  run: InspectorRun,
-  session: SessionInfo | null,
-  exportedAt: string,
-): RunTimelineEvent[] {
-  const events: RunTimelineEvent[] = [
-    {
-      timestamp: run.createdAt,
-      kind: "run-created",
-      severity: "info",
-      title: "Run declared",
-      details: `Run ${run.id} was declared for session ${run.sessionId}.`,
-      logId: null,
-      status: null,
-      model: null,
-      provider: null,
-      latencyMs: null,
-      isStreaming: null,
-    },
-  ];
-
-  if (session !== null && session.createdAt !== null) {
-    events.push({
-      timestamp: session.createdAt,
-      kind: "session-started",
-      severity: "info",
-      title: "Session started",
-      details: `Inspector session ${session.id} started with source ${session.source ?? "unknown"}.`,
-      logId: null,
-      status: null,
-      model: null,
-      provider: null,
-      latencyMs: null,
-      isStreaming: null,
-    });
-  }
-
-  if (session !== null) {
-    const logs = [...session.latestLogs].sort((left, right) =>
-      left.timestamp.localeCompare(right.timestamp),
-    );
-    for (const log of logs) {
-      events.push({
-        timestamp: log.timestamp,
-        kind: "request",
-        severity: log.hasError ? "error" : "success",
-        title: logTitle(log),
-        details: logDetails(log),
-        logId: log.id,
-        status: log.status,
-        model: log.model,
-        provider: log.provider,
-        latencyMs: log.latencyMs,
-        isStreaming: log.isStreaming,
-      });
-    }
-
-    if (session.updatedAt !== null) {
-      events.push({
-        timestamp: session.updatedAt,
-        kind: "session-finished",
-        severity:
-          session.status === "failed"
-            ? "error"
-            : session.status === "active"
-              ? "warning"
-              : "success",
-        title: `Session ${session.status}`,
-        details: `${String(session.requestCount)} request(s), ${String(
-          session.errorCount,
-        )} error(s), ${String(session.tokenUsage.total)} token(s).`,
-        logId: session.lastLogId,
-        status: null,
-        model: session.lastModel,
-        provider: null,
-        latencyMs: null,
-        isStreaming: null,
-      });
-    }
-  }
-
-  events.push({
-    timestamp: exportedAt,
-    kind: "evidence-exported",
-    severity: "info",
-    title: "Evidence exported",
-    details: `Evidence pack exported at ${exportedAt}.`,
-    logId: null,
-    status: null,
-    model: null,
-    provider: null,
-    latencyMs: null,
-    isStreaming: null,
-  });
-
-  return events.sort((left, right) => left.timestamp.localeCompare(right.timestamp));
-}
-
-function reportStatus(classification: FailureClassification): JenkinsReport["status"] {
-  switch (classification.outcome) {
-    case "success":
-      return "PASS";
-    case "failure":
-      return "FAIL";
-    case "active":
-      return "RUNNING";
-    case "unknown":
-      return "UNKNOWN";
-  }
-}
-
-function timelineMarkdown(timeline: readonly RunTimelineEvent[]): string[] {
-  if (timeline.length === 0) return ["No timeline events were available."];
-  return [
-    "| Time | Event | Severity | Evidence |",
-    "| --- | --- | --- | --- |",
-    ...timeline.map((event) => {
-      const evidence = event.logId === null ? "n/a" : `log ${String(event.logId)}`;
-      return `| ${event.timestamp} | ${event.title} | ${event.severity} | ${evidence} |`;
-    }),
-  ];
-}
-
-export function buildJenkinsReport(input: {
-  run: InspectorRun;
-  session: SessionInfo | null;
-  timeline: readonly RunTimelineEvent[];
-  classification: FailureClassification;
-  evidencePaths: AnalyzeEvidenceInput["evidencePaths"];
-}): JenkinsReport {
-  const status = reportStatus(input.classification);
-  const session = input.session;
-  const summary = `[${status}] ${input.classification.summary}`;
-  const lines = [
-    `# Jenkins Report: ${input.run.title}`,
-    "",
-    `- Status: ${status}`,
-    `- Run ID: ${input.run.id}`,
-    `- Session ID: ${input.run.sessionId}`,
-    `- Inspector URL: ${session?.inspectorUrl ?? "n/a"}`,
-    `- Category: ${input.classification.category}`,
-    `- Severity: ${input.classification.severity}`,
-    `- Summary: ${input.classification.summary}`,
-    `- Evidence Logs: ${
-      input.classification.evidenceLogIds.length > 0
-        ? input.classification.evidenceLogIds.map((id) => `#${String(id)}`).join(", ")
-        : "n/a"
-    }`,
-    "",
-    "## Session Metrics",
-    "",
-    `- Requests: ${String(session?.requestCount ?? 0)}`,
-    `- Errors: ${String(session?.errorCount ?? 0)}`,
-    `- Models: ${formatList(session?.models ?? [])}`,
-    `- Providers: ${formatList(session?.providers ?? [])}`,
-    `- Tokens: ${String(session?.tokenUsage.total ?? 0)}`,
-    "",
-    "## Timeline",
-    "",
-    ...timelineMarkdown(input.timeline),
-    "",
-    "## Artifacts",
-    "",
-    `- JSON: ${input.evidencePaths.jsonPath}`,
-    `- Markdown: ${input.evidencePaths.markdownPath}`,
-    `- HTML: ${input.evidencePaths.htmlPath}`,
-    "",
-    "## Next Actions",
-    "",
-    ...input.classification.hints.map((hint) => `- ${hint}`),
-    "",
-  ];
-
-  return {
-    status,
-    summary,
-    markdown: lines.join("\n"),
-  };
-}
-
-export function analyzeEvidence(input: AnalyzeEvidenceInput): {
-  classification: FailureClassification;
-  timeline: RunTimelineEvent[];
-  jenkinsReport: JenkinsReport;
-} {
-  const classification = classifyRunFailure(input.run, input.session);
-  const timeline = buildRunTimeline(input.run, input.session, input.exportedAt);
-  const jenkinsReport = buildJenkinsReport({
-    run: input.run,
-    session: input.session,
-    timeline,
-    classification,
-    evidencePaths: input.evidencePaths,
-  });
-  return { classification, timeline, jenkinsReport };
-}