@tonyclaw/agent-inspector 2.0.25 → 2.0.27

package/src/proxy/evidenceAnalysis.ts

@@ -0,0 +1,522 @@
+import type { SessionInfo, SessionLogSummary } from "../lib/sessionInfoContract";
+import type {
+  FailureCategory,
+  FailureClassification,
+  InspectorRun,
+  JenkinsReport,
+  RunTimelineEvent,
+} from "../lib/runContract";
+
+type AnalyzeEvidenceInput = {
+  run: InspectorRun;
+  session: SessionInfo | null;
+  exportedAt: string;
+  evidencePaths: {
+    jsonPath: string;
+    markdownPath: string;
+    htmlPath: string;
+  };
+};
+
+type ErrorCandidate = {
+  log: SessionLogSummary | null;
+  text: string;
+};
+
+function formatNullable(value: string | number | null): string {
+  return value === null ? "n/a" : String(value);
+}
+
+function formatList(values: readonly string[]): string {
+  return values.length > 0 ? values.join(", ") : "n/a";
+}
+
+function lower(value: string): string {
+  return value.toLowerCase();
+}
+
+function logEvidenceText(log: SessionLogSummary): string {
+  return lower(
+    [
+      log.error,
+      log.path,
+      log.model,
+      log.provider,
+      log.status === null ? null : String(log.status),
+      log.isStreaming ? "streaming" : "non-streaming",
+    ]
+      .filter((value) => value !== null)
+      .join(" "),
+  );
+}
+
+function collectErrorCandidates(session: SessionInfo): ErrorCandidate[] {
+  const candidates: ErrorCandidate[] = session.latestLogs
+    .filter((log) => log.hasError)
+    .map((log) => ({ log, text: logEvidenceText(log) }));
+
+  if (session.lastTaskError !== null) {
+    candidates.push({ log: null, text: lower(session.lastTaskError) });
+  }
+
+  return candidates;
+}
+
+function hasText(candidates: readonly ErrorCandidate[], patterns: readonly string[]): boolean {
+  return candidates.some((candidate) =>
+    patterns.some((pattern) => candidate.text.includes(pattern)),
+  );
+}
+
+function hasStatus(session: SessionInfo, predicate: (status: number) => boolean): boolean {
+  return session.latestLogs.some((log) => log.status !== null && predicate(log.status));
+}
+
+function evidenceIds(candidates: readonly ErrorCandidate[]): number[] {
+  const ids = new Set<number>();
+  for (const candidate of candidates) {
+    if (candidate.log !== null) {
+      ids.add(candidate.log.id);
+    }
+  }
+  return [...ids].sort((left, right) => left - right);
+}
+
+function hasStreamingMismatch(session: SessionInfo): boolean {
+  for (const failed of session.latestLogs.filter((log) => log.hasError)) {
+    const matchingSuccess = session.latestLogs.some(
+      (log) =>
+        !log.hasError &&
+        log.model === failed.model &&
+        log.provider === failed.provider &&
+        log.isStreaming !== failed.isStreaming,
+    );
+    if (matchingSuccess) return true;
+  }
+  return false;
+}
+
+function categorySummary(category: FailureCategory): string {
+  switch (category) {
+    case "none":
+      return "Session completed without captured errors.";
+    case "active":
+      return "Session still has active requests or running tasks.";
+    case "provider-timeout":
+      return "Provider call appears to have timed out.";
+    case "provider-http-error":
+      return "Provider returned an HTTP error.";
+    case "provider-auth-error":
+      return "Provider authentication or API key configuration appears invalid.";
+    case "provider-rate-limit":
+      return "Provider rejected the request because of rate limit or quota pressure.";
+    case "network-or-container":
+      return "Network connectivity looks suspicious; check container boundaries and localhost use.";
+    case "streaming-mismatch":
+      return "Streaming and non-streaming behavior differ for the same provider/model.";
+    case "context-pressure":
+      return "The failure looks related to context window or token pressure.";
+    case "runtime-task-error":
+      return "Inspector session runtime reported a task error.";
+    case "model-output-error":
+      return "The model output or upstream payload shape appears invalid.";
+    case "unknown-failure":
+      return "Inspector captured errors, but the root cause is not yet classified.";
+    case "no-session-data":
+      return "No Inspector session data was available for this run.";
+  }
+}
+
+function categoryHints(category: FailureCategory): string[] {
+  switch (category) {
+    case "none":
+      return ["Keep the evidence pack as the success proof for CI or release notes."];
+    case "active":
+      return ["Wait for the session to settle, then export evidence again."];
+    case "provider-timeout":
+      return [
+        "Increase the provider test timeout for slow models.",
+        "Compare streaming and non-streaming probes for the same model.",
+      ];
+    case "provider-http-error":
+      return [
+        "Open the cited log ids and inspect upstream status, response body, and provider URL.",
+      ];
+    case "provider-auth-error":
+      return ["Verify the provider API key, auth header mode, and configured base URL."];
+    case "provider-rate-limit":
+      return [
+        "Retry after provider quota recovers or switch to a less constrained provider/model.",
+      ];
+    case "network-or-container":
+      return [
+        "If the coding agent and Inspector are in different containers, replace localhost with a reachable host.",
+        "Check DNS, proxy, firewall, and container network routes.",
+      ];
+    case "streaming-mismatch":
+      return [
+        "Compare the streaming and non-streaming captured logs side by side.",
+        "Check whether the provider's OpenAI-compatible endpoint handles stream=false differently.",
+      ];
+    case "context-pressure":
+      return [
+        "Reduce prompt/tool schema/history size or configure the model context window metadata.",
+      ];
+    case "runtime-task-error":
+      return ["Inspect the session runtime error and retry after clearing stuck session tasks."];
+    case "model-output-error":
+      return ["Inspect the raw response and schema parsing details for the cited log ids."];
+    case "unknown-failure":
+      return [
+        "Open the cited logs and inspect error/status/latency fields to refine the diagnosis.",
+      ];
+    case "no-session-data":
+      return [
+        "Confirm the run's sessionId matches captured traffic and that history scanning is enabled when needed.",
+      ];
+  }
+}
+
+function classifyCategory(
+  session: SessionInfo,
+  candidates: readonly ErrorCandidate[],
+): FailureCategory {
+  if (session.status === "active") return "active";
+  if (session.errorCount === 0 && session.failedRequests === 0 && session.lastTaskError === null) {
+    return "none";
+  }
+  if (hasStreamingMismatch(session)) return "streaming-mismatch";
+  if (
+    hasText(candidates, [
+      "timeout",
+      "timed out",
+      "etimedout",
+      "deadline",
+      "aborterror",
+      "econnaborted",
+    ])
+  ) {
+    return "provider-timeout";
+  }
+  if (hasText(candidates, ["localhost", "econnrefused", "enotfound", "network", "fetch failed"])) {
+    return "network-or-container";
+  }
+  if (hasStatus(session, (status) => status === 401 || status === 403)) {
+    return "provider-auth-error";
+  }
+  if (hasText(candidates, ["unauthorized", "forbidden", "invalid api key", "api key"])) {
+    return "provider-auth-error";
+  }
+  if (hasStatus(session, (status) => status === 429)) return "provider-rate-limit";
+  if (hasText(candidates, ["rate limit", "quota"])) return "provider-rate-limit";
+  if (hasText(candidates, ["context", "too many tokens", "token limit", "maximum context"])) {
+    return "context-pressure";
+  }
+  if (session.lastTaskError !== null) return "runtime-task-error";
+  if (hasText(candidates, ["invalid json", "schema", "parse", "malformed"])) {
+    return "model-output-error";
+  }
+  if (hasStatus(session, (status) => status >= 400)) return "provider-http-error";
+  return "unknown-failure";
+}
+
+function severityFor(category: FailureCategory): FailureClassification["severity"] {
+  switch (category) {
+    case "none":
+      return "none";
+    case "active":
+      return "low";
+    case "provider-timeout":
+    case "provider-http-error":
+    case "provider-auth-error":
+    case "provider-rate-limit":
+    case "network-or-container":
+    case "streaming-mismatch":
+    case "runtime-task-error":
+      return "high";
+    case "context-pressure":
+    case "model-output-error":
+    case "unknown-failure":
+    case "no-session-data":
+      return "medium";
+  }
+}
+
+function outcomeFor(category: FailureCategory): FailureClassification["outcome"] {
+  switch (category) {
+    case "none":
+      return "success";
+    case "active":
+      return "active";
+    case "no-session-data":
+      return "unknown";
+    case "provider-timeout":
+    case "provider-http-error":
+    case "provider-auth-error":
+    case "provider-rate-limit":
+    case "network-or-container":
+    case "streaming-mismatch":
+    case "context-pressure":
+    case "runtime-task-error":
+    case "model-output-error":
+    case "unknown-failure":
+      return "failure";
+  }
+}
+
+export function classifyRunFailure(
+  run: InspectorRun,
+  session: SessionInfo | null,
+): FailureClassification {
+  if (session === null) {
+    const category: FailureCategory = "no-session-data";
+    return {
+      outcome: outcomeFor(category),
+      severity: severityFor(category),
+      category,
+      summary: `${categorySummary(category)} Run ${run.id} is bound to session ${run.sessionId}.`,
+      evidenceLogIds: [],
+      hints: categoryHints(category),
+    };
+  }
+
+  const candidates = collectErrorCandidates(session);
+  const category = classifyCategory(session, candidates);
+  const summary =
+    category === "none"
+      ? `${categorySummary(category)} ${String(session.requestCount)} request(s), ${String(
+          session.tokenUsage.total,
+        )} token(s).`
+      : `${categorySummary(category)} ${String(session.errorCount)} error log(s), ${String(
+          session.failedRequests,
+        )} failed request(s).`;
+
+  return {
+    outcome: outcomeFor(category),
+    severity: severityFor(category),
+    category,
+    summary,
+    evidenceLogIds: evidenceIds(candidates),
+    hints: categoryHints(category),
+  };
+}
+
+function logTitle(log: SessionLogSummary): string {
+  if (log.hasError) return `Request failed: log ${String(log.id)}`;
+  if (log.status !== null && log.status >= 200 && log.status < 400) {
+    return `Request succeeded: log ${String(log.id)}`;
+  }
+  return `Request captured: log ${String(log.id)}`;
+}
+
+function logDetails(log: SessionLogSummary): string {
+  const parts = [
+    `path=${log.path}`,
+    `status=${formatNullable(log.status)}`,
+    `model=${formatNullable(log.model)}`,
+    `provider=${formatNullable(log.provider)}`,
+    `streaming=${log.isStreaming ? "yes" : "no"}`,
+    `latencyMs=${formatNullable(log.latencyMs)}`,
+  ];
+  if (log.error !== null) {
+    parts.push(`error=${log.error}`);
+  }
+  return parts.join(", ");
+}
+
+export function buildRunTimeline(
+  run: InspectorRun,
+  session: SessionInfo | null,
+  exportedAt: string,
+): RunTimelineEvent[] {
+  const events: RunTimelineEvent[] = [
+    {
+      timestamp: run.createdAt,
+      kind: "run-created",
+      severity: "info",
+      title: "Run declared",
+      details: `Run ${run.id} was declared for session ${run.sessionId}.`,
+      logId: null,
+      status: null,
+      model: null,
+      provider: null,
+      latencyMs: null,
+      isStreaming: null,
+    },
+  ];
+
+  if (session !== null && session.createdAt !== null) {
+    events.push({
+      timestamp: session.createdAt,
+      kind: "session-started",
+      severity: "info",
+      title: "Session started",
+      details: `Inspector session ${session.id} started with source ${session.source ?? "unknown"}.`,
+      logId: null,
+      status: null,
+      model: null,
+      provider: null,
+      latencyMs: null,
+      isStreaming: null,
+    });
+  }
+
+  if (session !== null) {
+    const logs = [...session.latestLogs].sort((left, right) =>
+      left.timestamp.localeCompare(right.timestamp),
+    );
+    for (const log of logs) {
+      events.push({
+        timestamp: log.timestamp,
+        kind: "request",
+        severity: log.hasError ? "error" : "success",
+        title: logTitle(log),
+        details: logDetails(log),
+        logId: log.id,
+        status: log.status,
+        model: log.model,
+        provider: log.provider,
+        latencyMs: log.latencyMs,
+        isStreaming: log.isStreaming,
+      });
+    }
+
+    if (session.updatedAt !== null) {
+      events.push({
+        timestamp: session.updatedAt,
+        kind: "session-finished",
+        severity:
+          session.status === "failed"
+            ? "error"
+            : session.status === "active"
+              ? "warning"
+              : "success",
+        title: `Session ${session.status}`,
+        details: `${String(session.requestCount)} request(s), ${String(
+          session.errorCount,
+        )} error(s), ${String(session.tokenUsage.total)} token(s).`,
+        logId: session.lastLogId,
+        status: null,
+        model: session.lastModel,
+        provider: null,
+        latencyMs: null,
+        isStreaming: null,
+      });
+    }
+  }
+
+  events.push({
+    timestamp: exportedAt,
+    kind: "evidence-exported",
+    severity: "info",
+    title: "Evidence exported",
+    details: `Evidence pack exported at ${exportedAt}.`,
+    logId: null,
+    status: null,
+    model: null,
+    provider: null,
+    latencyMs: null,
+    isStreaming: null,
+  });
+
+  return events.sort((left, right) => left.timestamp.localeCompare(right.timestamp));
+}
+
+function reportStatus(classification: FailureClassification): JenkinsReport["status"] {
+  switch (classification.outcome) {
+    case "success":
+      return "PASS";
+    case "failure":
+      return "FAIL";
+    case "active":
+      return "RUNNING";
+    case "unknown":
+      return "UNKNOWN";
+  }
+}
+
+function timelineMarkdown(timeline: readonly RunTimelineEvent[]): string[] {
+  if (timeline.length === 0) return ["No timeline events were available."];
+  return [
+    "| Time | Event | Severity | Evidence |",
+    "| --- | --- | --- | --- |",
+    ...timeline.map((event) => {
+      const evidence = event.logId === null ? "n/a" : `log ${String(event.logId)}`;
+      return `| ${event.timestamp} | ${event.title} | ${event.severity} | ${evidence} |`;
+    }),
+  ];
+}
+
+export function buildJenkinsReport(input: {
+  run: InspectorRun;
+  session: SessionInfo | null;
+  timeline: readonly RunTimelineEvent[];
+  classification: FailureClassification;
+  evidencePaths: AnalyzeEvidenceInput["evidencePaths"];
+}): JenkinsReport {
+  const status = reportStatus(input.classification);
+  const session = input.session;
+  const summary = `[${status}] ${input.classification.summary}`;
+  const lines = [
+    `# Jenkins Report: ${input.run.title}`,
+    "",
+    `- Status: ${status}`,
+    `- Run ID: ${input.run.id}`,
+    `- Session ID: ${input.run.sessionId}`,
+    `- Inspector URL: ${session?.inspectorUrl ?? "n/a"}`,
+    `- Category: ${input.classification.category}`,
+    `- Severity: ${input.classification.severity}`,
+    `- Summary: ${input.classification.summary}`,
+    `- Evidence Logs: ${
+      input.classification.evidenceLogIds.length > 0
+        ? input.classification.evidenceLogIds.map((id) => `#${String(id)}`).join(", ")
+        : "n/a"
+    }`,
+    "",
+    "## Session Metrics",
+    "",
+    `- Requests: ${String(session?.requestCount ?? 0)}`,
+    `- Errors: ${String(session?.errorCount ?? 0)}`,
+    `- Models: ${formatList(session?.models ?? [])}`,
+    `- Providers: ${formatList(session?.providers ?? [])}`,
+    `- Tokens: ${String(session?.tokenUsage.total ?? 0)}`,
+    "",
+    "## Timeline",
+    "",
+    ...timelineMarkdown(input.timeline),
+    "",
+    "## Artifacts",
+    "",
+    `- JSON: ${input.evidencePaths.jsonPath}`,
+    `- Markdown: ${input.evidencePaths.markdownPath}`,
+    `- HTML: ${input.evidencePaths.htmlPath}`,
+    "",
+    "## Next Actions",
+    "",
+    ...input.classification.hints.map((hint) => `- ${hint}`),
+    "",
+  ];
+
+  return {
+    status,
+    summary,
+    markdown: lines.join("\n"),
+  };
+}
+
+export function analyzeEvidence(input: AnalyzeEvidenceInput): {
+  classification: FailureClassification;
+  timeline: RunTimelineEvent[];
+  jenkinsReport: JenkinsReport;
+} {
+  const classification = classifyRunFailure(input.run, input.session);
+  const timeline = buildRunTimeline(input.run, input.session, input.exportedAt);
+  const jenkinsReport = buildJenkinsReport({
+    run: input.run,
+    session: input.session,
+    timeline,
+    classification,
+    evidencePaths: input.evidencePaths,
+  });
+  return { classification, timeline, jenkinsReport };
+}