@tonyclaw/agent-inspector 2.1.3 → 2.1.4

package/src/proxy/alerts.ts

@@ -0,0 +1,664 @@
+import { createHash } from "node:crypto";
+import type { JsonValue } from "../contracts";
+import type {
+  Alert,
+  AlertCategory,
+  AlertEvidenceLink,
+  AlertListResponse,
+  AlertSeverity,
+  AlertSource,
+  AlertSummary,
+} from "../lib/alertContract";
+import type { InspectorGroup, InspectorGroupMember } from "../lib/groupContract";
+import type { RecentFailure } from "../lib/runContract";
+import { getSessionPath } from "../lib/sessionUrl";
+import { getConfig } from "./config";
+import { listGroups } from "./groupStore";
+import { listRecentFailures } from "./runFailures";
+import type { CapturedLog } from "./schemas";
+import { listLogsCursorPage } from "./store";
+
+export const DEFAULT_ALERT_LIMIT = 50;
+export const MAX_ALERT_LIMIT = 200;
+export const DEFAULT_ALERT_SCAN_LIMIT = 500;
+export const MAX_ALERT_SCAN_LIMIT = 2_000;
+
+type AlertFilters = {
+  severity?: AlertSeverity;
+  category?: AlertCategory;
+  source?: AlertSource;
+};
+
+export type ListAlertsOptions = AlertFilters & {
+  limit?: number;
+  scanLimit?: number;
+};
+
+type AlertObservation = {
+  fingerprint: string;
+  severity: AlertSeverity;
+  category: AlertCategory;
+  source: AlertSource;
+  title: string;
+  message: string;
+  observedAt: string;
+  logId: number | null;
+  sessionId: string | null;
+  runId: string | null;
+  groupId: string | null;
+  provider: string | null;
+  model: string | null;
+  apiFormat: Alert["apiFormat"];
+  evidence: AlertEvidenceLink[];
+  metadata: Record<string, JsonValue>;
+};
+
+export type AlertCollectionInput = {
+  logs: readonly CapturedLog[];
+  recentFailures: readonly RecentFailure[];
+  groups: readonly InspectorGroup[];
+  slowResponseThresholdSeconds: number;
+};
+
+function clampPositiveInt(value: number | undefined, fallback: number, max: number): number {
+  if (value === undefined) return fallback;
+  if (!Number.isInteger(value) || value < 1) return fallback;
+  return Math.min(value, max);
+}
+
+export function clampAlertLimit(value: number | undefined): number {
+  return clampPositiveInt(value, DEFAULT_ALERT_LIMIT, MAX_ALERT_LIMIT);
+}
+
+export function clampAlertScanLimit(value: number | undefined): number {
+  return clampPositiveInt(value, DEFAULT_ALERT_SCAN_LIMIT, MAX_ALERT_SCAN_LIMIT);
+}
+
+function alertIdFor(value: string): string {
+  const digest = createHash("sha1").update(value).digest("hex").slice(0, 16);
+  return `alert_${digest}`;
+}
+
+function sanitizeFingerprintPart(value: string | number | null): string {
+  if (value === null) return "-";
+  return String(value).trim().toLowerCase().replace(/\s+/g, " ").slice(0, 240);
+}
+
+function fingerprint(parts: readonly (string | number | null)[]): string {
+  return parts.map(sanitizeFingerprintPart).join("|");
+}
+
+function uniqueStrings(values: readonly string[]): string[] {
+  return [...new Set(values)].sort();
+}
+
+function uniqueNumbers(values: readonly number[]): number[] {
+  return [...new Set(values)].sort((left, right) => left - right);
+}
+
+function logEvidence(logId: number): AlertEvidenceLink {
+  const id = String(logId);
+  return { kind: "log", id, label: `Log #${id}`, href: `#log-${id}` };
+}
+
+function sessionEvidence(sessionId: string): AlertEvidenceLink {
+  return {
+    kind: "session",
+    id: sessionId,
+    label: `Session ${sessionId}`,
+    href: getSessionPath(sessionId),
+  };
+}
+
+function runEvidence(runId: string): AlertEvidenceLink {
+  return { kind: "run", id: runId, label: `Run ${runId}`, href: null };
+}
+
+function groupEvidence(groupId: string): AlertEvidenceLink {
+  return { kind: "group", id: groupId, label: `Group ${groupId}`, href: null };
+}
+
+function baseEvidence(
+  logId: number | null,
+  sessionId: string | null,
+  runId: string | null,
+  groupId: string | null,
+): AlertEvidenceLink[] {
+  const links: AlertEvidenceLink[] = [];
+  if (logId !== null) links.push(logEvidence(logId));
+  if (sessionId !== null) links.push(sessionEvidence(sessionId));
+  if (runId !== null) links.push(runEvidence(runId));
+  if (groupId !== null) links.push(groupEvidence(groupId));
+  return links;
+}
+
+function hasEquivalentEvidence(
+  links: readonly AlertEvidenceLink[],
+  candidate: AlertEvidenceLink,
+): boolean {
+  return links.some((link) => link.kind === candidate.kind && link.id === candidate.id);
+}
+
+function mergeEvidence(
+  left: readonly AlertEvidenceLink[],
+  right: readonly AlertEvidenceLink[],
+): AlertEvidenceLink[] {
+  const merged: AlertEvidenceLink[] = [...left];
+  for (const link of right) {
+    if (!hasEquivalentEvidence(merged, link)) merged.push(link);
+  }
+  return merged;
+}
+
+function firstString(left: string | null, right: string | null): string | null {
+  return left ?? right;
+}
+
+function mergeAlert(existing: Alert, observation: AlertObservation): Alert {
+  const logIds =
+    observation.logId === null ? existing.logIds : [...existing.logIds, observation.logId];
+  const sessionIds =
+    observation.sessionId === null
+      ? existing.sessionIds
+      : [...existing.sessionIds, observation.sessionId];
+  const runIds =
+    observation.runId === null ? existing.runIds : [...existing.runIds, observation.runId];
+  const groupIds =
+    observation.groupId === null ? existing.groupIds : [...existing.groupIds, observation.groupId];
+
+  return {
+    ...existing,
+    count: existing.count + 1,
+    firstSeenAt:
+      observation.observedAt < existing.firstSeenAt ? observation.observedAt : existing.firstSeenAt,
+    lastSeenAt:
+      observation.observedAt > existing.lastSeenAt ? observation.observedAt : existing.lastSeenAt,
+    logIds: uniqueNumbers(logIds),
+    sessionIds: uniqueStrings(sessionIds),
+    runIds: uniqueStrings(runIds),
+    groupIds: uniqueStrings(groupIds),
+    provider: firstString(existing.provider, observation.provider),
+    model: firstString(existing.model, observation.model),
+    apiFormat: existing.apiFormat ?? observation.apiFormat,
+    evidence: mergeEvidence(existing.evidence, observation.evidence),
+    metadata: { ...existing.metadata, ...observation.metadata },
+  };
+}
+
+function observationToAlert(observation: AlertObservation): Alert {
+  return {
+    id: alertIdFor(observation.fingerprint),
+    fingerprint: observation.fingerprint,
+    severity: observation.severity,
+    category: observation.category,
+    source: observation.source,
+    status: "open",
+    title: observation.title,
+    message: observation.message,
+    count: 1,
+    firstSeenAt: observation.observedAt,
+    lastSeenAt: observation.observedAt,
+    logIds: observation.logId === null ? [] : [observation.logId],
+    sessionIds: observation.sessionId === null ? [] : [observation.sessionId],
+    runIds: observation.runId === null ? [] : [observation.runId],
+    groupIds: observation.groupId === null ? [] : [observation.groupId],
+    provider: observation.provider,
+    model: observation.model,
+    apiFormat: observation.apiFormat,
+    evidence: observation.evidence,
+    metadata: observation.metadata,
+  };
+}
+
+function addObservation(map: Map<string, Alert>, observation: AlertObservation): void {
+  const existing = map.get(observation.fingerprint);
+  if (existing === undefined) {
+    map.set(observation.fingerprint, observationToAlert(observation));
+    return;
+  }
+  map.set(observation.fingerprint, mergeAlert(existing, observation));
+}
+
+function lowerText(value: string | null | undefined): string {
+  if (value === null || value === undefined) return "";
+  return value.toLowerCase();
+}
+
+function isTimeoutStatus(status: number | null): boolean {
+  return status === 408 || status === 504;
+}
+
+function isTimeoutLog(log: CapturedLog): boolean {
+  const text = `${lowerText(log.error)} ${lowerText(log.responseText)}`;
+  return (
+    isTimeoutStatus(log.responseStatus) ||
+    text.includes("timeout") ||
+    text.includes("timed out") ||
+    text.includes("abort")
+  );
+}
+
+function hasFailureStatus(status: number | null): boolean {
+  return status !== null && status >= 400;
+}
+
+function isFailedLog(log: CapturedLog): boolean {
+  return log.error !== null && log.error !== undefined && log.error.length > 0
+    ? true
+    : hasFailureStatus(log.responseStatus);
+}
+
+function statusLabel(status: number | null): string {
+  return status === null ? "no status" : `HTTP ${String(status)}`;
+}
+
+function errorDetail(log: CapturedLog): string {
+  if (log.error !== null && log.error !== undefined && log.error.length > 0) return log.error;
+  return statusLabel(log.responseStatus);
+}
+
+function requestFailureTitle(log: CapturedLog, timeout: boolean): string {
+  if (log.isTest === true && timeout) return "Provider test timed out";
+  if (log.isTest === true) return "Provider test failed";
+  return timeout ? "Request timed out" : "Request failed";
+}
+
+function requestFailureCategory(timeout: boolean): AlertCategory {
+  return timeout ? "request-timeout" : "request-failure";
+}
+
+function logProvider(log: CapturedLog): string | null {
+  return log.providerName ?? null;
+}
+
+function requestFailureObservation(log: CapturedLog): AlertObservation | null {
+  if (!isFailedLog(log)) return null;
+
+  const timeout = isTimeoutLog(log);
+  const category = requestFailureCategory(timeout);
+  const message = `${errorDetail(log)} while calling ${log.path}.`;
+  return {
+    fingerprint: fingerprint([
+      "proxy",
+      category,
+      logProvider(log),
+      log.model,
+      log.responseStatus,
+      errorDetail(log),
+    ]),
+    severity: "critical",
+    category,
+    source: "proxy",
+    title: requestFailureTitle(log, timeout),
+    message,
+    observedAt: log.timestamp,
+    logId: log.id,
+    sessionId: log.sessionId,
+    runId: null,
+    groupId: null,
+    provider: logProvider(log),
+    model: log.model,
+    apiFormat: log.apiFormat,
+    evidence: baseEvidence(log.id, log.sessionId, null, null),
+    metadata: {
+      path: log.path,
+      status: log.responseStatus,
+      error: log.error ?? null,
+      isTest: log.isTest,
+      streaming: log.streaming,
+    },
+  };
+}
+
+function slowResponseObservation(
+  log: CapturedLog,
+  slowResponseThresholdSeconds: number,
+): AlertObservation | null {
+  if (slowResponseThresholdSeconds <= 0) return null;
+  if (isFailedLog(log)) return null;
+  if (log.elapsedMs === null) return null;
+
+  const thresholdMs = slowResponseThresholdSeconds * 1000;
+  if (log.elapsedMs < thresholdMs) return null;
+
+  const elapsedSeconds = (log.elapsedMs / 1000).toFixed(1);
+  return {
+    fingerprint: fingerprint(["proxy", "slow-response", logProvider(log), log.model, log.path]),
+    severity: "warning",
+    category: "slow-response",
+    source: "proxy",
+    title: "Slow model response",
+    message: `Response took ${elapsedSeconds}s, above the ${String(
+      slowResponseThresholdSeconds,
+    )}s threshold.`,
+    observedAt: log.timestamp,
+    logId: log.id,
+    sessionId: log.sessionId,
+    runId: null,
+    groupId: null,
+    provider: logProvider(log),
+    model: log.model,
+    apiFormat: log.apiFormat,
+    evidence: baseEvidence(log.id, log.sessionId, null, null),
+    metadata: {
+      path: log.path,
+      elapsedMs: log.elapsedMs,
+      thresholdMs,
+      streaming: log.streaming,
+      firstChunkMs: log.firstChunkMs ?? null,
+      totalStreamMs: log.totalStreamMs ?? null,
+    },
+  };
+}
+
+function warningObservation(log: CapturedLog, warning: string): AlertObservation {
+  return {
+    fingerprint: fingerprint(["tool-schema", logProvider(log), log.model, warning]),
+    severity: "warning",
+    category: "tool-schema",
+    source: "tool-schema",
+    title: "Tool calling argument warning",
+    message: warning,
+    observedAt: log.timestamp,
+    logId: log.id,
+    sessionId: log.sessionId,
+    runId: null,
+    groupId: null,
+    provider: logProvider(log),
+    model: log.model,
+    apiFormat: log.apiFormat,
+    evidence: baseEvidence(log.id, log.sessionId, null, null),
+    metadata: {
+      path: log.path,
+      warning,
+    },
+  };
+}
+
+function addLogObservations(
+  map: Map<string, Alert>,
+  logs: readonly CapturedLog[],
+  slowResponseThresholdSeconds: number,
+): void {
+  for (const log of logs) {
+    const failure = requestFailureObservation(log);
+    if (failure !== null) addObservation(map, failure);
+
+    const slow = slowResponseObservation(log, slowResponseThresholdSeconds);
+    if (slow !== null) addObservation(map, slow);
+
+    for (const warning of log.warnings ?? []) {
+      addObservation(map, warningObservation(log, warning));
+    }
+  }
+}
+
+function alertSeverityFromFailure(failure: RecentFailure): AlertSeverity {
+  switch (failure.classification.severity) {
+    case "high":
+      return "critical";
+    case "medium":
+    case "low":
+      return "warning";
+    case "none":
+      return "notice";
+  }
+}
+
+function providerFromRun(failure: RecentFailure): string | null {
+  const provider = failure.run.metadata["provider"] ?? failure.run.metadata["providerName"];
+  return typeof provider === "string" && provider.length > 0 ? provider : null;
+}
+
+function modelFromRun(failure: RecentFailure): string | null {
+  const model = failure.run.metadata["model"] ?? failure.run.metadata["modelName"];
+  return typeof model === "string" && model.length > 0 ? model : null;
+}
+
+function runFailureObservation(failure: RecentFailure): AlertObservation {
+  const groupId = failure.run.groupId;
+  const logEvidenceLinks = failure.classification.evidenceLogIds.map(logEvidence);
+  const links = mergeEvidence(
+    baseEvidence(null, failure.sessionId, failure.run.id, groupId),
+    logEvidenceLinks,
+  );
+
+  return {
+    fingerprint: fingerprint(["run", failure.run.id, failure.classification.category]),
+    severity: alertSeverityFromFailure(failure),
+    category: "run-failure",
+    source: "run",
+    title: `Run ${failure.run.status}`,
+    message: failure.classification.summary,
+    observedAt: failure.updatedAt,
+    logId: failure.classification.evidenceLogIds[0] ?? null,
+    sessionId: failure.sessionId,
+    runId: failure.run.id,
+    groupId,
+    provider: providerFromRun(failure),
+    model: modelFromRun(failure),
+    apiFormat: null,
+    evidence: links,
+    metadata: {
+      runStatus: failure.run.status,
+      failureCategory: failure.classification.category,
+      evidenceMarkdownPath: failure.evidenceMarkdownPath,
+    },
+  };
+}
+
+function addRunFailureObservations(
+  map: Map<string, Alert>,
+  recentFailures: readonly RecentFailure[],
+): void {
+  for (const failure of recentFailures) {
+    addObservation(map, runFailureObservation(failure));
+  }
+}
+
+function memberProvider(member: InspectorGroupMember): string | null {
+  return member.provider !== null && member.provider.length > 0 ? member.provider : null;
+}
+
+function memberModel(member: InspectorGroupMember): string | null {
+  return member.model !== null && member.model.length > 0 ? member.model : null;
+}
+
+function isFailedMember(member: InspectorGroupMember): boolean {
+  return member.status === "failed" || member.status === "cancelled";
+}
+
+function groupFailureObservation(group: InspectorGroup): AlertObservation | null {
+  if (group.status !== "failed" && group.status !== "cancelled") return null;
+  return {
+    fingerprint: fingerprint(["group", group.id, group.status]),
+    severity: "critical",
+    category: "group-failure",
+    source: "group",
+    title: `Group ${group.status}`,
+    message: `${group.title} is marked ${group.status}.`,
+    observedAt: group.updatedAt,
+    logId: null,
+    sessionId: null,
+    runId: null,
+    groupId: group.id,
+    provider: null,
+    model: null,
+    apiFormat: null,
+    evidence: baseEvidence(null, null, null, group.id),
+    metadata: {
+      groupStatus: group.status,
+      memberCount: group.members.length,
+    },
+  };
+}
+
+function groupMemberFailureObservation(
+  group: InspectorGroup,
+  member: InspectorGroupMember,
+): AlertObservation {
+  return {
+    fingerprint: fingerprint(["group-member", group.id, member.id, member.status]),
+    severity: "critical",
+    category: "group-failure",
+    source: "group",
+    title: "Group member failed",
+    message: `${member.label ?? member.sessionId} is marked ${member.status ?? "unknown"}.`,
+    observedAt: member.updatedAt,
+    logId: null,
+    sessionId: member.sessionId,
+    runId: member.runId,
+    groupId: group.id,
+    provider: memberProvider(member),
+    model: memberModel(member),
+    apiFormat: null,
+    evidence: baseEvidence(null, member.sessionId, member.runId, group.id),
+    metadata: {
+      groupTitle: group.title,
+      memberId: member.id,
+      memberStatus: member.status,
+    },
+  };
+}
+
+function groupIncompleteObservation(
+  group: InspectorGroup,
+  missingStatusCount: number,
+): AlertObservation {
+  return {
+    fingerprint: fingerprint(["group-incomplete", group.id, missingStatusCount]),
+    severity: "notice",
+    category: "group-incomplete",
+    source: "group",
+    title: "Group has unclassified members",
+    message: `${group.title} has ${String(missingStatusCount)} member(s) without a final status.`,
+    observedAt: group.updatedAt,
+    logId: null,
+    sessionId: null,
+    runId: null,
+    groupId: group.id,
+    provider: null,
+    model: null,
+    apiFormat: null,
+    evidence: baseEvidence(null, null, null, group.id),
+    metadata: {
+      groupTitle: group.title,
+      missingStatusCount,
+    },
+  };
+}
+
+function addGroupObservations(map: Map<string, Alert>, groups: readonly InspectorGroup[]): void {
+  for (const group of groups) {
+    const groupFailure = groupFailureObservation(group);
+    if (groupFailure !== null) addObservation(map, groupFailure);
+
+    let missingStatusCount = 0;
+    for (const member of group.members) {
+      if (isFailedMember(member)) {
+        addObservation(map, groupMemberFailureObservation(group, member));
+      }
+      if (member.status === null) missingStatusCount += 1;
+    }
+
+    if (missingStatusCount > 0) {
+      addObservation(map, groupIncompleteObservation(group, missingStatusCount));
+    }
+  }
+}
+
+function alertMatchesFilters(alert: Alert, filters: AlertFilters): boolean {
+  if (filters.severity !== undefined && alert.severity !== filters.severity) return false;
+  if (filters.category !== undefined && alert.category !== filters.category) return false;
+  if (filters.source !== undefined && alert.source !== filters.source) return false;
+  return true;
+}
+
+function sortAlerts(alerts: readonly Alert[]): Alert[] {
+  return [...alerts].sort((left, right) => {
+    const timeCompare = right.lastSeenAt.localeCompare(left.lastSeenAt);
+    if (timeCompare !== 0) return timeCompare;
+    return left.id.localeCompare(right.id);
+  });
+}
+
+export function buildAlertSummary(alerts: readonly Alert[]): AlertSummary {
+  let critical = 0;
+  let warning = 0;
+  let notice = 0;
+  const byCategory = new Map<AlertCategory, number>();
+
+  for (const alert of alerts) {
+    switch (alert.severity) {
+      case "critical":
+        critical += 1;
+        break;
+      case "warning":
+        warning += 1;
+        break;
+      case "notice":
+        notice += 1;
+        break;
+    }
+    byCategory.set(alert.category, (byCategory.get(alert.category) ?? 0) + 1);
+  }
+
+  return {
+    total: alerts.length,
+    open: alerts.length,
+    critical,
+    warning,
+    notice,
+    byCategory: [...byCategory.entries()]
+      .map(([category, count]) => ({ category, count }))
+      .sort(
+        (left, right) => right.count - left.count || left.category.localeCompare(right.category),
+      ),
+  };
+}
+
+export function collectAlertsFromEvidence(input: AlertCollectionInput): Alert[] {
+  const map = new Map<string, Alert>();
+  addLogObservations(map, input.logs, input.slowResponseThresholdSeconds);
+  addRunFailureObservations(map, input.recentFailures);
+  addGroupObservations(map, input.groups);
+  return sortAlerts([...map.values()]);
+}
+
+function filterAlerts(alerts: readonly Alert[], filters: AlertFilters): Alert[] {
+  return alerts.filter((alert) => alertMatchesFilters(alert, filters));
+}
+
+export async function listAlerts(options: ListAlertsOptions = {}): Promise<AlertListResponse> {
+  const limit = clampAlertLimit(options.limit);
+  const scanLimit = clampAlertScanLimit(options.scanLimit);
+  const logPage = await listLogsCursorPage({
+    limit: scanLimit,
+    includeBodies: false,
+    anchor: "newest",
+  });
+  const config = getConfig();
+  const alerts = collectAlertsFromEvidence({
+    logs: logPage.logs,
+    recentFailures: listRecentFailures(scanLimit).failures,
+    groups: listGroups(),
+    slowResponseThresholdSeconds: config.slowResponseThresholdSeconds,
+  });
+  const filtered = filterAlerts(alerts, options);
+  return {
+    alerts: filtered.slice(0, limit),
+    total: filtered.length,
+    limit,
+    summary: buildAlertSummary(filtered),
+  };
+}
+
+export async function getAlertSummary(
+  options: Pick<ListAlertsOptions, "scanLimit"> = {},
+): Promise<AlertSummary> {
+  const response = await listAlerts({
+    limit: MAX_ALERT_LIMIT,
+    scanLimit: options.scanLimit,
+  });
+  return response.summary;
+}