@tonyclaw/agent-inspector 4.0.1 → 4.0.3

package/src/proxy/alerts.ts

@@ -1,664 +0,0 @@
-import { createHash } from "node:crypto";
-import type { JsonValue } from "../contracts";
-import type {
-  Alert,
-  AlertCategory,
-  AlertEvidenceLink,
-  AlertListResponse,
-  AlertSeverity,
-  AlertSource,
-  AlertSummary,
-} from "../lib/alertContract";
-import type { InspectorGroup, InspectorGroupMember } from "../lib/groupContract";
-import type { RecentFailure } from "../lib/runContract";
-import { getSessionPath } from "../lib/sessionUrl";
-import { getConfig } from "./config";
-import { listGroups } from "./groupStore";
-import { listRecentFailures } from "./runFailures";
-import type { CapturedLog } from "./schemas";
-import { listLogsCursorPage } from "./store";
-
-export const DEFAULT_ALERT_LIMIT = 50;
-export const MAX_ALERT_LIMIT = 200;
-export const DEFAULT_ALERT_SCAN_LIMIT = 500;
-export const MAX_ALERT_SCAN_LIMIT = 2_000;
-
-type AlertFilters = {
-  severity?: AlertSeverity;
-  category?: AlertCategory;
-  source?: AlertSource;
-};
-
-export type ListAlertsOptions = AlertFilters & {
-  limit?: number;
-  scanLimit?: number;
-};
-
-type AlertObservation = {
-  fingerprint: string;
-  severity: AlertSeverity;
-  category: AlertCategory;
-  source: AlertSource;
-  title: string;
-  message: string;
-  observedAt: string;
-  logId: number | null;
-  sessionId: string | null;
-  runId: string | null;
-  groupId: string | null;
-  provider: string | null;
-  model: string | null;
-  apiFormat: Alert["apiFormat"];
-  evidence: AlertEvidenceLink[];
-  metadata: Record<string, JsonValue>;
-};
-
-export type AlertCollectionInput = {
-  logs: readonly CapturedLog[];
-  recentFailures: readonly RecentFailure[];
-  groups: readonly InspectorGroup[];
-  slowResponseThresholdSeconds: number;
-};
-
-function clampPositiveInt(value: number | undefined, fallback: number, max: number): number {
-  if (value === undefined) return fallback;
-  if (!Number.isInteger(value) || value < 1) return fallback;
-  return Math.min(value, max);
-}
-
-export function clampAlertLimit(value: number | undefined): number {
-  return clampPositiveInt(value, DEFAULT_ALERT_LIMIT, MAX_ALERT_LIMIT);
-}
-
-export function clampAlertScanLimit(value: number | undefined): number {
-  return clampPositiveInt(value, DEFAULT_ALERT_SCAN_LIMIT, MAX_ALERT_SCAN_LIMIT);
-}
-
-function alertIdFor(value: string): string {
-  const digest = createHash("sha1").update(value).digest("hex").slice(0, 16);
-  return `alert_${digest}`;
-}
-
-function sanitizeFingerprintPart(value: string | number | null): string {
-  if (value === null) return "-";
-  return String(value).trim().toLowerCase().replace(/\s+/g, " ").slice(0, 240);
-}
-
-function fingerprint(parts: readonly (string | number | null)[]): string {
-  return parts.map(sanitizeFingerprintPart).join("|");
-}
-
-function uniqueStrings(values: readonly string[]): string[] {
-  return [...new Set(values)].sort();
-}
-
-function uniqueNumbers(values: readonly number[]): number[] {
-  return [...new Set(values)].sort((left, right) => left - right);
-}
-
-function logEvidence(logId: number): AlertEvidenceLink {
-  const id = String(logId);
-  return { kind: "log", id, label: `Log #${id}`, href: `#log-${id}` };
-}
-
-function sessionEvidence(sessionId: string): AlertEvidenceLink {
-  return {
-    kind: "session",
-    id: sessionId,
-    label: `Session ${sessionId}`,
-    href: getSessionPath(sessionId),
-  };
-}
-
-function runEvidence(runId: string): AlertEvidenceLink {
-  return { kind: "run", id: runId, label: `Run ${runId}`, href: null };
-}
-
-function groupEvidence(groupId: string): AlertEvidenceLink {
-  return { kind: "group", id: groupId, label: `Group ${groupId}`, href: null };
-}
-
-function baseEvidence(
-  logId: number | null,
-  sessionId: string | null,
-  runId: string | null,
-  groupId: string | null,
-): AlertEvidenceLink[] {
-  const links: AlertEvidenceLink[] = [];
-  if (logId !== null) links.push(logEvidence(logId));
-  if (sessionId !== null) links.push(sessionEvidence(sessionId));
-  if (runId !== null) links.push(runEvidence(runId));
-  if (groupId !== null) links.push(groupEvidence(groupId));
-  return links;
-}
-
-function hasEquivalentEvidence(
-  links: readonly AlertEvidenceLink[],
-  candidate: AlertEvidenceLink,
-): boolean {
-  return links.some((link) => link.kind === candidate.kind && link.id === candidate.id);
-}
-
-function mergeEvidence(
-  left: readonly AlertEvidenceLink[],
-  right: readonly AlertEvidenceLink[],
-): AlertEvidenceLink[] {
-  const merged: AlertEvidenceLink[] = [...left];
-  for (const link of right) {
-    if (!hasEquivalentEvidence(merged, link)) merged.push(link);
-  }
-  return merged;
-}
-
-function firstString(left: string | null, right: string | null): string | null {
-  return left ?? right;
-}
-
-function mergeAlert(existing: Alert, observation: AlertObservation): Alert {
-  const logIds =
-    observation.logId === null ? existing.logIds : [...existing.logIds, observation.logId];
-  const sessionIds =
-    observation.sessionId === null
-      ? existing.sessionIds
-      : [...existing.sessionIds, observation.sessionId];
-  const runIds =
-    observation.runId === null ? existing.runIds : [...existing.runIds, observation.runId];
-  const groupIds =
-    observation.groupId === null ? existing.groupIds : [...existing.groupIds, observation.groupId];
-
-  return {
-    ...existing,
-    count: existing.count + 1,
-    firstSeenAt:
-      observation.observedAt < existing.firstSeenAt ? observation.observedAt : existing.firstSeenAt,
-    lastSeenAt:
-      observation.observedAt > existing.lastSeenAt ? observation.observedAt : existing.lastSeenAt,
-    logIds: uniqueNumbers(logIds),
-    sessionIds: uniqueStrings(sessionIds),
-    runIds: uniqueStrings(runIds),
-    groupIds: uniqueStrings(groupIds),
-    provider: firstString(existing.provider, observation.provider),
-    model: firstString(existing.model, observation.model),
-    apiFormat: existing.apiFormat ?? observation.apiFormat,
-    evidence: mergeEvidence(existing.evidence, observation.evidence),
-    metadata: { ...existing.metadata, ...observation.metadata },
-  };
-}
-
-function observationToAlert(observation: AlertObservation): Alert {
-  return {
-    id: alertIdFor(observation.fingerprint),
-    fingerprint: observation.fingerprint,
-    severity: observation.severity,
-    category: observation.category,
-    source: observation.source,
-    status: "open",
-    title: observation.title,
-    message: observation.message,
-    count: 1,
-    firstSeenAt: observation.observedAt,
-    lastSeenAt: observation.observedAt,
-    logIds: observation.logId === null ? [] : [observation.logId],
-    sessionIds: observation.sessionId === null ? [] : [observation.sessionId],
-    runIds: observation.runId === null ? [] : [observation.runId],
-    groupIds: observation.groupId === null ? [] : [observation.groupId],
-    provider: observation.provider,
-    model: observation.model,
-    apiFormat: observation.apiFormat,
-    evidence: observation.evidence,
-    metadata: observation.metadata,
-  };
-}
-
-function addObservation(map: Map<string, Alert>, observation: AlertObservation): void {
-  const existing = map.get(observation.fingerprint);
-  if (existing === undefined) {
-    map.set(observation.fingerprint, observationToAlert(observation));
-    return;
-  }
-  map.set(observation.fingerprint, mergeAlert(existing, observation));
-}
-
-function lowerText(value: string | null | undefined): string {
-  if (value === null || value === undefined) return "";
-  return value.toLowerCase();
-}
-
-function isTimeoutStatus(status: number | null): boolean {
-  return status === 408 || status === 504;
-}
-
-function isTimeoutLog(log: CapturedLog): boolean {
-  const text = `${lowerText(log.error)} ${lowerText(log.responseText)}`;
-  return (
-    isTimeoutStatus(log.responseStatus) ||
-    text.includes("timeout") ||
-    text.includes("timed out") ||
-    text.includes("abort")
-  );
-}
-
-function hasFailureStatus(status: number | null): boolean {
-  return status !== null && status >= 400;
-}
-
-function isFailedLog(log: CapturedLog): boolean {
-  return log.error !== null && log.error !== undefined && log.error.length > 0
-    ? true
-    : hasFailureStatus(log.responseStatus);
-}
-
-function statusLabel(status: number | null): string {
-  return status === null ? "no status" : `HTTP ${String(status)}`;
-}
-
-function errorDetail(log: CapturedLog): string {
-  if (log.error !== null && log.error !== undefined && log.error.length > 0) return log.error;
-  return statusLabel(log.responseStatus);
-}
-
-function requestFailureTitle(log: CapturedLog, timeout: boolean): string {
-  if (log.isTest === true && timeout) return "Provider test timed out";
-  if (log.isTest === true) return "Provider test failed";
-  return timeout ? "Request timed out" : "Request failed";
-}
-
-function requestFailureCategory(timeout: boolean): AlertCategory {
-  return timeout ? "request-timeout" : "request-failure";
-}
-
-function logProvider(log: CapturedLog): string | null {
-  return log.providerName ?? null;
-}
-
-function requestFailureObservation(log: CapturedLog): AlertObservation | null {
-  if (!isFailedLog(log)) return null;
-
-  const timeout = isTimeoutLog(log);
-  const category = requestFailureCategory(timeout);
-  const message = `${errorDetail(log)} while calling ${log.path}.`;
-  return {
-    fingerprint: fingerprint([
-      "proxy",
-      category,
-      logProvider(log),
-      log.model,
-      log.responseStatus,
-      errorDetail(log),
-    ]),
-    severity: "critical",
-    category,
-    source: "proxy",
-    title: requestFailureTitle(log, timeout),
-    message,
-    observedAt: log.timestamp,
-    logId: log.id,
-    sessionId: log.sessionId,
-    runId: null,
-    groupId: null,
-    provider: logProvider(log),
-    model: log.model,
-    apiFormat: log.apiFormat,
-    evidence: baseEvidence(log.id, log.sessionId, null, null),
-    metadata: {
-      path: log.path,
-      status: log.responseStatus,
-      error: log.error ?? null,
-      isTest: log.isTest,
-      streaming: log.streaming,
-    },
-  };
-}
-
-function slowResponseObservation(
-  log: CapturedLog,
-  slowResponseThresholdSeconds: number,
-): AlertObservation | null {
-  if (slowResponseThresholdSeconds <= 0) return null;
-  if (isFailedLog(log)) return null;
-  if (log.elapsedMs === null) return null;
-
-  const thresholdMs = slowResponseThresholdSeconds * 1000;
-  if (log.elapsedMs < thresholdMs) return null;
-
-  const elapsedSeconds = (log.elapsedMs / 1000).toFixed(1);
-  return {
-    fingerprint: fingerprint(["proxy", "slow-response", logProvider(log), log.model, log.path]),
-    severity: "warning",
-    category: "slow-response",
-    source: "proxy",
-    title: "Slow model response",
-    message: `Response took ${elapsedSeconds}s, above the ${String(
-      slowResponseThresholdSeconds,
-    )}s threshold.`,
-    observedAt: log.timestamp,
-    logId: log.id,
-    sessionId: log.sessionId,
-    runId: null,
-    groupId: null,
-    provider: logProvider(log),
-    model: log.model,
-    apiFormat: log.apiFormat,
-    evidence: baseEvidence(log.id, log.sessionId, null, null),
-    metadata: {
-      path: log.path,
-      elapsedMs: log.elapsedMs,
-      thresholdMs,
-      streaming: log.streaming,
-      firstChunkMs: log.firstChunkMs ?? null,
-      totalStreamMs: log.totalStreamMs ?? null,
-    },
-  };
-}
-
-function warningObservation(log: CapturedLog, warning: string): AlertObservation {
-  return {
-    fingerprint: fingerprint(["tool-schema", logProvider(log), log.model, warning]),
-    severity: "warning",
-    category: "tool-schema",
-    source: "tool-schema",
-    title: "Tool calling argument warning",
-    message: warning,
-    observedAt: log.timestamp,
-    logId: log.id,
-    sessionId: log.sessionId,
-    runId: null,
-    groupId: null,
-    provider: logProvider(log),
-    model: log.model,
-    apiFormat: log.apiFormat,
-    evidence: baseEvidence(log.id, log.sessionId, null, null),
-    metadata: {
-      path: log.path,
-      warning,
-    },
-  };
-}
-
-function addLogObservations(
-  map: Map<string, Alert>,
-  logs: readonly CapturedLog[],
-  slowResponseThresholdSeconds: number,
-): void {
-  for (const log of logs) {
-    const failure = requestFailureObservation(log);
-    if (failure !== null) addObservation(map, failure);
-
-    const slow = slowResponseObservation(log, slowResponseThresholdSeconds);
-    if (slow !== null) addObservation(map, slow);
-
-    for (const warning of log.warnings ?? []) {
-      addObservation(map, warningObservation(log, warning));
-    }
-  }
-}
-
-function alertSeverityFromFailure(failure: RecentFailure): AlertSeverity {
-  switch (failure.classification.severity) {
-    case "high":
-      return "critical";
-    case "medium":
-    case "low":
-      return "warning";
-    case "none":
-      return "notice";
-  }
-}
-
-function providerFromRun(failure: RecentFailure): string | null {
-  const provider = failure.run.metadata["provider"] ?? failure.run.metadata["providerName"];
-  return typeof provider === "string" && provider.length > 0 ? provider : null;
-}
-
-function modelFromRun(failure: RecentFailure): string | null {
-  const model = failure.run.metadata["model"] ?? failure.run.metadata["modelName"];
-  return typeof model === "string" && model.length > 0 ? model : null;
-}
-
-function runFailureObservation(failure: RecentFailure): AlertObservation {
-  const groupId = failure.run.groupId;
-  const logEvidenceLinks = failure.classification.evidenceLogIds.map(logEvidence);
-  const links = mergeEvidence(
-    baseEvidence(null, failure.sessionId, failure.run.id, groupId),
-    logEvidenceLinks,
-  );
-
-  return {
-    fingerprint: fingerprint(["run", failure.run.id, failure.classification.category]),
-    severity: alertSeverityFromFailure(failure),
-    category: "run-failure",
-    source: "run",
-    title: `Run ${failure.run.status}`,
-    message: failure.classification.summary,
-    observedAt: failure.updatedAt,
-    logId: failure.classification.evidenceLogIds[0] ?? null,
-    sessionId: failure.sessionId,
-    runId: failure.run.id,
-    groupId,
-    provider: providerFromRun(failure),
-    model: modelFromRun(failure),
-    apiFormat: null,
-    evidence: links,
-    metadata: {
-      runStatus: failure.run.status,
-      failureCategory: failure.classification.category,
-      evidenceMarkdownPath: failure.evidenceMarkdownPath,
-    },
-  };
-}
-
-function addRunFailureObservations(
-  map: Map<string, Alert>,
-  recentFailures: readonly RecentFailure[],
-): void {
-  for (const failure of recentFailures) {
-    addObservation(map, runFailureObservation(failure));
-  }
-}
-
-function memberProvider(member: InspectorGroupMember): string | null {
-  return member.provider !== null && member.provider.length > 0 ? member.provider : null;
-}
-
-function memberModel(member: InspectorGroupMember): string | null {
-  return member.model !== null && member.model.length > 0 ? member.model : null;
-}
-
-function isFailedMember(member: InspectorGroupMember): boolean {
-  return member.status === "failed" || member.status === "cancelled";
-}
-
-function groupFailureObservation(group: InspectorGroup): AlertObservation | null {
-  if (group.status !== "failed" && group.status !== "cancelled") return null;
-  return {
-    fingerprint: fingerprint(["group", group.id, group.status]),
-    severity: "critical",
-    category: "group-failure",
-    source: "group",
-    title: `Group ${group.status}`,
-    message: `${group.title} is marked ${group.status}.`,
-    observedAt: group.updatedAt,
-    logId: null,
-    sessionId: null,
-    runId: null,
-    groupId: group.id,
-    provider: null,
-    model: null,
-    apiFormat: null,
-    evidence: baseEvidence(null, null, null, group.id),
-    metadata: {
-      groupStatus: group.status,
-      memberCount: group.members.length,
-    },
-  };
-}
-
-function groupMemberFailureObservation(
-  group: InspectorGroup,
-  member: InspectorGroupMember,
-): AlertObservation {
-  return {
-    fingerprint: fingerprint(["group-member", group.id, member.id, member.status]),
-    severity: "critical",
-    category: "group-failure",
-    source: "group",
-    title: "Group member failed",
-    message: `${member.label ?? member.sessionId} is marked ${member.status ?? "unknown"}.`,
-    observedAt: member.updatedAt,
-    logId: null,
-    sessionId: member.sessionId,
-    runId: member.runId,
-    groupId: group.id,
-    provider: memberProvider(member),
-    model: memberModel(member),
-    apiFormat: null,
-    evidence: baseEvidence(null, member.sessionId, member.runId, group.id),
-    metadata: {
-      groupTitle: group.title,
-      memberId: member.id,
-      memberStatus: member.status,
-    },
-  };
-}
-
-function groupIncompleteObservation(
-  group: InspectorGroup,
-  missingStatusCount: number,
-): AlertObservation {
-  return {
-    fingerprint: fingerprint(["group-incomplete", group.id, missingStatusCount]),
-    severity: "notice",
-    category: "group-incomplete",
-    source: "group",
-    title: "Group has unclassified members",
-    message: `${group.title} has ${String(missingStatusCount)} member(s) without a final status.`,
-    observedAt: group.updatedAt,
-    logId: null,
-    sessionId: null,
-    runId: null,
-    groupId: group.id,
-    provider: null,
-    model: null,
-    apiFormat: null,
-    evidence: baseEvidence(null, null, null, group.id),
-    metadata: {
-      groupTitle: group.title,
-      missingStatusCount,
-    },
-  };
-}
-
-function addGroupObservations(map: Map<string, Alert>, groups: readonly InspectorGroup[]): void {
-  for (const group of groups) {
-    const groupFailure = groupFailureObservation(group);
-    if (groupFailure !== null) addObservation(map, groupFailure);
-
-    let missingStatusCount = 0;
-    for (const member of group.members) {
-      if (isFailedMember(member)) {
-        addObservation(map, groupMemberFailureObservation(group, member));
-      }
-      if (member.status === null) missingStatusCount += 1;
-    }
-
-    if (missingStatusCount > 0) {
-      addObservation(map, groupIncompleteObservation(group, missingStatusCount));
-    }
-  }
-}
-
-function alertMatchesFilters(alert: Alert, filters: AlertFilters): boolean {
-  if (filters.severity !== undefined && alert.severity !== filters.severity) return false;
-  if (filters.category !== undefined && alert.category !== filters.category) return false;
-  if (filters.source !== undefined && alert.source !== filters.source) return false;
-  return true;
-}
-
-function sortAlerts(alerts: readonly Alert[]): Alert[] {
-  return [...alerts].sort((left, right) => {
-    const timeCompare = right.lastSeenAt.localeCompare(left.lastSeenAt);
-    if (timeCompare !== 0) return timeCompare;
-    return left.id.localeCompare(right.id);
-  });
-}
-
-export function buildAlertSummary(alerts: readonly Alert[]): AlertSummary {
-  let critical = 0;
-  let warning = 0;
-  let notice = 0;
-  const byCategory = new Map<AlertCategory, number>();
-
-  for (const alert of alerts) {
-    switch (alert.severity) {
-      case "critical":
-        critical += 1;
-        break;
-      case "warning":
-        warning += 1;
-        break;
-      case "notice":
-        notice += 1;
-        break;
-    }
-    byCategory.set(alert.category, (byCategory.get(alert.category) ?? 0) + 1);
-  }
-
-  return {
-    total: alerts.length,
-    open: alerts.length,
-    critical,
-    warning,
-    notice,
-    byCategory: [...byCategory.entries()]
-      .map(([category, count]) => ({ category, count }))
-      .sort(
-        (left, right) => right.count - left.count || left.category.localeCompare(right.category),
-      ),
-  };
-}
-
-export function collectAlertsFromEvidence(input: AlertCollectionInput): Alert[] {
-  const map = new Map<string, Alert>();
-  addLogObservations(map, input.logs, input.slowResponseThresholdSeconds);
-  addRunFailureObservations(map, input.recentFailures);
-  addGroupObservations(map, input.groups);
-  return sortAlerts([...map.values()]);
-}
-
-function filterAlerts(alerts: readonly Alert[], filters: AlertFilters): Alert[] {
-  return alerts.filter((alert) => alertMatchesFilters(alert, filters));
-}
-
-export async function listAlerts(options: ListAlertsOptions = {}): Promise<AlertListResponse> {
-  const limit = clampAlertLimit(options.limit);
-  const scanLimit = clampAlertScanLimit(options.scanLimit);
-  const logPage = await listLogsCursorPage({
-    limit: scanLimit,
-    includeBodies: false,
-    anchor: "newest",
-  });
-  const config = getConfig();
-  const alerts = collectAlertsFromEvidence({
-    logs: logPage.logs,
-    recentFailures: listRecentFailures(scanLimit).failures,
-    groups: listGroups(),
-    slowResponseThresholdSeconds: config.slowResponseThresholdSeconds,
-  });
-  const filtered = filterAlerts(alerts, options);
-  return {
-    alerts: filtered.slice(0, limit),
-    total: filtered.length,
-    limit,
-    summary: buildAlertSummary(filtered),
-  };
-}
-
-export async function getAlertSummary(
-  options: Pick<ListAlertsOptions, "scanLimit"> = {},
-): Promise<AlertSummary> {
-  const response = await listAlerts({
-    limit: MAX_ALERT_LIMIT,
-    scanLimit: options.scanLimit,
-  });
-  return response.summary;
-}