@oneuptime/common 11.5.4 → 11.5.6

package/Server/Utils/AI/Eval/EvalCorpus.ts

@@ -0,0 +1,369 @@
+import ObjectID from "../../../../Types/ObjectID";
+import SortOrder from "../../../../Types/BaseDatabase/SortOrder";
+import LIMIT_MAX from "../../../../Types/Database/LimitMax";
+import AIRunType from "../../../../Types/AI/AIRunType";
+import AIRunStatus from "../../../../Types/AI/AIRunStatus";
+import AIRunEventType from "../../../../Types/AI/AIRunEventType";
+import AIRunHumanVerdict from "../../../../Types/AI/AIRunHumanVerdict";
+import AIRunAutoGrade from "../../../../Types/AI/AIRunAutoGrade";
+import { AIRunEventResultSummary } from "../../../../Types/AI/AIChatTypes";
+import AIRun from "../../../../Models/DatabaseModels/AIRun";
+import AIRunEvent from "../../../../Models/DatabaseModels/AIRunEvent";
+import { IncidentFeedEventType } from "../../../../Models/DatabaseModels/IncidentFeed";
+import { AlertFeedEventType } from "../../../../Models/DatabaseModels/AlertFeed";
+import QueryHelper from "../../../Types/Database/QueryHelper";
+import Query from "../../../Types/Database/Query";
+import AIRunService from "../../../Services/AIRunService";
+import AIRunEventService from "../../../Services/AIRunEventService";
+import IncidentFeedService from "../../../Services/IncidentFeedService";
+import AlertFeedService from "../../../Services/AlertFeedService";
+import AIConfidenceSignal from "../SRE/ConfidenceSignal";
+import logger from "../../Logger";
+import CaptureSpan from "../../Telemetry/CaptureSpan";
+
+/*
+ * AI eval harness — golden-incident corpus (G3 bootstrap).
+ *
+ * A GoldenCase is one LABELED, completed AI investigation, exported as
+ * plain JSON: the run's identity, its human/auto label (AIRun.humanVerdict /
+ * AIRun.autoGrade — the Phase 2 measurement columns), deterministic per-run
+ * evidence stats derived from the recorded AIRunEvent trail, and the trail
+ * itself in skeletal form. Everything here is computable WITHOUT re-running
+ * any LLM — that is the point of the bootstrap: recorded runs + labels make
+ * the four G3 scores (EvalScores.ts) measurable today.
+ *
+ * WHAT THE TRAIL DOES AND DOES NOT CONTAIN (consequence for replay):
+ * AIRunEvent stores the validated tool ARGUMENTS (`toolArguments`) and a
+ * result SUMMARY (`resultSummary`: rowCount, durationInMs, isTruncated,
+ * bytesSentToLlm, errorMessage) — never the tool's actual result rows. So the
+ * corpus supports labeled-run SCORING (this file + EvalScores.ts) but not
+ * offline REPLAY; see ReplayInvestigation.ts for exactly what full replay
+ * would need.
+ *
+ * "Data-bearing" is defined ONCE, in
+ * AIConfidenceSignal.isDataBearingRowCount (rowCount > 0; rowCount 0 is
+ * proof of absence — evidence, but not data-bearing), and reused here so the
+ * offline stats can never drift from the live confidence floor's definition.
+ */
+
+// Default number of labeled runs exported when the caller does not say.
+export const DEFAULT_CORPUS_LIMIT: number = 200;
+
+export type GoldenCaseSubjectType = "Incident" | "Alert";
+
+export interface GoldenCaseLabel {
+  humanVerdict?: AIRunHumanVerdict | undefined;
+  autoGrade?: AIRunAutoGrade | undefined;
+}
+
+/*
+ * Deterministic per-run evidence stats, derived from the recorded event
+ * trail (server-side state — the model never wrote any of these numbers).
+ */
+export interface GoldenCaseConfidenceStats {
+  // Events that minted a server-side citation (AIRunEvent.citationId set).
+  citationsMinted: number;
+  /*
+   * Successful tool calls whose recorded resultSummary.rowCount passes
+   * AIConfidenceSignal.isDataBearingRowCount (> 0).
+   */
+  dataBearingToolCalls: number;
+  // ToolCallCompleted + ToolCallFailed events (calls that finished either way).
+  toolCallsTotal: number;
+  toolCallsFailed: number;
+}
+
+export interface GoldenCaseEvent {
+  sequence: number;
+  eventType: string;
+  toolName?: string | undefined;
+  // true = ToolCallCompleted, false = ToolCallFailed, undefined = not a tool result.
+  ok?: boolean | undefined;
+}
+
+export interface GoldenCase {
+  runId: string;
+  projectId: string;
+  subjectType: GoldenCaseSubjectType;
+  // ISO timestamp of the run's Completed transition.
+  completedAt?: string | undefined;
+  label: GoldenCaseLabel;
+  confidence: GoldenCaseConfidenceStats;
+  /*
+   * Whether a AI RootCause feed item exists for the run's subject.
+   * Honest caveat: feed items are keyed by SUBJECT, not by run — the AI's
+   * postAnalysis is their only writer (the same sourcing rule
+   * InvestigationGrader uses), but when several investigations ran for one
+   * subject they share this flag.
+   */
+  analysisPosted: boolean;
+  events: Array<GoldenCaseEvent>;
+}
+
+/*
+ * Structural projection of AIRunEvent used by the pure derivation helpers, so
+ * tests can feed synthetic event lists without constructing database models.
+ */
+export interface CorpusEventInput {
+  sequence?: number | undefined;
+  eventType?: AIRunEventType | undefined;
+  toolName?: string | undefined;
+  citationId?: string | undefined;
+  resultSummary?: AIRunEventResultSummary | undefined;
+}
+
+export interface ExportCorpusOptions {
+  projectId?: ObjectID | undefined;
+  limit?: number | undefined;
+}
+
+export default class EvalCorpus {
+  /*
+   * Derive the deterministic evidence stats from a run's recorded event trail
+   * (pure, exported for tests). Definitions, kept consistent with the live
+   * confidence floor:
+   *   - citationsMinted: events carrying a citationId — the engine mints one
+   *     citation per successful tool call, server-side, from validated args.
+   *   - toolCallsTotal / toolCallsFailed: ToolCallCompleted + ToolCallFailed
+   *     events. ToolCallStarted is deliberately NOT counted — a call that
+   *     never finished (pod death mid-call) has no outcome to score.
+   *   - dataBearingToolCalls: ToolCallCompleted events whose
+   *     resultSummary.rowCount passes isDataBearingRowCount — the SAME helper
+   *     the confidence floor applies to citation rowCounts.
+   */
+  public static deriveConfidenceStats(
+    events: Array<CorpusEventInput>,
+  ): GoldenCaseConfidenceStats {
+    let citationsMinted: number = 0;
+    let dataBearingToolCalls: number = 0;
+    let toolCallsTotal: number = 0;
+    let toolCallsFailed: number = 0;
+
+    for (const event of events) {
+      if (event.citationId) {
+        citationsMinted++;
+      }
+
+      if (event.eventType === AIRunEventType.ToolCallCompleted) {
+        toolCallsTotal++;
+
+        if (
+          AIConfidenceSignal.isDataBearingRowCount(
+            event.resultSummary?.rowCount,
+          )
+        ) {
+          dataBearingToolCalls++;
+        }
+      } else if (event.eventType === AIRunEventType.ToolCallFailed) {
+        toolCallsTotal++;
+        toolCallsFailed++;
+      }
+    }
+
+    return {
+      citationsMinted,
+      dataBearingToolCalls,
+      toolCallsTotal,
+      toolCallsFailed,
+    };
+  }
+
+  // Skeletal event projection for the exported JSON (pure, exported for tests).
+  public static toGoldenCaseEvents(
+    events: Array<CorpusEventInput>,
+  ): Array<GoldenCaseEvent> {
+    return events.map((event: CorpusEventInput) => {
+      let ok: boolean | undefined = undefined;
+
+      if (event.eventType === AIRunEventType.ToolCallCompleted) {
+        ok = true;
+      } else if (event.eventType === AIRunEventType.ToolCallFailed) {
+        ok = false;
+      }
+
+      return {
+        sequence: event.sequence || 0,
+        eventType: (event.eventType || "").toString(),
+        toolName: event.toolName,
+        ok,
+      };
+    });
+  }
+
+  /*
+   * Export the golden corpus: completed Investigation runs that HAVE a label
+   * (humanVerdict or autoGrade set — unlabeled runs carry no ground truth and
+   * are deliberately excluded), newest first, with their event trails.
+   */
+  @CaptureSpan()
+  public static async exportCorpus(
+    options: ExportCorpusOptions,
+  ): Promise<Array<GoldenCase>> {
+    const limit: number = Math.min(
+      options.limit || DEFAULT_CORPUS_LIMIT,
+      LIMIT_MAX,
+    );
+
+    const runSelect: {
+      _id: boolean;
+      projectId: boolean;
+      completedAt: boolean;
+      humanVerdict: boolean;
+      autoGrade: boolean;
+      triggeredByIncidentId: boolean;
+      triggeredByAlertId: boolean;
+    } = {
+      _id: true,
+      projectId: true,
+      completedAt: true,
+      humanVerdict: true,
+      autoGrade: true,
+      triggeredByIncidentId: true,
+      triggeredByAlertId: true,
+    };
+
+    const baseQuery: Query<AIRun> = {
+      runType: AIRunType.Investigation,
+      status: AIRunStatus.Completed,
+      ...(options.projectId ? { projectId: options.projectId } : {}),
+    };
+
+    /*
+     * "Has a label" is an OR (humanVerdict set OR autoGrade set); the query
+     * layer composes ANDs, so run the two legs separately and union by id.
+     */
+    const [humanLabeled, autoGraded]: [Array<AIRun>, Array<AIRun>] =
+      await Promise.all([
+        AIRunService.findBy({
+          query: { ...baseQuery, humanVerdict: QueryHelper.notNull() },
+          select: runSelect,
+          sort: { completedAt: SortOrder.Descending },
+          limit,
+          skip: 0,
+          props: { isRoot: true },
+        }),
+        AIRunService.findBy({
+          query: { ...baseQuery, autoGrade: QueryHelper.notNull() },
+          select: runSelect,
+          sort: { completedAt: SortOrder.Descending },
+          limit,
+          skip: 0,
+          props: { isRoot: true },
+        }),
+      ]);
+
+    const runsById: Map<string, AIRun> = new Map<string, AIRun>();
+
+    for (const run of [...humanLabeled, ...autoGraded]) {
+      if (run.id) {
+        runsById.set(run.id.toString(), run);
+      }
+    }
+
+    const labeledRuns: Array<AIRun> = Array.from(runsById.values())
+      .sort((a: AIRun, b: AIRun) => {
+        return (
+          (b.completedAt ? b.completedAt.getTime() : 0) -
+          (a.completedAt ? a.completedAt.getTime() : 0)
+        );
+      })
+      .slice(0, limit);
+
+    const goldenCases: Array<GoldenCase> = [];
+
+    for (const run of labeledRuns) {
+      const subjectType: GoldenCaseSubjectType | null =
+        this.getSubjectType(run);
+
+      if (!subjectType) {
+        // An Investigation run always has a subject; be defensive anyway.
+        logger.debug(
+          `Eval corpus: run ${run.id?.toString()} has no incident/alert subject — skipped.`,
+        );
+        continue;
+      }
+
+      const events: Array<AIRunEvent> = await AIRunEventService.findBy({
+        query: { aiRunId: run.id! },
+        select: {
+          sequence: true,
+          eventType: true,
+          toolName: true,
+          citationId: true,
+          resultSummary: true,
+        },
+        sort: { sequence: SortOrder.Ascending },
+        limit: LIMIT_MAX,
+        skip: 0,
+        props: { isRoot: true },
+      });
+
+      goldenCases.push({
+        runId: run.id!.toString(),
+        projectId: run.projectId!.toString(),
+        subjectType,
+        completedAt: run.completedAt
+          ? new Date(run.completedAt).toISOString()
+          : undefined,
+        label: {
+          humanVerdict: run.humanVerdict,
+          autoGrade: run.autoGrade,
+        },
+        confidence: this.deriveConfidenceStats(events),
+        analysisPosted: await this.hasPostedAnalysis(run, subjectType),
+        events: this.toGoldenCaseEvents(events),
+      });
+    }
+
+    return goldenCases;
+  }
+
+  private static getSubjectType(run: AIRun): GoldenCaseSubjectType | null {
+    if (run.triggeredByIncidentId) {
+      return "Incident";
+    }
+
+    if (run.triggeredByAlertId) {
+      return "Alert";
+    }
+
+    return null;
+  }
+
+  /*
+   * Whether the AI's analysis was posted for the run's subject: a RootCause
+   * feed item exists (postAnalysis is its only writer — the sourcing rule
+   * shared with InvestigationGrader). Subject-level, not run-level; see the
+   * GoldenCase.analysisPosted doc comment.
+   */
+  private static async hasPostedAnalysis(
+    run: AIRun,
+    subjectType: GoldenCaseSubjectType,
+  ): Promise<boolean> {
+    if (subjectType === "Incident") {
+      const count: number = (
+        await IncidentFeedService.countBy({
+          query: {
+            incidentId: run.triggeredByIncidentId!,
+            incidentFeedEventType: IncidentFeedEventType.RootCause,
+          },
+          props: { isRoot: true },
+        })
+      ).toNumber();
+
+      return count > 0;
+    }
+
+    const count: number = (
+      await AlertFeedService.countBy({
+        query: {
+          alertId: run.triggeredByAlertId!,
+          alertFeedEventType: AlertFeedEventType.RootCause,
+        },
+        props: { isRoot: true },
+      })
+    ).toNumber();
+
+    return count > 0;
+  }
+}

package/Server/Utils/AI/Eval/EvalScores.ts

@@ -0,0 +1,172 @@
+import AIRunHumanVerdict from "../../../../Types/AI/AIRunHumanVerdict";
+import AIRunAutoGrade from "../../../../Types/AI/AIRunAutoGrade";
+import AIConfidenceSignal from "../SRE/ConfidenceSignal";
+import { GoldenCase, GoldenCaseConfidenceStats } from "./EvalCorpus";
+
+/*
+ * AI eval harness — scoring (G3 bootstrap). Pure: given a GoldenCase
+ * corpus (EvalCorpus.ts), compute the four roadmap scores from recorded runs
+ * and their labels — no LLM is re-run, no database is touched.
+ *
+ * HONESTY RULE (same as the fix-PR acceptance-rate work): every score is
+ * {value, numerator, denominator} and value is NULL when the denominator is
+ * zero. A score that cannot be measured reports "not measurable", never a
+ * fake 0% or 100%.
+ *
+ * The two accuracy signals are deliberately NOT blended: autoGrade (the
+ * on-resolve LLM comparison against Incident.rootCause) and humanVerdict (the
+ * one-click panel judgment) measure different things with different biases —
+ * report both, let the reader weigh them.
+ */
+
+export interface EvalScore {
+  // numerator / denominator, or null when denominator is 0 (not measurable).
+  value: number | null;
+  numerator: number;
+  denominator: number;
+}
+
+/*
+ * Below this many labeled cases the numbers are noise, not measurement. The
+ * runner must surface this loudly; nothing downstream (G3 graduation, public
+ * scorecards) may consume a report whose sampleTooSmall is true.
+ */
+export const MINIMUM_TRUSTWORTHY_CORPUS_SIZE: number = 20;
+
+export interface EvalScoreReport {
+  caseCount: number;
+  sampleTooSmall: boolean;
+  /*
+   * Of the auto-graded cases: fraction graded Match.
+   * numerator = autoGrade === Match; denominator = autoGrade set (any value).
+   */
+  topHypothesisPrecision: EvalScore;
+  /*
+   * Of the human-labeled cases: fraction Confirmed. Reported separately from
+   * topHypothesisPrecision — never blended.
+   */
+  humanConfirmedRate: EvalScore;
+  /*
+   * Of the cases whose analysis was posted: fraction with at least one
+   * server-minted citation backing it.
+   */
+  citationGroundingRate: EvalScore;
+  /*
+   * Of ALL finished tool calls across the corpus (aggregated, not
+   * mean-of-per-run-means — small runs must not dominate): fraction that
+   * returned data.
+   */
+  toolSelectionAccuracy: EvalScore;
+  /*
+   * Of the human-REJECTED cases: fraction the deterministic evidence floor
+   * (ConfidenceSignal) would have flagged inconclusive on its own — i.e. the
+   * floor's recall of bad analyses, measured against human ground truth.
+   */
+  inconclusiveRecall: EvalScore;
+}
+
+export default class EvalScores {
+  // Build a score with the null-when-unmeasurable rule applied.
+  public static makeScore(numerator: number, denominator: number): EvalScore {
+    return {
+      value: denominator === 0 ? null : numerator / denominator,
+      numerator,
+      denominator,
+    };
+  }
+
+  /*
+   * Would the deterministic evidence floor have flagged this run inconclusive?
+   * Reuses AIConfidenceSignal.hasVerifiableEvidence — the LIVE floor —
+   * over the corpus stats, so the offline answer can never drift from what
+   * production would actually have decided: zero citations minted AND zero
+   * data-bearing tool calls.
+   */
+  public static wouldDeterministicFloorFlagInconclusive(
+    confidence: GoldenCaseConfidenceStats,
+  ): boolean {
+    return !AIConfidenceSignal.hasVerifiableEvidence({
+      citationCount: confidence.citationsMinted,
+      dataBearingToolCallCount: confidence.dataBearingToolCalls,
+      anyToolReturnedData: confidence.dataBearingToolCalls > 0,
+    });
+  }
+
+  public static computeScores(cases: Array<GoldenCase>): EvalScoreReport {
+    let gradedCount: number = 0;
+    let gradedMatchCount: number = 0;
+
+    let humanLabeledCount: number = 0;
+    let humanConfirmedCount: number = 0;
+
+    let analysisPostedCount: number = 0;
+    let analysisPostedWithCitationCount: number = 0;
+
+    let toolCallsTotal: number = 0;
+    let dataBearingToolCalls: number = 0;
+
+    let humanRejectedCount: number = 0;
+    let humanRejectedFloorFlaggedCount: number = 0;
+
+    for (const goldenCase of cases) {
+      if (goldenCase.label.autoGrade) {
+        gradedCount++;
+
+        if (goldenCase.label.autoGrade === AIRunAutoGrade.Match) {
+          gradedMatchCount++;
+        }
+      }
+
+      if (goldenCase.label.humanVerdict) {
+        humanLabeledCount++;
+
+        if (goldenCase.label.humanVerdict === AIRunHumanVerdict.Confirmed) {
+          humanConfirmedCount++;
+        }
+
+        if (goldenCase.label.humanVerdict === AIRunHumanVerdict.Rejected) {
+          humanRejectedCount++;
+
+          if (
+            this.wouldDeterministicFloorFlagInconclusive(goldenCase.confidence)
+          ) {
+            humanRejectedFloorFlaggedCount++;
+          }
+        }
+      }
+
+      if (goldenCase.analysisPosted) {
+        analysisPostedCount++;
+
+        if (goldenCase.confidence.citationsMinted > 0) {
+          analysisPostedWithCitationCount++;
+        }
+      }
+
+      toolCallsTotal += goldenCase.confidence.toolCallsTotal;
+      dataBearingToolCalls += goldenCase.confidence.dataBearingToolCalls;
+    }
+
+    return {
+      caseCount: cases.length,
+      sampleTooSmall: cases.length < MINIMUM_TRUSTWORTHY_CORPUS_SIZE,
+      topHypothesisPrecision: this.makeScore(gradedMatchCount, gradedCount),
+      humanConfirmedRate: this.makeScore(
+        humanConfirmedCount,
+        humanLabeledCount,
+      ),
+      citationGroundingRate: this.makeScore(
+        analysisPostedWithCitationCount,
+        analysisPostedCount,
+      ),
+      toolSelectionAccuracy: this.makeScore(
+        dataBearingToolCalls,
+        toolCallsTotal,
+      ),
+      inconclusiveRecall: this.makeScore(
+        humanRejectedFloorFlaggedCount,
+        humanRejectedCount,
+      ),
+    };
+  }
+}

package/Server/Utils/AI/Eval/ReplayInvestigation.ts

@@ -0,0 +1,87 @@
+import NotImplementedException from "../../../../Types/Exception/NotImplementedException";
+import { AIChatCitation } from "../../../../Types/AI/AIChatTypes";
+import { GoldenCase } from "./EvalCorpus";
+
+/*
+ * AI eval harness — OFFLINE REPLAY (not implemented; this file is the
+ * explicit extension point, so the gap is documented instead of pretended
+ * away).
+ *
+ * WHAT REPLAY MEANS HERE: re-run the investigation agent loop against the
+ * RECORDED tool results of a golden case, so a prompt/persona/model change can
+ * be scored on the exact same evidence the original run saw — deterministic
+ * inputs, no live telemetry, no flakiness from data that has since expired.
+ *
+ * WHY IT CANNOT BE BUILT FROM TODAY'S RECORDED TRAILS (verified against
+ * AIInvestigationEngine.emitEvent + ChatAgentRunner.emitEvent):
+ * AIRunEvent persists the validated tool ARGUMENTS (`toolArguments`) and a
+ * result SUMMARY only — `resultSummary` carries rowCount, durationInMs,
+ * isTruncated, bytesSentToLlm and errorMessage (AIRunEventResultSummary). The
+ * actual result payload the LLM read (the serialized, redacted, truncated
+ * tool output) is never persisted anywhere: not on the event, not on the run,
+ * not in LlmLog (investigations set storeContentPreviews: false by design —
+ * G8). We know HOW MANY bytes each tool sent to the LLM, but not WHICH bytes.
+ *
+ * CONSEQUENCE: a "replay" today could only re-EXECUTE the recorded
+ * `toolArguments` against live telemetry (they are replayable by construction
+ * — citations carry them for exactly that reason). That is a re-query, not a
+ * replay: ClickHouse retention windows (metrics default to days, not months)
+ * mean the data behind a weeks-old golden case is usually gone, and even
+ * fresh data may have changed. Scoring a model change on re-queried data is
+ * NOT scoring it on the golden corpus.
+ *
+ * WHAT FULL REPLAY NEEDS (in dependency order):
+ *
+ *   1. RECORD TOOL RESULTS: persist the serialized (redacted + truncated —
+ *      the exact text handed to the LLM, AFTER Toolbox/Serializer.ts ran)
+ *      result payload per ToolCallCompleted event. Design constraints: G8
+ *      applies (these payloads embed telemetry with narrower ACLs than the
+ *      event table's project-member read — they need the same access-tiering
+ *      review as LlmLog previews, plus a retention policy); size is bounded
+ *      by the serializer's existing truncation caps, so a JSON/text column on
+ *      AIRunEvent or a side table both work.
+ *
+ *   2. A RECORDED-RESULT TOOL TRANSPORT: a Toolbox-shaped shim the agent loop
+ *      can run against, serving recorded results keyed by (toolName,
+ *      canonicalized arguments) and falling to an explicit "not recorded"
+ *      tool error when the replayed model asks something the original never
+ *      asked. That divergence is itself signal — count it, don't hide it.
+ *
+ *   3. PINNED MODEL CONFIG + SCORING HOOKUP: replay with an explicit
+ *      provider/model/temperature (otherwise scores measure provider drift,
+ *      not the change under test), then feed the replayed analyses through
+ *      the same labels/scores as the recorded runs (EvalScores.ts) so
+ *      before/after is apples-to-apples.
+ *
+ * Until (1) exists for newly recorded runs, no historical run can ever be
+ * replayed — recording tool results is the prerequisite that only pays off
+ * for runs recorded AFTER it ships. That is why the corpus/scoring bootstrap
+ * shipped first: labels make recorded runs scorable today, replay makes
+ * FUTURE changes testable against them tomorrow.
+ */
+
+// What a completed replay would produce for scoring.
+export interface ReplayResult {
+  runId: string;
+  // The replayed agent's analysis, ready for grading against the case labels.
+  analysisMarkdown: string;
+  // Citations minted during the replay (server-side, as in live runs).
+  citations: Array<AIChatCitation>;
+  /*
+   * Tool calls the replayed model made that the original run never recorded —
+   * replay cannot answer them; a high count means the replayed prompt/model
+   * diverged too far from the recording for scores to be comparable.
+   */
+  unrecordedToolCallCount: number;
+}
+
+export default class ReplayInvestigation {
+  /*
+   * NOT IMPLEMENTED — see the module doc block for exactly what is missing
+   * (recorded tool RESULT payloads; today's AIRunEvent trail carries argument
+   * + summary only).
+   */
+  public static async replay(_goldenCase: GoldenCase): Promise<ReplayResult> {
+    throw new NotImplementedException();
+  }
+}