@oneuptime/common 11.5.3 → 11.5.5

package/build/dist/Server/Utils/AI/Eval/EvalCorpus.js

@@ -0,0 +1,245 @@
+var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+};
+var __metadata = (this && this.__metadata) || function (k, v) {
+    if (typeof Reflect === "object" && typeof Reflect.metadata === "function") return Reflect.metadata(k, v);
+};
+import SortOrder from "../../../../Types/BaseDatabase/SortOrder";
+import LIMIT_MAX from "../../../../Types/Database/LimitMax";
+import AIRunType from "../../../../Types/AI/AIRunType";
+import AIRunStatus from "../../../../Types/AI/AIRunStatus";
+import AIRunEventType from "../../../../Types/AI/AIRunEventType";
+import { IncidentFeedEventType } from "../../../../Models/DatabaseModels/IncidentFeed";
+import { AlertFeedEventType } from "../../../../Models/DatabaseModels/AlertFeed";
+import QueryHelper from "../../../Types/Database/QueryHelper";
+import AIRunService from "../../../Services/AIRunService";
+import AIRunEventService from "../../../Services/AIRunEventService";
+import IncidentFeedService from "../../../Services/IncidentFeedService";
+import AlertFeedService from "../../../Services/AlertFeedService";
+import AIConfidenceSignal from "../SRE/ConfidenceSignal";
+import logger from "../../Logger";
+import CaptureSpan from "../../Telemetry/CaptureSpan";
+/*
+ * AI eval harness — golden-incident corpus (G3 bootstrap).
+ *
+ * A GoldenCase is one LABELED, completed AI investigation, exported as
+ * plain JSON: the run's identity, its human/auto label (AIRun.humanVerdict /
+ * AIRun.autoGrade — the Phase 2 measurement columns), deterministic per-run
+ * evidence stats derived from the recorded AIRunEvent trail, and the trail
+ * itself in skeletal form. Everything here is computable WITHOUT re-running
+ * any LLM — that is the point of the bootstrap: recorded runs + labels make
+ * the four G3 scores (EvalScores.ts) measurable today.
+ *
+ * WHAT THE TRAIL DOES AND DOES NOT CONTAIN (consequence for replay):
+ * AIRunEvent stores the validated tool ARGUMENTS (`toolArguments`) and a
+ * result SUMMARY (`resultSummary`: rowCount, durationInMs, isTruncated,
+ * bytesSentToLlm, errorMessage) — never the tool's actual result rows. So the
+ * corpus supports labeled-run SCORING (this file + EvalScores.ts) but not
+ * offline REPLAY; see ReplayInvestigation.ts for exactly what full replay
+ * would need.
+ *
+ * "Data-bearing" is defined ONCE, in
+ * AIConfidenceSignal.isDataBearingRowCount (rowCount > 0; rowCount 0 is
+ * proof of absence — evidence, but not data-bearing), and reused here so the
+ * offline stats can never drift from the live confidence floor's definition.
+ */
+// Default number of labeled runs exported when the caller does not say.
+export const DEFAULT_CORPUS_LIMIT = 200;
+export default class EvalCorpus {
+    /*
+     * Derive the deterministic evidence stats from a run's recorded event trail
+     * (pure, exported for tests). Definitions, kept consistent with the live
+     * confidence floor:
+     *   - citationsMinted: events carrying a citationId — the engine mints one
+     *     citation per successful tool call, server-side, from validated args.
+     *   - toolCallsTotal / toolCallsFailed: ToolCallCompleted + ToolCallFailed
+     *     events. ToolCallStarted is deliberately NOT counted — a call that
+     *     never finished (pod death mid-call) has no outcome to score.
+     *   - dataBearingToolCalls: ToolCallCompleted events whose
+     *     resultSummary.rowCount passes isDataBearingRowCount — the SAME helper
+     *     the confidence floor applies to citation rowCounts.
+     */
+    static deriveConfidenceStats(events) {
+        var _a;
+        let citationsMinted = 0;
+        let dataBearingToolCalls = 0;
+        let toolCallsTotal = 0;
+        let toolCallsFailed = 0;
+        for (const event of events) {
+            if (event.citationId) {
+                citationsMinted++;
+            }
+            if (event.eventType === AIRunEventType.ToolCallCompleted) {
+                toolCallsTotal++;
+                if (AIConfidenceSignal.isDataBearingRowCount((_a = event.resultSummary) === null || _a === void 0 ? void 0 : _a.rowCount)) {
+                    dataBearingToolCalls++;
+                }
+            }
+            else if (event.eventType === AIRunEventType.ToolCallFailed) {
+                toolCallsTotal++;
+                toolCallsFailed++;
+            }
+        }
+        return {
+            citationsMinted,
+            dataBearingToolCalls,
+            toolCallsTotal,
+            toolCallsFailed,
+        };
+    }
+    // Skeletal event projection for the exported JSON (pure, exported for tests).
+    static toGoldenCaseEvents(events) {
+        return events.map((event) => {
+            let ok = undefined;
+            if (event.eventType === AIRunEventType.ToolCallCompleted) {
+                ok = true;
+            }
+            else if (event.eventType === AIRunEventType.ToolCallFailed) {
+                ok = false;
+            }
+            return {
+                sequence: event.sequence || 0,
+                eventType: (event.eventType || "").toString(),
+                toolName: event.toolName,
+                ok,
+            };
+        });
+    }
+    /*
+     * Export the golden corpus: completed Investigation runs that HAVE a label
+     * (humanVerdict or autoGrade set — unlabeled runs carry no ground truth and
+     * are deliberately excluded), newest first, with their event trails.
+     */
+    static async exportCorpus(options) {
+        var _a;
+        const limit = Math.min(options.limit || DEFAULT_CORPUS_LIMIT, LIMIT_MAX);
+        const runSelect = {
+            _id: true,
+            projectId: true,
+            completedAt: true,
+            humanVerdict: true,
+            autoGrade: true,
+            triggeredByIncidentId: true,
+            triggeredByAlertId: true,
+        };
+        const baseQuery = Object.assign({ runType: AIRunType.Investigation, status: AIRunStatus.Completed }, (options.projectId ? { projectId: options.projectId } : {}));
+        /*
+         * "Has a label" is an OR (humanVerdict set OR autoGrade set); the query
+         * layer composes ANDs, so run the two legs separately and union by id.
+         */
+        const [humanLabeled, autoGraded] = await Promise.all([
+            AIRunService.findBy({
+                query: Object.assign(Object.assign({}, baseQuery), { humanVerdict: QueryHelper.notNull() }),
+                select: runSelect,
+                sort: { completedAt: SortOrder.Descending },
+                limit,
+                skip: 0,
+                props: { isRoot: true },
+            }),
+            AIRunService.findBy({
+                query: Object.assign(Object.assign({}, baseQuery), { autoGrade: QueryHelper.notNull() }),
+                select: runSelect,
+                sort: { completedAt: SortOrder.Descending },
+                limit,
+                skip: 0,
+                props: { isRoot: true },
+            }),
+        ]);
+        const runsById = new Map();
+        for (const run of [...humanLabeled, ...autoGraded]) {
+            if (run.id) {
+                runsById.set(run.id.toString(), run);
+            }
+        }
+        const labeledRuns = Array.from(runsById.values())
+            .sort((a, b) => {
+            return ((b.completedAt ? b.completedAt.getTime() : 0) -
+                (a.completedAt ? a.completedAt.getTime() : 0));
+        })
+            .slice(0, limit);
+        const goldenCases = [];
+        for (const run of labeledRuns) {
+            const subjectType = this.getSubjectType(run);
+            if (!subjectType) {
+                // An Investigation run always has a subject; be defensive anyway.
+                logger.debug(`Eval corpus: run ${(_a = run.id) === null || _a === void 0 ? void 0 : _a.toString()} has no incident/alert subject — skipped.`);
+                continue;
+            }
+            const events = await AIRunEventService.findBy({
+                query: { aiRunId: run.id },
+                select: {
+                    sequence: true,
+                    eventType: true,
+                    toolName: true,
+                    citationId: true,
+                    resultSummary: true,
+                },
+                sort: { sequence: SortOrder.Ascending },
+                limit: LIMIT_MAX,
+                skip: 0,
+                props: { isRoot: true },
+            });
+            goldenCases.push({
+                runId: run.id.toString(),
+                projectId: run.projectId.toString(),
+                subjectType,
+                completedAt: run.completedAt
+                    ? new Date(run.completedAt).toISOString()
+                    : undefined,
+                label: {
+                    humanVerdict: run.humanVerdict,
+                    autoGrade: run.autoGrade,
+                },
+                confidence: this.deriveConfidenceStats(events),
+                analysisPosted: await this.hasPostedAnalysis(run, subjectType),
+                events: this.toGoldenCaseEvents(events),
+            });
+        }
+        return goldenCases;
+    }
+    static getSubjectType(run) {
+        if (run.triggeredByIncidentId) {
+            return "Incident";
+        }
+        if (run.triggeredByAlertId) {
+            return "Alert";
+        }
+        return null;
+    }
+    /*
+     * Whether the AI's analysis was posted for the run's subject: a RootCause
+     * feed item exists (postAnalysis is its only writer — the sourcing rule
+     * shared with InvestigationGrader). Subject-level, not run-level; see the
+     * GoldenCase.analysisPosted doc comment.
+     */
+    static async hasPostedAnalysis(run, subjectType) {
+        if (subjectType === "Incident") {
+            const count = (await IncidentFeedService.countBy({
+                query: {
+                    incidentId: run.triggeredByIncidentId,
+                    incidentFeedEventType: IncidentFeedEventType.RootCause,
+                },
+                props: { isRoot: true },
+            })).toNumber();
+            return count > 0;
+        }
+        const count = (await AlertFeedService.countBy({
+            query: {
+                alertId: run.triggeredByAlertId,
+                alertFeedEventType: AlertFeedEventType.RootCause,
+            },
+            props: { isRoot: true },
+        })).toNumber();
+        return count > 0;
+    }
+}
+__decorate([
+    CaptureSpan(),
+    __metadata("design:type", Function),
+    __metadata("design:paramtypes", [Object]),
+    __metadata("design:returntype", Promise)
+], EvalCorpus, "exportCorpus", null);
+//# sourceMappingURL=EvalCorpus.js.map

package/build/dist/Server/Utils/AI/Eval/EvalCorpus.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"EvalCorpus.js","sourceRoot":"","sources":["../../../../../../Server/Utils/AI/Eval/EvalCorpus.ts"],"names":[],"mappings":";;;;;;;;;AACA,OAAO,SAAS,MAAM,0CAA0C,CAAC;AACjE,OAAO,SAAS,MAAM,qCAAqC,CAAC;AAC5D,OAAO,SAAS,MAAM,gCAAgC,CAAC;AACvD,OAAO,WAAW,MAAM,kCAAkC,CAAC;AAC3D,OAAO,cAAc,MAAM,qCAAqC,CAAC;AAMjE,OAAO,EAAE,qBAAqB,EAAE,MAAM,gDAAgD,CAAC;AACvF,OAAO,EAAE,kBAAkB,EAAE,MAAM,6CAA6C,CAAC;AACjF,OAAO,WAAW,MAAM,qCAAqC,CAAC;AAE9D,OAAO,YAAY,MAAM,gCAAgC,CAAC;AAC1D,OAAO,iBAAiB,MAAM,qCAAqC,CAAC;AACpE,OAAO,mBAAmB,MAAM,uCAAuC,CAAC;AACxE,OAAO,gBAAgB,MAAM,oCAAoC,CAAC;AAClE,OAAO,kBAAkB,MAAM,yBAAyB,CAAC;AACzD,OAAO,MAAM,MAAM,cAAc,CAAC;AAClC,OAAO,WAAW,MAAM,6BAA6B,CAAC;AAEtD;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH,wEAAwE;AACxE,MAAM,CAAC,MAAM,oBAAoB,GAAW,GAAG,CAAC;AAsEhD,MAAM,CAAC,OAAO,OAAO,UAAU;IAC7B;;;;;;;;;;;;OAYG;IACI,MAAM,CAAC,qBAAqB,CACjC,MAA+B;;QAE/B,IAAI,eAAe,GAAW,CAAC,CAAC;QAChC,IAAI,oBAAoB,GAAW,CAAC,CAAC;QACrC,IAAI,cAAc,GAAW,CAAC,CAAC;QAC/B,IAAI,eAAe,GAAW,CAAC,CAAC;QAEhC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;gBACrB,eAAe,EAAE,CAAC;YACpB,CAAC;YAED,IAAI,KAAK,CAAC,SAAS,KAAK,cAAc,CAAC,iBAAiB,EAAE,CAAC;gBACzD,cAAc,EAAE,CAAC;gBAEjB,IACE,kBAAkB,CAAC,qBAAqB,CACtC,MAAA,KAAK,CAAC,aAAa,0CAAE,QAAQ,CAC9B,EACD,CAAC;oBACD,oBAAoB,EAAE,CAAC;gBACzB,CAAC;YACH,CAAC;iBAAM,IAAI,KAAK,CAAC,SAAS,KAAK,cAAc,CAAC,cAAc,EAAE,CAAC;gBAC7D,cAAc,EAAE,CAAC;gBACjB,eAAe,EAAE,CAAC;YACpB,CAAC;QACH,CAAC;QAED,OAAO;YACL,eAAe;YACf,oBAAoB;YACpB,cAAc;YACd,eAAe;SAChB,CAAC;IACJ,CAAC;IAED,8EAA8E;IACvE,MAAM,CAAC,kBAAkB,CAC9B,MAA+B;QAE/B,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAuB,EAAE,EAAE;YAC5C,IAAI,EAAE,GAAwB,SAAS,CAAC;YAExC,IAAI,KAAK,CAAC,SAAS,KAAK,cAAc,CAAC,iBAAiB,EAAE,CAAC;gBACzD,EAAE,GAAG,IAAI,CAAC;YACZ,CAAC;iBAAM,IAAI,KAAK,CAAC,SAAS,KAAK,cAAc,CAAC,cAAc,EAAE,CAAC;gBAC7D,EAAE,GAAG,KAAK,CAAC;YACb,CAAC;YAED,OAAO;gBACL,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,CAAC;gBAC7B,SAAS,EAAE,CAAC,KAAK,CAAC,SAAS,IAAI,EAAE,CAAC,CAAC,QAAQ,EAAE;gBAC7C,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,EAAE;aACH,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC;IAED;;;;OAIG;IAEiB,AAAb,MAAM,CAAC,KAAK,CAAC,YAAY,CAC9B,OAA4B;;QAE5B,MAAM,KAAK,GAAW,IAAI,CAAC,GAAG,CAC5B,OAAO,CAAC,KAAK,IAAI,oBAAoB,EACrC,SAAS,CACV,CAAC;QAEF,MAAM,SAAS,GAQX;YACF,GAAG,EAAE,IAAI;YACT,SAAS,EAAE,IAAI;YACf,WAAW,EAAE,IAAI;YACjB,YAAY,EAAE,IAAI;YAClB,SAAS,EAAE,IAAI;YACf,qBAAqB,EAAE,IAAI;YAC3B,kBAAkB,EAAE,IAAI;SACzB,CAAC;QAEF,MAAM,SAAS,mBACb,OAAO,EAAE,SAAS,CAAC,aAAa,EAChC,MAAM,EAAE,WAAW,CAAC,SAAS,IAC1B,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,CAC/D,CAAC;QAEF;;;WAGG;QACH,MAAM,CAAC,YAAY,EAAE,UAAU,CAAC,GAC9B,MAAM,OAAO,CAAC,GAAG,CAAC;YAChB,YAAY,CAAC,MAAM,CAAC;gBAClB,KAAK,kCAAO,SAAS,KAAE,YAAY,EAAE,WAAW,CAAC,OAAO,EAAE,GAAE;gBAC5D,MAAM,EAAE,SAAS;gBACjB,IAAI,EAAE,EAAE,WAAW,EAAE,SAAS,CAAC,UAAU,EAAE;gBAC3C,KAAK;gBACL,IAAI,EAAE,CAAC;gBACP,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACxB,CAAC;YACF,YAAY,CAAC,MAAM,CAAC;gBAClB,KAAK,kCAAO,SAAS,KAAE,SAAS,EAAE,WAAW,CAAC,OAAO,EAAE,GAAE;gBACzD,MAAM,EAAE,SAAS;gBACjB,IAAI,EAAE,EAAE,WAAW,EAAE,SAAS,CAAC,UAAU,EAAE;gBAC3C,KAAK;gBACL,IAAI,EAAE,CAAC;gBACP,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACxB,CAAC;SACH,CAAC,CAAC;QAEL,MAAM,QAAQ,GAAuB,IAAI,GAAG,EAAiB,CAAC;QAE9D,KAAK,MAAM,GAAG,IAAI,CAAC,GAAG,YAAY,EAAE,GAAG,UAAU,CAAC,EAAE,CAAC;YACnD,IAAI,GAAG,CAAC,EAAE,EAAE,CAAC;gBACX,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,QAAQ,EAAE,EAAE,GAAG,CAAC,CAAC;YACvC,CAAC;QACH,CAAC;QAED,MAAM,WAAW,GAAiB,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC;aAC5D,IAAI,CAAC,CAAC,CAAQ,EAAE,CAAQ,EAAE,EAAE;YAC3B,OAAO,CACL,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC7C,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAC9C,CAAC;QACJ,CAAC,CAAC;aACD,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;QAEnB,MAAM,WAAW,GAAsB,EAAE,CAAC;QAE1C,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC9B,MAAM,WAAW,GACf,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC;YAE3B,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,kEAAkE;gBAClE,MAAM,CAAC,KAAK,CACV,oBAAoB,MAAA,GAAG,CAAC,EAAE,0CAAE,QAAQ,EAAE,2CAA2C,CAClF,CAAC;gBACF,SAAS;YACX,CAAC;YAED,MAAM,MAAM,GAAsB,MAAM,iBAAiB,CAAC,MAAM,CAAC;gBAC/D,KAAK,EAAE,EAAE,OAAO,EAAE,GAAG,CAAC,EAAG,EAAE;gBAC3B,MAAM,EAAE;oBACN,QAAQ,EAAE,IAAI;oBACd,SAAS,EAAE,IAAI;oBACf,QAAQ,EAAE,IAAI;oBACd,UAAU,EAAE,IAAI;oBAChB,aAAa,EAAE,IAAI;iBACpB;gBACD,IAAI,EAAE,EAAE,QAAQ,EAAE,SAAS,CAAC,SAAS,EAAE;gBACvC,KAAK,EAAE,SAAS;gBAChB,IAAI,EAAE,CAAC;gBACP,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACxB,CAAC,CAAC;YAEH,WAAW,CAAC,IAAI,CAAC;gBACf,KAAK,EAAE,GAAG,CAAC,EAAG,CAAC,QAAQ,EAAE;gBACzB,SAAS,EAAE,GAAG,CAAC,SAAU,CAAC,QAAQ,EAAE;gBACpC,WAAW;gBACX,WAAW,EAAE,GAAG,CAAC,WAAW;oBAC1B,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,WAAW,EAAE;oBACzC,CAAC,CAAC,SAAS;gBACb,KAAK,EAAE;oBACL,YAAY,EAAE,GAAG,CAAC,YAAY;oBAC9B,SAAS,EAAE,GAAG,CAAC,SAAS;iBACzB;gBACD,UAAU,EAAE,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC;gBAC9C,cAAc,EAAE,MAAM,IAAI,CAAC,iBAAiB,CAAC,GAAG,EAAE,WAAW,CAAC;gBAC9D,MAAM,EAAE,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC;aACxC,CAAC,CAAC;QACL,CAAC;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,MAAM,CAAC,cAAc,CAAC,GAAU;QACtC,IAAI,GAAG,CAAC,qBAAqB,EAAE,CAAC;YAC9B,OAAO,UAAU,CAAC;QACpB,CAAC;QAED,IAAI,GAAG,CAAC,kBAAkB,EAAE,CAAC;YAC3B,OAAO,OAAO,CAAC;QACjB,CAAC;QAED,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;OAKG;IACK,MAAM,CAAC,KAAK,CAAC,iBAAiB,CACpC,GAAU,EACV,WAAkC;QAElC,IAAI,WAAW,KAAK,UAAU,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAW,CACpB,MAAM,mBAAmB,CAAC,OAAO,CAAC;gBAChC,KAAK,EAAE;oBACL,UAAU,EAAE,GAAG,CAAC,qBAAsB;oBACtC,qBAAqB,EAAE,qBAAqB,CAAC,SAAS;iBACvD;gBACD,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;aACxB,CAAC,CACH,CAAC,QAAQ,EAAE,CAAC;YAEb,OAAO,KAAK,GAAG,CAAC,CAAC;QACnB,CAAC;QAED,MAAM,KAAK,GAAW,CACpB,MAAM,gBAAgB,CAAC,OAAO,CAAC;YAC7B,KAAK,EAAE;gBACL,OAAO,EAAE,GAAG,CAAC,kBAAmB;gBAChC,kBAAkB,EAAE,kBAAkB,CAAC,SAAS;aACjD;YACD,KAAK,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE;SACxB,CAAC,CACH,CAAC,QAAQ,EAAE,CAAC;QAEb,OAAO,KAAK,GAAG,CAAC,CAAC;IACnB,CAAC;CACF;AA1KqB;IADnB,WAAW,EAAE;;;;oCAyHb"}

package/build/dist/Server/Utils/AI/Eval/EvalScores.js

@@ -0,0 +1,83 @@
+import AIRunHumanVerdict from "../../../../Types/AI/AIRunHumanVerdict";
+import AIRunAutoGrade from "../../../../Types/AI/AIRunAutoGrade";
+import AIConfidenceSignal from "../SRE/ConfidenceSignal";
+/*
+ * Below this many labeled cases the numbers are noise, not measurement. The
+ * runner must surface this loudly; nothing downstream (G3 graduation, public
+ * scorecards) may consume a report whose sampleTooSmall is true.
+ */
+export const MINIMUM_TRUSTWORTHY_CORPUS_SIZE = 20;
+export default class EvalScores {
+    // Build a score with the null-when-unmeasurable rule applied.
+    static makeScore(numerator, denominator) {
+        return {
+            value: denominator === 0 ? null : numerator / denominator,
+            numerator,
+            denominator,
+        };
+    }
+    /*
+     * Would the deterministic evidence floor have flagged this run inconclusive?
+     * Reuses AIConfidenceSignal.hasVerifiableEvidence — the LIVE floor —
+     * over the corpus stats, so the offline answer can never drift from what
+     * production would actually have decided: zero citations minted AND zero
+     * data-bearing tool calls.
+     */
+    static wouldDeterministicFloorFlagInconclusive(confidence) {
+        return !AIConfidenceSignal.hasVerifiableEvidence({
+            citationCount: confidence.citationsMinted,
+            dataBearingToolCallCount: confidence.dataBearingToolCalls,
+            anyToolReturnedData: confidence.dataBearingToolCalls > 0,
+        });
+    }
+    static computeScores(cases) {
+        let gradedCount = 0;
+        let gradedMatchCount = 0;
+        let humanLabeledCount = 0;
+        let humanConfirmedCount = 0;
+        let analysisPostedCount = 0;
+        let analysisPostedWithCitationCount = 0;
+        let toolCallsTotal = 0;
+        let dataBearingToolCalls = 0;
+        let humanRejectedCount = 0;
+        let humanRejectedFloorFlaggedCount = 0;
+        for (const goldenCase of cases) {
+            if (goldenCase.label.autoGrade) {
+                gradedCount++;
+                if (goldenCase.label.autoGrade === AIRunAutoGrade.Match) {
+                    gradedMatchCount++;
+                }
+            }
+            if (goldenCase.label.humanVerdict) {
+                humanLabeledCount++;
+                if (goldenCase.label.humanVerdict === AIRunHumanVerdict.Confirmed) {
+                    humanConfirmedCount++;
+                }
+                if (goldenCase.label.humanVerdict === AIRunHumanVerdict.Rejected) {
+                    humanRejectedCount++;
+                    if (this.wouldDeterministicFloorFlagInconclusive(goldenCase.confidence)) {
+                        humanRejectedFloorFlaggedCount++;
+                    }
+                }
+            }
+            if (goldenCase.analysisPosted) {
+                analysisPostedCount++;
+                if (goldenCase.confidence.citationsMinted > 0) {
+                    analysisPostedWithCitationCount++;
+                }
+            }
+            toolCallsTotal += goldenCase.confidence.toolCallsTotal;
+            dataBearingToolCalls += goldenCase.confidence.dataBearingToolCalls;
+        }
+        return {
+            caseCount: cases.length,
+            sampleTooSmall: cases.length < MINIMUM_TRUSTWORTHY_CORPUS_SIZE,
+            topHypothesisPrecision: this.makeScore(gradedMatchCount, gradedCount),
+            humanConfirmedRate: this.makeScore(humanConfirmedCount, humanLabeledCount),
+            citationGroundingRate: this.makeScore(analysisPostedWithCitationCount, analysisPostedCount),
+            toolSelectionAccuracy: this.makeScore(dataBearingToolCalls, toolCallsTotal),
+            inconclusiveRecall: this.makeScore(humanRejectedFloorFlaggedCount, humanRejectedCount),
+        };
+    }
+}
+//# sourceMappingURL=EvalScores.js.map

package/build/dist/Server/Utils/AI/Eval/EvalScores.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"EvalScores.js","sourceRoot":"","sources":["../../../../../../Server/Utils/AI/Eval/EvalScores.ts"],"names":[],"mappings":"AAAA,OAAO,iBAAiB,MAAM,wCAAwC,CAAC;AACvE,OAAO,cAAc,MAAM,qCAAqC,CAAC;AACjE,OAAO,kBAAkB,MAAM,yBAAyB,CAAC;AA0BzD;;;;GAIG;AACH,MAAM,CAAC,MAAM,+BAA+B,GAAW,EAAE,CAAC;AAkC1D,MAAM,CAAC,OAAO,OAAO,UAAU;IAC7B,8DAA8D;IACvD,MAAM,CAAC,SAAS,CAAC,SAAiB,EAAE,WAAmB;QAC5D,OAAO;YACL,KAAK,EAAE,WAAW,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,GAAG,WAAW;YACzD,SAAS;YACT,WAAW;SACZ,CAAC;IACJ,CAAC;IAED;;;;;;OAMG;IACI,MAAM,CAAC,uCAAuC,CACnD,UAAqC;QAErC,OAAO,CAAC,kBAAkB,CAAC,qBAAqB,CAAC;YAC/C,aAAa,EAAE,UAAU,CAAC,eAAe;YACzC,wBAAwB,EAAE,UAAU,CAAC,oBAAoB;YACzD,mBAAmB,EAAE,UAAU,CAAC,oBAAoB,GAAG,CAAC;SACzD,CAAC,CAAC;IACL,CAAC;IAEM,MAAM,CAAC,aAAa,CAAC,KAAwB;QAClD,IAAI,WAAW,GAAW,CAAC,CAAC;QAC5B,IAAI,gBAAgB,GAAW,CAAC,CAAC;QAEjC,IAAI,iBAAiB,GAAW,CAAC,CAAC;QAClC,IAAI,mBAAmB,GAAW,CAAC,CAAC;QAEpC,IAAI,mBAAmB,GAAW,CAAC,CAAC;QACpC,IAAI,+BAA+B,GAAW,CAAC,CAAC;QAEhD,IAAI,cAAc,GAAW,CAAC,CAAC;QAC/B,IAAI,oBAAoB,GAAW,CAAC,CAAC;QAErC,IAAI,kBAAkB,GAAW,CAAC,CAAC;QACnC,IAAI,8BAA8B,GAAW,CAAC,CAAC;QAE/C,KAAK,MAAM,UAAU,IAAI,KAAK,EAAE,CAAC;YAC/B,IAAI,UAAU,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;gBAC/B,WAAW,EAAE,CAAC;gBAEd,IAAI,UAAU,CAAC,KAAK,CAAC,SAAS,KAAK,cAAc,CAAC,KAAK,EAAE,CAAC;oBACxD,gBAAgB,EAAE,CAAC;gBACrB,CAAC;YACH,CAAC;YAED,IAAI,UAAU,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;gBAClC,iBAAiB,EAAE,CAAC;gBAEpB,IAAI,UAAU,CAAC,KAAK,CAAC,YAAY,KAAK,iBAAiB,CAAC,SAAS,EAAE,CAAC;oBAClE,mBAAmB,EAAE,CAAC;gBACxB,CAAC;gBAED,IAAI,UAAU,CAAC,KAAK,CAAC,YAAY,KAAK,iBAAiB,CAAC,QAAQ,EAAE,CAAC;oBACjE,kBAAkB,EAAE,CAAC;oBAErB,IACE,IAAI,CAAC,uCAAuC,CAAC,UAAU,CAAC,UAAU,CAAC,EACnE,CAAC;wBACD,8BAA8B,EAAE,CAAC;oBACnC,CAAC;gBACH,CAAC;YACH,CAAC;YAED,IAAI,UAAU,CAAC,cAAc,EAAE,CAAC;gBAC9B,mBAAmB,EAAE,CAAC;gBAEtB,IAAI,UAAU,CAAC,UAAU,CAAC,eAAe,GAAG,CAAC,EAAE,CAAC;oBAC9C,+BAA+B,EAAE,CAAC;gBACpC,CAAC;YACH,CAAC;YAED,cAAc,IAAI,UAAU,CAAC,UAAU,CAAC,cAAc,CAAC;YACvD,oBAAoB,IAAI,UAAU,CAAC,UAAU,CAAC,oBAAoB,CAAC;QACrE,CAAC;QAED,OAAO;YACL,SAAS,EAAE,KAAK,CAAC,MAAM;YACvB,cAAc,EAAE,KAAK,CAAC,MAAM,GAAG,+BAA+B;YAC9D,sBAAsB,EAAE,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE,WAAW,CAAC;YACrE,kBAAkB,EAAE,IAAI,CAAC,SAAS,CAChC,mBAAmB,EACnB,iBAAiB,CAClB;YACD,qBAAqB,EAAE,IAAI,CAAC,SAAS,CACnC,+BAA+B,EAC/B,mBAAmB,CACpB;YACD,qBAAqB,EAAE,IAAI,CAAC,SAAS,CACnC,oBAAoB,EACpB,cAAc,CACf;YACD,kBAAkB,EAAE,IAAI,CAAC,SAAS,CAChC,8BAA8B,EAC9B,kBAAkB,CACnB;SACF,CAAC;IACJ,CAAC;CACF"}

package/build/dist/Server/Utils/AI/Eval/ReplayInvestigation.js

@@ -0,0 +1,12 @@
+import NotImplementedException from "../../../../Types/Exception/NotImplementedException";
+export default class ReplayInvestigation {
+    /*
+     * NOT IMPLEMENTED — see the module doc block for exactly what is missing
+     * (recorded tool RESULT payloads; today's AIRunEvent trail carries argument
+     * + summary only).
+     */
+    static async replay(_goldenCase) {
+        throw new NotImplementedException();
+    }
+}
+//# sourceMappingURL=ReplayInvestigation.js.map

package/build/dist/Server/Utils/AI/Eval/ReplayInvestigation.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"ReplayInvestigation.js","sourceRoot":"","sources":["../../../../../../Server/Utils/AI/Eval/ReplayInvestigation.ts"],"names":[],"mappings":"AAAA,OAAO,uBAAuB,MAAM,qDAAqD,CAAC;AA6E1F,MAAM,CAAC,OAAO,OAAO,mBAAmB;IACtC;;;;OAIG;IACI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,WAAuB;QAChD,MAAM,IAAI,uBAAuB,EAAE,CAAC;IACtC,CAAC;CACF"}