@resolveio/server-lib 22.3.221 → 22.3.222

package/util/ai-run-evidence-dashboard.d.ts

@@ -0,0 +1,88 @@
+import { AIRun, AIRunCost } from './ai-run-evidence';
+import { buildAIRunTrainingExampleFromRun } from './ai-run-evidence-eval';
+export interface AIRunTimelineItem {
+    time?: string;
+    type: string;
+    label: string;
+    status?: string;
+    message?: string;
+    evidenceRefs: string[];
+    metadata?: Record<string, any>;
+}
+export interface AIRunQaHealth {
+    outcome?: string;
+    infra: {
+        pass: number;
+        fail: number;
+        blocked: number;
+        warn: number;
+        unknown: number;
+    };
+    compile?: {
+        status?: string;
+        stale?: boolean;
+        artifactPath?: string;
+        message?: string;
+    };
+    route: {
+        pass: number;
+        fail: number;
+        shellOnly: number;
+    };
+    business: {
+        pass: number;
+        fail: number;
+        blocked: number;
+        missing: boolean;
+    };
+}
+export interface AIRunCostPanel {
+    totalTokens: number;
+    estimatedUsd: number;
+    models: string[];
+    categories: Array<{
+        category: string;
+        totalTokens: number;
+        estimatedUsd: number;
+        count: number;
+    }>;
+    sources?: Array<{
+        source: string;
+        totalTokens: number;
+        estimatedUsd: number;
+        count: number;
+        manualCount: number;
+        untrackedCount: number;
+    }>;
+    surfaces?: Array<{
+        surface: string;
+        totalTokens: number;
+        estimatedUsd: number;
+        count: number;
+    }>;
+    manualEstimatedUsd?: number;
+    untrackedEstimatedUsd?: number;
+    untrackedWarnings?: string[];
+    observabilityStatus?: string;
+    missingLedger?: boolean;
+    externalCodexMonitorUsed?: boolean;
+    nextAction?: string;
+}
+export interface AIRunOperatorView {
+    runId?: string;
+    source: string;
+    title?: string;
+    outcome: string;
+    failureClass: string;
+    nextAction?: string;
+    warnings: string[];
+    timeline: AIRunTimelineItem[];
+    qaHealth: AIRunQaHealth;
+    costPanel: AIRunCostPanel;
+    trainingExample: ReturnType<typeof buildAIRunTrainingExampleFromRun>;
+}
+export declare function buildAIRunTimeline(run: AIRun): AIRunTimelineItem[];
+export declare function buildAIRunQaHealth(run: AIRun): AIRunQaHealth;
+export declare function buildAIRunCostPanel(cost?: AIRunCost, run?: AIRun): AIRunCostPanel;
+export declare function buildAIRunFalsePassWarnings(run: AIRun): string[];
+export declare function buildAIRunOperatorView(run: AIRun): AIRunOperatorView;

package/util/ai-run-evidence-dashboard.js

@@ -0,0 +1,343 @@
+"use strict";
+var __values = (this && this.__values) || function(o) {
+    var s = typeof Symbol === "function" && Symbol.iterator, m = s && o[s], i = 0;
+    if (m) return m.call(o);
+    if (o && typeof o.length === "number") return {
+        next: function () {
+            if (o && i >= o.length) o = void 0;
+            return { value: o && o[i++], done: !o };
+        }
+    };
+    throw new TypeError(s ? "Object is not iterable." : "Symbol.iterator is not defined.");
+};
+var __read = (this && this.__read) || function (o, n) {
+    var m = typeof Symbol === "function" && o[Symbol.iterator];
+    if (!m) return o;
+    var i = m.call(o), r, ar = [], e;
+    try {
+        while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
+    }
+    catch (error) { e = { error: error }; }
+    finally {
+        try {
+            if (r && !r.done && (m = i["return"])) m.call(i);
+        }
+        finally { if (e) throw e.error; }
+    }
+    return ar;
+};
+var __spreadArray = (this && this.__spreadArray) || function (to, from, pack) {
+    if (pack || arguments.length === 2) for (var i = 0, l = from.length, ar; i < l; i++) {
+        if (ar || !(i in from)) {
+            if (!ar) ar = Array.prototype.slice.call(from, 0, i);
+            ar[i] = from[i];
+        }
+    }
+    return to.concat(ar || Array.prototype.slice.call(from));
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildAIRunTimeline = buildAIRunTimeline;
+exports.buildAIRunQaHealth = buildAIRunQaHealth;
+exports.buildAIRunCostPanel = buildAIRunCostPanel;
+exports.buildAIRunFalsePassWarnings = buildAIRunFalsePassWarnings;
+exports.buildAIRunOperatorView = buildAIRunOperatorView;
+var ai_run_evidence_eval_1 = require("./ai-run-evidence-eval");
+function cleanText(value, max) {
+    if (max === void 0) { max = 1000; }
+    return String(value || '').replace(/\s+/g, ' ').trim().slice(0, max);
+}
+function toIso(value) {
+    if (value instanceof Date && Number.isFinite(value.getTime())) {
+        return value.toISOString();
+    }
+    if (value) {
+        var parsed = new Date(value);
+        if (Number.isFinite(parsed.getTime())) {
+            return parsed.toISOString();
+        }
+    }
+    return undefined;
+}
+function countStatuses(values) {
+    var e_1, _a;
+    var result = {
+        pass: 0,
+        fail: 0,
+        blocked: 0,
+        warn: 0,
+        unknown: 0
+    };
+    try {
+        for (var values_1 = __values(values), values_1_1 = values_1.next(); !values_1_1.done; values_1_1 = values_1.next()) {
+            var value = values_1_1.value;
+            var status_1 = cleanText(value.status, 40).toLowerCase();
+            if (status_1 === 'pass' || status_1 === 'fail' || status_1 === 'blocked' || status_1 === 'warn') {
+                result[status_1] += 1;
+            }
+            else {
+                result.unknown += 1;
+            }
+        }
+    }
+    catch (e_1_1) { e_1 = { error: e_1_1 }; }
+    finally {
+        try {
+            if (values_1_1 && !values_1_1.done && (_a = values_1.return)) _a.call(values_1);
+        }
+        finally { if (e_1) throw e_1.error; }
+    }
+    return result;
+}
+function sortTimeline(items) {
+    return __spreadArray([], __read(items), false).sort(function (left, right) {
+        var leftTime = left.time ? new Date(left.time).getTime() : 0;
+        var rightTime = right.time ? new Date(right.time).getTime() : 0;
+        return leftTime - rightTime;
+    });
+}
+function buildAIRunTimeline(run) {
+    var e_2, _a, e_3, _b, e_4, _c, e_5, _d, e_6, _e;
+    var _f, _g, _h, _j;
+    var items = [];
+    try {
+        for (var _k = __values(run.events || []), _l = _k.next(); !_l.done; _l = _k.next()) {
+            var event_1 = _l.value;
+            items.push({
+                time: toIso(event_1.recordedAt),
+                type: event_1.type,
+                label: event_1.category || event_1.type,
+                message: cleanText(event_1.message, 2000),
+                evidenceRefs: Array.isArray(event_1.artifactPaths) ? event_1.artifactPaths : [],
+                metadata: event_1.metadata
+            });
+        }
+    }
+    catch (e_2_1) { e_2 = { error: e_2_1 }; }
+    finally {
+        try {
+            if (_l && !_l.done && (_a = _k.return)) _a.call(_k);
+        }
+        finally { if (e_2) throw e_2.error; }
+    }
+    try {
+        for (var _m = __values(run.gates || []), _o = _m.next(); !_o.done; _o = _m.next()) {
+            var gate = _o.value;
+            items.push({
+                time: toIso(gate.recordedAt),
+                type: 'gate',
+                label: gate.label || gate.key,
+                status: gate.status,
+                message: gate.reason,
+                evidenceRefs: gate.evidenceRefs || [],
+                metadata: gate.metadata
+            });
+        }
+    }
+    catch (e_3_1) { e_3 = { error: e_3_1 }; }
+    finally {
+        try {
+            if (_o && !_o.done && (_b = _m.return)) _b.call(_m);
+        }
+        finally { if (e_3) throw e_3.error; }
+    }
+    try {
+        for (var _p = __values(((_f = run.qa) === null || _f === void 0 ? void 0 : _f.infraChecks) || []), _q = _p.next(); !_q.done; _q = _p.next()) {
+            var check = _q.value;
+            items.push({
+                time: toIso(check.checkedAt),
+                type: 'qa_infra',
+                label: check.name,
+                status: check.status,
+                message: check.message,
+                evidenceRefs: check.path ? [check.path] : [],
+                metadata: check.metadata
+            });
+        }
+    }
+    catch (e_4_1) { e_4 = { error: e_4_1 }; }
+    finally {
+        try {
+            if (_q && !_q.done && (_c = _p.return)) _c.call(_p);
+        }
+        finally { if (e_4) throw e_4.error; }
+    }
+    if ((_g = run.qa) === null || _g === void 0 ? void 0 : _g.compile) {
+        items.push({
+            time: toIso(run.qa.compile.recordedAt),
+            type: 'qa_compile',
+            label: run.qa.compile.command || 'Compile/build',
+            status: run.qa.compile.status,
+            message: run.qa.compile.message,
+            evidenceRefs: run.qa.compile.artifactPath ? [run.qa.compile.artifactPath] : [],
+            metadata: run.qa.compile.metadata
+        });
+    }
+    try {
+        for (var _r = __values(((_h = run.qa) === null || _h === void 0 ? void 0 : _h.routeProbes) || []), _s = _r.next(); !_s.done; _s = _r.next()) {
+            var probe = _s.value;
+            items.push({
+                time: toIso(probe.recordedAt),
+                type: 'qa_route_probe',
+                label: probe.route,
+                status: probe.status,
+                message: probe.message || probe.caption,
+                evidenceRefs: probe.screenshot ? [probe.screenshot] : [],
+                metadata: probe.metadata
+            });
+        }
+    }
+    catch (e_5_1) { e_5 = { error: e_5_1 }; }
+    finally {
+        try {
+            if (_s && !_s.done && (_d = _r.return)) _d.call(_r);
+        }
+        finally { if (e_5) throw e_5.error; }
+    }
+    try {
+        for (var _t = __values(((_j = run.qa) === null || _j === void 0 ? void 0 : _j.businessAssertions) || []), _u = _t.next(); !_u.done; _u = _t.next()) {
+            var assertion = _u.value;
+            items.push({
+                time: toIso(assertion.recordedAt),
+                type: 'qa_business_assertion',
+                label: assertion.assertion,
+                status: assertion.status,
+                message: assertion.observed || assertion.dataProof || assertion.message,
+                evidenceRefs: assertion.artifactPaths || [],
+                metadata: assertion.metadata
+            });
+        }
+    }
+    catch (e_6_1) { e_6 = { error: e_6_1 }; }
+    finally {
+        try {
+            if (_u && !_u.done && (_e = _t.return)) _e.call(_t);
+        }
+        finally { if (e_6) throw e_6.error; }
+    }
+    return sortTimeline(items);
+}
+function buildAIRunQaHealth(run) {
+    var _a, _b, _c, _d, _e;
+    var infraCounts = countStatuses(((_a = run.qa) === null || _a === void 0 ? void 0 : _a.infraChecks) || []);
+    var routeProbes = ((_b = run.qa) === null || _b === void 0 ? void 0 : _b.routeProbes) || [];
+    var routePass = routeProbes.filter(function (probe) { return probe.status === 'pass' && probe.shellOnly !== true; }).length;
+    var routeFail = routeProbes.filter(function (probe) { return probe.status === 'fail' || probe.status === 'blocked'; }).length;
+    var shellOnly = routeProbes.filter(function (probe) { return probe.shellOnly === true; }).length;
+    var assertions = ((_c = run.qa) === null || _c === void 0 ? void 0 : _c.businessAssertions) || [];
+    var businessPass = assertions.filter(function (assertion) { return assertion.status === 'pass'; }).length;
+    var businessFail = assertions.filter(function (assertion) { return assertion.status === 'fail'; }).length;
+    var businessBlocked = assertions.filter(function (assertion) { return assertion.status === 'blocked'; }).length;
+    return {
+        outcome: (_d = run.qa) === null || _d === void 0 ? void 0 : _d.outcome,
+        infra: {
+            pass: infraCounts.pass,
+            fail: infraCounts.fail,
+            blocked: infraCounts.blocked,
+            warn: infraCounts.warn,
+            unknown: infraCounts.unknown
+        },
+        compile: ((_e = run.qa) === null || _e === void 0 ? void 0 : _e.compile) ? {
+            status: run.qa.compile.status,
+            stale: run.qa.compile.staleEvidence === true || run.qa.compile.status === 'stale',
+            artifactPath: run.qa.compile.artifactPath,
+            message: run.qa.compile.message
+        } : undefined,
+        route: {
+            pass: routePass,
+            fail: routeFail,
+            shellOnly: shellOnly
+        },
+        business: {
+            pass: businessPass,
+            fail: businessFail,
+            blocked: businessBlocked,
+            missing: assertions.length === 0
+        }
+    };
+}
+function buildAIRunCostPanel(cost, run) {
+    var _a, _b, _c, _d;
+    var categories = Object.keys((cost === null || cost === void 0 ? void 0 : cost.categories) || {}).sort().map(function (category) {
+        var _a;
+        var entry = (_a = cost === null || cost === void 0 ? void 0 : cost.categories) === null || _a === void 0 ? void 0 : _a[category];
+        return {
+            category: category,
+            totalTokens: Number((entry === null || entry === void 0 ? void 0 : entry.totalTokens) || 0),
+            estimatedUsd: Number((entry === null || entry === void 0 ? void 0 : entry.estimatedUsd) || 0),
+            count: Number((entry === null || entry === void 0 ? void 0 : entry.count) || 0)
+        };
+    });
+    var sources = Object.keys((cost === null || cost === void 0 ? void 0 : cost.sources) || {}).sort().map(function (source) {
+        var _a;
+        var entry = (_a = cost === null || cost === void 0 ? void 0 : cost.sources) === null || _a === void 0 ? void 0 : _a[source];
+        return {
+            source: source,
+            totalTokens: Number((entry === null || entry === void 0 ? void 0 : entry.totalTokens) || 0),
+            estimatedUsd: Number((entry === null || entry === void 0 ? void 0 : entry.estimatedUsd) || 0),
+            count: Number((entry === null || entry === void 0 ? void 0 : entry.count) || 0),
+            manualCount: Number((entry === null || entry === void 0 ? void 0 : entry.manualCount) || 0),
+            untrackedCount: Number((entry === null || entry === void 0 ? void 0 : entry.untrackedCount) || 0)
+        };
+    });
+    var surfaces = Object.keys((cost === null || cost === void 0 ? void 0 : cost.surfaces) || {}).sort().map(function (surface) {
+        var _a;
+        var entry = (_a = cost === null || cost === void 0 ? void 0 : cost.surfaces) === null || _a === void 0 ? void 0 : _a[surface];
+        return {
+            surface: surface,
+            totalTokens: Number((entry === null || entry === void 0 ? void 0 : entry.totalTokens) || 0),
+            estimatedUsd: Number((entry === null || entry === void 0 ? void 0 : entry.estimatedUsd) || 0),
+            count: Number((entry === null || entry === void 0 ? void 0 : entry.count) || 0)
+        };
+    });
+    var costGate = (_a = run === null || run === void 0 ? void 0 : run.gates) === null || _a === void 0 ? void 0 : _a.find(function (gate) { return gate.key === 'cost_observability'; });
+    var observability = ((_b = run === null || run === void 0 ? void 0 : run.metadata) === null || _b === void 0 ? void 0 : _b.costObservability) || (costGate === null || costGate === void 0 ? void 0 : costGate.metadata) || {};
+    return {
+        totalTokens: Number((cost === null || cost === void 0 ? void 0 : cost.totalTokens) || 0),
+        estimatedUsd: Number((cost === null || cost === void 0 ? void 0 : cost.estimatedUsd) || 0),
+        models: Array.isArray(cost === null || cost === void 0 ? void 0 : cost.models) ? cost.models : [],
+        categories: categories,
+        sources: sources,
+        surfaces: surfaces,
+        manualEstimatedUsd: Number((cost === null || cost === void 0 ? void 0 : cost.manualEstimatedUsd) || 0),
+        untrackedEstimatedUsd: Number((cost === null || cost === void 0 ? void 0 : cost.untrackedEstimatedUsd) || 0),
+        untrackedWarnings: Array.isArray(cost === null || cost === void 0 ? void 0 : cost.untrackedWarnings) ? cost.untrackedWarnings : [],
+        observabilityStatus: cleanText(observability.status || (costGate === null || costGate === void 0 ? void 0 : costGate.status), 80),
+        missingLedger: observability.missingLedger === true || ((_c = costGate === null || costGate === void 0 ? void 0 : costGate.metadata) === null || _c === void 0 ? void 0 : _c.missingLedger) === true,
+        externalCodexMonitorUsed: observability.externalCodexMonitorUsed === true || ((_d = costGate === null || costGate === void 0 ? void 0 : costGate.metadata) === null || _d === void 0 ? void 0 : _d.externalCodexMonitorUsed) === true,
+        nextAction: cleanText(observability.nextAction, 500)
+    };
+}
+function buildAIRunFalsePassWarnings(run) {
+    var _a, _b, _c, _d;
+    var warnings = Array.isArray(run.warnings) ? __spreadArray([], __read(run.warnings), false) : [];
+    if (run.outcome === 'false_pass' && !warnings.includes('false_pass_outcome')) {
+        warnings.push('false_pass_outcome');
+    }
+    if ((((_a = run.qa) === null || _a === void 0 ? void 0 : _a.outcome) === 'route_only_pass' || ((_b = run.qa) === null || _b === void 0 ? void 0 : _b.outcome) === 'incomplete') && !warnings.includes('missing_business_assertion')) {
+        warnings.push('missing_business_assertion');
+    }
+    if (((_d = (_c = run.qa) === null || _c === void 0 ? void 0 : _c.routeProbes) === null || _d === void 0 ? void 0 : _d.some(function (probe) { return probe.shellOnly === true; })) && !warnings.includes('shell_only_route')) {
+        warnings.push('shell_only_route');
+    }
+    if (run.outcome === 'release_blocked' && !warnings.includes('release_blocked_after_scorecard')) {
+        warnings.push('release_blocked_after_scorecard');
+    }
+    return warnings;
+}
+function buildAIRunOperatorView(run) {
+    return {
+        runId: run.id || run._id,
+        source: run.source,
+        title: run.title,
+        outcome: run.outcome,
+        failureClass: (0, ai_run_evidence_eval_1.classifyAIRunFailureClass)(run),
+        nextAction: run.nextAction,
+        warnings: buildAIRunFalsePassWarnings(run),
+        timeline: buildAIRunTimeline(run),
+        qaHealth: buildAIRunQaHealth(run),
+        costPanel: buildAIRunCostPanel(run.cost, run),
+        trainingExample: (0, ai_run_evidence_eval_1.buildAIRunTrainingExampleFromRun)(run)
+    };
+}
+
+//# sourceMappingURL=ai-run-evidence-dashboard.js.map

package/util/ai-run-evidence-dashboard.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["../../src/util/ai-run-evidence-dashboard.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAqIA,gDAoEC;AAED,gDAqCC;AAED,kDA+CC;AAED,kEAeC;AAED,wDAcC;AAjUD,+DAAqG;AAuFrG,SAAS,SAAS,CAAC,KAAU,EAAE,GAAU;IAAV,oBAAA,EAAA,UAAU;IACxC,OAAO,MAAM,CAAC,KAAK,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,KAAK,CAAC,KAAU;IACxB,IAAI,KAAK,YAAY,IAAI,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;QAC/D,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;IAC5B,CAAC;IACD,IAAI,KAAK,EAAE,CAAC;QACX,IAAM,MAAM,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC;QAC/B,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;YACvC,OAAO,MAAM,CAAC,WAAW,EAAE,CAAC;QAC7B,CAAC;IACF,CAAC;IACD,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,SAAS,aAAa,CAAC,MAAkC;;IACxD,IAAM,MAAM,GAA2B;QACtC,IAAI,EAAE,CAAC;QACP,IAAI,EAAE,CAAC;QACP,OAAO,EAAE,CAAC;QACV,IAAI,EAAE,CAAC;QACP,OAAO,EAAE,CAAC;KACV,CAAC;;QACF,KAAoB,IAAA,WAAA,SAAA,MAAM,CAAA,8BAAA,kDAAE,CAAC;YAAxB,IAAM,KAAK,mBAAA;YACf,IAAM,QAAM,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;YACzD,IAAI,QAAM,KAAK,MAAM,IAAI,QAAM,KAAK,MAAM,IAAI,QAAM,KAAK,SAAS,IAAI,QAAM,KAAK,MAAM,EAAE,CAAC;gBACzF,MAAM,CAAC,QAAM,CAAC,IAAI,CAAC,CAAC;YACrB,CAAC;iBACI,CAAC;gBACL,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;YACrB,CAAC;QACF,CAAC;;;;;;;;;IACD,OAAO,MAAM,CAAC;AACf,CAAC;AAED,SAAS,YAAY,CAAC,KAA0B;IAC/C,OAAO,yBAAI,KAAK,UAAE,IAAI,CAAC,UAAC,IAAI,EAAE,KAAK;QAClC,IAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/D,IAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAClE,OAAO,QAAQ,GAAG,SAAS,CAAC;IAC7B,CAAC,CAAC,CAAC;AACJ,CAAC;AAED,SAAgB,kBAAkB,CAAC,GAAU;;;IAC5C,IAAM,KAAK,GAAwB,EAAE,CAAC;;QACtC,KAAoB,IAAA,KAAA,SAAA,GAAG,CAAC,MAAM,IAAI,EAAE,CAAA,gBAAA,4BAAE,CAAC;YAAlC,IAAM,OAAK,WAAA;YACf,KAAK,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,KAAK,CAAC,OAAK,CAAC,UAAU,CAAC;gBAC7B,IAAI,EAAE,OAAK,CAAC,IAAI;gBAChB,KAAK,EAAE,OAAK,CAAC,QAAQ,IAAI,OAAK,CAAC,IAAI;gBACnC,OAAO,EAAE,SAAS,CAAC,OAAK,CAAC,OAAO,EAAE,IAAI,CAAC;gBACvC,YAAY,EAAE,KAAK,CAAC,OAAO,CAAC,OAAK,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,OAAK,CAAC,aAAa,CAAC,CAAC,CAAC,EAAE;gBAC3E,QAAQ,EAAE,OAAK,CAAC,QAAQ;aACxB,CAAC,CAAC;QACJ,CAAC;;;;;;;;;;QACD,KAAmB,IAAA,KAAA,SAAA,GAAG,CAAC,KAAK,IAAI,EAAE,CAAA,gBAAA,4BAAE,CAAC;YAAhC,IAAM,IAAI,WAAA;YACd,KAAK,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC;gBAC5B,IAAI,EAAE,MAAM;gBACZ,KAAK,EAAE,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,GAAG;gBAC7B,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,OAAO,EAAE,IAAI,CAAC,MAAM;gBACpB,YAAY,EAAE,IAAI,CAAC,YAAY,IAAI,EAAE;gBACrC,QAAQ,EAAE,IAAI,CAAC,QAAQ;aACvB,CAAC,CAAC;QACJ,CAAC;;;;;;;;;;QACD,KAAoB,IAAA,KAAA,SAAA,CAAA,MAAA,GAAG,CAAC,EAAE,0CAAE,WAAW,KAAI,EAAE,CAAA,gBAAA,4BAAE,CAAC;YAA3C,IAAM,KAAK,WAAA;YACf,KAAK,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,SAAS,CAAC;gBAC5B,IAAI,EAAE,UAAU;gBAChB,KAAK,EAAE,KAAK,CAAC,IAAI;gBACjB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,OAAO,EAAE,KAAK,CAAC,OAAO;gBACtB,YAAY,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE;gBAC5C,QAAQ,EAAE,KAAK,CAAC,QAAQ;aACxB,CAAC,CAAC;QACJ,CAAC;;;;;;;;;IACD,IAAI,MAAA,GAAG,CAAC,EAAE,0CAAE,OAAO,EAAE,CAAC;QACrB,KAAK,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,UAAU,CAAC;YACtC,IAAI,EAAE,YAAY;YAClB,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO,IAAI,eAAe;YAChD,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM;YAC7B,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO;YAC/B,YAAY,EAAE,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,EAAE;YAC9E,QAAQ,EAAE,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,QAAQ;SACjC,CAAC,CAAC;IACJ,CAAC;;QACD,KAAoB,IAAA,KAAA,SAAA,CAAA,MAAA,GAAG,CAAC,EAAE,0CAAE,WAAW,KAAI,EAAE,CAAA,gBAAA,4BAAE,CAAC;YAA3C,IAAM,KAAK,WAAA;YACf,KAAK,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,UAAU,CAAC;gBAC7B,IAAI,EAAE,gBAAgB;gBACtB,KAAK,EAAE,KAAK,CAAC,KAAK;gBAClB,MAAM,EAAE,KAAK,CAAC,MAAM;gBACpB,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO;gBACvC,YAAY,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,EAAE;gBACxD,QAAQ,EAAE,KAAK,CAAC,QAAQ;aACxB,CAAC,CAAC;QACJ,CAAC;;;;;;;;;;QACD,KAAwB,IAAA,KAAA,SAAA,CAAA,MAAA,GAAG,CAAC,EAAE,0CAAE,kBAAkB,KAAI,EAAE,CAAA,gBAAA,4BAAE,CAAC;YAAtD,IAAM,SAAS,WAAA;YACnB,KAAK,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,KAAK,CAAC,SAAS,CAAC,UAAU,CAAC;gBACjC,IAAI,EAAE,uBAAuB;gBAC7B,KAAK,EAAE,SAAS,CAAC,SAAS;gBAC1B,MAAM,EAAE,SAAS,CAAC,MAAM;gBACxB,OAAO,EAAE,SAAS,CAAC,QAAQ,IAAI,SAAS,CAAC,SAAS,IAAI,SAAS,CAAC,OAAO;gBACvE,YAAY,EAAE,SAAS,CAAC,aAAa,IAAI,EAAE;gBAC3C,QAAQ,EAAE,SAAS,CAAC,QAAQ;aAC5B,CAAC,CAAC;QACJ,CAAC;;;;;;;;;IACD,OAAO,YAAY,CAAC,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED,SAAgB,kBAAkB,CAAC,GAAU;;IAC5C,IAAM,WAAW,GAAG,aAAa,CAAC,CAAA,MAAA,GAAG,CAAC,EAAE,0CAAE,WAAW,KAAI,EAAE,CAAC,CAAC;IAC7D,IAAM,WAAW,GAAG,CAAA,MAAA,GAAG,CAAC,EAAE,0CAAE,WAAW,KAAI,EAAE,CAAC;IAC9C,IAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,UAAC,KAAK,IAAK,OAAA,KAAK,CAAC,MAAM,KAAK,MAAM,IAAI,KAAK,CAAC,SAAS,KAAK,IAAI,EAAnD,CAAmD,CAAC,CAAC,MAAM,CAAC;IAC5G,IAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,UAAC,KAAK,IAAK,OAAA,KAAK,CAAC,MAAM,KAAK,MAAM,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAArD,CAAqD,CAAC,CAAC,MAAM,CAAC;IAC9G,IAAM,SAAS,GAAG,WAAW,CAAC,MAAM,CAAC,UAAC,KAAK,IAAK,OAAA,KAAK,CAAC,SAAS,KAAK,IAAI,EAAxB,CAAwB,CAAC,CAAC,MAAM,CAAC;IACjF,IAAM,UAAU,GAAG,CAAA,MAAA,GAAG,CAAC,EAAE,0CAAE,kBAAkB,KAAI,EAAE,CAAC;IACpD,IAAM,YAAY,GAAG,UAAU,CAAC,MAAM,CAAC,UAAC,SAAS,IAAK,OAAA,SAAS,CAAC,MAAM,KAAK,MAAM,EAA3B,CAA2B,CAAC,CAAC,MAAM,CAAC;IAC1F,IAAM,YAAY,GAAG,UAAU,CAAC,MAAM,CAAC,UAAC,SAAS,IAAK,OAAA,SAAS,CAAC,MAAM,KAAK,MAAM,EAA3B,CAA2B,CAAC,CAAC,MAAM,CAAC;IAC1F,IAAM,eAAe,GAAG,UAAU,CAAC,MAAM,CAAC,UAAC,SAAS,IAAK,OAAA,SAAS,CAAC,MAAM,KAAK,SAAS,EAA9B,CAA8B,CAAC,CAAC,MAAM,CAAC;IAChG,OAAO;QACN,OAAO,EAAE,MAAA,GAAG,CAAC,EAAE,0CAAE,OAAO;QACxB,KAAK,EAAE;YACN,IAAI,EAAE,WAAW,CAAC,IAAI;YACtB,IAAI,EAAE,WAAW,CAAC,IAAI;YACtB,OAAO,EAAE,WAAW,CAAC,OAAO;YAC5B,IAAI,EAAE,WAAW,CAAC,IAAI;YACtB,OAAO,EAAE,WAAW,CAAC,OAAO;SAC5B;QACD,OAAO,EAAE,CAAA,MAAA,GAAG,CAAC,EAAE,0CAAE,OAAO,EAAC,CAAC,CAAC;YAC1B,MAAM,EAAE,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM;YAC7B,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,aAAa,KAAK,IAAI,IAAI,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,KAAK,OAAO;YACjF,YAAY,EAAE,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,YAAY;YACzC,OAAO,EAAE,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,OAAO;SAC/B,CAAC,CAAC,CAAC,SAAS;QACb,KAAK,EAAE;YACN,IAAI,EAAE,SAAS;YACf,IAAI,EAAE,SAAS;YACf,SAAS,WAAA;SACT;QACD,QAAQ,EAAE;YACT,IAAI,EAAE,YAAY;YAClB,IAAI,EAAE,YAAY;YAClB,OAAO,EAAE,eAAe;YACxB,OAAO,EAAE,UAAU,CAAC,MAAM,KAAK,CAAC;SAChC;KACD,CAAC;AACH,CAAC;AAED,SAAgB,mBAAmB,CAAC,IAAgB,EAAE,GAAW;;IAChE,IAAM,UAAU,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,UAAU,KAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,UAAC,QAAQ;;QAC1E,IAAM,KAAK,GAAG,MAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,UAAU,0CAAG,QAAQ,CAAC,CAAC;QAC3C,OAAO;YACN,QAAQ,UAAA;YACR,WAAW,EAAE,MAAM,CAAC,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,WAAW,KAAI,CAAC,CAAC;YAC5C,YAAY,EAAE,MAAM,CAAC,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,YAAY,KAAI,CAAC,CAAC;YAC9C,KAAK,EAAE,MAAM,CAAC,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,KAAK,KAAI,CAAC,CAAC;SAChC,CAAC;IACH,CAAC,CAAC,CAAC;IACH,IAAM,OAAO,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,OAAO,KAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,UAAC,MAAM;;QAClE,IAAM,KAAK,GAAG,MAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,OAAO,0CAAG,MAAM,CAAC,CAAC;QACtC,OAAO;YACN,MAAM,QAAA;YACN,WAAW,EAAE,MAAM,CAAC,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,WAAW,KAAI,CAAC,CAAC;YAC5C,YAAY,EAAE,MAAM,CAAC,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,YAAY,KAAI,CAAC,CAAC;YAC9C,KAAK,EAAE,MAAM,CAAC,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,KAAK,KAAI,CAAC,CAAC;YAChC,WAAW,EAAE,MAAM,CAAC,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,WAAW,KAAI,CAAC,CAAC;YAC5C,cAAc,EAAE,MAAM,CAAC,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,cAAc,KAAI,CAAC,CAAC;SAClD,CAAC;IACH,CAAC,CAAC,CAAC;IACH,IAAM,QAAQ,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,QAAQ,KAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,UAAC,OAAO;;QACrE,IAAM,KAAK,GAAG,MAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,QAAQ,0CAAG,OAAO,CAAC,CAAC;QACxC,OAAO;YACN,OAAO,SAAA;YACP,WAAW,EAAE,MAAM,CAAC,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,WAAW,KAAI,CAAC,CAAC;YAC5C,YAAY,EAAE,MAAM,CAAC,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,YAAY,KAAI,CAAC,CAAC;YAC9C,KAAK,EAAE,MAAM,CAAC,CAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,KAAK,KAAI,CAAC,CAAC;SAChC,CAAC;IACH,CAAC,CAAC,CAAC;IACH,IAAM,QAAQ,GAAG,MAAA,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,KAAK,0CAAE,IAAI,CAAC,UAAC,IAAI,IAAK,OAAA,IAAI,CAAC,GAAG,KAAK,oBAAoB,EAAjC,CAAiC,CAAC,CAAC;IAC/E,IAAM,aAAa,GAAG,CAAA,MAAC,GAAG,aAAH,GAAG,uBAAH,GAAG,CAAE,QAAgB,0CAAE,iBAAiB,MAAI,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,QAAQ,CAAA,IAAI,EAAE,CAAC;IAC5F,OAAO;QACN,WAAW,EAAE,MAAM,CAAC,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,WAAW,KAAI,CAAC,CAAC;QAC3C,YAAY,EAAE,MAAM,CAAC,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,YAAY,KAAI,CAAC,CAAC;QAC7C,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE;QACtD,UAAU,YAAA;QACV,OAAO,SAAA;QACP,QAAQ,UAAA;QACR,kBAAkB,EAAE,MAAM,CAAC,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,kBAAkB,KAAI,CAAC,CAAC;QACzD,qBAAqB,EAAE,MAAM,CAAC,CAAA,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,qBAAqB,KAAI,CAAC,CAAC;QAC/D,iBAAiB,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,aAAJ,IAAI,uBAAJ,IAAI,CAAE,iBAAiB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE;QACvF,mBAAmB,EAAE,SAAS,CAAC,aAAa,CAAC,MAAM,KAAI,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,MAAM,CAAA,EAAE,EAAE,CAAC;QAC5E,aAAa,EAAE,aAAa,CAAC,aAAa,KAAK,IAAI,IAAI,CAAA,MAAA,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,QAAQ,0CAAE,aAAa,MAAK,IAAI;QACjG,wBAAwB,EAAE,aAAa,CAAC,wBAAwB,KAAK,IAAI,IAAI,CAAA,MAAA,QAAQ,aAAR,QAAQ,uBAAR,QAAQ,CAAE,QAAQ,0CAAE,wBAAwB,MAAK,IAAI;QAClI,UAAU,EAAE,SAAS,CAAC,aAAa,CAAC,UAAU,EAAE,GAAG,CAAC;KACpD,CAAC;AACH,CAAC;AAED,SAAgB,2BAA2B,CAAC,GAAU;;IACrD,IAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,0BAAK,GAAG,CAAC,QAAQ,UAAE,CAAC,CAAC,EAAE,CAAC;IACtE,IAAI,GAAG,CAAC,OAAO,KAAK,YAAY,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,oBAAoB,CAAC,EAAE,CAAC;QAC9E,QAAQ,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAC;IACrC,CAAC;IACD,IAAI,CAAC,CAAA,MAAA,GAAG,CAAC,EAAE,0CAAE,OAAO,MAAK,iBAAiB,IAAI,CAAA,MAAA,GAAG,CAAC,EAAE,0CAAE,OAAO,MAAK,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,4BAA4B,CAAC,EAAE,CAAC;QACrI,QAAQ,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;IAC7C,CAAC;IACD,IAAI,CAAA,MAAA,MAAA,GAAG,CAAC,EAAE,0CAAE,WAAW,0CAAE,IAAI,CAAC,UAAC,KAAK,IAAK,OAAA,KAAK,CAAC,SAAS,KAAK,IAAI,EAAxB,CAAwB,CAAC,KAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;QAC9G,QAAQ,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC;IACnC,CAAC;IACD,IAAI,GAAG,CAAC,OAAO,KAAK,iBAAiB,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,iCAAiC,CAAC,EAAE,CAAC;QAChG,QAAQ,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;IAClD,CAAC;IACD,OAAO,QAAQ,CAAC;AACjB,CAAC;AAED,SAAgB,sBAAsB,CAAC,GAAU;IAChD,OAAO;QACN,KAAK,EAAE,GAAG,CAAC,EAAE,IAAK,GAAW,CAAC,GAAG;QACjC,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,KAAK,EAAE,GAAG,CAAC,KAAK;QAChB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,YAAY,EAAE,IAAA,gDAAyB,EAAC,GAAG,CAAC;QAC5C,UAAU,EAAE,GAAG,CAAC,UAAU;QAC1B,QAAQ,EAAE,2BAA2B,CAAC,GAAG,CAAC;QAC1C,QAAQ,EAAE,kBAAkB,CAAC,GAAG,CAAC;QACjC,QAAQ,EAAE,kBAAkB,CAAC,GAAG,CAAC;QACjC,SAAS,EAAE,mBAAmB,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC;QAC7C,eAAe,EAAE,IAAA,uDAAgC,EAAC,GAAG,CAAC;KACtD,CAAC;AACH,CAAC","file":"ai-run-evidence-dashboard.js","sourcesContent":["import { AIRun, AIRunCost } from './ai-run-evidence';\nimport { buildAIRunTrainingExampleFromRun, classifyAIRunFailureClass } from './ai-run-evidence-eval';\n\nexport interface AIRunTimelineItem {\n\ttime?: string;\n\ttype: string;\n\tlabel: string;\n\tstatus?: string;\n\tmessage?: string;\n\tevidenceRefs: string[];\n\tmetadata?: Record<string, any>;\n}\n\nexport interface AIRunQaHealth {\n\toutcome?: string;\n\tinfra: {\n\t\tpass: number;\n\t\tfail: number;\n\t\tblocked: number;\n\t\twarn: number;\n\t\tunknown: number;\n\t};\n\tcompile?: {\n\t\tstatus?: string;\n\t\tstale?: boolean;\n\t\tartifactPath?: string;\n\t\tmessage?: string;\n\t};\n\troute: {\n\t\tpass: number;\n\t\tfail: number;\n\t\tshellOnly: number;\n\t};\n\tbusiness: {\n\t\tpass: number;\n\t\tfail: number;\n\t\tblocked: number;\n\t\tmissing: boolean;\n\t};\n}\n\nexport interface AIRunCostPanel {\n\ttotalTokens: number;\n\testimatedUsd: number;\n\tmodels: string[];\n\tcategories: Array<{\n\t\tcategory: string;\n\t\ttotalTokens: number;\n\t\testimatedUsd: number;\n\t\tcount: number;\n\t}>;\n\tsources?: Array<{\n\t\tsource: string;\n\t\ttotalTokens: number;\n\t\testimatedUsd: number;\n\t\tcount: number;\n\t\tmanualCount: number;\n\t\tuntrackedCount: number;\n\t}>;\n\tsurfaces?: Array<{\n\t\tsurface: string;\n\t\ttotalTokens: number;\n\t\testimatedUsd: number;\n\t\tcount: number;\n\t}>;\n\tmanualEstimatedUsd?: number;\n\tuntrackedEstimatedUsd?: number;\n\tuntrackedWarnings?: string[];\n\tobservabilityStatus?: string;\n\tmissingLedger?: boolean;\n\texternalCodexMonitorUsed?: boolean;\n\tnextAction?: string;\n}\n\nexport interface AIRunOperatorView {\n\trunId?: string;\n\tsource: string;\n\ttitle?: string;\n\toutcome: string;\n\tfailureClass: string;\n\tnextAction?: string;\n\twarnings: string[];\n\ttimeline: AIRunTimelineItem[];\n\tqaHealth: AIRunQaHealth;\n\tcostPanel: AIRunCostPanel;\n\ttrainingExample: ReturnType<typeof buildAIRunTrainingExampleFromRun>;\n}\n\nfunction cleanText(value: any, max = 1000): string {\n\treturn String(value || '').replace(/\\s+/g, ' ').trim().slice(0, max);\n}\n\nfunction toIso(value: any): string | undefined {\n\tif (value instanceof Date && Number.isFinite(value.getTime())) {\n\t\treturn value.toISOString();\n\t}\n\tif (value) {\n\t\tconst parsed = new Date(value);\n\t\tif (Number.isFinite(parsed.getTime())) {\n\t\t\treturn parsed.toISOString();\n\t\t}\n\t}\n\treturn undefined;\n}\n\nfunction countStatuses(values: Array<{ status?: string }>): Record<string, number> {\n\tconst result: Record<string, number> = {\n\t\tpass: 0,\n\t\tfail: 0,\n\t\tblocked: 0,\n\t\twarn: 0,\n\t\tunknown: 0\n\t};\n\tfor (const value of values) {\n\t\tconst status = cleanText(value.status, 40).toLowerCase();\n\t\tif (status === 'pass' || status === 'fail' || status === 'blocked' || status === 'warn') {\n\t\t\tresult[status] += 1;\n\t\t}\n\t\telse {\n\t\t\tresult.unknown += 1;\n\t\t}\n\t}\n\treturn result;\n}\n\nfunction sortTimeline(items: AIRunTimelineItem[]): AIRunTimelineItem[] {\n\treturn [...items].sort((left, right) => {\n\t\tconst leftTime = left.time ? new Date(left.time).getTime() : 0;\n\t\tconst rightTime = right.time ? new Date(right.time).getTime() : 0;\n\t\treturn leftTime - rightTime;\n\t});\n}\n\nexport function buildAIRunTimeline(run: AIRun): AIRunTimelineItem[] {\n\tconst items: AIRunTimelineItem[] = [];\n\tfor (const event of run.events || []) {\n\t\titems.push({\n\t\t\ttime: toIso(event.recordedAt),\n\t\t\ttype: event.type,\n\t\t\tlabel: event.category || event.type,\n\t\t\tmessage: cleanText(event.message, 2000),\n\t\t\tevidenceRefs: Array.isArray(event.artifactPaths) ? event.artifactPaths : [],\n\t\t\tmetadata: event.metadata\n\t\t});\n\t}\n\tfor (const gate of run.gates || []) {\n\t\titems.push({\n\t\t\ttime: toIso(gate.recordedAt),\n\t\t\ttype: 'gate',\n\t\t\tlabel: gate.label || gate.key,\n\t\t\tstatus: gate.status,\n\t\t\tmessage: gate.reason,\n\t\t\tevidenceRefs: gate.evidenceRefs || [],\n\t\t\tmetadata: gate.metadata\n\t\t});\n\t}\n\tfor (const check of run.qa?.infraChecks || []) {\n\t\titems.push({\n\t\t\ttime: toIso(check.checkedAt),\n\t\t\ttype: 'qa_infra',\n\t\t\tlabel: check.name,\n\t\t\tstatus: check.status,\n\t\t\tmessage: check.message,\n\t\t\tevidenceRefs: check.path ? [check.path] : [],\n\t\t\tmetadata: check.metadata\n\t\t});\n\t}\n\tif (run.qa?.compile) {\n\t\titems.push({\n\t\t\ttime: toIso(run.qa.compile.recordedAt),\n\t\t\ttype: 'qa_compile',\n\t\t\tlabel: run.qa.compile.command || 'Compile/build',\n\t\t\tstatus: run.qa.compile.status,\n\t\t\tmessage: run.qa.compile.message,\n\t\t\tevidenceRefs: run.qa.compile.artifactPath ? [run.qa.compile.artifactPath] : [],\n\t\t\tmetadata: run.qa.compile.metadata\n\t\t});\n\t}\n\tfor (const probe of run.qa?.routeProbes || []) {\n\t\titems.push({\n\t\t\ttime: toIso(probe.recordedAt),\n\t\t\ttype: 'qa_route_probe',\n\t\t\tlabel: probe.route,\n\t\t\tstatus: probe.status,\n\t\t\tmessage: probe.message || probe.caption,\n\t\t\tevidenceRefs: probe.screenshot ? [probe.screenshot] : [],\n\t\t\tmetadata: probe.metadata\n\t\t});\n\t}\n\tfor (const assertion of run.qa?.businessAssertions || []) {\n\t\titems.push({\n\t\t\ttime: toIso(assertion.recordedAt),\n\t\t\ttype: 'qa_business_assertion',\n\t\t\tlabel: assertion.assertion,\n\t\t\tstatus: assertion.status,\n\t\t\tmessage: assertion.observed || assertion.dataProof || assertion.message,\n\t\t\tevidenceRefs: assertion.artifactPaths || [],\n\t\t\tmetadata: assertion.metadata\n\t\t});\n\t}\n\treturn sortTimeline(items);\n}\n\nexport function buildAIRunQaHealth(run: AIRun): AIRunQaHealth {\n\tconst infraCounts = countStatuses(run.qa?.infraChecks || []);\n\tconst routeProbes = run.qa?.routeProbes || [];\n\tconst routePass = routeProbes.filter((probe) => probe.status === 'pass' && probe.shellOnly !== true).length;\n\tconst routeFail = routeProbes.filter((probe) => probe.status === 'fail' || probe.status === 'blocked').length;\n\tconst shellOnly = routeProbes.filter((probe) => probe.shellOnly === true).length;\n\tconst assertions = run.qa?.businessAssertions || [];\n\tconst businessPass = assertions.filter((assertion) => assertion.status === 'pass').length;\n\tconst businessFail = assertions.filter((assertion) => assertion.status === 'fail').length;\n\tconst businessBlocked = assertions.filter((assertion) => assertion.status === 'blocked').length;\n\treturn {\n\t\toutcome: run.qa?.outcome,\n\t\tinfra: {\n\t\t\tpass: infraCounts.pass,\n\t\t\tfail: infraCounts.fail,\n\t\t\tblocked: infraCounts.blocked,\n\t\t\twarn: infraCounts.warn,\n\t\t\tunknown: infraCounts.unknown\n\t\t},\n\t\tcompile: run.qa?.compile ? {\n\t\t\tstatus: run.qa.compile.status,\n\t\t\tstale: run.qa.compile.staleEvidence === true || run.qa.compile.status === 'stale',\n\t\t\tartifactPath: run.qa.compile.artifactPath,\n\t\t\tmessage: run.qa.compile.message\n\t\t} : undefined,\n\t\troute: {\n\t\t\tpass: routePass,\n\t\t\tfail: routeFail,\n\t\t\tshellOnly\n\t\t},\n\t\tbusiness: {\n\t\t\tpass: businessPass,\n\t\t\tfail: businessFail,\n\t\t\tblocked: businessBlocked,\n\t\t\tmissing: assertions.length === 0\n\t\t}\n\t};\n}\n\nexport function buildAIRunCostPanel(cost?: AIRunCost, run?: AIRun): AIRunCostPanel {\n\tconst categories = Object.keys(cost?.categories || {}).sort().map((category) => {\n\t\tconst entry = cost?.categories?.[category];\n\t\treturn {\n\t\t\tcategory,\n\t\t\ttotalTokens: Number(entry?.totalTokens || 0),\n\t\t\testimatedUsd: Number(entry?.estimatedUsd || 0),\n\t\t\tcount: Number(entry?.count || 0)\n\t\t};\n\t});\n\tconst sources = Object.keys(cost?.sources || {}).sort().map((source) => {\n\t\tconst entry = cost?.sources?.[source];\n\t\treturn {\n\t\t\tsource,\n\t\t\ttotalTokens: Number(entry?.totalTokens || 0),\n\t\t\testimatedUsd: Number(entry?.estimatedUsd || 0),\n\t\t\tcount: Number(entry?.count || 0),\n\t\t\tmanualCount: Number(entry?.manualCount || 0),\n\t\t\tuntrackedCount: Number(entry?.untrackedCount || 0)\n\t\t};\n\t});\n\tconst surfaces = Object.keys(cost?.surfaces || {}).sort().map((surface) => {\n\t\tconst entry = cost?.surfaces?.[surface];\n\t\treturn {\n\t\t\tsurface,\n\t\t\ttotalTokens: Number(entry?.totalTokens || 0),\n\t\t\testimatedUsd: Number(entry?.estimatedUsd || 0),\n\t\t\tcount: Number(entry?.count || 0)\n\t\t};\n\t});\n\tconst costGate = run?.gates?.find((gate) => gate.key === 'cost_observability');\n\tconst observability = (run?.metadata as any)?.costObservability || costGate?.metadata || {};\n\treturn {\n\t\ttotalTokens: Number(cost?.totalTokens || 0),\n\t\testimatedUsd: Number(cost?.estimatedUsd || 0),\n\t\tmodels: Array.isArray(cost?.models) ? cost.models : [],\n\t\tcategories,\n\t\tsources,\n\t\tsurfaces,\n\t\tmanualEstimatedUsd: Number(cost?.manualEstimatedUsd || 0),\n\t\tuntrackedEstimatedUsd: Number(cost?.untrackedEstimatedUsd || 0),\n\t\tuntrackedWarnings: Array.isArray(cost?.untrackedWarnings) ? cost.untrackedWarnings : [],\n\t\tobservabilityStatus: cleanText(observability.status || costGate?.status, 80),\n\t\tmissingLedger: observability.missingLedger === true || costGate?.metadata?.missingLedger === true,\n\t\texternalCodexMonitorUsed: observability.externalCodexMonitorUsed === true || costGate?.metadata?.externalCodexMonitorUsed === true,\n\t\tnextAction: cleanText(observability.nextAction, 500)\n\t};\n}\n\nexport function buildAIRunFalsePassWarnings(run: AIRun): string[] {\n\tconst warnings = Array.isArray(run.warnings) ? [...run.warnings] : [];\n\tif (run.outcome === 'false_pass' && !warnings.includes('false_pass_outcome')) {\n\t\twarnings.push('false_pass_outcome');\n\t}\n\tif ((run.qa?.outcome === 'route_only_pass' || run.qa?.outcome === 'incomplete') && !warnings.includes('missing_business_assertion')) {\n\t\twarnings.push('missing_business_assertion');\n\t}\n\tif (run.qa?.routeProbes?.some((probe) => probe.shellOnly === true) && !warnings.includes('shell_only_route')) {\n\t\twarnings.push('shell_only_route');\n\t}\n\tif (run.outcome === 'release_blocked' && !warnings.includes('release_blocked_after_scorecard')) {\n\t\twarnings.push('release_blocked_after_scorecard');\n\t}\n\treturn warnings;\n}\n\nexport function buildAIRunOperatorView(run: AIRun): AIRunOperatorView {\n\treturn {\n\t\trunId: run.id || (run as any)._id,\n\t\tsource: run.source,\n\t\ttitle: run.title,\n\t\toutcome: run.outcome,\n\t\tfailureClass: classifyAIRunFailureClass(run),\n\t\tnextAction: run.nextAction,\n\t\twarnings: buildAIRunFalsePassWarnings(run),\n\t\ttimeline: buildAIRunTimeline(run),\n\t\tqaHealth: buildAIRunQaHealth(run),\n\t\tcostPanel: buildAIRunCostPanel(run.cost, run),\n\t\ttrainingExample: buildAIRunTrainingExampleFromRun(run)\n\t};\n}\n"]}

package/util/ai-run-evidence-eval.d.ts

@@ -0,0 +1,86 @@
+import { AIRun, AIRunOutcomeLabel, AIRunSource, AIRunTrainingExample } from './ai-run-evidence';
+export type AIRunFailureClass = 'none' | 'infra' | 'compile' | 'route' | 'journey_contract' | 'business' | 'release' | 'false_pass' | 'assistant_missing_evidence' | 'assistant_query_error' | 'assistant_permission' | 'assistant_date_window' | 'assistant_query_shape' | 'assistant_correctness' | 'manual' | 'stopped' | 'unknown';
+export type AIRunRetryAction = 'none' | 'stop' | 'retry_infra_repair' | 'retry_build_repair' | 'retry_route_repair' | 'retry_journey_contract' | 'retry_business_repair' | 'retry_release_repair' | 'retry_assistant_answer_quality' | 'retry_assistant_query' | 'request_assistant_permission' | 'retry_assistant_date_window' | 'retry_assistant_query_shape' | 'manual_handoff';
+export interface AIRunEvalExpectedGate {
+    key: string;
+    status?: string;
+}
+export interface AIRunEvalExpected {
+    outcome: AIRunOutcomeLabel;
+    failureClass?: AIRunFailureClass;
+    nextActionIncludes?: string;
+    gates?: AIRunEvalExpectedGate[];
+}
+export interface AIRunEvalFixture {
+    id: string;
+    title: string;
+    source: AIRunSource;
+    run: AIRun;
+    expected: AIRunEvalExpected;
+    tags: string[];
+}
+export interface AIRunEvalResult {
+    id: string;
+    title: string;
+    source: AIRunSource;
+    passed: boolean;
+    actualOutcome: AIRunOutcomeLabel;
+    expectedOutcome: AIRunOutcomeLabel;
+    actualFailureClass: AIRunFailureClass;
+    expectedFailureClass?: AIRunFailureClass;
+    failures: string[];
+    tags: string[];
+}
+export interface AIRunEvalSummary {
+    passed: boolean;
+    total: number;
+    passedCount: number;
+    failedCount: number;
+    results: AIRunEvalResult[];
+    failuresByTag: Record<string, number>;
+    outcomes: Record<string, number>;
+}
+export interface AIRunDatasetExport {
+    generatedAt: string;
+    totalRuns: number;
+    summary: {
+        bySource: Record<string, number>;
+        byOutcome: Record<string, number>;
+        byFailureClass: Record<string, number>;
+        estimatedUsd: number;
+        totalTokens: number;
+    };
+    runs: AIRun[];
+    trainingExamples: AIRunTrainingExample[];
+}
+export interface AIRunDatasetExportOptions {
+    redact?: boolean;
+    includeRuns?: boolean;
+    maxEvidenceItems?: number;
+    now?: Date | string;
+}
+export interface AIRunRetryPolicyInput {
+    currentRun: AIRun;
+    previousRun?: AIRun;
+    newEvidenceCount?: number;
+    unchangedFailureLimit?: number;
+    sameFailureCount?: number;
+    historicalAcceptanceRate?: number;
+    estimatedNextRetryCostUsd?: number;
+    minRetryExpectedValue?: number;
+}
+export interface AIRunRetryDecision {
+    action: AIRunRetryAction;
+    failureClass: AIRunFailureClass;
+    repeatedFailure: boolean;
+    reason: string;
+    nextAction: string;
+    retryExpectedValue?: number;
+}
+export declare function classifyAIRunFailureClass(run: AIRun): AIRunFailureClass;
+export declare function evaluateAIRunFixture(fixture: AIRunEvalFixture): AIRunEvalResult;
+export declare function evaluateAIRunFixtures(fixtures?: AIRunEvalFixture[]): AIRunEvalSummary;
+export declare function buildAIRunTrainingExampleFromRun(run: AIRun, maxEvidenceItems?: number): AIRunTrainingExample;
+export declare function buildAIRunDatasetExport(runs: AIRun[], options?: AIRunDatasetExportOptions): AIRunDatasetExport;
+export declare function decideAIRunRetry(input: AIRunRetryPolicyInput): AIRunRetryDecision;
+export declare const AI_RUN_SEED_EVAL_FIXTURES: AIRunEvalFixture[];