@resolveio/server-lib 22.3.221 → 22.3.223

package/util/ai-run-evidence-eval.js

@@ -0,0 +1,1018 @@
+"use strict";
+var __assign = (this && this.__assign) || function () {
+    __assign = Object.assign || function(t) {
+        for (var s, i = 1, n = arguments.length; i < n; i++) {
+            s = arguments[i];
+            for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
+                t[p] = s[p];
+        }
+        return t;
+    };
+    return __assign.apply(this, arguments);
+};
+var __values = (this && this.__values) || function(o) {
+    var s = typeof Symbol === "function" && Symbol.iterator, m = s && o[s], i = 0;
+    if (m) return m.call(o);
+    if (o && typeof o.length === "number") return {
+        next: function () {
+            if (o && i >= o.length) o = void 0;
+            return { value: o && o[i++], done: !o };
+        }
+    };
+    throw new TypeError(s ? "Object is not iterable." : "Symbol.iterator is not defined.");
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.AI_RUN_SEED_EVAL_FIXTURES = void 0;
+exports.classifyAIRunFailureClass = classifyAIRunFailureClass;
+exports.evaluateAIRunFixture = evaluateAIRunFixture;
+exports.evaluateAIRunFixtures = evaluateAIRunFixtures;
+exports.buildAIRunTrainingExampleFromRun = buildAIRunTrainingExampleFromRun;
+exports.buildAIRunDatasetExport = buildAIRunDatasetExport;
+exports.decideAIRunRetry = decideAIRunRetry;
+var ai_run_evidence_1 = require("./ai-run-evidence");
+function cleanText(value, max) {
+    if (max === void 0) { max = 1000; }
+    return String(value || '').replace(/\s+/g, ' ').trim().slice(0, max);
+}
+function countBy(values) {
+    var e_1, _a;
+    var result = {};
+    try {
+        for (var values_1 = __values(values), values_1_1 = values_1.next(); !values_1_1.done; values_1_1 = values_1.next()) {
+            var value = values_1_1.value;
+            result[value] = (result[value] || 0) + 1;
+        }
+    }
+    catch (e_1_1) { e_1 = { error: e_1_1 }; }
+    finally {
+        try {
+            if (values_1_1 && !values_1_1.done && (_a = values_1.return)) _a.call(values_1);
+        }
+        finally { if (e_1) throw e_1.error; }
+    }
+    return result;
+}
+function isoNow(value) {
+    if (value instanceof Date) {
+        return value.toISOString();
+    }
+    var parsed = value ? new Date(value) : new Date();
+    if (Number.isFinite(parsed.getTime())) {
+        return parsed.toISOString();
+    }
+    return new Date().toISOString();
+}
+function addUnique(values, value, max) {
+    if (max === void 0) { max = 1000; }
+    var normalized = cleanText(value, max);
+    if (normalized && !values.includes(normalized)) {
+        values.push(normalized);
+    }
+}
+function normalizedStatus(value) {
+    return cleanText(value, 160).toLowerCase().replace(/[\s-]+/g, '_');
+}
+function plainObject(value) {
+    return value && typeof value === 'object' && !Array.isArray(value) ? value : {};
+}
+function assistantAnswerQualityMetadata(run) {
+    var gate = (run.gates || []).find(function (entry) { return entry.key === 'assistant_answer_quality'; });
+    var gateMetadata = plainObject(gate === null || gate === void 0 ? void 0 : gate.metadata);
+    var runMetadata = plainObject(run.metadata);
+    var answerQuality = plainObject(runMetadata.answerQuality || runMetadata.answer_quality);
+    return __assign(__assign({}, gateMetadata), answerQuality);
+}
+function classifyAssistantFailureClass(run) {
+    var metadata = assistantAnswerQualityMetadata(run);
+    var status = normalizedStatus(metadata.status || metadata.answerQualityStatus || metadata.answer_quality_status);
+    var queryStatus = normalizedStatus(metadata.queryStatus || metadata.query_status);
+    var missingEvidenceStatuses = new Set([
+        'missing_data_source',
+        'missing_query_status',
+        'missing_query_proof',
+        'missing_citations',
+        'missing_citation',
+        'low_confidence',
+        'missing_next_action',
+        'no_data_unverified',
+        'blocked'
+    ]);
+    if (queryStatus === 'permission_error' || status === 'permission_error') {
+        return 'assistant_permission';
+    }
+    if (queryStatus === 'query_error' || status === 'query_error') {
+        return 'assistant_query_error';
+    }
+    if (status === 'date_incorrect' || status === 'missing_date_window') {
+        return 'assistant_date_window';
+    }
+    if (status === 'illegal_query_shape') {
+        return 'assistant_query_shape';
+    }
+    if (missingEvidenceStatuses.has(status) || !status) {
+        return 'assistant_missing_evidence';
+    }
+    return 'assistant_correctness';
+}
+function classifyAIRunFailureClass(run) {
+    var _a, _b, _c, _d, _e;
+    if (run.outcome === 'accepted') {
+        return 'none';
+    }
+    if (run.outcome === 'qa_infra_failed') {
+        return 'infra';
+    }
+    if (run.outcome === 'build_failed') {
+        return 'compile';
+    }
+    if (run.outcome === 'release_blocked') {
+        return 'release';
+    }
+    if (run.source === 'aicoder_app' && (run.gates || []).some(function (gate) { return gate.key === 'aicoder_journey_contract' && (gate.status === 'blocked' || gate.status === 'fail'); })) {
+        return 'journey_contract';
+    }
+    if (run.source === 'ai_assistant' && run.outcome === 'false_pass') {
+        return classifyAssistantFailureClass(run);
+    }
+    if (run.outcome === 'false_pass') {
+        return 'false_pass';
+    }
+    if (run.outcome === 'manual_handoff') {
+        return 'manual';
+    }
+    if (run.outcome === 'stopped') {
+        return 'stopped';
+    }
+    if (run.source === 'ai_assistant' && (run.outcome === 'rejected' || run.outcome === 'qa_incomplete')) {
+        return classifyAssistantFailureClass(run);
+    }
+    if (((_a = run.qa) === null || _a === void 0 ? void 0 : _a.outcome) === 'compile_failed') {
+        return 'compile';
+    }
+    if (((_b = run.qa) === null || _b === void 0 ? void 0 : _b.outcome) === 'route_failed') {
+        return 'route';
+    }
+    if (((_c = run.qa) === null || _c === void 0 ? void 0 : _c.outcome) === 'route_only_pass' || ((_d = run.qa) === null || _d === void 0 ? void 0 : _d.outcome) === 'incomplete') {
+        return 'business';
+    }
+    if (((_e = run.qa) === null || _e === void 0 ? void 0 : _e.outcome) === 'business_assertion_failed') {
+        return 'business';
+    }
+    if (run.outcome === 'qa_incomplete') {
+        return 'business';
+    }
+    if (run.outcome === 'rejected') {
+        return 'business';
+    }
+    return 'unknown';
+}
+function evaluateAIRunFixture(fixture) {
+    var e_2, _a, e_3, _b;
+    var failures = [];
+    var actualFailureClass = classifyAIRunFailureClass(fixture.run);
+    if (fixture.run.outcome !== fixture.expected.outcome) {
+        failures.push("Expected outcome ".concat(fixture.expected.outcome, ", got ").concat(fixture.run.outcome, "."));
+    }
+    if (fixture.expected.failureClass && actualFailureClass !== fixture.expected.failureClass) {
+        failures.push("Expected failure class ".concat(fixture.expected.failureClass, ", got ").concat(actualFailureClass, "."));
+    }
+    if (fixture.expected.nextActionIncludes) {
+        var expectedText = fixture.expected.nextActionIncludes.toLowerCase();
+        var actualText = cleanText(fixture.run.nextAction, 2000).toLowerCase();
+        if (!actualText.includes(expectedText)) {
+            failures.push("Expected next action to include \"".concat(fixture.expected.nextActionIncludes, "\", got \"").concat(fixture.run.nextAction || '', "\"."));
+        }
+    }
+    var gatesByKey = {};
+    try {
+        for (var _c = __values(fixture.run.gates || []), _d = _c.next(); !_d.done; _d = _c.next()) {
+            var gate = _d.value;
+            gatesByKey[gate.key] = gate.status;
+        }
+    }
+    catch (e_2_1) { e_2 = { error: e_2_1 }; }
+    finally {
+        try {
+            if (_d && !_d.done && (_a = _c.return)) _a.call(_c);
+        }
+        finally { if (e_2) throw e_2.error; }
+    }
+    try {
+        for (var _e = __values(fixture.expected.gates || []), _f = _e.next(); !_f.done; _f = _e.next()) {
+            var expectedGate = _f.value;
+            if (!gatesByKey[expectedGate.key]) {
+                failures.push("Expected gate ".concat(expectedGate.key, " to be present."));
+            }
+            else if (expectedGate.status && gatesByKey[expectedGate.key] !== expectedGate.status) {
+                failures.push("Expected gate ".concat(expectedGate.key, " status ").concat(expectedGate.status, ", got ").concat(gatesByKey[expectedGate.key], "."));
+            }
+        }
+    }
+    catch (e_3_1) { e_3 = { error: e_3_1 }; }
+    finally {
+        try {
+            if (_f && !_f.done && (_b = _e.return)) _b.call(_e);
+        }
+        finally { if (e_3) throw e_3.error; }
+    }
+    return {
+        id: fixture.id,
+        title: fixture.title,
+        source: fixture.source,
+        passed: failures.length === 0,
+        actualOutcome: fixture.run.outcome,
+        expectedOutcome: fixture.expected.outcome,
+        actualFailureClass: actualFailureClass,
+        expectedFailureClass: fixture.expected.failureClass,
+        failures: failures,
+        tags: fixture.tags
+    };
+}
+function evaluateAIRunFixtures(fixtures) {
+    var e_4, _a, e_5, _b;
+    if (fixtures === void 0) { fixtures = exports.AI_RUN_SEED_EVAL_FIXTURES; }
+    var results = fixtures.map(function (fixture) { return evaluateAIRunFixture(fixture); });
+    var failuresByTag = {};
+    try {
+        for (var results_1 = __values(results), results_1_1 = results_1.next(); !results_1_1.done; results_1_1 = results_1.next()) {
+            var result = results_1_1.value;
+            if (result.passed) {
+                continue;
+            }
+            try {
+                for (var _c = (e_5 = void 0, __values(result.tags)), _d = _c.next(); !_d.done; _d = _c.next()) {
+                    var tag = _d.value;
+                    failuresByTag[tag] = (failuresByTag[tag] || 0) + 1;
+                }
+            }
+            catch (e_5_1) { e_5 = { error: e_5_1 }; }
+            finally {
+                try {
+                    if (_d && !_d.done && (_b = _c.return)) _b.call(_c);
+                }
+                finally { if (e_5) throw e_5.error; }
+            }
+        }
+    }
+    catch (e_4_1) { e_4 = { error: e_4_1 }; }
+    finally {
+        try {
+            if (results_1_1 && !results_1_1.done && (_a = results_1.return)) _a.call(results_1);
+        }
+        finally { if (e_4) throw e_4.error; }
+    }
+    var outcomes = countBy(results.map(function (result) { return result.actualOutcome; }));
+    var passedCount = results.filter(function (result) { return result.passed; }).length;
+    return {
+        passed: passedCount === results.length,
+        total: results.length,
+        passedCount: passedCount,
+        failedCount: results.length - passedCount,
+        results: results,
+        failuresByTag: failuresByTag,
+        outcomes: outcomes
+    };
+}
+function buildAIRunTrainingExampleFromRun(run, maxEvidenceItems) {
+    var e_6, _a, e_7, _b, e_8, _c;
+    var _d, _e;
+    if (maxEvidenceItems === void 0) { maxEvidenceItems = 30; }
+    var evidence = [];
+    try {
+        for (var _f = __values(run.gates || []), _g = _f.next(); !_g.done; _g = _f.next()) {
+            var gate = _g.value;
+            addUnique(evidence, "".concat(gate.key, ":").concat(gate.status, ": ").concat(gate.reason), 1600);
+        }
+    }
+    catch (e_6_1) { e_6 = { error: e_6_1 }; }
+    finally {
+        try {
+            if (_g && !_g.done && (_a = _f.return)) _a.call(_f);
+        }
+        finally { if (e_6) throw e_6.error; }
+    }
+    try {
+        for (var _h = __values(run.events || []), _j = _h.next(); !_j.done; _j = _h.next()) {
+            var event_1 = _j.value;
+            addUnique(evidence, "".concat(event_1.type, ":").concat(event_1.category || '', ": ").concat(event_1.message || ''), 1400);
+            if (evidence.length >= maxEvidenceItems) {
+                break;
+            }
+        }
+    }
+    catch (e_7_1) { e_7 = { error: e_7_1 }; }
+    finally {
+        try {
+            if (_j && !_j.done && (_b = _h.return)) _b.call(_h);
+        }
+        finally { if (e_7) throw e_7.error; }
+    }
+    if (evidence.length < maxEvidenceItems) {
+        try {
+            for (var _k = __values(((_d = run.qa) === null || _d === void 0 ? void 0 : _d.businessAssertions) || []), _l = _k.next(); !_l.done; _l = _k.next()) {
+                var assertion = _l.value;
+                addUnique(evidence, "assertion:".concat(assertion.status, ": ").concat(assertion.assertion, " observed=").concat(assertion.observed || assertion.dataProof || assertion.message || ''), 1600);
+                if (evidence.length >= maxEvidenceItems) {
+                    break;
+                }
+            }
+        }
+        catch (e_8_1) { e_8 = { error: e_8_1 }; }
+        finally {
+            try {
+                if (_l && !_l.done && (_c = _k.return)) _c.call(_k);
+            }
+            finally { if (e_8) throw e_8.error; }
+        }
+    }
+    return {
+        runId: run.id || run.sourceIds.jobId || run.sourceIds.ticketId || run.sourceIds.appId || run.sourceIds.conversationId || "".concat(run.source, ":unknown"),
+        source: run.source,
+        inputSummary: [
+            "source=".concat(run.source),
+            "title=".concat(run.title || ''),
+            "status=".concat(run.status || ''),
+            "phase=".concat(run.phase || ''),
+            "ids=".concat(JSON.stringify(run.sourceIds || {}))
+        ].join(' | '),
+        actionSummary: [
+            "outcome=".concat(run.outcome),
+            "failureClass=".concat(classifyAIRunFailureClass(run)),
+            "nextAction=".concat(run.nextAction || '')
+        ].join(' | '),
+        outcome: run.outcome,
+        evidence: evidence.slice(0, maxEvidenceItems),
+        cost: run.cost,
+        metadata: {
+            warnings: run.warnings || [],
+            gateCount: (run.gates || []).length,
+            eventCount: (run.events || []).length,
+            qaOutcome: (_e = run.qa) === null || _e === void 0 ? void 0 : _e.outcome
+        }
+    };
+}
+function buildAIRunDatasetExport(runs, options) {
+    if (options === void 0) { options = {}; }
+    var generatedAt = isoNow(options.now);
+    var trainingExamples = runs.map(function (run) {
+        var example = buildAIRunTrainingExampleFromRun(run, options.maxEvidenceItems || 30);
+        return options.redact === false ? example : (0, ai_run_evidence_1.redactAIRunTrainingExample)(example);
+    });
+    var estimatedUsd = runs.reduce(function (sum, run) { var _a; return sum + (Number((_a = run.cost) === null || _a === void 0 ? void 0 : _a.estimatedUsd) || 0); }, 0);
+    var totalTokens = runs.reduce(function (sum, run) { var _a; return sum + (Number((_a = run.cost) === null || _a === void 0 ? void 0 : _a.totalTokens) || 0); }, 0);
+    return {
+        generatedAt: generatedAt,
+        totalRuns: runs.length,
+        summary: {
+            bySource: countBy(runs.map(function (run) { return run.source; })),
+            byOutcome: countBy(runs.map(function (run) { return run.outcome; })),
+            byFailureClass: countBy(runs.map(function (run) { return classifyAIRunFailureClass(run); })),
+            estimatedUsd: Number(estimatedUsd.toFixed(6)),
+            totalTokens: totalTokens
+        },
+        runs: options.includeRuns === false ? [] : runs,
+        trainingExamples: trainingExamples
+    };
+}
+function decideAIRunRetry(input) {
+    var failureClass = classifyAIRunFailureClass(input.currentRun);
+    var previousFailureClass = input.previousRun ? classifyAIRunFailureClass(input.previousRun) : undefined;
+    var sameFailureCount = Number(input.sameFailureCount || 0);
+    var unchangedFailureLimit = Number(input.unchangedFailureLimit || 2);
+    var repeatedFailure = previousFailureClass === failureClass && (Number(input.newEvidenceCount || 0) <= 0 || sameFailureCount >= unchangedFailureLimit);
+    var retryExpectedValue;
+    if (input.historicalAcceptanceRate !== undefined && input.estimatedNextRetryCostUsd !== undefined) {
+        retryExpectedValue = Number((Number(input.historicalAcceptanceRate) - Number(input.estimatedNextRetryCostUsd)).toFixed(6));
+    }
+    if (input.currentRun.outcome === 'accepted') {
+        return {
+            action: 'none',
+            failureClass: failureClass,
+            repeatedFailure: false,
+            reason: 'Run is already accepted.',
+            nextAction: 'No retry required.',
+            retryExpectedValue: retryExpectedValue
+        };
+    }
+    if (failureClass === 'manual') {
+        return {
+            action: 'manual_handoff',
+            failureClass: failureClass,
+            repeatedFailure: repeatedFailure,
+            reason: 'Run is marked for manual handoff.',
+            nextAction: input.currentRun.nextAction || 'Ask an operator to resolve the run.',
+            retryExpectedValue: retryExpectedValue
+        };
+    }
+    if (failureClass === 'stopped') {
+        return {
+            action: 'stop',
+            failureClass: failureClass,
+            repeatedFailure: repeatedFailure,
+            reason: 'Run was stopped before accepted evidence was recorded.',
+            nextAction: input.currentRun.nextAction || 'Do not spend another model cycle unless the request is resumed.',
+            retryExpectedValue: retryExpectedValue
+        };
+    }
+    if (repeatedFailure) {
+        return {
+            action: 'stop',
+            failureClass: failureClass,
+            repeatedFailure: repeatedFailure,
+            reason: 'The same failure class repeated with no new evidence. Stop model loops until the evidence changes.',
+            nextAction: input.currentRun.nextAction || 'Inspect the failing gate before retrying.',
+            retryExpectedValue: retryExpectedValue
+        };
+    }
+    if (retryExpectedValue !== undefined && retryExpectedValue < Number(input.minRetryExpectedValue || 0)) {
+        return {
+            action: 'stop',
+            failureClass: failureClass,
+            repeatedFailure: repeatedFailure,
+            reason: 'Historical retry expected value is below the configured threshold.',
+            nextAction: input.currentRun.nextAction || 'Do not retry without new evidence or a cheaper targeted repair.',
+            retryExpectedValue: retryExpectedValue
+        };
+    }
+    if (failureClass === 'infra') {
+        return {
+            action: 'retry_infra_repair',
+            failureClass: failureClass,
+            repeatedFailure: repeatedFailure,
+            reason: 'QA infrastructure failed before business validation.',
+            nextAction: input.currentRun.nextAction || 'Repair browser, settings, Mongo, ports, or startup before model repair.',
+            retryExpectedValue: retryExpectedValue
+        };
+    }
+    if (failureClass === 'compile') {
+        return {
+            action: 'retry_build_repair',
+            failureClass: failureClass,
+            repeatedFailure: repeatedFailure,
+            reason: 'Compile/build evidence failed or was stale.',
+            nextAction: input.currentRun.nextAction || 'Repair build before browser QA.',
+            retryExpectedValue: retryExpectedValue
+        };
+    }
+    if (failureClass === 'route') {
+        return {
+            action: 'retry_route_repair',
+            failureClass: failureClass,
+            repeatedFailure: repeatedFailure,
+            reason: 'Route probe failed before business proof.',
+            nextAction: input.currentRun.nextAction || 'Repair route/auth/startup and rerun the exact probe.',
+            retryExpectedValue: retryExpectedValue
+        };
+    }
+    if (failureClass === 'journey_contract') {
+        return {
+            action: 'retry_journey_contract',
+            failureClass: failureClass,
+            repeatedFailure: repeatedFailure,
+            reason: 'AICoder Journey Contract is missing or invalid before workflow build/proof.',
+            nextAction: input.currentRun.nextAction || 'Generate or repair docs/APP_JOURNEY_CONTRACT.md, validate the north-star workflow, then regenerate workflow QA rows.',
+            retryExpectedValue: retryExpectedValue
+        };
+    }
+    if (failureClass === 'assistant_missing_evidence') {
+        return {
+            action: 'retry_assistant_answer_quality',
+            failureClass: failureClass,
+            repeatedFailure: repeatedFailure,
+            reason: 'Assistant answer is missing structured answerQuality, query proof, citations, confidence, or next-action evidence.',
+            nextAction: input.currentRun.nextAction || 'Rebuild the answerQuality record from the exact query/tool evidence before another broad assistant response.',
+            retryExpectedValue: retryExpectedValue
+        };
+    }
+    if (failureClass === 'assistant_query_error') {
+        return {
+            action: 'retry_assistant_query',
+            failureClass: failureClass,
+            repeatedFailure: repeatedFailure,
+            reason: 'Assistant data access failed with a query error.',
+            nextAction: input.currentRun.nextAction || 'Repair and rerun the failing query/tool call, then answer only from the new query evidence.',
+            retryExpectedValue: retryExpectedValue
+        };
+    }
+    if (failureClass === 'assistant_permission') {
+        return {
+            action: 'request_assistant_permission',
+            failureClass: failureClass,
+            repeatedFailure: repeatedFailure,
+            reason: 'Assistant data access was blocked by permission or authorization.',
+            nextAction: input.currentRun.nextAction || 'Ask for permission, escalate to an authorized user, or answer with the permission blocker instead of claiming no data.',
+            retryExpectedValue: retryExpectedValue
+        };
+    }
+    if (failureClass === 'assistant_date_window') {
+        return {
+            action: 'retry_assistant_date_window',
+            failureClass: failureClass,
+            repeatedFailure: repeatedFailure,
+            reason: 'Assistant date-sensitive answer used the wrong current date or omitted the concrete date window.',
+            nextAction: input.currentRun.nextAction || 'Rerun the query using the actual current date and record exact start/end date-window evidence.',
+            retryExpectedValue: retryExpectedValue
+        };
+    }
+    if (failureClass === 'assistant_query_shape') {
+        return {
+            action: 'retry_assistant_query_shape',
+            failureClass: failureClass,
+            repeatedFailure: repeatedFailure,
+            reason: 'Assistant query shape is illegal or unsafe for Mongo.',
+            nextAction: input.currentRun.nextAction || 'Rewrite the query with a legal projection or aggregation shape before retrying the data answer.',
+            retryExpectedValue: retryExpectedValue
+        };
+    }
+    if (failureClass === 'release') {
+        return {
+            action: 'retry_release_repair',
+            failureClass: failureClass,
+            repeatedFailure: repeatedFailure,
+            reason: 'Business proof exists but deploy, publish, or sample-data gates block release.',
+            nextAction: input.currentRun.nextAction || 'Repair release gates without another broad model planning loop.',
+            retryExpectedValue: retryExpectedValue
+        };
+    }
+    if (failureClass === 'business' || failureClass === 'false_pass' || failureClass === 'assistant_correctness') {
+        return {
+            action: 'retry_business_repair',
+            failureClass: failureClass,
+            repeatedFailure: repeatedFailure,
+            reason: 'Business or assistant correctness proof failed or is missing.',
+            nextAction: input.currentRun.nextAction || 'Repair the exact failing workflow/query and rerun the deterministic assertion.',
+            retryExpectedValue: retryExpectedValue
+        };
+    }
+    return {
+        action: 'manual_handoff',
+        failureClass: failureClass,
+        repeatedFailure: repeatedFailure,
+        reason: 'Run lacks enough normalized evidence to choose a safe retry lane.',
+        nextAction: input.currentRun.nextAction || 'Ingest more run evidence before retrying.',
+        retryExpectedValue: retryExpectedValue
+    };
+}
+var supportAccepted004131 = (0, ai_run_evidence_1.buildAIRun)({
+    id: 'support-004131',
+    source: 'support_ticket',
+    sourceIds: { ticketNumber: '004131', ticketId: 'ticket-004131' },
+    title: '004131 accepted support fix',
+    status: 'Resolved',
+    phase: 'COMPLETE',
+    events: [{ type: 'git_commit', message: 'Fix support workflow and verified before/action/after proof.' }],
+    qa: (0, ai_run_evidence_1.buildAIQaRun)({
+        compile: { status: 'pass', artifactPath: '.build-output/build-server.log' },
+        routeProbes: [{ route: '/support/004131', status: 'pass', screenshot: 'qa-artifacts/004131-after.jpg' }],
+        businessAssertions: [{
+                assertion: 'Before/action/after workflow proves the ticket fix.',
+                status: 'pass',
+                action: 'Run the ticket workflow against seeded data.',
+                expected: 'The corrected state is visible.',
+                observed: 'The corrected state is visible after the action.',
+                artifactPaths: ['qa-artifacts/004131-proof.json']
+            }]
+    }),
+    explicitAccepted: true,
+    terminal: true
+});
+var supportBuildFailed004333 = (0, ai_run_evidence_1.buildAIRun)({
+    id: 'support-004333',
+    source: 'support_ticket',
+    sourceIds: { ticketNumber: '004333', ticketId: 'ticket-004333' },
+    title: '004333 compile failure',
+    status: 'Running',
+    phase: 'QA',
+    events: [{ type: 'log', message: 'Angular build failed before browser QA.' }],
+    qa: (0, ai_run_evidence_1.buildAIQaRun)({
+        compile: { status: 'fail', artifactPath: '.build-output/build-client.log', message: 'TypeScript compile failed.' }
+    })
+});
+var supportInfraFailed004341 = (0, ai_run_evidence_1.buildAIRun)({
+    id: 'support-004341',
+    source: 'support_ticket',
+    sourceIds: { ticketNumber: '004341', ticketId: 'ticket-004341' },
+    title: '004341 missing Puppeteer',
+    status: 'Running',
+    phase: 'QA',
+    events: [{ type: 'qa_infra', message: 'Puppeteer module could not be loaded.' }],
+    qa: (0, ai_run_evidence_1.buildAIQaRun)({
+        infraChecks: [{ name: 'puppeteer_require', status: 'blocked', message: 'Cannot find module puppeteer.' }]
+    })
+});
+var supportFalsePass004421 = (0, ai_run_evidence_1.buildAIRun)({
+    id: 'support-004421',
+    source: 'support_ticket',
+    sourceIds: { ticketNumber: '004421', ticketId: 'ticket-004421' },
+    title: '004421 route-only support QA',
+    status: 'Passed',
+    phase: 'QA',
+    events: [{ type: 'scorecard', message: 'Scorecard passed after route loaded.' }],
+    qa: (0, ai_run_evidence_1.buildAIQaRun)({
+        infraChecks: [{ name: 'chrome_launch', status: 'pass' }],
+        compile: { status: 'pass' },
+        routeProbes: [{ route: '/support/004421', status: 'pass', screenshot: 'qa-artifacts/004421-route.jpg' }]
+    }),
+    scorecardPassed: true
+});
+var supportBusinessFailed004423 = (0, ai_run_evidence_1.buildAIRun)({
+    id: 'support-004423',
+    source: 'support_ticket',
+    sourceIds: { ticketNumber: '004423', ticketId: 'ticket-004423' },
+    title: '004423 business assertion failed',
+    status: 'Running',
+    phase: 'QA',
+    events: [{ type: 'qa_business_assertion', message: 'The expected workflow state did not change.' }],
+    qa: (0, ai_run_evidence_1.buildAIQaRun)({
+        compile: { status: 'pass' },
+        routeProbes: [{ route: '/support/004423', status: 'pass' }],
+        businessAssertions: [{
+                assertion: 'Ticket workflow changes the requested field.',
+                status: 'fail',
+                expected: 'Field updates after save.',
+                observed: 'Field stayed unchanged.'
+            }]
+    })
+});
+var supportStaleBuild004430 = (0, ai_run_evidence_1.buildAIRun)({
+    id: 'support-004430',
+    source: 'support_ticket',
+    sourceIds: { ticketNumber: '004430', ticketId: 'ticket-004430' },
+    title: '004430 stale build evidence',
+    status: 'Running',
+    phase: 'QA',
+    events: [{ type: 'qa_compile', message: 'Build log predates the final code diff.' }],
+    qa: (0, ai_run_evidence_1.buildAIQaRun)({
+        compile: { status: 'stale', staleEvidence: true, artifactPath: '.build-output/build-client.log' }
+    })
+});
+var cryptoPortfolioAccepted = (0, ai_run_evidence_1.buildAIRun)({
+    id: 'aicoder-crypto-v6-golden',
+    source: 'aicoder_app',
+    sourceIds: { appId: 'crypto-v6-golden', jobId: 'crypto-job-golden' },
+    title: 'Crypto Portfolio Tracker V6 Golden',
+    status: 'Passed',
+    phase: 'COMPLETE',
+    events: [{ type: 'scorecard', message: 'Quality scorecard passed with workflow proof.' }],
+    qa: (0, ai_run_evidence_1.buildAIQaRun)({
+        compile: { status: 'pass' },
+        routeProbes: [{ route: '/portfolio', status: 'pass', screenshot: 'qa-artifacts/crypto-dashboard.jpg' }],
+        businessAssertions: [{
+                assertion: 'Portfolio dashboard calculates seeded crypto holdings.',
+                status: 'pass',
+                dataProof: 'Seeded BTC and ETH holdings render totals, allocation, and change.'
+            }]
+    }),
+    scorecardPassed: true,
+    deployStatus: 'pass',
+    publishStatus: 'published',
+    sampleDataStatus: 'pass',
+    terminal: true
+});
+var layflatReleaseBlocked = (0, ai_run_evidence_1.buildAIRun)({
+    id: 'aicoder-layflat-pressure-optimizer',
+    source: 'aicoder_app',
+    sourceIds: { appId: 'layflat-pressure-optimizer', jobId: 'layflat-job' },
+    title: 'Layflat Pressure Optimizer',
+    status: 'Passed',
+    phase: 'COMPLETE',
+    events: [{ type: 'scorecard', message: 'Scorecard passed but sample data/publish gate failed.' }],
+    qa: (0, ai_run_evidence_1.buildAIQaRun)({
+        compile: { status: 'pass' },
+        routeProbes: [{ route: '/optimizer', status: 'pass', screenshot: 'qa-artifacts/layflat-route.jpg' }],
+        businessAssertions: [{
+                assertion: 'Pressure optimizer computes a layflat hose result.',
+                status: 'pass',
+                dataProof: 'A seeded hose diameter and flow rate produce PSI loss.'
+            }]
+    }),
+    scorecardPassed: true,
+    deployStatus: 'pass',
+    publishStatus: 'publish failed',
+    sampleDataStatus: 'empty seed collections',
+    terminal: true
+});
+var assistantWrongDateWindow = (0, ai_run_evidence_1.buildAIRun)({
+    id: 'assistant-wrong-month-window',
+    source: 'ai_assistant',
+    sourceIds: { conversationId: 'assistant-date-window' },
+    title: 'Assistant wrong month/date-window answer',
+    status: 'Closed',
+    phase: 'ANSWERED',
+    events: [{ type: 'assistant_message', message: 'Answered with the wrong month for today-sensitive invoice data.' }],
+    qa: (0, ai_run_evidence_1.buildAIQaRun)({
+        businessAssertions: [{
+                assertion: 'Assistant date window uses the actual current date.',
+                status: 'fail',
+                expected: 'Use June 2026 for current-month windows.',
+                observed: 'Used a stale month.'
+            }]
+    }),
+    gates: [{
+            key: 'assistant_answer_quality',
+            label: 'Assistant answer quality',
+            status: 'fail',
+            reason: 'Assistant answer did not prove it used the actual current date.',
+            evidenceRefs: [],
+            recordedAt: '2026-06-16T00:00:00.000Z',
+            metadata: {
+                status: 'date_incorrect',
+                queryStatus: 'ok',
+                dateBasis: '2026-05-01',
+                expectedCurrentDate: '2026-06-16'
+            }
+        }],
+    metadata: {
+        answerQuality: {
+            status: 'date_incorrect',
+            queryStatus: 'ok',
+            dateBasis: '2026-05-01',
+            expectedCurrentDate: '2026-06-16'
+        }
+    },
+    rejected: true,
+    terminal: true
+});
+var assistantIllegalProjection = (0, ai_run_evidence_1.buildAIRun)({
+    id: 'assistant-illegal-mongo-projection',
+    source: 'ai_assistant',
+    sourceIds: { conversationId: 'assistant-positional-projection' },
+    title: 'Assistant illegal Mongo positional projection',
+    status: 'Closed',
+    phase: 'ANSWERED',
+    events: [{ type: 'assistant_message', message: 'Mongo query used an illegal positional projection shape.' }],
+    qa: (0, ai_run_evidence_1.buildAIQaRun)({
+        businessAssertions: [{
+                assertion: 'Assistant avoids illegal Mongo positional projection.',
+                status: 'fail',
+                expected: 'Use legal aggregation or field projection shape.',
+                observed: 'Used invalid positional projection.'
+            }]
+    }),
+    gates: [{
+            key: 'assistant_answer_quality',
+            label: 'Assistant answer quality',
+            status: 'fail',
+            reason: 'Assistant must use legal Mongo query/projection shapes before answering.',
+            evidenceRefs: [],
+            recordedAt: '2026-06-16T00:00:00.000Z',
+            metadata: {
+                status: 'illegal_query_shape',
+                queryStatus: 'ok',
+                legalQueryShape: false
+            }
+        }],
+    metadata: {
+        answerQuality: {
+            status: 'illegal_query_shape',
+            queryStatus: 'ok',
+            legalQueryShape: false
+        }
+    },
+    rejected: true,
+    terminal: true
+});
+var qaMissingPuppeteer = (0, ai_run_evidence_1.buildAIRun)({
+    id: 'qa-missing-puppeteer',
+    source: 'qa_runner',
+    sourceIds: { jobId: 'qa-missing-puppeteer' },
+    title: 'QA runner missing Puppeteer',
+    status: 'Running',
+    phase: 'PREFLIGHT',
+    events: [{ type: 'qa_infra', message: 'Puppeteer failed to require.' }],
+    qa: (0, ai_run_evidence_1.buildAIQaRun)({
+        infraChecks: [{ name: 'puppeteer_require', status: 'blocked', message: 'Cannot find module puppeteer.' }]
+    })
+});
+var qaBadChromePath = (0, ai_run_evidence_1.buildAIRun)({
+    id: 'qa-bad-chrome-path',
+    source: 'qa_runner',
+    sourceIds: { jobId: 'qa-bad-chrome-path' },
+    title: 'QA runner bad Chrome path',
+    status: 'Running',
+    phase: 'PREFLIGHT',
+    events: [{ type: 'qa_infra', message: 'Chrome executable path was missing.' }],
+    qa: (0, ai_run_evidence_1.buildAIQaRun)({
+        infraChecks: [{ name: 'chrome_executable', status: 'fail', message: 'Chrome executable not found.', path: '/bad/chrome' }]
+    })
+});
+var qaCompileFailure = (0, ai_run_evidence_1.buildAIRun)({
+    id: 'qa-compile-failure',
+    source: 'qa_runner',
+    sourceIds: { jobId: 'qa-compile-failure' },
+    title: 'QA runner compile failure',
+    status: 'Running',
+    phase: 'COMPILE',
+    events: [{ type: 'qa_compile', message: 'Compile failed before browser QA.' }],
+    qa: (0, ai_run_evidence_1.buildAIQaRun)({
+        compile: { status: 'fail', command: 'npm run build', artifactPath: '.build-output/build.log' }
+    })
+});
+var qaRouteOnlyPass = (0, ai_run_evidence_1.buildAIRun)({
+    id: 'qa-route-only-pass',
+    source: 'qa_runner',
+    sourceIds: { jobId: 'qa-route-only-pass' },
+    title: 'QA runner route-only pass',
+    status: 'Passed',
+    phase: 'BROWSER_QA',
+    events: [{ type: 'qa_route_probe', message: 'Route loaded but no business proof exists.' }],
+    qa: (0, ai_run_evidence_1.buildAIQaRun)({
+        compile: { status: 'pass' },
+        routeProbes: [{ route: '/dashboard', status: 'pass', screenshot: 'qa-artifacts/dashboard.jpg' }]
+    }),
+    scorecardPassed: true
+});
+var qaShellOnlyPage = (0, ai_run_evidence_1.buildAIRun)({
+    id: 'qa-shell-only-page',
+    source: 'qa_runner',
+    sourceIds: { jobId: 'qa-shell-only-page' },
+    title: 'QA runner shell-only page',
+    status: 'Running',
+    phase: 'BROWSER_QA',
+    events: [{ type: 'qa_route_probe', message: 'Shell-only route loaded.' }],
+    qa: (0, ai_run_evidence_1.buildAIQaRun)({
+        compile: { status: 'pass' },
+        routeProbes: [{ route: '/dashboard', status: 'pass', shellOnly: true, screenshot: 'qa-artifacts/shell.jpg' }]
+    })
+});
+var qaEmptyData = (0, ai_run_evidence_1.buildAIRun)({
+    id: 'qa-empty-data',
+    source: 'qa_runner',
+    sourceIds: { jobId: 'qa-empty-data' },
+    title: 'QA runner empty data',
+    status: 'Failed',
+    phase: 'BROWSER_QA',
+    events: [{ type: 'qa_business_assertion', message: 'Workflow rendered empty data.' }],
+    qa: (0, ai_run_evidence_1.buildAIQaRun)({
+        compile: { status: 'pass' },
+        routeProbes: [{ route: '/invoices', status: 'pass' }],
+        businessAssertions: [{
+                assertion: 'Invoice dashboard renders non-empty meaningful data.',
+                status: 'fail',
+                expected: 'At least one meaningful invoice row.',
+                observed: 'Empty table.'
+            }]
+    }),
+    terminal: true
+});
+var qaPublishFailureAfterScorecard = (0, ai_run_evidence_1.buildAIRun)({
+    id: 'qa-scorecard-pass-publish-failure',
+    source: 'qa_runner',
+    sourceIds: { jobId: 'qa-publish-failure' },
+    title: 'QA scorecard pass followed by publish failure',
+    status: 'Passed',
+    phase: 'RELEASE',
+    events: [{ type: 'publish', message: 'Scorecard passed but publish failed.' }],
+    qa: (0, ai_run_evidence_1.buildAIQaRun)({
+        compile: { status: 'pass' },
+        routeProbes: [{ route: '/app', status: 'pass' }],
+        businessAssertions: [{
+                assertion: 'Primary app workflow runs.',
+                status: 'pass',
+                dataProof: 'Workflow action returned expected state.'
+            }]
+    }),
+    scorecardPassed: true,
+    publishStatus: 'publish failed',
+    sampleDataStatus: 'pass'
+});
+exports.AI_RUN_SEED_EVAL_FIXTURES = [
+    {
+        id: 'support-004131',
+        title: 'Support 004131 accepted proof',
+        source: 'support_ticket',
+        run: supportAccepted004131,
+        expected: { outcome: 'accepted', failureClass: 'none', gates: [{ key: 'qa_business_assertion', status: 'pass' }] },
+        tags: ['support', 'accepted', 'business_assertion']
+    },
+    {
+        id: 'support-004333',
+        title: 'Support 004333 compile failure',
+        source: 'support_ticket',
+        run: supportBuildFailed004333,
+        expected: { outcome: 'build_failed', failureClass: 'compile', nextActionIncludes: 'compile' },
+        tags: ['support', 'compile']
+    },
+    {
+        id: 'support-004341',
+        title: 'Support 004341 Puppeteer infra failure',
+        source: 'support_ticket',
+        run: supportInfraFailed004341,
+        expected: { outcome: 'qa_infra_failed', failureClass: 'infra', nextActionIncludes: 'browser' },
+        tags: ['support', 'infra', 'puppeteer']
+    },
+    {
+        id: 'support-004421',
+        title: 'Support 004421 route-only false pass',
+        source: 'support_ticket',
+        run: supportFalsePass004421,
+        expected: { outcome: 'false_pass', failureClass: 'false_pass', gates: [{ key: 'qa_business_assertion', status: 'blocked' }] },
+        tags: ['support', 'route_only', 'false_pass']
+    },
+    {
+        id: 'support-004423',
+        title: 'Support 004423 business assertion failed',
+        source: 'support_ticket',
+        run: supportBusinessFailed004423,
+        expected: { outcome: 'qa_incomplete', failureClass: 'business', nextActionIncludes: 'assertion' },
+        tags: ['support', 'business_assertion']
+    },
+    {
+        id: 'support-004430',
+        title: 'Support 004430 stale build evidence',
+        source: 'support_ticket',
+        run: supportStaleBuild004430,
+        expected: { outcome: 'build_failed', failureClass: 'compile', nextActionIncludes: 'build' },
+        tags: ['support', 'compile', 'stale_build']
+    },
+    {
+        id: 'aicoder-crypto-v6-golden',
+        title: 'AICoder Crypto Portfolio Tracker V6 Golden',
+        source: 'aicoder_app',
+        run: cryptoPortfolioAccepted,
+        expected: { outcome: 'accepted', failureClass: 'none', gates: [{ key: 'qa_business_assertion', status: 'pass' }] },
+        tags: ['aicoder', 'crypto', 'accepted']
+    },
+    {
+        id: 'aicoder-layflat-pressure-optimizer',
+        title: 'AICoder Layflat Pressure Optimizer release blocked',
+        source: 'aicoder_app',
+        run: layflatReleaseBlocked,
+        expected: { outcome: 'release_blocked', failureClass: 'release', gates: [{ key: 'release', status: 'fail' }] },
+        tags: ['aicoder', 'layflat', 'release_blocked', 'sample_data']
+    },
+    {
+        id: 'assistant-wrong-month-window',
+        title: 'Assistant wrong month/date-window answer',
+        source: 'ai_assistant',
+        run: assistantWrongDateWindow,
+        expected: { outcome: 'rejected', failureClass: 'assistant_date_window' },
+        tags: ['assistant', 'date_window']
+    },
+    {
+        id: 'assistant-illegal-mongo-projection',
+        title: 'Assistant illegal Mongo positional projection',
+        source: 'ai_assistant',
+        run: assistantIllegalProjection,
+        expected: { outcome: 'rejected', failureClass: 'assistant_query_shape' },
+        tags: ['assistant', 'mongo_query']
+    },
+    {
+        id: 'qa-missing-puppeteer',
+        title: 'QA missing Puppeteer',
+        source: 'qa_runner',
+        run: qaMissingPuppeteer,
+        expected: { outcome: 'qa_infra_failed', failureClass: 'infra' },
+        tags: ['qa_runner', 'infra', 'puppeteer']
+    },
+    {
+        id: 'qa-bad-chrome-path',
+        title: 'QA bad Chrome path',
+        source: 'qa_runner',
+        run: qaBadChromePath,
+        expected: { outcome: 'qa_infra_failed', failureClass: 'infra' },
+        tags: ['qa_runner', 'infra', 'chrome']
+    },
+    {
+        id: 'qa-compile-failure',
+        title: 'QA compile failure',
+        source: 'qa_runner',
+        run: qaCompileFailure,
+        expected: { outcome: 'build_failed', failureClass: 'compile' },
+        tags: ['qa_runner', 'compile']
+    },
+    {
+        id: 'qa-route-only-pass',
+        title: 'QA route-only pass',
+        source: 'qa_runner',
+        run: qaRouteOnlyPass,
+        expected: { outcome: 'false_pass', failureClass: 'false_pass' },
+        tags: ['qa_runner', 'route_only', 'false_pass']
+    },
+    {
+        id: 'qa-shell-only-page',
+        title: 'QA shell-only page',
+        source: 'qa_runner',
+        run: qaShellOnlyPage,
+        expected: { outcome: 'qa_incomplete', failureClass: 'route' },
+        tags: ['qa_runner', 'route', 'shell_only']
+    },
+    {
+        id: 'qa-empty-data',
+        title: 'QA empty data',
+        source: 'qa_runner',
+        run: qaEmptyData,
+        expected: { outcome: 'rejected', failureClass: 'business' },
+        tags: ['qa_runner', 'empty_data', 'business_assertion']
+    },
+    {
+        id: 'qa-scorecard-pass-publish-failure',
+        title: 'QA scorecard pass followed by publish failure',
+        source: 'qa_runner',
+        run: qaPublishFailureAfterScorecard,
+        expected: { outcome: 'release_blocked', failureClass: 'release' },
+        tags: ['qa_runner', 'release_blocked', 'publish']
+    }
+];
+
+//# sourceMappingURL=ai-run-evidence-eval.js.map