@mcoda/codali 0.1.87 → 0.1.89

package/dist/eval/GatewayEvalSuite.js

@@ -0,0 +1,673 @@
+import { randomUUID } from "node:crypto";
+export const DEFAULT_CODALI_GATEWAY_EVAL_THRESHOLDS = {
+    plannerSchemaValidityMin: 1,
+    evidencePrecisionMin: 1,
+    citationSourceCorrectnessMin: 1,
+    disabledToolLeakageMax: 0,
+    finalAnswerDirectnessMin: 1,
+    finalLargeModelRateMin: 1,
+    budgetComplianceMin: 1,
+    latencyP95MsMax: 120000,
+    costP95UsdMax: 10,
+    latencyRegressionRatioMax: 0.25,
+    costRegressionRatioMax: 0.25,
+};
+export const CODALI_GATEWAY_EVAL_CASES = [
+    {
+        id: "gateway-generic-question",
+        type: "generic_question",
+        prompt: "Answer a simple product-neutral question without calling tools.",
+        expectations: {
+            allowedTools: [],
+            deniedTools: [
+                "github_search",
+                "jira_search",
+                "microsoft_graph",
+                "smartclick_search",
+            ],
+            requiresFinalLargeModel: true,
+            maxLatencyMs: 2000,
+            maxTokens: 800,
+            maxToolCalls: 0,
+            maxModelCalls: 3,
+            maxCostUsd: 0.05,
+        },
+    },
+    {
+        id: "gateway-code-repo-question",
+        type: "code_repo_question",
+        prompt: "Answer a repository question using Docdex evidence from the tenant repo.",
+        expectations: {
+            allowedTools: ["docdex_search", "docdex_open"],
+            requiredTools: ["docdex_search"],
+            requiredSourceTypes: ["docdex"],
+            deniedTools: ["shell", "write_file", "git_push"],
+            requiresEvidence: true,
+            requiresFinalLargeModel: true,
+            maxLatencyMs: 5000,
+            maxTokens: 2000,
+            maxToolCalls: 3,
+            maxModelCalls: 4,
+            maxCostUsd: 0.15,
+        },
+    },
+    {
+        id: "gateway-encrypted-docdex-search-question",
+        type: "encrypted_docdex_search_question",
+        prompt: "Answer a tenant-scoped encrypted Docdex search question.",
+        expectations: {
+            allowedTools: ["docdex_search", "docdex_open"],
+            requiredTools: ["docdex_search"],
+            requiredSourceTypes: ["encrypted_docdex"],
+            deniedTools: ["shell", "write_file", "local_docdex_fallback"],
+            requiresEvidence: true,
+            requiresFinalLargeModel: true,
+            maxLatencyMs: 8000,
+            maxTokens: 2200,
+            maxToolCalls: 4,
+            maxModelCalls: 4,
+            maxCostUsd: 0.2,
+        },
+    },
+    {
+        id: "gateway-product-tool-question",
+        type: "product_tool_question",
+        prompt: "Answer using a runtime-provided read-only app tool contract.",
+        expectations: {
+            allowedTools: ["app_tool_gateway"],
+            requiredTools: ["app_tool_gateway"],
+            requiredSourceTypes: ["app_tool_gateway"],
+            deniedTools: ["shell", "write_file", "github_issue_create"],
+            requiresEvidence: true,
+            requiresFinalLargeModel: true,
+            maxLatencyMs: 6000,
+            maxTokens: 2000,
+            maxToolCalls: 2,
+            maxModelCalls: 4,
+            maxCostUsd: 0.2,
+        },
+    },
+    {
+        id: "gateway-disabled-integration-question",
+        type: "disabled_integration_question",
+        prompt: "Handle a question that mentions disabled SmartClick/GitHub/Jira/Microsoft integrations.",
+        expectations: {
+            allowedTools: ["docdex_search"],
+            deniedTools: [
+                "smartclick_search",
+                "github_search",
+                "jira_search",
+                "microsoft_graph",
+            ],
+            requiresFinalLargeModel: true,
+            maxLatencyMs: 4000,
+            maxTokens: 1000,
+            maxToolCalls: 1,
+            maxModelCalls: 3,
+            maxCostUsd: 0.1,
+        },
+    },
+    {
+        id: "gateway-image-generation-question",
+        type: "image_generation_question",
+        prompt: "Route an image request to the image worker and preserve artifact metadata.",
+        expectations: {
+            allowedTools: ["image_generate"],
+            requiredTools: ["image_generate"],
+            deniedTools: ["shell", "write_file"],
+            requiresImageArtifact: true,
+            requiresFinalLargeModel: true,
+            maxLatencyMs: 15000,
+            maxTokens: 2500,
+            maxToolCalls: 1,
+            maxModelCalls: 5,
+            maxCostUsd: 0.5,
+        },
+    },
+    {
+        id: "gateway-missing-evidence-question",
+        type: "missing_evidence_question",
+        prompt: "Handle a repo question with insufficient evidence without fabricating.",
+        expectations: {
+            allowedTools: ["docdex_search"],
+            requiredTools: ["docdex_search"],
+            deniedTools: ["shell", "write_file"],
+            expectsMissingEvidence: true,
+            requiresFinalLargeModel: true,
+            maxLatencyMs: 4000,
+            maxTokens: 1200,
+            maxToolCalls: 2,
+            maxModelCalls: 4,
+            maxCostUsd: 0.15,
+        },
+    },
+];
+const rateMetric = (numerator, denominator, missing = 0) => ({
+    numerator,
+    denominator,
+    missing,
+    value: denominator === 0 ? null : numerator / denominator,
+});
+const percentileMetric = (values) => {
+    const samples = values.filter((value) => (value !== undefined && Number.isFinite(value))).sort((left, right) => left - right);
+    const missing = values.length - samples.length;
+    if (samples.length === 0) {
+        return { sample_size: 0, missing, median: null, p95: null };
+    }
+    const medianIndex = Math.floor((samples.length - 1) / 2);
+    const p95Index = Math.min(samples.length - 1, Math.ceil(samples.length * 0.95) - 1);
+    return {
+        sample_size: samples.length,
+        missing,
+        median: samples[medianIndex] ?? null,
+        p95: samples[p95Index] ?? null,
+    };
+};
+const includesAll = (actual, expected) => {
+    if (!expected || expected.length === 0)
+        return true;
+    const actualSet = new Set(actual);
+    return expected.every((entry) => actualSet.has(entry));
+};
+const intersects = (actual, denied) => {
+    if (!denied || denied.length === 0)
+        return false;
+    const actualSet = new Set(actual);
+    return denied.some((entry) => actualSet.has(entry));
+};
+const usedUnallowedTool = (calledTools, allowedTools) => {
+    if (!allowedTools)
+        return false;
+    const allowed = new Set(allowedTools);
+    return calledTools.some((tool) => !allowed.has(tool));
+};
+const calculateEvidencePrecision = (record, evalCase) => {
+    if (!evalCase.expectations.requiresEvidence)
+        return null;
+    if (record.evidence.length === 0)
+        return 0;
+    const relevant = record.evidence.filter((item) => item.relevant !== false).length;
+    return relevant / record.evidence.length;
+};
+const citationMatchesEvidence = (citation, evidenceById, evidenceSourceTypes) => {
+    if (citation.evidenceId && evidenceById.has(citation.evidenceId))
+        return true;
+    if (citation.sourceType && evidenceSourceTypes.has(citation.sourceType))
+        return true;
+    return false;
+};
+const hasCorrectCitations = (record, evalCase) => {
+    const requiredSourceTypes = evalCase.expectations.requiredSourceTypes ?? [];
+    if (!evalCase.expectations.requiresEvidence && requiredSourceTypes.length === 0)
+        return true;
+    if (record.citations.length === 0)
+        return false;
+    const evidenceById = new Map(record.evidence.map((item) => [item.id, item]));
+    const evidenceSourceTypes = new Set(record.evidence.map((item) => item.sourceType));
+    const citedSourceTypes = new Set();
+    for (const evidence of record.evidence) {
+        if (evidence.cited)
+            citedSourceTypes.add(evidence.sourceType);
+    }
+    for (const citation of record.citations) {
+        if (!citationMatchesEvidence(citation, evidenceById, evidenceSourceTypes)) {
+            return false;
+        }
+        if (citation.sourceType)
+            citedSourceTypes.add(citation.sourceType);
+        if (citation.evidenceId) {
+            const evidence = evidenceById.get(citation.evidenceId);
+            if (evidence)
+                citedSourceTypes.add(evidence.sourceType);
+        }
+    }
+    return includesAll(citedSourceTypes, requiredSourceTypes);
+};
+const isBudgetCompliant = (record, evalCase) => {
+    const expectations = evalCase.expectations;
+    const checks = [
+        [record.latencyMs, expectations.maxLatencyMs],
+        [record.tokensUsed, expectations.maxTokens],
+        [record.toolCallCount ?? record.calledTools.length, expectations.maxToolCalls],
+        [record.modelCallCount, expectations.maxModelCalls],
+        [record.costUsd, expectations.maxCostUsd],
+    ];
+    return checks.every(([actual, max]) => (max === undefined || (actual !== undefined && actual <= max)));
+};
+export const evaluateCodaliGatewayEvalCase = (evalCase, record, thresholds = DEFAULT_CODALI_GATEWAY_EVAL_THRESHOLDS) => {
+    const failures = [];
+    const warnings = [...(record.warnings ?? [])];
+    const errors = [...(record.errors ?? [])];
+    const expectations = evalCase.expectations;
+    const plannerSchemaValid = record.plannerSchemaValid
+        && record.taskType === evalCase.type
+        && (!record.selectedTaskType || record.selectedTaskType === evalCase.type);
+    const evidencePrecision = calculateEvidencePrecision(record, evalCase);
+    const minEvidencePrecision = expectations.minEvidencePrecision ?? thresholds.evidencePrecisionMin;
+    const evidencePrecisionPassed = !expectations.requiresEvidence
+        || (evidencePrecision !== null && evidencePrecision >= minEvidencePrecision);
+    const citationSourceCorrect = hasCorrectCitations(record, evalCase);
+    const disabledToolLeakageDetected = intersects(record.calledTools, expectations.deniedTools);
+    const unallowedToolCalled = usedUnallowedTool(record.calledTools, expectations.allowedTools);
+    const finalAnswerDirect = record.finalAnswerDirect === true && record.finalAnswer.trim().length > 0;
+    const finalLargeModelUsed = !expectations.requiresFinalLargeModel
+        || record.finalModelTier === "large";
+    const budgetCompliant = isBudgetCompliant(record, evalCase);
+    const imageArtifactPresent = !expectations.requiresImageArtifact
+        || !!record.imageArtifact?.uri
+        || !!record.imageArtifact?.id;
+    const missingEvidenceHandled = !expectations.expectsMissingEvidence
+        || record.missingEvidenceHandled === true;
+    if (record.status !== "passed")
+        failures.push("gateway_eval_runner_status_failed");
+    if (!plannerSchemaValid)
+        failures.push("gateway_planner_schema_or_task_type_invalid");
+    if (!includesAll(record.calledTools, expectations.requiredTools)) {
+        failures.push("gateway_required_tool_not_called");
+    }
+    if (unallowedToolCalled)
+        failures.push("gateway_unallowed_tool_called");
+    if (!evidencePrecisionPassed)
+        failures.push("gateway_evidence_precision_below_threshold");
+    if (!citationSourceCorrect)
+        failures.push("gateway_citation_source_incorrect");
+    if (disabledToolLeakageDetected)
+        failures.push("gateway_disabled_tool_leakage_detected");
+    if (!finalAnswerDirect)
+        failures.push("gateway_final_answer_not_direct");
+    if (!finalLargeModelUsed)
+        failures.push("gateway_final_large_model_missing");
+    if (!budgetCompliant)
+        failures.push("gateway_budget_exceeded");
+    if (!imageArtifactPresent)
+        failures.push("gateway_image_artifact_missing");
+    if (!missingEvidenceHandled)
+        failures.push("gateway_missing_evidence_not_handled");
+    return {
+        caseId: evalCase.id,
+        taskType: evalCase.type,
+        status: failures.length === 0 ? "passed" : "failed",
+        passed: failures.length === 0,
+        plannerSchemaValid,
+        evidencePrecision,
+        evidencePrecisionPassed,
+        citationSourceCorrect,
+        disabledToolLeakageDetected,
+        finalAnswerDirect,
+        finalLargeModelUsed,
+        budgetCompliant,
+        imageArtifactPresent,
+        missingEvidenceHandled,
+        calledTools: [...record.calledTools],
+        failures,
+        warnings,
+        errors,
+        latencyMs: record.latencyMs,
+        tokensUsed: record.tokensUsed,
+        costUsd: record.costUsd,
+        toolCallCount: record.toolCallCount ?? record.calledTools.length,
+        modelCallCount: record.modelCallCount,
+    };
+};
+export const aggregateCodaliGatewayEvalMetrics = (cases) => {
+    const taskCount = cases.length;
+    const evidenceCases = cases.filter((result) => result.evidencePrecision !== null);
+    const citationCases = cases.filter((result) => result.evidencePrecision !== null);
+    const finalLargeCases = cases.filter((result) => (result.finalLargeModelUsed || result.failures.includes("gateway_final_large_model_missing")));
+    return {
+        schemaVersion: 1,
+        generatedAt: new Date().toISOString(),
+        taskCount,
+        plannerSchemaValidityRate: rateMetric(cases.filter((result) => result.plannerSchemaValid).length, taskCount),
+        evidencePrecisionRate: rateMetric(evidenceCases.filter((result) => result.evidencePrecisionPassed).length, evidenceCases.length, taskCount - evidenceCases.length),
+        citationSourceCorrectnessRate: rateMetric(citationCases.filter((result) => result.citationSourceCorrect).length, citationCases.length, taskCount - citationCases.length),
+        disabledToolLeakageRate: rateMetric(cases.filter((result) => result.disabledToolLeakageDetected).length, taskCount),
+        finalAnswerDirectnessRate: rateMetric(cases.filter((result) => result.finalAnswerDirect).length, taskCount),
+        finalLargeModelRate: rateMetric(finalLargeCases.filter((result) => result.finalLargeModelUsed).length, finalLargeCases.length, taskCount - finalLargeCases.length),
+        budgetComplianceRate: rateMetric(cases.filter((result) => result.budgetCompliant).length, taskCount),
+        latencyMs: percentileMetric(cases.map((result) => result.latencyMs)),
+        tokensUsed: percentileMetric(cases.map((result) => result.tokensUsed)),
+        costUsd: percentileMetric(cases.map((result) => result.costUsd)),
+        toolCallCount: percentileMetric(cases.map((result) => result.toolCallCount)),
+        modelCallCount: percentileMetric(cases.map((result) => result.modelCallCount)),
+    };
+};
+const comparisonInput = (baseline) => {
+    if (!baseline)
+        return {};
+    if ("metrics" in baseline) {
+        return {
+            metrics: baseline.metrics,
+            reportId: baseline.reportId,
+            generatedAt: baseline.metrics.generatedAt,
+        };
+    }
+    return { metrics: baseline, generatedAt: baseline.generatedAt };
+};
+const metricDelta = (key, unit, current, baseline) => {
+    if (current === null || baseline === null) {
+        return {
+            key,
+            unit,
+            baseline,
+            current,
+            delta: null,
+            relativeDelta: null,
+            regression: false,
+        };
+    }
+    const delta = current - baseline;
+    const relativeDelta = baseline === 0 ? null : delta / baseline;
+    return {
+        key,
+        unit,
+        baseline,
+        current,
+        delta,
+        relativeDelta,
+        regression: delta > 0,
+    };
+};
+export const compareCodaliGatewayEvalBaseline = (params) => {
+    const baseline = comparisonInput(params.baseline);
+    const deltas = [
+        metricDelta("latency_ms.p95", "ms", params.current.latencyMs.p95, baseline.metrics?.latencyMs.p95 ?? null),
+        metricDelta("cost_usd.p95", "usd", params.current.costUsd.p95, baseline.metrics?.costUsd.p95 ?? null),
+        metricDelta("tokens_used.p95", "tokens", params.current.tokensUsed.p95, baseline.metrics?.tokensUsed.p95 ?? null),
+    ];
+    return {
+        schemaVersion: 1,
+        status: baseline.metrics ? "compared" : "baseline_missing",
+        baselineReportId: baseline.reportId,
+        baselineGeneratedAt: baseline.generatedAt,
+        deltas,
+        regressionCount: deltas.filter((delta) => delta.regression).length,
+    };
+};
+const valueBelowMinimum = (value, minimum) => (value === null || value < minimum);
+const valueAboveMaximum = (value, maximum) => (value === null || value > maximum);
+const addRateMinimumFailure = (failures, params) => {
+    if (valueBelowMinimum(params.actual, params.threshold)) {
+        failures.push(params);
+    }
+};
+const addRateMaximumFailure = (failures, params) => {
+    if (valueAboveMaximum(params.actual, params.threshold)) {
+        failures.push(params);
+    }
+};
+const findRegressionDelta = (regression, key) => {
+    if (!regression || regression.status !== "compared")
+        return undefined;
+    return regression.deltas.find((delta) => delta.key === key);
+};
+export const evaluateCodaliGatewayEvalGates = (params) => {
+    const thresholds = {
+        ...DEFAULT_CODALI_GATEWAY_EVAL_THRESHOLDS,
+        ...(params.thresholds ?? {}),
+    };
+    const failures = [];
+    const { metrics } = params;
+    addRateMinimumFailure(failures, {
+        code: "gateway_planner_schema_validity_below_min",
+        metric: "plannerSchemaValidityRate",
+        message: "Planner schema validity is below threshold.",
+        threshold: thresholds.plannerSchemaValidityMin,
+        actual: metrics.plannerSchemaValidityRate.value,
+    });
+    addRateMinimumFailure(failures, {
+        code: "gateway_evidence_precision_below_min",
+        metric: "evidencePrecisionRate",
+        message: "Evidence precision is below threshold.",
+        threshold: thresholds.evidencePrecisionMin,
+        actual: metrics.evidencePrecisionRate.value,
+    });
+    addRateMinimumFailure(failures, {
+        code: "gateway_citation_source_correctness_below_min",
+        metric: "citationSourceCorrectnessRate",
+        message: "Citation/source correctness is below threshold.",
+        threshold: thresholds.citationSourceCorrectnessMin,
+        actual: metrics.citationSourceCorrectnessRate.value,
+    });
+    addRateMaximumFailure(failures, {
+        code: "gateway_disabled_tool_leakage_exceeded",
+        metric: "disabledToolLeakageRate",
+        message: "Disabled-tool leakage exceeded threshold.",
+        threshold: thresholds.disabledToolLeakageMax,
+        actual: metrics.disabledToolLeakageRate.value,
+    });
+    addRateMinimumFailure(failures, {
+        code: "gateway_final_answer_directness_below_min",
+        metric: "finalAnswerDirectnessRate",
+        message: "Final-answer directness is below threshold.",
+        threshold: thresholds.finalAnswerDirectnessMin,
+        actual: metrics.finalAnswerDirectnessRate.value,
+    });
+    addRateMinimumFailure(failures, {
+        code: "gateway_final_large_model_rate_below_min",
+        metric: "finalLargeModelRate",
+        message: "Final answers were not produced by the required large tier.",
+        threshold: thresholds.finalLargeModelRateMin,
+        actual: metrics.finalLargeModelRate.value,
+    });
+    addRateMinimumFailure(failures, {
+        code: "gateway_budget_compliance_below_min",
+        metric: "budgetComplianceRate",
+        message: "Gateway budget compliance is below threshold.",
+        threshold: thresholds.budgetComplianceMin,
+        actual: metrics.budgetComplianceRate.value,
+    });
+    if (thresholds.latencyP95MsMax !== undefined) {
+        addRateMaximumFailure(failures, {
+            code: "gateway_latency_p95_exceeded",
+            metric: "latencyMs.p95",
+            message: "Gateway p95 latency exceeded threshold.",
+            threshold: thresholds.latencyP95MsMax,
+            actual: metrics.latencyMs.p95,
+        });
+    }
+    if (thresholds.costP95UsdMax !== undefined) {
+        addRateMaximumFailure(failures, {
+            code: "gateway_cost_p95_exceeded",
+            metric: "costUsd.p95",
+            message: "Gateway p95 cost exceeded threshold.",
+            threshold: thresholds.costP95UsdMax,
+            actual: metrics.costUsd.p95,
+        });
+    }
+    const latencyDelta = findRegressionDelta(params.regression, "latency_ms.p95");
+    if (latencyDelta?.relativeDelta !== null
+        && latencyDelta?.relativeDelta !== undefined
+        && latencyDelta.relativeDelta > thresholds.latencyRegressionRatioMax) {
+        failures.push({
+            code: "gateway_latency_regression_exceeded",
+            metric: "latency_ms.p95",
+            message: "Gateway p95 latency regressed beyond the allowed ratio.",
+            threshold: thresholds.latencyRegressionRatioMax,
+            actual: latencyDelta.current,
+            baseline: latencyDelta.baseline,
+            delta: latencyDelta.relativeDelta,
+        });
+    }
+    const costDelta = findRegressionDelta(params.regression, "cost_usd.p95");
+    if (costDelta?.relativeDelta !== null
+        && costDelta?.relativeDelta !== undefined
+        && costDelta.relativeDelta > thresholds.costRegressionRatioMax) {
+        failures.push({
+            code: "gateway_cost_regression_exceeded",
+            metric: "cost_usd.p95",
+            message: "Gateway p95 cost regressed beyond the allowed ratio.",
+            threshold: thresholds.costRegressionRatioMax,
+            actual: costDelta.current,
+            baseline: costDelta.baseline,
+            delta: costDelta.relativeDelta,
+        });
+    }
+    return {
+        schemaVersion: 1,
+        passed: failures.length === 0,
+        thresholds,
+        failures,
+    };
+};
+const defaultEvidenceForCase = (evalCase) => {
+    const [sourceType] = evalCase.expectations.requiredSourceTypes ?? [];
+    if (!sourceType)
+        return [];
+    return [{
+            id: `${evalCase.id}-evidence-1`,
+            sourceType,
+            sourceId: `${sourceType}:tenant-scope-1`,
+            cited: true,
+            relevant: true,
+        }];
+};
+export const createDefaultCodaliGatewayEvalRunner = () => (async (evalCase) => {
+    const evidence = defaultEvidenceForCase(evalCase);
+    const calledTools = evalCase.expectations.requiredTools
+        ? [...evalCase.expectations.requiredTools]
+        : [];
+    const citations = evidence.map((item) => ({
+        evidenceId: item.id,
+        sourceType: item.sourceType,
+        sourceId: item.sourceId,
+    }));
+    return {
+        caseId: evalCase.id,
+        taskType: evalCase.type,
+        status: "passed",
+        plannerSchemaValid: true,
+        selectedTaskType: evalCase.type,
+        calledTools,
+        evidence,
+        citations,
+        finalAnswer: evalCase.expectations.expectsMissingEvidence
+            ? "I do not have enough tenant-scoped evidence to answer that directly."
+            : "Tenant-scoped evidence supports this direct answer.",
+        finalAnswerDirect: true,
+        finalModelTier: evalCase.expectations.requiresFinalLargeModel ? "large" : "medium",
+        latencyMs: Math.min(evalCase.expectations.maxLatencyMs ?? 1000, 1000),
+        tokensUsed: Math.min(evalCase.expectations.maxTokens ?? 500, 500),
+        costUsd: Math.min(evalCase.expectations.maxCostUsd ?? 0.01, 0.01),
+        toolCallCount: calledTools.length,
+        modelCallCount: Math.min(evalCase.expectations.maxModelCalls ?? 2, 2),
+        imageArtifact: evalCase.expectations.requiresImageArtifact
+            ? { id: `${evalCase.id}-artifact`, uri: "artifact://codali-gateway-eval/image.png", mimeType: "image/png" }
+            : undefined,
+        missingEvidenceHandled: evalCase.expectations.expectsMissingEvidence ? true : undefined,
+        warnings: [],
+        errors: [],
+    };
+});
+export const runCodaliGatewayEvalSuite = async (options = {}) => {
+    const runId = options.runId ?? randomUUID();
+    const reportId = options.reportId ?? randomUUID();
+    const evalCases = options.cases ?? CODALI_GATEWAY_EVAL_CASES;
+    const runner = options.runner ?? createDefaultCodaliGatewayEvalRunner();
+    const thresholds = {
+        ...DEFAULT_CODALI_GATEWAY_EVAL_THRESHOLDS,
+        ...(options.thresholds ?? {}),
+    };
+    const startedAtMs = Date.now();
+    const startedAt = new Date(startedAtMs).toISOString();
+    const results = [];
+    const warnings = [];
+    const errors = [];
+    for (const evalCase of evalCases) {
+        try {
+            const record = await runner(evalCase);
+            const result = evaluateCodaliGatewayEvalCase(evalCase, record, thresholds);
+            results.push(result);
+            warnings.push(...result.warnings.map((warning) => `${evalCase.id}:${warning}`));
+            errors.push(...result.errors.map((error) => `${evalCase.id}:${error}`));
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            errors.push(`${evalCase.id}:${message}`);
+            results.push({
+                caseId: evalCase.id,
+                taskType: evalCase.type,
+                status: "failed",
+                passed: false,
+                plannerSchemaValid: false,
+                evidencePrecision: null,
+                evidencePrecisionPassed: false,
+                citationSourceCorrect: false,
+                disabledToolLeakageDetected: false,
+                finalAnswerDirect: false,
+                finalLargeModelUsed: false,
+                budgetCompliant: false,
+                imageArtifactPresent: !evalCase.expectations.requiresImageArtifact,
+                missingEvidenceHandled: !evalCase.expectations.expectsMissingEvidence,
+                calledTools: [],
+                failures: ["gateway_eval_runner_threw"],
+                warnings: [],
+                errors: [message],
+            });
+        }
+    }
+    const metrics = aggregateCodaliGatewayEvalMetrics(results);
+    const regression = compareCodaliGatewayEvalBaseline({
+        current: metrics,
+        baseline: options.baseline,
+    });
+    const gates = evaluateCodaliGatewayEvalGates({
+        metrics,
+        thresholds,
+        regression,
+    });
+    const endedAtMs = Date.now();
+    const passed = results.filter((result) => result.passed).length;
+    const failed = results.length - passed;
+    return {
+        schemaVersion: 1,
+        reportId,
+        runId,
+        runtime: "codali_gateway_eval",
+        mode: "gateway_smoke",
+        startedAt,
+        endedAt: new Date(endedAtMs).toISOString(),
+        durationMs: endedAtMs - startedAtMs,
+        cases: results,
+        metrics,
+        regression,
+        gates,
+        summary: {
+            status: failed === 0 && gates.passed ? "passed" : "failed",
+            total: results.length,
+            passed,
+            failed,
+        },
+        warnings,
+        errors,
+    };
+};
+const formatRate = (metric) => (metric.value === null ? "n/a" : `${(metric.value * 100).toFixed(2)}%`);
+const formatValue = (value, suffix = "") => (value === null ? "n/a" : `${value.toFixed(2)}${suffix}`);
+export const formatCodaliGatewayEvalTextReport = (report) => {
+    const gateSummary = report.gates.passed
+        ? "passed"
+        : `failed (${report.gates.failures.map((failure) => failure.code).join(", ")})`;
+    const lines = [
+        `Codali gateway eval smoke: ${report.summary.status}`,
+        `Run: ${report.runId}`,
+        `Cases: ${report.summary.passed}/${report.summary.total} passed`,
+        `Planner schema validity: ${formatRate(report.metrics.plannerSchemaValidityRate)}`,
+        `Evidence precision: ${formatRate(report.metrics.evidencePrecisionRate)}`,
+        `Citation/source correctness: ${formatRate(report.metrics.citationSourceCorrectnessRate)}`,
+        `Disabled-tool leakage: ${formatRate(report.metrics.disabledToolLeakageRate)}`,
+        `Final-answer directness: ${formatRate(report.metrics.finalAnswerDirectnessRate)}`,
+        `Final large-model usage: ${formatRate(report.metrics.finalLargeModelRate)}`,
+        `Budget compliance: ${formatRate(report.metrics.budgetComplianceRate)}`,
+        `Latency median/p95: ${formatValue(report.metrics.latencyMs.median, "ms")}/${formatValue(report.metrics.latencyMs.p95, "ms")}`,
+        `Tokens median/p95: ${formatValue(report.metrics.tokensUsed.median)}/${formatValue(report.metrics.tokensUsed.p95)}`,
+        `Cost median/p95: ${formatValue(report.metrics.costUsd.median, " USD")}/${formatValue(report.metrics.costUsd.p95, " USD")}`,
+        `Regression: ${report.regression.status}`,
+        `Gates: ${gateSummary}`,
+    ];
+    for (const result of report.cases) {
+        const suffix = result.failures.length > 0 ? ` (${result.failures.join(", ")})` : "";
+        lines.push(`Case ${result.caseId}: ${result.status}${suffix}`);
+    }
+    return lines.join("\n");
+};