@mcoda/codali 0.1.90 → 0.1.94

package/dist/eval/CodaliGatewayLiveHarness.js

@@ -191,7 +191,7 @@ export const parseCodaliGatewayLiveInventory = (payload) => {
     return [];
 };
 const secretKeyPattern = /(api[_-]?key|authorization|bearer|password|secret|token|credential)/i;
-const safeTokenMetricKeyPattern = /^(cachedInputTokens|completionTokens|contextWindow|inputTokens|maxContextPackTokens|maxOutputTokens|maxTokens|outputTokens|promptTokens|tokenEstimate|tokensUsed|totalTokens|usageTokensTotal|usage_tokens_total|cached_input_tokens|completion_tokens|context_window|input_tokens|max_context_pack_tokens|max_output_tokens|max_tokens|output_tokens|prompt_tokens|token_estimate|tokens_used|total_tokens)$/i;
+const safeTokenMetricKeyPattern = /^(cachedInputTokens|completionTokens|contextWindow|inputTokens|maxContextPackTokens|maxOutputTokens|maxTokens|outputTokens|outputTokensPerSecond|promptTokens|tokenEstimate|tokenUse|tokensPerSecond|tokensUsed|totalTokens|usageTokensTotal|usage_tokens_total|cached_input_tokens|completion_tokens|context_window|input_tokens|max_context_pack_tokens|max_output_tokens|max_tokens|output_tokens|output_tokens_per_second|prompt_tokens|token_estimate|token_use|tokens_per_second|tokens_used|total_tokens)$/i;
 const bearerPattern = /\bBearer\s+[A-Za-z0-9._~+/=-]+/gi;
 const apiKeyPattern = /\b(?:sk|pk|mswarm|mcoda)_[A-Za-z0-9_-]{12,}\b/gi;
 export const redactCodaliGatewayLiveValue = (value) => {
@@ -477,11 +477,21 @@ const normalizeArtifact = (value) => {
 const summarizeAgentRunMetadata = (metadata) => {
     const cli = asRecord(metadata?.cli);
     const usage = asRecord(metadata?.usage);
+    const localInference = asRecord(metadata?.localInference) ??
+        asRecord(metadata?.local_inference);
+    const queue = asRecord(metadata?.queue);
+    const throughput = asRecord(metadata?.throughput);
     const output = {};
     const mode = readString(metadata, ["mode"]);
     const adapterType = readString(metadata, ["adapterType", "adapter_type"]);
     const authMode = readString(metadata, ["authMode", "auth_mode"]);
     const cliVersion = readString(cli, ["version"]);
+    const costUsd = readNumber(metadata, [
+        "costUsd",
+        "cost_usd",
+        "estimatedCostUsd",
+        "estimated_cost_usd",
+    ]);
     if (mode)
         output.mode = mode;
     if (adapterType)
@@ -490,8 +500,17 @@ const summarizeAgentRunMetadata = (metadata) => {
         output.authMode = authMode;
     if (cliVersion)
         output.cli = { version: cliVersion };
+    if (costUsd !== undefined)
+        output.costUsd = costUsd;
     if (usage)
         output.usage = redactCodaliGatewayLiveValue(usage);
+    if (queue)
+        output.queue = redactCodaliGatewayLiveValue(queue);
+    if (throughput)
+        output.throughput = redactCodaliGatewayLiveValue(throughput);
+    if (localInference) {
+        output.localInference = redactCodaliGatewayLiveValue(localInference);
+    }
     return output;
 };
 const outputPreview = (text) => {
@@ -505,13 +524,26 @@ const classifyAgentRunCommandFailure = (result) => {
         normalized.includes("invalid model id") ||
         normalized.includes("model_not_found") ||
         normalized.includes("model not found");
-    const status = knownCatalogMismatch
+    const upstreamUnavailable = normalized.includes("mswarm_error") ||
+        normalized.includes("upstream_error") ||
+        normalized.includes("self-hosted node is not currently reachable") ||
+        normalized.includes("self hosted node is not currently reachable") ||
+        normalized.includes("node is not currently reachable") ||
+        normalized.includes("connection_error") ||
+        normalized.includes("econnrefused") ||
+        normalized.includes("fetch failed");
+    const degradedFailureClass = knownCatalogMismatch
+        ? "agent_run_model_catalog_mismatch"
+        : upstreamUnavailable
+            ? "agent_run_upstream_unavailable"
+            : undefined;
+    const status = degradedFailureClass
         ? "degraded"
         : "failed";
     return {
         status,
-        warnings: knownCatalogMismatch
-            ? ["agent_run_model_catalog_mismatch"]
+        warnings: degradedFailureClass
+            ? [degradedFailureClass]
             : [],
         errors: [
             `agent_run_exit_${result.exitCode}`,
@@ -520,9 +552,7 @@ const classifyAgentRunCommandFailure = (result) => {
         metadata: {
             runner: "mcoda_agent_run",
             exitCode: result.exitCode,
-            failureClass: knownCatalogMismatch
-                ? "agent_run_model_catalog_mismatch"
-                : "agent_run_command_failed",
+            failureClass: degradedFailureClass ?? "agent_run_command_failed",
         },
     };
 };
@@ -735,6 +765,333 @@ const summarizeHarness = (classification, scenarios) => {
 const assignmentForScenario = (classification, scenario) => classification.assignments[scenario.role] ??
     classification.resolution.assignments[ROLE_TO_RESOLVER_ROLE[scenario.role]];
 const redactScenario = (scenario) => redactCodaliGatewayLiveValue(scenario);
+const normalizeShadowComparisonPolicy = (policy) => ({
+    enabled: policy?.enabled === true,
+    maxCandidatesPerScenario: Math.max(0, Math.min(5, Math.trunc(policy?.maxCandidatesPerScenario ?? 1))),
+    includePrimary: policy?.includePrimary !== false,
+    requireHealthy: policy?.requireHealthy !== false,
+    roles: policy?.roles?.length
+        ? unique(policy.roles)
+        : [...new Set(CODALI_GATEWAY_LIVE_SCENARIOS.map((scenario) => scenario.role))],
+});
+const buildRuntimeAgentInput = (candidate) => ({
+    slug: candidate.slug,
+    adapter: candidate.adapter,
+    provider: candidate.provider,
+    model: candidate.model,
+    baseUrl: candidate.baseUrl,
+    runnerKind: candidate.runnerKind,
+    supportsTools: candidate.supportsTools,
+    capabilities: candidate.capabilities,
+    contextWindow: candidate.contextWindow,
+    maxOutputTokens: candidate.maxOutputTokens,
+});
+const createComparisonAssignment = (params) => ({
+    role: params.resolverRole,
+    policy: params.primary?.policy ?? { tier: params.candidate.tier },
+    candidate: params.candidate,
+    agent: buildRuntimeAgentInput(params.candidate),
+    score: params.score ?? 0,
+    reasons: params.reasons ?? [],
+});
+const resolveShadowAssignmentsForScenario = (params) => {
+    if (!params.policy.enabled || params.policy.maxCandidatesPerScenario <= 0) {
+        return [];
+    }
+    if (!params.policy.roles.includes(params.scenario.role)) {
+        return [];
+    }
+    const resolverRole = ROLE_TO_RESOLVER_ROLE[params.scenario.role];
+    const primarySlug = params.primary?.candidate.slug;
+    return params.classification.resolution.diagnostics
+        .filter((diagnostic) => diagnostic.role === resolverRole &&
+        diagnostic.eligible &&
+        diagnostic.slug !== primarySlug &&
+        diagnostic.score !== undefined)
+        .map((diagnostic) => {
+        const candidate = params.classification.resolution.candidates.find((entry) => entry.slug === diagnostic.slug);
+        if (!candidate)
+            return undefined;
+        if (params.policy.requireHealthy && candidate.healthStatus !== "healthy") {
+            return undefined;
+        }
+        return createComparisonAssignment({
+            resolverRole,
+            primary: params.primary,
+            candidate,
+            score: diagnostic.score,
+            reasons: diagnostic.reasons,
+        });
+    })
+        .filter((assignment) => Boolean(assignment))
+        .sort((left, right) => {
+        const scoreDelta = right.score - left.score;
+        if (scoreDelta !== 0)
+            return scoreDelta;
+        return left.candidate.slug.localeCompare(right.candidate.slug);
+    })
+        .slice(0, params.policy.maxCandidatesPerScenario);
+};
+const metricRecords = (metadata) => {
+    const responseMetadata = asRecord(metadata?.responseMetadata);
+    const localInference = asRecord(metadata?.localInference) ??
+        asRecord(metadata?.local_inference) ??
+        asRecord(responseMetadata?.localInference) ??
+        asRecord(responseMetadata?.local_inference);
+    const usage = asRecord(metadata?.usage) ??
+        asRecord(responseMetadata?.usage) ??
+        asRecord(localInference?.usage);
+    const queue = asRecord(metadata?.queue) ??
+        asRecord(responseMetadata?.queue) ??
+        asRecord(localInference?.queue);
+    const throughput = asRecord(metadata?.throughput) ??
+        asRecord(responseMetadata?.throughput) ??
+        asRecord(localInference?.throughput);
+    return [metadata, responseMetadata, localInference, usage, queue, throughput]
+        .filter((record) => Boolean(record));
+};
+const firstMetricNumber = (records, keys) => {
+    for (const record of records) {
+        const value = readNumber(record, keys);
+        if (value !== undefined)
+            return value;
+    }
+    return undefined;
+};
+const firstMetricString = (records, keys) => {
+    for (const record of records) {
+        const value = readString(record, keys);
+        if (value)
+            return value;
+    }
+    return undefined;
+};
+const extractTokenUse = (records) => {
+    const inputTokens = firstMetricNumber(records, [
+        "inputTokens",
+        "input_tokens",
+        "promptTokens",
+        "prompt_tokens",
+    ]);
+    const outputTokens = firstMetricNumber(records, [
+        "outputTokens",
+        "output_tokens",
+        "completionTokens",
+        "completion_tokens",
+    ]);
+    const totalTokens = firstMetricNumber(records, ["totalTokens", "total_tokens", "tokensUsed", "tokens_used"]) ??
+        (inputTokens !== undefined || outputTokens !== undefined
+            ? (inputTokens ?? 0) + (outputTokens ?? 0)
+            : undefined);
+    if (inputTokens === undefined && outputTokens === undefined && totalTokens === undefined) {
+        return undefined;
+    }
+    return { inputTokens, outputTokens, totalTokens };
+};
+const extractQueueMetrics = (metadata) => {
+    const responseMetadata = asRecord(metadata?.responseMetadata);
+    const localInference = asRecord(metadata?.localInference) ??
+        asRecord(metadata?.local_inference) ??
+        asRecord(responseMetadata?.localInference) ??
+        asRecord(responseMetadata?.local_inference);
+    const queue = asRecord(metadata?.queue) ??
+        asRecord(responseMetadata?.queue) ??
+        asRecord(localInference?.queue);
+    const records = [queue, metadata, responseMetadata, localInference]
+        .filter((record) => Boolean(record));
+    const waitMs = firstMetricNumber(records, [
+        "waitMs",
+        "wait_ms",
+        "queueWaitMs",
+        "queue_wait_ms",
+        "queuedMs",
+        "queued_ms",
+    ]);
+    const depth = firstMetricNumber(records, ["depth", "queueDepth", "queue_depth"]);
+    const status = firstMetricString(records, ["status", "queueStatus", "queue_status"]);
+    if (waitMs === undefined && depth === undefined && status === undefined) {
+        return undefined;
+    }
+    return { waitMs, depth, status };
+};
+const extractThroughputMetrics = (params) => {
+    const responseMetadata = asRecord(params.metadata?.responseMetadata);
+    const localInference = asRecord(params.metadata?.localInference) ??
+        asRecord(params.metadata?.local_inference) ??
+        asRecord(responseMetadata?.localInference) ??
+        asRecord(responseMetadata?.local_inference);
+    const throughput = asRecord(params.metadata?.throughput) ??
+        asRecord(responseMetadata?.throughput) ??
+        asRecord(localInference?.throughput);
+    const records = [throughput, params.metadata, responseMetadata, localInference]
+        .filter((record) => Boolean(record));
+    const explicitTokensPerSecond = firstMetricNumber(records, [
+        "tokensPerSecond",
+        "tokens_per_second",
+        "outputTokensPerSecond",
+        "output_tokens_per_second",
+    ]);
+    const tokensPerSecond = explicitTokensPerSecond ??
+        (params.tokenUse?.outputTokens !== undefined && params.latencyMs && params.latencyMs > 0
+            ? params.tokenUse.outputTokens / (params.latencyMs / 1000)
+            : undefined);
+    const requestsPerMinute = firstMetricNumber(records, [
+        "requestsPerMinute",
+        "requests_per_minute",
+        "requestPerMinute",
+        "request_per_minute",
+    ]);
+    if (tokensPerSecond === undefined && requestsPerMinute === undefined) {
+        return undefined;
+    }
+    return { tokensPerSecond, requestsPerMinute };
+};
+const extractLocalInferenceMetrics = (metadata) => {
+    const responseMetadata = asRecord(metadata?.responseMetadata);
+    const localInference = asRecord(metadata?.localInference) ??
+        asRecord(metadata?.local_inference) ??
+        asRecord(responseMetadata?.localInference) ??
+        asRecord(responseMetadata?.local_inference);
+    return localInference
+        ? asRecord(redactCodaliGatewayLiveValue(localInference))
+        : undefined;
+};
+const qualityScore = (scenario) => {
+    if (scenario.status === "passed")
+        return 1;
+    if (scenario.status === "degraded")
+        return 0.5;
+    return 0;
+};
+const comparisonMetrics = (scenario, assignment) => {
+    const records = metricRecords(scenario.metadata);
+    const tokenUse = extractTokenUse(records);
+    const costUsd = firstMetricNumber(records, ["costUsd", "cost_usd", "estimatedCostUsd", "estimated_cost_usd"]) ??
+        (assignment?.candidate.costPerMillion !== undefined && tokenUse?.totalTokens !== undefined
+            ? (assignment.candidate.costPerMillion * tokenUse.totalTokens) / 1000000
+            : undefined);
+    const queue = extractQueueMetrics(scenario.metadata);
+    const throughput = extractThroughputMetrics({
+        metadata: scenario.metadata,
+        tokenUse,
+        latencyMs: scenario.latencyMs,
+    });
+    const failureStatus = scenario.status === "passed"
+        ? "none"
+        : scenario.status;
+    return {
+        quality: {
+            status: scenario.status,
+            score: qualityScore(scenario),
+            jsonValid: scenario.jsonValid,
+            toolCallCount: scenario.toolCallCount,
+            artifactPresent: scenario.artifact !== undefined,
+            finalAnswerSucceeded: scenario.finalAnswerStatus === "succeeded",
+        },
+        latencyMs: scenario.latencyMs,
+        costUsd,
+        tokenUse,
+        queue,
+        throughput,
+        failure: {
+            status: failureStatus,
+            reasons: failureStatus === "none"
+                ? []
+                : unique([...scenario.errors, ...scenario.warnings]),
+        },
+        localInference: extractLocalInferenceMetrics(scenario.metadata),
+    };
+};
+const comparisonRecord = (params) => {
+    const candidate = params.assignment?.candidate;
+    const agentSlug = candidate?.slug ?? params.result.agentSlug;
+    const comparisonRole = params.primary ? "primary" : "shadow";
+    return {
+        id: `${params.scenario.id}:${comparisonRole}:${agentSlug ?? "unassigned"}`,
+        scenarioId: params.scenario.id,
+        role: params.scenario.role,
+        resolverRole: ROLE_TO_RESOLVER_ROLE[params.scenario.role],
+        comparisonRole,
+        primary: params.primary,
+        candidateRank: params.candidateRank,
+        agentSlug,
+        tier: candidate?.tier ?? params.result.tier,
+        model: params.result.model ?? candidate?.model,
+        adapter: params.result.adapter ?? candidate?.adapter,
+        source: candidate?.source,
+        healthStatus: candidate?.healthStatus,
+        capabilities: candidate?.capabilities,
+        resultStatus: params.result.status,
+        selectedByPolicy: true,
+        metrics: comparisonMetrics(params.result, params.assignment),
+        warnings: params.result.warnings,
+        errors: params.result.errors,
+        metadata: params.result.metadata,
+    };
+};
+const warningForRoleError = (role, error) => ({
+    code: "GATEWAY_LIVE_ROLE_UNAVAILABLE",
+    message: `No eligible inventory candidate found for live role ${role} (${error.role ?? ROLE_TO_RESOLVER_ROLE[role]}).`,
+    reason: error.code,
+    role,
+    resolverRole: error.role,
+    details: error.details,
+});
+const buildEnvironmentWarnings = (params) => {
+    const warnings = [];
+    if (params.discovery.status === "failed") {
+        warnings.push({
+            code: "GATEWAY_LIVE_INVENTORY_DISCOVERY_FAILED",
+            message: "Agent inventory discovery failed before live model validation.",
+            reason: params.discovery.errors.join("; ") || "inventory_command_failed",
+            details: {
+                command: params.discovery.command,
+                args: params.discovery.args,
+            },
+        });
+    }
+    for (const [role, summary] of Object.entries(params.classification.roles)) {
+        if (summary.status !== "assigned") {
+            const matchingError = params.classification.errors.find((error) => error.role === summary.resolverRole);
+            warnings.push(warningForRoleError(role, matchingError ?? {
+                code: summary.errorCodes[0] ?? "GATEWAY_AGENT_ROLE_UNRESOLVED",
+                message: `No eligible agent candidate found for role ${summary.resolverRole}.`,
+                role: summary.resolverRole,
+            }));
+            continue;
+        }
+        const assignment = params.classification.assignments[role];
+        const healthStatus = assignment?.candidate.healthStatus;
+        if (assignment && healthStatus !== "healthy") {
+            warnings.push({
+                code: "GATEWAY_LIVE_AGENT_HEALTH_DEGRADED",
+                message: `Assigned inventory candidate ${assignment.candidate.slug} for live role ${role} is ${healthStatus}.`,
+                reason: `health_${healthStatus}`,
+                role,
+                resolverRole: summary.resolverRole,
+                agentSlug: assignment.candidate.slug,
+                healthStatus,
+            });
+        }
+    }
+    return warnings;
+};
+const buildScenarioEnvironmentWarnings = (scenarios) => scenarios.flatMap((scenario) => scenario.warnings
+    .filter((warning) => warning === "agent_run_model_catalog_mismatch" ||
+    warning === "agent_run_upstream_unavailable")
+    .map((warning) => ({
+    code: "GATEWAY_LIVE_AGENT_RUN_DEGRADED",
+    message: `Live agent-run for scenario ${scenario.id} degraded due to an environment/runtime condition.`,
+    reason: warning,
+    role: scenario.role,
+    resolverRole: ROLE_TO_RESOLVER_ROLE[scenario.role],
+    agentSlug: scenario.agentSlug,
+    details: {
+        scenarioId: scenario.id,
+        status: scenario.status,
+        errors: scenario.errors,
+    },
+})));
 export const runCodaliGatewayLiveHarness = async (options = {}) => {
     const startedMs = Date.now();
     const startedAt = isoNow();
@@ -747,29 +1104,100 @@ export const runCodaliGatewayLiveHarness = async (options = {}) => {
         allowImageWorker: options.allowImageWorker ?? true,
         agentPolicy: options.agentPolicy,
     });
+    const environmentWarnings = buildEnvironmentWarnings({ discovery, classification });
+    const shadowPolicy = normalizeShadowComparisonPolicy(options.shadowComparison);
+    const shadowRecords = [];
+    const shadowEnvironmentWarnings = [];
     const scenarioRunner = options.scenarioRunner ?? createMcodaAgentRunScenarioRunner();
     const scenarios = options.scenarios?.length
         ? options.scenarios.map(scenarioById)
         : CODALI_GATEWAY_LIVE_SCENARIOS;
     const scenarioResults = [];
     for (const scenario of scenarios) {
+        const primaryAssignment = assignmentForScenario(classification, scenario);
         const result = await scenarioRunner({
             runId,
             scenario,
-            assignment: assignmentForScenario(classification, scenario),
+            assignment: primaryAssignment,
             classification,
             commandRunner,
             command: options.command ?? DEFAULT_MCODA_COMMAND,
             timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS,
             forceAgentRun: options.forceAgentRun === true,
         });
-        scenarioResults.push(redactScenario(result));
+        const redactedResult = redactScenario(result);
+        scenarioResults.push(redactedResult);
+        if (!shadowPolicy.enabled) {
+            continue;
+        }
+        if (shadowPolicy.includePrimary) {
+            shadowRecords.push(comparisonRecord({
+                scenario,
+                result: redactedResult,
+                assignment: primaryAssignment,
+                primary: true,
+                candidateRank: 0,
+            }));
+        }
+        const shadowAssignments = resolveShadowAssignmentsForScenario({
+            classification,
+            scenario,
+            primary: primaryAssignment,
+            policy: shadowPolicy,
+        });
+        if (shadowAssignments.length === 0) {
+            shadowEnvironmentWarnings.push({
+                code: "GATEWAY_SHADOW_COMPARISON_CANDIDATE_UNAVAILABLE",
+                message: `No eligible shadow comparison candidate found for live role ${scenario.role}.`,
+                reason: shadowPolicy.requireHealthy
+                    ? "no_healthy_alternate_candidate"
+                    : "no_alternate_candidate",
+                role: scenario.role,
+                resolverRole: ROLE_TO_RESOLVER_ROLE[scenario.role],
+            });
+        }
+        for (const [candidateIndex, shadowAssignment] of shadowAssignments.entries()) {
+            const shadowResult = await scenarioRunner({
+                runId,
+                scenario,
+                assignment: shadowAssignment,
+                classification,
+                commandRunner,
+                command: options.command ?? DEFAULT_MCODA_COMMAND,
+                timeoutMs: options.timeoutMs ?? DEFAULT_TIMEOUT_MS,
+                forceAgentRun: options.forceAgentRun === true,
+            });
+            shadowRecords.push(comparisonRecord({
+                scenario,
+                result: redactScenario(shadowResult),
+                assignment: shadowAssignment,
+                primary: false,
+                candidateRank: candidateIndex + 1,
+            }));
+        }
     }
     const endedAt = isoNow();
     const summary = summarizeHarness(classification, scenarioResults);
+    const scenarioEnvironmentWarnings = buildScenarioEnvironmentWarnings(scenarioResults);
+    const allEnvironmentWarnings = [
+        ...environmentWarnings,
+        ...scenarioEnvironmentWarnings,
+    ];
+    const shadowComparison = {
+        policy: shadowPolicy,
+        status: !shadowPolicy.enabled
+            ? "disabled"
+            : shadowRecords.length > 0
+                ? "compared"
+                : "skipped",
+        records: shadowRecords,
+        environmentWarnings: shadowEnvironmentWarnings,
+    };
     const warnings = unique([
         ...classification.warnings.map((warning) => warning.code),
         ...scenarioResults.flatMap((scenario) => scenario.warnings),
+        ...allEnvironmentWarnings.map((warning) => warning.code),
+        ...shadowEnvironmentWarnings.map((warning) => warning.code),
     ]);
     const errors = unique([
         ...discovery.errors,
@@ -787,6 +1215,8 @@ export const runCodaliGatewayLiveHarness = async (options = {}) => {
         discovery,
         classification: redactCodaliGatewayLiveValue(classification),
         scenarios: scenarioResults,
+        shadowComparison: redactCodaliGatewayLiveValue(shadowComparison),
+        environmentWarnings: redactCodaliGatewayLiveValue(allEnvironmentWarnings),
         summary,
         warnings,
         errors,
@@ -818,6 +1248,10 @@ export const formatCodaliGatewayLiveHarnessTextReport = (result) => {
         `JSON-capable agents: ${result.summary.jsonValidAgents.length ? result.summary.jsonValidAgents.join(", ") : "none"}`,
         `Large final synthesizer: ${result.summary.largeFinalSynthesizerOk ? "ok" : "missing or not proven"}`,
         `Image artifact: ${result.summary.imageArtifactOk ? "ok" : "missing or not proven"}`,
+        `Shadow comparison: ${result.shadowComparison.status} (${result.shadowComparison.records.length} records)`,
+        result.environmentWarnings.length
+            ? `Environment warnings: ${result.environmentWarnings.map((warning) => `${warning.code}:${warning.reason}`).join(", ")}`
+            : "Environment warnings: none",
         result.warnings.length ? `Warnings: ${result.warnings.join(", ")}` : "Warnings: none",
         result.errors.length ? `Errors: ${result.errors.join(", ")}` : "Errors: none",
     ].join("\n");

package/dist/eval/GatewayDatasetEval.d.ts

@@ -0,0 +1,70 @@
+import { type CodaliStorageDatasetKind, type CodaliStorageExportKind, type CodaliStorageObjectRef } from "../storage/CodaliStorageContracts.js";
+import { CODALI_DATASET_REPLAY_FIXTURE_SCHEMA_VERSION } from "../storage/DatasetExportJob.js";
+import { type GatewayDatasetObjectStore } from "../storage/GatewayDatasetStore.js";
+import type { CodaliGatewayEvalCase } from "./GatewayEvalSuite.js";
+export declare const CODALI_GATEWAY_DATASET_EVAL_SCHEMA_VERSION: 1;
+export type CodaliGatewayDatasetEvalStage = "classifier" | "planner" | "tool_router" | "rag_retrieval" | "evidence_extractor" | "verifier" | "context_pack" | "final_answer" | "schema_repair" | "policy_event";
+export declare const CODALI_GATEWAY_DATASET_EVAL_STAGES: CodaliGatewayDatasetEvalStage[];
+export declare const CODALI_GATEWAY_DATASET_EVAL_PROMPT_VERSIONS: Record<CodaliGatewayDatasetEvalStage, string>;
+export interface CodaliGatewayDatasetReplayFixtureRecord {
+    recordId: string;
+    datasetKind: CodaliStorageDatasetKind;
+    sourceGatewayRecordId?: string;
+    inputRef: CodaliStorageObjectRef;
+    outputRef?: CodaliStorageObjectRef;
+    evidenceRefs?: CodaliStorageObjectRef[];
+    quality?: {
+        score?: number;
+        labels?: string[];
+        reviewed?: boolean;
+    };
+    metadata?: Record<string, unknown>;
+}
+export interface CodaliGatewayDatasetReplayFixture {
+    schemaVersion: typeof CODALI_DATASET_REPLAY_FIXTURE_SCHEMA_VERSION;
+    exportKind: CodaliStorageExportKind;
+    generatedAt: string;
+    records: CodaliGatewayDatasetReplayFixtureRecord[];
+}
+export interface CodaliGatewayDatasetEvalSkippedRecord {
+    recordId: string;
+    stage?: CodaliGatewayDatasetEvalStage;
+    reason: string;
+}
+export interface CodaliGatewayDatasetEvalImportLineage {
+    schemaVersion: typeof CODALI_GATEWAY_DATASET_EVAL_SCHEMA_VERSION;
+    source: "dataset_replay_fixture";
+    fixtureId?: string;
+    fixtureSchemaVersion: typeof CODALI_DATASET_REPLAY_FIXTURE_SCHEMA_VERSION;
+    exportKind: CodaliStorageExportKind;
+    generatedAt: string;
+    importedAt: string;
+    selectedRecordCount: number;
+    skippedRecordCount: number;
+    sourceRecordIds: string[];
+    sourceGatewayRecordIds: string[];
+    sourceObjectHashes: string[];
+    stageCounts: Record<CodaliGatewayDatasetEvalStage, number>;
+    skippedRecords: CodaliGatewayDatasetEvalSkippedRecord[];
+}
+export interface CodaliGatewayDatasetEvalImportResult {
+    cases: CodaliGatewayEvalCase[];
+    lineage: CodaliGatewayDatasetEvalImportLineage;
+}
+export interface CodaliGatewayDatasetReplayFixtureImportOptions {
+    fixture: unknown;
+    fixtureId?: string;
+    objectStore?: GatewayDatasetObjectStore;
+    stages?: CodaliGatewayDatasetEvalStage[];
+    limitPerStage?: number;
+    now?: () => Date;
+}
+export declare class CodaliGatewayDatasetEvalImportError extends Error {
+    readonly code: string;
+    constructor(code: string, message: string);
+}
+export declare const parseCodaliGatewayDatasetReplayFixture: (fixture: unknown) => CodaliGatewayDatasetReplayFixture;
+export declare const importCodaliGatewayDatasetReplayFixture: (options: CodaliGatewayDatasetReplayFixtureImportOptions) => Promise<CodaliGatewayDatasetEvalImportResult>;
+export declare const createDefaultCodaliGatewayDatasetReplayFixture: () => CodaliGatewayDatasetReplayFixture;
+export declare const createDefaultCodaliGatewayDatasetEvalImport: () => Promise<CodaliGatewayDatasetEvalImportResult>;
+//# sourceMappingURL=GatewayDatasetEval.d.ts.map

package/dist/eval/GatewayDatasetEval.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"GatewayDatasetEval.d.ts","sourceRoot":"","sources":["../../src/eval/GatewayDatasetEval.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,wBAAwB,EAC7B,KAAK,uBAAuB,EAC5B,KAAK,sBAAsB,EAC5B,MAAM,sCAAsC,CAAC;AAC9C,OAAO,EAAE,4CAA4C,EAAE,MAAM,gCAAgC,CAAC;AAC9F,OAAO,EAEL,KAAK,yBAAyB,EAC/B,MAAM,mCAAmC,CAAC;AAC3C,OAAO,KAAK,EACV,qBAAqB,EAItB,MAAM,uBAAuB,CAAC;AAE/B,eAAO,MAAM,0CAA0C,EAAG,CAAU,CAAC;AAErE,MAAM,MAAM,6BAA6B,GACrC,YAAY,GACZ,SAAS,GACT,aAAa,GACb,eAAe,GACf,oBAAoB,GACpB,UAAU,GACV,cAAc,GACd,cAAc,GACd,eAAe,GACf,cAAc,CAAC;AAEnB,eAAO,MAAM,kCAAkC,EAAE,6BAA6B,EAW7E,CAAC;AAEF,eAAO,MAAM,2CAA2C,EAAE,MAAM,CAC9D,6BAA6B,EAC7B,MAAM,CAYP,CAAC;AAEF,MAAM,WAAW,uCAAuC;IACtD,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,EAAE,wBAAwB,CAAC;IACtC,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAC/B,QAAQ,EAAE,sBAAsB,CAAC;IACjC,SAAS,CAAC,EAAE,sBAAsB,CAAC;IACnC,YAAY,CAAC,EAAE,sBAAsB,EAAE,CAAC;IACxC,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,MAAM,CAAC;QACf,MAAM,CAAC,EAAE,MAAM,EAAE,CAAC;QAClB,QAAQ,CAAC,EAAE,OAAO,CAAC;KACpB,CAAC;IACF,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,MAAM,WAAW,iCAAiC;IAChD,aAAa,EAAE,OAAO,4CAA4C,CAAC;IACnE,UAAU,EAAE,uBAAuB,CAAC;IACpC,WAAW,EAAE,MAAM,CAAC;IACpB,OAAO,EAAE,uCAAuC,EAAE,CAAC;CACpD;AAED,MAAM,WAAW,qCAAqC;IACpD,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,6BAA6B,CAAC;IACtC,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,qCAAqC;IACpD,aAAa,EAAE,OAAO,0CAA0C,CAAC;IACjE,MAAM,EAAE,wBAAwB,CAAC;IACjC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,oBAAoB,EAAE,OAAO,4CAA4C,CAAC;IAC1E,UAAU,EAAE,uBAAuB,CAAC;IACpC,WAAW,EAAE,MAAM,CAAC;IACpB,UAAU,EAAE,MAAM,CAAC;IACnB,mBAAmB,EAAE,MAAM,CAAC;IAC5B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,sBAAsB,EAAE,MAAM,EAAE,CAAC;IACjC,kBAAkB,EAAE,MAAM,EAAE,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC,6BAA6B,EAAE,MAAM,CAAC,CAAC;IAC3D,cAAc,EAAE,qCAAqC,EAAE,CAAC;CACzD;AAED,MAAM,WAAW,oCAAoC;IACnD,KAAK,EAAE,qBAAqB,EAAE,CAAC;IAC/B,OAAO,EAAE,qCAAqC,CAAC;CAChD;AAED,MAAM,WAAW,8CAA8C;IAC7D,OAAO,EAAE,OAAO,CAAC;IACjB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,WAAW,CAAC,EAAE,yBAAyB,CAAC;IACxC,MAAM,CAAC,EAAE,6BAA6B,EAAE,CAAC;IACzC,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;CAClB;AAED,qBAAa,mCAAoC,SAAQ,KAAK;IAC5D,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;gBAEV,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;CAK1C;AA6SD,eAAO,MAAM,sCAAsC,GACjD,SAAS,OAAO,KACf,iCAqCF,CAAC;AAEF,eAAO,MAAM,uCAAuC,GAClD,SAAS,8CAA8C,KACtD,OAAO,CAAC,oCAAoC,CAwD9C,CAAC;AA2EF,eAAO,MAAM,8CAA8C,QACrD,iCAyDF,CAAC;AAEL,eAAO,MAAM,2CAA2C,QAAa,OAAO,CAC1E,oCAAoC,CAMlC,CAAC"}