@sanity/ailf 2.0.0 → 2.0.2

package/dist/pipeline/calculate-scores.js

@@ -25,7 +25,7 @@
  * All functions accept rootDir as a parameter — no module-level constants.
  * No process.argv parsing. No env var fallbacks.
  *
- * @see docs/exec-plans/eliminate-lib-layer.md
+ * @see docs/archive/exec-plans/eliminate-lib-layer.md
  */
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
 import { join } from "path";
@@ -454,40 +454,55 @@ function readAndNormalizeResults(resultsPath, log) {
         resultCount: wrapper.results.length,
         stats: wrapper.stats,
     });
-    const all = wrapper.results.map((r) => ({
-        cost: r.cost ?? 0,
-        description: r.testCase?.description ?? "unknown",
-        error: r.error,
-        gradingResult: r.gradingResult,
-        metadata: r.metadata,
-        provider: r.provider?.label ?? r.provider?.id,
-        providerId: r.provider?.id,
-        providerLabel: r.provider?.label,
-        response: r.response,
-        vars: r.vars ?? r.testCase?.vars ?? {},
-    }));
-    // Filter out results where gradingResult is null (errored/timed-out tests).
-    // Promptfoo sets gradingResult to null when a test errors before grading.
-    const valid = all.filter((r) => r.gradingResult !== null);
-    const skipped = all.length - valid.length;
-    _log.debug("Filtered null gradingResults", {
-        totalResults: all.length,
-        validResults: valid.length,
-        skippedCount: skipped,
-    });
-    if (skipped > 0) {
-        _log.warn(`⚠ Skipping ${skipped} of ${all.length} result(s) with null gradingResult (errored tests):`);
-        for (const r of all) {
-            if (r.gradingResult === null) {
-                const providerLabel = r.provider ? `[${r.provider}] ` : "";
-                const errorMsg = r.error
-                    ? r.error.slice(0, 150)
-                    : "unknown error (no error field in result)";
-                _log.warn(`✗ ${providerLabel}"${r.description}" — ${errorMsg}`);
-            }
+    // Normalize results. Errored tests (null gradingResult) get a synthetic
+    // zero-score result so they flow through scoring and failure mode
+    // classification as "api-error" instead of being silently dropped.
+    const results = [];
+    let synthesizedCount = 0;
+    for (const r of wrapper.results) {
+        const base = {
+            cost: r.cost ?? 0,
+            description: r.testCase?.description ?? "unknown",
+            metadata: r.metadata,
+            providerId: r.provider?.id,
+            providerLabel: r.provider?.label,
+            response: r.response ?? { output: "" },
+            vars: r.vars ?? r.testCase?.vars ?? {},
+        };
+        if (r.gradingResult === null || r.gradingResult === undefined) {
+            // Synthesize a zero-score result so errored tests are visible in
+            // scoring, gap analysis, and failure mode classification.
+            const errorMsg = r.error ?? "unknown error (null gradingResult)";
+            synthesizedCount++;
+            const providerLabel = r.provider?.label ?? r.provider?.id ?? "";
+            _log.warn(`⚠ [api-error] ${providerLabel ? `[${providerLabel}] ` : ""}"${base.description}" — ${errorMsg.slice(0, 150)}`);
+            results.push({
+                ...base,
+                gradingResult: {
+                    pass: false,
+                    componentResults: [
+                        {
+                            assertion: { type: "llm-rubric" },
+                            pass: false,
+                            reason: `[api-error] ${errorMsg}`,
+                            score: 0,
+                        },
+                    ],
+                },
+            });
         }
+        else {
+            results.push({ ...base, gradingResult: r.gradingResult });
+        }
+    }
+    _log.debug("Normalized results", {
+        totalResults: wrapper.results.length,
+        synthesizedApiErrors: synthesizedCount,
+    });
+    if (synthesizedCount > 0) {
+        _log.warn(`⚠ Synthesized ${synthesizedCount} zero-score result(s) for errored tests (api-error)`);
     }
-    return valid;
+    return results;
 }
 /**
  * Core scoring logic: takes a pre-filtered array of TestResult and produces
@@ -805,12 +820,34 @@ function computeTestSummary(resultsPath) {
             failed++;
         }
     }
+    // Extract per-test timing from latencyMs (when available from Promptfoo)
+    const durations = rawResults
+        .filter((r) => typeof r.latencyMs === "number")
+        .map((r) => ({
+        task: r.testCase?.description ?? "unknown",
+        model: r.provider?.label ?? r.provider?.id ?? "unknown",
+        durationMs: r.latencyMs,
+    }));
+    let timing;
+    if (durations.length > 0) {
+        const sorted = durations.map((d) => d.durationMs).sort((a, b) => a - b);
+        const medianMs = sorted[Math.floor(sorted.length / 2)];
+        const p95Ms = sorted[Math.floor(sorted.length * 0.95)];
+        const maxMs = sorted[sorted.length - 1];
+        // Flag tests exceeding 2x median (min 60s) as "slow"
+        const slowThreshold = Math.max(medianMs * 2, 60_000);
+        const slowTests = durations
+            .filter((d) => d.durationMs > slowThreshold)
+            .sort((a, b) => b.durationMs - a.durationMs);
+        timing = { medianMs, p95Ms, maxMs, slowTests };
+    }
     return {
         total: rawResults.length,
         passed,
         failed,
         errored,
         ...(errors.length > 0 ? { errors } : {}),
+        ...(timing ? { timing } : {}),
     };
 }
 function printPerModelReport(perModel, log) {

package/dist/pipeline/checks.d.ts

@@ -23,10 +23,15 @@ export declare function checkContextsExist(rootDir: string, areas: string[]): Va
  */
 export declare function checkEnvironment(rootDir: string): ValidationIssue[];
 /**
- * Check that the baseline `promptfooconfig.yaml` exists. Optionally check
- * for `promptfooconfig.observed.yaml` and `promptfooconfig.agentic.yaml`.
+ * Check that the generated promptfoo config for a given mode exists.
+ *
+ * When `mode` is provided, checks only for that mode's config file
+ * (e.g. `promptfooconfig.agent-harness.yaml` for mode `"agent-harness"`).
+ *
+ * When `mode` is omitted, falls back to the legacy literacy check:
+ * baseline `promptfooconfig.yaml` (required) plus optional observed/agentic.
  */
-export declare function checkGeneratedConfigsExist(rootDir: string): ValidationIssue[];
+export declare function checkGeneratedConfigsExist(rootDir: string, mode?: string): ValidationIssue[];
 /**
  * Check that the eval results JSON file exists, is valid JSON, and contains
  * a `results` array.

package/dist/pipeline/checks.js

@@ -8,6 +8,7 @@
 import { config as loadEnv } from "dotenv";
 import { existsSync, readFileSync, statSync } from "fs";
 import { join, resolve } from "path";
+import { configFileForMode } from "./eval-constants.js";
 // ---------------------------------------------------------------------------
 // Precondition: contexts exist for each feature area
 // ---------------------------------------------------------------------------
@@ -109,11 +110,30 @@ export function checkEnvironment(rootDir) {
 // Postcondition: score summary is valid
 // ---------------------------------------------------------------------------
 /**
- * Check that the baseline `promptfooconfig.yaml` exists. Optionally check
- * for `promptfooconfig.observed.yaml` and `promptfooconfig.agentic.yaml`.
+ * Check that the generated promptfoo config for a given mode exists.
+ *
+ * When `mode` is provided, checks only for that mode's config file
+ * (e.g. `promptfooconfig.agent-harness.yaml` for mode `"agent-harness"`).
+ *
+ * When `mode` is omitted, falls back to the legacy literacy check:
+ * baseline `promptfooconfig.yaml` (required) plus optional observed/agentic.
  */
-export function checkGeneratedConfigsExist(rootDir) {
+export function checkGeneratedConfigsExist(rootDir, mode) {
     const issues = [];
+    if (mode) {
+        const configName = configFileForMode(mode);
+        const configPath = resolve(rootDir, configName);
+        if (!existsSync(configPath)) {
+            issues.push({
+                message: `Config '${configName}' not found for mode '${mode}'. Run the pipeline to generate it.`,
+                path: configPath,
+                severity: "error",
+                source: "checkGeneratedConfigsExist",
+            });
+        }
+        return issues;
+    }
+    // Legacy literacy check: baseline required, observed/agentic optional
     const baselinePath = resolve(rootDir, "promptfooconfig.yaml");
     if (!existsSync(baselinePath)) {
         issues.push({

package/dist/pipeline/chronic-failures.d.ts

@@ -0,0 +1,55 @@
+/**
+ * pipeline/chronic-failures.ts
+ *
+ * Aggregates error data across recent reports to identify tasks that
+ * consistently fail (>threshold error rate). This catches structurally
+ * broken tasks — wrong model config, tasks too complex for the provider,
+ * persistent API incompatibility — that would otherwise be invisible.
+ *
+ * @see docs/exec-plans/eval-pipeline-timeout-resilience.md — Phase 5
+ */
+import type { ReportStore } from "../report-store.js";
+export interface ChronicFailureEntry {
+    /** Task ID */
+    task: string;
+    /** Error rate as a fraction (0–1) */
+    errorRate: number;
+    /** Number of runs with errors / total runs analyzed */
+    errorCount: number;
+    totalRuns: number;
+    /** Which models are affected and how often */
+    modelBreakdown: {
+        model: string;
+        errorCount: number;
+    }[];
+    /** Most common error message */
+    commonError: string;
+}
+export interface ChronicFailureReport {
+    /** Number of reports analyzed */
+    lookback: number;
+    /** Threshold used for classification */
+    threshold: number;
+    /** Tasks exceeding the error threshold */
+    failures: ChronicFailureEntry[];
+    /** Total reports found (may be less than lookback if not enough history) */
+    reportsFound: number;
+}
+export interface ChronicFailureOptions {
+    /** Number of recent reports to analyze (default: 10) */
+    lookback?: number;
+    /** Error rate threshold (0–1) for "chronic" classification (default: 0.5) */
+    threshold?: number;
+}
+/**
+ * Query recent reports and identify tasks with chronic failures.
+ *
+ * @param reportStore - The report store to query
+ * @param options - Lookback window and threshold
+ * @returns Chronic failure report, or null if no reports found
+ */
+export declare function detectChronicFailures(reportStore: ReportStore, options?: ChronicFailureOptions): Promise<ChronicFailureReport>;
+/**
+ * Format a chronic failure report for console output.
+ */
+export declare function formatChronicFailuresConsole(report: ChronicFailureReport): string;

package/dist/pipeline/chronic-failures.js

@@ -0,0 +1,110 @@
+/**
+ * pipeline/chronic-failures.ts
+ *
+ * Aggregates error data across recent reports to identify tasks that
+ * consistently fail (>threshold error rate). This catches structurally
+ * broken tasks — wrong model config, tasks too complex for the provider,
+ * persistent API incompatibility — that would otherwise be invisible.
+ *
+ * @see docs/exec-plans/eval-pipeline-timeout-resilience.md — Phase 5
+ */
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Query recent reports and identify tasks with chronic failures.
+ *
+ * @param reportStore - The report store to query
+ * @param options - Lookback window and threshold
+ * @returns Chronic failure report, or null if no reports found
+ */
+export async function detectChronicFailures(reportStore, options = {}) {
+    const lookback = options.lookback ?? 10;
+    const threshold = options.threshold ?? 0.5;
+    const reports = await reportStore.queryRecentErrors(lookback);
+    if (reports.length === 0) {
+        return { lookback, threshold, failures: [], reportsFound: 0 };
+    }
+    // Aggregate errors by task
+    const taskErrors = new Map();
+    for (const report of reports) {
+        for (const error of report.errors) {
+            let entry = taskErrors.get(error.task);
+            if (!entry) {
+                entry = {
+                    runsWith: new Set(),
+                    modelErrors: new Map(),
+                    errors: [],
+                };
+                taskErrors.set(error.task, entry);
+            }
+            entry.runsWith.add(report.reportId);
+            entry.modelErrors.set(error.model, (entry.modelErrors.get(error.model) ?? 0) + 1);
+            entry.errors.push(error.error);
+        }
+    }
+    // Identify chronic failures (error rate > threshold)
+    const failures = [];
+    const totalRuns = reports.length;
+    for (const [task, data] of taskErrors) {
+        const errorRate = data.runsWith.size / totalRuns;
+        if (errorRate >= threshold) {
+            // Find the most common error message
+            const errorCounts = new Map();
+            for (const err of data.errors) {
+                const truncated = err.slice(0, 200);
+                errorCounts.set(truncated, (errorCounts.get(truncated) ?? 0) + 1);
+            }
+            const commonError = [...errorCounts.entries()].sort((a, b) => b[1] - a[1])[0]?.[0] ??
+                "unknown";
+            const modelBreakdown = [...data.modelErrors.entries()]
+                .map(([model, errorCount]) => ({ model, errorCount }))
+                .sort((a, b) => b.errorCount - a.errorCount);
+            failures.push({
+                task,
+                errorRate,
+                errorCount: data.runsWith.size,
+                totalRuns,
+                modelBreakdown,
+                commonError,
+            });
+        }
+    }
+    // Sort by error rate descending
+    failures.sort((a, b) => b.errorRate - a.errorRate);
+    return { lookback, threshold, failures, reportsFound: reports.length };
+}
+// ---------------------------------------------------------------------------
+// Formatting
+// ---------------------------------------------------------------------------
+/**
+ * Format a chronic failure report for console output.
+ */
+export function formatChronicFailuresConsole(report) {
+    const lines = [];
+    lines.push(`Chronic Failure Report (last ${report.reportsFound} runs)`);
+    lines.push("━".repeat(50));
+    lines.push("");
+    if (report.failures.length === 0) {
+        lines.push("  ✅ No chronic failures detected (all tasks below " +
+            `${(report.threshold * 100).toFixed(0)}% error threshold)`);
+        lines.push("");
+        return lines.join("\n");
+    }
+    lines.push(`  ⚠ ${report.failures.length} task(s) with chronic failures ` +
+        `(>${(report.threshold * 100).toFixed(0)}% error rate):`);
+    lines.push("");
+    for (const f of report.failures) {
+        lines.push(`  ${f.task}`);
+        lines.push(`    Error rate: ${f.errorCount}/${f.totalRuns} runs ` +
+            `(${(f.errorRate * 100).toFixed(0)}%)`);
+        const models = f.modelBreakdown
+            .map((m) => `${m.model} (${m.errorCount})`)
+            .join(", ");
+        lines.push(`    Models affected: ${models}`);
+        lines.push(`    Common error: "${f.commonError}"`);
+        lines.push("    Suggested action: Increase timeoutMs for affected models or simplify task");
+        lines.push("");
+    }
+    return lines.join("\n");
+}

package/dist/pipeline/compiler/__tests__/mcp-server-handler.test.js

@@ -194,6 +194,39 @@ describe("compileMCPTask", () => {
         const server = serverCfg(result.providers[0]);
         assert.deepEqual(server.auth, { type: "bearer", token: "{{env.MY_TOKEN}}" });
     });
+    it("maps headers to mcpServer config", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            serverConfig: {
+                transport: "streamable-http",
+                url: "https://mcp.example.com",
+                headers: {
+                    Authorization: "Bearer {{env.MY_TOKEN}}",
+                    "X-Custom": "value",
+                },
+            },
+        }), { models: TEST_MODELS });
+        const server = serverCfg(result.providers[0]);
+        assert.deepEqual(server.headers, {
+            Authorization: "Bearer {{env.MY_TOKEN}}",
+            "X-Custom": "value",
+        });
+    });
+    it("passes both headers and auth when both present", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            serverConfig: {
+                transport: "streamable-http",
+                url: "https://mcp.example.com",
+                headers: { "X-Custom": "value" },
+                auth: { type: "bearer", token: "{{env.MY_TOKEN}}" },
+            },
+        }), { models: TEST_MODELS });
+        const server = serverCfg(result.providers[0]);
+        assert.deepEqual(server.headers, { "X-Custom": "value" });
+        assert.deepEqual(server.auth, {
+            type: "bearer",
+            token: "{{env.MY_TOKEN}}",
+        });
+    });
     it("maps capabilities to mcpTools config", () => {
         const result = compileMCPTask(makeMinimalMCPTask({
             capabilities: ["query_documents", "get_schema"],

package/dist/pipeline/compiler/__tests__/promptfoo-compiler.test.js

@@ -9,7 +9,6 @@
 import assert from "node:assert/strict";
 import { describe, it } from "node:test";
 import { tmpdir } from "os";
-import { LiteracyVariant } from "../../normalize-mode.js";
 import { compileToPromptfoo } from "../promptfoo-compiler.js";
 // ---------------------------------------------------------------------------
 // Helpers
@@ -123,8 +122,8 @@ describe("compileToPromptfoo", () => {
         const result = compileToPromptfoo(graph, {
             mode: "literacy",
             models: makeModels([
-                { id: "model-a", label: "A", modes: [LiteracyVariant.STANDARD] },
-                { id: "model-b", label: "B", modes: ["agentic-naive"] },
+                { id: "model-a", label: "A", modes: ["literacy"] },
+                { id: "model-b", label: "B", modes: ["mcp-server"] },
             ]),
             rootDir: tmpdir(),
         });

package/dist/pipeline/compiler/__tests__/task-bridge.test.d.ts

@@ -0,0 +1,9 @@
+/**
+ * task-bridge.test.ts — Unit tests for the TaskDefinition ↔ LiteracyTaskDefinition bridge.
+ *
+ * Covers round-trip fidelity, edge cases (missing optionals, all optionals),
+ * assertion type mapping, and all four CanonicalDocRef / GeneralizedDocRef variants.
+ *
+ * Run: npx tsx --test src/pipeline/compiler/__tests__/task-bridge.test.ts
+ */
+export {};