@sanity/ailf 1.0.0 → 2.0.1

package/dist/pipeline/calculate-scores.js

@@ -25,7 +25,7 @@
  * All functions accept rootDir as a parameter — no module-level constants.
  * No process.argv parsing. No env var fallbacks.
  *
- * @see docs/exec-plans/eliminate-lib-layer.md
+ * @see docs/archive/exec-plans/eliminate-lib-layer.md
  */
 import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
 import { join } from "path";
@@ -454,40 +454,55 @@ function readAndNormalizeResults(resultsPath, log) {
         resultCount: wrapper.results.length,
         stats: wrapper.stats,
     });
-    const all = wrapper.results.map((r) => ({
-        cost: r.cost ?? 0,
-        description: r.testCase?.description ?? "unknown",
-        error: r.error,
-        gradingResult: r.gradingResult,
-        metadata: r.metadata,
-        provider: r.provider?.label ?? r.provider?.id,
-        providerId: r.provider?.id,
-        providerLabel: r.provider?.label,
-        response: r.response,
-        vars: r.vars ?? r.testCase?.vars ?? {},
-    }));
-    // Filter out results where gradingResult is null (errored/timed-out tests).
-    // Promptfoo sets gradingResult to null when a test errors before grading.
-    const valid = all.filter((r) => r.gradingResult !== null);
-    const skipped = all.length - valid.length;
-    _log.debug("Filtered null gradingResults", {
-        totalResults: all.length,
-        validResults: valid.length,
-        skippedCount: skipped,
-    });
-    if (skipped > 0) {
-        _log.warn(`⚠ Skipping ${skipped} of ${all.length} result(s) with null gradingResult (errored tests):`);
-        for (const r of all) {
-            if (r.gradingResult === null) {
-                const providerLabel = r.provider ? `[${r.provider}] ` : "";
-                const errorMsg = r.error
-                    ? r.error.slice(0, 150)
-                    : "unknown error (no error field in result)";
-                _log.warn(`✗ ${providerLabel}"${r.description}" — ${errorMsg}`);
-            }
+    // Normalize results. Errored tests (null gradingResult) get a synthetic
+    // zero-score result so they flow through scoring and failure mode
+    // classification as "api-error" instead of being silently dropped.
+    const results = [];
+    let synthesizedCount = 0;
+    for (const r of wrapper.results) {
+        const base = {
+            cost: r.cost ?? 0,
+            description: r.testCase?.description ?? "unknown",
+            metadata: r.metadata,
+            providerId: r.provider?.id,
+            providerLabel: r.provider?.label,
+            response: r.response ?? { output: "" },
+            vars: r.vars ?? r.testCase?.vars ?? {},
+        };
+        if (r.gradingResult === null || r.gradingResult === undefined) {
+            // Synthesize a zero-score result so errored tests are visible in
+            // scoring, gap analysis, and failure mode classification.
+            const errorMsg = r.error ?? "unknown error (null gradingResult)";
+            synthesizedCount++;
+            const providerLabel = r.provider?.label ?? r.provider?.id ?? "";
+            _log.warn(`⚠ [api-error] ${providerLabel ? `[${providerLabel}] ` : ""}"${base.description}" — ${errorMsg.slice(0, 150)}`);
+            results.push({
+                ...base,
+                gradingResult: {
+                    pass: false,
+                    componentResults: [
+                        {
+                            assertion: { type: "llm-rubric" },
+                            pass: false,
+                            reason: `[api-error] ${errorMsg}`,
+                            score: 0,
+                        },
+                    ],
+                },
+            });
         }
+        else {
+            results.push({ ...base, gradingResult: r.gradingResult });
+        }
+    }
+    _log.debug("Normalized results", {
+        totalResults: wrapper.results.length,
+        synthesizedApiErrors: synthesizedCount,
+    });
+    if (synthesizedCount > 0) {
+        _log.warn(`⚠ Synthesized ${synthesizedCount} zero-score result(s) for errored tests (api-error)`);
     }
-    return valid;
+    return results;
 }
 /**
  * Core scoring logic: takes a pre-filtered array of TestResult and produces
@@ -805,12 +820,34 @@ function computeTestSummary(resultsPath) {
             failed++;
         }
     }
+    // Extract per-test timing from latencyMs (when available from Promptfoo)
+    const durations = rawResults
+        .filter((r) => typeof r.latencyMs === "number")
+        .map((r) => ({
+        task: r.testCase?.description ?? "unknown",
+        model: r.provider?.label ?? r.provider?.id ?? "unknown",
+        durationMs: r.latencyMs,
+    }));
+    let timing;
+    if (durations.length > 0) {
+        const sorted = durations.map((d) => d.durationMs).sort((a, b) => a - b);
+        const medianMs = sorted[Math.floor(sorted.length / 2)];
+        const p95Ms = sorted[Math.floor(sorted.length * 0.95)];
+        const maxMs = sorted[sorted.length - 1];
+        // Flag tests exceeding 2x median (min 60s) as "slow"
+        const slowThreshold = Math.max(medianMs * 2, 60_000);
+        const slowTests = durations
+            .filter((d) => d.durationMs > slowThreshold)
+            .sort((a, b) => b.durationMs - a.durationMs);
+        timing = { medianMs, p95Ms, maxMs, slowTests };
+    }
     return {
         total: rawResults.length,
         passed,
         failed,
         errored,
         ...(errors.length > 0 ? { errors } : {}),
+        ...(timing ? { timing } : {}),
     };
 }
 function printPerModelReport(perModel, log) {

package/dist/pipeline/chronic-failures.d.ts

@@ -0,0 +1,55 @@
+/**
+ * pipeline/chronic-failures.ts
+ *
+ * Aggregates error data across recent reports to identify tasks that
+ * consistently fail (>threshold error rate). This catches structurally
+ * broken tasks — wrong model config, tasks too complex for the provider,
+ * persistent API incompatibility — that would otherwise be invisible.
+ *
+ * @see docs/exec-plans/eval-pipeline-timeout-resilience.md — Phase 5
+ */
+import type { ReportStore } from "../report-store.js";
+export interface ChronicFailureEntry {
+    /** Task ID */
+    task: string;
+    /** Error rate as a fraction (0–1) */
+    errorRate: number;
+    /** Number of runs with errors / total runs analyzed */
+    errorCount: number;
+    totalRuns: number;
+    /** Which models are affected and how often */
+    modelBreakdown: {
+        model: string;
+        errorCount: number;
+    }[];
+    /** Most common error message */
+    commonError: string;
+}
+export interface ChronicFailureReport {
+    /** Number of reports analyzed */
+    lookback: number;
+    /** Threshold used for classification */
+    threshold: number;
+    /** Tasks exceeding the error threshold */
+    failures: ChronicFailureEntry[];
+    /** Total reports found (may be less than lookback if not enough history) */
+    reportsFound: number;
+}
+export interface ChronicFailureOptions {
+    /** Number of recent reports to analyze (default: 10) */
+    lookback?: number;
+    /** Error rate threshold (0–1) for "chronic" classification (default: 0.5) */
+    threshold?: number;
+}
+/**
+ * Query recent reports and identify tasks with chronic failures.
+ *
+ * @param reportStore - The report store to query
+ * @param options - Lookback window and threshold
+ * @returns Chronic failure report, or null if no reports found
+ */
+export declare function detectChronicFailures(reportStore: ReportStore, options?: ChronicFailureOptions): Promise<ChronicFailureReport>;
+/**
+ * Format a chronic failure report for console output.
+ */
+export declare function formatChronicFailuresConsole(report: ChronicFailureReport): string;

package/dist/pipeline/chronic-failures.js

@@ -0,0 +1,110 @@
+/**
+ * pipeline/chronic-failures.ts
+ *
+ * Aggregates error data across recent reports to identify tasks that
+ * consistently fail (>threshold error rate). This catches structurally
+ * broken tasks — wrong model config, tasks too complex for the provider,
+ * persistent API incompatibility — that would otherwise be invisible.
+ *
+ * @see docs/exec-plans/eval-pipeline-timeout-resilience.md — Phase 5
+ */
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Query recent reports and identify tasks with chronic failures.
+ *
+ * @param reportStore - The report store to query
+ * @param options - Lookback window and threshold
+ * @returns Chronic failure report, or null if no reports found
+ */
+export async function detectChronicFailures(reportStore, options = {}) {
+    const lookback = options.lookback ?? 10;
+    const threshold = options.threshold ?? 0.5;
+    const reports = await reportStore.queryRecentErrors(lookback);
+    if (reports.length === 0) {
+        return { lookback, threshold, failures: [], reportsFound: 0 };
+    }
+    // Aggregate errors by task
+    const taskErrors = new Map();
+    for (const report of reports) {
+        for (const error of report.errors) {
+            let entry = taskErrors.get(error.task);
+            if (!entry) {
+                entry = {
+                    runsWith: new Set(),
+                    modelErrors: new Map(),
+                    errors: [],
+                };
+                taskErrors.set(error.task, entry);
+            }
+            entry.runsWith.add(report.reportId);
+            entry.modelErrors.set(error.model, (entry.modelErrors.get(error.model) ?? 0) + 1);
+            entry.errors.push(error.error);
+        }
+    }
+    // Identify chronic failures (error rate > threshold)
+    const failures = [];
+    const totalRuns = reports.length;
+    for (const [task, data] of taskErrors) {
+        const errorRate = data.runsWith.size / totalRuns;
+        if (errorRate >= threshold) {
+            // Find the most common error message
+            const errorCounts = new Map();
+            for (const err of data.errors) {
+                const truncated = err.slice(0, 200);
+                errorCounts.set(truncated, (errorCounts.get(truncated) ?? 0) + 1);
+            }
+            const commonError = [...errorCounts.entries()].sort((a, b) => b[1] - a[1])[0]?.[0] ??
+                "unknown";
+            const modelBreakdown = [...data.modelErrors.entries()]
+                .map(([model, errorCount]) => ({ model, errorCount }))
+                .sort((a, b) => b.errorCount - a.errorCount);
+            failures.push({
+                task,
+                errorRate,
+                errorCount: data.runsWith.size,
+                totalRuns,
+                modelBreakdown,
+                commonError,
+            });
+        }
+    }
+    // Sort by error rate descending
+    failures.sort((a, b) => b.errorRate - a.errorRate);
+    return { lookback, threshold, failures, reportsFound: reports.length };
+}
+// ---------------------------------------------------------------------------
+// Formatting
+// ---------------------------------------------------------------------------
+/**
+ * Format a chronic failure report for console output.
+ */
+export function formatChronicFailuresConsole(report) {
+    const lines = [];
+    lines.push(`Chronic Failure Report (last ${report.reportsFound} runs)`);
+    lines.push("━".repeat(50));
+    lines.push("");
+    if (report.failures.length === 0) {
+        lines.push("  ✅ No chronic failures detected (all tasks below " +
+            `${(report.threshold * 100).toFixed(0)}% error threshold)`);
+        lines.push("");
+        return lines.join("\n");
+    }
+    lines.push(`  ⚠ ${report.failures.length} task(s) with chronic failures ` +
+        `(>${(report.threshold * 100).toFixed(0)}% error rate):`);
+    lines.push("");
+    for (const f of report.failures) {
+        lines.push(`  ${f.task}`);
+        lines.push(`    Error rate: ${f.errorCount}/${f.totalRuns} runs ` +
+            `(${(f.errorRate * 100).toFixed(0)}%)`);
+        const models = f.modelBreakdown
+            .map((m) => `${m.model} (${m.errorCount})`)
+            .join(", ");
+        lines.push(`    Models affected: ${models}`);
+        lines.push(`    Common error: "${f.commonError}"`);
+        lines.push("    Suggested action: Increase timeoutMs for affected models or simplify task");
+        lines.push("");
+    }
+    return lines.join("\n");
+}

package/dist/pipeline/compiler/__tests__/agent-harness-handler.test.js

@@ -10,7 +10,7 @@
 import assert from "node:assert/strict";
 import { describe, it } from "node:test";
 import { LiteracyVariant } from "../../normalize-mode.js";
-import { compileAgentHarnessTask, AGENT_HARNESS_PROMPT_TEMPLATES, handler as agentHandler, validateAgentHarnessTask, } from "../mode-handlers/agent-harness-handler.js";
+import { compileAgentHarnessTask, AGENT_HARNESS_PROMPT_TEMPLATES, handler as agentHandler, validateAgentHarnessTask, } from "../mode-handlers/agent-harness/index.js";
 import { allAgentHarnessExampleTasks, scaffoldProjectTask, modifyCodeTask, multiFileRefactorTask, } from "../mode-handlers/__fixtures__/agent-harness-example-tasks.js";
 // ---------------------------------------------------------------------------
 // Helpers

package/dist/pipeline/compiler/__tests__/knowledge-probe-handler.test.js

@@ -10,7 +10,7 @@
 import assert from "node:assert/strict";
 import { describe, it } from "node:test";
 import { LiteracyVariant } from "../../normalize-mode.js";
-import { compileKnowledgeProbeTask, handler as probeHandler, KNOWLEDGE_PROBE_PROMPT_TEMPLATES, validateKnowledgeProbeTask, } from "../mode-handlers/knowledge-probe-handler.js";
+import { compileKnowledgeProbeTask, handler as probeHandler, KNOWLEDGE_PROBE_PROMPT_TEMPLATES, validateKnowledgeProbeTask, } from "../mode-handlers/knowledge-probe/index.js";
 import { allKnowledgeProbeExampleTasks, groqProjectionTask, defineTypeApiTask, ecosystemComparisonTask, } from "../mode-handlers/__fixtures__/knowledge-probe-example-tasks.js";
 // ---------------------------------------------------------------------------
 // Helpers

package/dist/pipeline/compiler/__tests__/literacy-handler.test.js

@@ -10,7 +10,7 @@
 import assert from "node:assert/strict";
 import { describe, it } from "node:test";
 import { LiteracyVariant } from "../../normalize-mode.js";
-import { compileLiteracyTask, validateLiteracyTask, } from "../mode-handlers/literacy-handler.js";
+import { compileLiteracyTask, validateLiteracyTask, } from "../mode-handlers/literacy/index.js";
 import { compileLiteracyTasks, compareCompilerOutputs, } from "../literacy-bridge.js";
 // ---------------------------------------------------------------------------
 // Helpers

package/dist/pipeline/compiler/__tests__/mcp-server-handler.test.js

@@ -10,8 +10,7 @@
 import assert from "node:assert/strict";
 import { describe, it } from "node:test";
 import { LiteracyVariant } from "../../normalize-mode.js";
-import { compileMCPTask, handler as mcpHandler, MCP_PROMPT_TEMPLATES, validateMCPTask, } from "../mode-handlers/mcp-server-handler.js";
-import { buildMCPAssertions } from "../mode-handlers/mcp-assertions.js";
+import { buildMCPAssertions, compileMCPTask, handler as mcpHandler, MCP_PROMPT_TEMPLATES, validateMCPTask, } from "../mode-handlers/mcp-server/index.js";
 import { allMCPExampleTasks, createAndPublishTask, inspectSchemaTask, queryDocumentsTask, semanticSearchTask, stdioServerTask, } from "../mode-handlers/__fixtures__/mcp-example-tasks.js";
 // ---------------------------------------------------------------------------
 // Helpers
@@ -26,6 +25,29 @@ function makeMinimalMCPTask(overrides) {
         ...overrides,
     };
 }
+/** Test models for compilation — simulates models from the registry */
+const TEST_MODELS = [
+    {
+        id: "anthropic:messages:claude-opus-4-6",
+        label: "Claude Opus 4.6",
+        config: { temperature: 0.2 },
+    },
+    {
+        id: "openai:responses:gpt-5.4",
+        label: "GPT 5.4",
+        config: { reasoning_effort: "medium" },
+    },
+];
+/** The custom MCP provider file:// path */
+const MCP_PROVIDER_PATH = "file://dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js";
+/** Helper to get provider config */
+function cfg(provider) {
+    return provider.config;
+}
+/** Helper to get mcpServer sub-config from provider */
+function serverCfg(provider) {
+    return cfg(provider)?.mcpServer;
+}
 // ---------------------------------------------------------------------------
 // handler.getPrompts() — prompt template ownership
 // ---------------------------------------------------------------------------
@@ -38,11 +60,9 @@ describe("MCPServerHandler.getPrompts", () => {
     it("returns templates keyed by MCP-specific IDs (not literacy names)", () => {
         const prompts = mcpHandler.getPrompts();
         const keys = Object.keys(prompts);
-        // Must not use literacy template names
         assert.ok(!keys.includes("with-docs"), "should not use literacy key 'with-docs'");
         assert.ok(!keys.includes("without-docs"), "should not use literacy key 'without-docs'");
         assert.ok(!keys.includes(LiteracyVariant.AGENTIC), "should not use literacy key 'agentic'");
-        // Must have MCP-appropriate key(s)
         assert.ok(keys.includes("mcp-server"), "should include 'mcp-server' template");
     });
     it("mcp-server template instructs model to use MCP tools", () => {
@@ -50,7 +70,6 @@ describe("MCPServerHandler.getPrompts", () => {
         const template = prompts["mcp-server"];
         assert.ok(template, "mcp-server template should exist");
         assert.ok(template.template.includes("{{task}}"), "should include {{task}} placeholder");
-        // Should reference MCP tools / tool usage
         assert.ok(/tool/i.test(template.template), "template should mention tools (MCP-appropriate content)");
     });
     it("template has correct PromptTemplate shape", () => {
@@ -121,70 +140,124 @@ describe("validateMCPTask", () => {
     });
 });
 // ---------------------------------------------------------------------------
-// compileMCPTask
+// compileMCPTask — provider assembly
 // ---------------------------------------------------------------------------
 describe("compileMCPTask", () => {
     it("produces provider, tests, and prompts", () => {
-        const result = compileMCPTask(makeMinimalMCPTask());
+        const result = compileMCPTask(makeMinimalMCPTask(), { models: TEST_MODELS });
         assert.ok(result.providers.length > 0, "Should produce providers");
         assert.ok(result.tests.length > 0, "Should produce test cases");
         assert.ok(result.prompts.length > 0, "Should produce prompts");
     });
-    it("builds Promptfoo-native MCP provider for stdio", () => {
+    it("emits file:// providers using the custom MCP tool provider", () => {
         const result = compileMCPTask(makeMinimalMCPTask({
             serverConfig: {
                 transport: "stdio",
                 command: "node dist/server.js --flag",
             },
-        }));
-        assert.equal(result.providers.length, 1);
-        assert.equal(result.providers[0].id, "mcp");
-        const config = result.providers[0].config;
-        assert.equal(config.enabled, true);
-        const server = config.server;
-        assert.equal(server.command, "node");
-        assert.deepEqual(server.args, ["dist/server.js", "--flag"]);
-    });
-    it("builds Promptfoo-native MCP provider for URL-based transport", () => {
+        }), { models: TEST_MODELS });
+        assert.equal(result.providers.length, 2, "One provider per model");
+        // All providers use the custom MCP tool provider path
+        assert.equal(result.providers[0].id, MCP_PROVIDER_PATH);
+        assert.equal(result.providers[1].id, MCP_PROVIDER_PATH);
+        // Model ID is passed in config
+        assert.equal(cfg(result.providers[0]).model, "anthropic:messages:claude-opus-4-6");
+        assert.equal(cfg(result.providers[1]).model, "openai:responses:gpt-5.4");
+        // MCP server config is in config.mcpServer
+        const server = serverCfg(result.providers[0]);
+        assert.equal(server.command, "node dist/server.js --flag");
+    });
+    it("preserves model config in provider config", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            serverConfig: { transport: "sse", url: "http://localhost:3000/sse" },
+        }), { models: TEST_MODELS });
+        const c = cfg(result.providers[0]);
+        assert.equal(c.temperature, 0.2, "Model config preserved");
+        assert.ok(c.mcpServer, "MCP server config present");
+        assert.equal(c.maxToolRounds, 5, "Default maxToolRounds");
+    });
+    it("builds MCP server config for URL-based transport", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            serverConfig: { transport: "sse", url: "http://localhost:3000/sse" },
+        }), { models: TEST_MODELS });
+        const server = serverCfg(result.providers[0]);
+        assert.equal(server.url, "http://localhost:3000/sse");
+    });
+    it("maps auth config to mcpServer config", () => {
         const result = compileMCPTask(makeMinimalMCPTask({
             serverConfig: {
-                transport: "sse",
-                url: "http://localhost:3000/sse",
+                transport: "streamable-http",
+                url: "https://mcp.example.com",
+                auth: { type: "bearer", token: "{{env.MY_TOKEN}}" },
             },
-        }));
-        assert.equal(result.providers[0].id, "mcp");
-        const config = result.providers[0].config;
-        const server = config.server;
-        assert.equal(server.url, "http://localhost:3000/sse");
+        }), { models: TEST_MODELS });
+        const server = serverCfg(result.providers[0]);
+        assert.deepEqual(server.auth, { type: "bearer", token: "{{env.MY_TOKEN}}" });
     });
-    it("maps auth config to Promptfoo provider", () => {
+    it("maps headers to mcpServer config", () => {
         const result = compileMCPTask(makeMinimalMCPTask({
             serverConfig: {
                 transport: "streamable-http",
                 url: "https://mcp.example.com",
-                auth: {
-                    type: "bearer",
-                    token: "{{env.MY_TOKEN}}",
+                headers: {
+                    Authorization: "Bearer {{env.MY_TOKEN}}",
+                    "X-Custom": "value",
                 },
             },
-        }));
-        const config = result.providers[0].config;
-        const server = config.server;
+        }), { models: TEST_MODELS });
+        const server = serverCfg(result.providers[0]);
+        assert.deepEqual(server.headers, {
+            Authorization: "Bearer {{env.MY_TOKEN}}",
+            "X-Custom": "value",
+        });
+    });
+    it("passes both headers and auth when both present", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            serverConfig: {
+                transport: "streamable-http",
+                url: "https://mcp.example.com",
+                headers: { "X-Custom": "value" },
+                auth: { type: "bearer", token: "{{env.MY_TOKEN}}" },
+            },
+        }), { models: TEST_MODELS });
+        const server = serverCfg(result.providers[0]);
+        assert.deepEqual(server.headers, { "X-Custom": "value" });
         assert.deepEqual(server.auth, {
             type: "bearer",
             token: "{{env.MY_TOKEN}}",
         });
     });
-    it("maps capabilities to Promptfoo tools filter", () => {
+    it("maps capabilities to mcpTools config", () => {
         const result = compileMCPTask(makeMinimalMCPTask({
             capabilities: ["query_documents", "get_schema"],
             serverConfig: {
                 transport: "streamable-http",
                 url: "https://mcp.example.com",
             },
-        }));
-        const config = result.providers[0].config;
-        assert.deepEqual(config.tools, ["query_documents", "get_schema"]);
+        }), { models: TEST_MODELS });
+        assert.deepEqual(cfg(result.providers[0]).mcpTools, [
+            "query_documents",
+            "get_schema",
+        ]);
+    });
+    it("uses task-level models override when specified", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({
+            models: ["anthropic:messages:claude-sonnet-4-20250514"],
+            serverConfig: { transport: "sse", url: "http://localhost:3000" },
+        }), { models: TEST_MODELS });
+        assert.equal(result.providers.length, 1);
+        assert.equal(cfg(result.providers[0]).model, "anthropic:messages:claude-sonnet-4-20250514");
+    });
+    it("respects task-level maxToolRounds", () => {
+        const result = compileMCPTask(makeMinimalMCPTask({ maxToolRounds: 10 }), {
+            models: TEST_MODELS,
+        });
+        assert.equal(cfg(result.providers[0]).maxToolRounds, 10);
+    });
+    it("falls back to default model when no models provided", () => {
+        const result = compileMCPTask(makeMinimalMCPTask());
+        assert.ok(result.providers.length > 0, "Should have a fallback provider");
+        assert.ok(result.warnings.some((w) => w.includes("no models")));
     });
     it("uses task description as prompt text", () => {
         const result = compileMCPTask(makeMinimalMCPTask({
@@ -217,7 +290,6 @@ describe("compileMCPTask", () => {
                 ],
             },
         }));
-        // Primary + multi-turn test cases
         assert.equal(result.tests.length, 2);
         assert.ok(result.tests[1].description.includes("[multi-turn]"));
     });
@@ -293,63 +365,61 @@ describe("buildMCPAssertions", () => {
 // Example task compilation (end-to-end)
 // ---------------------------------------------------------------------------
 describe("example MCP tasks — end-to-end compilation", () => {
+    const opts = { models: TEST_MODELS };
     it("compiles all example tasks without errors", () => {
         for (const task of allMCPExampleTasks) {
-            const result = compileMCPTask(task);
+            const result = compileMCPTask(task, opts);
             assert.ok(result.providers.length > 0, `${task.id}: should produce providers`);
             assert.ok(result.tests.length > 0, `${task.id}: should produce test cases`);
             assert.ok(result.prompts.length > 0, `${task.id}: should produce prompts`);
         }
     });
     it("query task has tool-called + contains + llm-rubric assertions", () => {
-        const result = compileMCPTask(queryDocumentsTask);
+        const result = compileMCPTask(queryDocumentsTask, opts);
         const asserts = result.tests[0].assert;
-        // tool-called (→ javascript), contains × 2, llm-rubric
         assert.equal(asserts.length, 4);
-        assert.equal(asserts[0].type, "javascript"); // tool-called → javascript
+        assert.equal(asserts[0].type, "javascript");
         assert.equal(asserts[1].type, "contains");
         assert.equal(asserts[2].type, "contains");
         assert.equal(asserts[3].type, "llm-rubric");
     });
     it("schema task uses get_schema tool", () => {
-        const result = compileMCPTask(inspectSchemaTask);
+        const result = compileMCPTask(inspectSchemaTask, opts);
         const asserts = result.tests[0].assert;
         assert.ok(asserts.some((a) => a.type === "javascript" && a.value.includes("get_schema")), "Should have tool-called assertion for get_schema");
     });
     it("create-publish task produces multi-turn test case", () => {
-        const result = compileMCPTask(createAndPublishTask);
-        // Primary + multi-turn
+        const result = compileMCPTask(createAndPublishTask, opts);
         assert.equal(result.tests.length, 2);
         assert.ok(result.tests[1].description?.includes("[multi-turn]"));
     });
-    it("stdio task has Promptfoo-native MCP provider with command", () => {
-        const result = compileMCPTask(stdioServerTask);
-        assert.equal(result.providers[0].id, "mcp");
-        const config = result.providers[0].config;
-        assert.equal(config.enabled, true);
-        const server = config.server;
-        assert.equal(server.command, "node");
-        assert.deepEqual(server.args, ["dist/sanity-mcp-server.js"]);
+    it("stdio task uses custom provider with command config", () => {
+        const result = compileMCPTask(stdioServerTask, opts);
+        assert.equal(result.providers[0].id, MCP_PROVIDER_PATH);
+        assert.equal(cfg(result.providers[0]).model, "anthropic:messages:claude-opus-4-6");
+        const server = serverCfg(result.providers[0]);
+        assert.equal(server.command, "node dist/sanity-mcp-server.js");
     });
     it("semantic search task has two tool-called + one llm-rubric assertion", () => {
-        const result = compileMCPTask(semanticSearchTask);
+        const result = compileMCPTask(semanticSearchTask, opts);
         const asserts = result.tests[0].assert;
-        // tool-called × 2 (→ javascript) + llm-rubric
         assert.equal(asserts.length, 3);
-        assert.equal(asserts[0].type, "javascript"); // tool-called → javascript
-        assert.ok(asserts[0].value.includes("list_embeddings_indices"), "Should have tool-called assertion for list_embeddings_indices");
-        assert.equal(asserts[1].type, "javascript"); // tool-called → javascript
-        assert.ok(asserts[1].value.includes("semantic_search"), "Should have tool-called assertion for semantic_search");
+        assert.equal(asserts[0].type, "javascript");
+        assert.ok(asserts[0].value.includes("list_embeddings_indices"));
+        assert.equal(asserts[1].type, "javascript");
+        assert.ok(asserts[1].value.includes("semantic_search"));
         assert.equal(asserts[2].type, "llm-rubric");
     });
     it("remote task has bearer auth and tools filter", () => {
-        const result = compileMCPTask(queryDocumentsTask);
-        const config = result.providers[0].config;
-        const server = config.server;
+        const result = compileMCPTask(queryDocumentsTask, opts);
+        const server = serverCfg(result.providers[0]);
         assert.deepEqual(server.auth, {
             type: "bearer",
             token: "{{env.SANITY_MCP_AUTH_TOKEN}}",
         });
-        assert.deepEqual(config.tools, ["query_documents", "get_schema"]);
+        assert.deepEqual(cfg(result.providers[0]).mcpTools, [
+            "query_documents",
+            "get_schema",
+        ]);
     });
 });