@sanity/ailf 1.0.0 → 2.0.1

package/dist/pipeline/compiler/mode-handlers/mcp-server/compiler.js

@@ -0,0 +1,100 @@
+/**
+ * MCP server task compilation — core compiler logic.
+ *
+ * Produces Promptfoo configuration from MCP server task definitions:
+ * 1. A provider config pointing to the MCP server
+ * 2. Test cases with tool-call assertions
+ * 3. Appropriate prompts for the evaluation
+ */
+import { buildMCPAssertions } from "./assertions.js";
+import { buildMCPProvider } from "./provider-config.js";
+import { validateMCPTask } from "./validation.js";
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Compile an MCP server task definition into Promptfoo configuration.
+ *
+ * This is the core of the MCP mode handler. It produces:
+ * 1. A provider config pointing to the MCP server
+ * 2. Test cases with tool-call assertions
+ * 3. Appropriate prompts for the evaluation
+ */
+export function compileMCPTask(task, options) {
+    const warnings = [];
+    // Validate
+    const validationErrors = validateMCPTask(task);
+    if (validationErrors.length > 0) {
+        for (const err of validationErrors) {
+            warnings.push(`MCP task "${task.id}": ${err.field} — ${err.message}`);
+        }
+    }
+    // Build providers (one LLM provider per model, each with MCP config)
+    const providers = buildMCPProvider(task, options?.models ?? [], warnings);
+    // Build prompts
+    const prompts = buildMCPPrompts(task);
+    // Build test cases
+    const tests = buildMCPTestCases(task, options, warnings);
+    return { providers, tests, prompts, warnings };
+}
+// ---------------------------------------------------------------------------
+// Prompt assembly
+// ---------------------------------------------------------------------------
+function buildMCPPrompts(task) {
+    // MCP mode uses a single prompt — the task description
+    const promptText = task.prompt?.text ??
+        task.prompt?.vars?.task ??
+        task.description ??
+        `Test MCP server: ${task.title}`;
+    return [
+        {
+            id: "mcp-test",
+            label: `MCP: ${task.title}`,
+            raw: String(promptText),
+        },
+    ];
+}
+// ---------------------------------------------------------------------------
+// Test case assembly
+// ---------------------------------------------------------------------------
+function buildMCPTestCases(task, options, warnings) {
+    const tests = [];
+    // Build assertion context
+    const assertionContext = {
+        capabilities: task.capabilities ?? [],
+        graderProvider: options?.graderProvider,
+        taskId: task.id,
+    };
+    // Compile assertions
+    // Cast GeneralizedAssertionDefinition[] → AssertionInput[] (structurally compatible)
+    const assertions = [];
+    if (task.assertions) {
+        const rawAssertions = task.assertions;
+        const { assertions: mapped, warnings: assertionWarnings } = buildMCPAssertions(rawAssertions, assertionContext);
+        assertions.push(...mapped);
+        warnings.push(...assertionWarnings);
+    }
+    // Build test case vars
+    const vars = {
+        task: task.prompt?.vars?.task ?? task.description ?? `Test: ${task.title}`,
+        ...(task.prompt?.vars ?? {}),
+    };
+    // Primary test case
+    tests.push({
+        description: `${task.id} — ${task.title}`,
+        vars,
+        ...(assertions.length > 0 ? { assert: assertions } : {}),
+    });
+    // Multi-turn test cases
+    if (task.multiTurn?.turns && task.multiTurn.turns.length > 0) {
+        tests.push({
+            description: `${task.id} — ${task.title} [multi-turn]`,
+            vars: {
+                ...vars,
+                __multiTurn: task.multiTurn.turns,
+            },
+            ...(assertions.length > 0 ? { assert: assertions } : {}),
+        });
+    }
+    return tests;
+}

package/dist/pipeline/compiler/mode-handlers/mcp-server/index.d.ts

@@ -0,0 +1,27 @@
+/**
+ * MCP Server mode handler — directory barrel.
+ *
+ * MCPServerModeHandler — compilation rules for `mcp-server` evaluation mode.
+ *
+ * This is the first non-literacy mode handler, proving the compiler
+ * architecture works end-to-end. It translates MCP server task definitions
+ * into Promptfoo configuration with:
+ *
+ * - An MCP provider that wraps the server under test
+ * - Tool-call assertions compiled to Promptfoo `javascript` assertions
+ * - Server lifecycle management via Promptfoo provider hooks
+ * - Multi-turn conversation support via Promptfoo's `steps` syntax
+ *
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-3-mcp-server-mode.md
+ * @see packages/core/src/types/eval-mode-config.ts — MCPServerModeConfig
+ * @see packages/core/src/types/generalized-task.ts — MCPServerTaskDefinition
+ */
+import type { ModeHandler } from "../../../../_vendor/ailf-core/index.d.ts";
+/** ModeHandler-conformant export for the mcp-server evaluation mode. */
+export declare const handler: ModeHandler;
+export type { MCPAssertionContext, MCPCompileOptions, MCPCompileResult, MCPValidationError, } from "./types.js";
+export { buildMCPAssertions } from "./assertions.js";
+export { compileMCPTask } from "./compiler.js";
+export { validateMCPTask } from "./validation.js";
+export { MCP_PROMPT_TEMPLATES } from "./prompts.js";
+export { DEFAULT_MAX_TOOL_ROUNDS, MCP_PROVIDER_PATH, } from "./provider-config.js";

package/dist/pipeline/compiler/mode-handlers/mcp-server/index.js

@@ -0,0 +1,54 @@
+/**
+ * MCP Server mode handler — directory barrel.
+ *
+ * MCPServerModeHandler — compilation rules for `mcp-server` evaluation mode.
+ *
+ * This is the first non-literacy mode handler, proving the compiler
+ * architecture works end-to-end. It translates MCP server task definitions
+ * into Promptfoo configuration with:
+ *
+ * - An MCP provider that wraps the server under test
+ * - Tool-call assertions compiled to Promptfoo `javascript` assertions
+ * - Server lifecycle management via Promptfoo provider hooks
+ * - Multi-turn conversation support via Promptfoo's `steps` syntax
+ *
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-3-mcp-server-mode.md
+ * @see packages/core/src/types/eval-mode-config.ts — MCPServerModeConfig
+ * @see packages/core/src/types/generalized-task.ts — MCPServerTaskDefinition
+ */
+import { compileMCPTask } from "./compiler.js";
+import { MCP_PROMPT_TEMPLATES } from "./prompts.js";
+// ---------------------------------------------------------------------------
+// ModeHandler adapter
+// ---------------------------------------------------------------------------
+/** ModeHandler-conformant export for the mcp-server evaluation mode. */
+export const handler = {
+    getPrompts() {
+        return MCP_PROMPT_TEMPLATES;
+    },
+    compileTask(task, ctx) {
+        if (!("mode" in task) || task.mode !== "mcp-server") {
+            throw new Error(`MCP server handler received task with mode "${task.mode ?? "undefined"}" — expected "mcp-server"`);
+        }
+        const result = compileMCPTask(task, {
+            graderProvider: ctx.graderProvider,
+            models: ctx.models,
+        });
+        return {
+            providers: result.providers,
+            tests: result.tests,
+            prompts: result.prompts,
+            warnings: result.warnings,
+        };
+    },
+};
+// Assertions
+export { buildMCPAssertions } from "./assertions.js";
+// Compilation
+export { compileMCPTask } from "./compiler.js";
+// Validation
+export { validateMCPTask } from "./validation.js";
+// Prompts
+export { MCP_PROMPT_TEMPLATES } from "./prompts.js";
+// Provider config
+export { DEFAULT_MAX_TOOL_ROUNDS, MCP_PROVIDER_PATH, } from "./provider-config.js";

package/dist/pipeline/compiler/mode-handlers/mcp-server/prompts.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Canonical MCP server prompt templates.
+ *
+ * Handler-owned prompts for MCP server evaluations. Instructs the model to
+ * interact with MCP tools rather than writing standalone code.
+ */
+import type { PromptTemplate } from "../../../../_vendor/ailf-core/index.d.ts";
+export declare const MCP_PROMPT_TEMPLATES: Record<string, PromptTemplate>;

package/dist/pipeline/compiler/mode-handlers/mcp-server/prompts.js

@@ -0,0 +1,28 @@
+/**
+ * Canonical MCP server prompt templates.
+ *
+ * Handler-owned prompts for MCP server evaluations. Instructs the model to
+ * interact with MCP tools rather than writing standalone code.
+ */
+export const MCP_PROMPT_TEMPLATES = {
+    "mcp-server": {
+        id: "mcp-server",
+        label: "MCP Server Tool Use",
+        template: `You are an AI assistant with access to an MCP (Model Context Protocol) server that provides tools for interacting with a Sanity content backend.
+
+## Task
+{{task}}
+
+## Instructions
+
+1. Use the available MCP tools to complete the task
+2. Call tools with the correct parameters as described in their schemas
+3. Interpret tool responses and use the results to accomplish the goal
+4. If a tool returns an error, explain the issue clearly
+5. Prefer using specific tools over broad queries when possible
+
+Complete the task using the MCP tools provided:
+`,
+        variables: ["task"],
+    },
+};

package/dist/pipeline/compiler/mode-handlers/mcp-server/provider-config.d.ts

@@ -0,0 +1,28 @@
+/**
+ * MCP server provider assembly — builds Promptfoo provider configs.
+ */
+import type { MCPServerTaskDefinition, ModeProviderEntry } from "../../../../_vendor/ailf-core/index.d.ts";
+import type { PromptfooProvider } from "../../promptfoo-compiler.js";
+/** Default max tool rounds for MCP multi-turn execution */
+export declare const DEFAULT_MAX_TOOL_ROUNDS = 5;
+/** Provider path relative to eval package dist */
+export declare const MCP_PROVIDER_PATH = "file://dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js";
+/**
+ * Build custom MCP tool provider configs — one per model.
+ *
+ * Each provider uses the custom mcp-tool-provider.ts which implements a
+ * multi-turn tool execution loop. The LLM receives a prompt, discovers
+ * MCP tools, calls them, gets results, and continues until it produces
+ * a final text answer or exhausts maxToolRounds.
+ *
+ * Config shape passed to the custom provider:
+ *   { model, mcpServer: { url, auth, name }, mcpTools, maxToolRounds, temperature, ... }
+ */
+export declare function buildMCPProvider(task: MCPServerTaskDefinition, models: ModeProviderEntry[], warnings: string[]): PromptfooProvider[];
+/**
+ * Build the MCP server connection config for the custom provider.
+ *
+ * Shape: { url?, command?, name?, auth? }
+ * The custom mcp-tool-provider.ts uses this to connect to the MCP server.
+ */
+export declare function buildMCPServerConfig(task: MCPServerTaskDefinition, warnings: string[]): Record<string, unknown>;

package/dist/pipeline/compiler/mode-handlers/mcp-server/provider-config.js

@@ -0,0 +1,108 @@
+/**
+ * MCP server provider assembly — builds Promptfoo provider configs.
+ */
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+/** Default max tool rounds for MCP multi-turn execution */
+export const DEFAULT_MAX_TOOL_ROUNDS = 5;
+/** Provider path relative to eval package dist */
+export const MCP_PROVIDER_PATH = "file://dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js";
+// ---------------------------------------------------------------------------
+// Provider assembly
+// ---------------------------------------------------------------------------
+/**
+ * Build custom MCP tool provider configs — one per model.
+ *
+ * Each provider uses the custom mcp-tool-provider.ts which implements a
+ * multi-turn tool execution loop. The LLM receives a prompt, discovers
+ * MCP tools, calls them, gets results, and continues until it produces
+ * a final text answer or exhausts maxToolRounds.
+ *
+ * Config shape passed to the custom provider:
+ *   { model, mcpServer: { url, auth, name }, mcpTools, maxToolRounds, temperature, ... }
+ */
+export function buildMCPProvider(task, models, warnings) {
+    // Build the MCP server config
+    const mcpServer = buildMCPServerConfig(task, warnings);
+    const mcpTools = task.capabilities ?? undefined;
+    const maxToolRounds = task.maxToolRounds ?? DEFAULT_MAX_TOOL_ROUNDS;
+    // Helper to build a provider entry for a given model
+    function makeProvider(modelId, label, modelConfig) {
+        return {
+            id: MCP_PROVIDER_PATH,
+            label: `${label} + MCP`,
+            config: {
+                model: modelId,
+                mcpServer,
+                ...(mcpTools ? { mcpTools } : {}),
+                maxToolRounds,
+                ...(modelConfig ?? {}),
+            },
+        };
+    }
+    // Task-level model override takes precedence over registry models
+    const taskModels = task.models;
+    if (taskModels && taskModels.length > 0) {
+        return taskModels.map((modelId) => makeProvider(modelId, modelId));
+    }
+    // Use registry models (already filtered to mcp-server mode)
+    if (models.length === 0) {
+        warnings.push(`MCP task "${task.id}": no models available. Add "mcp-server" to a ` +
+            "model's modes array in config/models.ts, or set models on the task.");
+        return [
+            makeProvider("anthropic:messages:claude-sonnet-4-20250514", "Claude Sonnet 4"),
+        ];
+    }
+    return models.map((model) => makeProvider(model.id, model.label, model.config));
+}
+/**
+ * Build the MCP server connection config for the custom provider.
+ *
+ * Shape: { url?, command?, name?, auth? }
+ * The custom mcp-tool-provider.ts uses this to connect to the MCP server.
+ */
+export function buildMCPServerConfig(task, warnings) {
+    const config = task.serverConfig;
+    if (!config) {
+        warnings.push(`MCP task "${task.id}": no serverConfig — using placeholder. ` +
+            "Set serverConfig.command or serverConfig.url to point to your MCP server.");
+        return { name: task.id };
+    }
+    const serverConfig = { name: task.id };
+    if (config.transport === "stdio") {
+        serverConfig.command = config.command;
+    }
+    else {
+        serverConfig.url = config.url;
+    }
+    // Explicit headers for HTTP transports
+    if (config.headers) {
+        serverConfig.headers = config.headers;
+    }
+    // Auth config
+    if (config.auth) {
+        serverConfig.auth = config.auth;
+    }
+    else if (config.env) {
+        const tokenKey = Object.keys(config.env).find((k) => /token|auth|key/i.test(k));
+        if (tokenKey) {
+            const val = config.env[tokenKey];
+            let envVar = val;
+            if (val.startsWith("$env(") && val.endsWith(")")) {
+                envVar = val.slice(5, -1);
+            }
+            if (!envVar || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(envVar)) {
+                warnings.push(`MCP task: env var name "${envVar}" from "${val}" is not a valid ` +
+                    "identifier — skipping auth config");
+            }
+            else {
+                serverConfig.auth = {
+                    type: "bearer",
+                    token: `{{env.${envVar}}}`,
+                };
+            }
+        }
+    }
+    return serverConfig;
+}

package/dist/pipeline/compiler/mode-handlers/mcp-server/types.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Shared types for the MCP server mode handler.
+ */
+import type { ModeProviderEntry } from "../../../../_vendor/ailf-core/index.d.ts";
+import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../../promptfoo-compiler.js";
+/** Options for compiling an MCP server task */
+export interface MCPCompileOptions {
+    /** Grader provider for LLM-graded assertions */
+    graderProvider?: string;
+    /** Model providers to evaluate with (from registry, filtered by mcp-server mode) */
+    models?: ModeProviderEntry[];
+}
+/** Result of compiling a single MCP task */
+export interface MCPCompileResult {
+    /** Promptfoo provider config for the MCP server */
+    providers: PromptfooProvider[];
+    /** Compiled test cases */
+    tests: PromptfooTestCase[];
+    /** Prompts for MCP evaluation */
+    prompts: PromptfooPrompt[];
+    /** Warnings generated during compilation */
+    warnings: string[];
+}
+/** Validation errors for MCP task definitions */
+export interface MCPValidationError {
+    field: string;
+    message: string;
+}
+/** Context for building MCP assertions */
+export interface MCPAssertionContext {
+    /** Task ID (for error messages) */
+    taskId: string;
+    /** Expected server capabilities */
+    capabilities: string[];
+    /** Grader provider for LLM-graded assertions */
+    graderProvider?: string;
+}

package/dist/pipeline/compiler/mode-handlers/mcp-server/types.js

@@ -0,0 +1,4 @@
+/**
+ * Shared types for the MCP server mode handler.
+ */
+export {};

package/dist/pipeline/compiler/mode-handlers/mcp-server/validation.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Validation for MCP server task definitions.
+ */
+import type { MCPServerTaskDefinition } from "../../../../_vendor/ailf-core/index.d.ts";
+import type { MCPValidationError } from "./types.js";
+/**
+ * Validate that an MCP task definition has all required fields.
+ */
+export declare function validateMCPTask(task: MCPServerTaskDefinition): MCPValidationError[];

package/dist/pipeline/compiler/mode-handlers/mcp-server/validation.js

@@ -0,0 +1,43 @@
+/**
+ * Validation for MCP server task definitions.
+ */
+/**
+ * Validate that an MCP task definition has all required fields.
+ */
+export function validateMCPTask(task) {
+    const errors = [];
+    if (!task.id) {
+        errors.push({ field: "id", message: "Task ID is required" });
+    }
+    if (!task.title) {
+        errors.push({ field: "title", message: "Task title is required" });
+    }
+    if (task.serverConfig) {
+        const { transport, command, url } = task.serverConfig;
+        if (transport === "stdio" && !command) {
+            errors.push({
+                field: "serverConfig.command",
+                message: "Server command is required for stdio transport (e.g., 'node dist/server.js')",
+            });
+        }
+        if ((transport === "sse" || transport === "streamable-http") && !url) {
+            errors.push({
+                field: "serverConfig.url",
+                message: `Server URL is required for ${transport} transport`,
+            });
+        }
+    }
+    // Assertions should reference MCP-compatible types
+    if (task.assertions) {
+        for (const assertion of task.assertions) {
+            if (assertion.type === "tool-called" &&
+                !("value" in assertion && assertion.value)) {
+                errors.push({
+                    field: "assertions",
+                    message: 'tool-called assertion requires a "value" specifying the tool name',
+                });
+            }
+        }
+    }
+    return errors;
+}

package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.d.ts

@@ -26,13 +26,15 @@
  * @see packages/core/src/types/eval-mode-config.ts — MCPServerModeConfig
  * @see packages/core/src/types/generalized-task.ts — MCPServerTaskDefinition
  */
-import type { MCPServerTaskDefinition, ModeHandler, PromptTemplate } from "../../../_vendor/ailf-core/index.d.ts";
+import type { MCPServerTaskDefinition, ModeHandler, ModeProviderEntry, PromptTemplate } from "../../../_vendor/ailf-core/index.d.ts";
 import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../promptfoo-compiler.js";
 export declare const MCP_PROMPT_TEMPLATES: Record<string, PromptTemplate>;
 /** Options for compiling an MCP server task */
 export interface MCPCompileOptions {
     /** Grader provider for LLM-graded assertions */
     graderProvider?: string;
+    /** Model providers to evaluate with (from registry, filtered by mcp-server mode) */
+    models?: ModeProviderEntry[];
 }
 /** Result of compiling a single MCP task */
 export interface MCPCompileResult {

package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.js

@@ -114,8 +114,8 @@ export function compileMCPTask(task, options) {
             warnings.push(`MCP task "${task.id}": ${err.field} — ${err.message}`);
         }
     }
-    // Build provider
-    const providers = buildMCPProvider(task, warnings);
+    // Build providers (one LLM provider per model, each with MCP config)
+    const providers = buildMCPProvider(task, options?.models ?? [], warnings);
     // Build prompts
     const prompts = buildMCPPrompts(task);
     // Build test cases
@@ -125,103 +125,100 @@ export function compileMCPTask(task, options) {
 // ---------------------------------------------------------------------------
 // Provider assembly
 // ---------------------------------------------------------------------------
+/** Default max tool rounds for MCP multi-turn execution */
+const DEFAULT_MAX_TOOL_ROUNDS = 5;
+/** Provider path relative to eval package dist */
+const MCP_PROVIDER_PATH = "file://dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js";
 /**
- * Build a Promptfoo-native MCP provider config.
+ * Build custom MCP tool provider configs — one per model.
  *
- * Promptfoo supports MCP servers natively via `id: "mcp"` with a
- * structured config. See: https://www.promptfoo.dev/docs/providers/mcp/
+ * Each provider uses the custom mcp-tool-provider.ts which implements a
+ * multi-turn tool execution loop. The LLM receives a prompt, discovers
+ * MCP tools, calls them, gets results, and continues until it produces
+ * a final text answer or exhausts maxToolRounds.
  *
- * Key config shape:
- *   { enabled: true, server: { url?, command?, args?, name?, auth?, headers? },
- *     tools?, exclude_tools?, timeout?, debug? }
+ * Config shape passed to the custom provider:
+ *   { model, mcpServer: { url, auth, name }, mcpTools, maxToolRounds, temperature, ... }
  */
-function buildMCPProvider(task, warnings) {
+function buildMCPProvider(task, models, warnings) {
+    // Build the MCP server config
+    const mcpServer = buildMCPServerConfig(task, warnings);
+    const mcpTools = task.capabilities ?? undefined;
+    const maxToolRounds = task.maxToolRounds ?? DEFAULT_MAX_TOOL_ROUNDS;
+    // Helper to build a provider entry for a given model
+    function makeProvider(modelId, label, modelConfig) {
+        return {
+            id: MCP_PROVIDER_PATH,
+            label: `${label} + MCP`,
+            config: {
+                model: modelId,
+                mcpServer,
+                ...(mcpTools ? { mcpTools } : {}),
+                maxToolRounds,
+                ...(modelConfig ?? {}),
+            },
+        };
+    }
+    // Task-level model override takes precedence over registry models
+    const taskModels = task.models;
+    if (taskModels && taskModels.length > 0) {
+        return taskModels.map((modelId) => makeProvider(modelId, modelId));
+    }
+    // Use registry models (already filtered to mcp-server mode)
+    if (models.length === 0) {
+        warnings.push(`MCP task "${task.id}": no models available. Add "mcp-server" to a ` +
+            "model's modes array in config/models.ts, or set models on the task.");
+        return [
+            makeProvider("anthropic:messages:claude-sonnet-4-20250514", "Claude Sonnet 4"),
+        ];
+    }
+    return models.map((model) => makeProvider(model.id, model.label, model.config));
+}
+/**
+ * Build the MCP server connection config for the custom provider.
+ *
+ * Shape: { url?, command?, name?, auth? }
+ * The custom mcp-tool-provider.ts uses this to connect to the MCP server.
+ */
+function buildMCPServerConfig(task, warnings) {
     const config = task.serverConfig;
     if (!config) {
-        warnings.push(`MCP task "${task.id}": no serverConfig — using placeholder provider. ` +
+        warnings.push(`MCP task "${task.id}": no serverConfig — using placeholder. ` +
             "Set serverConfig.command or serverConfig.url to point to your MCP server.");
-        return [
-            {
-                id: "mcp",
-                label: `MCP Server: ${task.title}`,
-                config: { enabled: true, server: { name: task.id } },
-            },
-        ];
+        return { name: task.id };
     }
-    // Build the server sub-config (Promptfoo's native format)
-    const server = { name: task.id };
+    const serverConfig = { name: task.id };
     if (config.transport === "stdio") {
-        // Promptfoo expects command + args as separate fields
-        const parts = config.command?.split(/\s+/) ?? [];
-        server.command = parts[0] ?? "node";
-        if (parts.length > 1) {
-            server.args = parts.slice(1);
-        }
+        serverConfig.command = config.command;
     }
     else {
-        // sse or streamable-http — use URL-based connection
-        server.url = config.url;
+        serverConfig.url = config.url;
     }
-    // Auth config (Promptfoo supports bearer, basic, api_key, oauth)
+    // Auth config
     if (config.auth) {
-        server.auth = config.auth;
+        serverConfig.auth = config.auth;
     }
     else if (config.env) {
-        // Backward compat: if env has a token-like variable, convert to
-        // bearer auth using Promptfoo's {{env.VAR}} template syntax
         const tokenKey = Object.keys(config.env).find((k) => /token|auth|key/i.test(k));
         if (tokenKey) {
             const val = config.env[tokenKey];
-            // Convert $env(VAR) syntax to Promptfoo's {{env.VAR}} syntax
             let envVar = val;
             if (val.startsWith("$env(") && val.endsWith(")")) {
-                envVar = val.slice(5, -1); // $env(VAR_NAME) → VAR_NAME
+                envVar = val.slice(5, -1);
             }
-            // Validate extracted env var name is non-empty and valid
             if (!envVar || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(envVar)) {
                 warnings.push(`MCP task: env var name "${envVar}" from "${val}" is not a valid ` +
                     "identifier — skipping auth config");
             }
             else {
-                server.auth = {
+                serverConfig.auth = {
                     type: "bearer",
                     token: `{{env.${envVar}}}`,
                 };
             }
         }
     }
-    // Custom headers (if any non-auth env vars remain)
-    if (config.env) {
-        const headers = {};
-        for (const [key, val] of Object.entries(config.env)) {
-            if (/header[_.]?/i.test(key)) {
-                headers[key.replace(/^header[_.]?/i, "")] = val;
-            }
-        }
-        if (Object.keys(headers).length > 0) {
-            server.headers = headers;
-        }
-    }
-    // Build top-level provider config
-    const providerConfig = {
-        enabled: true,
-        server,
-    };
-    // Tool filtering — map AILF capabilities to Promptfoo tools
-    if (task.capabilities && task.capabilities.length > 0) {
-        providerConfig.tools = task.capabilities;
-    }
-    // Timeout
-    if (config.startupTimeoutMs) {
-        providerConfig.timeout = config.startupTimeoutMs;
-    }
-    return [
-        {
-            id: "mcp",
-            label: `MCP Server: ${task.title}`,
-            config: providerConfig,
-        },
-    ];
+    return serverConfig;
 }
 // ---------------------------------------------------------------------------
 // Prompt assembly
@@ -298,6 +295,7 @@ export const handler = {
         }
         const result = compileMCPTask(task, {
             graderProvider: ctx.graderProvider,
+            models: ctx.models,
         });
         return {
             providers: result.providers,