@sanity/ailf 1.0.0 → 2.0.0

package/dist/pipeline/compiler/mode-handlers/mcp-server/compiler.js

@@ -0,0 +1,100 @@
+/**
+ * MCP server task compilation — core compiler logic.
+ *
+ * Produces Promptfoo configuration from MCP server task definitions:
+ * 1. A provider config pointing to the MCP server
+ * 2. Test cases with tool-call assertions
+ * 3. Appropriate prompts for the evaluation
+ */
+import { buildMCPAssertions } from "./assertions.js";
+import { buildMCPProvider } from "./provider-config.js";
+import { validateMCPTask } from "./validation.js";
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+/**
+ * Compile an MCP server task definition into Promptfoo configuration.
+ *
+ * This is the core of the MCP mode handler. It produces:
+ * 1. A provider config pointing to the MCP server
+ * 2. Test cases with tool-call assertions
+ * 3. Appropriate prompts for the evaluation
+ */
+export function compileMCPTask(task, options) {
+    const warnings = [];
+    // Validate
+    const validationErrors = validateMCPTask(task);
+    if (validationErrors.length > 0) {
+        for (const err of validationErrors) {
+            warnings.push(`MCP task "${task.id}": ${err.field} — ${err.message}`);
+        }
+    }
+    // Build providers (one LLM provider per model, each with MCP config)
+    const providers = buildMCPProvider(task, options?.models ?? [], warnings);
+    // Build prompts
+    const prompts = buildMCPPrompts(task);
+    // Build test cases
+    const tests = buildMCPTestCases(task, options, warnings);
+    return { providers, tests, prompts, warnings };
+}
+// ---------------------------------------------------------------------------
+// Prompt assembly
+// ---------------------------------------------------------------------------
+function buildMCPPrompts(task) {
+    // MCP mode uses a single prompt — the task description
+    const promptText = task.prompt?.text ??
+        task.prompt?.vars?.task ??
+        task.description ??
+        `Test MCP server: ${task.title}`;
+    return [
+        {
+            id: "mcp-test",
+            label: `MCP: ${task.title}`,
+            raw: String(promptText),
+        },
+    ];
+}
+// ---------------------------------------------------------------------------
+// Test case assembly
+// ---------------------------------------------------------------------------
+function buildMCPTestCases(task, options, warnings) {
+    const tests = [];
+    // Build assertion context
+    const assertionContext = {
+        capabilities: task.capabilities ?? [],
+        graderProvider: options?.graderProvider,
+        taskId: task.id,
+    };
+    // Compile assertions
+    // Cast GeneralizedAssertionDefinition[] → AssertionInput[] (structurally compatible)
+    const assertions = [];
+    if (task.assertions) {
+        const rawAssertions = task.assertions;
+        const { assertions: mapped, warnings: assertionWarnings } = buildMCPAssertions(rawAssertions, assertionContext);
+        assertions.push(...mapped);
+        warnings.push(...assertionWarnings);
+    }
+    // Build test case vars
+    const vars = {
+        task: task.prompt?.vars?.task ?? task.description ?? `Test: ${task.title}`,
+        ...(task.prompt?.vars ?? {}),
+    };
+    // Primary test case
+    tests.push({
+        description: `${task.id} — ${task.title}`,
+        vars,
+        ...(assertions.length > 0 ? { assert: assertions } : {}),
+    });
+    // Multi-turn test cases
+    if (task.multiTurn?.turns && task.multiTurn.turns.length > 0) {
+        tests.push({
+            description: `${task.id} — ${task.title} [multi-turn]`,
+            vars: {
+                ...vars,
+                __multiTurn: task.multiTurn.turns,
+            },
+            ...(assertions.length > 0 ? { assert: assertions } : {}),
+        });
+    }
+    return tests;
+}

package/dist/pipeline/compiler/mode-handlers/mcp-server/index.d.ts

@@ -0,0 +1,27 @@
+/**
+ * MCP Server mode handler — directory barrel.
+ *
+ * MCPServerModeHandler — compilation rules for `mcp-server` evaluation mode.
+ *
+ * This is the first non-literacy mode handler, proving the compiler
+ * architecture works end-to-end. It translates MCP server task definitions
+ * into Promptfoo configuration with:
+ *
+ * - An MCP provider that wraps the server under test
+ * - Tool-call assertions compiled to Promptfoo `javascript` assertions
+ * - Server lifecycle management via Promptfoo provider hooks
+ * - Multi-turn conversation support via Promptfoo's `steps` syntax
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-3-mcp-server-mode.md
+ * @see packages/core/src/types/eval-mode-config.ts — MCPServerModeConfig
+ * @see packages/core/src/types/generalized-task.ts — MCPServerTaskDefinition
+ */
+import type { ModeHandler } from "../../../../_vendor/ailf-core/index.d.ts";
+/** ModeHandler-conformant export for the mcp-server evaluation mode. */
+export declare const handler: ModeHandler;
+export type { MCPAssertionContext, MCPCompileOptions, MCPCompileResult, MCPValidationError, } from "./types.js";
+export { buildMCPAssertions } from "./assertions.js";
+export { compileMCPTask } from "./compiler.js";
+export { validateMCPTask } from "./validation.js";
+export { MCP_PROMPT_TEMPLATES } from "./prompts.js";
+export { DEFAULT_MAX_TOOL_ROUNDS, MCP_PROVIDER_PATH, } from "./provider-config.js";

package/dist/pipeline/compiler/mode-handlers/mcp-server/index.js

@@ -0,0 +1,54 @@
+/**
+ * MCP Server mode handler — directory barrel.
+ *
+ * MCPServerModeHandler — compilation rules for `mcp-server` evaluation mode.
+ *
+ * This is the first non-literacy mode handler, proving the compiler
+ * architecture works end-to-end. It translates MCP server task definitions
+ * into Promptfoo configuration with:
+ *
+ * - An MCP provider that wraps the server under test
+ * - Tool-call assertions compiled to Promptfoo `javascript` assertions
+ * - Server lifecycle management via Promptfoo provider hooks
+ * - Multi-turn conversation support via Promptfoo's `steps` syntax
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-3-mcp-server-mode.md
+ * @see packages/core/src/types/eval-mode-config.ts — MCPServerModeConfig
+ * @see packages/core/src/types/generalized-task.ts — MCPServerTaskDefinition
+ */
+import { compileMCPTask } from "./compiler.js";
+import { MCP_PROMPT_TEMPLATES } from "./prompts.js";
+// ---------------------------------------------------------------------------
+// ModeHandler adapter
+// ---------------------------------------------------------------------------
+/** ModeHandler-conformant export for the mcp-server evaluation mode. */
+export const handler = {
+    getPrompts() {
+        return MCP_PROMPT_TEMPLATES;
+    },
+    compileTask(task, ctx) {
+        if (!("mode" in task) || task.mode !== "mcp-server") {
+            throw new Error(`MCP server handler received task with mode "${task.mode ?? "undefined"}" — expected "mcp-server"`);
+        }
+        const result = compileMCPTask(task, {
+            graderProvider: ctx.graderProvider,
+            models: ctx.models,
+        });
+        return {
+            providers: result.providers,
+            tests: result.tests,
+            prompts: result.prompts,
+            warnings: result.warnings,
+        };
+    },
+};
+// Assertions
+export { buildMCPAssertions } from "./assertions.js";
+// Compilation
+export { compileMCPTask } from "./compiler.js";
+// Validation
+export { validateMCPTask } from "./validation.js";
+// Prompts
+export { MCP_PROMPT_TEMPLATES } from "./prompts.js";
+// Provider config
+export { DEFAULT_MAX_TOOL_ROUNDS, MCP_PROVIDER_PATH, } from "./provider-config.js";

package/dist/pipeline/compiler/mode-handlers/mcp-server/prompts.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Canonical MCP server prompt templates.
+ *
+ * Handler-owned prompts for MCP server evaluations. Instructs the model to
+ * interact with MCP tools rather than writing standalone code.
+ */
+import type { PromptTemplate } from "../../../../_vendor/ailf-core/index.d.ts";
+export declare const MCP_PROMPT_TEMPLATES: Record<string, PromptTemplate>;

package/dist/pipeline/compiler/mode-handlers/mcp-server/prompts.js

@@ -0,0 +1,28 @@
+/**
+ * Canonical MCP server prompt templates.
+ *
+ * Handler-owned prompts for MCP server evaluations. Instructs the model to
+ * interact with MCP tools rather than writing standalone code.
+ */
+export const MCP_PROMPT_TEMPLATES = {
+    "mcp-server": {
+        id: "mcp-server",
+        label: "MCP Server Tool Use",
+        template: `You are an AI assistant with access to an MCP (Model Context Protocol) server that provides tools for interacting with a Sanity content backend.
+
+## Task
+{{task}}
+
+## Instructions
+
+1. Use the available MCP tools to complete the task
+2. Call tools with the correct parameters as described in their schemas
+3. Interpret tool responses and use the results to accomplish the goal
+4. If a tool returns an error, explain the issue clearly
+5. Prefer using specific tools over broad queries when possible
+
+Complete the task using the MCP tools provided:
+`,
+        variables: ["task"],
+    },
+};

package/dist/pipeline/compiler/mode-handlers/mcp-server/provider-config.d.ts

@@ -0,0 +1,28 @@
+/**
+ * MCP server provider assembly — builds Promptfoo provider configs.
+ */
+import type { MCPServerTaskDefinition, ModeProviderEntry } from "../../../../_vendor/ailf-core/index.d.ts";
+import type { PromptfooProvider } from "../../promptfoo-compiler.js";
+/** Default max tool rounds for MCP multi-turn execution */
+export declare const DEFAULT_MAX_TOOL_ROUNDS = 5;
+/** Provider path relative to eval package dist */
+export declare const MCP_PROVIDER_PATH = "file://dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js";
+/**
+ * Build custom MCP tool provider configs — one per model.
+ *
+ * Each provider uses the custom mcp-tool-provider.ts which implements a
+ * multi-turn tool execution loop. The LLM receives a prompt, discovers
+ * MCP tools, calls them, gets results, and continues until it produces
+ * a final text answer or exhausts maxToolRounds.
+ *
+ * Config shape passed to the custom provider:
+ *   { model, mcpServer: { url, auth, name }, mcpTools, maxToolRounds, temperature, ... }
+ */
+export declare function buildMCPProvider(task: MCPServerTaskDefinition, models: ModeProviderEntry[], warnings: string[]): PromptfooProvider[];
+/**
+ * Build the MCP server connection config for the custom provider.
+ *
+ * Shape: { url?, command?, name?, auth? }
+ * The custom mcp-tool-provider.ts uses this to connect to the MCP server.
+ */
+export declare function buildMCPServerConfig(task: MCPServerTaskDefinition, warnings: string[]): Record<string, unknown>;

package/dist/pipeline/compiler/mode-handlers/mcp-server/provider-config.js

@@ -0,0 +1,104 @@
+/**
+ * MCP server provider assembly — builds Promptfoo provider configs.
+ */
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+/** Default max tool rounds for MCP multi-turn execution */
+export const DEFAULT_MAX_TOOL_ROUNDS = 5;
+/** Provider path relative to eval package dist */
+export const MCP_PROVIDER_PATH = "file://dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js";
+// ---------------------------------------------------------------------------
+// Provider assembly
+// ---------------------------------------------------------------------------
+/**
+ * Build custom MCP tool provider configs — one per model.
+ *
+ * Each provider uses the custom mcp-tool-provider.ts which implements a
+ * multi-turn tool execution loop. The LLM receives a prompt, discovers
+ * MCP tools, calls them, gets results, and continues until it produces
+ * a final text answer or exhausts maxToolRounds.
+ *
+ * Config shape passed to the custom provider:
+ *   { model, mcpServer: { url, auth, name }, mcpTools, maxToolRounds, temperature, ... }
+ */
+export function buildMCPProvider(task, models, warnings) {
+    // Build the MCP server config
+    const mcpServer = buildMCPServerConfig(task, warnings);
+    const mcpTools = task.capabilities ?? undefined;
+    const maxToolRounds = task.maxToolRounds ?? DEFAULT_MAX_TOOL_ROUNDS;
+    // Helper to build a provider entry for a given model
+    function makeProvider(modelId, label, modelConfig) {
+        return {
+            id: MCP_PROVIDER_PATH,
+            label: `${label} + MCP`,
+            config: {
+                model: modelId,
+                mcpServer,
+                ...(mcpTools ? { mcpTools } : {}),
+                maxToolRounds,
+                ...(modelConfig ?? {}),
+            },
+        };
+    }
+    // Task-level model override takes precedence over registry models
+    const taskModels = task.models;
+    if (taskModels && taskModels.length > 0) {
+        return taskModels.map((modelId) => makeProvider(modelId, modelId));
+    }
+    // Use registry models (already filtered to mcp-server mode)
+    if (models.length === 0) {
+        warnings.push(`MCP task "${task.id}": no models available. Add "mcp-server" to a ` +
+            "model's modes array in config/models.ts, or set models on the task.");
+        return [
+            makeProvider("anthropic:messages:claude-sonnet-4-20250514", "Claude Sonnet 4"),
+        ];
+    }
+    return models.map((model) => makeProvider(model.id, model.label, model.config));
+}
+/**
+ * Build the MCP server connection config for the custom provider.
+ *
+ * Shape: { url?, command?, name?, auth? }
+ * The custom mcp-tool-provider.ts uses this to connect to the MCP server.
+ */
+export function buildMCPServerConfig(task, warnings) {
+    const config = task.serverConfig;
+    if (!config) {
+        warnings.push(`MCP task "${task.id}": no serverConfig — using placeholder. ` +
+            "Set serverConfig.command or serverConfig.url to point to your MCP server.");
+        return { name: task.id };
+    }
+    const serverConfig = { name: task.id };
+    if (config.transport === "stdio") {
+        serverConfig.command = config.command;
+    }
+    else {
+        serverConfig.url = config.url;
+    }
+    // Auth config
+    if (config.auth) {
+        serverConfig.auth = config.auth;
+    }
+    else if (config.env) {
+        const tokenKey = Object.keys(config.env).find((k) => /token|auth|key/i.test(k));
+        if (tokenKey) {
+            const val = config.env[tokenKey];
+            let envVar = val;
+            if (val.startsWith("$env(") && val.endsWith(")")) {
+                envVar = val.slice(5, -1);
+            }
+            if (!envVar || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(envVar)) {
+                warnings.push(`MCP task: env var name "${envVar}" from "${val}" is not a valid ` +
+                    "identifier — skipping auth config");
+            }
+            else {
+                serverConfig.auth = {
+                    type: "bearer",
+                    token: `{{env.${envVar}}}`,
+                };
+            }
+        }
+    }
+    return serverConfig;
+}

package/dist/pipeline/compiler/mode-handlers/mcp-server/types.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Shared types for the MCP server mode handler.
+ */
+import type { ModeProviderEntry } from "../../../../_vendor/ailf-core/index.d.ts";
+import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../../promptfoo-compiler.js";
+/** Options for compiling an MCP server task */
+export interface MCPCompileOptions {
+    /** Grader provider for LLM-graded assertions */
+    graderProvider?: string;
+    /** Model providers to evaluate with (from registry, filtered by mcp-server mode) */
+    models?: ModeProviderEntry[];
+}
+/** Result of compiling a single MCP task */
+export interface MCPCompileResult {
+    /** Promptfoo provider config for the MCP server */
+    providers: PromptfooProvider[];
+    /** Compiled test cases */
+    tests: PromptfooTestCase[];
+    /** Prompts for MCP evaluation */
+    prompts: PromptfooPrompt[];
+    /** Warnings generated during compilation */
+    warnings: string[];
+}
+/** Validation errors for MCP task definitions */
+export interface MCPValidationError {
+    field: string;
+    message: string;
+}
+/** Context for building MCP assertions */
+export interface MCPAssertionContext {
+    /** Task ID (for error messages) */
+    taskId: string;
+    /** Expected server capabilities */
+    capabilities: string[];
+    /** Grader provider for LLM-graded assertions */
+    graderProvider?: string;
+}

package/dist/pipeline/compiler/mode-handlers/mcp-server/types.js

@@ -0,0 +1,4 @@
+/**
+ * Shared types for the MCP server mode handler.
+ */
+export {};

package/dist/pipeline/compiler/mode-handlers/mcp-server/validation.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Validation for MCP server task definitions.
+ */
+import type { MCPServerTaskDefinition } from "../../../../_vendor/ailf-core/index.d.ts";
+import type { MCPValidationError } from "./types.js";
+/**
+ * Validate that an MCP task definition has all required fields.
+ */
+export declare function validateMCPTask(task: MCPServerTaskDefinition): MCPValidationError[];

package/dist/pipeline/compiler/mode-handlers/mcp-server/validation.js

@@ -0,0 +1,43 @@
+/**
+ * Validation for MCP server task definitions.
+ */
+/**
+ * Validate that an MCP task definition has all required fields.
+ */
+export function validateMCPTask(task) {
+    const errors = [];
+    if (!task.id) {
+        errors.push({ field: "id", message: "Task ID is required" });
+    }
+    if (!task.title) {
+        errors.push({ field: "title", message: "Task title is required" });
+    }
+    if (task.serverConfig) {
+        const { transport, command, url } = task.serverConfig;
+        if (transport === "stdio" && !command) {
+            errors.push({
+                field: "serverConfig.command",
+                message: "Server command is required for stdio transport (e.g., 'node dist/server.js')",
+            });
+        }
+        if ((transport === "sse" || transport === "streamable-http") && !url) {
+            errors.push({
+                field: "serverConfig.url",
+                message: `Server URL is required for ${transport} transport`,
+            });
+        }
+    }
+    // Assertions should reference MCP-compatible types
+    if (task.assertions) {
+        for (const assertion of task.assertions) {
+            if (assertion.type === "tool-called" &&
+                !("value" in assertion && assertion.value)) {
+                errors.push({
+                    field: "assertions",
+                    message: 'tool-called assertion requires a "value" specifying the tool name',
+                });
+            }
+        }
+    }
+    return errors;
+}

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.d.ts

@@ -0,0 +1,33 @@
+/**
+ * MCPToolProvider — Custom Promptfoo provider for MCP tool-use evaluation.
+ *
+ * Orchestrates the MCP evaluation flow:
+ * 1. Connects to the MCP server and discovers available tools
+ * 2. Selects the appropriate LLM backend based on model ID prefix
+ * 3. Delegates the multi-turn tool loop to the backend
+ * 4. Formats the result for Promptfoo (including tool call summary)
+ *
+ * Promptfoo config usage:
+ *
+ *   providers:
+ *     - id: file://dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js
+ *       label: "Claude Opus 4.6 + MCP"
+ *       config:
+ *         model: anthropic:messages:claude-opus-4-6
+ *         maxToolRounds: 5
+ *         temperature: 0.2
+ *         max_tokens: 4096
+ *         mcpServer:
+ *           url: https://mcp.sanity.io
+ *           auth: { type: bearer, token: "{{env.SANITY_API_TOKEN}}" }
+ *           name: mcp-live-query-documents
+ *         mcpTools: [query_documents, get_schema]
+ */
+import type { CallApiContextParams, ProviderOptions, ProviderResponse } from "./types.js";
+export default class MCPToolProvider {
+    config: Record<string, unknown>;
+    private providerId;
+    constructor(options?: ProviderOptions);
+    id(): string;
+    callApi(prompt: string, _context?: CallApiContextParams): Promise<ProviderResponse>;
+}

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js

@@ -0,0 +1,174 @@
+/**
+ * MCPToolProvider — Custom Promptfoo provider for MCP tool-use evaluation.
+ *
+ * Orchestrates the MCP evaluation flow:
+ * 1. Connects to the MCP server and discovers available tools
+ * 2. Selects the appropriate LLM backend based on model ID prefix
+ * 3. Delegates the multi-turn tool loop to the backend
+ * 4. Formats the result for Promptfoo (including tool call summary)
+ *
+ * Promptfoo config usage:
+ *
+ *   providers:
+ *     - id: file://dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js
+ *       label: "Claude Opus 4.6 + MCP"
+ *       config:
+ *         model: anthropic:messages:claude-opus-4-6
+ *         maxToolRounds: 5
+ *         temperature: 0.2
+ *         max_tokens: 4096
+ *         mcpServer:
+ *           url: https://mcp.sanity.io
+ *           auth: { type: bearer, token: "{{env.SANITY_API_TOKEN}}" }
+ *           name: mcp-live-query-documents
+ *         mcpTools: [query_documents, get_schema]
+ */
+import { config as loadDotenv } from "dotenv";
+import { connectMCP } from "./mcp-connection.js";
+import { runAnthropicToolLoop } from "./tool-loop-anthropic.js";
+import { runOpenAIToolLoop } from "./tool-loop-openai.js";
+loadDotenv({
+    override: true,
+    path: new URL("../../../../../.env", import.meta.url).pathname,
+});
+// ---------------------------------------------------------------------------
+// Backend registry — maps model ID prefixes to tool loop implementations
+// ---------------------------------------------------------------------------
+const BACKENDS = {
+    anthropic: runAnthropicToolLoop,
+    openai: runOpenAIToolLoop,
+};
+/**
+ * Resolve the LLM backend from a model ID.
+ *
+ * Model IDs follow the pattern `provider:type:model-name` (e.g.,
+ * `anthropic:messages:claude-opus-4-6`). The first segment determines
+ * which backend handles the tool loop.
+ */
+function resolveBackend(modelId) {
+    const prefix = modelId.split(":")[0];
+    const backend = BACKENDS[prefix];
+    if (!backend) {
+        const supported = Object.keys(BACKENDS).join(", ");
+        throw new Error(`No backend for model "${modelId}". Supported prefixes: ${supported}`);
+    }
+    // Extract the model name for the API (e.g., "claude-opus-4-6" from "anthropic:messages:claude-opus-4-6")
+    const parts = modelId.split(":");
+    const modelName = parts.length > 2 ? parts.slice(2).join(":") : parts[parts.length - 1];
+    return { backend, modelName };
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/** Append a machine-readable tool call summary for assertion detection */
+function appendToolSummary(text, log) {
+    if (log.length === 0)
+        return text;
+    const names = JSON.stringify(log.map((tc) => tc.name));
+    return `${text}\n\n<!-- MCP_TOOLS_CALLED: ${names} -->`;
+}
+/** Resolve the API key for a given model prefix */
+function resolveApiKey(prefix, config) {
+    if (config.apiKey)
+        return String(config.apiKey);
+    const envMap = {
+        anthropic: "ANTHROPIC_API_KEY",
+        openai: "OPENAI_API_KEY",
+    };
+    const envVar = envMap[prefix];
+    return envVar ? process.env[envVar] : undefined;
+}
+// ---------------------------------------------------------------------------
+// Provider class
+// ---------------------------------------------------------------------------
+export default class MCPToolProvider {
+    config;
+    providerId;
+    constructor(options = {}) {
+        this.config = options.config || {};
+        this.providerId = options.id || "mcp-tool-provider";
+    }
+    id() {
+        return this.providerId;
+    }
+    async callApi(prompt, _context) {
+        const mcpServerConfig = this.config.mcpServer;
+        if (!mcpServerConfig) {
+            return { error: "mcpServer config is required", output: undefined };
+        }
+        // Resolve model and backend
+        const modelId = this.config.model || "anthropic:messages:claude-opus-4-6";
+        let backend;
+        let modelName;
+        try {
+            const resolved = resolveBackend(modelId);
+            backend = resolved.backend;
+            modelName = resolved.modelName;
+        }
+        catch (err) {
+            return {
+                error: err instanceof Error ? err.message : String(err),
+                output: undefined,
+            };
+        }
+        // Resolve API key
+        const prefix = modelId.split(":")[0];
+        const apiKey = resolveApiKey(prefix, this.config);
+        if (!apiKey) {
+            return {
+                error: `API key not found for ${prefix}. Set ${prefix.toUpperCase()}_API_KEY in env or config.apiKey.`,
+                output: undefined,
+            };
+        }
+        // Connect to MCP server
+        let mcpClient;
+        try {
+            mcpClient = await connectMCP(mcpServerConfig);
+        }
+        catch (err) {
+            return {
+                error: `Failed to connect to MCP server: ${err instanceof Error ? err.message : String(err)}`,
+                output: undefined,
+            };
+        }
+        try {
+            // Filter tools by capabilities
+            const allTools = mcpClient.getAllTools();
+            const toolFilter = this.config.mcpTools;
+            const tools = toolFilter
+                ? allTools.filter((t) => toolFilter.includes(t.name))
+                : allTools;
+            if (tools.length === 0) {
+                return {
+                    error: "No MCP tools available after filtering. Check mcpTools config and server capabilities.",
+                    output: undefined,
+                };
+            }
+            // Run the tool loop
+            const result = await backend({
+                prompt,
+                tools,
+                callTool: mcpClient.callTool,
+                maxToolRounds: this.config.maxToolRounds || 5,
+                model: modelName,
+                temperature: this.config.temperature ?? 0.2,
+                maxTokens: this.config.max_tokens || 4096,
+                apiKey,
+            });
+            return {
+                cost: 0,
+                metadata: {
+                    toolRounds: result.toolRounds,
+                    toolCallLog: result.toolCallLog,
+                    exhaustedRounds: result.exhaustedRounds,
+                    latencyMs: result.latencyMs,
+                },
+                output: appendToolSummary(result.output, result.toolCallLog),
+                tokenUsage: result.tokenUsage,
+            };
+        }
+        finally {
+            await mcpClient.cleanup().catch(() => { });
+        }
+    }
+}