@sanity/ailf 2.0.1 → 2.1.0

package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.js

@@ -1,307 +0,0 @@
-/**
- * MCPServerModeHandler — compilation rules for `mcp-server` evaluation mode.
- *
- * This is the first non-literacy mode handler, proving the compiler
- * architecture works end-to-end. It translates MCP server task definitions
- * into Promptfoo configuration with:
- *
- * - An MCP provider that wraps the server under test
- * - Tool-call assertions compiled to Promptfoo `javascript` assertions
- * - Server lifecycle management via Promptfoo provider hooks
- * - Multi-turn conversation support via Promptfoo's `steps` syntax
- *
- * Promptfoo supports MCP servers as providers natively:
- * ```yaml
- * providers:
- *   - id: mcp:./my-server
- *     config:
- *       command: node
- *       args: [./dist/server.js]
- *       env: { API_KEY: "..." }
- * ```
- *
- * This handler assembles that config from AILF's `MCPServerTaskDefinition`.
- *
- * @see docs/exec-plans/architecture-overhaul/phase-3-mcp-server-mode.md
- * @see packages/core/src/types/eval-mode-config.ts — MCPServerModeConfig
- * @see packages/core/src/types/generalized-task.ts — MCPServerTaskDefinition
- */
-import { buildMCPAssertions, } from "./mcp-assertions.js";
-// ---------------------------------------------------------------------------
-// Canonical MCP server prompt templates
-// ---------------------------------------------------------------------------
-// Handler-owned prompts for MCP server evaluations. Instructs the model to
-// interact with MCP tools rather than writing standalone code.
-export const MCP_PROMPT_TEMPLATES = {
-    "mcp-server": {
-        id: "mcp-server",
-        label: "MCP Server Tool Use",
-        template: `You are an AI assistant with access to an MCP (Model Context Protocol) server that provides tools for interacting with a Sanity content backend.
-
-## Task
-{{task}}
-
-## Instructions
-
-1. Use the available MCP tools to complete the task
-2. Call tools with the correct parameters as described in their schemas
-3. Interpret tool responses and use the results to accomplish the goal
-4. If a tool returns an error, explain the issue clearly
-5. Prefer using specific tools over broad queries when possible
-
-Complete the task using the MCP tools provided:
-`,
-        variables: ["task"],
-    },
-};
-/**
- * Validate that an MCP task definition has all required fields.
- */
-export function validateMCPTask(task) {
-    const errors = [];
-    if (!task.id) {
-        errors.push({ field: "id", message: "Task ID is required" });
-    }
-    if (!task.title) {
-        errors.push({ field: "title", message: "Task title is required" });
-    }
-    if (task.serverConfig) {
-        const { transport, command, url } = task.serverConfig;
-        if (transport === "stdio" && !command) {
-            errors.push({
-                field: "serverConfig.command",
-                message: "Server command is required for stdio transport (e.g., 'node dist/server.js')",
-            });
-        }
-        if ((transport === "sse" || transport === "streamable-http") && !url) {
-            errors.push({
-                field: "serverConfig.url",
-                message: `Server URL is required for ${transport} transport`,
-            });
-        }
-    }
-    // Assertions should reference MCP-compatible types
-    if (task.assertions) {
-        for (const assertion of task.assertions) {
-            if (assertion.type === "tool-called" &&
-                !("value" in assertion && assertion.value)) {
-                errors.push({
-                    field: "assertions",
-                    message: 'tool-called assertion requires a "value" specifying the tool name',
-                });
-            }
-        }
-    }
-    return errors;
-}
-// ---------------------------------------------------------------------------
-// Compilation
-// ---------------------------------------------------------------------------
-/**
- * Compile an MCP server task definition into Promptfoo configuration.
- *
- * This is the core of the MCP mode handler. It produces:
- * 1. A provider config pointing to the MCP server
- * 2. Test cases with tool-call assertions
- * 3. Appropriate prompts for the evaluation
- */
-export function compileMCPTask(task, options) {
-    const warnings = [];
-    // Validate
-    const validationErrors = validateMCPTask(task);
-    if (validationErrors.length > 0) {
-        for (const err of validationErrors) {
-            warnings.push(`MCP task "${task.id}": ${err.field} — ${err.message}`);
-        }
-    }
-    // Build providers (one LLM provider per model, each with MCP config)
-    const providers = buildMCPProvider(task, options?.models ?? [], warnings);
-    // Build prompts
-    const prompts = buildMCPPrompts(task);
-    // Build test cases
-    const tests = buildMCPTestCases(task, options, warnings);
-    return { providers, tests, prompts, warnings };
-}
-// ---------------------------------------------------------------------------
-// Provider assembly
-// ---------------------------------------------------------------------------
-/** Default max tool rounds for MCP multi-turn execution */
-const DEFAULT_MAX_TOOL_ROUNDS = 5;
-/** Provider path relative to eval package dist */
-const MCP_PROVIDER_PATH = "file://dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js";
-/**
- * Build custom MCP tool provider configs — one per model.
- *
- * Each provider uses the custom mcp-tool-provider.ts which implements a
- * multi-turn tool execution loop. The LLM receives a prompt, discovers
- * MCP tools, calls them, gets results, and continues until it produces
- * a final text answer or exhausts maxToolRounds.
- *
- * Config shape passed to the custom provider:
- *   { model, mcpServer: { url, auth, name }, mcpTools, maxToolRounds, temperature, ... }
- */
-function buildMCPProvider(task, models, warnings) {
-    // Build the MCP server config
-    const mcpServer = buildMCPServerConfig(task, warnings);
-    const mcpTools = task.capabilities ?? undefined;
-    const maxToolRounds = task.maxToolRounds ?? DEFAULT_MAX_TOOL_ROUNDS;
-    // Helper to build a provider entry for a given model
-    function makeProvider(modelId, label, modelConfig) {
-        return {
-            id: MCP_PROVIDER_PATH,
-            label: `${label} + MCP`,
-            config: {
-                model: modelId,
-                mcpServer,
-                ...(mcpTools ? { mcpTools } : {}),
-                maxToolRounds,
-                ...(modelConfig ?? {}),
-            },
-        };
-    }
-    // Task-level model override takes precedence over registry models
-    const taskModels = task.models;
-    if (taskModels && taskModels.length > 0) {
-        return taskModels.map((modelId) => makeProvider(modelId, modelId));
-    }
-    // Use registry models (already filtered to mcp-server mode)
-    if (models.length === 0) {
-        warnings.push(`MCP task "${task.id}": no models available. Add "mcp-server" to a ` +
-            "model's modes array in config/models.ts, or set models on the task.");
-        return [
-            makeProvider("anthropic:messages:claude-sonnet-4-20250514", "Claude Sonnet 4"),
-        ];
-    }
-    return models.map((model) => makeProvider(model.id, model.label, model.config));
-}
-/**
- * Build the MCP server connection config for the custom provider.
- *
- * Shape: { url?, command?, name?, auth? }
- * The custom mcp-tool-provider.ts uses this to connect to the MCP server.
- */
-function buildMCPServerConfig(task, warnings) {
-    const config = task.serverConfig;
-    if (!config) {
-        warnings.push(`MCP task "${task.id}": no serverConfig — using placeholder. ` +
-            "Set serverConfig.command or serverConfig.url to point to your MCP server.");
-        return { name: task.id };
-    }
-    const serverConfig = { name: task.id };
-    if (config.transport === "stdio") {
-        serverConfig.command = config.command;
-    }
-    else {
-        serverConfig.url = config.url;
-    }
-    // Auth config
-    if (config.auth) {
-        serverConfig.auth = config.auth;
-    }
-    else if (config.env) {
-        const tokenKey = Object.keys(config.env).find((k) => /token|auth|key/i.test(k));
-        if (tokenKey) {
-            const val = config.env[tokenKey];
-            let envVar = val;
-            if (val.startsWith("$env(") && val.endsWith(")")) {
-                envVar = val.slice(5, -1);
-            }
-            if (!envVar || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(envVar)) {
-                warnings.push(`MCP task: env var name "${envVar}" from "${val}" is not a valid ` +
-                    "identifier — skipping auth config");
-            }
-            else {
-                serverConfig.auth = {
-                    type: "bearer",
-                    token: `{{env.${envVar}}}`,
-                };
-            }
-        }
-    }
-    return serverConfig;
-}
-// ---------------------------------------------------------------------------
-// Prompt assembly
-// ---------------------------------------------------------------------------
-function buildMCPPrompts(task) {
-    // MCP mode uses a single prompt — the task description
-    const promptText = task.prompt?.text ??
-        task.prompt?.vars?.task ??
-        task.description ??
-        `Test MCP server: ${task.title}`;
-    return [
-        {
-            id: "mcp-test",
-            label: `MCP: ${task.title}`,
-            raw: String(promptText),
-        },
-    ];
-}
-// ---------------------------------------------------------------------------
-// Test case assembly
-// ---------------------------------------------------------------------------
-function buildMCPTestCases(task, options, warnings) {
-    const tests = [];
-    // Build assertion context
-    const assertionContext = {
-        capabilities: task.capabilities ?? [],
-        graderProvider: options?.graderProvider,
-        taskId: task.id,
-    };
-    // Compile assertions
-    // Cast GeneralizedAssertionDefinition[] → AssertionInput[] (structurally compatible)
-    const assertions = [];
-    if (task.assertions) {
-        const rawAssertions = task.assertions;
-        const { assertions: mapped, warnings: assertionWarnings } = buildMCPAssertions(rawAssertions, assertionContext);
-        assertions.push(...mapped);
-        warnings.push(...assertionWarnings);
-    }
-    // Build test case vars
-    const vars = {
-        task: task.prompt?.vars?.task ?? task.description ?? `Test: ${task.title}`,
-        ...(task.prompt?.vars ?? {}),
-    };
-    // Primary test case
-    tests.push({
-        description: `${task.id} — ${task.title}`,
-        vars,
-        ...(assertions.length > 0 ? { assert: assertions } : {}),
-    });
-    // Multi-turn test cases
-    if (task.multiTurn?.turns && task.multiTurn.turns.length > 0) {
-        tests.push({
-            description: `${task.id} — ${task.title} [multi-turn]`,
-            vars: {
-                ...vars,
-                __multiTurn: task.multiTurn.turns,
-            },
-            ...(assertions.length > 0 ? { assert: assertions } : {}),
-        });
-    }
-    return tests;
-}
-// ---------------------------------------------------------------------------
-// ModeHandler adapter
-// ---------------------------------------------------------------------------
-/** ModeHandler-conformant export for the mcp-server evaluation mode. */
-export const handler = {
-    getPrompts() {
-        return MCP_PROMPT_TEMPLATES;
-    },
-    compileTask(task, ctx) {
-        if (!("mode" in task) || task.mode !== "mcp-server") {
-            throw new Error(`MCP server handler received task with mode "${task.mode ?? "undefined"}" — expected "mcp-server"`);
-        }
-        const result = compileMCPTask(task, {
-            graderProvider: ctx.graderProvider,
-            models: ctx.models,
-        });
-        return {
-            providers: result.providers,
-            tests: result.tests,
-            prompts: result.prompts,
-            warnings: result.warnings,
-        };
-    },
-};

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider.d.ts

@@ -1,65 +0,0 @@
-/**
- * mcp-tool-provider.ts — Custom Promptfoo provider for MCP tool-use evaluation.
- *
- * Implements a multi-turn tool execution loop: the LLM receives a prompt,
- * discovers MCP tools, calls them, gets results, and continues until it
- * produces a final text answer or exhausts maxToolRounds.
- *
- * Promptfoo's built-in Anthropic/OpenAI providers with config.mcp only do
- * single-turn tool calls. This provider fills that gap by managing the
- * full conversation loop, similar to the agentic-provider.ts pattern.
- *
- * Promptfoo config usage:
- *
- *   providers:
- *     - id: file://dist/pipeline/compiler/mode-handlers/mcp-tool-provider.js
- *       label: "Claude Opus 4.6 + MCP"
- *       config:
- *         model: anthropic:messages:claude-opus-4-6
- *         maxToolRounds: 5
- *         temperature: 0.2
- *         max_tokens: 4096
- *         mcpServer:
- *           url: https://mcp.sanity.io
- *           auth: { type: bearer, token: "{{env.SANITY_API_TOKEN}}" }
- *           name: mcp-live-query-documents
- *         mcpTools: [query_documents, get_schema]
- */
-interface CallApiContextParams {
-    prompt?: {
-        raw: string;
-        label?: string;
-    };
-    vars?: Record<string, object | string>;
-}
-interface ProviderOptions {
-    config?: Record<string, unknown>;
-    id?: string;
-}
-interface ProviderResponse {
-    cached?: boolean;
-    cost?: number;
-    error?: string;
-    metadata?: Record<string, unknown>;
-    output?: object | string;
-    tokenUsage?: {
-        total?: number;
-        prompt?: number;
-        completion?: number;
-        cached?: number;
-    };
-}
-export default class MCPToolProvider {
-    config: Record<string, unknown>;
-    private providerId;
-    constructor(options?: ProviderOptions);
-    id(): string;
-    callApi(prompt: string, _context?: CallApiContextParams): Promise<ProviderResponse>;
-    private runAnthropicLoop;
-    private connectMCP;
-    /**
-     * Resolve {{env.VAR}} templates in config values.
-     */
-    private resolveEnvTemplates;
-}
-export {};