@sanity/ailf 1.0.0 → 2.0.0

package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.js

@@ -1,309 +0,0 @@
-/**
- * MCPServerModeHandler — compilation rules for `mcp-server` evaluation mode.
- *
- * This is the first non-literacy mode handler, proving the compiler
- * architecture works end-to-end. It translates MCP server task definitions
- * into Promptfoo configuration with:
- *
- * - An MCP provider that wraps the server under test
- * - Tool-call assertions compiled to Promptfoo `javascript` assertions
- * - Server lifecycle management via Promptfoo provider hooks
- * - Multi-turn conversation support via Promptfoo's `steps` syntax
- *
- * Promptfoo supports MCP servers as providers natively:
- * ```yaml
- * providers:
- *   - id: mcp:./my-server
- *     config:
- *       command: node
- *       args: [./dist/server.js]
- *       env: { API_KEY: "..." }
- * ```
- *
- * This handler assembles that config from AILF's `MCPServerTaskDefinition`.
- *
- * @see docs/exec-plans/architecture-overhaul/phase-3-mcp-server-mode.md
- * @see packages/core/src/types/eval-mode-config.ts — MCPServerModeConfig
- * @see packages/core/src/types/generalized-task.ts — MCPServerTaskDefinition
- */
-import { buildMCPAssertions, } from "./mcp-assertions.js";
-// ---------------------------------------------------------------------------
-// Canonical MCP server prompt templates
-// ---------------------------------------------------------------------------
-// Handler-owned prompts for MCP server evaluations. Instructs the model to
-// interact with MCP tools rather than writing standalone code.
-export const MCP_PROMPT_TEMPLATES = {
-    "mcp-server": {
-        id: "mcp-server",
-        label: "MCP Server Tool Use",
-        template: `You are an AI assistant with access to an MCP (Model Context Protocol) server that provides tools for interacting with a Sanity content backend.
-
-## Task
-{{task}}
-
-## Instructions
-
-1. Use the available MCP tools to complete the task
-2. Call tools with the correct parameters as described in their schemas
-3. Interpret tool responses and use the results to accomplish the goal
-4. If a tool returns an error, explain the issue clearly
-5. Prefer using specific tools over broad queries when possible
-
-Complete the task using the MCP tools provided:
-`,
-        variables: ["task"],
-    },
-};
-/**
- * Validate that an MCP task definition has all required fields.
- */
-export function validateMCPTask(task) {
-    const errors = [];
-    if (!task.id) {
-        errors.push({ field: "id", message: "Task ID is required" });
-    }
-    if (!task.title) {
-        errors.push({ field: "title", message: "Task title is required" });
-    }
-    if (task.serverConfig) {
-        const { transport, command, url } = task.serverConfig;
-        if (transport === "stdio" && !command) {
-            errors.push({
-                field: "serverConfig.command",
-                message: "Server command is required for stdio transport (e.g., 'node dist/server.js')",
-            });
-        }
-        if ((transport === "sse" || transport === "streamable-http") && !url) {
-            errors.push({
-                field: "serverConfig.url",
-                message: `Server URL is required for ${transport} transport`,
-            });
-        }
-    }
-    // Assertions should reference MCP-compatible types
-    if (task.assertions) {
-        for (const assertion of task.assertions) {
-            if (assertion.type === "tool-called" &&
-                !("value" in assertion && assertion.value)) {
-                errors.push({
-                    field: "assertions",
-                    message: 'tool-called assertion requires a "value" specifying the tool name',
-                });
-            }
-        }
-    }
-    return errors;
-}
-// ---------------------------------------------------------------------------
-// Compilation
-// ---------------------------------------------------------------------------
-/**
- * Compile an MCP server task definition into Promptfoo configuration.
- *
- * This is the core of the MCP mode handler. It produces:
- * 1. A provider config pointing to the MCP server
- * 2. Test cases with tool-call assertions
- * 3. Appropriate prompts for the evaluation
- */
-export function compileMCPTask(task, options) {
-    const warnings = [];
-    // Validate
-    const validationErrors = validateMCPTask(task);
-    if (validationErrors.length > 0) {
-        for (const err of validationErrors) {
-            warnings.push(`MCP task "${task.id}": ${err.field} — ${err.message}`);
-        }
-    }
-    // Build provider
-    const providers = buildMCPProvider(task, warnings);
-    // Build prompts
-    const prompts = buildMCPPrompts(task);
-    // Build test cases
-    const tests = buildMCPTestCases(task, options, warnings);
-    return { providers, tests, prompts, warnings };
-}
-// ---------------------------------------------------------------------------
-// Provider assembly
-// ---------------------------------------------------------------------------
-/**
- * Build a Promptfoo-native MCP provider config.
- *
- * Promptfoo supports MCP servers natively via `id: "mcp"` with a
- * structured config. See: https://www.promptfoo.dev/docs/providers/mcp/
- *
- * Key config shape:
- *   { enabled: true, server: { url?, command?, args?, name?, auth?, headers? },
- *     tools?, exclude_tools?, timeout?, debug? }
- */
-function buildMCPProvider(task, warnings) {
-    const config = task.serverConfig;
-    if (!config) {
-        warnings.push(`MCP task "${task.id}": no serverConfig — using placeholder provider. ` +
-            "Set serverConfig.command or serverConfig.url to point to your MCP server.");
-        return [
-            {
-                id: "mcp",
-                label: `MCP Server: ${task.title}`,
-                config: { enabled: true, server: { name: task.id } },
-            },
-        ];
-    }
-    // Build the server sub-config (Promptfoo's native format)
-    const server = { name: task.id };
-    if (config.transport === "stdio") {
-        // Promptfoo expects command + args as separate fields
-        const parts = config.command?.split(/\s+/) ?? [];
-        server.command = parts[0] ?? "node";
-        if (parts.length > 1) {
-            server.args = parts.slice(1);
-        }
-    }
-    else {
-        // sse or streamable-http — use URL-based connection
-        server.url = config.url;
-    }
-    // Auth config (Promptfoo supports bearer, basic, api_key, oauth)
-    if (config.auth) {
-        server.auth = config.auth;
-    }
-    else if (config.env) {
-        // Backward compat: if env has a token-like variable, convert to
-        // bearer auth using Promptfoo's {{env.VAR}} template syntax
-        const tokenKey = Object.keys(config.env).find((k) => /token|auth|key/i.test(k));
-        if (tokenKey) {
-            const val = config.env[tokenKey];
-            // Convert $env(VAR) syntax to Promptfoo's {{env.VAR}} syntax
-            let envVar = val;
-            if (val.startsWith("$env(") && val.endsWith(")")) {
-                envVar = val.slice(5, -1); // $env(VAR_NAME) → VAR_NAME
-            }
-            // Validate extracted env var name is non-empty and valid
-            if (!envVar || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(envVar)) {
-                warnings.push(`MCP task: env var name "${envVar}" from "${val}" is not a valid ` +
-                    "identifier — skipping auth config");
-            }
-            else {
-                server.auth = {
-                    type: "bearer",
-                    token: `{{env.${envVar}}}`,
-                };
-            }
-        }
-    }
-    // Custom headers (if any non-auth env vars remain)
-    if (config.env) {
-        const headers = {};
-        for (const [key, val] of Object.entries(config.env)) {
-            if (/header[_.]?/i.test(key)) {
-                headers[key.replace(/^header[_.]?/i, "")] = val;
-            }
-        }
-        if (Object.keys(headers).length > 0) {
-            server.headers = headers;
-        }
-    }
-    // Build top-level provider config
-    const providerConfig = {
-        enabled: true,
-        server,
-    };
-    // Tool filtering — map AILF capabilities to Promptfoo tools
-    if (task.capabilities && task.capabilities.length > 0) {
-        providerConfig.tools = task.capabilities;
-    }
-    // Timeout
-    if (config.startupTimeoutMs) {
-        providerConfig.timeout = config.startupTimeoutMs;
-    }
-    return [
-        {
-            id: "mcp",
-            label: `MCP Server: ${task.title}`,
-            config: providerConfig,
-        },
-    ];
-}
-// ---------------------------------------------------------------------------
-// Prompt assembly
-// ---------------------------------------------------------------------------
-function buildMCPPrompts(task) {
-    // MCP mode uses a single prompt — the task description
-    const promptText = task.prompt?.text ??
-        task.prompt?.vars?.task ??
-        task.description ??
-        `Test MCP server: ${task.title}`;
-    return [
-        {
-            id: "mcp-test",
-            label: `MCP: ${task.title}`,
-            raw: String(promptText),
-        },
-    ];
-}
-// ---------------------------------------------------------------------------
-// Test case assembly
-// ---------------------------------------------------------------------------
-function buildMCPTestCases(task, options, warnings) {
-    const tests = [];
-    // Build assertion context
-    const assertionContext = {
-        capabilities: task.capabilities ?? [],
-        graderProvider: options?.graderProvider,
-        taskId: task.id,
-    };
-    // Compile assertions
-    // Cast GeneralizedAssertionDefinition[] → AssertionInput[] (structurally compatible)
-    const assertions = [];
-    if (task.assertions) {
-        const rawAssertions = task.assertions;
-        const { assertions: mapped, warnings: assertionWarnings } = buildMCPAssertions(rawAssertions, assertionContext);
-        assertions.push(...mapped);
-        warnings.push(...assertionWarnings);
-    }
-    // Build test case vars
-    const vars = {
-        task: task.prompt?.vars?.task ?? task.description ?? `Test: ${task.title}`,
-        ...(task.prompt?.vars ?? {}),
-    };
-    // Primary test case
-    tests.push({
-        description: `${task.id} — ${task.title}`,
-        vars,
-        ...(assertions.length > 0 ? { assert: assertions } : {}),
-    });
-    // Multi-turn test cases
-    if (task.multiTurn?.turns && task.multiTurn.turns.length > 0) {
-        tests.push({
-            description: `${task.id} — ${task.title} [multi-turn]`,
-            vars: {
-                ...vars,
-                __multiTurn: task.multiTurn.turns,
-            },
-            ...(assertions.length > 0 ? { assert: assertions } : {}),
-        });
-    }
-    return tests;
-}
-// ---------------------------------------------------------------------------
-// ModeHandler adapter
-// ---------------------------------------------------------------------------
-/** ModeHandler-conformant export for the mcp-server evaluation mode. */
-export const handler = {
-    getPrompts() {
-        return MCP_PROMPT_TEMPLATES;
-    },
-    compileTask(task, ctx) {
-        if (!("mode" in task) || task.mode !== "mcp-server") {
-            throw new Error(`MCP server handler received task with mode "${task.mode ?? "undefined"}" — expected "mcp-server"`);
-        }
-        const result = compileMCPTask(task, {
-            graderProvider: ctx.graderProvider,
-        });
-        return {
-            providers: result.providers,
-            tests: result.tests,
-            prompts: result.prompts,
-            warnings: result.warnings,
-        };
-    },
-};