@sanity/ailf 2.0.0 → 2.0.1

package/dist/pipeline/compiler/mode-handlers/mcp-server-handler.js

@@ -0,0 +1,307 @@
+/**
+ * MCPServerModeHandler — compilation rules for `mcp-server` evaluation mode.
+ *
+ * This is the first non-literacy mode handler, proving the compiler
+ * architecture works end-to-end. It translates MCP server task definitions
+ * into Promptfoo configuration with:
+ *
+ * - An MCP provider that wraps the server under test
+ * - Tool-call assertions compiled to Promptfoo `javascript` assertions
+ * - Server lifecycle management via Promptfoo provider hooks
+ * - Multi-turn conversation support via Promptfoo's `steps` syntax
+ *
+ * Promptfoo supports MCP servers as providers natively:
+ * ```yaml
+ * providers:
+ *   - id: mcp:./my-server
+ *     config:
+ *       command: node
+ *       args: [./dist/server.js]
+ *       env: { API_KEY: "..." }
+ * ```
+ *
+ * This handler assembles that config from AILF's `MCPServerTaskDefinition`.
+ *
+ * @see docs/exec-plans/architecture-overhaul/phase-3-mcp-server-mode.md
+ * @see packages/core/src/types/eval-mode-config.ts — MCPServerModeConfig
+ * @see packages/core/src/types/generalized-task.ts — MCPServerTaskDefinition
+ */
+import { buildMCPAssertions, } from "./mcp-assertions.js";
+// ---------------------------------------------------------------------------
+// Canonical MCP server prompt templates
+// ---------------------------------------------------------------------------
+// Handler-owned prompts for MCP server evaluations. Instructs the model to
+// interact with MCP tools rather than writing standalone code.
+export const MCP_PROMPT_TEMPLATES = {
+    "mcp-server": {
+        id: "mcp-server",
+        label: "MCP Server Tool Use",
+        template: `You are an AI assistant with access to an MCP (Model Context Protocol) server that provides tools for interacting with a Sanity content backend.
+
+## Task
+{{task}}
+
+## Instructions
+
+1. Use the available MCP tools to complete the task
+2. Call tools with the correct parameters as described in their schemas
+3. Interpret tool responses and use the results to accomplish the goal
+4. If a tool returns an error, explain the issue clearly
+5. Prefer using specific tools over broad queries when possible
+
+Complete the task using the MCP tools provided:
+`,
+        variables: ["task"],
+    },
+};
+/**
+ * Validate that an MCP task definition has all required fields.
+ */
+export function validateMCPTask(task) {
+    const errors = [];
+    if (!task.id) {
+        errors.push({ field: "id", message: "Task ID is required" });
+    }
+    if (!task.title) {
+        errors.push({ field: "title", message: "Task title is required" });
+    }
+    if (task.serverConfig) {
+        const { transport, command, url } = task.serverConfig;
+        if (transport === "stdio" && !command) {
+            errors.push({
+                field: "serverConfig.command",
+                message: "Server command is required for stdio transport (e.g., 'node dist/server.js')",
+            });
+        }
+        if ((transport === "sse" || transport === "streamable-http") && !url) {
+            errors.push({
+                field: "serverConfig.url",
+                message: `Server URL is required for ${transport} transport`,
+            });
+        }
+    }
+    // Assertions should reference MCP-compatible types
+    if (task.assertions) {
+        for (const assertion of task.assertions) {
+            if (assertion.type === "tool-called" &&
+                !("value" in assertion && assertion.value)) {
+                errors.push({
+                    field: "assertions",
+                    message: 'tool-called assertion requires a "value" specifying the tool name',
+                });
+            }
+        }
+    }
+    return errors;
+}
+// ---------------------------------------------------------------------------
+// Compilation
+// ---------------------------------------------------------------------------
+/**
+ * Compile an MCP server task definition into Promptfoo configuration.
+ *
+ * This is the core of the MCP mode handler. It produces:
+ * 1. A provider config pointing to the MCP server
+ * 2. Test cases with tool-call assertions
+ * 3. Appropriate prompts for the evaluation
+ */
+export function compileMCPTask(task, options) {
+    const warnings = [];
+    // Validate
+    const validationErrors = validateMCPTask(task);
+    if (validationErrors.length > 0) {
+        for (const err of validationErrors) {
+            warnings.push(`MCP task "${task.id}": ${err.field} — ${err.message}`);
+        }
+    }
+    // Build providers (one LLM provider per model, each with MCP config)
+    const providers = buildMCPProvider(task, options?.models ?? [], warnings);
+    // Build prompts
+    const prompts = buildMCPPrompts(task);
+    // Build test cases
+    const tests = buildMCPTestCases(task, options, warnings);
+    return { providers, tests, prompts, warnings };
+}
+// ---------------------------------------------------------------------------
+// Provider assembly
+// ---------------------------------------------------------------------------
+/** Default max tool rounds for MCP multi-turn execution */
+const DEFAULT_MAX_TOOL_ROUNDS = 5;
+/** Provider path relative to eval package dist */
+const MCP_PROVIDER_PATH = "file://dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js";
+/**
+ * Build custom MCP tool provider configs — one per model.
+ *
+ * Each provider uses the custom mcp-tool-provider.ts which implements a
+ * multi-turn tool execution loop. The LLM receives a prompt, discovers
+ * MCP tools, calls them, gets results, and continues until it produces
+ * a final text answer or exhausts maxToolRounds.
+ *
+ * Config shape passed to the custom provider:
+ *   { model, mcpServer: { url, auth, name }, mcpTools, maxToolRounds, temperature, ... }
+ */
+function buildMCPProvider(task, models, warnings) {
+    // Build the MCP server config
+    const mcpServer = buildMCPServerConfig(task, warnings);
+    const mcpTools = task.capabilities ?? undefined;
+    const maxToolRounds = task.maxToolRounds ?? DEFAULT_MAX_TOOL_ROUNDS;
+    // Helper to build a provider entry for a given model
+    function makeProvider(modelId, label, modelConfig) {
+        return {
+            id: MCP_PROVIDER_PATH,
+            label: `${label} + MCP`,
+            config: {
+                model: modelId,
+                mcpServer,
+                ...(mcpTools ? { mcpTools } : {}),
+                maxToolRounds,
+                ...(modelConfig ?? {}),
+            },
+        };
+    }
+    // Task-level model override takes precedence over registry models
+    const taskModels = task.models;
+    if (taskModels && taskModels.length > 0) {
+        return taskModels.map((modelId) => makeProvider(modelId, modelId));
+    }
+    // Use registry models (already filtered to mcp-server mode)
+    if (models.length === 0) {
+        warnings.push(`MCP task "${task.id}": no models available. Add "mcp-server" to a ` +
+            "model's modes array in config/models.ts, or set models on the task.");
+        return [
+            makeProvider("anthropic:messages:claude-sonnet-4-20250514", "Claude Sonnet 4"),
+        ];
+    }
+    return models.map((model) => makeProvider(model.id, model.label, model.config));
+}
+/**
+ * Build the MCP server connection config for the custom provider.
+ *
+ * Shape: { url?, command?, name?, auth? }
+ * The custom mcp-tool-provider.ts uses this to connect to the MCP server.
+ */
+function buildMCPServerConfig(task, warnings) {
+    const config = task.serverConfig;
+    if (!config) {
+        warnings.push(`MCP task "${task.id}": no serverConfig — using placeholder. ` +
+            "Set serverConfig.command or serverConfig.url to point to your MCP server.");
+        return { name: task.id };
+    }
+    const serverConfig = { name: task.id };
+    if (config.transport === "stdio") {
+        serverConfig.command = config.command;
+    }
+    else {
+        serverConfig.url = config.url;
+    }
+    // Auth config
+    if (config.auth) {
+        serverConfig.auth = config.auth;
+    }
+    else if (config.env) {
+        const tokenKey = Object.keys(config.env).find((k) => /token|auth|key/i.test(k));
+        if (tokenKey) {
+            const val = config.env[tokenKey];
+            let envVar = val;
+            if (val.startsWith("$env(") && val.endsWith(")")) {
+                envVar = val.slice(5, -1);
+            }
+            if (!envVar || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(envVar)) {
+                warnings.push(`MCP task: env var name "${envVar}" from "${val}" is not a valid ` +
+                    "identifier — skipping auth config");
+            }
+            else {
+                serverConfig.auth = {
+                    type: "bearer",
+                    token: `{{env.${envVar}}}`,
+                };
+            }
+        }
+    }
+    return serverConfig;
+}
+// ---------------------------------------------------------------------------
+// Prompt assembly
+// ---------------------------------------------------------------------------
+function buildMCPPrompts(task) {
+    // MCP mode uses a single prompt — the task description
+    const promptText = task.prompt?.text ??
+        task.prompt?.vars?.task ??
+        task.description ??
+        `Test MCP server: ${task.title}`;
+    return [
+        {
+            id: "mcp-test",
+            label: `MCP: ${task.title}`,
+            raw: String(promptText),
+        },
+    ];
+}
+// ---------------------------------------------------------------------------
+// Test case assembly
+// ---------------------------------------------------------------------------
+function buildMCPTestCases(task, options, warnings) {
+    const tests = [];
+    // Build assertion context
+    const assertionContext = {
+        capabilities: task.capabilities ?? [],
+        graderProvider: options?.graderProvider,
+        taskId: task.id,
+    };
+    // Compile assertions
+    // Cast GeneralizedAssertionDefinition[] → AssertionInput[] (structurally compatible)
+    const assertions = [];
+    if (task.assertions) {
+        const rawAssertions = task.assertions;
+        const { assertions: mapped, warnings: assertionWarnings } = buildMCPAssertions(rawAssertions, assertionContext);
+        assertions.push(...mapped);
+        warnings.push(...assertionWarnings);
+    }
+    // Build test case vars
+    const vars = {
+        task: task.prompt?.vars?.task ?? task.description ?? `Test: ${task.title}`,
+        ...(task.prompt?.vars ?? {}),
+    };
+    // Primary test case
+    tests.push({
+        description: `${task.id} — ${task.title}`,
+        vars,
+        ...(assertions.length > 0 ? { assert: assertions } : {}),
+    });
+    // Multi-turn test cases
+    if (task.multiTurn?.turns && task.multiTurn.turns.length > 0) {
+        tests.push({
+            description: `${task.id} — ${task.title} [multi-turn]`,
+            vars: {
+                ...vars,
+                __multiTurn: task.multiTurn.turns,
+            },
+            ...(assertions.length > 0 ? { assert: assertions } : {}),
+        });
+    }
+    return tests;
+}
+// ---------------------------------------------------------------------------
+// ModeHandler adapter
+// ---------------------------------------------------------------------------
+/** ModeHandler-conformant export for the mcp-server evaluation mode. */
+export const handler = {
+    getPrompts() {
+        return MCP_PROMPT_TEMPLATES;
+    },
+    compileTask(task, ctx) {
+        if (!("mode" in task) || task.mode !== "mcp-server") {
+            throw new Error(`MCP server handler received task with mode "${task.mode ?? "undefined"}" — expected "mcp-server"`);
+        }
+        const result = compileMCPTask(task, {
+            graderProvider: ctx.graderProvider,
+            models: ctx.models,
+        });
+        return {
+            providers: result.providers,
+            tests: result.tests,
+            prompts: result.prompts,
+            warnings: result.warnings,
+        };
+    },
+};

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js

@@ -43,19 +43,29 @@ const BACKENDS = {
  *
  * Model IDs follow the pattern `provider:type:model-name` (e.g.,
  * `anthropic:messages:claude-opus-4-6`). The first segment determines
- * which backend handles the tool loop.
+ * which backend handles the tool loop. For OpenAI, the second segment
+ * determines the API variant (`chat` → Chat Completions, `responses` →
+ * Responses API).
  */
 function resolveBackend(modelId) {
-    const prefix = modelId.split(":")[0];
+    const parts = modelId.split(":");
+    const prefix = parts[0];
     const backend = BACKENDS[prefix];
     if (!backend) {
         const supported = Object.keys(BACKENDS).join(", ");
         throw new Error(`No backend for model "${modelId}". Supported prefixes: ${supported}`);
     }
     // Extract the model name for the API (e.g., "claude-opus-4-6" from "anthropic:messages:claude-opus-4-6")
-    const parts = modelId.split(":");
     const modelName = parts.length > 2 ? parts.slice(2).join(":") : parts[parts.length - 1];
-    return { backend, modelName };
+    // For OpenAI, extract the API variant from the second segment
+    let apiVariant;
+    if (prefix === "openai" && parts.length > 2) {
+        const variant = parts[1];
+        if (variant === "responses" || variant === "chat") {
+            apiVariant = variant;
+        }
+    }
+    return { backend, modelName, apiVariant };
 }
 // ---------------------------------------------------------------------------
 // Helpers
@@ -100,10 +110,12 @@ export default class MCPToolProvider {
         const modelId = this.config.model || "anthropic:messages:claude-opus-4-6";
         let backend;
         let modelName;
+        let apiVariant;
         try {
             const resolved = resolveBackend(modelId);
             backend = resolved.backend;
             modelName = resolved.modelName;
+            apiVariant = resolved.apiVariant;
         }
         catch (err) {
             return {
@@ -152,8 +164,13 @@ export default class MCPToolProvider {
                 maxToolRounds: this.config.maxToolRounds || 5,
                 model: modelName,
                 temperature: this.config.temperature ?? 0.2,
-                maxTokens: this.config.max_tokens || 4096,
+                maxTokens: this.config.max_output_tokens ||
+                    this.config.max_completion_tokens ||
+                    this.config.max_tokens ||
+                    4096,
                 apiKey,
+                apiVariant,
+                providerConfig: this.config,
             });
             return {
                 cost: 0,

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/mcp-connection.js

@@ -33,10 +33,16 @@ export async function connectMCP(serverConfig) {
     else if (resolvedConfig.url) {
         const { StreamableHTTPClientTransport } = await import("@modelcontextprotocol/sdk/client/streamableHttp.js");
         const headers = {};
+        // Auth-derived headers (structured auth config)
         const auth = resolvedConfig.auth;
         if (auth?.type === "bearer" && auth.token) {
             headers["Authorization"] = `Bearer ${auth.token}`;
         }
+        // Explicit headers override auth-derived ones
+        const customHeaders = resolvedConfig.headers;
+        if (customHeaders) {
+            Object.assign(headers, customHeaders);
+        }
         const transport = new StreamableHTTPClientTransport(new URL(String(resolvedConfig.url)), { requestInit: { headers } });
         await client.connect(transport);
         closeTransport = () => transport.close();

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/tool-loop-openai.d.ts

@@ -1,14 +1,19 @@
 /**
  * OpenAI multi-turn tool execution loop.
  *
- * Placeholder for future implementation. Will use the OpenAI Chat
- * Completions or Responses API with function calling to execute MCP
- * tools in a multi-turn loop.
+ * Supports two OpenAI API surfaces:
+ * - **Chat Completions** (`/v1/chat/completions`) — used by `openai:chat:*` models
+ * - **Responses** (`/v1/responses`) — used by `openai:responses:*` models (GPT-5.x)
+ *
+ * Both follow the same loop pattern: send prompt → model calls tools → execute
+ * via MCP → feed results back → repeat until final text or maxToolRounds.
  */
 import type { ToolLoopConfig, ToolLoopResult } from "./types.js";
 /**
  * Run a multi-turn tool loop using the OpenAI API.
  *
- * @throws {Error} Not yet implemented
+ * Routes to Chat Completions or Responses API based on `config.apiVariant`:
+ * - `"responses"` → Responses API (`/v1/responses`)
+ * - `"chat"` or undefined → Chat Completions API (`/v1/chat/completions`)
  */
-export declare function runOpenAIToolLoop(_config: ToolLoopConfig): Promise<ToolLoopResult>;
+export declare function runOpenAIToolLoop(config: ToolLoopConfig): Promise<ToolLoopResult>;