@sanity/ailf 0.5.0 → 2.0.0

package/dist/pipeline/compiler/mode-handlers/mcp-server/provider-config.js

@@ -0,0 +1,104 @@
+/**
+ * MCP server provider assembly — builds Promptfoo provider configs.
+ */
+// ---------------------------------------------------------------------------
+// Constants
+// ---------------------------------------------------------------------------
+/** Default max tool rounds for MCP multi-turn execution */
+export const DEFAULT_MAX_TOOL_ROUNDS = 5;
+/** Provider path relative to eval package dist */
+export const MCP_PROVIDER_PATH = "file://dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js";
+// ---------------------------------------------------------------------------
+// Provider assembly
+// ---------------------------------------------------------------------------
+/**
+ * Build custom MCP tool provider configs — one per model.
+ *
+ * Each provider uses the custom mcp-tool-provider.ts which implements a
+ * multi-turn tool execution loop. The LLM receives a prompt, discovers
+ * MCP tools, calls them, gets results, and continues until it produces
+ * a final text answer or exhausts maxToolRounds.
+ *
+ * Config shape passed to the custom provider:
+ *   { model, mcpServer: { url, auth, name }, mcpTools, maxToolRounds, temperature, ... }
+ */
+export function buildMCPProvider(task, models, warnings) {
+    // Build the MCP server config
+    const mcpServer = buildMCPServerConfig(task, warnings);
+    const mcpTools = task.capabilities ?? undefined;
+    const maxToolRounds = task.maxToolRounds ?? DEFAULT_MAX_TOOL_ROUNDS;
+    // Helper to build a provider entry for a given model
+    function makeProvider(modelId, label, modelConfig) {
+        return {
+            id: MCP_PROVIDER_PATH,
+            label: `${label} + MCP`,
+            config: {
+                model: modelId,
+                mcpServer,
+                ...(mcpTools ? { mcpTools } : {}),
+                maxToolRounds,
+                ...(modelConfig ?? {}),
+            },
+        };
+    }
+    // Task-level model override takes precedence over registry models
+    const taskModels = task.models;
+    if (taskModels && taskModels.length > 0) {
+        return taskModels.map((modelId) => makeProvider(modelId, modelId));
+    }
+    // Use registry models (already filtered to mcp-server mode)
+    if (models.length === 0) {
+        warnings.push(`MCP task "${task.id}": no models available. Add "mcp-server" to a ` +
+            "model's modes array in config/models.ts, or set models on the task.");
+        return [
+            makeProvider("anthropic:messages:claude-sonnet-4-20250514", "Claude Sonnet 4"),
+        ];
+    }
+    return models.map((model) => makeProvider(model.id, model.label, model.config));
+}
+/**
+ * Build the MCP server connection config for the custom provider.
+ *
+ * Shape: { url?, command?, name?, auth? }
+ * The custom mcp-tool-provider.ts uses this to connect to the MCP server.
+ */
+export function buildMCPServerConfig(task, warnings) {
+    const config = task.serverConfig;
+    if (!config) {
+        warnings.push(`MCP task "${task.id}": no serverConfig — using placeholder. ` +
+            "Set serverConfig.command or serverConfig.url to point to your MCP server.");
+        return { name: task.id };
+    }
+    const serverConfig = { name: task.id };
+    if (config.transport === "stdio") {
+        serverConfig.command = config.command;
+    }
+    else {
+        serverConfig.url = config.url;
+    }
+    // Auth config
+    if (config.auth) {
+        serverConfig.auth = config.auth;
+    }
+    else if (config.env) {
+        const tokenKey = Object.keys(config.env).find((k) => /token|auth|key/i.test(k));
+        if (tokenKey) {
+            const val = config.env[tokenKey];
+            let envVar = val;
+            if (val.startsWith("$env(") && val.endsWith(")")) {
+                envVar = val.slice(5, -1);
+            }
+            if (!envVar || !/^[A-Za-z_][A-Za-z0-9_]*$/.test(envVar)) {
+                warnings.push(`MCP task: env var name "${envVar}" from "${val}" is not a valid ` +
+                    "identifier — skipping auth config");
+            }
+            else {
+                serverConfig.auth = {
+                    type: "bearer",
+                    token: `{{env.${envVar}}}`,
+                };
+            }
+        }
+    }
+    return serverConfig;
+}

package/dist/pipeline/compiler/mode-handlers/mcp-server/types.d.ts

@@ -0,0 +1,37 @@
+/**
+ * Shared types for the MCP server mode handler.
+ */
+import type { ModeProviderEntry } from "../../../../_vendor/ailf-core/index.d.ts";
+import type { PromptfooPrompt, PromptfooProvider, PromptfooTestCase } from "../../promptfoo-compiler.js";
+/** Options for compiling an MCP server task */
+export interface MCPCompileOptions {
+    /** Grader provider for LLM-graded assertions */
+    graderProvider?: string;
+    /** Model providers to evaluate with (from registry, filtered by mcp-server mode) */
+    models?: ModeProviderEntry[];
+}
+/** Result of compiling a single MCP task */
+export interface MCPCompileResult {
+    /** Promptfoo provider config for the MCP server */
+    providers: PromptfooProvider[];
+    /** Compiled test cases */
+    tests: PromptfooTestCase[];
+    /** Prompts for MCP evaluation */
+    prompts: PromptfooPrompt[];
+    /** Warnings generated during compilation */
+    warnings: string[];
+}
+/** Validation errors for MCP task definitions */
+export interface MCPValidationError {
+    field: string;
+    message: string;
+}
+/** Context for building MCP assertions */
+export interface MCPAssertionContext {
+    /** Task ID (for error messages) */
+    taskId: string;
+    /** Expected server capabilities */
+    capabilities: string[];
+    /** Grader provider for LLM-graded assertions */
+    graderProvider?: string;
+}

package/dist/pipeline/compiler/mode-handlers/mcp-server/types.js

@@ -0,0 +1,4 @@
+/**
+ * Shared types for the MCP server mode handler.
+ */
+export {};

package/dist/pipeline/compiler/mode-handlers/mcp-server/validation.d.ts

@@ -0,0 +1,9 @@
+/**
+ * Validation for MCP server task definitions.
+ */
+import type { MCPServerTaskDefinition } from "../../../../_vendor/ailf-core/index.d.ts";
+import type { MCPValidationError } from "./types.js";
+/**
+ * Validate that an MCP task definition has all required fields.
+ */
+export declare function validateMCPTask(task: MCPServerTaskDefinition): MCPValidationError[];

package/dist/pipeline/compiler/mode-handlers/mcp-server/validation.js

@@ -0,0 +1,43 @@
+/**
+ * Validation for MCP server task definitions.
+ */
+/**
+ * Validate that an MCP task definition has all required fields.
+ */
+export function validateMCPTask(task) {
+    const errors = [];
+    if (!task.id) {
+        errors.push({ field: "id", message: "Task ID is required" });
+    }
+    if (!task.title) {
+        errors.push({ field: "title", message: "Task title is required" });
+    }
+    if (task.serverConfig) {
+        const { transport, command, url } = task.serverConfig;
+        if (transport === "stdio" && !command) {
+            errors.push({
+                field: "serverConfig.command",
+                message: "Server command is required for stdio transport (e.g., 'node dist/server.js')",
+            });
+        }
+        if ((transport === "sse" || transport === "streamable-http") && !url) {
+            errors.push({
+                field: "serverConfig.url",
+                message: `Server URL is required for ${transport} transport`,
+            });
+        }
+    }
+    // Assertions should reference MCP-compatible types
+    if (task.assertions) {
+        for (const assertion of task.assertions) {
+            if (assertion.type === "tool-called" &&
+                !("value" in assertion && assertion.value)) {
+                errors.push({
+                    field: "assertions",
+                    message: 'tool-called assertion requires a "value" specifying the tool name',
+                });
+            }
+        }
+    }
+    return errors;
+}

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.d.ts

@@ -0,0 +1,33 @@
+/**
+ * MCPToolProvider — Custom Promptfoo provider for MCP tool-use evaluation.
+ *
+ * Orchestrates the MCP evaluation flow:
+ * 1. Connects to the MCP server and discovers available tools
+ * 2. Selects the appropriate LLM backend based on model ID prefix
+ * 3. Delegates the multi-turn tool loop to the backend
+ * 4. Formats the result for Promptfoo (including tool call summary)
+ *
+ * Promptfoo config usage:
+ *
+ *   providers:
+ *     - id: file://dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js
+ *       label: "Claude Opus 4.6 + MCP"
+ *       config:
+ *         model: anthropic:messages:claude-opus-4-6
+ *         maxToolRounds: 5
+ *         temperature: 0.2
+ *         max_tokens: 4096
+ *         mcpServer:
+ *           url: https://mcp.sanity.io
+ *           auth: { type: bearer, token: "{{env.SANITY_API_TOKEN}}" }
+ *           name: mcp-live-query-documents
+ *         mcpTools: [query_documents, get_schema]
+ */
+import type { CallApiContextParams, ProviderOptions, ProviderResponse } from "./types.js";
+export default class MCPToolProvider {
+    config: Record<string, unknown>;
+    private providerId;
+    constructor(options?: ProviderOptions);
+    id(): string;
+    callApi(prompt: string, _context?: CallApiContextParams): Promise<ProviderResponse>;
+}

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js

@@ -0,0 +1,174 @@
+/**
+ * MCPToolProvider — Custom Promptfoo provider for MCP tool-use evaluation.
+ *
+ * Orchestrates the MCP evaluation flow:
+ * 1. Connects to the MCP server and discovers available tools
+ * 2. Selects the appropriate LLM backend based on model ID prefix
+ * 3. Delegates the multi-turn tool loop to the backend
+ * 4. Formats the result for Promptfoo (including tool call summary)
+ *
+ * Promptfoo config usage:
+ *
+ *   providers:
+ *     - id: file://dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js
+ *       label: "Claude Opus 4.6 + MCP"
+ *       config:
+ *         model: anthropic:messages:claude-opus-4-6
+ *         maxToolRounds: 5
+ *         temperature: 0.2
+ *         max_tokens: 4096
+ *         mcpServer:
+ *           url: https://mcp.sanity.io
+ *           auth: { type: bearer, token: "{{env.SANITY_API_TOKEN}}" }
+ *           name: mcp-live-query-documents
+ *         mcpTools: [query_documents, get_schema]
+ */
+import { config as loadDotenv } from "dotenv";
+import { connectMCP } from "./mcp-connection.js";
+import { runAnthropicToolLoop } from "./tool-loop-anthropic.js";
+import { runOpenAIToolLoop } from "./tool-loop-openai.js";
+loadDotenv({
+    override: true,
+    path: new URL("../../../../../.env", import.meta.url).pathname,
+});
+// ---------------------------------------------------------------------------
+// Backend registry — maps model ID prefixes to tool loop implementations
+// ---------------------------------------------------------------------------
+const BACKENDS = {
+    anthropic: runAnthropicToolLoop,
+    openai: runOpenAIToolLoop,
+};
+/**
+ * Resolve the LLM backend from a model ID.
+ *
+ * Model IDs follow the pattern `provider:type:model-name` (e.g.,
+ * `anthropic:messages:claude-opus-4-6`). The first segment determines
+ * which backend handles the tool loop.
+ */
+function resolveBackend(modelId) {
+    const prefix = modelId.split(":")[0];
+    const backend = BACKENDS[prefix];
+    if (!backend) {
+        const supported = Object.keys(BACKENDS).join(", ");
+        throw new Error(`No backend for model "${modelId}". Supported prefixes: ${supported}`);
+    }
+    // Extract the model name for the API (e.g., "claude-opus-4-6" from "anthropic:messages:claude-opus-4-6")
+    const parts = modelId.split(":");
+    const modelName = parts.length > 2 ? parts.slice(2).join(":") : parts[parts.length - 1];
+    return { backend, modelName };
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/** Append a machine-readable tool call summary for assertion detection */
+function appendToolSummary(text, log) {
+    if (log.length === 0)
+        return text;
+    const names = JSON.stringify(log.map((tc) => tc.name));
+    return `${text}\n\n<!-- MCP_TOOLS_CALLED: ${names} -->`;
+}
+/** Resolve the API key for a given model prefix */
+function resolveApiKey(prefix, config) {
+    if (config.apiKey)
+        return String(config.apiKey);
+    const envMap = {
+        anthropic: "ANTHROPIC_API_KEY",
+        openai: "OPENAI_API_KEY",
+    };
+    const envVar = envMap[prefix];
+    return envVar ? process.env[envVar] : undefined;
+}
+// ---------------------------------------------------------------------------
+// Provider class
+// ---------------------------------------------------------------------------
+export default class MCPToolProvider {
+    config;
+    providerId;
+    constructor(options = {}) {
+        this.config = options.config || {};
+        this.providerId = options.id || "mcp-tool-provider";
+    }
+    id() {
+        return this.providerId;
+    }
+    async callApi(prompt, _context) {
+        const mcpServerConfig = this.config.mcpServer;
+        if (!mcpServerConfig) {
+            return { error: "mcpServer config is required", output: undefined };
+        }
+        // Resolve model and backend
+        const modelId = this.config.model || "anthropic:messages:claude-opus-4-6";
+        let backend;
+        let modelName;
+        try {
+            const resolved = resolveBackend(modelId);
+            backend = resolved.backend;
+            modelName = resolved.modelName;
+        }
+        catch (err) {
+            return {
+                error: err instanceof Error ? err.message : String(err),
+                output: undefined,
+            };
+        }
+        // Resolve API key
+        const prefix = modelId.split(":")[0];
+        const apiKey = resolveApiKey(prefix, this.config);
+        if (!apiKey) {
+            return {
+                error: `API key not found for ${prefix}. Set ${prefix.toUpperCase()}_API_KEY in env or config.apiKey.`,
+                output: undefined,
+            };
+        }
+        // Connect to MCP server
+        let mcpClient;
+        try {
+            mcpClient = await connectMCP(mcpServerConfig);
+        }
+        catch (err) {
+            return {
+                error: `Failed to connect to MCP server: ${err instanceof Error ? err.message : String(err)}`,
+                output: undefined,
+            };
+        }
+        try {
+            // Filter tools by capabilities
+            const allTools = mcpClient.getAllTools();
+            const toolFilter = this.config.mcpTools;
+            const tools = toolFilter
+                ? allTools.filter((t) => toolFilter.includes(t.name))
+                : allTools;
+            if (tools.length === 0) {
+                return {
+                    error: "No MCP tools available after filtering. Check mcpTools config and server capabilities.",
+                    output: undefined,
+                };
+            }
+            // Run the tool loop
+            const result = await backend({
+                prompt,
+                tools,
+                callTool: mcpClient.callTool,
+                maxToolRounds: this.config.maxToolRounds || 5,
+                model: modelName,
+                temperature: this.config.temperature ?? 0.2,
+                maxTokens: this.config.max_tokens || 4096,
+                apiKey,
+            });
+            return {
+                cost: 0,
+                metadata: {
+                    toolRounds: result.toolRounds,
+                    toolCallLog: result.toolCallLog,
+                    exhaustedRounds: result.exhaustedRounds,
+                    latencyMs: result.latencyMs,
+                },
+                output: appendToolSummary(result.output, result.toolCallLog),
+                tokenUsage: result.tokenUsage,
+            };
+        }
+        finally {
+            await mcpClient.cleanup().catch(() => { });
+        }
+    }
+}

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/mcp-connection.d.ts

@@ -0,0 +1,19 @@
+/**
+ * MCP server connection and tool discovery.
+ *
+ * Handles connecting to an MCP server via streamable-http or stdio transport,
+ * discovering available tools, and resolving {{env.VAR}} templates in config.
+ */
+import type { MCPClient } from "./types.js";
+/**
+ * Connect to an MCP server and return a client for tool discovery and execution.
+ *
+ * Supports two transport types:
+ * - `url` → streamable-http (remote MCP servers like mcp.sanity.io)
+ * - `command` → stdio (local MCP server processes)
+ */
+export declare function connectMCP(serverConfig: Record<string, unknown>): Promise<MCPClient>;
+/**
+ * Resolve `{{env.VAR}}` templates in config values, recursively.
+ */
+export declare function resolveEnvTemplates(config: Record<string, unknown>): Record<string, unknown>;

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/mcp-connection.js

@@ -0,0 +1,95 @@
+/**
+ * MCP server connection and tool discovery.
+ *
+ * Handles connecting to an MCP server via streamable-http or stdio transport,
+ * discovering available tools, and resolving {{env.VAR}} templates in config.
+ */
+/**
+ * Connect to an MCP server and return a client for tool discovery and execution.
+ *
+ * Supports two transport types:
+ * - `url` → streamable-http (remote MCP servers like mcp.sanity.io)
+ * - `command` → stdio (local MCP server processes)
+ */
+export async function connectMCP(serverConfig) {
+    const { Client } = await import("@modelcontextprotocol/sdk/client/index.js");
+    const client = new Client({
+        name: "ailf-mcp-eval",
+        version: "1.0.0",
+    });
+    const resolvedConfig = resolveEnvTemplates(serverConfig);
+    let closeTransport;
+    if (resolvedConfig.command) {
+        const { StdioClientTransport } = await import("@modelcontextprotocol/sdk/client/stdio.js");
+        const parts = String(resolvedConfig.command).split(/\s+/);
+        const transport = new StdioClientTransport({
+            command: parts[0],
+            args: parts.slice(1),
+            env: process.env,
+        });
+        await client.connect(transport);
+        closeTransport = () => transport.close();
+    }
+    else if (resolvedConfig.url) {
+        const { StreamableHTTPClientTransport } = await import("@modelcontextprotocol/sdk/client/streamableHttp.js");
+        const headers = {};
+        const auth = resolvedConfig.auth;
+        if (auth?.type === "bearer" && auth.token) {
+            headers["Authorization"] = `Bearer ${auth.token}`;
+        }
+        const transport = new StreamableHTTPClientTransport(new URL(String(resolvedConfig.url)), { requestInit: { headers } });
+        await client.connect(transport);
+        closeTransport = () => transport.close();
+    }
+    else {
+        throw new Error("MCP server config must have either 'command' (stdio) or 'url' (http)");
+    }
+    // Discover tools
+    const { tools: toolsList } = await client.listTools();
+    const allTools = toolsList.map((t) => ({
+        name: t.name,
+        description: t.description,
+        inputSchema: t.inputSchema,
+    }));
+    return {
+        getAllTools: () => allTools,
+        callTool: async (name, args) => {
+            const result = await client.callTool({ name, arguments: args });
+            let content = "";
+            if (result?.content) {
+                if (Array.isArray(result.content)) {
+                    content = result.content
+                        .map((c) => c.text || JSON.stringify(c))
+                        .join("\n");
+                }
+                else {
+                    content = String(result.content);
+                }
+            }
+            return { content, error: result.isError ? content : undefined };
+        },
+        cleanup: async () => {
+            await closeTransport().catch(() => { });
+        },
+    };
+}
+/**
+ * Resolve `{{env.VAR}}` templates in config values, recursively.
+ */
+export function resolveEnvTemplates(config) {
+    const resolved = {};
+    for (const [key, value] of Object.entries(config)) {
+        if (typeof value === "string") {
+            resolved[key] = value.replace(/\{\{env\.(\w+)\}\}/g, (_, varName) => {
+                return process.env[varName] || "";
+            });
+        }
+        else if (value && typeof value === "object" && !Array.isArray(value)) {
+            resolved[key] = resolveEnvTemplates(value);
+        }
+        else {
+            resolved[key] = value;
+        }
+    }
+    return resolved;
+}

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/tool-loop-anthropic.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Anthropic multi-turn tool execution loop.
+ *
+ * Sends a prompt to the Anthropic Messages API with MCP tools attached.
+ * When the model calls a tool, executes it via the MCP client, feeds
+ * the result back, and continues until the model produces a final text
+ * response or maxToolRounds is exhausted.
+ */
+import type { ToolLoopConfig, ToolLoopResult } from "./types.js";
+/**
+ * Run a multi-turn tool loop using the Anthropic Messages API.
+ *
+ * The loop:
+ * 1. Sends the prompt with available tools to Claude
+ * 2. If Claude calls tools → executes them via MCP, sends results back
+ * 3. Repeats until Claude produces a text-only response or maxToolRounds is hit
+ * 4. On the last round, omits tools to force a synthesis response
+ */
+export declare function runAnthropicToolLoop(config: ToolLoopConfig): Promise<ToolLoopResult>;