@sanity/ailf 1.0.0 → 2.0.1

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.d.ts

@@ -0,0 +1,33 @@
+/**
+ * MCPToolProvider — Custom Promptfoo provider for MCP tool-use evaluation.
+ *
+ * Orchestrates the MCP evaluation flow:
+ * 1. Connects to the MCP server and discovers available tools
+ * 2. Selects the appropriate LLM backend based on model ID prefix
+ * 3. Delegates the multi-turn tool loop to the backend
+ * 4. Formats the result for Promptfoo (including tool call summary)
+ *
+ * Promptfoo config usage:
+ *
+ *   providers:
+ *     - id: file://dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js
+ *       label: "Claude Opus 4.6 + MCP"
+ *       config:
+ *         model: anthropic:messages:claude-opus-4-6
+ *         maxToolRounds: 5
+ *         temperature: 0.2
+ *         max_tokens: 4096
+ *         mcpServer:
+ *           url: https://mcp.sanity.io
+ *           auth: { type: bearer, token: "{{env.SANITY_API_TOKEN}}" }
+ *           name: mcp-live-query-documents
+ *         mcpTools: [query_documents, get_schema]
+ */
+import type { CallApiContextParams, ProviderOptions, ProviderResponse } from "./types.js";
+export default class MCPToolProvider {
+    config: Record<string, unknown>;
+    private providerId;
+    constructor(options?: ProviderOptions);
+    id(): string;
+    callApi(prompt: string, _context?: CallApiContextParams): Promise<ProviderResponse>;
+}

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js

@@ -0,0 +1,191 @@
+/**
+ * MCPToolProvider — Custom Promptfoo provider for MCP tool-use evaluation.
+ *
+ * Orchestrates the MCP evaluation flow:
+ * 1. Connects to the MCP server and discovers available tools
+ * 2. Selects the appropriate LLM backend based on model ID prefix
+ * 3. Delegates the multi-turn tool loop to the backend
+ * 4. Formats the result for Promptfoo (including tool call summary)
+ *
+ * Promptfoo config usage:
+ *
+ *   providers:
+ *     - id: file://dist/pipeline/compiler/mode-handlers/mcp-tool-provider/index.js
+ *       label: "Claude Opus 4.6 + MCP"
+ *       config:
+ *         model: anthropic:messages:claude-opus-4-6
+ *         maxToolRounds: 5
+ *         temperature: 0.2
+ *         max_tokens: 4096
+ *         mcpServer:
+ *           url: https://mcp.sanity.io
+ *           auth: { type: bearer, token: "{{env.SANITY_API_TOKEN}}" }
+ *           name: mcp-live-query-documents
+ *         mcpTools: [query_documents, get_schema]
+ */
+import { config as loadDotenv } from "dotenv";
+import { connectMCP } from "./mcp-connection.js";
+import { runAnthropicToolLoop } from "./tool-loop-anthropic.js";
+import { runOpenAIToolLoop } from "./tool-loop-openai.js";
+loadDotenv({
+    override: true,
+    path: new URL("../../../../../.env", import.meta.url).pathname,
+});
+// ---------------------------------------------------------------------------
+// Backend registry — maps model ID prefixes to tool loop implementations
+// ---------------------------------------------------------------------------
+const BACKENDS = {
+    anthropic: runAnthropicToolLoop,
+    openai: runOpenAIToolLoop,
+};
+/**
+ * Resolve the LLM backend from a model ID.
+ *
+ * Model IDs follow the pattern `provider:type:model-name` (e.g.,
+ * `anthropic:messages:claude-opus-4-6`). The first segment determines
+ * which backend handles the tool loop. For OpenAI, the second segment
+ * determines the API variant (`chat` → Chat Completions, `responses` →
+ * Responses API).
+ */
+function resolveBackend(modelId) {
+    const parts = modelId.split(":");
+    const prefix = parts[0];
+    const backend = BACKENDS[prefix];
+    if (!backend) {
+        const supported = Object.keys(BACKENDS).join(", ");
+        throw new Error(`No backend for model "${modelId}". Supported prefixes: ${supported}`);
+    }
+    // Extract the model name for the API (e.g., "claude-opus-4-6" from "anthropic:messages:claude-opus-4-6")
+    const modelName = parts.length > 2 ? parts.slice(2).join(":") : parts[parts.length - 1];
+    // For OpenAI, extract the API variant from the second segment
+    let apiVariant;
+    if (prefix === "openai" && parts.length > 2) {
+        const variant = parts[1];
+        if (variant === "responses" || variant === "chat") {
+            apiVariant = variant;
+        }
+    }
+    return { backend, modelName, apiVariant };
+}
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+/** Append a machine-readable tool call summary for assertion detection */
+function appendToolSummary(text, log) {
+    if (log.length === 0)
+        return text;
+    const names = JSON.stringify(log.map((tc) => tc.name));
+    return `${text}\n\n<!-- MCP_TOOLS_CALLED: ${names} -->`;
+}
+/** Resolve the API key for a given model prefix */
+function resolveApiKey(prefix, config) {
+    if (config.apiKey)
+        return String(config.apiKey);
+    const envMap = {
+        anthropic: "ANTHROPIC_API_KEY",
+        openai: "OPENAI_API_KEY",
+    };
+    const envVar = envMap[prefix];
+    return envVar ? process.env[envVar] : undefined;
+}
+// ---------------------------------------------------------------------------
+// Provider class
+// ---------------------------------------------------------------------------
+export default class MCPToolProvider {
+    config;
+    providerId;
+    constructor(options = {}) {
+        this.config = options.config || {};
+        this.providerId = options.id || "mcp-tool-provider";
+    }
+    id() {
+        return this.providerId;
+    }
+    async callApi(prompt, _context) {
+        const mcpServerConfig = this.config.mcpServer;
+        if (!mcpServerConfig) {
+            return { error: "mcpServer config is required", output: undefined };
+        }
+        // Resolve model and backend
+        const modelId = this.config.model || "anthropic:messages:claude-opus-4-6";
+        let backend;
+        let modelName;
+        let apiVariant;
+        try {
+            const resolved = resolveBackend(modelId);
+            backend = resolved.backend;
+            modelName = resolved.modelName;
+            apiVariant = resolved.apiVariant;
+        }
+        catch (err) {
+            return {
+                error: err instanceof Error ? err.message : String(err),
+                output: undefined,
+            };
+        }
+        // Resolve API key
+        const prefix = modelId.split(":")[0];
+        const apiKey = resolveApiKey(prefix, this.config);
+        if (!apiKey) {
+            return {
+                error: `API key not found for ${prefix}. Set ${prefix.toUpperCase()}_API_KEY in env or config.apiKey.`,
+                output: undefined,
+            };
+        }
+        // Connect to MCP server
+        let mcpClient;
+        try {
+            mcpClient = await connectMCP(mcpServerConfig);
+        }
+        catch (err) {
+            return {
+                error: `Failed to connect to MCP server: ${err instanceof Error ? err.message : String(err)}`,
+                output: undefined,
+            };
+        }
+        try {
+            // Filter tools by capabilities
+            const allTools = mcpClient.getAllTools();
+            const toolFilter = this.config.mcpTools;
+            const tools = toolFilter
+                ? allTools.filter((t) => toolFilter.includes(t.name))
+                : allTools;
+            if (tools.length === 0) {
+                return {
+                    error: "No MCP tools available after filtering. Check mcpTools config and server capabilities.",
+                    output: undefined,
+                };
+            }
+            // Run the tool loop
+            const result = await backend({
+                prompt,
+                tools,
+                callTool: mcpClient.callTool,
+                maxToolRounds: this.config.maxToolRounds || 5,
+                model: modelName,
+                temperature: this.config.temperature ?? 0.2,
+                maxTokens: this.config.max_output_tokens ||
+                    this.config.max_completion_tokens ||
+                    this.config.max_tokens ||
+                    4096,
+                apiKey,
+                apiVariant,
+                providerConfig: this.config,
+            });
+            return {
+                cost: 0,
+                metadata: {
+                    toolRounds: result.toolRounds,
+                    toolCallLog: result.toolCallLog,
+                    exhaustedRounds: result.exhaustedRounds,
+                    latencyMs: result.latencyMs,
+                },
+                output: appendToolSummary(result.output, result.toolCallLog),
+                tokenUsage: result.tokenUsage,
+            };
+        }
+        finally {
+            await mcpClient.cleanup().catch(() => { });
+        }
+    }
+}

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/mcp-connection.d.ts

@@ -0,0 +1,19 @@
+/**
+ * MCP server connection and tool discovery.
+ *
+ * Handles connecting to an MCP server via streamable-http or stdio transport,
+ * discovering available tools, and resolving {{env.VAR}} templates in config.
+ */
+import type { MCPClient } from "./types.js";
+/**
+ * Connect to an MCP server and return a client for tool discovery and execution.
+ *
+ * Supports two transport types:
+ * - `url` → streamable-http (remote MCP servers like mcp.sanity.io)
+ * - `command` → stdio (local MCP server processes)
+ */
+export declare function connectMCP(serverConfig: Record<string, unknown>): Promise<MCPClient>;
+/**
+ * Resolve `{{env.VAR}}` templates in config values, recursively.
+ */
+export declare function resolveEnvTemplates(config: Record<string, unknown>): Record<string, unknown>;

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/mcp-connection.js

@@ -0,0 +1,101 @@
+/**
+ * MCP server connection and tool discovery.
+ *
+ * Handles connecting to an MCP server via streamable-http or stdio transport,
+ * discovering available tools, and resolving {{env.VAR}} templates in config.
+ */
+/**
+ * Connect to an MCP server and return a client for tool discovery and execution.
+ *
+ * Supports two transport types:
+ * - `url` → streamable-http (remote MCP servers like mcp.sanity.io)
+ * - `command` → stdio (local MCP server processes)
+ */
+export async function connectMCP(serverConfig) {
+    const { Client } = await import("@modelcontextprotocol/sdk/client/index.js");
+    const client = new Client({
+        name: "ailf-mcp-eval",
+        version: "1.0.0",
+    });
+    const resolvedConfig = resolveEnvTemplates(serverConfig);
+    let closeTransport;
+    if (resolvedConfig.command) {
+        const { StdioClientTransport } = await import("@modelcontextprotocol/sdk/client/stdio.js");
+        const parts = String(resolvedConfig.command).split(/\s+/);
+        const transport = new StdioClientTransport({
+            command: parts[0],
+            args: parts.slice(1),
+            env: process.env,
+        });
+        await client.connect(transport);
+        closeTransport = () => transport.close();
+    }
+    else if (resolvedConfig.url) {
+        const { StreamableHTTPClientTransport } = await import("@modelcontextprotocol/sdk/client/streamableHttp.js");
+        const headers = {};
+        // Auth-derived headers (structured auth config)
+        const auth = resolvedConfig.auth;
+        if (auth?.type === "bearer" && auth.token) {
+            headers["Authorization"] = `Bearer ${auth.token}`;
+        }
+        // Explicit headers override auth-derived ones
+        const customHeaders = resolvedConfig.headers;
+        if (customHeaders) {
+            Object.assign(headers, customHeaders);
+        }
+        const transport = new StreamableHTTPClientTransport(new URL(String(resolvedConfig.url)), { requestInit: { headers } });
+        await client.connect(transport);
+        closeTransport = () => transport.close();
+    }
+    else {
+        throw new Error("MCP server config must have either 'command' (stdio) or 'url' (http)");
+    }
+    // Discover tools
+    const { tools: toolsList } = await client.listTools();
+    const allTools = toolsList.map((t) => ({
+        name: t.name,
+        description: t.description,
+        inputSchema: t.inputSchema,
+    }));
+    return {
+        getAllTools: () => allTools,
+        callTool: async (name, args) => {
+            const result = await client.callTool({ name, arguments: args });
+            let content = "";
+            if (result?.content) {
+                if (Array.isArray(result.content)) {
+                    content = result.content
+                        .map((c) => c.text || JSON.stringify(c))
+                        .join("\n");
+                }
+                else {
+                    content = String(result.content);
+                }
+            }
+            return { content, error: result.isError ? content : undefined };
+        },
+        cleanup: async () => {
+            await closeTransport().catch(() => { });
+        },
+    };
+}
+/**
+ * Resolve `{{env.VAR}}` templates in config values, recursively.
+ */
+export function resolveEnvTemplates(config) {
+    const resolved = {};
+    for (const [key, value] of Object.entries(config)) {
+        if (typeof value === "string") {
+            resolved[key] = value.replace(/\{\{env\.(\w+)\}\}/g, (_, varName) => {
+                return process.env[varName] || "";
+            });
+        }
+        else if (value && typeof value === "object" && !Array.isArray(value)) {
+            resolved[key] = resolveEnvTemplates(value);
+        }
+        else {
+            resolved[key] = value;
+        }
+    }
+    return resolved;
+}

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/tool-loop-anthropic.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Anthropic multi-turn tool execution loop.
+ *
+ * Sends a prompt to the Anthropic Messages API with MCP tools attached.
+ * When the model calls a tool, executes it via the MCP client, feeds
+ * the result back, and continues until the model produces a final text
+ * response or maxToolRounds is exhausted.
+ */
+import type { ToolLoopConfig, ToolLoopResult } from "./types.js";
+/**
+ * Run a multi-turn tool loop using the Anthropic Messages API.
+ *
+ * The loop:
+ * 1. Sends the prompt with available tools to Claude
+ * 2. If Claude calls tools → executes them via MCP, sends results back
+ * 3. Repeats until Claude produces a text-only response or maxToolRounds is hit
+ * 4. On the last round, omits tools to force a synthesis response
+ */
+export declare function runAnthropicToolLoop(config: ToolLoopConfig): Promise<ToolLoopResult>;

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/tool-loop-anthropic.js

@@ -0,0 +1,172 @@
+/**
+ * Anthropic multi-turn tool execution loop.
+ *
+ * Sends a prompt to the Anthropic Messages API with MCP tools attached.
+ * When the model calls a tool, executes it via the MCP client, feeds
+ * the result back, and continues until the model produces a final text
+ * response or maxToolRounds is exhausted.
+ */
+// ---------------------------------------------------------------------------
+// Tool loop implementation
+// ---------------------------------------------------------------------------
+/**
+ * Run a multi-turn tool loop using the Anthropic Messages API.
+ *
+ * The loop:
+ * 1. Sends the prompt with available tools to Claude
+ * 2. If Claude calls tools → executes them via MCP, sends results back
+ * 3. Repeats until Claude produces a text-only response or maxToolRounds is hit
+ * 4. On the last round, omits tools to force a synthesis response
+ */
+export async function runAnthropicToolLoop(config) {
+    const { prompt, tools, callTool, maxToolRounds, model, temperature, maxTokens, apiKey, } = config;
+    const anthropicTools = tools.map((t) => ({
+        name: t.name,
+        description: t.description || `MCP tool: ${t.name}`,
+        input_schema: t.inputSchema || { type: "object", properties: {} },
+    }));
+    const systemPrompt = "You are an AI assistant with access to tools provided by an MCP server. " +
+        "Use the available tools to complete the task. Call tools with correct parameters, " +
+        "interpret responses, and provide a complete answer.";
+    const messages = [{ content: prompt, role: "user" }];
+    let inputTokens = 0;
+    let outputTokens = 0;
+    const startTime = Date.now();
+    const toolCallLog = [];
+    for (let round = 0; round <= maxToolRounds; round++) {
+        const isLastRound = round === maxToolRounds;
+        // On the last round, omit tools to force a text-only response.
+        // Anthropic doesn't support tool_choice: "none" — the way to disable
+        // tools is to simply not include them in the request.
+        if (isLastRound) {
+            const lastMsg = messages[messages.length - 1];
+            const synthesisText = "You've used the tools available. Based on the information gathered, " +
+                "provide your complete, final answer now.";
+            if (lastMsg?.role === "user" && Array.isArray(lastMsg.content)) {
+                ;
+                lastMsg.content.push({
+                    type: "text",
+                    text: synthesisText,
+                });
+            }
+            else {
+                messages.push({ content: synthesisText, role: "user" });
+            }
+        }
+        const body = {
+            max_tokens: maxTokens,
+            messages,
+            model,
+            system: systemPrompt,
+            temperature,
+        };
+        if (!isLastRound) {
+            body.tools = anthropicTools;
+        }
+        const response = await fetch("https://api.anthropic.com/v1/messages", {
+            body: JSON.stringify(body),
+            headers: {
+                "anthropic-version": "2023-06-01",
+                "Content-Type": "application/json",
+                "x-api-key": apiKey,
+            },
+            method: "POST",
+        });
+        const data = (await response.json());
+        if (data.error) {
+            throw new Error(data.error.message ??
+                `Anthropic API error: ${JSON.stringify(data.error)}`);
+        }
+        inputTokens += data.usage?.input_tokens ?? 0;
+        outputTokens += data.usage?.output_tokens ?? 0;
+        if (!data.content?.length) {
+            return {
+                output: "",
+                toolCallLog,
+                tokenUsage: {
+                    prompt: inputTokens,
+                    completion: outputTokens,
+                    total: inputTokens + outputTokens,
+                },
+                toolRounds: round,
+                latencyMs: Date.now() - startTime,
+            };
+        }
+        // Add assistant response to history
+        messages.push({ content: data.content, role: "assistant" });
+        // Check if model wants to use tools
+        const toolUseBlocks = data.content.filter((b) => b.type === "tool_use");
+        if (data.stop_reason !== "tool_use" || toolUseBlocks.length === 0) {
+            // Model is done — extract text
+            const textBlocks = data.content.filter((b) => b.type === "text");
+            const output = textBlocks.map((b) => b.text || "").join("\n") || "";
+            return {
+                output,
+                toolCallLog,
+                tokenUsage: {
+                    prompt: inputTokens,
+                    completion: outputTokens,
+                    total: inputTokens + outputTokens,
+                },
+                toolRounds: round,
+                latencyMs: Date.now() - startTime,
+            };
+        }
+        // Execute each tool call via MCP
+        const toolResults = [];
+        for (const toolUse of toolUseBlocks) {
+            const toolName = toolUse.name;
+            const toolInput = (toolUse.input || {});
+            try {
+                const result = await callTool(toolName, toolInput);
+                const content = result.error
+                    ? JSON.stringify({ error: result.error })
+                    : result.content;
+                toolCallLog.push({ name: toolName, input: toolInput, output: content });
+                toolResults.push({
+                    content,
+                    tool_use_id: toolUse.id,
+                    type: "tool_result",
+                });
+            }
+            catch (err) {
+                const errMsg = err instanceof Error ? err.message : String(err);
+                toolCallLog.push({
+                    name: toolName,
+                    input: toolInput,
+                    output: `Error: ${errMsg}`,
+                });
+                toolResults.push({
+                    content: JSON.stringify({ error: errMsg }),
+                    tool_use_id: toolUse.id,
+                    type: "tool_result",
+                });
+            }
+        }
+        // Add tool results to conversation
+        messages.push({ content: toolResults, role: "user" });
+    }
+    // Exhausted rounds — extract last text
+    const lastAssistant = [...messages]
+        .reverse()
+        .find((m) => m.role === "assistant");
+    let lastText = "";
+    if (lastAssistant && Array.isArray(lastAssistant.content)) {
+        lastText = lastAssistant.content
+            .filter((b) => b.type === "text")
+            .map((b) => b.text || "")
+            .join("\n");
+    }
+    return {
+        output: lastText || "[Exhausted tool rounds without final answer]",
+        toolCallLog,
+        tokenUsage: {
+            prompt: inputTokens,
+            completion: outputTokens,
+            total: inputTokens + outputTokens,
+        },
+        toolRounds: maxToolRounds,
+        exhaustedRounds: true,
+        latencyMs: Date.now() - startTime,
+    };
+}

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/tool-loop-openai.d.ts

@@ -0,0 +1,19 @@
+/**
+ * OpenAI multi-turn tool execution loop.
+ *
+ * Supports two OpenAI API surfaces:
+ * - **Chat Completions** (`/v1/chat/completions`) — used by `openai:chat:*` models
+ * - **Responses** (`/v1/responses`) — used by `openai:responses:*` models (GPT-5.x)
+ *
+ * Both follow the same loop pattern: send prompt → model calls tools → execute
+ * via MCP → feed results back → repeat until final text or maxToolRounds.
+ */
+import type { ToolLoopConfig, ToolLoopResult } from "./types.js";
+/**
+ * Run a multi-turn tool loop using the OpenAI API.
+ *
+ * Routes to Chat Completions or Responses API based on `config.apiVariant`:
+ * - `"responses"` → Responses API (`/v1/responses`)
+ * - `"chat"` or undefined → Chat Completions API (`/v1/chat/completions`)
+ */
+export declare function runOpenAIToolLoop(config: ToolLoopConfig): Promise<ToolLoopResult>;