@sanity/ailf 2.0.1 → 2.1.0

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider.js

@@ -1,368 +0,0 @@
-/**
- * mcp-tool-provider.ts — Custom Promptfoo provider for MCP tool-use evaluation.
- *
- * Implements a multi-turn tool execution loop: the LLM receives a prompt,
- * discovers MCP tools, calls them, gets results, and continues until it
- * produces a final text answer or exhausts maxToolRounds.
- *
- * Promptfoo's built-in Anthropic/OpenAI providers with config.mcp only do
- * single-turn tool calls. This provider fills that gap by managing the
- * full conversation loop, similar to the agentic-provider.ts pattern.
- *
- * Promptfoo config usage:
- *
- *   providers:
- *     - id: file://dist/pipeline/compiler/mode-handlers/mcp-tool-provider.js
- *       label: "Claude Opus 4.6 + MCP"
- *       config:
- *         model: anthropic:messages:claude-opus-4-6
- *         maxToolRounds: 5
- *         temperature: 0.2
- *         max_tokens: 4096
- *         mcpServer:
- *           url: https://mcp.sanity.io
- *           auth: { type: bearer, token: "{{env.SANITY_API_TOKEN}}" }
- *           name: mcp-live-query-documents
- *         mcpTools: [query_documents, get_schema]
- */
-import { config as loadDotenv } from "dotenv";
-loadDotenv({
-    override: true,
-    path: new URL("../../../../.env", import.meta.url).pathname,
-});
-// ---------------------------------------------------------------------------
-// Provider
-// ---------------------------------------------------------------------------
-export default class MCPToolProvider {
-    config;
-    providerId;
-    constructor(options = {}) {
-        this.config = options.config || {};
-        this.providerId = options.id || "mcp-tool-provider";
-    }
-    id() {
-        return this.providerId;
-    }
-    async callApi(prompt, _context) {
-        const mcpServerConfig = this.config.mcpServer;
-        if (!mcpServerConfig) {
-            return { error: "mcpServer config is required", output: undefined };
-        }
-        // Resolve model provider
-        const modelId = this.config.model ||
-            "anthropic:messages:claude-sonnet-4-20250514";
-        if (modelId.startsWith("anthropic:")) {
-            return this.runAnthropicLoop(prompt, mcpServerConfig, modelId);
-        }
-        // For now, only Anthropic is supported. OpenAI support can be added later.
-        return {
-            error: `MCP tool provider only supports Anthropic models for now. Got: ${modelId}`,
-            output: undefined,
-        };
-    }
-    // -------------------------------------------------------------------------
-    // Anthropic multi-turn MCP tool loop
-    // -------------------------------------------------------------------------
-    async runAnthropicLoop(prompt, mcpServerConfig, modelId) {
-        // Parse model name from provider ID (e.g., "anthropic:messages:claude-opus-4-6" → "claude-opus-4-6")
-        const modelParts = modelId.split(":");
-        const model = modelParts.length > 2
-            ? modelParts.slice(2).join(":")
-            : modelParts[modelParts.length - 1];
-        const temperature = this.config.temperature ?? 0.2;
-        const maxTokens = this.config.max_tokens || 4096;
-        const maxToolRounds = this.config.maxToolRounds || 5;
-        const apiKey = this.config.apiKey || process.env.ANTHROPIC_API_KEY;
-        if (!apiKey) {
-            return {
-                error: "ANTHROPIC_API_KEY not set. Configure it in env or provider config.",
-                output: undefined,
-            };
-        }
-        // Connect to MCP server and discover tools
-        let mcpClient;
-        try {
-            mcpClient = await this.connectMCP(mcpServerConfig);
-        }
-        catch (err) {
-            return {
-                error: `Failed to connect to MCP server: ${err instanceof Error ? err.message : String(err)}`,
-                output: undefined,
-            };
-        }
-        try {
-            // Get available tools and convert to Anthropic format
-            const mcpTools = mcpClient.getAllTools();
-            const toolFilter = this.config.mcpTools;
-            const filteredTools = toolFilter
-                ? mcpTools.filter((t) => toolFilter.includes(t.name))
-                : mcpTools;
-            const tools = filteredTools.map((t) => ({
-                name: t.name,
-                description: t.description || `MCP tool: ${t.name}`,
-                input_schema: t.inputSchema || { type: "object", properties: {} },
-            }));
-            if (tools.length === 0) {
-                return {
-                    error: "No MCP tools available after filtering. Check mcpTools config and server capabilities.",
-                    output: undefined,
-                };
-            }
-            /** Append a machine-readable tool call summary to output for assertion detection */
-            function appendToolSummary(text, log) {
-                if (log.length === 0)
-                    return text;
-                const names = JSON.stringify(log.map((tc) => tc.name));
-                return `${text}\n\n<!-- MCP_TOOLS_CALLED: ${names} -->`;
-            }
-            const systemPrompt = "You are an AI assistant with access to tools provided by an MCP server. " +
-                "Use the available tools to complete the task. Call tools with correct parameters, " +
-                "interpret responses, and provide a complete answer.";
-            const messages = [{ content: prompt, role: "user" }];
-            let inputTokens = 0;
-            let outputTokens = 0;
-            const startTime = Date.now();
-            const toolCallLog = [];
-            for (let round = 0; round <= maxToolRounds; round++) {
-                const isLastRound = round === maxToolRounds;
-                // On last round, omit tools to force a final text response
-                if (isLastRound) {
-                    const lastMsg = messages[messages.length - 1];
-                    const synthesisText = "You've used the tools available. Based on the information gathered, " +
-                        "provide your complete, final answer now.";
-                    if (lastMsg?.role === "user" && Array.isArray(lastMsg.content)) {
-                        ;
-                        lastMsg.content.push({
-                            type: "text",
-                            text: synthesisText,
-                        });
-                    }
-                    else {
-                        messages.push({ content: synthesisText, role: "user" });
-                    }
-                }
-                const body = {
-                    max_tokens: maxTokens,
-                    messages,
-                    model,
-                    system: systemPrompt,
-                    temperature,
-                };
-                if (!isLastRound) {
-                    body.tools = tools;
-                }
-                const response = await fetch("https://api.anthropic.com/v1/messages", {
-                    body: JSON.stringify(body),
-                    headers: {
-                        "anthropic-version": "2023-06-01",
-                        "Content-Type": "application/json",
-                        "x-api-key": apiKey,
-                    },
-                    method: "POST",
-                });
-                const data = (await response.json());
-                if (data.error) {
-                    return {
-                        error: data.error.message ??
-                            `Anthropic API error: ${JSON.stringify(data.error)}`,
-                        output: undefined,
-                    };
-                }
-                inputTokens += data.usage?.input_tokens ?? 0;
-                outputTokens += data.usage?.output_tokens ?? 0;
-                if (!data.content?.length) {
-                    return {
-                        cost: 0,
-                        metadata: { toolRounds: round, toolCallLog },
-                        output: "",
-                        tokenUsage: {
-                            completion: outputTokens,
-                            prompt: inputTokens,
-                            total: inputTokens + outputTokens,
-                        },
-                    };
-                }
-                // Add assistant response to history
-                messages.push({ content: data.content, role: "assistant" });
-                // Check if model wants to use tools
-                const toolUseBlocks = data.content.filter((b) => b.type === "tool_use");
-                if (data.stop_reason !== "tool_use" || toolUseBlocks.length === 0) {
-                    // Model is done — extract text
-                    const textBlocks = data.content.filter((b) => b.type === "text");
-                    const rawOutput = textBlocks.map((b) => b.text || "").join("\n") || "";
-                    return {
-                        cost: 0,
-                        metadata: {
-                            toolRounds: round,
-                            toolCallLog,
-                            latencyMs: Date.now() - startTime,
-                        },
-                        output: appendToolSummary(rawOutput, toolCallLog),
-                        tokenUsage: {
-                            completion: outputTokens,
-                            prompt: inputTokens,
-                            total: inputTokens + outputTokens,
-                        },
-                    };
-                }
-                // Execute each tool call via MCP
-                const toolResults = [];
-                for (const toolUse of toolUseBlocks) {
-                    const toolName = toolUse.name;
-                    const toolInput = (toolUse.input || {});
-                    try {
-                        const result = await mcpClient.callTool(toolName, toolInput);
-                        const content = result.error
-                            ? JSON.stringify({ error: result.error })
-                            : result.content;
-                        toolCallLog.push({
-                            name: toolName,
-                            input: toolInput,
-                            output: content,
-                        });
-                        toolResults.push({
-                            content,
-                            tool_use_id: toolUse.id,
-                            type: "tool_result",
-                        });
-                    }
-                    catch (err) {
-                        const errMsg = err instanceof Error ? err.message : String(err);
-                        toolCallLog.push({
-                            name: toolName,
-                            input: toolInput,
-                            output: `Error: ${errMsg}`,
-                        });
-                        toolResults.push({
-                            content: JSON.stringify({ error: errMsg }),
-                            tool_use_id: toolUse.id,
-                            type: "tool_result",
-                        });
-                    }
-                }
-                // Add tool results to conversation
-                messages.push({ content: toolResults, role: "user" });
-            }
-            // Exhausted rounds
-            const lastAssistant = [...messages]
-                .reverse()
-                .find((m) => m.role === "assistant");
-            let lastText = "";
-            if (lastAssistant && Array.isArray(lastAssistant.content)) {
-                lastText = lastAssistant.content
-                    .filter((b) => b.type === "text")
-                    .map((b) => b.text || "")
-                    .join("\n");
-            }
-            return {
-                cost: 0,
-                metadata: {
-                    toolRounds: maxToolRounds,
-                    exhaustedRounds: true,
-                    toolCallLog,
-                    latencyMs: Date.now() - startTime,
-                },
-                output: appendToolSummary(lastText || "[Exhausted tool rounds without final answer]", toolCallLog),
-                tokenUsage: {
-                    completion: outputTokens,
-                    prompt: inputTokens,
-                    total: inputTokens + outputTokens,
-                },
-            };
-        }
-        finally {
-            await mcpClient.cleanup().catch(() => { });
-        }
-    }
-    // -------------------------------------------------------------------------
-    // MCP client management
-    // -------------------------------------------------------------------------
-    async connectMCP(serverConfig) {
-        // Dynamically import Promptfoo's MCPClient — reuse its MCP SDK integration
-        // rather than adding a direct dependency on @modelcontextprotocol/sdk
-        const { Client } = await import("@modelcontextprotocol/sdk/client/index.js");
-        const client = new Client({
-            name: "ailf-mcp-eval",
-            version: "1.0.0",
-        });
-        // Resolve auth — render {{env.VAR}} templates
-        const resolvedConfig = this.resolveEnvTemplates(serverConfig);
-        // Determine transport type and connect
-        let closeTransport;
-        if (resolvedConfig.command) {
-            // stdio transport
-            const { StdioClientTransport } = await import("@modelcontextprotocol/sdk/client/stdio.js");
-            const parts = String(resolvedConfig.command).split(/\s+/);
-            const transport = new StdioClientTransport({
-                command: parts[0],
-                args: parts.slice(1),
-                env: process.env,
-            });
-            await client.connect(transport);
-            closeTransport = () => transport.close();
-        }
-        else if (resolvedConfig.url) {
-            // streamable-http transport
-            const { StreamableHTTPClientTransport } = await import("@modelcontextprotocol/sdk/client/streamableHttp.js");
-            const headers = {};
-            const auth = resolvedConfig.auth;
-            if (auth?.type === "bearer" && auth.token) {
-                headers["Authorization"] = `Bearer ${auth.token}`;
-            }
-            const transport = new StreamableHTTPClientTransport(new URL(String(resolvedConfig.url)), { requestInit: { headers } });
-            await client.connect(transport);
-            closeTransport = () => transport.close();
-        }
-        else {
-            throw new Error("MCP server config must have either 'command' (stdio) or 'url' (http)");
-        }
-        // Discover tools
-        const { tools: toolsList } = await client.listTools();
-        const allTools = toolsList.map((t) => ({
-            name: t.name,
-            description: t.description,
-            inputSchema: t.inputSchema,
-        }));
-        return {
-            getAllTools: () => allTools,
-            callTool: async (name, args) => {
-                const result = await client.callTool({ name, arguments: args });
-                let content = "";
-                if (result?.content) {
-                    if (Array.isArray(result.content)) {
-                        content = result.content
-                            .map((c) => c.text || JSON.stringify(c))
-                            .join("\n");
-                    }
-                    else {
-                        content = String(result.content);
-                    }
-                }
-                return { content, error: result.isError ? content : undefined };
-            },
-            cleanup: async () => {
-                await closeTransport().catch(() => { });
-            },
-        };
-    }
-    /**
-     * Resolve {{env.VAR}} templates in config values.
-     */
-    resolveEnvTemplates(config) {
-        const resolved = {};
-        for (const [key, value] of Object.entries(config)) {
-            if (typeof value === "string") {
-                resolved[key] = value.replace(/\{\{env\.(\w+)\}\}/g, (_, varName) => {
-                    return process.env[varName] || "";
-                });
-            }
-            else if (value && typeof value === "object" && !Array.isArray(value)) {
-                resolved[key] = this.resolveEnvTemplates(value);
-            }
-            else {
-                resolved[key] = value;
-            }
-        }
-        return resolved;
-    }
-}

package/dist/pipeline/compiler/task-bridge.d.ts

@@ -1,41 +0,0 @@
-/**
- * task-bridge.ts — Bidirectional bridge between old TaskDefinition and new LiteracyTaskDefinition.
- *
- * Enables incremental migration: consumers can convert between the two types
- * without changing their internal logic. Once all consumers use
- * GeneralizedTaskDefinition, this module is deleted (Wave 3 task 6).
- *
- * Field mapping (TaskDefinition ↔ LiteracyTaskDefinition):
- *   id              ↔ id
- *   description     ↔ title
- *   featureArea     ↔ area
- *   taskPrompt      ↔ prompt.text (fallback: prompt.template)
- *   canonicalDocs   ↔ context.docs
- *   referenceSolution ↔ referenceSolution
- *   docCoverage     ↔ docCoverage
- *   assertions      ↔ assertions       (structurally identical)
- *   baseline        ↔ baseline         (structurally identical)
- *   tags            ↔ tags
- *   status          ↔ status
- *   extraVars       ↔ prompt.vars
- *
- * The assertion and doc-ref sub-types are structurally identical between
- * the old and new type systems, so no field-level remapping is needed
- * for those — only a TypeScript-level cast.
- */
-import type { LiteracyTaskDefinition, TaskDefinition } from "../../_vendor/ailf-core/index.d.ts";
-/**
- * Convert an old-style TaskDefinition to the new LiteracyTaskDefinition.
- *
- * Every field of TaskDefinition has a corresponding field in LiteracyTaskDefinition,
- * so this conversion is lossless.
- */
-export declare function toGeneralized(task: TaskDefinition): LiteracyTaskDefinition;
-/**
- * Convert a new LiteracyTaskDefinition to the old TaskDefinition shape.
- *
- * Fields that only exist on LiteracyTaskDefinition (description, difficulty,
- * metadata, rubric, providers, options, context.fixtures, prompt.systemMessage)
- * are dropped — the old type has no place for them.
- */
-export declare function toLiteracyTask(task: LiteracyTaskDefinition): TaskDefinition;

package/dist/pipeline/compiler/task-bridge.js

@@ -1,92 +0,0 @@
-/**
- * task-bridge.ts — Bidirectional bridge between old TaskDefinition and new LiteracyTaskDefinition.
- *
- * Enables incremental migration: consumers can convert between the two types
- * without changing their internal logic. Once all consumers use
- * GeneralizedTaskDefinition, this module is deleted (Wave 3 task 6).
- *
- * Field mapping (TaskDefinition ↔ LiteracyTaskDefinition):
- *   id              ↔ id
- *   description     ↔ title
- *   featureArea     ↔ area
- *   taskPrompt      ↔ prompt.text (fallback: prompt.template)
- *   canonicalDocs   ↔ context.docs
- *   referenceSolution ↔ referenceSolution
- *   docCoverage     ↔ docCoverage
- *   assertions      ↔ assertions       (structurally identical)
- *   baseline        ↔ baseline         (structurally identical)
- *   tags            ↔ tags
- *   status          ↔ status
- *   extraVars       ↔ prompt.vars
- *
- * The assertion and doc-ref sub-types are structurally identical between
- * the old and new type systems, so no field-level remapping is needed
- * for those — only a TypeScript-level cast.
- */
-// ---------------------------------------------------------------------------
-// toGeneralized — old TaskDefinition → LiteracyTaskDefinition
-// ---------------------------------------------------------------------------
-/**
- * Convert an old-style TaskDefinition to the new LiteracyTaskDefinition.
- *
- * Every field of TaskDefinition has a corresponding field in LiteracyTaskDefinition,
- * so this conversion is lossless.
- */
-export function toGeneralized(task) {
-    const result = {
-        mode: "literacy",
-        id: task.id,
-        title: task.description,
-        area: task.featureArea,
-        prompt: {
-            text: task.taskPrompt,
-            ...(task.extraVars != null ? { vars: task.extraVars } : {}),
-        },
-        context: {
-            docs: task.canonicalDocs,
-        },
-        referenceSolution: task.referenceSolution,
-        docCoverage: task.docCoverage,
-        assertions: task.assertions,
-    };
-    // Only set optional fields when present to preserve round-trip identity
-    if (task.baseline != null)
-        result.baseline = task.baseline;
-    if (task.tags != null)
-        result.tags = task.tags;
-    if (task.status != null)
-        result.status = task.status;
-    return result;
-}
-// ---------------------------------------------------------------------------
-// toLiteracyTask — LiteracyTaskDefinition → old TaskDefinition
-// ---------------------------------------------------------------------------
-/**
- * Convert a new LiteracyTaskDefinition to the old TaskDefinition shape.
- *
- * Fields that only exist on LiteracyTaskDefinition (description, difficulty,
- * metadata, rubric, providers, options, context.fixtures, prompt.systemMessage)
- * are dropped — the old type has no place for them.
- */
-export function toLiteracyTask(task) {
-    const result = {
-        id: task.id,
-        description: task.title,
-        featureArea: task.area ?? "",
-        taskPrompt: task.prompt?.text ?? task.prompt?.template ?? "",
-        canonicalDocs: (task.context?.docs ?? []),
-        referenceSolution: task.referenceSolution ?? "",
-        docCoverage: task.docCoverage ?? false,
-        assertions: (task.assertions ?? []),
-    };
-    // Only set optional fields when present to preserve round-trip identity
-    if (task.baseline != null)
-        result.baseline = task.baseline;
-    if (task.tags != null)
-        result.tags = task.tags;
-    if (task.status != null)
-        result.status = task.status;
-    if (task.prompt?.vars != null)
-        result.extraVars = task.prompt.vars;
-    return result;
-}