@sanity/ailf 1.0.0 → 2.0.1

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider.js

@@ -0,0 +1,368 @@
+/**
+ * mcp-tool-provider.ts — Custom Promptfoo provider for MCP tool-use evaluation.
+ *
+ * Implements a multi-turn tool execution loop: the LLM receives a prompt,
+ * discovers MCP tools, calls them, gets results, and continues until it
+ * produces a final text answer or exhausts maxToolRounds.
+ *
+ * Promptfoo's built-in Anthropic/OpenAI providers with config.mcp only do
+ * single-turn tool calls. This provider fills that gap by managing the
+ * full conversation loop, similar to the agentic-provider.ts pattern.
+ *
+ * Promptfoo config usage:
+ *
+ *   providers:
+ *     - id: file://dist/pipeline/compiler/mode-handlers/mcp-tool-provider.js
+ *       label: "Claude Opus 4.6 + MCP"
+ *       config:
+ *         model: anthropic:messages:claude-opus-4-6
+ *         maxToolRounds: 5
+ *         temperature: 0.2
+ *         max_tokens: 4096
+ *         mcpServer:
+ *           url: https://mcp.sanity.io
+ *           auth: { type: bearer, token: "{{env.SANITY_API_TOKEN}}" }
+ *           name: mcp-live-query-documents
+ *         mcpTools: [query_documents, get_schema]
+ */
+import { config as loadDotenv } from "dotenv";
+loadDotenv({
+    override: true,
+    path: new URL("../../../../.env", import.meta.url).pathname,
+});
+// ---------------------------------------------------------------------------
+// Provider
+// ---------------------------------------------------------------------------
+export default class MCPToolProvider {
+    config;
+    providerId;
+    constructor(options = {}) {
+        this.config = options.config || {};
+        this.providerId = options.id || "mcp-tool-provider";
+    }
+    id() {
+        return this.providerId;
+    }
+    async callApi(prompt, _context) {
+        const mcpServerConfig = this.config.mcpServer;
+        if (!mcpServerConfig) {
+            return { error: "mcpServer config is required", output: undefined };
+        }
+        // Resolve model provider
+        const modelId = this.config.model ||
+            "anthropic:messages:claude-sonnet-4-20250514";
+        if (modelId.startsWith("anthropic:")) {
+            return this.runAnthropicLoop(prompt, mcpServerConfig, modelId);
+        }
+        // For now, only Anthropic is supported. OpenAI support can be added later.
+        return {
+            error: `MCP tool provider only supports Anthropic models for now. Got: ${modelId}`,
+            output: undefined,
+        };
+    }
+    // -------------------------------------------------------------------------
+    // Anthropic multi-turn MCP tool loop
+    // -------------------------------------------------------------------------
+    async runAnthropicLoop(prompt, mcpServerConfig, modelId) {
+        // Parse model name from provider ID (e.g., "anthropic:messages:claude-opus-4-6" → "claude-opus-4-6")
+        const modelParts = modelId.split(":");
+        const model = modelParts.length > 2
+            ? modelParts.slice(2).join(":")
+            : modelParts[modelParts.length - 1];
+        const temperature = this.config.temperature ?? 0.2;
+        const maxTokens = this.config.max_tokens || 4096;
+        const maxToolRounds = this.config.maxToolRounds || 5;
+        const apiKey = this.config.apiKey || process.env.ANTHROPIC_API_KEY;
+        if (!apiKey) {
+            return {
+                error: "ANTHROPIC_API_KEY not set. Configure it in env or provider config.",
+                output: undefined,
+            };
+        }
+        // Connect to MCP server and discover tools
+        let mcpClient;
+        try {
+            mcpClient = await this.connectMCP(mcpServerConfig);
+        }
+        catch (err) {
+            return {
+                error: `Failed to connect to MCP server: ${err instanceof Error ? err.message : String(err)}`,
+                output: undefined,
+            };
+        }
+        try {
+            // Get available tools and convert to Anthropic format
+            const mcpTools = mcpClient.getAllTools();
+            const toolFilter = this.config.mcpTools;
+            const filteredTools = toolFilter
+                ? mcpTools.filter((t) => toolFilter.includes(t.name))
+                : mcpTools;
+            const tools = filteredTools.map((t) => ({
+                name: t.name,
+                description: t.description || `MCP tool: ${t.name}`,
+                input_schema: t.inputSchema || { type: "object", properties: {} },
+            }));
+            if (tools.length === 0) {
+                return {
+                    error: "No MCP tools available after filtering. Check mcpTools config and server capabilities.",
+                    output: undefined,
+                };
+            }
+            /** Append a machine-readable tool call summary to output for assertion detection */
+            function appendToolSummary(text, log) {
+                if (log.length === 0)
+                    return text;
+                const names = JSON.stringify(log.map((tc) => tc.name));
+                return `${text}\n\n<!-- MCP_TOOLS_CALLED: ${names} -->`;
+            }
+            const systemPrompt = "You are an AI assistant with access to tools provided by an MCP server. " +
+                "Use the available tools to complete the task. Call tools with correct parameters, " +
+                "interpret responses, and provide a complete answer.";
+            const messages = [{ content: prompt, role: "user" }];
+            let inputTokens = 0;
+            let outputTokens = 0;
+            const startTime = Date.now();
+            const toolCallLog = [];
+            for (let round = 0; round <= maxToolRounds; round++) {
+                const isLastRound = round === maxToolRounds;
+                // On last round, omit tools to force a final text response
+                if (isLastRound) {
+                    const lastMsg = messages[messages.length - 1];
+                    const synthesisText = "You've used the tools available. Based on the information gathered, " +
+                        "provide your complete, final answer now.";
+                    if (lastMsg?.role === "user" && Array.isArray(lastMsg.content)) {
+                        ;
+                        lastMsg.content.push({
+                            type: "text",
+                            text: synthesisText,
+                        });
+                    }
+                    else {
+                        messages.push({ content: synthesisText, role: "user" });
+                    }
+                }
+                const body = {
+                    max_tokens: maxTokens,
+                    messages,
+                    model,
+                    system: systemPrompt,
+                    temperature,
+                };
+                if (!isLastRound) {
+                    body.tools = tools;
+                }
+                const response = await fetch("https://api.anthropic.com/v1/messages", {
+                    body: JSON.stringify(body),
+                    headers: {
+                        "anthropic-version": "2023-06-01",
+                        "Content-Type": "application/json",
+                        "x-api-key": apiKey,
+                    },
+                    method: "POST",
+                });
+                const data = (await response.json());
+                if (data.error) {
+                    return {
+                        error: data.error.message ??
+                            `Anthropic API error: ${JSON.stringify(data.error)}`,
+                        output: undefined,
+                    };
+                }
+                inputTokens += data.usage?.input_tokens ?? 0;
+                outputTokens += data.usage?.output_tokens ?? 0;
+                if (!data.content?.length) {
+                    return {
+                        cost: 0,
+                        metadata: { toolRounds: round, toolCallLog },
+                        output: "",
+                        tokenUsage: {
+                            completion: outputTokens,
+                            prompt: inputTokens,
+                            total: inputTokens + outputTokens,
+                        },
+                    };
+                }
+                // Add assistant response to history
+                messages.push({ content: data.content, role: "assistant" });
+                // Check if model wants to use tools
+                const toolUseBlocks = data.content.filter((b) => b.type === "tool_use");
+                if (data.stop_reason !== "tool_use" || toolUseBlocks.length === 0) {
+                    // Model is done — extract text
+                    const textBlocks = data.content.filter((b) => b.type === "text");
+                    const rawOutput = textBlocks.map((b) => b.text || "").join("\n") || "";
+                    return {
+                        cost: 0,
+                        metadata: {
+                            toolRounds: round,
+                            toolCallLog,
+                            latencyMs: Date.now() - startTime,
+                        },
+                        output: appendToolSummary(rawOutput, toolCallLog),
+                        tokenUsage: {
+                            completion: outputTokens,
+                            prompt: inputTokens,
+                            total: inputTokens + outputTokens,
+                        },
+                    };
+                }
+                // Execute each tool call via MCP
+                const toolResults = [];
+                for (const toolUse of toolUseBlocks) {
+                    const toolName = toolUse.name;
+                    const toolInput = (toolUse.input || {});
+                    try {
+                        const result = await mcpClient.callTool(toolName, toolInput);
+                        const content = result.error
+                            ? JSON.stringify({ error: result.error })
+                            : result.content;
+                        toolCallLog.push({
+                            name: toolName,
+                            input: toolInput,
+                            output: content,
+                        });
+                        toolResults.push({
+                            content,
+                            tool_use_id: toolUse.id,
+                            type: "tool_result",
+                        });
+                    }
+                    catch (err) {
+                        const errMsg = err instanceof Error ? err.message : String(err);
+                        toolCallLog.push({
+                            name: toolName,
+                            input: toolInput,
+                            output: `Error: ${errMsg}`,
+                        });
+                        toolResults.push({
+                            content: JSON.stringify({ error: errMsg }),
+                            tool_use_id: toolUse.id,
+                            type: "tool_result",
+                        });
+                    }
+                }
+                // Add tool results to conversation
+                messages.push({ content: toolResults, role: "user" });
+            }
+            // Exhausted rounds
+            const lastAssistant = [...messages]
+                .reverse()
+                .find((m) => m.role === "assistant");
+            let lastText = "";
+            if (lastAssistant && Array.isArray(lastAssistant.content)) {
+                lastText = lastAssistant.content
+                    .filter((b) => b.type === "text")
+                    .map((b) => b.text || "")
+                    .join("\n");
+            }
+            return {
+                cost: 0,
+                metadata: {
+                    toolRounds: maxToolRounds,
+                    exhaustedRounds: true,
+                    toolCallLog,
+                    latencyMs: Date.now() - startTime,
+                },
+                output: appendToolSummary(lastText || "[Exhausted tool rounds without final answer]", toolCallLog),
+                tokenUsage: {
+                    completion: outputTokens,
+                    prompt: inputTokens,
+                    total: inputTokens + outputTokens,
+                },
+            };
+        }
+        finally {
+            await mcpClient.cleanup().catch(() => { });
+        }
+    }
+    // -------------------------------------------------------------------------
+    // MCP client management
+    // -------------------------------------------------------------------------
+    async connectMCP(serverConfig) {
+        // Dynamically import Promptfoo's MCPClient — reuse its MCP SDK integration
+        // rather than adding a direct dependency on @modelcontextprotocol/sdk
+        const { Client } = await import("@modelcontextprotocol/sdk/client/index.js");
+        const client = new Client({
+            name: "ailf-mcp-eval",
+            version: "1.0.0",
+        });
+        // Resolve auth — render {{env.VAR}} templates
+        const resolvedConfig = this.resolveEnvTemplates(serverConfig);
+        // Determine transport type and connect
+        let closeTransport;
+        if (resolvedConfig.command) {
+            // stdio transport
+            const { StdioClientTransport } = await import("@modelcontextprotocol/sdk/client/stdio.js");
+            const parts = String(resolvedConfig.command).split(/\s+/);
+            const transport = new StdioClientTransport({
+                command: parts[0],
+                args: parts.slice(1),
+                env: process.env,
+            });
+            await client.connect(transport);
+            closeTransport = () => transport.close();
+        }
+        else if (resolvedConfig.url) {
+            // streamable-http transport
+            const { StreamableHTTPClientTransport } = await import("@modelcontextprotocol/sdk/client/streamableHttp.js");
+            const headers = {};
+            const auth = resolvedConfig.auth;
+            if (auth?.type === "bearer" && auth.token) {
+                headers["Authorization"] = `Bearer ${auth.token}`;
+            }
+            const transport = new StreamableHTTPClientTransport(new URL(String(resolvedConfig.url)), { requestInit: { headers } });
+            await client.connect(transport);
+            closeTransport = () => transport.close();
+        }
+        else {
+            throw new Error("MCP server config must have either 'command' (stdio) or 'url' (http)");
+        }
+        // Discover tools
+        const { tools: toolsList } = await client.listTools();
+        const allTools = toolsList.map((t) => ({
+            name: t.name,
+            description: t.description,
+            inputSchema: t.inputSchema,
+        }));
+        return {
+            getAllTools: () => allTools,
+            callTool: async (name, args) => {
+                const result = await client.callTool({ name, arguments: args });
+                let content = "";
+                if (result?.content) {
+                    if (Array.isArray(result.content)) {
+                        content = result.content
+                            .map((c) => c.text || JSON.stringify(c))
+                            .join("\n");
+                    }
+                    else {
+                        content = String(result.content);
+                    }
+                }
+                return { content, error: result.isError ? content : undefined };
+            },
+            cleanup: async () => {
+                await closeTransport().catch(() => { });
+            },
+        };
+    }
+    /**
+     * Resolve {{env.VAR}} templates in config values.
+     */
+    resolveEnvTemplates(config) {
+        const resolved = {};
+        for (const [key, value] of Object.entries(config)) {
+            if (typeof value === "string") {
+                resolved[key] = value.replace(/\{\{env\.(\w+)\}\}/g, (_, varName) => {
+                    return process.env[varName] || "";
+                });
+            }
+            else if (value && typeof value === "object" && !Array.isArray(value)) {
+                resolved[key] = this.resolveEnvTemplates(value);
+            }
+            else {
+                resolved[key] = value;
+            }
+        }
+        return resolved;
+    }
+}

package/dist/pipeline/compiler/preset-loader.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Preset loader — resolves and loads external PresetDefinition files.
+ *
+ * Supports two reference formats:
+ * - Relative path (./foo.ts, ../presets/bar.ts) → loaded from disk via jiti
+ * - npm package name (my-preset-package) → resolved via Node require
+ *
+ * Uses jiti for synchronous TypeScript/JavaScript loading, matching the
+ * sync constraint of createAppContext() and createRegistry().
+ *
+ * @see packages/eval/src/composition-root.ts — where this is called
+ */
+import type { PresetDefinition } from "../../_vendor/ailf-core/index.d.ts";
+/**
+ * Load external presets from file paths or npm package names.
+ *
+ * @param refs - Preset references (relative paths or package names)
+ * @param rootDir - Base directory for resolving relative paths
+ * @returns Loaded PresetDefinition array (preserves input order)
+ * @throws If a preset file is not found or doesn't export a valid PresetDefinition
+ */
+export declare function loadExternalPresets(refs: string[], rootDir: string): PresetDefinition[];

package/dist/pipeline/compiler/preset-loader.js

@@ -0,0 +1,99 @@
+/**
+ * Preset loader — resolves and loads external PresetDefinition files.
+ *
+ * Supports two reference formats:
+ * - Relative path (./foo.ts, ../presets/bar.ts) → loaded from disk via jiti
+ * - npm package name (my-preset-package) → resolved via Node require
+ *
+ * Uses jiti for synchronous TypeScript/JavaScript loading, matching the
+ * sync constraint of createAppContext() and createRegistry().
+ *
+ * @see packages/eval/src/composition-root.ts — where this is called
+ */
+import { existsSync } from "fs";
+import { resolve } from "path";
+import { pathToFileURL } from "url";
+import { createJiti } from "jiti";
+/** Thrown for preset-specific load errors (distinguishes from third-party errors) */
+class PresetLoadError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "PresetLoadError";
+    }
+}
+/**
+ * Load external presets from file paths or npm package names.
+ *
+ * @param refs - Preset references (relative paths or package names)
+ * @param rootDir - Base directory for resolving relative paths
+ * @returns Loaded PresetDefinition array (preserves input order)
+ * @throws If a preset file is not found or doesn't export a valid PresetDefinition
+ */
+export function loadExternalPresets(refs, rootDir) {
+    const presets = [];
+    for (const ref of refs) {
+        const preset = loadSinglePreset(ref, rootDir);
+        presets.push(preset);
+    }
+    return presets;
+}
+function loadSinglePreset(ref, rootDir) {
+    const isPath = ref.startsWith("./") || ref.startsWith("../") || ref.startsWith("/");
+    const filePath = isPath ? resolve(rootDir, ref) : ref;
+    // Containment check: resolved path must be within rootDir to prevent
+    // path traversal (e.g., "../../other-project/secret.ts")
+    if (isPath) {
+        const resolvedRoot = resolve(rootDir);
+        if (!resolve(filePath).startsWith(resolvedRoot)) {
+            throw new PresetLoadError(`Preset "${ref}" resolves outside the project root (${resolvedRoot}). ` +
+                `Preset paths must be within the project directory.`);
+        }
+        if (!existsSync(filePath)) {
+            throw new PresetLoadError(`Preset file not found: ${filePath} (resolved from "${ref}" relative to ${rootDir})`);
+        }
+    }
+    try {
+        const jiti = createJiti(pathToFileURL(rootDir).href, {
+            interopDefault: true,
+            requireCache: true,
+        });
+        // jiti() is the synchronous loader
+        const mod = jiti(filePath);
+        const preset = extractDefault(mod);
+        if (!preset || typeof preset !== "object") {
+            throw new PresetLoadError(`Preset "${ref}" does not export a default value. ` +
+                `Use \`export default definePreset({ ... })\`.`);
+        }
+        // Validate minimal shape
+        const p = preset;
+        if (!p.name || !p.mode || !p.manifest) {
+            throw new PresetLoadError(`Preset "${ref}" is missing required fields. ` +
+                `A preset must have at least \`name\`, \`mode\`, and \`manifest\`.`);
+        }
+        return p;
+    }
+    catch (cause) {
+        if (cause instanceof PresetLoadError) {
+            throw cause; // Re-throw our own errors
+        }
+        const message = cause instanceof Error ? cause.message : String(cause);
+        throw new Error(`Failed to load preset "${ref}": ${message}`, {
+            cause: cause,
+        });
+    }
+}
+/**
+ * Extract the default export from a module, handling both ESM and CJS patterns.
+ */
+function extractDefault(mod) {
+    if (mod === null || mod === undefined)
+        return undefined;
+    if (typeof mod !== "object")
+        return mod;
+    const obj = mod;
+    // ESM: { default: value }
+    if ("default" in obj)
+        return obj.default;
+    // CJS: the module itself is the value
+    return obj;
+}

package/dist/pipeline/compiler/presets/sanity-literacy.d.ts

@@ -1,20 +1,17 @@
 /**
- * sanity-literacy preset — the built-in documentation literacy evaluation.
+ * sanity-literacy preset — Sanity-specific domain configuration for literacy evaluation.
  *
- * Packages ALL Sanity-specific configuration into a single preset:
- * - Literacy mode handler registration
- * - Prompt templates (with-docs, without-docs, agentic)
- * - Rubric templates (task-completion, code-correctness, doc-coverage)
- * - Scoring profiles (default, output-only)
+ * This is a domain preset that targets the `literacy` mode base. It provides
+ * Sanity-specific configuration:
  * - Sanity doc source definitions (production, branch, local)
  * - Product feature registry for coverage auditing
  * - DocFetcher factory (SanityDocFetcher)
- * - Standard assertions and fixture resolvers
+ * - Sanity fixture resolver (sanity:// scheme)
  *
- * Use `createSanityLiteracyPreset()` to get a fully configured preset
- * with a docFetcher factory bound to a specific rootDir.
+ * Evaluation methodology (rubrics, scoring, prompts) is inherited from the
+ * `literacy` mode base — see mode-bases/literacy.ts.
  *
- * @see docs/exec-plans/architecture-overhaul/phase-8-scoring-storage-presets.md
+ * @see docs/archive/exec-plans/architecture-overhaul/phase-8-scoring-storage-presets.md
  */
 import { type PresetDefinition } from "../../../_vendor/ailf-core/index.d.ts";
 export interface SanityLiteracyPresetOptions {