@sanity/ailf 1.0.0 → 2.0.1

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/tool-loop-openai.js

@@ -0,0 +1,323 @@
+/**
+ * OpenAI multi-turn tool execution loop.
+ *
+ * Supports two OpenAI API surfaces:
+ * - **Chat Completions** (`/v1/chat/completions`) — used by `openai:chat:*` models
+ * - **Responses** (`/v1/responses`) — used by `openai:responses:*` models (GPT-5.x)
+ *
+ * Both follow the same loop pattern: send prompt → model calls tools → execute
+ * via MCP → feed results back → repeat until final text or maxToolRounds.
+ */
+/** Convert MCP tools to OpenAI Chat Completions function tool format */
+function toOpenAIChatTools(tools) {
+    return tools.map((t) => ({
+        type: "function",
+        function: {
+            name: t.name,
+            description: t.description || `MCP tool: ${t.name}`,
+            parameters: t.inputSchema || { type: "object", properties: {} },
+        },
+    }));
+}
+/** Convert MCP tools to OpenAI Responses API function tool format */
+function toOpenAIResponsesTools(tools) {
+    return tools.map((t) => ({
+        type: "function",
+        name: t.name,
+        description: t.description || `MCP tool: ${t.name}`,
+        parameters: t.inputSchema || { type: "object", properties: {} },
+    }));
+}
+const SYSTEM_PROMPT = "You are an AI assistant with access to tools provided by an MCP server. " +
+    "Use the available tools to complete the task. Call tools with correct parameters, " +
+    "interpret responses, and provide a complete answer.";
+// ---------------------------------------------------------------------------
+// Chat Completions tool loop
+// ---------------------------------------------------------------------------
+async function runChatCompletionsLoop(config) {
+    const { prompt, tools, callTool, maxToolRounds, model, temperature, maxTokens, apiKey, providerConfig, } = config;
+    const openaiTools = toOpenAIChatTools(tools);
+    // GPT-5.x and o-series use max_completion_tokens instead of max_tokens.
+    // NOTE: This model-name heuristic must be updated when OpenAI ships new
+    // model families. Prefer setting max_completion_tokens in providerConfig
+    // for new models. Matches the pattern in agentic-provider.ts.
+    const useMaxCompletionTokens = providerConfig?.max_output_tokens != null ||
+        providerConfig?.max_completion_tokens != null ||
+        model.startsWith("gpt-5") ||
+        model.startsWith("o3") ||
+        model.startsWith("o4");
+    const tokenLimitParam = useMaxCompletionTokens
+        ? { max_completion_tokens: maxTokens }
+        : { max_tokens: maxTokens };
+    const messages = [
+        { content: SYSTEM_PROMPT, role: "system" },
+        { content: prompt, role: "user" },
+    ];
+    let promptTokens = 0;
+    let completionTokens = 0;
+    let totalTokens = 0;
+    const startTime = Date.now();
+    const toolCallLog = [];
+    for (let round = 0; round <= maxToolRounds; round++) {
+        const isLastRound = round === maxToolRounds;
+        const response = await fetch("https://api.openai.com/v1/chat/completions", {
+            body: JSON.stringify({
+                ...tokenLimitParam,
+                messages,
+                model,
+                temperature,
+                tool_choice: isLastRound ? "none" : "auto",
+                tools: openaiTools,
+            }),
+            headers: {
+                Authorization: `Bearer ${apiKey}`,
+                "Content-Type": "application/json",
+            },
+            method: "POST",
+        });
+        if (!response.ok) {
+            const text = await response.text().catch(() => "");
+            throw new Error(`OpenAI Chat Completions API returned HTTP ${response.status}: ${text.slice(0, 200)}`);
+        }
+        const data = (await response.json());
+        if (data.error) {
+            throw new Error(data.error.message ?? "Unknown OpenAI Chat Completions error");
+        }
+        promptTokens += data.usage?.prompt_tokens ?? 0;
+        completionTokens += data.usage?.completion_tokens ?? 0;
+        totalTokens += data.usage?.total_tokens ?? 0;
+        const assistantMessage = data.choices?.[0]?.message;
+        const finishReason = data.choices?.[0]?.finish_reason;
+        if (!assistantMessage) {
+            return {
+                output: "",
+                toolCallLog,
+                tokenUsage: {
+                    prompt: promptTokens,
+                    completion: completionTokens,
+                    total: totalTokens,
+                },
+                toolRounds: round,
+                latencyMs: Date.now() - startTime,
+            };
+        }
+        messages.push(assistantMessage);
+        // If the model didn't call tools, it's done
+        if (finishReason !== "tool_calls" || !assistantMessage.tool_calls?.length) {
+            return {
+                output: assistantMessage.content ?? "",
+                toolCallLog,
+                tokenUsage: {
+                    prompt: promptTokens,
+                    completion: completionTokens,
+                    total: totalTokens,
+                },
+                toolRounds: round,
+                latencyMs: Date.now() - startTime,
+            };
+        }
+        // Execute each tool call via MCP
+        for (const toolCall of assistantMessage.tool_calls) {
+            const toolName = toolCall.function.name;
+            let toolInput;
+            try {
+                toolInput = JSON.parse(toolCall.function.arguments);
+            }
+            catch {
+                toolInput = { _raw: toolCall.function.arguments };
+            }
+            try {
+                const result = await callTool(toolName, toolInput);
+                const content = result.error
+                    ? JSON.stringify({ error: result.error })
+                    : result.content;
+                toolCallLog.push({ name: toolName, input: toolInput, output: content });
+                messages.push({
+                    content,
+                    role: "tool",
+                    tool_call_id: toolCall.id,
+                });
+            }
+            catch (err) {
+                const errMsg = err instanceof Error ? err.message : String(err);
+                toolCallLog.push({
+                    name: toolName,
+                    input: toolInput,
+                    output: `Error: ${errMsg}`,
+                });
+                messages.push({
+                    content: JSON.stringify({ error: errMsg }),
+                    role: "tool",
+                    tool_call_id: toolCall.id,
+                });
+            }
+        }
+    }
+    // Exhausted rounds — extract last assistant text
+    const lastAssistant = messages
+        .filter((m) => m.role === "assistant" && m.content)
+        .pop();
+    return {
+        output: lastAssistant?.content ?? "[Exhausted tool rounds without final answer]",
+        toolCallLog,
+        tokenUsage: {
+            prompt: promptTokens,
+            completion: completionTokens,
+            total: totalTokens,
+        },
+        toolRounds: maxToolRounds,
+        exhaustedRounds: true,
+        latencyMs: Date.now() - startTime,
+    };
+}
+// ---------------------------------------------------------------------------
+// Responses API tool loop
+// ---------------------------------------------------------------------------
+async function runResponsesLoop(config) {
+    const { prompt, tools, callTool, maxToolRounds, model, temperature, maxTokens, apiKey, providerConfig, } = config;
+    const openaiTools = toOpenAIResponsesTools(tools);
+    // Input for the current round — replaced each iteration with tool outputs
+    let input = [
+        { type: "message", role: "system", content: SYSTEM_PROMPT },
+        { type: "message", role: "user", content: prompt },
+    ];
+    let inputTokens = 0;
+    let outputTokens = 0;
+    let totalTokens = 0;
+    const startTime = Date.now();
+    const toolCallLog = [];
+    // Track the previous response ID for conversation chaining
+    let previousResponseId;
+    // Pass through reasoning_effort if configured
+    const reasoningEffort = providerConfig?.reasoning_effort;
+    for (let round = 0; round <= maxToolRounds; round++) {
+        const isLastRound = round === maxToolRounds;
+        const body = {
+            model,
+            max_output_tokens: maxTokens,
+            tools: openaiTools,
+            tool_choice: isLastRound ? "none" : "auto",
+        };
+        // Reasoning models (GPT-5.x, o-series) use reasoning.effort instead of
+        // temperature — the Responses API rejects temperature when reasoning is set.
+        if (reasoningEffort) {
+            body.reasoning = { effort: reasoningEffort };
+        }
+        else {
+            body.temperature = temperature;
+        }
+        body.input = input;
+        if (previousResponseId) {
+            body.previous_response_id = previousResponseId;
+        }
+        const response = await fetch("https://api.openai.com/v1/responses", {
+            body: JSON.stringify(body),
+            headers: {
+                Authorization: `Bearer ${apiKey}`,
+                "Content-Type": "application/json",
+            },
+            method: "POST",
+        });
+        if (!response.ok) {
+            const text = await response.text().catch(() => "");
+            throw new Error(`OpenAI Responses API returned HTTP ${response.status}: ${text.slice(0, 200)}`);
+        }
+        const data = (await response.json());
+        if (data.error) {
+            throw new Error(data.error.message ?? "Unknown OpenAI Responses API error");
+        }
+        inputTokens += data.usage?.input_tokens ?? 0;
+        outputTokens += data.usage?.output_tokens ?? 0;
+        totalTokens += data.usage?.total_tokens ?? 0;
+        previousResponseId = data.id;
+        // Extract function calls and message outputs
+        const functionCalls = data.output.filter((item) => item.type === "function_call");
+        const messageItems = data.output.filter((item) => item.type === "message");
+        // If no function calls, model is done — extract text
+        if (functionCalls.length === 0) {
+            const output = messageItems
+                .flatMap((m) => m.content)
+                .filter((c) => c.type === "output_text")
+                .map((c) => c.text)
+                .join("\n") || "";
+            return {
+                output,
+                toolCallLog,
+                tokenUsage: {
+                    prompt: inputTokens,
+                    completion: outputTokens,
+                    total: totalTokens,
+                },
+                toolRounds: round,
+                latencyMs: Date.now() - startTime,
+            };
+        }
+        // Execute each function call via MCP and build tool output items for next request
+        const toolOutputItems = [];
+        for (const fc of functionCalls) {
+            const toolName = fc.name;
+            let toolInput;
+            try {
+                toolInput = JSON.parse(fc.arguments);
+            }
+            catch {
+                toolInput = { _raw: fc.arguments };
+            }
+            try {
+                const result = await callTool(toolName, toolInput);
+                const content = result.error
+                    ? JSON.stringify({ error: result.error })
+                    : result.content;
+                toolCallLog.push({ name: toolName, input: toolInput, output: content });
+                toolOutputItems.push({
+                    type: "function_call_output",
+                    call_id: fc.call_id,
+                    output: content,
+                });
+            }
+            catch (err) {
+                const errMsg = err instanceof Error ? err.message : String(err);
+                toolCallLog.push({
+                    name: toolName,
+                    input: toolInput,
+                    output: `Error: ${errMsg}`,
+                });
+                toolOutputItems.push({
+                    type: "function_call_output",
+                    call_id: fc.call_id,
+                    output: JSON.stringify({ error: errMsg }),
+                });
+            }
+        }
+        // For the next round, send tool outputs as input (chained via previous_response_id)
+        input = toolOutputItems;
+    }
+    // Exhausted rounds — no final text available
+    return {
+        output: "[Exhausted tool rounds without final answer]",
+        toolCallLog,
+        tokenUsage: {
+            prompt: inputTokens,
+            completion: outputTokens,
+            total: totalTokens,
+        },
+        toolRounds: maxToolRounds,
+        exhaustedRounds: true,
+        latencyMs: Date.now() - startTime,
+    };
+}
+// ---------------------------------------------------------------------------
+// Public entry point
+// ---------------------------------------------------------------------------
+/**
+ * Run a multi-turn tool loop using the OpenAI API.
+ *
+ * Routes to Chat Completions or Responses API based on `config.apiVariant`:
+ * - `"responses"` → Responses API (`/v1/responses`)
+ * - `"chat"` or undefined → Chat Completions API (`/v1/chat/completions`)
+ */
+export async function runOpenAIToolLoop(config) {
+    return config.apiVariant === "responses"
+        ? runResponsesLoop(config)
+        : runChatCompletionsLoop(config);
+}

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/types.d.ts

@@ -0,0 +1,103 @@
+/**
+ * Shared types for the MCP tool provider.
+ */
+export interface CallApiContextParams {
+    prompt?: {
+        raw: string;
+        label?: string;
+    };
+    vars?: Record<string, object | string>;
+}
+export interface ProviderOptions {
+    config?: Record<string, unknown>;
+    id?: string;
+}
+export interface ProviderResponse {
+    cached?: boolean;
+    cost?: number;
+    error?: string;
+    metadata?: Record<string, unknown>;
+    output?: object | string;
+    tokenUsage?: {
+        total?: number;
+        prompt?: number;
+        completion?: number;
+        cached?: number;
+    };
+}
+export interface MCPTool {
+    name: string;
+    description?: string;
+    inputSchema?: Record<string, unknown>;
+}
+export interface MCPClient {
+    getAllTools: () => MCPTool[];
+    callTool: (name: string, args: Record<string, unknown>) => Promise<{
+        content: string;
+        error?: string;
+    }>;
+    cleanup: () => Promise<void>;
+}
+export interface ToolCallEntry {
+    name: string;
+    input: unknown;
+    output: string;
+}
+/** Configuration passed from the provider to an LLM backend's tool loop */
+export interface ToolLoopConfig {
+    /** The user's prompt */
+    prompt: string;
+    /** MCP tools available (already filtered by capabilities) */
+    tools: MCPTool[];
+    /** Function to call an MCP tool by name */
+    callTool: (name: string, args: Record<string, unknown>) => Promise<{
+        content: string;
+        error?: string;
+    }>;
+    /** Max rounds of tool calls before forcing a final answer */
+    maxToolRounds: number;
+    /** LLM model name (provider-specific, e.g., "claude-opus-4-6") */
+    model: string;
+    /** Sampling temperature */
+    temperature: number;
+    /** Max output tokens */
+    maxTokens: number;
+    /** API key for the LLM provider */
+    apiKey: string;
+    /**
+     * OpenAI API variant — determines which endpoint to call.
+     * - `"chat"` → `/v1/chat/completions` (default)
+     * - `"responses"` → `/v1/responses`
+     *
+     * Extracted from model ID: `openai:responses:gpt-5.4` → `"responses"`
+     */
+    apiVariant?: "chat" | "responses";
+    /** Additional model-specific config passed through from the provider */
+    providerConfig?: Record<string, unknown>;
+}
+/** Result returned from an LLM backend's tool loop */
+export interface ToolLoopResult {
+    /** The LLM's final text output */
+    output: string;
+    /** Log of all tool calls made during the loop */
+    toolCallLog: ToolCallEntry[];
+    /** Token usage stats */
+    tokenUsage: {
+        prompt: number;
+        completion: number;
+        total: number;
+    };
+    /** Number of tool rounds completed */
+    toolRounds: number;
+    /** Whether the loop exhausted maxToolRounds */
+    exhaustedRounds?: boolean;
+    /** Wall-clock duration in ms */
+    latencyMs: number;
+}
+/**
+ * An LLM backend that can run a multi-turn tool loop.
+ *
+ * Each backend (Anthropic, OpenAI) implements this function signature.
+ * The provider selects the appropriate backend based on the model ID prefix.
+ */
+export type RunToolLoop = (config: ToolLoopConfig) => Promise<ToolLoopResult>;

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider/types.js

@@ -0,0 +1,4 @@
+/**
+ * Shared types for the MCP tool provider.
+ */
+export {};

package/dist/pipeline/compiler/mode-handlers/mcp-tool-provider.d.ts

@@ -0,0 +1,65 @@
+/**
+ * mcp-tool-provider.ts — Custom Promptfoo provider for MCP tool-use evaluation.
+ *
+ * Implements a multi-turn tool execution loop: the LLM receives a prompt,
+ * discovers MCP tools, calls them, gets results, and continues until it
+ * produces a final text answer or exhausts maxToolRounds.
+ *
+ * Promptfoo's built-in Anthropic/OpenAI providers with config.mcp only do
+ * single-turn tool calls. This provider fills that gap by managing the
+ * full conversation loop, similar to the agentic-provider.ts pattern.
+ *
+ * Promptfoo config usage:
+ *
+ *   providers:
+ *     - id: file://dist/pipeline/compiler/mode-handlers/mcp-tool-provider.js
+ *       label: "Claude Opus 4.6 + MCP"
+ *       config:
+ *         model: anthropic:messages:claude-opus-4-6
+ *         maxToolRounds: 5
+ *         temperature: 0.2
+ *         max_tokens: 4096
+ *         mcpServer:
+ *           url: https://mcp.sanity.io
+ *           auth: { type: bearer, token: "{{env.SANITY_API_TOKEN}}" }
+ *           name: mcp-live-query-documents
+ *         mcpTools: [query_documents, get_schema]
+ */
+interface CallApiContextParams {
+    prompt?: {
+        raw: string;
+        label?: string;
+    };
+    vars?: Record<string, object | string>;
+}
+interface ProviderOptions {
+    config?: Record<string, unknown>;
+    id?: string;
+}
+interface ProviderResponse {
+    cached?: boolean;
+    cost?: number;
+    error?: string;
+    metadata?: Record<string, unknown>;
+    output?: object | string;
+    tokenUsage?: {
+        total?: number;
+        prompt?: number;
+        completion?: number;
+        cached?: number;
+    };
+}
+export default class MCPToolProvider {
+    config: Record<string, unknown>;
+    private providerId;
+    constructor(options?: ProviderOptions);
+    id(): string;
+    callApi(prompt: string, _context?: CallApiContextParams): Promise<ProviderResponse>;
+    private runAnthropicLoop;
+    private connectMCP;
+    /**
+     * Resolve {{env.VAR}} templates in config values.
+     */
+    private resolveEnvTemplates;
+}
+export {};