@sanity/ailf 3.8.0 → 3.9.0

package/dist/pipeline/compiler/__tests__/tool-loop-openai.test.js

@@ -1,509 +0,0 @@
-/**
- * tool-loop-openai.test.ts — Tests for the OpenAI MCP tool loop.
- *
- * Tests both API variants (Chat Completions and Responses) with mocked
- * fetch to verify tool calling, error handling, token tracking, and
- * round exhaustion.
- *
- * Run: npx tsx --test src/pipeline/compiler/__tests__/tool-loop-openai.test.ts
- */
-import assert from "node:assert/strict";
-import { afterEach, beforeEach, describe, it } from "node:test";
-import { runOpenAIToolLoop } from "../mode-handlers/mcp-tool-provider/tool-loop-openai.js";
-// ---------------------------------------------------------------------------
-// Test fixtures
-// ---------------------------------------------------------------------------
-const TEST_TOOLS = [
-    {
-        name: "query_documents",
-        description: "Query Sanity documents using GROQ",
-        inputSchema: {
-            type: "object",
-            properties: {
-                query: { type: "string" },
-                projectId: { type: "string" },
-                dataset: { type: "string" },
-            },
-            required: ["query"],
-        },
-    },
-    {
-        name: "get_schema",
-        description: "Get the Sanity schema for a project",
-        inputSchema: {
-            type: "object",
-            properties: { projectId: { type: "string" } },
-        },
-    },
-];
-function makeCallTool(results = {}) {
-    return async (name, _args) => {
-        if (results[name]) {
-            return { content: results[name] };
-        }
-        return { content: `Result from ${name}` };
-    };
-}
-function makeThrowingCallTool(errorMessage) {
-    return async () => {
-        throw new Error(errorMessage);
-    };
-}
-function baseConfig(overrides) {
-    return {
-        prompt: "Query all documents from project abc123",
-        tools: TEST_TOOLS,
-        callTool: makeCallTool(),
-        maxToolRounds: 5,
-        model: "gpt-5.2",
-        temperature: 0.2,
-        maxTokens: 4096,
-        apiKey: "test-api-key",
-        ...overrides,
-    };
-}
-// ---------------------------------------------------------------------------
-// Chat Completions API response builders
-// ---------------------------------------------------------------------------
-function chatResponse(opts) {
-    const message = {
-        role: "assistant",
-        content: opts.content ?? null,
-    };
-    if (opts.toolCalls) {
-        message.tool_calls = opts.toolCalls.map((tc) => ({
-            id: tc.id,
-            type: "function",
-            function: { name: tc.name, arguments: tc.arguments },
-        }));
-    }
-    return {
-        choices: [
-            {
-                message,
-                finish_reason: opts.finishReason ?? (opts.toolCalls ? "tool_calls" : "stop"),
-            },
-        ],
-        usage: {
-            prompt_tokens: opts.promptTokens ?? 100,
-            completion_tokens: opts.completionTokens ?? 50,
-            total_tokens: (opts.promptTokens ?? 100) + (opts.completionTokens ?? 50),
-        },
-    };
-}
-function chatErrorResponse(message) {
-    return { error: { message } };
-}
-// ---------------------------------------------------------------------------
-// Responses API response builders
-// ---------------------------------------------------------------------------
-function responsesResponse(opts) {
-    const output = [];
-    if (opts.functionCalls) {
-        for (const fc of opts.functionCalls) {
-            output.push({
-                type: "function_call",
-                id: `fc_${fc.callId}`,
-                call_id: fc.callId,
-                name: fc.name,
-                arguments: fc.arguments,
-            });
-        }
-    }
-    if (opts.text !== undefined) {
-        output.push({
-            type: "message",
-            id: "msg_001",
-            role: "assistant",
-            content: [{ type: "output_text", text: opts.text }],
-        });
-    }
-    return {
-        id: opts.id ?? "resp_001",
-        status: "completed",
-        output,
-        usage: {
-            input_tokens: opts.inputTokens ?? 100,
-            output_tokens: opts.outputTokens ?? 50,
-            total_tokens: (opts.inputTokens ?? 100) + (opts.outputTokens ?? 50),
-        },
-    };
-}
-function responsesErrorResponse(message) {
-    return { error: { message } };
-}
-// ---------------------------------------------------------------------------
-// Mock fetch helper
-// ---------------------------------------------------------------------------
-let originalFetch;
-let fetchCalls;
-/** Mock fetch that returns JSON responses. All responses have ok: true (API-level errors are in the body). */
-function mockFetch(responses) {
-    let callIndex = 0;
-    fetchCalls = [];
-    globalThis.fetch = (async (url, init) => {
-        const body = init?.body ? JSON.parse(String(init.body)) : undefined;
-        fetchCalls.push({ url: url.toString(), body });
-        const responseData = responses[callIndex] ?? responses[responses.length - 1];
-        callIndex++;
-        return {
-            json: async () => responseData,
-            ok: true,
-            status: 200,
-        };
-    });
-}
-/** Mock fetch that returns an HTTP error (non-JSON body) */
-function mockHttpError(status, body) {
-    fetchCalls = [];
-    globalThis.fetch = (async (url, init) => {
-        const reqBody = init?.body ? JSON.parse(String(init.body)) : undefined;
-        fetchCalls.push({ url: url.toString(), body: reqBody });
-        return new Response(body, { status, statusText: "Error" });
-    });
-}
-// ---------------------------------------------------------------------------
-// Tests: Chat Completions API
-// ---------------------------------------------------------------------------
-describe("runOpenAIToolLoop — Chat Completions API", () => {
-    beforeEach(() => {
-        originalFetch = globalThis.fetch;
-    });
-    afterEach(() => {
-        globalThis.fetch = originalFetch;
-    });
-    it("returns text when model answers without tool calls", async () => {
-        mockFetch([
-            chatResponse({
-                content: "There are 42 documents.",
-                finishReason: "stop",
-            }),
-        ]);
-        const result = await runOpenAIToolLoop(baseConfig({ apiVariant: "chat" }));
-        assert.equal(result.output, "There are 42 documents.");
-        assert.equal(result.toolCallLog.length, 0);
-        assert.equal(result.toolRounds, 0);
-        assert.equal(result.exhaustedRounds, undefined);
-    });
-    it("executes a single tool call and returns final answer", async () => {
-        mockFetch([
-            // Round 0: model calls query_documents
-            chatResponse({
-                toolCalls: [
-                    {
-                        id: "call_1",
-                        name: "query_documents",
-                        arguments: '{"query":"*[_type==\\"post\\"]"}',
-                    },
-                ],
-            }),
-            // Round 1: model synthesizes answer
-            chatResponse({ content: "Found 10 posts.", finishReason: "stop" }),
-        ]);
-        const result = await runOpenAIToolLoop(baseConfig({ apiVariant: "chat" }));
-        assert.equal(result.output, "Found 10 posts.");
-        assert.equal(result.toolCallLog.length, 1);
-        assert.equal(result.toolCallLog[0].name, "query_documents");
-        assert.equal(result.toolRounds, 1);
-    });
-    it("executes multi-turn tool calls", async () => {
-        mockFetch([
-            // Round 0: model calls get_schema
-            chatResponse({
-                toolCalls: [
-                    {
-                        id: "call_1",
-                        name: "get_schema",
-                        arguments: '{"projectId":"abc123"}',
-                    },
-                ],
-            }),
-            // Round 1: model calls query_documents
-            chatResponse({
-                toolCalls: [
-                    { id: "call_2", name: "query_documents", arguments: '{"query":"*"}' },
-                ],
-            }),
-            // Round 2: model synthesizes
-            chatResponse({
-                content: "Schema has 5 types, 100 documents.",
-                finishReason: "stop",
-            }),
-        ]);
-        const result = await runOpenAIToolLoop(baseConfig({ apiVariant: "chat" }));
-        assert.equal(result.toolCallLog.length, 2);
-        assert.equal(result.toolCallLog[0].name, "get_schema");
-        assert.equal(result.toolCallLog[1].name, "query_documents");
-        assert.equal(result.toolRounds, 2);
-    });
-    it("captures tool execution errors in toolCallLog", async () => {
-        mockFetch([
-            chatResponse({
-                toolCalls: [
-                    { id: "call_1", name: "query_documents", arguments: '{"query":"*"}' },
-                ],
-            }),
-            chatResponse({
-                content: "Tool failed, but I'll answer.",
-                finishReason: "stop",
-            }),
-        ]);
-        const result = await runOpenAIToolLoop(baseConfig({
-            apiVariant: "chat",
-            callTool: makeThrowingCallTool("Connection refused"),
-        }));
-        assert.equal(result.toolCallLog.length, 1);
-        assert.equal(result.toolCallLog[0].output, "Error: Connection refused");
-        assert.equal(result.output, "Tool failed, but I'll answer.");
-    });
-    it("handles exhausted rounds", async () => {
-        // Model keeps calling tools for all 3 rounds (maxToolRounds=2 means rounds 0,1,2)
-        mockFetch([
-            chatResponse({
-                toolCalls: [{ id: "call_1", name: "get_schema", arguments: "{}" }],
-            }),
-            chatResponse({
-                toolCalls: [{ id: "call_2", name: "get_schema", arguments: "{}" }],
-            }),
-            // Last round: tool_choice "none" forces text, but model returns nothing useful
-            chatResponse({ content: null, finishReason: "stop" }),
-        ]);
-        const result = await runOpenAIToolLoop(baseConfig({ apiVariant: "chat", maxToolRounds: 2 }));
-        // Round 2 (the last) gets tool_choice: "none", model stops
-        assert.equal(result.toolCallLog.length, 2);
-        assert.equal(result.toolRounds, 2);
-        // The model returned content: null with finishReason: stop on the last round
-        assert.equal(result.output, "");
-    });
-    it("throws on API-level error in JSON body", async () => {
-        mockFetch([chatErrorResponse("Rate limit exceeded")]);
-        await assert.rejects(() => runOpenAIToolLoop(baseConfig({ apiVariant: "chat" })), { message: "Rate limit exceeded" });
-    });
-    it("throws on HTTP error with non-JSON body", async () => {
-        mockHttpError(502, "<html>Bad Gateway</html>");
-        await assert.rejects(() => runOpenAIToolLoop(baseConfig({ apiVariant: "chat" })), (err) => err.message.includes("HTTP 502") && err.message.includes("Bad Gateway"));
-    });
-    it("accumulates token usage across rounds", async () => {
-        mockFetch([
-            chatResponse({
-                toolCalls: [{ id: "call_1", name: "get_schema", arguments: "{}" }],
-                promptTokens: 200,
-                completionTokens: 50,
-            }),
-            chatResponse({
-                content: "Done.",
-                finishReason: "stop",
-                promptTokens: 300,
-                completionTokens: 80,
-            }),
-        ]);
-        const result = await runOpenAIToolLoop(baseConfig({ apiVariant: "chat" }));
-        assert.equal(result.tokenUsage.prompt, 500); // 200 + 300
-        assert.equal(result.tokenUsage.completion, 130); // 50 + 80
-    });
-    it("sends max_completion_tokens for GPT-5.x models", async () => {
-        mockFetch([chatResponse({ content: "Answer.", finishReason: "stop" })]);
-        await runOpenAIToolLoop(baseConfig({ apiVariant: "chat", model: "gpt-5.2" }));
-        assert.equal(fetchCalls.length, 1);
-        const body = fetchCalls[0].body;
-        assert.equal(body.max_completion_tokens, 4096);
-        assert.equal(body.max_tokens, undefined);
-    });
-    it("sends max_tokens for older models", async () => {
-        mockFetch([chatResponse({ content: "Answer.", finishReason: "stop" })]);
-        await runOpenAIToolLoop(baseConfig({ apiVariant: "chat", model: "gpt-4o" }));
-        assert.equal(fetchCalls.length, 1);
-        const body = fetchCalls[0].body;
-        assert.equal(body.max_tokens, 4096);
-        assert.equal(body.max_completion_tokens, undefined);
-    });
-    it("sends tool_choice 'none' on last round", async () => {
-        mockFetch([
-            chatResponse({
-                toolCalls: [{ id: "call_1", name: "get_schema", arguments: "{}" }],
-            }),
-            chatResponse({ content: "Final.", finishReason: "stop" }),
-        ]);
-        await runOpenAIToolLoop(baseConfig({ apiVariant: "chat", maxToolRounds: 1 }));
-        // Round 0: auto, Round 1 (last): none
-        assert.equal(fetchCalls.length, 2);
-        assert.equal(fetchCalls[0].body.tool_choice, "auto");
-        assert.equal(fetchCalls[1].body.tool_choice, "none");
-    });
-});
-// ---------------------------------------------------------------------------
-// Tests: Responses API
-// ---------------------------------------------------------------------------
-describe("runOpenAIToolLoop — Responses API", () => {
-    beforeEach(() => {
-        originalFetch = globalThis.fetch;
-    });
-    afterEach(() => {
-        globalThis.fetch = originalFetch;
-    });
-    it("returns text when model answers without tool calls", async () => {
-        mockFetch([responsesResponse({ text: "42 documents found." })]);
-        const result = await runOpenAIToolLoop(baseConfig({ apiVariant: "responses" }));
-        assert.equal(result.output, "42 documents found.");
-        assert.equal(result.toolCallLog.length, 0);
-        assert.equal(result.toolRounds, 0);
-    });
-    it("executes a single tool call and returns final answer", async () => {
-        mockFetch([
-            // Round 0: model calls query_documents
-            responsesResponse({
-                id: "resp_001",
-                functionCalls: [
-                    {
-                        callId: "call_1",
-                        name: "query_documents",
-                        arguments: '{"query":"*"}',
-                    },
-                ],
-            }),
-            // Round 1: model synthesizes
-            responsesResponse({ id: "resp_002", text: "Found 10 posts." }),
-        ]);
-        const result = await runOpenAIToolLoop(baseConfig({ apiVariant: "responses" }));
-        assert.equal(result.output, "Found 10 posts.");
-        assert.equal(result.toolCallLog.length, 1);
-        assert.equal(result.toolCallLog[0].name, "query_documents");
-        assert.equal(result.toolRounds, 1);
-    });
-    it("chains via previous_response_id", async () => {
-        mockFetch([
-            responsesResponse({
-                id: "resp_001",
-                functionCalls: [
-                    { callId: "call_1", name: "get_schema", arguments: "{}" },
-                ],
-            }),
-            responsesResponse({ id: "resp_002", text: "Schema loaded." }),
-        ]);
-        await runOpenAIToolLoop(baseConfig({ apiVariant: "responses" }));
-        // Second request should chain via previous_response_id
-        assert.equal(fetchCalls.length, 2);
-        const secondBody = fetchCalls[1].body;
-        assert.equal(secondBody.previous_response_id, "resp_001");
-    });
-    it("captures tool execution errors in toolCallLog", async () => {
-        mockFetch([
-            responsesResponse({
-                functionCalls: [
-                    {
-                        callId: "call_1",
-                        name: "query_documents",
-                        arguments: '{"query":"*"}',
-                    },
-                ],
-            }),
-            responsesResponse({ text: "Handled the error." }),
-        ]);
-        const result = await runOpenAIToolLoop(baseConfig({
-            apiVariant: "responses",
-            callTool: makeThrowingCallTool("Server unavailable"),
-        }));
-        assert.equal(result.toolCallLog.length, 1);
-        assert.equal(result.toolCallLog[0].output, "Error: Server unavailable");
-        assert.equal(result.output, "Handled the error.");
-    });
-    it("handles exhausted rounds", async () => {
-        mockFetch([
-            responsesResponse({
-                id: "resp_001",
-                functionCalls: [
-                    { callId: "call_1", name: "get_schema", arguments: "{}" },
-                ],
-            }),
-            responsesResponse({
-                id: "resp_002",
-                functionCalls: [
-                    { callId: "call_2", name: "get_schema", arguments: "{}" },
-                ],
-            }),
-            // Last round with tool_choice: "none" — model must return text
-            // But if it doesn't produce function calls, we get empty output
-            responsesResponse({ id: "resp_003", text: "" }),
-        ]);
-        const result = await runOpenAIToolLoop(baseConfig({ apiVariant: "responses", maxToolRounds: 2 }));
-        assert.equal(result.toolCallLog.length, 2);
-        assert.equal(result.toolRounds, 2);
-    });
-    it("throws on API-level error in JSON body", async () => {
-        mockFetch([responsesErrorResponse("Invalid model")]);
-        await assert.rejects(() => runOpenAIToolLoop(baseConfig({ apiVariant: "responses" })), { message: "Invalid model" });
-    });
-    it("throws on HTTP error with non-JSON body", async () => {
-        mockHttpError(503, "Service Unavailable");
-        await assert.rejects(() => runOpenAIToolLoop(baseConfig({ apiVariant: "responses" })), (err) => err.message.includes("HTTP 503") &&
-            err.message.includes("Service Unavailable"));
-    });
-    it("accumulates token usage across rounds", async () => {
-        mockFetch([
-            responsesResponse({
-                functionCalls: [
-                    { callId: "call_1", name: "get_schema", arguments: "{}" },
-                ],
-                inputTokens: 150,
-                outputTokens: 40,
-            }),
-            responsesResponse({
-                text: "Done.",
-                inputTokens: 250,
-                outputTokens: 60,
-            }),
-        ]);
-        const result = await runOpenAIToolLoop(baseConfig({ apiVariant: "responses" }));
-        assert.equal(result.tokenUsage.prompt, 400); // 150 + 250
-        assert.equal(result.tokenUsage.completion, 100); // 40 + 60
-    });
-    it("sends max_output_tokens (not max_tokens)", async () => {
-        mockFetch([responsesResponse({ text: "Answer." })]);
-        await runOpenAIToolLoop(baseConfig({ apiVariant: "responses" }));
-        const body = fetchCalls[0].body;
-        assert.equal(body.max_output_tokens, 4096);
-        assert.equal(body.max_tokens, undefined);
-        assert.equal(body.max_completion_tokens, undefined);
-    });
-    it("uses correct endpoint URL", async () => {
-        mockFetch([responsesResponse({ text: "Hi." })]);
-        await runOpenAIToolLoop(baseConfig({ apiVariant: "responses" }));
-        assert.ok(fetchCalls[0].url.includes("/v1/responses"));
-    });
-    it("passes reasoning_effort and omits temperature", async () => {
-        mockFetch([responsesResponse({ text: "Thought carefully." })]);
-        await runOpenAIToolLoop(baseConfig({
-            apiVariant: "responses",
-            providerConfig: { reasoning_effort: "medium" },
-        }));
-        const body = fetchCalls[0].body;
-        assert.deepEqual(body.reasoning, { effort: "medium" });
-        assert.equal(body.temperature, undefined);
-    });
-    it("includes temperature when reasoning_effort is not set", async () => {
-        mockFetch([responsesResponse({ text: "Answer." })]);
-        await runOpenAIToolLoop(baseConfig({ apiVariant: "responses" }));
-        const body = fetchCalls[0].body;
-        assert.equal(body.temperature, 0.2);
-        assert.equal(body.reasoning, undefined);
-    });
-});
-// ---------------------------------------------------------------------------
-// Tests: Default routing (no apiVariant)
-// ---------------------------------------------------------------------------
-describe("runOpenAIToolLoop — default routing", () => {
-    beforeEach(() => {
-        originalFetch = globalThis.fetch;
-    });
-    afterEach(() => {
-        globalThis.fetch = originalFetch;
-    });
-    it("defaults to Chat Completions when apiVariant is undefined", async () => {
-        mockFetch([
-            chatResponse({ content: "Default path.", finishReason: "stop" }),
-        ]);
-        const result = await runOpenAIToolLoop(baseConfig());
-        assert.equal(result.output, "Default path.");
-        assert.ok(fetchCalls[0].url.includes("/v1/chat/completions"));
-    });
-});