@draht/ai 2026.7.13-1 → 2026.7.30

package/dist/api/openai-completions.js

@@ -2,10 +2,13 @@ import OpenAI from "openai";
 import { calculateCost, clampThinkingLevel } from "../models.js";
 import { formatProviderError, normalizeProviderError } from "../utils/error-body.js";
 import { AssistantMessageEventStream } from "../utils/event-stream.js";
+import { shortHash } from "../utils/hash.js";
 import { headersToRecord } from "../utils/headers.js";
 import { parseStreamingJson } from "../utils/json-parse.js";
 import { getProviderEnvValue } from "../utils/provider-env.js";
+import { retryProviderRequest } from "../utils/provider-retry.js";
 import { sanitizeSurrogates } from "../utils/sanitize-unicode.js";
+import { appendGrammarToolInputJsonDelta, createGrammarToolInputProperties, getGrammarToolInput, resolveGrammarConstrainedSampling, resolveJsonSchemaStrictSampling, } from "./constrained-sampling.js";
 import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.js";
 import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.js";
 import { buildBaseOptions } from "./simple-options.js";
@@ -45,6 +48,25 @@ function hasToolHistory(messages) {
     }
     return false;
 }
+function getDeferredToolNames(messages) {
+    const names = new Set();
+    for (const message of messages) {
+        if (message.role === "toolResult") {
+            for (const name of message.addedToolNames ?? []) {
+                names.add(name);
+            }
+        }
+    }
+    return names;
+}
+function getToolsByName(tools, names) {
+    if (!tools)
+        return [];
+    const toolsByName = new Map(tools.map((tool) => [tool.name, tool]));
+    return Array.from(names)
+        .map((name) => toolsByName.get(name))
+        .filter((tool) => tool !== undefined);
+}
 function isTextContentBlock(block) {
     return block.type === "text";
 }
@@ -94,16 +116,17 @@ export const stream = (model, context, options) => {
                 totalTokens: 0,
                 cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
             },
-            stopReason: "stop",
+            stopReason: "pending",
             timestamp: Date.now(),
         };
         try {
             const apiKey = getClientApiKey(model.provider, options?.apiKey, options?.headers);
             const compat = getCompat(model);
+            const grammarToolInputProperties = createGrammarToolInputProperties(context.tools, compat.supportsOpenAIGrammarTools);
             const cacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env);
             const cacheSessionId = cacheRetention === "none" ? undefined : options?.sessionId;
-            const client = createClient(model, context, apiKey, options?.headers, cacheSessionId, compat);
-            let params = buildParams(model, context, options, compat, cacheRetention);
+            const client = createClient(model, context, apiKey, options?.headers, options?.fetch, cacheSessionId, compat);
+            let params = buildParams(model, context, options, compat, cacheRetention, grammarToolInputProperties);
             const nextParams = await options?.onPayload?.(params, model);
             if (nextParams !== undefined) {
                 params = nextParams;
@@ -111,11 +134,13 @@ export const stream = (model, context, options) => {
             const requestOptions = {
                 ...(options?.signal ? { signal: options.signal } : {}),
                 ...(options?.timeoutMs !== undefined ? { timeout: options.timeoutMs } : {}),
-                maxRetries: options?.maxRetries ?? 0,
+                maxRetries: 0,
             };
-            const { data: openaiStream, response } = await client.chat.completions
-                .create(params, requestOptions)
-                .withResponse();
+            const { data: openaiStream, response } = await retryProviderRequest(() => client.chat.completions.create(params, requestOptions).withResponse(), {
+                maxRetries: options?.maxRetries,
+                maxRetryDelayMs: options?.maxRetryDelayMs,
+                signal: options?.signal,
+            });
             await options?.onResponse?.({ status: response.status, headers: headersToRecord(response.headers) }, model);
             stream.push({ type: "start", partial: output });
             let textBlock = null;
@@ -126,6 +151,21 @@ export const stream = (model, context, options) => {
             const pendingReasoningDetailsByToolCallId = new Map();
             const blocks = output.content;
             const getContentIndex = (block) => blocks.indexOf(block);
+            const getCustomToolCallInput = (block) => {
+                const property = block.customInput?.property;
+                if (property === undefined)
+                    return "";
+                const value = block.arguments[property];
+                return typeof value === "string" ? value : "";
+            };
+            const appendCustomToolCallInput = (block, nextInput, close) => {
+                const customInput = block.customInput;
+                if (!customInput)
+                    return undefined;
+                const delta = appendGrammarToolInputJsonDelta(customInput.jsonBuffer, customInput.property, nextInput, close);
+                block.arguments = { [customInput.property]: nextInput };
+                return delta;
+            };
             const finishBlock = (block) => {
                 const contentIndex = getContentIndex(block);
                 if (contentIndex === -1) {
@@ -148,10 +188,24 @@ export const stream = (model, context, options) => {
                     });
                 }
                 else if (block.type === "toolCall") {
-                    block.arguments = parseStreamingJson(block.partialArgs);
+                    if (block.customInput) {
+                        const delta = appendCustomToolCallInput(block, getCustomToolCallInput(block), true);
+                        if (delta !== undefined) {
+                            stream.push({
+                                type: "toolcall_delta",
+                                contentIndex,
+                                delta,
+                                partial: output,
+                            });
+                        }
+                    }
+                    else {
+                        block.arguments = parseStreamingJson(block.partialArgs);
+                    }
                     // Finalize in-place and strip the scratch buffers so replay only
                     // carries parsed arguments.
                     delete block.partialArgs;
+                    delete block.customInput;
                     delete block.streamIndex;
                     stream.push({
                         type: "toolcall_end",
@@ -193,17 +247,25 @@ export const stream = (model, context, options) => {
             };
             const ensureToolCallBlock = (toolCall) => {
                 const streamIndex = typeof toolCall.index === "number" ? toolCall.index : undefined;
+                const name = toolCall.function?.name ?? toolCall.custom?.name ?? "";
                 let block = streamIndex !== undefined ? toolCallBlocksByIndex.get(streamIndex) : undefined;
                 if (!block && toolCall.id) {
                     block = toolCallBlocksById.get(toolCall.id);
                 }
                 if (!block) {
+                    // Note: the "input" fallback here should/must not be taken.  in case the LLM makes up
+                    // a tool we don't knwo about, we at least have a place to stash our stuff.
+                    const customInputProperty = toolCall.custom && !toolCall.function ? (grammarToolInputProperties.get(name) ?? "input") : undefined;
+                    const hasCustomInput = customInputProperty !== undefined;
                     block = {
                         type: "toolCall",
                         id: toolCall.id || "",
-                        name: toolCall.function?.name || "",
-                        arguments: {},
-                        partialArgs: "",
+                        name,
+                        arguments: hasCustomInput ? { [customInputProperty]: "" } : {},
+                        partialArgs: hasCustomInput ? undefined : "",
+                        customInput: hasCustomInput
+                            ? { property: customInputProperty, jsonBuffer: { input: "", started: false, closed: false } }
+                            : undefined,
                         streamIndex,
                     };
                     if (streamIndex !== undefined) {
@@ -226,6 +288,18 @@ export const stream = (model, context, options) => {
                 if (toolCall.id) {
                     toolCallBlocksById.set(toolCall.id, block);
                 }
+                if (!block.name && name) {
+                    block.name = name;
+                }
+                if (toolCall.custom && !toolCall.function && !block.customInput) {
+                    const customInputProperty = grammarToolInputProperties.get(block.name) ?? "input";
+                    block.arguments = { [customInputProperty]: "" };
+                    block.customInput = {
+                        property: customInputProperty,
+                        jsonBuffer: { input: "", started: false, closed: false },
+                    };
+                    delete block.partialArgs;
+                }
                 applyPendingReasoningDetail(block);
                 return block;
             };
@@ -250,6 +324,7 @@ export const stream = (model, context, options) => {
                     output.usage = parseChunkUsage(choice.usage, model);
                 }
                 if (choice.finish_reason) {
+                    output.rawStopReason = choice.finish_reason;
                     const finishReasonResult = mapStopReason(choice.finish_reason);
                     output.stopReason = finishReasonResult.stopReason;
                     if (finishReasonResult.errorMessage) {
@@ -307,8 +382,9 @@ export const stream = (model, context, options) => {
                                 block.id = toolCall.id;
                                 toolCallBlocksById.set(toolCall.id, block);
                             }
-                            if (!block.name && toolCall.function?.name) {
-                                block.name = toolCall.function.name;
+                            const name = toolCall.function?.name ?? toolCall.custom?.name;
+                            if (!block.name && name) {
+                                block.name = name;
                             }
                             let delta = "";
                             if (toolCall.function?.arguments) {
@@ -316,6 +392,10 @@ export const stream = (model, context, options) => {
                                 block.partialArgs = (block.partialArgs ?? "") + toolCall.function.arguments;
                                 block.arguments = parseStreamingJson(block.partialArgs);
                             }
+                            else if (toolCall.custom?.input) {
+                                const nextInput = getCustomToolCallInput(block) + toolCall.custom.input;
+                                delta = appendCustomToolCallInput(block, nextInput, false) ?? "";
+                            }
                             stream.push({
                                 type: "toolcall_delta",
                                 contentIndex: getContentIndex(block),
@@ -353,7 +433,7 @@ export const stream = (model, context, options) => {
             if (output.stopReason === "error") {
                 throw new Error(output.errorMessage || "Provider returned an error stop reason");
             }
-            if (!hasFinishReason) {
+            if (!hasFinishReason || output.stopReason === "pending") {
                 throw new Error("Stream ended without finish_reason");
             }
             stream.push({ type: "done", reason: output.stopReason, message: output });
@@ -364,6 +444,7 @@ export const stream = (model, context, options) => {
                 delete block.index;
                 // Streaming scratch buffers are only used during parsing; never persist them.
                 delete block.partialArgs;
+                delete block.customInput;
                 delete block.streamIndex;
             }
             output.stopReason = options?.signal?.aborted ? "aborted" : "error";
@@ -394,7 +475,7 @@ export const streamSimple = (model, context, options) => {
         toolChoice,
     });
 };
-function createClient(model, context, apiKey, optionsHeaders, sessionId, compat = getCompat(model)) {
+function createClient(model, context, apiKey, optionsHeaders, fetch, sessionId, compat = getCompat(model)) {
     const headers = { ...model.headers };
     if (model.provider === "github-copilot") {
         const hasImages = hasCopilotVisionInput(context.messages);
@@ -405,9 +486,16 @@ function createClient(model, context, apiKey, optionsHeaders, sessionId, compat
         Object.assign(headers, copilotHeaders);
     }
     if (sessionId && compat.sendSessionAffinityHeaders) {
-        headers.session_id = sessionId;
-        headers["x-client-request-id"] = sessionId;
-        headers["x-session-affinity"] = sessionId;
+        if (compat.sessionAffinityFormat === "openrouter") {
+            headers["x-session-id"] = sessionId;
+        }
+        else {
+            if (compat.sessionAffinityFormat === "openai") {
+                headers.session_id = sessionId;
+            }
+            headers["x-client-request-id"] = sessionId;
+            headers["x-session-affinity"] = sessionId;
+        }
     }
     // Merge options headers last so they can override defaults
     if (optionsHeaders) {
@@ -417,11 +505,12 @@ function createClient(model, context, apiKey, optionsHeaders, sessionId, compat
         apiKey,
         baseURL: model.baseUrl,
         dangerouslyAllowBrowser: true,
+        fetch,
         defaultHeaders: headers,
     });
 }
-function buildParams(model, context, options, compat = getCompat(model), cacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env)) {
-    const messages = convertMessages(model, context, compat);
+function buildParams(model, context, options, compat = getCompat(model), cacheRetention = resolveCacheRetention(options?.cacheRetention, options?.env), grammarToolInputProperties = createGrammarToolInputProperties(context.tools, compat.supportsOpenAIGrammarTools)) {
+    const messages = convertMessages(model, context, compat, { grammarToolInputProperties });
     const cacheControl = getCompatCacheControl(compat, cacheRetention);
     const params = {
         model: model.id,
@@ -450,8 +539,10 @@ function buildParams(model, context, options, compat = getCompat(model), cacheRe
     if (options?.temperature !== undefined) {
         params.temperature = options.temperature;
     }
-    if (context.tools && context.tools.length > 0) {
-        params.tools = convertTools(context.tools, compat);
+    const deferredToolNames = compat.deferredToolsMode === "kimi" ? getDeferredToolNames(context.messages) : new Set();
+    const activeTools = context.tools?.filter((tool) => !deferredToolNames.has(tool.name));
+    if (activeTools && activeTools.length > 0) {
+        params.tools = convertTools(activeTools, compat);
         if (compat.zaiToolStream) {
             params.tool_stream = true;
         }
@@ -479,6 +570,12 @@ function buildParams(model, context, options, compat = getCompat(model), cacheRe
     }
     else if (compat.thinkingFormat === "qwen" && model.reasoning) {
         params.enable_thinking = !!options?.reasoningEffort;
+        if (options?.reasoningEffort && compat.supportsReasoningEffort) {
+            const effort = model.thinkingLevelMap?.[options.reasoningEffort] ?? options.reasoningEffort;
+            if (typeof effort === "string") {
+                params.reasoning_effort = effort;
+            }
+        }
     }
     else if (compat.thinkingFormat === "qwen-chat-template" && model.reasoning) {
         params.chat_template_kwargs = {
@@ -613,7 +710,7 @@ function addCacheControlToSystemPrompt(messages, cacheControl) {
 function addCacheControlToLastConversationMessage(messages, cacheControl) {
     for (let i = messages.length - 1; i >= 0; i--) {
         const message = messages[i];
-        if (message.role === "user" || message.role === "assistant") {
+        if (message.role === "user" || message.role === "assistant" || message.role === "tool") {
             if (addCacheControlToMessage(message, cacheControl)) {
                 return;
             }
@@ -631,7 +728,7 @@ function addCacheControlToInstructionMessage(message, cacheControl) {
     return addCacheControlToTextContent(message, cacheControl);
 }
 function addCacheControlToMessage(message, cacheControl) {
-    if (message.role === "user" || message.role === "assistant") {
+    if (message.role === "user" || message.role === "assistant" || message.role === "tool") {
         return addCacheControlToTextContent(message, cacheControl);
     }
     return false;
@@ -664,17 +761,28 @@ function addCacheControlToTextContent(message, cacheControl) {
     }
     return false;
 }
-export function convertMessages(model, context, compat) {
+export function convertMessages(model, context, compat, options) {
     const params = [];
     const normalizeToolCallId = (id) => {
         // Handle pipe-separated IDs from OpenAI Responses API
         // Format: {call_id}|{id} where {id} can be 400+ chars with special chars (+, /, =)
         // These come from providers like github-copilot, openai-codex, opencode
         // Extract just the call_id part and normalize it
+        // Multiple tool calls in the same turn can share call_id but differ by item_id.
+        // Preserve item-level uniqueness when replaying into Chat Completions, which
+        // requires distinct tool call ids.
         if (id.includes("|")) {
-            const [callId] = id.split("|");
             // Sanitize to allowed chars and truncate to 40 chars (OpenAI limit)
-            return callId.replace(/[^a-zA-Z0-9_-]/g, "_").slice(0, 40);
+            const separatorIndex = id.indexOf("|");
+            const callId = id.slice(0, separatorIndex).replace(/[^a-zA-Z0-9_-]/g, "_");
+            const itemId = id.slice(separatorIndex + 1).replace(/[^a-zA-Z0-9_-]/g, "_");
+            const combinedId = itemId.length > 0 ? `${callId}_${itemId}` : callId;
+            if (combinedId.length <= 40) {
+                return combinedId;
+            }
+            const hash = shortHash(id).slice(0, 8);
+            const prefix = callId.slice(0, Math.max(1, 40 - hash.length - 1));
+            return `${prefix}_${hash}`;
         }
         if (model.provider === "openai")
             return id.length > 40 ? id.slice(0, 40) : id;
@@ -780,14 +888,27 @@ export function convertMessages(model, context, compat) {
             }
             const toolCalls = msg.content.filter(isToolCallBlock);
             if (toolCalls.length > 0) {
-                assistantMsg.tool_calls = toolCalls.map((tc) => ({
-                    id: tc.id,
-                    type: "function",
-                    function: {
-                        name: tc.name,
-                        arguments: JSON.stringify(tc.arguments),
-                    },
-                }));
+                assistantMsg.tool_calls = toolCalls.map((tc) => {
+                    const customInputProperty = options?.grammarToolInputProperties?.get(tc.name);
+                    if (customInputProperty !== undefined) {
+                        return {
+                            id: tc.id,
+                            type: "custom",
+                            custom: {
+                                name: tc.name,
+                                input: sanitizeSurrogates(getGrammarToolInput(tc.name, tc.arguments, customInputProperty)),
+                            },
+                        };
+                    }
+                    return {
+                        id: tc.id,
+                        type: "function",
+                        function: {
+                            name: tc.name,
+                            arguments: JSON.stringify(tc.arguments),
+                        },
+                    };
+                });
                 const reasoningDetails = toolCalls
                     .filter((tc) => tc.thoughtSignature)
                     .map((tc) => {
@@ -827,6 +948,7 @@ export function convertMessages(model, context, compat) {
         }
         else if (msg.role === "toolResult") {
             const imageBlocks = [];
+            const deferredToolNames = new Set();
             let j = i;
             for (; j < transformedMessages.length && transformedMessages[j].role === "toolResult"; j++) {
                 const toolMsg = transformedMessages[j];
@@ -849,6 +971,11 @@ export function convertMessages(model, context, compat) {
                     toolResultMsg.name = toolMsg.toolName;
                 }
                 params.push(toolResultMsg);
+                if (compat.deferredToolsMode === "kimi") {
+                    for (const name of toolMsg.addedToolNames ?? []) {
+                        deferredToolNames.add(name);
+                    }
+                }
                 if (hasImages && model.input.includes("image")) {
                     for (const block of toolMsg.content) {
                         if (isImageContentBlock(block)) {
@@ -885,6 +1012,17 @@ export function convertMessages(model, context, compat) {
             else {
                 lastRole = "toolResult";
             }
+            if (deferredToolNames.size > 0) {
+                const deferredTools = getToolsByName(context.tools, deferredToolNames);
+                if (deferredTools.length > 0) {
+                    const kimiToolMessage = {
+                        role: "system",
+                        tools: convertTools(deferredTools, compat),
+                    };
+                    // Kimi accepts a system message with tools but omits the standard content field.
+                    params.push(kimiToolMessage);
+                }
+            }
             continue;
         }
         lastRole = msg.role;
@@ -892,16 +1030,36 @@ export function convertMessages(model, context, compat) {
     return params;
 }
 function convertTools(tools, compat) {
-    return tools.map((tool) => ({
-        type: "function",
-        function: {
-            name: tool.name,
-            description: tool.description,
-            parameters: tool.parameters, // TypeBox already generates JSON Schema
-            // Only include strict if provider supports it. Some reject unknown fields.
-            ...(compat.supportsStrictMode !== false && { strict: false }),
-        },
-    }));
+    return tools.map((tool) => {
+        const grammar = resolveGrammarConstrainedSampling(tool, compat.supportsOpenAIGrammarTools);
+        if (grammar) {
+            return {
+                type: "custom",
+                custom: {
+                    name: tool.name,
+                    description: tool.description,
+                    format: {
+                        type: "grammar",
+                        grammar: {
+                            syntax: grammar.format,
+                            definition: grammar.definition,
+                        },
+                    },
+                },
+            };
+        }
+        const strict = resolveJsonSchemaStrictSampling(tool, compat.supportsStrictMode !== false);
+        return {
+            type: "function",
+            function: {
+                name: tool.name,
+                description: tool.description,
+                parameters: tool.parameters, // TypeBox already generates JSON Schema
+                // Only include strict if provider supports it. Some reject unknown fields.
+                ...(compat.supportsStrictMode !== false && { strict: strict ?? false }),
+            },
+        };
+    });
 }
 function parseChunkUsage(rawUsage, model) {
     const promptTokens = rawUsage.prompt_tokens || 0;
@@ -987,7 +1145,13 @@ function detectCompat(model) {
         isCloudflareWorkersAI ||
         isCloudflareAiGateway ||
         isAntLing;
-    const useMaxTokens = baseUrl.includes("chutes.ai") || isMoonshot || isCloudflareAiGateway || isTogether || isNvidia || isAntLing;
+    const useMaxTokens = baseUrl.includes("chutes.ai") ||
+        isMoonshot ||
+        isCloudflareAiGateway ||
+        isTogether ||
+        isNvidia ||
+        isAntLing ||
+        isZai;
     const isGrok = provider === "xai" || baseUrl.includes("api.x.ai");
     const isDeepSeek = provider === "deepseek" || baseUrl.includes("deepseek.com");
     const isOpenRouterDeveloperRoleModel = isOpenRouter && (model.id.startsWith("anthropic/") || model.id.startsWith("openai/"));
@@ -1018,8 +1182,11 @@ function detectCompat(model) {
         chatTemplateKwargs: {},
         zaiToolStream: false,
         supportsStrictMode: !isMoonshot && !isTogether && !isCloudflareAiGateway && !isNvidia,
+        supportsOpenAIGrammarTools: false,
         cacheControlFormat,
         sendSessionAffinityHeaders: false,
+        deferredToolsMode: undefined,
+        sessionAffinityFormat: isOpenRouter ? "openrouter" : "openai",
         supportsLongCacheRetention: !(isTogether ||
             isCloudflareWorkersAI ||
             isCloudflareAiGateway ||
@@ -1052,8 +1219,11 @@ function getCompat(model) {
         chatTemplateKwargs: model.compat.chatTemplateKwargs ?? detected.chatTemplateKwargs,
         zaiToolStream: model.compat.zaiToolStream ?? detected.zaiToolStream,
         supportsStrictMode: model.compat.supportsStrictMode ?? detected.supportsStrictMode,
+        supportsOpenAIGrammarTools: model.compat.supportsOpenAIGrammarTools ?? detected.supportsOpenAIGrammarTools,
         cacheControlFormat: model.compat.cacheControlFormat ?? detected.cacheControlFormat,
         sendSessionAffinityHeaders: model.compat.sendSessionAffinityHeaders ?? detected.sendSessionAffinityHeaders,
+        deferredToolsMode: model.compat.deferredToolsMode ?? detected.deferredToolsMode,
+        sessionAffinityFormat: model.compat.sessionAffinityFormat ?? detected.sessionAffinityFormat,
         supportsLongCacheRetention: model.compat.supportsLongCacheRetention ?? detected.supportsLongCacheRetention,
     };
 }