@earendil-works/pi-ai 1.4.0 → 1.5.1

package/dist/api/openai-completions.js

@@ -5,13 +5,14 @@ import { AssistantMessageEventStream } from "../utils/event-stream.js";
 import { shortHash } from "../utils/hash.js";
 import { headersToRecord } from "../utils/headers.js";
 import { getIdentityHeaders } from "../utils/identity.js";
-import { parseStreamingJson, parseToolCallArguments } from "../utils/json-parse.js";
+import { applyOpenRouterReportedCost, isOpenRouterModel, } from "../utils/openrouter-cost.js";
 import { getProviderEnvValue } from "../utils/provider-env.js";
 import { retryProviderRequest } from "../utils/provider-retry.js";
 import { sanitizeSurrogates } from "../utils/sanitize-unicode.js";
+import { ToolCallArgumentsStream } from "../utils/streaming-json.js";
 import { getSystemMessageText, renderSystemMessageUpdate } from "../utils/text.js";
 import { getDeclaredTools, resolveTranscript, resolveTranscriptTools, } from "../utils/transcript.js";
-import { appendGrammarToolInputJsonDelta, createGrammarToolInputProperties, getGrammarToolInput, getJsonSchemaToolParameters, resolveGrammarConstrainedSampling, resolveJsonSchemaStrictSampling, } from "./constrained-sampling.js";
+import { appendGrammarToolInputJsonDelta, closeGrammarToolInputJson, createGrammarToolInputProperties, getGrammarToolInput, getJsonSchemaToolParameters, resolveGrammarConstrainedSampling, resolveJsonSchemaStrictSampling, } from "./constrained-sampling.js";
 import { buildCopilotDynamicHeaders, hasCopilotVisionInput } from "./github-copilot-headers.js";
 import { clampOpenAIPromptCacheKey } from "./openai-prompt-cache.js";
 import { buildBaseOptions, clampThinkingBudgetToAnswerRoom, resolveSamplingParams, thinkingBudgetForLevel, } from "./simple-options.js";
@@ -206,23 +207,14 @@ export const stream = (model, context, options) => {
             let hasFinishReason = false;
             const toolCallBlocksByIndex = new Map();
             const toolCallBlocksById = new Map();
+            const toolCallStates = new Map();
             const blocks = output.content;
             const getContentIndex = (block) => blocks.indexOf(block);
-            const getCustomToolCallInput = (block) => {
-                const property = block.customInput?.property;
-                if (property === undefined)
-                    return "";
-                const value = block.arguments[property];
-                return typeof value === "string" ? value : "";
-            };
-            const appendCustomToolCallInput = (block, nextInput, close) => {
-                const customInput = block.customInput;
-                if (!customInput)
-                    return undefined;
-                const delta = appendGrammarToolInputJsonDelta(customInput.jsonBuffer, customInput.property, nextInput, close);
-                block.arguments = { [customInput.property]: nextInput };
-                return delta;
-            };
+            const customInput = (property) => ({
+                kind: "custom",
+                property,
+                buffer: { input: "", started: false, closed: false },
+            });
             const finishBlock = (block) => {
                 const contentIndex = getContentIndex(block);
                 if (contentIndex === -1) {
@@ -246,8 +238,10 @@ export const stream = (model, context, options) => {
                     });
                 }
                 else if (block.type === "toolCall") {
-                    if (block.customInput) {
-                        const delta = appendCustomToolCallInput(block, getCustomToolCallInput(block), true);
+                    const input = toolCallStates.get(block).input;
+                    toolCallStates.delete(block);
+                    if (input.kind === "custom") {
+                        const delta = closeGrammarToolInputJson(input.buffer, input.property, undefined);
                         if (delta !== undefined) {
                             stream.push({
                                 type: "toolcall_delta",
@@ -258,13 +252,8 @@ export const stream = (model, context, options) => {
                         }
                     }
                     else {
-                        Object.assign(block, parseToolCallArguments(block.partialArgs));
+                        Object.assign(block, input.arguments.finish());
                     }
-                    // Finalize in-place and strip the scratch buffers so replay only
-                    // carries parsed arguments.
-                    delete block.partialArgs;
-                    delete block.customInput;
-                    delete block.streamIndex;
                     stream.push({
                         type: "toolcall_end",
                         contentIndex,
@@ -304,18 +293,18 @@ export const stream = (model, context, options) => {
                     // Note: the "input" fallback here should/must not be taken.  in case the LLM makes up
                     // a tool we don't knwo about, we at least have a place to stash our stuff.
                     const customInputProperty = toolCall.custom && !toolCall.function ? (grammarToolInputProperties.get(name) ?? "input") : undefined;
-                    const hasCustomInput = customInputProperty !== undefined;
                     block = {
                         type: "toolCall",
                         id: toolCall.id || "",
                         name,
-                        arguments: hasCustomInput ? { [customInputProperty]: "" } : {},
-                        partialArgs: hasCustomInput ? undefined : "",
-                        customInput: hasCustomInput
-                            ? { property: customInputProperty, jsonBuffer: { input: "", started: false, closed: false } }
-                            : undefined,
-                        streamIndex,
+                        arguments: customInputProperty !== undefined ? { [customInputProperty]: "" } : {},
                     };
+                    toolCallStates.set(block, {
+                        streamIndex,
+                        input: customInputProperty !== undefined
+                            ? customInput(customInputProperty)
+                            : { kind: "json", arguments: new ToolCallArgumentsStream() },
+                    });
                     if (streamIndex !== undefined) {
                         toolCallBlocksByIndex.set(streamIndex, block);
                     }
@@ -329,8 +318,9 @@ export const stream = (model, context, options) => {
                         partial: output,
                     });
                 }
-                if (streamIndex !== undefined && block.streamIndex === undefined) {
-                    block.streamIndex = streamIndex;
+                const state = toolCallStates.get(block);
+                if (streamIndex !== undefined && state.streamIndex === undefined) {
+                    state.streamIndex = streamIndex;
                     toolCallBlocksByIndex.set(streamIndex, block);
                 }
                 if (toolCall.id) {
@@ -339,16 +329,12 @@ export const stream = (model, context, options) => {
                 if (!block.name && name) {
                     block.name = name;
                 }
-                if (toolCall.custom && !toolCall.function && !block.customInput) {
+                if (toolCall.custom && !toolCall.function && state.input.kind !== "custom") {
                     const customInputProperty = grammarToolInputProperties.get(block.name) ?? "input";
                     block.arguments = { [customInputProperty]: "" };
-                    block.customInput = {
-                        property: customInputProperty,
-                        jsonBuffer: { input: "", started: false, closed: false },
-                    };
-                    delete block.partialArgs;
+                    state.input = customInput(customInputProperty);
                 }
-                return block;
+                return { block, state };
             };
             for await (const chunk of openaiStream) {
                 await options?.onProviderStreamEvent?.(chunk, model);
@@ -440,7 +426,7 @@ export const stream = (model, context, options) => {
                     }
                     if (choice?.delta?.tool_calls) {
                         for (const toolCall of choice.delta.tool_calls) {
-                            const block = ensureToolCallBlock(toolCall);
+                            const { block, state } = ensureToolCallBlock(toolCall);
                             if (!block.id && toolCall.id) {
                                 block.id = toolCall.id;
                                 toolCallBlocksById.set(toolCall.id, block);
@@ -450,14 +436,15 @@ export const stream = (model, context, options) => {
                                 block.name = name;
                             }
                             let delta = "";
-                            if (toolCall.function?.arguments) {
+                            const input = state.input;
+                            if (toolCall.function?.arguments && input.kind === "json") {
                                 delta = toolCall.function.arguments;
-                                block.partialArgs = (block.partialArgs ?? "") + toolCall.function.arguments;
-                                block.arguments = parseStreamingJson(block.partialArgs);
+                                block.arguments = input.arguments.append(delta);
                             }
-                            else if (toolCall.custom?.input) {
-                                const nextInput = getCustomToolCallInput(block) + toolCall.custom.input;
-                                delta = appendCustomToolCallInput(block, nextInput, false) ?? "";
+                            else if (toolCall.custom?.input && input.kind === "custom") {
+                                delta =
+                                    appendGrammarToolInputJsonDelta(input.buffer, input.property, toolCall.custom.input) ?? "";
+                                block.arguments = { [input.property]: input.buffer.input };
                             }
                             stream.push({
                                 type: "toolcall_delta",
@@ -495,11 +482,6 @@ export const stream = (model, context, options) => {
                 if (block.type === "thinking") {
                     applyStreamedReasoningDetails(block);
                 }
-                delete block.index;
-                // Streaming scratch buffers are only used during parsing; never persist them.
-                delete block.partialArgs;
-                delete block.customInput;
-                delete block.streamIndex;
             }
             output.stopReason = options?.signal?.aborted ? "aborted" : "error";
             output.errorMessage = formatProviderError(normalizeProviderError(error));
@@ -573,12 +555,16 @@ function buildParams(model, context, options, compat = getCompat(model), cacheRe
         grammarToolInputProperties,
     });
     const cacheControl = getCompatCacheControl(compat, cacheRetention);
+    const promptCacheKeyMode = compat.promptCacheKeyMode ?? "auto";
+    const supportsPromptCacheKey = promptCacheKeyMode === "enabled" ||
+        (promptCacheKeyMode === "auto" &&
+            (model.baseUrl.includes("api.openai.com") ||
+                (cacheRetention === "long" && compat.supportsLongCacheRetention)));
     const params = {
         model: model.id,
         messages,
         stream: true,
-        prompt_cache_key: (model.baseUrl.includes("api.openai.com") && cacheRetention !== "none") ||
-            (cacheRetention === "long" && compat.supportsLongCacheRetention)
+        prompt_cache_key: cacheRetention !== "none" && supportsPromptCacheKey
             ? clampOpenAIPromptCacheKey(options?.sessionId)
             : undefined,
         prompt_cache_retention: cacheRetention === "long" && compat.supportsLongCacheRetention ? "24h" : undefined,
@@ -607,8 +593,7 @@ function buildParams(model, context, options, compat = getCompat(model), cacheRe
             params.tool_stream = true;
         }
     }
-    else if (hasToolHistory(context.messages)) {
-        // Anthropic (via LiteLLM/proxy) requires tools param when conversation has tool_calls/tool_results
+    else if (compat.requiresToolsWithToolHistory && hasToolHistory(context.messages)) {
         params.tools = [];
     }
     if (cacheControl) {
@@ -968,7 +953,7 @@ export function convertMessages(model, context, compat, options) {
             // Some providers don't accept null content, use empty string instead
             const assistantMsg = {
                 role: "assistant",
-                content: compat.requiresAssistantAfterToolResult ? "" : null,
+                content: compat.requiresAssistantAfterToolResult || compat.requiresNonNullAssistantContent ? "" : null,
             };
             const assistantTextParts = msg.content
                 .filter(isTextContentBlock)
@@ -1168,7 +1153,10 @@ function convertTools(tools, compat) {
         };
     });
 }
-function parseChunkUsage(rawUsage, model) {
+function parseChunkUsage(
+// OpenRouter adds billing fields to streamed usage at runtime,
+// although the OpenAI SDK does not type those fields.
+rawUsage, model) {
     const promptTokens = rawUsage.prompt_tokens || 0;
     const cacheReadTokens = rawUsage.prompt_tokens_details?.cached_tokens ?? rawUsage.prompt_cache_hit_tokens ?? rawUsage.cached_tokens ?? 0;
     const cacheWriteTokens = rawUsage.prompt_tokens_details?.cache_write_tokens || 0;
@@ -1196,6 +1184,9 @@ function parseChunkUsage(rawUsage, model) {
         cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
     };
     calculateCost(model, usage);
+    if (isOpenRouterModel(model)) {
+        applyOpenRouterReportedCost(usage, rawUsage);
+    }
     return usage;
 }
 function mapStopReason(reason) {
@@ -1276,6 +1267,8 @@ function detectCompat(model) {
         maxTokensField: useMaxTokens ? "max_tokens" : "max_completion_tokens",
         requiresToolResultName: false,
         requiresAssistantAfterToolResult: false,
+        requiresNonNullAssistantContent: false,
+        requiresToolsWithToolHistory: false,
         requiresThinkingAsText: false,
         requiresReasoningContentOnAssistantMessages: isDeepSeek,
         thinkingFormat: isDeepSeek
@@ -1328,6 +1321,8 @@ function getCompat(model) {
         maxTokensField: model.compat.maxTokensField ?? detected.maxTokensField,
         requiresToolResultName: model.compat.requiresToolResultName ?? detected.requiresToolResultName,
         requiresAssistantAfterToolResult: model.compat.requiresAssistantAfterToolResult ?? detected.requiresAssistantAfterToolResult,
+        requiresNonNullAssistantContent: model.compat.requiresNonNullAssistantContent ?? detected.requiresNonNullAssistantContent,
+        requiresToolsWithToolHistory: model.compat.requiresToolsWithToolHistory ?? detected.requiresToolsWithToolHistory,
         requiresThinkingAsText: model.compat.requiresThinkingAsText ?? detected.requiresThinkingAsText,
         requiresReasoningContentOnAssistantMessages: model.compat.requiresReasoningContentOnAssistantMessages ??
             detected.requiresReasoningContentOnAssistantMessages,
@@ -1347,6 +1342,7 @@ function getCompat(model) {
         sendSessionAffinityHeaders: model.compat.sendSessionAffinityHeaders ?? detected.sendSessionAffinityHeaders,
         sessionAffinityFormat: model.compat.sessionAffinityFormat ?? detected.sessionAffinityFormat,
         supportsLongCacheRetention: model.compat.supportsLongCacheRetention ?? detected.supportsLongCacheRetention,
+        promptCacheKeyMode: model.compat.promptCacheKeyMode,
         vllmPriority: model.compat.vllmPriority,
     };
 }