@mariozechner/pi-ai 0.67.68 → 0.68.0

package/dist/providers/openai-completions.js

@@ -26,6 +26,27 @@ function hasToolHistory(messages) {
     }
     return false;
 }
+function isTextContentBlock(block) {
+    return block.type === "text";
+}
+function isThinkingContentBlock(block) {
+    return block.type === "thinking";
+}
+function isToolCallBlock(block) {
+    return block.type === "toolCall";
+}
+function isImageContentBlock(block) {
+    return block.type === "image";
+}
+function resolveCacheRetention(cacheRetention) {
+    if (cacheRetention) {
+        return cacheRetention;
+    }
+    if (typeof process !== "undefined" && process.env.PI_CACHE_RETENTION === "long") {
+        return "long";
+    }
+    return "short";
+}
 export const streamOpenAICompletions = (model, context, options) => {
     const stream = new AssistantMessageEventStream();
     (async () => {
@@ -48,8 +69,11 @@ export const streamOpenAICompletions = (model, context, options) => {
         };
         try {
             const apiKey = options?.apiKey || getEnvApiKey(model.provider) || "";
-            const client = createClient(model, context, apiKey, options?.headers);
-            let params = buildParams(model, context, options);
+            const compat = getCompat(model);
+            const cacheRetention = resolveCacheRetention(options?.cacheRetention);
+            const cacheSessionId = cacheRetention === "none" ? undefined : options?.sessionId;
+            const client = createClient(model, context, apiKey, options?.headers, cacheSessionId, compat);
+            let params = buildParams(model, context, options, compat, cacheRetention);
             const nextParams = await options?.onPayload?.(params, model);
             if (nextParams !== undefined) {
                 params = nextParams;
@@ -270,7 +294,7 @@ export const streamSimpleOpenAICompletions = (model, context, options) => {
         toolChoice,
     });
 };
-function createClient(model, context, apiKey, optionsHeaders) {
+function createClient(model, context, apiKey, optionsHeaders, sessionId, compat = getCompat(model)) {
     if (!apiKey) {
         if (!process.env.OPENAI_API_KEY) {
             throw new Error("OpenAI API key is required. Set OPENAI_API_KEY environment variable or pass it as an argument.");
@@ -286,6 +310,11 @@ function createClient(model, context, apiKey, optionsHeaders) {
         });
         Object.assign(headers, copilotHeaders);
     }
+    if (sessionId && compat.sendSessionAffinityHeaders) {
+        headers.session_id = sessionId;
+        headers["x-client-request-id"] = sessionId;
+        headers["x-session-affinity"] = sessionId;
+    }
     // Merge options headers last so they can override defaults
     if (optionsHeaders) {
         Object.assign(headers, optionsHeaders);
@@ -297,14 +326,15 @@ function createClient(model, context, apiKey, optionsHeaders) {
         defaultHeaders: headers,
     });
 }
-function buildParams(model, context, options) {
-    const compat = getCompat(model);
+function buildParams(model, context, options, compat = getCompat(model), cacheRetention = resolveCacheRetention(options?.cacheRetention)) {
     const messages = convertMessages(model, context, compat);
-    maybeAddOpenRouterAnthropicCacheControl(model, messages);
+    const cacheControl = getCompatCacheControl(model, compat, cacheRetention);
     const params = {
         model: model.id,
         messages,
         stream: true,
+        prompt_cache_key: model.baseUrl.includes("api.openai.com") && cacheRetention !== "none" ? options?.sessionId : undefined,
+        prompt_cache_retention: model.baseUrl.includes("api.openai.com") && cacheRetention === "long" ? "24h" : undefined,
     };
     if (compat.supportsUsageInStreaming !== false) {
         params.stream_options = { include_usage: true };
@@ -333,6 +363,9 @@ function buildParams(model, context, options) {
         // Anthropic (via LiteLLM/proxy) requires tools param when conversation has tool_calls/tool_results
         params.tools = [];
     }
+    if (cacheControl) {
+        applyAnthropicCacheControl(messages, params.tools, cacheControl);
+    }
     if (options?.toolChoice) {
         params.tool_choice = options.toolChoice;
     }
@@ -385,34 +418,80 @@ function buildParams(model, context, options) {
 function mapReasoningEffort(effort, reasoningEffortMap) {
     return reasoningEffortMap[effort] ?? effort;
 }
-function maybeAddOpenRouterAnthropicCacheControl(model, messages) {
-    if (model.provider !== "openrouter" || !model.id.startsWith("anthropic/"))
-        return;
-    // Anthropic-style caching requires cache_control on a text part. Add a breakpoint
-    // on the last user/assistant message (walking backwards until we find text content).
-    for (let i = messages.length - 1; i >= 0; i--) {
-        const msg = messages[i];
-        if (msg.role !== "user" && msg.role !== "assistant")
-            continue;
-        const content = msg.content;
-        if (typeof content === "string") {
-            msg.content = [
-                Object.assign({ type: "text", text: content }, { cache_control: { type: "ephemeral" } }),
-            ];
+function getCompatCacheControl(model, compat, cacheRetention) {
+    if (compat.cacheControlFormat !== "anthropic" || cacheRetention === "none") {
+        return undefined;
+    }
+    const ttl = cacheRetention === "long" && model.baseUrl.includes("api.anthropic.com") ? "1h" : undefined;
+    return { type: "ephemeral", ...(ttl ? { ttl } : {}) };
+}
+function applyAnthropicCacheControl(messages, tools, cacheControl) {
+    addCacheControlToSystemPrompt(messages, cacheControl);
+    addCacheControlToLastTool(tools, cacheControl);
+    addCacheControlToLastConversationMessage(messages, cacheControl);
+}
+function addCacheControlToSystemPrompt(messages, cacheControl) {
+    for (const message of messages) {
+        if (message.role === "system" || message.role === "developer") {
+            addCacheControlToInstructionMessage(message, cacheControl);
             return;
         }
-        if (!Array.isArray(content))
-            continue;
-        // Find last text part and add cache_control
-        for (let j = content.length - 1; j >= 0; j--) {
-            const part = content[j];
-            if (part?.type === "text") {
-                Object.assign(part, { cache_control: { type: "ephemeral" } });
+    }
+}
+function addCacheControlToLastConversationMessage(messages, cacheControl) {
+    for (let i = messages.length - 1; i >= 0; i--) {
+        const message = messages[i];
+        if (message.role === "user" || message.role === "assistant") {
+            if (addCacheControlToMessage(message, cacheControl)) {
                 return;
             }
         }
     }
 }
+function addCacheControlToLastTool(tools, cacheControl) {
+    if (!tools || tools.length === 0) {
+        return;
+    }
+    const lastTool = tools[tools.length - 1];
+    lastTool.cache_control = cacheControl;
+}
+function addCacheControlToInstructionMessage(message, cacheControl) {
+    return addCacheControlToTextContent(message, cacheControl);
+}
+function addCacheControlToMessage(message, cacheControl) {
+    if (message.role === "user" || message.role === "assistant") {
+        return addCacheControlToTextContent(message, cacheControl);
+    }
+    return false;
+}
+function addCacheControlToTextContent(message, cacheControl) {
+    const content = message.content;
+    if (typeof content === "string") {
+        if (content.length === 0) {
+            return false;
+        }
+        message.content = [
+            {
+                type: "text",
+                text: content,
+                cache_control: cacheControl,
+            },
+        ];
+        return true;
+    }
+    if (!Array.isArray(content)) {
+        return false;
+    }
+    for (let i = content.length - 1; i >= 0; i--) {
+        const part = content[i];
+        if (part?.type === "text") {
+            const textPart = part;
+            textPart.cache_control = cacheControl;
+            return true;
+        }
+    }
+    return false;
+}
 export function convertMessages(model, context, compat) {
     const params = [];
     const normalizeToolCallId = (id) => {
@@ -470,14 +549,11 @@ export function convertMessages(model, context, compat) {
                         };
                     }
                 });
-                const filteredContent = !model.input.includes("image")
-                    ? content.filter((c) => c.type !== "image_url")
-                    : content;
-                if (filteredContent.length === 0)
+                if (content.length === 0)
                     continue;
                 params.push({
                     role: "user",
-                    content: filteredContent,
+                    content,
                 });
             }
         }
@@ -487,42 +563,50 @@ export function convertMessages(model, context, compat) {
                 role: "assistant",
                 content: compat.requiresAssistantAfterToolResult ? "" : null,
             };
-            const textBlocks = msg.content.filter((b) => b.type === "text");
-            // Filter out empty text blocks to avoid API validation errors
-            const nonEmptyTextBlocks = textBlocks.filter((b) => b.text && b.text.trim().length > 0);
-            if (nonEmptyTextBlocks.length > 0) {
-                // Always send assistant content as a plain string (OpenAI Chat Completions
-                // API standard format). Sending as an array of {type:"text", text:"..."}
-                // objects is non-standard and causes some models (e.g. DeepSeek V3.2 via
-                // NVIDIA NIM) to mirror the content-block structure literally in their
-                // output, producing recursive nesting like [{'type':'text','text':'[{...}]'}].
-                assistantMsg.content = nonEmptyTextBlocks.map((b) => sanitizeSurrogates(b.text)).join("");
-            }
-            // Handle thinking blocks
-            const thinkingBlocks = msg.content.filter((b) => b.type === "thinking");
-            // Filter out empty thinking blocks to avoid API validation errors
-            const nonEmptyThinkingBlocks = thinkingBlocks.filter((b) => b.thinking && b.thinking.trim().length > 0);
+            const assistantTextParts = msg.content
+                .filter(isTextContentBlock)
+                .filter((block) => block.text.trim().length > 0)
+                .map((block) => ({
+                type: "text",
+                text: sanitizeSurrogates(block.text),
+            }));
+            const assistantText = assistantTextParts.map((part) => part.text).join("");
+            const nonEmptyThinkingBlocks = msg.content
+                .filter(isThinkingContentBlock)
+                .filter((block) => block.thinking.trim().length > 0);
             if (nonEmptyThinkingBlocks.length > 0) {
                 if (compat.requiresThinkingAsText) {
                     // Convert thinking blocks to plain text (no tags to avoid model mimicking them)
-                    const thinkingText = nonEmptyThinkingBlocks.map((b) => b.thinking).join("\n\n");
-                    const textContent = assistantMsg.content;
-                    if (textContent) {
-                        textContent.unshift({ type: "text", text: thinkingText });
-                    }
-                    else {
-                        assistantMsg.content = [{ type: "text", text: thinkingText }];
-                    }
+                    const thinkingText = nonEmptyThinkingBlocks
+                        .map((block) => sanitizeSurrogates(block.thinking))
+                        .join("\n\n");
+                    assistantMsg.content = [{ type: "text", text: thinkingText }, ...assistantTextParts];
                 }
                 else {
+                    // Always send assistant content as a plain string (OpenAI Chat Completions
+                    // API standard format). Sending as an array of {type:"text", text:"..."}
+                    // objects is non-standard and causes some models (e.g. DeepSeek V3.2 via
+                    // NVIDIA NIM) to mirror the content-block structure literally in their
+                    // output, producing recursive nesting like [{'type':'text','text':'[{...}]'}].
+                    if (assistantText.length > 0) {
+                        assistantMsg.content = assistantText;
+                    }
                     // Use the signature from the first thinking block if available (for llama.cpp server + gpt-oss)
                     const signature = nonEmptyThinkingBlocks[0].thinkingSignature;
                     if (signature && signature.length > 0) {
-                        assistantMsg[signature] = nonEmptyThinkingBlocks.map((b) => b.thinking).join("\n");
+                        assistantMsg[signature] = nonEmptyThinkingBlocks.map((block) => block.thinking).join("\n");
                     }
                 }
             }
-            const toolCalls = msg.content.filter((b) => b.type === "toolCall");
+            else if (assistantText.length > 0) {
+                // Always send assistant content as a plain string (OpenAI Chat Completions
+                // API standard format). Sending as an array of {type:"text", text:"..."}
+                // objects is non-standard and causes some models (e.g. DeepSeek V3.2 via
+                // NVIDIA NIM) to mirror the content-block structure literally in their
+                // output, producing recursive nesting like [{'type':'text','text':'[{...}]'}].
+                assistantMsg.content = assistantText;
+            }
+            const toolCalls = msg.content.filter(isToolCallBlock);
             if (toolCalls.length > 0) {
                 assistantMsg.tool_calls = toolCalls.map((tc) => ({
                     id: tc.id,
@@ -567,8 +651,8 @@ export function convertMessages(model, context, compat) {
                 const toolMsg = transformedMessages[j];
                 // Extract text and image content
                 const textResult = toolMsg.content
-                    .filter((c) => c.type === "text")
-                    .map((c) => c.text)
+                    .filter(isTextContentBlock)
+                    .map((block) => block.text)
                     .join("\n");
                 const hasImages = toolMsg.content.some((c) => c.type === "image");
                 // Always send tool result with text (or placeholder if only images)
@@ -585,7 +669,7 @@ export function convertMessages(model, context, compat) {
                 params.push(toolResultMsg);
                 if (hasImages && model.input.includes("image")) {
                     for (const block of toolMsg.content) {
-                        if (block.type === "image") {
+                        if (isImageContentBlock(block)) {
                             imageBlocks.push({
                                 type: "image_url",
                                 image_url: {
@@ -707,6 +791,7 @@ function detectCompat(model) {
     const useMaxTokens = baseUrl.includes("chutes.ai");
     const isGrok = provider === "xai" || baseUrl.includes("api.x.ai");
     const isGroq = provider === "groq" || baseUrl.includes("groq.com");
+    const cacheControlFormat = provider === "openrouter" && model.id.startsWith("anthropic/") ? "anthropic" : undefined;
     const reasoningEffortMap = isGroq && model.id === "qwen/qwen3-32b"
         ? {
             minimal: "default",
@@ -735,6 +820,8 @@ function detectCompat(model) {
         vercelGatewayRouting: {},
         zaiToolStream: false,
         supportsStrictMode: true,
+        cacheControlFormat,
+        sendSessionAffinityHeaders: false,
     };
 }
 /**
@@ -760,6 +847,8 @@ function getCompat(model) {
         vercelGatewayRouting: model.compat.vercelGatewayRouting ?? detected.vercelGatewayRouting,
         zaiToolStream: model.compat.zaiToolStream ?? detected.zaiToolStream,
         supportsStrictMode: model.compat.supportsStrictMode ?? detected.supportsStrictMode,
+        cacheControlFormat: model.compat.cacheControlFormat ?? detected.cacheControlFormat,
+        sendSessionAffinityHeaders: model.compat.sendSessionAffinityHeaders ?? detected.sendSessionAffinityHeaders,
     };
 }
 //# sourceMappingURL=openai-completions.js.map