@hebo-ai/gateway 0.5.2 → 0.6.0-rc1

package/README.md

@@ -38,7 +38,7 @@ bun install @hebo-ai/gateway
 - Runtime Support
   - [Vercel Edge](#vercel-edge) | [Cloudflare Workers](#cloudflare-workers) | [Deno Deploy](#deno-deploy) | [AWS Lambda](#aws-lambda)
 - OpenAI Extensions
-  - [Reasoning](#reasoning)
+  - [Reasoning](#reasoning) | [Prompt Caching](#prompt-caching)
 - Advanced Usage
   - [Passing Framework State to Hooks](#passing-framework-state-to-hooks) | [Selective Route Mounting](#selective-route-mounting) | [Low-level Schemas & Converters](#low-level-schemas--converters)
 
@@ -565,6 +565,37 @@ Advanced models (like Anthropic Claude 3.7 or Gemini 3) surface structured reaso
 
 For **Gemini 3** models, returning the thought signature via `extra_content` is mandatory to resume the chain-of-thought; failing to do so may result in errors or degraded performance.
 
+### Prompt Caching
+
+The chat completions endpoint supports both implicit (provider-managed) and explicit prompt caching across OpenAI-compatible providers.
+
+Accepted request fields:
+
+- `prompt_cache_key` + `prompt_cache_retention` (OpenAI style)
+- `cache_control` (OpenRouter / Vercel / Claude style)
+- `cached_content` (Gemini style)
+
+```json
+{
+  "model": "anthropic/claude-sonnet-4.6",
+  "messages": [
+    {
+      "role": "system",
+      "content": "Reusable policy and instructions",
+      "cache_control": { "type": "ephemeral", "ttl": "1h" }
+    },
+    { "role": "user", "content": "Apply policy to this request." }
+  ]
+}
+```
+
+Provider behavior:
+
+- **OpenAI-compatible**: forwards `prompt_cache_key` and `prompt_cache_retention` as native provider options.
+- **Anthropic Claude**: maps top-level caching to Anthropic cache control, while message/part `cache_control` breakpoints are preserved.
+- **Google Gemini**: maps `cached_content` to Gemini `cachedContent`.
+- **Amazon Nova (Bedrock)**: maps `cache_control` to Bedrock `cachePoints` and inserts an automatic cache point on a stable prefix when none is provided.
+
 ## 🧪 Advanced Usage
 
 ### Logger Settings

package/dist/endpoints/chat-completions/converters.js

@@ -5,8 +5,9 @@ import { OpenAIError, toOpenAIError } from "../../errors/openai";
 import { toResponse } from "../../utils/response";
 // --- Request Flow ---
 export function convertToTextCallOptions(params) {
-    const { messages, tools, tool_choice, temperature, max_tokens, max_completion_tokens, response_format, reasoning_effort, reasoning, frequency_penalty, presence_penalty, seed, stop, top_p, ...rest } = params;
+    const { messages, tools, tool_choice, temperature, max_tokens, max_completion_tokens, response_format, reasoning_effort, reasoning, prompt_cache_key, prompt_cache_retention, cached_content, cache_control, frequency_penalty, presence_penalty, seed, stop, top_p, ...rest } = params;
     Object.assign(rest, parseReasoningOptions(reasoning_effort, reasoning));
+    Object.assign(rest, parsePromptCachingOptions(prompt_cache_key, prompt_cache_retention, cached_content, cache_control));
     const { toolChoice, activeTools } = convertToToolChoiceOptions(tool_choice);
     return {
         messages: convertToModelMessages(messages),
@@ -44,6 +45,11 @@ export function convertToModelMessages(messages) {
         if (message.role === "tool")
             continue;
         if (message.role === "system") {
+            if (message.cache_control) {
+                message.providerOptions = {
+                    unknown: { cache_control: message.cache_control },
+                };
+            }
             modelMessages.push(message);
             continue;
         }
@@ -67,15 +73,21 @@ function indexToolMessages(messages) {
     return map;
 }
 export function fromChatCompletionsUserMessage(message) {
-    return {
+    const out = {
         role: "user",
         content: Array.isArray(message.content)
             ? fromChatCompletionsContent(message.content)
             : message.content,
     };
+    if (message.cache_control) {
+        out.providerOptions = {
+            unknown: { cache_control: message.cache_control },
+        };
+    }
+    return out;
 }
 export function fromChatCompletionsAssistantMessage(message) {
-    const { tool_calls, role, content, extra_content, reasoning_details } = message;
+    const { tool_calls, role, content, extra_content, reasoning_details, cache_control } = message;
     const parts = [];
     if (reasoning_details?.length) {
         for (const detail of reasoning_details) {
@@ -111,10 +123,16 @@ export function fromChatCompletionsAssistantMessage(message) {
             : content;
         for (const part of inputContent) {
             if (part.type === "text") {
-                parts.push({
+                const textPart = {
                     type: "text",
                     text: part.text,
-                });
+                };
+                if (part.cache_control) {
+                    textPart.providerOptions = {
+                        unknown: { cache_control: part.cache_control },
+                    };
+                }
+                parts.push(textPart);
             }
         }
     }
@@ -141,6 +159,9 @@ export function fromChatCompletionsAssistantMessage(message) {
     if (extra_content) {
         out.providerOptions = extra_content;
     }
+    if (cache_control) {
+        ((out.providerOptions ??= { unknown: {} })["unknown"] ??= {})["cache_control"] = cache_control;
+    }
     return out;
 }
 export function fromChatCompletionsToolResultMessage(message, toolById) {
@@ -165,40 +186,68 @@ export function fromChatCompletionsContent(content) {
     return content.map((part) => {
         switch (part.type) {
             case "image_url":
-                return fromImageUrlPart(part.image_url.url);
+                return fromImageUrlPart(part.image_url.url, part.cache_control);
             case "file":
-                return fromFilePart(part.file.data, part.file.media_type, part.file.filename);
+                return fromFilePart(part.file.data, part.file.media_type, part.file.filename, part.cache_control);
             case "input_audio":
-                return fromFilePart(part.input_audio.data, `audio/${part.input_audio.format}`);
-            default:
-                return part;
+                return fromFilePart(part.input_audio.data, `audio/${part.input_audio.format}`, undefined, part.cache_control);
+            default: {
+                const out = {
+                    type: "text",
+                    text: part.text,
+                };
+                if (part.cache_control) {
+                    out.providerOptions = {
+                        unknown: { cache_control: part.cache_control },
+                    };
+                }
+                return out;
+            }
         }
     });
 }
-function fromImageUrlPart(url) {
+function fromImageUrlPart(url, cacheControl) {
     if (url.startsWith("data:")) {
         const { mimeType, base64Data } = parseDataUrl(url);
-        return fromFilePart(base64Data, mimeType);
+        return fromFilePart(base64Data, mimeType, undefined, cacheControl);
     }
-    return {
+    const out = {
         type: "image",
         image: new URL(url),
     };
+    if (cacheControl) {
+        out.providerOptions = {
+            unknown: { cache_control: cacheControl },
+        };
+    }
+    return out;
 }
-function fromFilePart(base64Data, mediaType, filename) {
+function fromFilePart(base64Data, mediaType, filename, cacheControl) {
     if (mediaType.startsWith("image/")) {
-        return {
+        const out = {
             type: "image",
             image: z.util.base64ToUint8Array(base64Data),
             mediaType,
         };
+        if (cacheControl) {
+            out.providerOptions = {
+                unknown: { cache_control: cacheControl },
+            };
+        }
+        return out;
     }
-    return {
+    const out = {
         type: "file",
         data: z.util.base64ToUint8Array(base64Data),
         filename,
         mediaType,
     };
+    if (cacheControl) {
+        out.providerOptions = {
+            unknown: { cache_control: cacheControl },
+        };
+    }
+    return out;
 }
 export const convertToToolSet = (tools) => {
     if (!tools) {
@@ -295,6 +344,31 @@ function parseReasoningOptions(reasoning_effort, reasoning) {
     }
     return out;
 }
+function parsePromptCachingOptions(prompt_cache_key, prompt_cache_retention, cached_content, cache_control) {
+    const out = {};
+    const syncedCacheKey = prompt_cache_key ?? cached_content;
+    const syncedCachedContent = cached_content ?? prompt_cache_key;
+    let syncedCacheRetention = prompt_cache_retention;
+    if (!syncedCacheRetention && cache_control?.ttl) {
+        syncedCacheRetention = cache_control.ttl === "24h" ? "24h" : "in_memory";
+    }
+    let syncedCacheControl = cache_control;
+    if (!syncedCacheControl && syncedCacheRetention) {
+        syncedCacheControl = {
+            type: "ephemeral",
+            ttl: syncedCacheRetention === "24h" ? "24h" : "5m",
+        };
+    }
+    if (syncedCacheKey)
+        out["prompt_cache_key"] = syncedCacheKey;
+    if (syncedCacheRetention)
+        out["prompt_cache_retention"] = syncedCacheRetention;
+    if (syncedCachedContent)
+        out["cached_content"] = syncedCachedContent;
+    if (syncedCacheControl)
+        out["cache_control"] = syncedCacheControl;
+    return out;
+}
 // --- Response Flow ---
 export function toChatCompletions(result, model) {
     return {
@@ -494,8 +568,16 @@ export function toChatCompletionsUsage(usage) {
     if (reasoning !== undefined)
         out.completion_tokens_details = { reasoning_tokens: reasoning };
     const cached = usage.inputTokenDetails?.cacheReadTokens;
-    if (cached !== undefined)
-        out.prompt_tokens_details = { cached_tokens: cached };
+    const cacheWrite = usage.inputTokenDetails?.cacheWriteTokens;
+    if (cached !== undefined || cacheWrite !== undefined) {
+        out.prompt_tokens_details = {};
+        if (cached !== undefined) {
+            out.prompt_tokens_details.cached_tokens = cached;
+        }
+        if (cacheWrite !== undefined) {
+            out.prompt_tokens_details.cache_write_tokens = cacheWrite;
+        }
+    }
     return out;
 }
 export function toChatCompletionsToolCall(id, name, args, providerMetadata) {

package/dist/endpoints/chat-completions/handler.js

@@ -28,6 +28,7 @@ export const chatCompletions = (config) => {
         catch {
             throw new GatewayError("Invalid JSON", 400);
         }
+        logger.trace({ requestId: ctx.requestId, body: ctx.body }, "[chat] ChatCompletionsBody");
         addSpanEvent("hebo.request.deserialized");
         const parsed = ChatCompletionsBodySchema.safeParse(ctx.body);
         if (!parsed.success) {
@@ -37,7 +38,8 @@ export const chatCompletions = (config) => {
         ctx.body = parsed.data;
         addSpanEvent("hebo.request.parsed");
         if (hooks?.before) {
-            ctx.body = (await hooks.before(ctx)) ?? ctx.body;
+            ctx.body =
+                (await hooks.before(ctx)) ?? ctx.body;
             addSpanEvent("hebo.hooks.before.completed");
         }
         // Resolve model + provider (hooks may override defaults).
@@ -70,7 +72,7 @@ export const chatCompletions = (config) => {
             options: textOptions,
         }, "[chat] AI SDK options");
         addSpanEvent("hebo.options.prepared");
-        setSpanAttributes(getChatRequestAttributes(inputs, genAiSignalLevel));
+        setSpanAttributes(getChatRequestAttributes(ctx.body, genAiSignalLevel));
         // Build middleware chain (model -> forward params -> provider).
         const languageModelWithMiddleware = wrapLanguageModel({
             model: languageModel,
@@ -93,6 +95,7 @@ export const chatCompletions = (config) => {
                 onFinish: (res) => {
                     addSpanEvent("hebo.ai-sdk.completed");
                     const streamResult = toChatCompletions(res, ctx.resolvedModelId);
+                    logger.trace({ requestId: ctx.requestId, result: streamResult }, "[chat] ChatCompletions");
                     addSpanEvent("hebo.result.transformed");
                     const genAiResponseAttrs = getChatResponseAttributes(streamResult, genAiSignalLevel);
                     setSpanAttributes(genAiResponseAttrs);
@@ -129,6 +132,7 @@ export const chatCompletions = (config) => {
         addSpanEvent("hebo.ai-sdk.completed");
         // Transform result.
         ctx.result = toChatCompletions(result, ctx.resolvedModelId);
+        logger.trace({ requestId: ctx.requestId, result: ctx.result }, "[chat] ChatCompletions");
         addSpanEvent("hebo.result.transformed");
         const genAiResponseAttrs = getChatResponseAttributes(ctx.result, genAiSignalLevel);
         setSpanAttributes(genAiResponseAttrs);

package/dist/endpoints/chat-completions/index.d.ts

@@ -1,3 +1,4 @@
 export * from "./converters";
 export * from "./handler";
 export * from "./schema";
+export * from "./otel";

package/dist/endpoints/chat-completions/index.js

@@ -1,3 +1,4 @@
 export * from "./converters";
 export * from "./handler";
 export * from "./schema";
+export * from "./otel";

package/dist/endpoints/chat-completions/otel.js

@@ -77,6 +77,7 @@ export const getChatRequestAttributes = (inputs, signalLevel) => {
     }
     if (signalLevel !== "required") {
         Object.assign(attrs, {
+            // FUTURE: add reasoning info
             "gen_ai.request.stream": inputs.stream,
             "gen_ai.request.frequency_penalty": inputs.frequency_penalty,
             "gen_ai.request.max_tokens": inputs.max_completion_tokens,