@hebo-ai/gateway 0.5.1 → 0.6.0-rc0

package/README.md

@@ -38,7 +38,7 @@ bun install @hebo-ai/gateway
 - Runtime Support
   - [Vercel Edge](#vercel-edge) | [Cloudflare Workers](#cloudflare-workers) | [Deno Deploy](#deno-deploy) | [AWS Lambda](#aws-lambda)
 - OpenAI Extensions
-  - [Reasoning](#reasoning)
+  - [Reasoning](#reasoning) | [Prompt Caching](#prompt-caching)
 - Advanced Usage
   - [Passing Framework State to Hooks](#passing-framework-state-to-hooks) | [Selective Route Mounting](#selective-route-mounting) | [Low-level Schemas & Converters](#low-level-schemas--converters)
 
@@ -565,6 +565,37 @@ Advanced models (like Anthropic Claude 3.7 or Gemini 3) surface structured reaso
 
 For **Gemini 3** models, returning the thought signature via `extra_content` is mandatory to resume the chain-of-thought; failing to do so may result in errors or degraded performance.
 
+### Prompt Caching
+
+The chat completions endpoint supports both implicit (provider-managed) and explicit prompt caching across OpenAI-compatible providers.
+
+Accepted request fields:
+
+- `prompt_cache_key` + `prompt_cache_retention` (OpenAI style)
+- `cache_control` (OpenRouter / Vercel / Claude style)
+- `cached_content` (Gemini style)
+
+```json
+{
+  "model": "anthropic/claude-sonnet-4.6",
+  "messages": [
+    {
+      "role": "system",
+      "content": "Reusable policy and instructions",
+      "cache_control": { "type": "ephemeral", "ttl": "1h" }
+    },
+    { "role": "user", "content": "Apply policy to this request." }
+  ]
+}
+```
+
+Provider behavior:
+
+- **OpenAI-compatible**: forwards `prompt_cache_key` and `prompt_cache_retention` as native provider options.
+- **Anthropic Claude**: maps top-level caching to Anthropic cache control, while message/part `cache_control` breakpoints are preserved.
+- **Google Gemini**: maps `cached_content` to Gemini `cachedContent`.
+- **Amazon Nova (Bedrock)**: maps `cache_control` to Bedrock `cachePoints` and inserts an automatic cache point on a stable prefix when none is provided.
+
 ## 🧪 Advanced Usage
 
 ### Logger Settings

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hebo-ai/gateway",
-  "version": "0.5.1",
+  "version": "0.6.0-rc0",
   "description": "AI gateway as a framework. For full control over models, routing & lifecycle. OpenAI-compatible /chat/completions, /embeddings & /models.",
   "keywords": [
     "ai",
@@ -140,6 +140,11 @@
       "types": "./dist/providers/voyage/index.d.ts",
       "import": "./dist/providers/voyage/index.js",
       "dev-source": "./src/providers/voyage/index.ts"
+    },
+    "./telemetry": {
+      "types": "./dist/telemetry/index.d.ts",
+      "import": "./dist/telemetry/index.js",
+      "dev-source": "./src/telemetry/index.ts"
     }
   },
   "scripts": {
@@ -155,33 +160,33 @@
   },
   "dependencies": {
     "@ai-sdk/provider": "^3.0.8",
-    "ai": "^6.0.97",
+    "ai": "^6.0.101",
     "zod": "^4.3.6"
   },
   "devDependencies": {
-    "@ai-sdk/amazon-bedrock": "^4.0.63",
-    "@ai-sdk/anthropic": "^3.0.46",
+    "@ai-sdk/amazon-bedrock": "^4.0.65",
+    "@ai-sdk/anthropic": "^3.0.47",
     "@ai-sdk/cohere": "^3.0.21",
-    "@ai-sdk/google-vertex": "^4.0.61",
+    "@ai-sdk/google-vertex": "^4.0.63",
     "@ai-sdk/groq": "^3.0.24",
-    "@ai-sdk/openai": "^3.0.30",
-    "@aws-sdk/credential-providers": "^3.995.0",
+    "@ai-sdk/openai": "^3.0.34",
+    "@aws-sdk/credential-providers": "^3.998.0",
     "@langfuse/otel": "^4.6.1",
     "@mjackson/node-fetch-server": "^0.7.0",
     "@opentelemetry/api": "^1.9.0",
     "@opentelemetry/context-async-hooks": "^2.5.1",
     "@opentelemetry/sdk-trace-base": "^2.5.1",
-    "@tanstack/react-router": "^1.161.3",
-    "@tanstack/react-start": "^1.161.3",
+    "@tanstack/react-router": "^1.163.2",
+    "@tanstack/react-start": "^1.163.2",
     "@types/bun": "latest",
     "@types/react": "^19.2.14",
     "@types/react-dom": "^19.2.3",
-    "elysia": "^1.4.25",
-    "hono": "^4.12.0",
+    "elysia": "^1.4.26",
+    "hono": "^4.12.2",
     "lefthook": "^2.1.1",
     "next": "^16.1.6",
     "oxfmt": "^0.24.0",
-    "oxlint": "^1.49.0",
+    "oxlint": "^1.50.0",
     "pino": "^10.3.1",
     "typescript": "^5.9.3",
     "vite": "^7.3.1",

package/src/endpoints/chat-completions/converters.test.ts

@@ -1,4 +1,4 @@
-import type { GenerateTextResult, ToolSet, Output } from "ai";
+import type { GenerateTextResult, ToolSet, Output, LanguageModelUsage } from "ai";
 
 import { describe, expect, test } from "bun:test";
 
@@ -8,6 +8,7 @@ import {
   convertToTextCallOptions,
   toChatCompletionsAssistantMessage,
   toChatCompletionsToolCall,
+  toChatCompletionsUsage,
   fromChatCompletionsAssistantMessage,
   fromChatCompletionsToolResultMessage,
 } from "./converters";
@@ -520,6 +521,89 @@ describe("Chat Completions Converters", () => {
       expect(result.tools).toBeDefined();
       expect(Object.keys(result.tools!)).toEqual(["get_weather"]);
     });
+
+    test("should map prompt cache options into providerOptions.unknown", () => {
+      const result = convertToTextCallOptions({
+        messages: [{ role: "system", content: "You are concise." }],
+        prompt_cache_key: "tenant:docs:v1",
+        prompt_cache_retention: "24h",
+      });
+
+      expect(result.providerOptions).toEqual({
+        unknown: {
+          prompt_cache_key: "tenant:docs:v1",
+          prompt_cache_retention: "24h",
+          cached_content: "tenant:docs:v1",
+          cache_control: {
+            type: "ephemeral",
+            ttl: "24h",
+          },
+        },
+      });
+    });
+
+    test("should sync retention from cache_control ttl", () => {
+      const result = convertToTextCallOptions({
+        messages: [{ role: "system", content: "You are concise." }],
+        cache_control: {
+          type: "ephemeral",
+          ttl: "5m",
+        },
+      });
+
+      expect(result.providerOptions).toEqual({
+        unknown: {
+          prompt_cache_retention: "in_memory",
+          cache_control: {
+            type: "ephemeral",
+            ttl: "5m",
+          },
+        },
+      });
+    });
+
+    test("should preserve cache_control on message and content parts", () => {
+      const result = convertToTextCallOptions({
+        messages: [
+          {
+            role: "system",
+            content: "Policy block",
+            cache_control: { type: "ephemeral", ttl: "1h" },
+          },
+          {
+            role: "user",
+            content: [{ type: "text", text: "Question", cache_control: { type: "ephemeral" } }],
+          },
+        ],
+      });
+
+      expect((result.messages[0] as any).providerOptions.unknown.cache_control).toEqual({
+        type: "ephemeral",
+        ttl: "1h",
+      });
+      expect((result.messages[1] as any).content[0].providerOptions.unknown.cache_control).toEqual({
+        type: "ephemeral",
+      });
+    });
+  });
+
+  describe("toChatCompletionsUsage", () => {
+    test("should include cached token details", () => {
+      const usage = toChatCompletionsUsage({
+        inputTokens: 100,
+        outputTokens: 20,
+        totalTokens: 120,
+        inputTokenDetails: {
+          cacheReadTokens: 60,
+          cacheWriteTokens: 10,
+        },
+      } as LanguageModelUsage);
+
+      expect(usage.prompt_tokens_details).toEqual({
+        cached_tokens: 60,
+        cache_write_tokens: 10,
+      });
+    });
   });
 
   describe("toChatCompletionsToolCall", () => {

package/src/endpoints/chat-completions/converters.ts

@@ -17,6 +17,9 @@ import type {
   AssistantModelMessage,
   ToolModelMessage,
   UserModelMessage,
+  TextPart,
+  ImagePart,
+  FilePart,
 } from "ai";
 
 import { Output, jsonSchema, tool } from "ai";
@@ -28,7 +31,6 @@ import type {
   ChatCompletionsToolChoice,
   ChatCompletionsContentPart,
   ChatCompletionsMessage,
-  ChatCompletionsSystemMessage,
   ChatCompletionsUserMessage,
   ChatCompletionsAssistantMessage,
   ChatCompletionsToolMessage,
@@ -46,6 +48,7 @@ import type {
   ChatCompletionsReasoningDetail,
   ChatCompletionsResponseFormat,
   ChatCompletionsContentPartText,
+  ChatCompletionsCacheControl,
 } from "./schema";
 
 import { GatewayError } from "../../errors/gateway";
@@ -81,6 +84,10 @@ export function convertToTextCallOptions(params: ChatCompletionsInputs): TextCal
     response_format,
     reasoning_effort,
     reasoning,
+    prompt_cache_key,
+    prompt_cache_retention,
+    cached_content,
+    cache_control,
     frequency_penalty,
     presence_penalty,
     seed,
@@ -90,6 +97,15 @@ export function convertToTextCallOptions(params: ChatCompletionsInputs): TextCal
   } = params;
 
   Object.assign(rest, parseReasoningOptions(reasoning_effort, reasoning));
+  Object.assign(
+    rest,
+    parsePromptCachingOptions(
+      prompt_cache_key,
+      prompt_cache_retention,
+      cached_content,
+      cache_control,
+    ),
+  );
 
   const { toolChoice, activeTools } = convertToToolChoiceOptions(tool_choice);
 
@@ -133,7 +149,12 @@ export function convertToModelMessages(messages: ChatCompletionsMessage[]): Mode
     if (message.role === "tool") continue;
 
     if (message.role === "system") {
-      modelMessages.push(message satisfies ChatCompletionsSystemMessage);
+      if (message.cache_control) {
+        (message as ModelMessage).providerOptions = {
+          unknown: { cache_control: message.cache_control },
+        };
+      }
+      modelMessages.push(message);
       continue;
     }
 
@@ -161,18 +182,24 @@ function indexToolMessages(messages: ChatCompletionsMessage[]) {
 export function fromChatCompletionsUserMessage(
   message: ChatCompletionsUserMessage,
 ): UserModelMessage {
-  return {
+  const out: UserModelMessage = {
     role: "user",
     content: Array.isArray(message.content)
       ? fromChatCompletionsContent(message.content)
       : message.content,
   };
+  if (message.cache_control) {
+    out.providerOptions = {
+      unknown: { cache_control: message.cache_control },
+    };
+  }
+  return out;
 }
 
 export function fromChatCompletionsAssistantMessage(
   message: ChatCompletionsAssistantMessage,
 ): AssistantModelMessage {
-  const { tool_calls, role, content, extra_content, reasoning_details } = message;
+  const { tool_calls, role, content, extra_content, reasoning_details, cache_control } = message;
 
   const parts: AssistantContent = [];
 
@@ -211,10 +238,16 @@ export function fromChatCompletionsAssistantMessage(
         : content;
     for (const part of inputContent) {
       if (part.type === "text") {
-        parts.push({
+        const textPart: TextPart = {
           type: "text",
           text: part.text,
-        });
+        };
+        if (part.cache_control) {
+          textPart.providerOptions = {
+            unknown: { cache_control: part.cache_control },
+          };
+        }
+        parts.push(textPart);
       }
     }
   }
@@ -245,6 +278,10 @@ export function fromChatCompletionsAssistantMessage(
     out.providerOptions = extra_content as SharedV3ProviderOptions;
   }
 
+  if (cache_control) {
+    ((out.providerOptions ??= { unknown: {} })["unknown"] ??= {})["cache_control"] = cache_control;
+  }
+
   return out;
 }
 
@@ -275,44 +312,87 @@ export function fromChatCompletionsContent(content: ChatCompletionsContentPart[]
   return content.map((part) => {
     switch (part.type) {
       case "image_url":
-        return fromImageUrlPart(part.image_url.url);
+        return fromImageUrlPart(part.image_url.url, part.cache_control);
       case "file":
-        return fromFilePart(part.file.data, part.file.media_type, part.file.filename);
+        return fromFilePart(
+          part.file.data,
+          part.file.media_type,
+          part.file.filename,
+          part.cache_control,
+        );
       case "input_audio":
-        return fromFilePart(part.input_audio.data, `audio/${part.input_audio.format}`);
-      default:
-        return part;
+        return fromFilePart(
+          part.input_audio.data,
+          `audio/${part.input_audio.format}`,
+          undefined,
+          part.cache_control,
+        );
+      default: {
+        const out: TextPart = {
+          type: "text" as const,
+          text: part.text,
+        };
+        if (part.cache_control) {
+          out.providerOptions = {
+            unknown: { cache_control: part.cache_control },
+          };
+        }
+        return out;
+      }
     }
   });
 }
 
-function fromImageUrlPart(url: string) {
+function fromImageUrlPart(url: string, cacheControl?: ChatCompletionsCacheControl) {
   if (url.startsWith("data:")) {
     const { mimeType, base64Data } = parseDataUrl(url);
-    return fromFilePart(base64Data, mimeType);
+    return fromFilePart(base64Data, mimeType, undefined, cacheControl);
   }
 
-  return {
+  const out: ImagePart = {
     type: "image" as const,
     image: new URL(url),
   };
+  if (cacheControl) {
+    out.providerOptions = {
+      unknown: { cache_control: cacheControl },
+    };
+  }
+  return out;
 }
 
-function fromFilePart(base64Data: string, mediaType: string, filename?: string) {
+function fromFilePart(
+  base64Data: string,
+  mediaType: string,
+  filename?: string,
+  cacheControl?: ChatCompletionsCacheControl,
+) {
   if (mediaType.startsWith("image/")) {
-    return {
+    const out: ImagePart = {
       type: "image" as const,
       image: z.util.base64ToUint8Array(base64Data),
       mediaType,
     };
+    if (cacheControl) {
+      out.providerOptions = {
+        unknown: { cache_control: cacheControl },
+      };
+    }
+    return out;
   }
 
-  return {
+  const out: FilePart = {
     type: "file" as const,
     data: z.util.base64ToUint8Array(base64Data),
     filename,
     mediaType,
   };
+  if (cacheControl) {
+    out.providerOptions = {
+      unknown: { cache_control: cacheControl },
+    };
+  }
+  return out;
 }
 
 export const convertToToolSet = (tools: ChatCompletionsTool[] | undefined): ToolSet | undefined => {
@@ -438,6 +518,38 @@ function parseReasoningOptions(
   return out;
 }
 
+function parsePromptCachingOptions(
+  prompt_cache_key: string | undefined,
+  prompt_cache_retention: "in_memory" | "24h" | undefined,
+  cached_content: string | undefined,
+  cache_control: ChatCompletionsCacheControl | undefined,
+) {
+  const out: Record<string, unknown> = {};
+
+  const syncedCacheKey = prompt_cache_key ?? cached_content;
+  const syncedCachedContent = cached_content ?? prompt_cache_key;
+
+  let syncedCacheRetention = prompt_cache_retention;
+  if (!syncedCacheRetention && cache_control?.ttl) {
+    syncedCacheRetention = cache_control.ttl === "24h" ? "24h" : "in_memory";
+  }
+
+  let syncedCacheControl = cache_control;
+  if (!syncedCacheControl && syncedCacheRetention) {
+    syncedCacheControl = {
+      type: "ephemeral",
+      ttl: syncedCacheRetention === "24h" ? "24h" : "5m",
+    };
+  }
+
+  if (syncedCacheKey) out["prompt_cache_key"] = syncedCacheKey;
+  if (syncedCacheRetention) out["prompt_cache_retention"] = syncedCacheRetention;
+  if (syncedCachedContent) out["cached_content"] = syncedCachedContent;
+  if (syncedCacheControl) out["cache_control"] = syncedCacheControl;
+
+  return out;
+}
+
 // --- Response Flow ---
 
 export function toChatCompletions(
@@ -731,7 +843,16 @@ export function toChatCompletionsUsage(usage: LanguageModelUsage): ChatCompletio
   if (reasoning !== undefined) out.completion_tokens_details = { reasoning_tokens: reasoning };
 
   const cached = usage.inputTokenDetails?.cacheReadTokens;
-  if (cached !== undefined) out.prompt_tokens_details = { cached_tokens: cached };
+  const cacheWrite = usage.inputTokenDetails?.cacheWriteTokens;
+  if (cached !== undefined || cacheWrite !== undefined) {
+    out.prompt_tokens_details = {};
+    if (cached !== undefined) {
+      out.prompt_tokens_details.cached_tokens = cached;
+    }
+    if (cacheWrite !== undefined) {
+      out.prompt_tokens_details.cache_write_tokens = cacheWrite;
+    }
+  }
 
   return out;
 }

package/src/endpoints/chat-completions/handler.test.ts

@@ -197,6 +197,7 @@ describe("Chat Completions Handler", () => {
         },
         prompt_tokens_details: {
           cached_tokens: 20,
+          cache_write_tokens: 0,
         },
       },
       provider_metadata: { provider: { key: "value" } },
@@ -284,6 +285,7 @@ describe("Chat Completions Handler", () => {
         },
         prompt_tokens_details: {
           cached_tokens: 20,
+          cache_write_tokens: 0,
         },
       },
       provider_metadata: { provider: { key: "value" } },

package/src/endpoints/chat-completions/handler.ts

@@ -36,7 +36,7 @@ import {
   getChatRequestAttributes,
   getChatResponseAttributes,
 } from "./otel";
-import { ChatCompletionsBodySchema } from "./schema";
+import { ChatCompletionsBodySchema, type ChatCompletionsBody } from "./schema";
 
 export const chatCompletions = (config: GatewayConfig): Endpoint => {
   const hooks = config.hooks;
@@ -57,6 +57,7 @@ export const chatCompletions = (config: GatewayConfig): Endpoint => {
     } catch {
       throw new GatewayError("Invalid JSON", 400);
     }
+    logger.trace({ requestId: ctx.requestId, body: ctx.body }, "[chat] ChatCompletionsBody");
     addSpanEvent("hebo.request.deserialized");
 
     const parsed = ChatCompletionsBodySchema.safeParse(ctx.body);
@@ -68,7 +69,8 @@ export const chatCompletions = (config: GatewayConfig): Endpoint => {
     addSpanEvent("hebo.request.parsed");
 
     if (hooks?.before) {
-      ctx.body = (await hooks.before(ctx as BeforeHookContext)) ?? ctx.body;
+      ctx.body =
+        ((await hooks.before(ctx as BeforeHookContext)) as ChatCompletionsBody) ?? ctx.body;
       addSpanEvent("hebo.hooks.before.completed");
     }
 
@@ -110,7 +112,7 @@ export const chatCompletions = (config: GatewayConfig): Endpoint => {
       "[chat] AI SDK options",
     );
     addSpanEvent("hebo.options.prepared");
-    setSpanAttributes(getChatRequestAttributes(inputs, genAiSignalLevel));
+    setSpanAttributes(getChatRequestAttributes(ctx.body, genAiSignalLevel));
 
     // Build middleware chain (model -> forward params -> provider).
     const languageModelWithMiddleware = wrapLanguageModel({
@@ -138,6 +140,10 @@ export const chatCompletions = (config: GatewayConfig): Endpoint => {
             res as unknown as GenerateTextResult<ToolSet, Output.Output>,
             ctx.resolvedModelId!,
           );
+          logger.trace(
+            { requestId: ctx.requestId, result: streamResult },
+            "[chat] ChatCompletions",
+          );
           addSpanEvent("hebo.result.transformed");
 
           const genAiResponseAttrs = getChatResponseAttributes(streamResult, genAiSignalLevel);
@@ -180,6 +186,7 @@ export const chatCompletions = (config: GatewayConfig): Endpoint => {
 
     // Transform result.
     ctx.result = toChatCompletions(result, ctx.resolvedModelId);
+    logger.trace({ requestId: ctx.requestId, result: ctx.result }, "[chat] ChatCompletions");
     addSpanEvent("hebo.result.transformed");
 
     const genAiResponseAttrs = getChatResponseAttributes(ctx.result, genAiSignalLevel);

package/src/endpoints/chat-completions/index.ts

@@ -1,3 +1,4 @@
 export * from "./converters";
 export * from "./handler";
 export * from "./schema";
+export * from "./otel";

package/src/endpoints/chat-completions/otel.ts

@@ -98,6 +98,7 @@ export const getChatRequestAttributes = (
 
   if (signalLevel !== "required") {
     Object.assign(attrs, {
+      // FUTURE: add reasoning info
       "gen_ai.request.stream": inputs.stream,
       "gen_ai.request.frequency_penalty": inputs.frequency_penalty,
       "gen_ai.request.max_tokens": inputs.max_completion_tokens,