@hebo-ai/gateway 0.5.2 → 0.6.0-rc0

package/src/lifecycle.ts

@@ -10,7 +10,7 @@ import { GatewayError } from "./errors/gateway";
 import { toOpenAIErrorResponse } from "./errors/openai";
 import { logger } from "./logger";
 import { getBaggageAttributes } from "./telemetry/baggage";
-import { initFetch } from "./telemetry/fetch";
+import { instrumentFetch } from "./telemetry/fetch";
 import { getRequestAttributes, getResponseAttributes } from "./telemetry/http";
 import { recordV8jsMemory } from "./telemetry/memory";
 import { addSpanEvent, setSpanEventsEnabled, setSpanTracer, startSpan } from "./telemetry/span";
@@ -27,7 +27,7 @@ export const winterCgHandler = (
   if (parsedConfig.telemetry?.enabled) {
     setSpanTracer(parsedConfig.telemetry?.tracer);
     setSpanEventsEnabled(parsedConfig.telemetry?.signals?.hebo);
-    initFetch(parsedConfig.telemetry?.signals?.hebo);
+    instrumentFetch(parsedConfig.telemetry?.signals?.hebo);
   }
 
   return async (request: Request, state?: Record<string, unknown>): Promise<Response> => {

package/src/models/anthropic/middleware.test.ts

@@ -3,7 +3,7 @@ import { expect, test } from "bun:test";
 
 import { modelMiddlewareMatcher } from "../../middleware/matcher";
 import { CANONICAL_MODEL_IDS } from "../../models/types";
-import { claudeReasoningMiddleware } from "./middleware";
+import { claudePromptCachingMiddleware, claudeReasoningMiddleware } from "./middleware";
 
 test("claudeReasoningMiddleware > matching patterns", () => {
   const matching = [
@@ -27,6 +27,7 @@ test("claudeReasoningMiddleware > matching patterns", () => {
   for (const id of matching) {
     const middleware = modelMiddlewareMatcher.resolve({ kind: "text", modelId: id });
     expect(middleware).toContain(claudeReasoningMiddleware);
+    expect(middleware).toContain(claudePromptCachingMiddleware);
   }
 
   for (const id of nonMatching) {
@@ -35,6 +36,49 @@ test("claudeReasoningMiddleware > matching patterns", () => {
   }
 });
 
+test("claudePromptCachingMiddleware > should not auto-enable top-level cache control", async () => {
+  const params = {
+    prompt: [],
+    providerOptions: {
+      unknown: {},
+    },
+  };
+
+  const result = await claudePromptCachingMiddleware.transformParams!({
+    type: "generate",
+    params,
+    model: new MockLanguageModelV3({ modelId: "anthropic/claude-sonnet-4.6" }),
+  });
+
+  expect(result.providerOptions).toEqual({
+    unknown: {},
+  });
+});
+
+test("claudePromptCachingMiddleware > should map cache_control ttl", async () => {
+  const params = {
+    prompt: [],
+    providerOptions: {
+      unknown: {
+        cache_control: { type: "ephemeral", ttl: "1h" },
+      },
+    },
+  };
+
+  const result = await claudePromptCachingMiddleware.transformParams!({
+    type: "generate",
+    params,
+    model: new MockLanguageModelV3({ modelId: "anthropic/claude-sonnet-4.6" }),
+  });
+
+  expect(result.providerOptions).toEqual({
+    anthropic: {
+      cacheControl: { type: "ephemeral", ttl: "1h" },
+    },
+    unknown: {},
+  });
+});
+
 test("claudeReasoningMiddleware > should transform reasoning_effort string to thinking budget", async () => {
   const params = {
     prompt: [],

package/src/models/anthropic/middleware.ts

@@ -1,6 +1,7 @@
 import type { LanguageModelMiddleware } from "ai";
 
 import type {
+  ChatCompletionsCacheControl,
   ChatCompletionsReasoningConfig,
   ChatCompletionsReasoningEffort,
 } from "../../endpoints/chat-completions/schema";
@@ -123,6 +124,25 @@ export const claudeReasoningMiddleware: LanguageModelMiddleware = {
   },
 };
 
+// https://platform.claude.com/docs/en/build-with-claude/prompt-caching
+export const claudePromptCachingMiddleware: LanguageModelMiddleware = {
+  specificationVersion: "v3",
+  // eslint-disable-next-line require-await
+  transformParams: async ({ params }) => {
+    const unknown = params.providerOptions?.["unknown"];
+    if (!unknown) return params;
+
+    const cacheControl = unknown["cache_control"] as ChatCompletionsCacheControl;
+    if (cacheControl) {
+      (params.providerOptions!["anthropic"] ??= {})["cacheControl"] = cacheControl;
+    }
+
+    delete unknown["cache_control"];
+
+    return params;
+  },
+};
+
 modelMiddlewareMatcher.useForModel(["anthropic/claude-*3*7*", "anthropic/claude-*4*"], {
-  language: [claudeReasoningMiddleware],
+  language: [claudeReasoningMiddleware, claudePromptCachingMiddleware],
 });

package/src/models/google/middleware.test.ts

@@ -4,7 +4,11 @@ import { expect, test } from "bun:test";
 import { modelMiddlewareMatcher } from "../../middleware/matcher";
 import { calculateReasoningBudgetFromEffort } from "../../middleware/utils";
 import { CANONICAL_MODEL_IDS } from "../../models/types";
-import { geminiDimensionsMiddleware, geminiReasoningMiddleware } from "./middleware";
+import {
+  geminiDimensionsMiddleware,
+  geminiPromptCachingMiddleware,
+  geminiReasoningMiddleware,
+} from "./middleware";
 
 test("geminiReasoningMiddleware > matching patterns", () => {
   const matching = [
@@ -21,6 +25,7 @@ test("geminiReasoningMiddleware > matching patterns", () => {
   for (const id of matching) {
     const middleware = modelMiddlewareMatcher.resolve({ kind: "text", modelId: id });
     expect(middleware).toContain(geminiReasoningMiddleware);
+    expect(middleware).toContain(geminiPromptCachingMiddleware);
   }
 
   for (const id of nonMatching) {
@@ -29,6 +34,30 @@ test("geminiReasoningMiddleware > matching patterns", () => {
   }
 });
 
+test("geminiPromptCachingMiddleware > should map normalized cached_content", async () => {
+  const params = {
+    prompt: [],
+    providerOptions: {
+      unknown: {
+        cached_content: "cachedContents/reusable",
+      },
+    },
+  };
+
+  const result = await geminiPromptCachingMiddleware.transformParams!({
+    type: "generate",
+    params,
+    model: new MockLanguageModelV3({ modelId: "google/gemini-2.5-flash" }),
+  });
+
+  expect(result.providerOptions).toEqual({
+    google: {
+      cachedContent: "cachedContents/reusable",
+    },
+    unknown: {},
+  });
+});
+
 test("geminiDimensionsMiddleware > matching patterns", () => {
   const matching = ["google/gemini-embedding-001"];
   const nonMatching = [

package/src/models/google/middleware.ts

@@ -123,10 +123,29 @@ export const geminiReasoningMiddleware: LanguageModelMiddleware = {
   },
 };
 
+// https://ai.google.dev/gemini-api/docs/caching
+// FUTURE: auto-create cached_content for message-level cache_control blocks
+export const geminiPromptCachingMiddleware: LanguageModelMiddleware = {
+  specificationVersion: "v3",
+  // eslint-disable-next-line require-await
+  transformParams: async ({ params }) => {
+    const unknown = params.providerOptions?.["unknown"];
+    if (!unknown) return params;
+
+    const cachedContent = unknown["cached_content"] as string | undefined;
+    if (cachedContent) {
+      (params.providerOptions!["google"] ??= {})["cachedContent"] = cachedContent;
+    }
+
+    delete unknown["cached_content"];
+    return params;
+  },
+};
+
 modelMiddlewareMatcher.useForModel("google/gemini-*embedding-*", {
   embedding: [geminiDimensionsMiddleware],
 });
 
 modelMiddlewareMatcher.useForModel(["google/gemini-2*", "google/gemini-3*"], {
-  language: [geminiReasoningMiddleware],
+  language: [geminiReasoningMiddleware, geminiPromptCachingMiddleware],
 });

package/src/models/openai/middleware.test.ts

@@ -3,7 +3,11 @@ import { expect, test } from "bun:test";
 
 import { modelMiddlewareMatcher } from "../../middleware/matcher";
 import { CANONICAL_MODEL_IDS } from "../../models/types";
-import { openAIDimensionsMiddleware, openAIReasoningMiddleware } from "./middleware";
+import {
+  openAIDimensionsMiddleware,
+  openAIPromptCachingMiddleware,
+  openAIReasoningMiddleware,
+} from "./middleware";
 
 test("openAI middleware > matching patterns", () => {
   const languageMatching = [
@@ -21,6 +25,7 @@ test("openAI middleware > matching patterns", () => {
   for (const id of languageMatching) {
     const middleware = modelMiddlewareMatcher.resolve({ kind: "text", modelId: id });
     expect(middleware).toContain(openAIReasoningMiddleware);
+    expect(middleware).toContain(openAIPromptCachingMiddleware);
   }
 
   for (const id of languageNonMatching) {
@@ -46,6 +51,32 @@ test("openAI middleware > matching patterns", () => {
   }
 });
 
+test("openAIPromptCachingMiddleware > should map key and retention", async () => {
+  const params = {
+    prompt: [],
+    providerOptions: {
+      unknown: {
+        prompt_cache_key: "tenant:shared:legal-v1",
+        prompt_cache_retention: "24h",
+      },
+    },
+  };
+
+  const result = await openAIPromptCachingMiddleware.transformParams!({
+    type: "generate",
+    params,
+    model: new MockLanguageModelV3({ modelId: "openai/gpt-5" }),
+  });
+
+  expect(result.providerOptions).toEqual({
+    openai: {
+      promptCacheKey: "tenant:shared:legal-v1",
+      promptCacheRetention: "24h",
+    },
+    unknown: {},
+  });
+});
+
 test("openAIReasoningMiddleware > should map reasoning effort to OpenAI provider options", async () => {
   const params = {
     prompt: [],

package/src/models/openai/middleware.ts

@@ -70,10 +70,34 @@ export const openAIReasoningMiddleware: LanguageModelMiddleware = {
   },
 };
 
+// https://developers.openai.com/api/docs/guides/prompt-caching/
+export const openAIPromptCachingMiddleware: LanguageModelMiddleware = {
+  specificationVersion: "v3",
+  // eslint-disable-next-line require-await
+  transformParams: async ({ params }) => {
+    const unknown = params.providerOptions?.["unknown"];
+    if (!unknown) return params;
+
+    const key = unknown["prompt_cache_key"] as string | undefined;
+    const retention = unknown["prompt_cache_retention"] as "in_memory" | "24h" | undefined;
+
+    if (key || retention) {
+      const target = (params.providerOptions!["openai"] ??= {});
+      if (key) target["promptCacheKey"] = key;
+      if (retention) target["promptCacheRetention"] = retention;
+    }
+
+    delete unknown["prompt_cache_key"];
+    delete unknown["prompt_cache_retention"];
+
+    return params;
+  },
+};
+
 modelMiddlewareMatcher.useForModel("openai/text-embedding-*", {
   embedding: [openAIDimensionsMiddleware],
 });
 
 modelMiddlewareMatcher.useForModel("openai/gpt-*", {
-  language: [openAIReasoningMiddleware],
+  language: [openAIReasoningMiddleware, openAIPromptCachingMiddleware],
 });

package/src/providers/bedrock/middleware.test.ts

@@ -2,7 +2,11 @@ import { MockLanguageModelV3 } from "ai/test";
 import { expect, test } from "bun:test";
 
 import { modelMiddlewareMatcher } from "../../middleware/matcher";
-import { bedrockClaudeReasoningMiddleware, bedrockGptReasoningMiddleware } from "./middleware";
+import {
+  bedrockClaudeReasoningMiddleware,
+  bedrockGptReasoningMiddleware,
+  bedrockPromptCachingMiddleware,
+} from "./middleware";
 
 test("bedrock middlewares > matching provider resolves GPT middleware", () => {
   const middleware = modelMiddlewareMatcher.resolve({
@@ -24,6 +28,26 @@ test("bedrock middlewares > matching provider resolves Claude middleware", () =>
   expect(middleware).toContain(bedrockClaudeReasoningMiddleware);
 });
 
+test("bedrock middlewares > matching provider resolves prompt caching middleware for Claude", () => {
+  const middleware = modelMiddlewareMatcher.resolve({
+    kind: "text",
+    modelId: "anthropic/claude-opus-4.6",
+    providerId: "amazon-bedrock",
+  });
+
+  expect(middleware).toContain(bedrockPromptCachingMiddleware);
+});
+
+test("bedrock middlewares > matching provider resolves prompt caching middleware for Nova", () => {
+  const middleware = modelMiddlewareMatcher.resolve({
+    kind: "text",
+    modelId: "amazon/nova-2-lite",
+    providerId: "amazon-bedrock",
+  });
+
+  expect(middleware).toContain(bedrockPromptCachingMiddleware);
+});
+
 test("bedrockGptReasoningMiddleware > should map reasoningEffort into reasoningConfig", async () => {
   const params = {
     prompt: [],
@@ -181,3 +205,99 @@ test("bedrockClaudeReasoningMiddleware > should not set maxReasoningEffort for C
     },
   });
 });
+
+test("bedrockPromptCachingMiddleware > should map message and part cacheControl to cachePoint", async () => {
+  const params = {
+    prompt: [
+      {
+        role: "system",
+        content: [
+          {
+            type: "text",
+            text: "Policy",
+            providerOptions: {
+              bedrock: {
+                cacheControl: { type: "ephemeral", ttl: "1h" },
+              },
+            },
+          },
+        ],
+        providerOptions: {
+          bedrock: {
+            cacheControl: { type: "ephemeral", ttl: "1h" },
+          },
+        },
+      },
+    ],
+    providerOptions: {
+      bedrock: {},
+    },
+  };
+
+  const result = await bedrockPromptCachingMiddleware.transformParams!({
+    type: "generate",
+    params: params as any,
+    model: new MockLanguageModelV3({ modelId: "amazon/nova-2-lite" }),
+  });
+
+  expect((result.prompt[0] as any).providerOptions.bedrock.cachePoint).toEqual({
+    type: "default",
+  });
+  expect((result.prompt[0] as any).providerOptions.bedrock.cacheControl).toBeUndefined();
+  expect((result.prompt[0] as any).content[0].providerOptions.bedrock.cachePoint).toEqual({
+    type: "default",
+  });
+  expect((result.prompt[0] as any).content[0].providerOptions.bedrock.cacheControl).toBeUndefined();
+});
+
+test("bedrockPromptCachingMiddleware > should fallback from top-level cacheControl", async () => {
+  const params = {
+    prompt: [
+      {
+        role: "system",
+        content: "Reusable context",
+      },
+      {
+        role: "user",
+        content: "Question",
+      },
+    ],
+    providerOptions: {
+      bedrock: {
+        cacheControl: { type: "ephemeral", ttl: "1h" },
+      },
+    },
+  };
+
+  const result = await bedrockPromptCachingMiddleware.transformParams!({
+    type: "generate",
+    params: params as any,
+    model: new MockLanguageModelV3({ modelId: "anthropic/claude-opus-4.6" }),
+  });
+
+  expect((result.prompt[1] as any).providerOptions).toBeUndefined();
+  expect((result.providerOptions as any).bedrock.cacheControl).toBeUndefined();
+});
+
+test("bedrockPromptCachingMiddleware > should skip non-claude non-nova models", async () => {
+  const params = {
+    prompt: [{ role: "user", content: "Hello" }],
+    providerOptions: {
+      bedrock: {
+        cacheControl: { type: "ephemeral", ttl: "1h" },
+      },
+    },
+  };
+
+  const result = await bedrockPromptCachingMiddleware.transformParams!({
+    type: "generate",
+    params: params as any,
+    model: new MockLanguageModelV3({ modelId: "openai/gpt-oss-20b" }),
+  });
+
+  expect((result.providerOptions as any).bedrock.cacheControl).toEqual({
+    type: "ephemeral",
+    ttl: "1h",
+  });
+  expect((result.prompt[0] as any).providerOptions).toBeUndefined();
+});

package/src/providers/bedrock/middleware.ts

@@ -1,5 +1,7 @@
 import type { LanguageModelMiddleware } from "ai";
 
+import type { ChatCompletionsCacheControl } from "../../endpoints/chat-completions/schema";
+
 import { modelMiddlewareMatcher } from "../../middleware/matcher";
 
 const isClaude46 = (modelId: string) => modelId.includes("-4-6");
@@ -63,6 +65,64 @@ export const bedrockClaudeReasoningMiddleware: LanguageModelMiddleware = {
   },
 };
 
+function toBedrockCachePoint(modelId: string, cacheControl?: ChatCompletionsCacheControl) {
+  const out: { type: "default"; ttl?: string } = { type: "default" };
+  // Nova currently only supports 5m
+  if (cacheControl?.ttl && !modelId.includes("nova")) {
+    out.ttl = cacheControl.ttl;
+  }
+  return out;
+}
+
+// https://docs.aws.amazon.com/bedrock/latest/userguide/prompt-caching.html
+export const bedrockPromptCachingMiddleware: LanguageModelMiddleware = {
+  specificationVersion: "v3",
+  // eslint-disable-next-line require-await
+  transformParams: async ({ params, model }) => {
+    if (!model.modelId.includes("nova") && !model.modelId.includes("claude")) return params;
+
+    let hasExplicitCacheControl = false;
+    let lastCacheableBlock;
+
+    const processCacheControl = (providerOptions?: Record<string, any>) => {
+      if (!providerOptions) return;
+
+      const entryBedrock = providerOptions["bedrock"] as Record<string, unknown> | undefined;
+      const entryCacheControl = entryBedrock?.["cacheControl"] as ChatCompletionsCacheControl;
+      if (!entryBedrock || !entryCacheControl) return;
+
+      hasExplicitCacheControl = true;
+      entryBedrock["cachePoint"] = toBedrockCachePoint(model.modelId, entryCacheControl);
+      delete entryBedrock["cacheControl"];
+    };
+
+    for (const message of params.prompt) {
+      processCacheControl(message["providerOptions"]);
+
+      if (!Array.isArray(message["content"])) continue;
+      for (const part of message["content"]) {
+        processCacheControl(part["providerOptions"]);
+      }
+      lastCacheableBlock = message;
+    }
+
+    const bedrock = params.providerOptions?.["bedrock"];
+    const cacheControl = bedrock?.["cacheControl"] as ChatCompletionsCacheControl;
+    if (cacheControl && !hasExplicitCacheControl && lastCacheableBlock) {
+      ((lastCacheableBlock["providerOptions"] ??= {})["bedrock"] ??= {})["cachePoint"] =
+        toBedrockCachePoint(model.modelId, cacheControl);
+    }
+
+    delete bedrock?.["cacheControl"];
+
+    return params;
+  },
+};
+
 modelMiddlewareMatcher.useForProvider("amazon-bedrock", {
-  language: [bedrockGptReasoningMiddleware, bedrockClaudeReasoningMiddleware],
+  language: [
+    bedrockGptReasoningMiddleware,
+    bedrockClaudeReasoningMiddleware,
+    bedrockPromptCachingMiddleware,
+  ],
 });

package/src/telemetry/fetch.ts

@@ -1,8 +1,8 @@
-import { SpanKind } from "@opentelemetry/api";
+import { SpanKind, type Attributes } from "@opentelemetry/api";
 
 import type { TelemetrySignalLevel } from "../types";
 
-import { withSpan } from "./span";
+import { setSpanAttributes, withSpan } from "./span";
 
 const ORIGINAL_FETCH_KEY = Symbol.for("@hebo/fetch/original-fetch");
 
@@ -13,6 +13,21 @@ type GlobalFetchState = typeof globalThis & {
 const g = globalThis as GlobalFetchState;
 let fetchTracingEnabled = false;
 
+const isRequest = (value: unknown): value is Request =>
+  typeof Request !== "undefined" && value instanceof Request;
+
+const getRequestAttributes = (input: RequestInfo | URL, init?: RequestInit): Attributes => {
+  const attrs: Attributes = {
+    "http.request.method": init?.method ?? (isRequest(input) ? input.method : "GET"),
+  };
+
+  if (input instanceof URL) attrs["url.full"] = input.href;
+  else if (typeof input === "string") attrs["url.full"] = input;
+  else if (isRequest(input)) attrs["url.full"] = input.url;
+
+  return attrs;
+};
+
 const shouldTraceFetch = (init?: RequestInit): boolean =>
   typeof (init?.headers as any)?.["user-agent"] === "string" &&
   (init!.headers as any)["user-agent"].indexOf("ai-sdk/provider-utils") !== -1;
@@ -22,10 +37,22 @@ const otelFetch = (input: RequestInfo | URL, init?: RequestInit) => {
 
   if (!fetchTracingEnabled) return original(input, init);
   if (!shouldTraceFetch(init)) return original(input, init);
-  return withSpan("fetch", () => original(input, init), { kind: SpanKind.CLIENT });
+
+  return withSpan(
+    "fetch",
+    async () => {
+      const response = await original(input, init);
+      setSpanAttributes({ "http.response.status_code": response.status });
+      return response;
+    },
+    {
+      kind: SpanKind.CLIENT,
+      attributes: getRequestAttributes(input, init),
+    },
+  );
 };
 
-export const initFetch = (level?: TelemetrySignalLevel) => {
+export const instrumentFetch = (level?: TelemetrySignalLevel) => {
   fetchTracingEnabled = level === "full";
   if (!fetchTracingEnabled) return;
   if (g[ORIGINAL_FETCH_KEY]) return;

package/src/telemetry/index.ts

@@ -0,0 +1 @@
+export * from "./fetch";

package/dist/config.d.ts

@@ -1,2 +0,0 @@
-import { type GatewayConfig, type GatewayConfigParsed } from "./types";
-export declare const parseConfig: (config: GatewayConfig) => GatewayConfigParsed;

package/dist/config.js

@@ -1,81 +0,0 @@
-import { isLogger, logger, setLoggerInstance } from "./logger";
-import { createDefaultLogger } from "./logger/default";
-import { installAiSdkWarningLogger } from "./telemetry/ai-sdk";
-import { kParsed, } from "./types";
-export const parseConfig = (config) => {
-    // If it has been parsed before, just return.
-    if (kParsed in config)
-        return config;
-    const providers = config.providers ?? {};
-    const parsedProviders = {};
-    const models = config.models ?? {};
-    // Set the global logger instance.
-    if (config.logger === undefined) {
-        setLoggerInstance(createDefaultLogger({}));
-    }
-    else if (config.logger !== null) {
-        setLoggerInstance(isLogger(config.logger) ? config.logger : createDefaultLogger(config.logger));
-        logger.info(isLogger(config.logger)
-            ? `[logger] custom logger configured`
-            : `[logger] logger configured: level=${config.logger.level}`);
-    }
-    // Strip providers that are not configured.
-    for (const id in providers) {
-        const provider = providers[id];
-        if (provider === undefined) {
-            logger.warn(`[config] ${id} provider removed (undefined)`);
-            continue;
-        }
-        parsedProviders[id] = provider;
-    }
-    if (Object.keys(parsedProviders).length === 0) {
-        throw new Error("No providers configured (config.providers is empty)");
-    }
-    // Strip providers that are not configured from models.
-    const parsedModels = {};
-    const warnings = new Set();
-    for (const id in models) {
-        const model = models[id];
-        const kept = [];
-        for (const p of model.providers) {
-            if (p in parsedProviders)
-                kept.push(p);
-            else
-                warnings.add(p);
-        }
-        if (kept.length > 0)
-            parsedModels[id] = { ...model, providers: kept };
-    }
-    for (const warning of warnings) {
-        logger.warn(`[config] ${warning} provider removed (not configured)`);
-    }
-    if (Object.keys(parsedModels).length === 0) {
-        throw new Error("No models configured (config.models is empty)");
-    }
-    // Default for the telemetry settings.
-    const telemetryEnabled = config.telemetry?.enabled ?? false;
-    const telemetrySignals = telemetryEnabled
-        ? {
-            http: config.telemetry?.signals?.http ?? "recommended",
-            gen_ai: config.telemetry?.signals?.gen_ai ?? "full",
-            hebo: config.telemetry?.signals?.hebo ?? "off",
-        }
-        : {
-            http: "off",
-            gen_ai: "off",
-            hebo: "off",
-        };
-    installAiSdkWarningLogger(telemetrySignals.gen_ai);
-    // Return parsed config.
-    return {
-        ...config,
-        telemetry: {
-            ...config.telemetry,
-            enabled: telemetryEnabled,
-            signals: telemetrySignals,
-        },
-        providers: parsedProviders,
-        models: parsedModels,
-        [kParsed]: true,
-    };
-};