@jambonz/llm 0.6.3 → 0.6.5

package/dist/index.js

@@ -86,6 +86,9 @@ async function* streamFromOpenAI(client, req, options) {
   if (req.cacheKey && options.includeCacheKey !== false) {
     body.prompt_cache_key = req.cacheKey;
   }
+  if (req.cacheKey && options.openRouterAutoCache && /^~?anthropic\//.test(req.model)) {
+    body.cache_control = { type: "ephemeral" };
+  }
   const effectiveMaxTokens = req.maxTokens ?? options.defaultMaxTokens;
   if (effectiveMaxTokens !== void 0) {
     const param = options.tokensParam ?? (needsMaxCompletionTokens(req.model) ? "max_completion_tokens" : "max_tokens");
@@ -510,10 +513,13 @@ var OpenAIAdapter = class {
   vendor = openAIManifest.vendor;
   acceptedAuth = ["apiKey"];
   client;
+  /** baseURL is OpenRouter: enables its automatic prompt caching for Anthropic models. */
+  isOpenRouter = false;
   init(auth, client) {
     if (!auth.apiKey) {
       throw new Error("OpenAIAdapter: apiKey is required");
     }
+    this.isOpenRouter = /(^|\/\/|\.)openrouter\.ai(\/|:|$)/i.test(auth.baseURL ?? "");
     this.client = new OpenAI({
       apiKey: auth.apiKey,
       ...auth.baseURL ? { baseURL: auth.baseURL } : {},
@@ -525,7 +531,8 @@ var OpenAIAdapter = class {
     return streamFromOpenAI(this.ensureClient(), req, {
       knownModels: this.knownModels(),
       vendorMetadataExtractor: OPENAI_METADATA_EXTRACTOR,
-      includeCacheKey: this.forwardCacheKey()
+      includeCacheKey: this.forwardCacheKey(),
+      openRouterAutoCache: this.isOpenRouter
     });
   }
   appendAssistantToolCall(history, toolCalls, assistantText) {
@@ -2799,11 +2806,12 @@ var BedrockAdapter = class {
     if (Object.keys(inferenceConfig).length > 0) {
       commandInput.inferenceConfig = inferenceConfig;
     }
-    if (req.cacheKey) {
+    const caching = req.cacheKey ? cachePointSupport(req.model) : void 0;
+    if (caching) {
       if (commandInput.system) {
         commandInput.system.push({ cachePoint: { type: "default" } });
       }
-      if (tools) {
+      if (tools && caching.tools) {
         tools.tools.push({ cachePoint: { type: "default" } });
       }
       markLastMessageForCaching2(messages);
@@ -3027,6 +3035,11 @@ function buildWireMessages3(req) {
   }
   return out;
 }
+function cachePointSupport(model) {
+  if (model.includes("anthropic.claude")) return { tools: true };
+  if (model.includes("amazon.nova")) return { tools: false };
+  return void 0;
+}
 function markLastMessageForCaching2(messages) {
   if (messages.length === 0) return;
   const last = messages[messages.length - 1];