@jambonz/llm 0.6.4 → 0.6.6

package/dist/index.js

@@ -86,6 +86,9 @@ async function* streamFromOpenAI(client, req, options) {
   if (req.cacheKey && options.includeCacheKey !== false) {
     body.prompt_cache_key = req.cacheKey;
   }
+  if (req.cacheKey && options.openRouterAutoCache && /^~?anthropic\//.test(req.model)) {
+    body.cache_control = { type: "ephemeral" };
+  }
   const effectiveMaxTokens = req.maxTokens ?? options.defaultMaxTokens;
   if (effectiveMaxTokens !== void 0) {
     const param = options.tokensParam ?? (needsMaxCompletionTokens(req.model) ? "max_completion_tokens" : "max_tokens");
@@ -139,12 +142,20 @@ async function* streamFromOpenAI(client, req, options) {
         throw new Error(`OpenAI-compatible stream returned an error: ${message}`);
       }
       if (chunk.usage) {
-        const cachedTokens = chunk.usage.prompt_tokens_details?.cached_tokens;
+        const details = chunk.usage.prompt_tokens_details;
+        const cachedTokens = details?.cached_tokens;
+        const cacheWriteTokens = details?.cache_write_tokens;
         usage = {
-          ...chunk.usage.prompt_tokens !== void 0 ? { inputTokens: chunk.usage.prompt_tokens - (cachedTokens ?? 0) } : {},
+          ...chunk.usage.prompt_tokens !== void 0 ? {
+            inputTokens: Math.max(
+              0,
+              chunk.usage.prompt_tokens - (cachedTokens ?? 0) - (cacheWriteTokens ?? 0)
+            )
+          } : {},
           ...chunk.usage.completion_tokens !== void 0 ? { outputTokens: chunk.usage.completion_tokens } : {},
           ...chunk.usage.total_tokens !== void 0 ? { totalTokens: chunk.usage.total_tokens } : {},
-          ...cachedTokens !== void 0 ? { cacheReadTokens: cachedTokens } : {}
+          ...cachedTokens !== void 0 ? { cacheReadTokens: cachedTokens } : {},
+          ...cacheWriteTokens !== void 0 ? { cacheWriteTokens } : {}
         };
       }
       const choice = chunk.choices?.[0];
@@ -510,10 +521,13 @@ var OpenAIAdapter = class {
   vendor = openAIManifest.vendor;
   acceptedAuth = ["apiKey"];
   client;
+  /** baseURL is OpenRouter: enables its automatic prompt caching for Anthropic models. */
+  isOpenRouter = false;
   init(auth, client) {
     if (!auth.apiKey) {
       throw new Error("OpenAIAdapter: apiKey is required");
     }
+    this.isOpenRouter = /(^|\/\/|\.)openrouter\.ai(\/|:|$)/i.test(auth.baseURL ?? "");
     this.client = new OpenAI({
       apiKey: auth.apiKey,
       ...auth.baseURL ? { baseURL: auth.baseURL } : {},
@@ -525,7 +539,8 @@ var OpenAIAdapter = class {
     return streamFromOpenAI(this.ensureClient(), req, {
       knownModels: this.knownModels(),
       vendorMetadataExtractor: OPENAI_METADATA_EXTRACTOR,
-      includeCacheKey: this.forwardCacheKey()
+      includeCacheKey: this.forwardCacheKey(),
+      openRouterAutoCache: this.isOpenRouter
     });
   }
   appendAssistantToolCall(history, toolCalls, assistantText) {