@jambonz/llm 0.5.7 → 0.6.1

package/dist/index.cjs

@@ -101,6 +101,9 @@ function assertValidRequest(req) {
       "PromptRequest.messages must not contain role: 'system'. Pass the system prompt via PromptRequest.system instead."
     );
   }
+  if (req.cacheKey !== void 0 && (typeof req.cacheKey !== "string" || req.cacheKey.length === 0)) {
+    throw new Error("PromptRequest.cacheKey must be a non-empty string");
+  }
 }
 
 // src/adapters/openai/_streaming.ts
@@ -125,6 +128,9 @@ async function* streamFromOpenAI(client, req, options) {
     body.tool_choice = "auto";
   }
   if (req.temperature !== void 0) body.temperature = req.temperature;
+  if (req.cacheKey && options.includeCacheKey !== false) {
+    body.prompt_cache_key = req.cacheKey;
+  }
   const effectiveMaxTokens = req.maxTokens ?? options.defaultMaxTokens;
   if (effectiveMaxTokens !== void 0) {
     const param = options.tokensParam ?? (needsMaxCompletionTokens(req.model) ? "max_completion_tokens" : "max_tokens");
@@ -178,10 +184,12 @@ async function* streamFromOpenAI(client, req, options) {
         throw new Error(`OpenAI-compatible stream returned an error: ${message}`);
       }
       if (chunk.usage) {
+        const cachedTokens = chunk.usage.prompt_tokens_details?.cached_tokens;
         usage = {
-          ...chunk.usage.prompt_tokens !== void 0 ? { inputTokens: chunk.usage.prompt_tokens } : {},
+          ...chunk.usage.prompt_tokens !== void 0 ? { inputTokens: chunk.usage.prompt_tokens - (cachedTokens ?? 0) } : {},
           ...chunk.usage.completion_tokens !== void 0 ? { outputTokens: chunk.usage.completion_tokens } : {},
-          ...chunk.usage.total_tokens !== void 0 ? { totalTokens: chunk.usage.total_tokens } : {}
+          ...chunk.usage.total_tokens !== void 0 ? { totalTokens: chunk.usage.total_tokens } : {},
+          ...cachedTokens !== void 0 ? { cacheReadTokens: cachedTokens } : {}
         };
       }
       const choice = chunk.choices?.[0];
@@ -560,7 +568,8 @@ var OpenAIAdapter = class {
   stream(req) {
     return streamFromOpenAI(this.ensureClient(), req, {
       knownModels: this.knownModels(),
-      vendorMetadataExtractor: OPENAI_METADATA_EXTRACTOR
+      vendorMetadataExtractor: OPENAI_METADATA_EXTRACTOR,
+      includeCacheKey: this.forwardCacheKey()
     });
   }
   appendAssistantToolCall(history, toolCalls) {
@@ -581,6 +590,20 @@ var OpenAIAdapter = class {
   knownModels() {
     return openAIManifest.knownModels;
   }
+  /**
+   * Whether to forward `req.cacheKey` as OpenAI's `prompt_cache_key` request
+   * param. Default true — verified accepted by OpenAI, Azure OpenAI,
+   * Vertex-OpenAI, and Baseten. OpenAI-compatible vendors whose param
+   * handling is strict or unverified (DeepSeek, Moonshot, Zai, xAI — and
+   * Groq via its own stream override) override to false: a strict backend
+   * rejects the whole request on an unknown body param, which is far worse
+   * than forgoing a cache-routing hint. Flip a vendor to true only after
+   * verifying its endpoint (including any proxy in front of it, e.g.
+   * Bedrock Mantle) tolerates the param.
+   */
+  forwardCacheKey() {
+    return true;
+  }
   async testCredential() {
     await this.ensureClient().models.list();
   }
@@ -693,6 +716,11 @@ var DeepSeekAdapter = class extends OpenAIAdapter {
   knownModels() {
     return deepseekManifest.knownModels;
   }
+  // DeepSeek caches automatically server-side; prompt_cache_key is
+  // unverified against its param handling — do not forward.
+  forwardCacheKey() {
+    return false;
+  }
 };
 var deepseekFactory = {
   vendor: deepseekManifest.vendor,
@@ -916,6 +944,12 @@ var MoonshotAdapter = class extends OpenAIAdapter {
   knownModels() {
     return moonshotManifest.knownModels;
   }
+  // Moonshot's param handling is strict (e.g. it rejects unknown template
+  // kwargs) and prompt_cache_key is unverified against both the native
+  // endpoint and Bedrock Mantle deployments — do not forward.
+  forwardCacheKey() {
+    return false;
+  }
 };
 var moonshotFactory = {
   vendor: moonshotManifest.vendor,
@@ -989,6 +1023,11 @@ var ZaiAdapter = class extends OpenAIAdapter {
   knownModels() {
     return zaiManifest.knownModels;
   }
+  // prompt_cache_key is unverified against Zai's OpenAI-compat endpoint —
+  // do not forward.
+  forwardCacheKey() {
+    return false;
+  }
 };
 var zaiFactory = {
   vendor: zaiManifest.vendor,
@@ -1084,6 +1123,11 @@ var XaiAdapter = class extends OpenAIAdapter {
   knownModels() {
     return xaiManifest.knownModels;
   }
+  // xAI caches automatically server-side; prompt_cache_key is unverified
+  // against its param handling — do not forward.
+  forwardCacheKey() {
+    return false;
+  }
 };
 var xaiFactory = {
   vendor: xaiManifest.vendor,
@@ -1234,6 +1278,16 @@ var AnthropicAdapter = class {
     if (req.system) body.system = req.system;
     if (tools) body.tools = tools;
     if (req.temperature !== void 0) body.temperature = req.temperature;
+    if (req.cacheKey) {
+      if (req.system) {
+        body.system = [{ type: "text", text: req.system, cache_control: { type: "ephemeral" } }];
+      }
+      if (tools) {
+        const lastTool = tools[tools.length - 1];
+        lastTool.cache_control = { type: "ephemeral" };
+      }
+      markLastMessageForCaching(messages);
+    }
     let stream;
     let responseHeaders;
     try {
@@ -1261,6 +1315,8 @@ var AnthropicAdapter = class {
     let stopReason;
     let inputTokens;
     let outputTokens;
+    let cacheReadTokens;
+    let cacheWriteTokens;
     try {
       for await (const event of stream) {
         if (req.signal?.aborted) {
@@ -1270,6 +1326,8 @@ var AnthropicAdapter = class {
         switch (event.type) {
           case "message_start":
             inputTokens = event.message?.usage?.input_tokens;
+            cacheReadTokens = event.message?.usage?.cache_read_input_tokens;
+            cacheWriteTokens = event.message?.usage?.cache_creation_input_tokens;
             break;
           case "content_block_start": {
             const block = event.content_block;
@@ -1351,7 +1409,11 @@ var AnthropicAdapter = class {
     const usage = inputTokens !== void 0 || outputTokens !== void 0 ? {
       ...inputTokens !== void 0 ? { inputTokens } : {},
       ...outputTokens !== void 0 ? { outputTokens } : {},
-      ...inputTokens !== void 0 && outputTokens !== void 0 ? { totalTokens: inputTokens + outputTokens } : {}
+      ...inputTokens !== void 0 && outputTokens !== void 0 ? {
+        totalTokens: inputTokens + outputTokens + (cacheReadTokens ?? 0) + (cacheWriteTokens ?? 0)
+      } : {},
+      ...cacheReadTokens !== void 0 ? { cacheReadTokens } : {},
+      ...cacheWriteTokens !== void 0 ? { cacheWriteTokens } : {}
     } : void 0;
     const endEvent = {
       type: "end",
@@ -1450,6 +1512,23 @@ function buildWireMessages2(req) {
   }
   return out;
 }
+function markLastMessageForCaching(messages) {
+  if (messages.length === 0) return;
+  const last = messages[messages.length - 1];
+  const cacheControl = { cache_control: { type: "ephemeral" } };
+  if (typeof last.content === "string" && last.content.length > 0) {
+    messages[messages.length - 1] = {
+      ...last,
+      content: [{ type: "text", text: last.content, ...cacheControl }]
+    };
+  } else if (Array.isArray(last.content) && last.content.length > 0) {
+    const blocks = last.content;
+    messages[messages.length - 1] = {
+      ...last,
+      content: [...blocks.slice(0, -1), { ...blocks[blocks.length - 1], ...cacheControl }]
+    };
+  }
+}
 function formatTools2(tools) {
   return tools.map((t) => ({
     name: t.name,
@@ -1512,6 +1591,11 @@ async function* streamFromGemini(client, req) {
   if (req.temperature !== void 0) config.temperature = req.temperature;
   if (req.maxTokens !== void 0) config.maxOutputTokens = req.maxTokens;
   if (req.signal) config.abortSignal = req.signal;
+  if (req.providerParams) {
+    for (const [key, value] of Object.entries(req.providerParams)) {
+      if (!(key in config)) config[key] = value;
+    }
+  }
   let stream;
   try {
     stream = await client.models.generateContentStream({
@@ -1532,6 +1616,7 @@ async function* streamFromGemini(client, req) {
   let inputTokens;
   let outputTokens;
   let totalTokens;
+  let cacheReadTokens;
   try {
     for await (const chunk of stream) {
       if (req.signal?.aborted) {
@@ -1540,7 +1625,12 @@ async function* streamFromGemini(client, req) {
       }
       const usage2 = chunk.usageMetadata;
       if (usage2) {
-        if (usage2.promptTokenCount !== void 0) inputTokens = usage2.promptTokenCount;
+        if (usage2.cachedContentTokenCount !== void 0) {
+          cacheReadTokens = usage2.cachedContentTokenCount;
+        }
+        if (usage2.promptTokenCount !== void 0) {
+          inputTokens = usage2.promptTokenCount - (usage2.cachedContentTokenCount ?? 0);
+        }
         if (usage2.candidatesTokenCount !== void 0) outputTokens = usage2.candidatesTokenCount;
         if (usage2.totalTokenCount !== void 0) totalTokens = usage2.totalTokenCount;
       }
@@ -1594,7 +1684,8 @@ async function* streamFromGemini(client, req) {
   const usage = inputTokens !== void 0 || outputTokens !== void 0 || totalTokens !== void 0 ? {
     ...inputTokens !== void 0 ? { inputTokens } : {},
     ...outputTokens !== void 0 ? { outputTokens } : {},
-    ...totalTokens !== void 0 ? { totalTokens } : {}
+    ...totalTokens !== void 0 ? { totalTokens } : {},
+    ...cacheReadTokens !== void 0 ? { cacheReadTokens } : {}
   } : void 0;
   const endEvent = {
     type: "end",
@@ -1736,6 +1827,39 @@ function isAbortError3(err) {
 
 // src/adapters/google/manifest.ts
 var GOOGLE_KNOWN_MODELS = [
+  {
+    id: "gemini-3.6-flash",
+    displayName: "Gemini 3.6 Flash",
+    capabilities: {
+      streaming: true,
+      tools: true,
+      vision: true,
+      systemPrompt: true,
+      maxContextTokens: 1e6
+    }
+  },
+  {
+    id: "gemini-3.5-flash",
+    displayName: "Gemini 3.5 Flash",
+    capabilities: {
+      streaming: true,
+      tools: true,
+      vision: true,
+      systemPrompt: true,
+      maxContextTokens: 1e6
+    }
+  },
+  {
+    id: "gemini-3.5-flash-lite",
+    displayName: "Gemini 3.5 Flash-Lite",
+    capabilities: {
+      streaming: true,
+      tools: true,
+      vision: true,
+      systemPrompt: true,
+      maxContextTokens: 1e6
+    }
+  },
   {
     id: "gemini-2.5-pro",
     displayName: "Gemini 2.5 Pro",
@@ -1879,6 +2003,39 @@ var import_genai2 = require("@google/genai");
 
 // src/adapters/vertex-gemini/manifest.ts
 var VERTEX_GEMINI_KNOWN_MODELS = [
+  {
+    id: "gemini-3.6-flash",
+    displayName: "Gemini 3.6 Flash",
+    capabilities: {
+      streaming: true,
+      tools: true,
+      vision: true,
+      systemPrompt: true,
+      maxContextTokens: 1e6
+    }
+  },
+  {
+    id: "gemini-3.5-flash",
+    displayName: "Gemini 3.5 Flash",
+    capabilities: {
+      streaming: true,
+      tools: true,
+      vision: true,
+      systemPrompt: true,
+      maxContextTokens: 1e6
+    }
+  },
+  {
+    id: "gemini-3.5-flash-lite",
+    displayName: "Gemini 3.5 Flash-Lite",
+    capabilities: {
+      streaming: true,
+      tools: true,
+      vision: true,
+      systemPrompt: true,
+      maxContextTokens: 1e6
+    }
+  },
   {
     id: "gemini-2.5-pro",
     displayName: "Gemini 2.5 Pro",
@@ -2562,6 +2719,15 @@ var BedrockAdapter = class {
     if (Object.keys(inferenceConfig).length > 0) {
       commandInput.inferenceConfig = inferenceConfig;
     }
+    if (req.cacheKey) {
+      if (commandInput.system) {
+        commandInput.system.push({ cachePoint: { type: "default" } });
+      }
+      if (tools) {
+        tools.tools.push({ cachePoint: { type: "default" } });
+      }
+      markLastMessageForCaching2(messages);
+    }
     const command = new import_client_bedrock_runtime.ConverseStreamCommand(
       commandInput
     );
@@ -2585,6 +2751,8 @@ var BedrockAdapter = class {
     let inputTokens;
     let outputTokens;
     let totalTokens;
+    let cacheReadTokens;
+    let cacheWriteTokens;
     try {
       for await (const event of response.stream) {
         if (req.signal?.aborted) {
@@ -2637,6 +2805,8 @@ var BedrockAdapter = class {
             inputTokens = event.metadata.usage.inputTokens;
             outputTokens = event.metadata.usage.outputTokens;
             totalTokens = event.metadata.usage.totalTokens;
+            cacheReadTokens = event.metadata.usage.cacheReadInputTokens;
+            cacheWriteTokens = event.metadata.usage.cacheWriteInputTokens;
           }
         }
       }
@@ -2663,7 +2833,9 @@ var BedrockAdapter = class {
     const usage = inputTokens !== void 0 || outputTokens !== void 0 || totalTokens !== void 0 ? {
       ...inputTokens !== void 0 ? { inputTokens } : {},
       ...outputTokens !== void 0 ? { outputTokens } : {},
-      ...totalTokens !== void 0 ? { totalTokens } : {}
+      ...totalTokens !== void 0 ? { totalTokens } : {},
+      ...cacheReadTokens !== void 0 ? { cacheReadTokens } : {},
+      ...cacheWriteTokens !== void 0 ? { cacheWriteTokens } : {}
     } : void 0;
     const endEvent = {
       type: "end",
@@ -2771,6 +2943,16 @@ function buildWireMessages3(req) {
   }
   return out;
 }
+function markLastMessageForCaching2(messages) {
+  if (messages.length === 0) return;
+  const last = messages[messages.length - 1];
+  if (Array.isArray(last.content) && last.content.length > 0) {
+    messages[messages.length - 1] = {
+      ...last,
+      content: [...last.content, { cachePoint: { type: "default" } }]
+    };
+  }
+}
 function formatTools4(tools) {
   return tools.map((t) => ({
     toolSpec: {
@@ -3126,7 +3308,10 @@ var GroqAdapter = class extends OpenAIAdapter {
   stream(req) {
     return streamFromOpenAI(this.ensureClient(), req, {
       knownModels: groqManifest.knownModels,
-      vendorMetadataExtractor: GROQ_METADATA_EXTRACTOR
+      vendorMetadataExtractor: GROQ_METADATA_EXTRACTOR,
+      // Groq caches automatically server-side and rejects/ignores undocumented
+      // OpenAI params — do not forward prompt_cache_key.
+      includeCacheKey: false
     });
   }
 };
@@ -3247,7 +3432,10 @@ var HuggingfaceAdapter = class extends OpenAIAdapter {
   stream(req) {
     return streamFromOpenAI(this.ensureClient(), req, {
       knownModels: huggingfaceManifest.knownModels,
-      vendorMetadataExtractor: HUGGINGFACE_METADATA_EXTRACTOR
+      vendorMetadataExtractor: HUGGINGFACE_METADATA_EXTRACTOR,
+      // The HF router fronts many third-party inference providers;
+      // prompt_cache_key is unverified against them — do not forward.
+      includeCacheKey: false
     });
   }
 };