@jambonz/llm 0.5.6 → 0.6.0

package/dist/index.cjs

@@ -101,6 +101,9 @@ function assertValidRequest(req) {
       "PromptRequest.messages must not contain role: 'system'. Pass the system prompt via PromptRequest.system instead."
     );
   }
+  if (req.cacheKey !== void 0 && (typeof req.cacheKey !== "string" || req.cacheKey.length === 0)) {
+    throw new Error("PromptRequest.cacheKey must be a non-empty string");
+  }
 }
 
 // src/adapters/openai/_streaming.ts
@@ -125,11 +128,20 @@ async function* streamFromOpenAI(client, req, options) {
     body.tool_choice = "auto";
   }
   if (req.temperature !== void 0) body.temperature = req.temperature;
+  if (req.cacheKey && options.includeCacheKey !== false) {
+    body.prompt_cache_key = req.cacheKey;
+  }
   const effectiveMaxTokens = req.maxTokens ?? options.defaultMaxTokens;
   if (effectiveMaxTokens !== void 0) {
     const param = options.tokensParam ?? (needsMaxCompletionTokens(req.model) ? "max_completion_tokens" : "max_tokens");
     body[param] = effectiveMaxTokens;
   }
+  if (req.providerParams) {
+    const bodyRecord = body;
+    for (const [key, value] of Object.entries(req.providerParams)) {
+      if (!(key in bodyRecord)) bodyRecord[key] = value;
+    }
+  }
   const t0 = process.hrtime.bigint();
   let t1;
   let t2;
@@ -172,10 +184,12 @@ async function* streamFromOpenAI(client, req, options) {
         throw new Error(`OpenAI-compatible stream returned an error: ${message}`);
       }
       if (chunk.usage) {
+        const cachedTokens = chunk.usage.prompt_tokens_details?.cached_tokens;
         usage = {
-          ...chunk.usage.prompt_tokens !== void 0 ? { inputTokens: chunk.usage.prompt_tokens } : {},
+          ...chunk.usage.prompt_tokens !== void 0 ? { inputTokens: chunk.usage.prompt_tokens - (cachedTokens ?? 0) } : {},
           ...chunk.usage.completion_tokens !== void 0 ? { outputTokens: chunk.usage.completion_tokens } : {},
-          ...chunk.usage.total_tokens !== void 0 ? { totalTokens: chunk.usage.total_tokens } : {}
+          ...chunk.usage.total_tokens !== void 0 ? { totalTokens: chunk.usage.total_tokens } : {},
+          ...cachedTokens !== void 0 ? { cacheReadTokens: cachedTokens } : {}
         };
       }
       const choice = chunk.choices?.[0];
@@ -1228,6 +1242,16 @@ var AnthropicAdapter = class {
     if (req.system) body.system = req.system;
     if (tools) body.tools = tools;
     if (req.temperature !== void 0) body.temperature = req.temperature;
+    if (req.cacheKey) {
+      if (req.system) {
+        body.system = [{ type: "text", text: req.system, cache_control: { type: "ephemeral" } }];
+      }
+      if (tools) {
+        const lastTool = tools[tools.length - 1];
+        lastTool.cache_control = { type: "ephemeral" };
+      }
+      markLastMessageForCaching(messages);
+    }
     let stream;
     let responseHeaders;
     try {
@@ -1255,6 +1279,8 @@ var AnthropicAdapter = class {
     let stopReason;
     let inputTokens;
     let outputTokens;
+    let cacheReadTokens;
+    let cacheWriteTokens;
     try {
       for await (const event of stream) {
         if (req.signal?.aborted) {
@@ -1264,6 +1290,8 @@ var AnthropicAdapter = class {
         switch (event.type) {
           case "message_start":
             inputTokens = event.message?.usage?.input_tokens;
+            cacheReadTokens = event.message?.usage?.cache_read_input_tokens;
+            cacheWriteTokens = event.message?.usage?.cache_creation_input_tokens;
             break;
           case "content_block_start": {
             const block = event.content_block;
@@ -1345,7 +1373,11 @@ var AnthropicAdapter = class {
     const usage = inputTokens !== void 0 || outputTokens !== void 0 ? {
       ...inputTokens !== void 0 ? { inputTokens } : {},
       ...outputTokens !== void 0 ? { outputTokens } : {},
-      ...inputTokens !== void 0 && outputTokens !== void 0 ? { totalTokens: inputTokens + outputTokens } : {}
+      ...inputTokens !== void 0 && outputTokens !== void 0 ? {
+        totalTokens: inputTokens + outputTokens + (cacheReadTokens ?? 0) + (cacheWriteTokens ?? 0)
+      } : {},
+      ...cacheReadTokens !== void 0 ? { cacheReadTokens } : {},
+      ...cacheWriteTokens !== void 0 ? { cacheWriteTokens } : {}
     } : void 0;
     const endEvent = {
       type: "end",
@@ -1444,6 +1476,23 @@ function buildWireMessages2(req) {
   }
   return out;
 }
+function markLastMessageForCaching(messages) {
+  if (messages.length === 0) return;
+  const last = messages[messages.length - 1];
+  const cacheControl = { cache_control: { type: "ephemeral" } };
+  if (typeof last.content === "string" && last.content.length > 0) {
+    messages[messages.length - 1] = {
+      ...last,
+      content: [{ type: "text", text: last.content, ...cacheControl }]
+    };
+  } else if (Array.isArray(last.content) && last.content.length > 0) {
+    const blocks = last.content;
+    messages[messages.length - 1] = {
+      ...last,
+      content: [...blocks.slice(0, -1), { ...blocks[blocks.length - 1], ...cacheControl }]
+    };
+  }
+}
 function formatTools2(tools) {
   return tools.map((t) => ({
     name: t.name,
@@ -1506,6 +1555,11 @@ async function* streamFromGemini(client, req) {
   if (req.temperature !== void 0) config.temperature = req.temperature;
   if (req.maxTokens !== void 0) config.maxOutputTokens = req.maxTokens;
   if (req.signal) config.abortSignal = req.signal;
+  if (req.providerParams) {
+    for (const [key, value] of Object.entries(req.providerParams)) {
+      if (!(key in config)) config[key] = value;
+    }
+  }
   let stream;
   try {
     stream = await client.models.generateContentStream({
@@ -1526,6 +1580,7 @@ async function* streamFromGemini(client, req) {
   let inputTokens;
   let outputTokens;
   let totalTokens;
+  let cacheReadTokens;
   try {
     for await (const chunk of stream) {
       if (req.signal?.aborted) {
@@ -1534,7 +1589,12 @@ async function* streamFromGemini(client, req) {
       }
       const usage2 = chunk.usageMetadata;
       if (usage2) {
-        if (usage2.promptTokenCount !== void 0) inputTokens = usage2.promptTokenCount;
+        if (usage2.cachedContentTokenCount !== void 0) {
+          cacheReadTokens = usage2.cachedContentTokenCount;
+        }
+        if (usage2.promptTokenCount !== void 0) {
+          inputTokens = usage2.promptTokenCount - (usage2.cachedContentTokenCount ?? 0);
+        }
         if (usage2.candidatesTokenCount !== void 0) outputTokens = usage2.candidatesTokenCount;
         if (usage2.totalTokenCount !== void 0) totalTokens = usage2.totalTokenCount;
       }
@@ -1588,7 +1648,8 @@ async function* streamFromGemini(client, req) {
   const usage = inputTokens !== void 0 || outputTokens !== void 0 || totalTokens !== void 0 ? {
     ...inputTokens !== void 0 ? { inputTokens } : {},
     ...outputTokens !== void 0 ? { outputTokens } : {},
-    ...totalTokens !== void 0 ? { totalTokens } : {}
+    ...totalTokens !== void 0 ? { totalTokens } : {},
+    ...cacheReadTokens !== void 0 ? { cacheReadTokens } : {}
   } : void 0;
   const endEvent = {
     type: "end",
@@ -1730,6 +1791,39 @@ function isAbortError3(err) {
 
 // src/adapters/google/manifest.ts
 var GOOGLE_KNOWN_MODELS = [
+  {
+    id: "gemini-3.6-flash",
+    displayName: "Gemini 3.6 Flash",
+    capabilities: {
+      streaming: true,
+      tools: true,
+      vision: true,
+      systemPrompt: true,
+      maxContextTokens: 1e6
+    }
+  },
+  {
+    id: "gemini-3.5-flash",
+    displayName: "Gemini 3.5 Flash",
+    capabilities: {
+      streaming: true,
+      tools: true,
+      vision: true,
+      systemPrompt: true,
+      maxContextTokens: 1e6
+    }
+  },
+  {
+    id: "gemini-3.5-flash-lite",
+    displayName: "Gemini 3.5 Flash-Lite",
+    capabilities: {
+      streaming: true,
+      tools: true,
+      vision: true,
+      systemPrompt: true,
+      maxContextTokens: 1e6
+    }
+  },
   {
     id: "gemini-2.5-pro",
     displayName: "Gemini 2.5 Pro",
@@ -1873,6 +1967,39 @@ var import_genai2 = require("@google/genai");
 
 // src/adapters/vertex-gemini/manifest.ts
 var VERTEX_GEMINI_KNOWN_MODELS = [
+  {
+    id: "gemini-3.6-flash",
+    displayName: "Gemini 3.6 Flash",
+    capabilities: {
+      streaming: true,
+      tools: true,
+      vision: true,
+      systemPrompt: true,
+      maxContextTokens: 1e6
+    }
+  },
+  {
+    id: "gemini-3.5-flash",
+    displayName: "Gemini 3.5 Flash",
+    capabilities: {
+      streaming: true,
+      tools: true,
+      vision: true,
+      systemPrompt: true,
+      maxContextTokens: 1e6
+    }
+  },
+  {
+    id: "gemini-3.5-flash-lite",
+    displayName: "Gemini 3.5 Flash-Lite",
+    capabilities: {
+      streaming: true,
+      tools: true,
+      vision: true,
+      systemPrompt: true,
+      maxContextTokens: 1e6
+    }
+  },
   {
     id: "gemini-2.5-pro",
     displayName: "Gemini 2.5 Pro",
@@ -2556,6 +2683,15 @@ var BedrockAdapter = class {
     if (Object.keys(inferenceConfig).length > 0) {
       commandInput.inferenceConfig = inferenceConfig;
     }
+    if (req.cacheKey) {
+      if (commandInput.system) {
+        commandInput.system.push({ cachePoint: { type: "default" } });
+      }
+      if (tools) {
+        tools.tools.push({ cachePoint: { type: "default" } });
+      }
+      markLastMessageForCaching2(messages);
+    }
     const command = new import_client_bedrock_runtime.ConverseStreamCommand(
       commandInput
     );
@@ -2579,6 +2715,8 @@ var BedrockAdapter = class {
     let inputTokens;
     let outputTokens;
     let totalTokens;
+    let cacheReadTokens;
+    let cacheWriteTokens;
     try {
       for await (const event of response.stream) {
         if (req.signal?.aborted) {
@@ -2631,6 +2769,8 @@ var BedrockAdapter = class {
             inputTokens = event.metadata.usage.inputTokens;
             outputTokens = event.metadata.usage.outputTokens;
             totalTokens = event.metadata.usage.totalTokens;
+            cacheReadTokens = event.metadata.usage.cacheReadInputTokens;
+            cacheWriteTokens = event.metadata.usage.cacheWriteInputTokens;
           }
         }
       }
@@ -2657,7 +2797,9 @@ var BedrockAdapter = class {
     const usage = inputTokens !== void 0 || outputTokens !== void 0 || totalTokens !== void 0 ? {
       ...inputTokens !== void 0 ? { inputTokens } : {},
       ...outputTokens !== void 0 ? { outputTokens } : {},
-      ...totalTokens !== void 0 ? { totalTokens } : {}
+      ...totalTokens !== void 0 ? { totalTokens } : {},
+      ...cacheReadTokens !== void 0 ? { cacheReadTokens } : {},
+      ...cacheWriteTokens !== void 0 ? { cacheWriteTokens } : {}
     } : void 0;
     const endEvent = {
       type: "end",
@@ -2765,6 +2907,16 @@ function buildWireMessages3(req) {
   }
   return out;
 }
+function markLastMessageForCaching2(messages) {
+  if (messages.length === 0) return;
+  const last = messages[messages.length - 1];
+  if (Array.isArray(last.content) && last.content.length > 0) {
+    messages[messages.length - 1] = {
+      ...last,
+      content: [...last.content, { cachePoint: { type: "default" } }]
+    };
+  }
+}
 function formatTools4(tools) {
   return tools.map((t) => ({
     toolSpec: {
@@ -3120,7 +3272,10 @@ var GroqAdapter = class extends OpenAIAdapter {
   stream(req) {
     return streamFromOpenAI(this.ensureClient(), req, {
       knownModels: groqManifest.knownModels,
-      vendorMetadataExtractor: GROQ_METADATA_EXTRACTOR
+      vendorMetadataExtractor: GROQ_METADATA_EXTRACTOR,
+      // Groq caches automatically server-side and rejects/ignores undocumented
+      // OpenAI params — do not forward prompt_cache_key.
+      includeCacheKey: false
     });
   }
 };