@jambonz/llm 0.6.0 → 0.6.1

package/dist/index.js

@@ -523,7 +523,8 @@ var OpenAIAdapter = class {
   stream(req) {
     return streamFromOpenAI(this.ensureClient(), req, {
       knownModels: this.knownModels(),
-      vendorMetadataExtractor: OPENAI_METADATA_EXTRACTOR
+      vendorMetadataExtractor: OPENAI_METADATA_EXTRACTOR,
+      includeCacheKey: this.forwardCacheKey()
     });
   }
   appendAssistantToolCall(history, toolCalls) {
@@ -544,6 +545,20 @@ var OpenAIAdapter = class {
   knownModels() {
     return openAIManifest.knownModels;
   }
+  /**
+   * Whether to forward `req.cacheKey` as OpenAI's `prompt_cache_key` request
+   * param. Default true — verified accepted by OpenAI, Azure OpenAI,
+   * Vertex-OpenAI, and Baseten. OpenAI-compatible vendors whose param
+   * handling is strict or unverified (DeepSeek, Moonshot, Zai, xAI — and
+   * Groq via its own stream override) override to false: a strict backend
+   * rejects the whole request on an unknown body param, which is far worse
+   * than forgoing a cache-routing hint. Flip a vendor to true only after
+   * verifying its endpoint (including any proxy in front of it, e.g.
+   * Bedrock Mantle) tolerates the param.
+   */
+  forwardCacheKey() {
+    return true;
+  }
   async testCredential() {
     await this.ensureClient().models.list();
   }
@@ -656,6 +671,11 @@ var DeepSeekAdapter = class extends OpenAIAdapter {
   knownModels() {
     return deepseekManifest.knownModels;
   }
+  // DeepSeek caches automatically server-side; prompt_cache_key is
+  // unverified against its param handling — do not forward.
+  forwardCacheKey() {
+    return false;
+  }
 };
 var deepseekFactory = {
   vendor: deepseekManifest.vendor,
@@ -879,6 +899,12 @@ var MoonshotAdapter = class extends OpenAIAdapter {
   knownModels() {
     return moonshotManifest.knownModels;
   }
+  // Moonshot's param handling is strict (e.g. it rejects unknown template
+  // kwargs) and prompt_cache_key is unverified against both the native
+  // endpoint and Bedrock Mantle deployments — do not forward.
+  forwardCacheKey() {
+    return false;
+  }
 };
 var moonshotFactory = {
   vendor: moonshotManifest.vendor,
@@ -952,6 +978,11 @@ var ZaiAdapter = class extends OpenAIAdapter {
   knownModels() {
     return zaiManifest.knownModels;
   }
+  // prompt_cache_key is unverified against Zai's OpenAI-compat endpoint —
+  // do not forward.
+  forwardCacheKey() {
+    return false;
+  }
 };
 var zaiFactory = {
   vendor: zaiManifest.vendor,
@@ -1047,6 +1078,11 @@ var XaiAdapter = class extends OpenAIAdapter {
   knownModels() {
     return xaiManifest.knownModels;
   }
+  // xAI caches automatically server-side; prompt_cache_key is unverified
+  // against its param handling — do not forward.
+  forwardCacheKey() {
+    return false;
+  }
 };
 var xaiFactory = {
   vendor: xaiManifest.vendor,
@@ -3357,7 +3393,10 @@ var HuggingfaceAdapter = class extends OpenAIAdapter {
   stream(req) {
     return streamFromOpenAI(this.ensureClient(), req, {
       knownModels: huggingfaceManifest.knownModels,
-      vendorMetadataExtractor: HUGGINGFACE_METADATA_EXTRACTOR
+      vendorMetadataExtractor: HUGGINGFACE_METADATA_EXTRACTOR,
+      // The HF router fronts many third-party inference providers;
+      // prompt_cache_key is unverified against them — do not forward.
+      includeCacheKey: false
     });
   }
 };