@jambonz/llm 0.6.0 → 0.6.2

package/dist/index.cjs

@@ -568,7 +568,8 @@ var OpenAIAdapter = class {
   stream(req) {
     return streamFromOpenAI(this.ensureClient(), req, {
       knownModels: this.knownModels(),
-      vendorMetadataExtractor: OPENAI_METADATA_EXTRACTOR
+      vendorMetadataExtractor: OPENAI_METADATA_EXTRACTOR,
+      includeCacheKey: this.forwardCacheKey()
     });
   }
   appendAssistantToolCall(history, toolCalls) {
@@ -589,6 +590,20 @@ var OpenAIAdapter = class {
   knownModels() {
     return openAIManifest.knownModels;
   }
+  /**
+   * Whether to forward `req.cacheKey` as OpenAI's `prompt_cache_key` request
+   * param. Default true — verified accepted by OpenAI, Azure OpenAI,
+   * Vertex-OpenAI, and Baseten. OpenAI-compatible vendors whose param
+   * handling is strict or unverified (DeepSeek, Moonshot, Zai, xAI — and
+   * Groq via its own stream override) override to false: a strict backend
+   * rejects the whole request on an unknown body param, which is far worse
+   * than forgoing a cache-routing hint. Flip a vendor to true only after
+   * verifying its endpoint (including any proxy in front of it, e.g.
+   * Bedrock Mantle) tolerates the param.
+   */
+  forwardCacheKey() {
+    return true;
+  }
   async testCredential() {
     await this.ensureClient().models.list();
   }
@@ -701,6 +716,11 @@ var DeepSeekAdapter = class extends OpenAIAdapter {
   knownModels() {
     return deepseekManifest.knownModels;
   }
+  // DeepSeek caches automatically server-side; prompt_cache_key is
+  // unverified against its param handling — do not forward.
+  forwardCacheKey() {
+    return false;
+  }
 };
 var deepseekFactory = {
   vendor: deepseekManifest.vendor,
@@ -924,6 +944,12 @@ var MoonshotAdapter = class extends OpenAIAdapter {
   knownModels() {
     return moonshotManifest.knownModels;
   }
+  // Moonshot's param handling is strict (e.g. it rejects unknown template
+  // kwargs) and prompt_cache_key is unverified against both the native
+  // endpoint and Bedrock Mantle deployments — do not forward.
+  forwardCacheKey() {
+    return false;
+  }
 };
 var moonshotFactory = {
   vendor: moonshotManifest.vendor,
@@ -997,6 +1023,11 @@ var ZaiAdapter = class extends OpenAIAdapter {
   knownModels() {
     return zaiManifest.knownModels;
   }
+  // prompt_cache_key is unverified against Zai's OpenAI-compat endpoint —
+  // do not forward.
+  forwardCacheKey() {
+    return false;
+  }
 };
 var zaiFactory = {
   vendor: zaiManifest.vendor,
@@ -1092,12 +1123,127 @@ var XaiAdapter = class extends OpenAIAdapter {
   knownModels() {
     return xaiManifest.knownModels;
   }
+  // xAI caches automatically server-side; prompt_cache_key is unverified
+  // against its param handling — do not forward.
+  forwardCacheKey() {
+    return false;
+  }
 };
 var xaiFactory = {
   vendor: xaiManifest.vendor,
   manifest: xaiManifest,
   create: () => new XaiAdapter()
 };
+var BEDROCK_MANTLE_DEFAULT_BASE_URL = "https://bedrock-mantle.us-east-1.api.aws/v1";
+var bedrockMantleManifest = {
+  vendor: "bedrock-mantle",
+  displayName: "AWS Bedrock (OpenAI-compatible)",
+  authKinds: [
+    {
+      kind: "apiKey",
+      displayName: "API Key",
+      fields: [
+        {
+          name: "apiKey",
+          label: "Bedrock API Key",
+          type: "password",
+          required: true,
+          help: "Generate a long-term Bedrock API key in the AWS console (Amazon Bedrock \u2192 API keys)."
+        },
+        {
+          name: "baseURL",
+          label: "Base URL",
+          type: "url",
+          required: false,
+          default: BEDROCK_MANTLE_DEFAULT_BASE_URL,
+          help: "Change the region to match your key. xAI models are served only on the /openai/v1 path."
+        }
+      ]
+    }
+  ],
+  /* A convenience seed, NOT the source of truth: the live `/v1/models`
+   * endpoint is authoritative and grows continuously, which is why
+   * `supportsModelListing` is true. Any model the endpoint lists can be used
+   * by id without appearing here.
+   *
+   * AWS publishes no context-window metadata (neither `/v1/models` nor
+   * `GetFoundationModel` returns it), so these limits are carried over from
+   * the native-vendor manifests above for the same underlying models. Vision
+   * follows Bedrock's `inputModalities`. */
+  knownModels: [
+    {
+      id: "zai.glm-5",
+      displayName: "GLM 5",
+      capabilities: {
+        streaming: true,
+        tools: true,
+        vision: false,
+        systemPrompt: true,
+        maxContextTokens: 202800
+      }
+    },
+    {
+      id: "zai.glm-4.7",
+      displayName: "GLM 4.7",
+      capabilities: {
+        streaming: true,
+        tools: true,
+        vision: false,
+        systemPrompt: true,
+        maxContextTokens: 2e5
+      }
+    },
+    {
+      id: "moonshotai.kimi-k2.5",
+      displayName: "Kimi K2.5",
+      capabilities: {
+        streaming: true,
+        tools: true,
+        vision: true,
+        systemPrompt: true,
+        maxContextTokens: 262144
+      }
+    },
+    {
+      id: "xai.grok-4.3",
+      displayName: "Grok 4.3",
+      capabilities: {
+        streaming: true,
+        tools: true,
+        vision: true,
+        systemPrompt: true,
+        maxContextTokens: 1e6
+      }
+    }
+  ],
+  supportsModelListing: true,
+  docsUrl: "https://docs.aws.amazon.com/bedrock/latest/userguide/inference-chat-completions.html"
+};
+var BedrockMantleAdapter = class extends OpenAIAdapter {
+  vendor = bedrockMantleManifest.vendor;
+  init(auth, client) {
+    super.init(
+      {
+        ...auth,
+        baseURL: auth.baseURL ?? BEDROCK_MANTLE_DEFAULT_BASE_URL
+      },
+      client
+    );
+  }
+  knownModels() {
+    return bedrockMantleManifest.knownModels;
+  }
+  // Mantle fronts many third-party runtimes with differing param strictness,
+  // and prompt_cache_key is unverified against all of them — do not forward.
+  forwardCacheKey() {
+    return false;
+  }
+};
+var bedrockMantleFactory = {
+  vendor: bedrockMantleManifest.vendor,
+  manifest: bedrockMantleManifest,
+  create: () => new BedrockMantleAdapter()
+};
 
 // src/adapters/anthropic/adapter.ts
 var import_sdk = __toESM(require("@anthropic-ai/sdk"), 1);
@@ -3396,7 +3542,10 @@ var HuggingfaceAdapter = class extends OpenAIAdapter {
   stream(req) {
     return streamFromOpenAI(this.ensureClient(), req, {
       knownModels: huggingfaceManifest.knownModels,
-      vendorMetadataExtractor: HUGGINGFACE_METADATA_EXTRACTOR
+      vendorMetadataExtractor: HUGGINGFACE_METADATA_EXTRACTOR,
+      // The HF router fronts many third-party inference providers;
+      // prompt_cache_key is unverified against them — do not forward.
+      includeCacheKey: false
     });
   }
 };
@@ -3415,6 +3564,7 @@ registerAdapter(basetenFactory);
 registerAdapter(moonshotFactory);
 registerAdapter(zaiFactory);
 registerAdapter(xaiFactory);
+registerAdapter(bedrockMantleFactory);
 registerAdapter(anthropicFactory);
 registerAdapter(googleFactory);
 registerAdapter(vertexGeminiFactory);
@@ -3434,6 +3584,7 @@ function normalizeAuth(vendor, raw) {
     case "moonshot":
     case "zai":
     case "xai":
+    case "bedrock-mantle":
       return requireApiKey(vendor, raw);
     case "deepseek":
       return {