@jambonz/llm 0.4.5 → 0.5.0

package/dist/test-kit/index.cjs

@@ -118,8 +118,17 @@ async function* streamFromOpenAI(client, req, options) {
     body[param] = effectiveMaxTokens;
   }
   let stream;
+  let responseHeaders;
   try {
-    stream = await client.chat.completions.create(body, { signal: req.signal });
+    const apiPromise = client.chat.completions.create(body, { signal: req.signal });
+    const maybeWithResponse = apiPromise.withResponse;
+    if (typeof maybeWithResponse === "function") {
+      const result = await maybeWithResponse.call(apiPromise);
+      stream = result.data;
+      responseHeaders = result.response?.headers;
+    } else {
+      stream = await apiPromise;
+    }
   } catch (err) {
     if (isAbortError(err)) {
       yield { type: "end", finishReason: "aborted" };
@@ -214,6 +223,12 @@ async function* streamFromOpenAI(client, req, options) {
     ...usage ? { usage } : {},
     ...finalRawReason ? { rawReason: finalRawReason } : {}
   };
+  if (options.vendorMetadataExtractor && responseHeaders) {
+    const metadata = options.vendorMetadataExtractor(responseHeaders);
+    if (metadata && Object.keys(metadata).length > 0) {
+      endEvent.vendorMetadata = metadata;
+    }
+  }
   yield endEvent;
 }
 function appendOpenAIAssistantToolCall(history, toolCalls) {
@@ -341,6 +356,40 @@ function isAbortError(err) {
   return e.name === "APIUserAbortError" || e.name === "AbortError" || e.code === "ABORT_ERR" || typeof e.message === "string" && /aborted/i.test(e.message);
 }
 
+// src/adapters/_metadata.ts
+function makeMetadataExtractor(fields) {
+  return (headers) => {
+    const out = {};
+    for (const f of fields) {
+      const raw = headers.get(f.header);
+      if (raw === null || raw === void 0 || raw === "") continue;
+      if (f.numeric) {
+        const n = Number(raw);
+        if (Number.isFinite(n)) out[f.key] = n;
+      } else {
+        out[f.key] = raw;
+      }
+    }
+    return out;
+  };
+}
+function extractMetadata(fields, headers) {
+  const headersInstance = headers instanceof Headers ? headers : recordToHeaders(headers);
+  return makeMetadataExtractor(fields)(headersInstance);
+}
+function recordToHeaders(record) {
+  const h = new Headers();
+  for (const [name, value] of Object.entries(record)) {
+    if (value === void 0) continue;
+    if (Array.isArray(value)) {
+      for (const v of value) h.append(name, v);
+    } else {
+      h.set(name, value);
+    }
+  }
+  return h;
+}
+
 // src/adapters/openai/manifest.ts
 var OPENAI_KNOWN_MODELS = [
   {
@@ -442,6 +491,12 @@ var openAIManifest = {
 };
 
 // src/adapters/openai/adapter.ts
+var OPENAI_METADATA_EXTRACTOR = makeMetadataExtractor([
+  { header: "x-request-id", key: "request_id" },
+  { header: "openai-processing-ms", key: "processing_ms", numeric: true },
+  { header: "x-ratelimit-remaining-requests", key: "requests_remaining", numeric: true },
+  { header: "x-ratelimit-remaining-tokens", key: "tokens_remaining", numeric: true }
+]);
 var OpenAIAdapter = class {
   vendor = openAIManifest.vendor;
   acceptedAuth = ["apiKey"];
@@ -459,7 +514,8 @@ var OpenAIAdapter = class {
   }
   stream(req) {
     return streamFromOpenAI(this.ensureClient(), req, {
-      knownModels: openAIManifest.knownModels
+      knownModels: openAIManifest.knownModels,
+      vendorMetadataExtractor: OPENAI_METADATA_EXTRACTOR
     });
   }
   appendAssistantToolCall(history, toolCalls) {
@@ -691,6 +747,12 @@ var anthropicManifest = {
 };
 
 // src/adapters/anthropic/adapter.ts
+var ANTHROPIC_METADATA_EXTRACTOR = makeMetadataExtractor([
+  { header: "request-id", key: "request_id" },
+  { header: "anthropic-ratelimit-requests-remaining", key: "requests_remaining", numeric: true },
+  { header: "anthropic-ratelimit-input-tokens-remaining", key: "input_tokens_remaining", numeric: true },
+  { header: "anthropic-ratelimit-output-tokens-remaining", key: "output_tokens_remaining", numeric: true }
+]);
 var AnthropicAdapter = class {
   vendor = anthropicManifest.vendor;
   acceptedAuth = ["apiKey"];
@@ -725,11 +787,20 @@ var AnthropicAdapter = class {
     if (tools) body.tools = tools;
     if (req.temperature !== void 0) body.temperature = req.temperature;
     let stream;
+    let responseHeaders;
     try {
-      stream = await client.messages.create(
+      const apiPromise = client.messages.create(
         body,
         { signal: req.signal }
       );
+      const maybeWithResponse = apiPromise.withResponse;
+      if (typeof maybeWithResponse === "function") {
+        const result = await maybeWithResponse.call(apiPromise);
+        stream = result.data;
+        responseHeaders = result.response?.headers;
+      } else {
+        stream = await apiPromise;
+      }
     } catch (err) {
       if (isAbortError2(err)) {
         yield { type: "end", finishReason: "aborted" };
@@ -840,6 +911,12 @@ var AnthropicAdapter = class {
       ...usage ? { usage } : {},
       ...stopReason ? { rawReason: stopReason } : {}
     };
+    if (responseHeaders) {
+      const metadata = ANTHROPIC_METADATA_EXTRACTOR(responseHeaders);
+      if (Object.keys(metadata).length > 0) {
+        endEvent.vendorMetadata = metadata;
+      }
+    }
     yield endEvent;
   }
   appendAssistantToolCall(history, toolCalls) {
@@ -1632,6 +1709,9 @@ var vertexOpenAIManifest = {
 };
 
 // src/adapters/vertex-openai/adapter.ts
+var VERTEX_OPENAI_METADATA_EXTRACTOR = makeMetadataExtractor([
+  { header: "x-request-id", key: "request_id" }
+]);
 var VertexOpenAIAdapter = class {
   vendor = vertexOpenAIManifest.vendor;
   acceptedAuth = ["vertexServiceAccount"];
@@ -1684,7 +1764,8 @@ var VertexOpenAIAdapter = class {
       // endpoint predates `max_completion_tokens` and may not recognize it.
       // The model-id heuristic wouldn't catch a partner model named in the
       // OpenAI style (unlikely today, but defensive).
-      tokensParam: "max_tokens"
+      tokensParam: "max_tokens",
+      vendorMetadataExtractor: VERTEX_OPENAI_METADATA_EXTRACTOR
     });
   }
   appendAssistantToolCall(history, toolCalls) {
@@ -2142,6 +2223,21 @@ var BedrockAdapter = class {
       ...usage ? { usage } : {},
       ...stopReason ? { rawReason: stopReason } : {}
     };
+    const httpHeaders = response.$metadata?.httpHeaders;
+    if (httpHeaders) {
+      const metadata = extractMetadata(
+        [
+          { header: "x-amzn-bedrock-invocation-latency", key: "invocation_latency_ms", numeric: true },
+          { header: "x-amzn-bedrock-cache-read-input-token-count", key: "cache_read_input_tokens", numeric: true },
+          { header: "x-amzn-bedrock-cache-write-input-token-count", key: "cache_write_input_tokens", numeric: true },
+          { header: "x-amzn-bedrock-trace-id", key: "trace_id" }
+        ],
+        httpHeaders
+      );
+      if (Object.keys(metadata).length > 0) {
+        endEvent.vendorMetadata = metadata;
+      }
+    }
     yield endEvent;
   }
   appendAssistantToolCall(history, toolCalls) {
@@ -2393,6 +2489,14 @@ var azureOpenAIManifest = {
 };
 
 // src/adapters/azure-openai/adapter.ts
+var AZURE_OPENAI_METADATA_EXTRACTOR = makeMetadataExtractor([
+  { header: "x-ms-region", key: "region" },
+  { header: "x-ms-deployment-name", key: "deployment" },
+  { header: "apim-request-id", key: "request_id" },
+  { header: "azureml-model-session", key: "model_session" },
+  { header: "x-ratelimit-remaining-requests", key: "requests_remaining", numeric: true },
+  { header: "x-ratelimit-remaining-tokens", key: "tokens_remaining", numeric: true }
+]);
 var AzureOpenAIAdapter = class {
   vendor = azureOpenAIManifest.vendor;
   acceptedAuth = ["azureOpenAIApiKey"];
@@ -2427,7 +2531,8 @@ var AzureOpenAIAdapter = class {
   stream(req) {
     return streamFromOpenAI(this.ensureClient(), req, {
       knownModels: azureOpenAIManifest.knownModels,
-      tokensParam: "max_completion_tokens"
+      tokensParam: "max_completion_tokens",
+      vendorMetadataExtractor: AZURE_OPENAI_METADATA_EXTRACTOR
     });
   }
   appendAssistantToolCall(history, toolCalls) {
@@ -2552,6 +2657,12 @@ var groqManifest = {
 };
 
 // src/adapters/groq/adapter.ts
+var GROQ_METADATA_EXTRACTOR = makeMetadataExtractor([
+  { header: "x-request-id", key: "request_id" },
+  { header: "x-groq-region", key: "region" },
+  { header: "x-ratelimit-remaining-requests", key: "requests_remaining", numeric: true },
+  { header: "x-ratelimit-remaining-tokens", key: "tokens_remaining", numeric: true }
+]);
 var GroqAdapter = class extends OpenAIAdapter {
   vendor = groqManifest.vendor;
   init(auth, client) {
@@ -2563,6 +2674,12 @@ var GroqAdapter = class extends OpenAIAdapter {
       client
     );
   }
+  stream(req) {
+    return streamFromOpenAI(this.ensureClient(), req, {
+      knownModels: groqManifest.knownModels,
+      vendorMetadataExtractor: GROQ_METADATA_EXTRACTOR
+    });
+  }
 };
 
 // src/adapters/groq/factory.ts
@@ -2572,6 +2689,127 @@ var groqFactory = {
   create: () => new GroqAdapter()
 };
 
+// src/adapters/huggingface/manifest.ts
+var HF_KNOWN_MODELS = [
+  {
+    id: "meta-llama/Llama-3.3-70B-Instruct",
+    displayName: "Llama 3.3 70B Instruct (HF Providers)",
+    capabilities: {
+      streaming: true,
+      tools: true,
+      vision: false,
+      systemPrompt: true,
+      maxContextTokens: 128e3
+    }
+  },
+  {
+    id: "meta-llama/Llama-3.1-8B-Instruct",
+    displayName: "Llama 3.1 8B Instruct (HF Providers)",
+    capabilities: {
+      streaming: true,
+      tools: true,
+      vision: false,
+      systemPrompt: true,
+      maxContextTokens: 128e3
+    }
+  },
+  {
+    id: "Qwen/Qwen2.5-72B-Instruct",
+    displayName: "Qwen 2.5 72B Instruct (HF Providers)",
+    capabilities: {
+      streaming: true,
+      tools: true,
+      vision: false,
+      systemPrompt: true,
+      maxContextTokens: 128e3
+    }
+  },
+  {
+    id: "mistralai/Mistral-7B-Instruct-v0.3",
+    displayName: "Mistral 7B Instruct v0.3 (HF Providers)",
+    capabilities: {
+      streaming: true,
+      tools: true,
+      vision: false,
+      systemPrompt: true,
+      maxContextTokens: 32e3
+    }
+  },
+  {
+    id: "deepseek-ai/DeepSeek-V3",
+    displayName: "DeepSeek V3 (HF Providers)",
+    capabilities: {
+      streaming: true,
+      tools: true,
+      vision: false,
+      systemPrompt: true,
+      maxContextTokens: 64e3
+    }
+  }
+];
+var huggingfaceManifest = {
+  vendor: "huggingface",
+  displayName: "HuggingFace Inference Providers",
+  authKinds: [
+    {
+      kind: "apiKey",
+      displayName: "API Key",
+      fields: [
+        {
+          name: "apiKey",
+          label: "HuggingFace Token",
+          type: "password",
+          required: true,
+          help: "Get a token from https://huggingface.co/settings/tokens (Read scope is sufficient). Inference Providers requires a credit balance \u2014 claim the free monthly tier or attach a payment method at https://huggingface.co/billing."
+        },
+        {
+          name: "baseURL",
+          label: "Base URL",
+          type: "url",
+          required: false,
+          default: "https://router.huggingface.co/v1",
+          help: "Override only for proxies. Default is the HF Providers router endpoint."
+        }
+      ]
+    }
+  ],
+  knownModels: HF_KNOWN_MODELS,
+  // The HF router exposes /v1/models with the same Bearer auth.
+  supportsModelListing: true,
+  docsUrl: "https://huggingface.co/docs/inference-providers"
+};
+
+// src/adapters/huggingface/adapter.ts
+var HUGGINGFACE_METADATA_EXTRACTOR = makeMetadataExtractor([
+  { header: "x-inference-provider", key: "provider" },
+  { header: "x-request-id", key: "request_id" }
+]);
+var HuggingfaceAdapter = class extends OpenAIAdapter {
+  vendor = huggingfaceManifest.vendor;
+  init(auth, client) {
+    super.init(
+      {
+        ...auth,
+        baseURL: auth.baseURL ?? "https://router.huggingface.co/v1"
+      },
+      client
+    );
+  }
+  stream(req) {
+    return streamFromOpenAI(this.ensureClient(), req, {
+      knownModels: huggingfaceManifest.knownModels,
+      vendorMetadataExtractor: HUGGINGFACE_METADATA_EXTRACTOR
+    });
+  }
+};
+
+// src/adapters/huggingface/factory.ts
+var huggingfaceFactory = {
+  vendor: huggingfaceManifest.vendor,
+  manifest: huggingfaceManifest,
+  create: () => new HuggingfaceAdapter()
+};
+
 // src/adapters/index.ts
 registerAdapter(openAIFactory);
 registerAdapter(deepseekFactory);
@@ -2582,6 +2820,7 @@ registerAdapter(vertexOpenAIFactory);
 registerAdapter(bedrockFactory);
 registerAdapter(azureOpenAIFactory);
 registerAdapter(groqFactory);
+registerAdapter(huggingfaceFactory);
 
 // src/index.ts
 async function createLlm(args) {