@jambonz/llm 0.4.6 → 0.5.1

package/dist/test-kit/index.cjs

@@ -117,9 +117,22 @@ async function* streamFromOpenAI(client, req, options) {
     const param = options.tokensParam ?? (needsMaxCompletionTokens(req.model) ? "max_completion_tokens" : "max_tokens");
     body[param] = effectiveMaxTokens;
   }
+  const t0 = process.hrtime.bigint();
+  let t1;
+  let t2;
   let stream;
+  let responseHeaders;
   try {
-    stream = await client.chat.completions.create(body, { signal: req.signal });
+    const apiPromise = client.chat.completions.create(body, { signal: req.signal });
+    const maybeWithResponse = apiPromise.withResponse;
+    if (typeof maybeWithResponse === "function") {
+      const result = await maybeWithResponse.call(apiPromise);
+      stream = result.data;
+      responseHeaders = result.response?.headers;
+    } else {
+      stream = await apiPromise;
+    }
+    t1 = process.hrtime.bigint();
   } catch (err) {
     if (isAbortError(err)) {
       yield { type: "end", finishReason: "aborted" };
@@ -151,6 +164,7 @@ async function* streamFromOpenAI(client, req, options) {
       const delta = choice.delta;
       const finishReason = choice.finish_reason;
       if (delta?.content) {
+        if (t2 === void 0) t2 = process.hrtime.bigint();
         yield { type: "token", text: delta.content };
       }
       if (delta?.tool_calls) {
@@ -214,6 +228,20 @@ async function* streamFromOpenAI(client, req, options) {
     ...usage ? { usage } : {},
     ...finalRawReason ? { rawReason: finalRawReason } : {}
   };
+  if (options.vendorMetadataExtractor && responseHeaders) {
+    const metadata = options.vendorMetadataExtractor(responseHeaders);
+    if (metadata && Object.keys(metadata).length > 0) {
+      endEvent.vendorMetadata = metadata;
+    }
+  }
+  if (t1 !== void 0) {
+    const t3 = process.hrtime.bigint();
+    endEvent.clientTiming = {
+      requestToHeadersMs: Number(t1 - t0) / 1e6,
+      ...t2 !== void 0 ? { headersToFirstTokenMs: Number(t2 - t1) / 1e6 } : {},
+      totalMs: Number(t3 - t0) / 1e6
+    };
+  }
   yield endEvent;
 }
 function appendOpenAIAssistantToolCall(history, toolCalls) {
@@ -341,6 +369,40 @@ function isAbortError(err) {
   return e.name === "APIUserAbortError" || e.name === "AbortError" || e.code === "ABORT_ERR" || typeof e.message === "string" && /aborted/i.test(e.message);
 }
 
+// src/adapters/_metadata.ts
+function makeMetadataExtractor(fields) {
+  return (headers) => {
+    const out = {};
+    for (const f of fields) {
+      const raw = headers.get(f.header);
+      if (raw === null || raw === void 0 || raw === "") continue;
+      if (f.numeric) {
+        const n = Number(raw);
+        if (Number.isFinite(n)) out[f.key] = n;
+      } else {
+        out[f.key] = raw;
+      }
+    }
+    return out;
+  };
+}
+function extractMetadata(fields, headers) {
+  const headersInstance = headers instanceof Headers ? headers : recordToHeaders(headers);
+  return makeMetadataExtractor(fields)(headersInstance);
+}
+function recordToHeaders(record) {
+  const h = new Headers();
+  for (const [name, value] of Object.entries(record)) {
+    if (value === void 0) continue;
+    if (Array.isArray(value)) {
+      for (const v of value) h.append(name, v);
+    } else {
+      h.set(name, value);
+    }
+  }
+  return h;
+}
+
 // src/adapters/openai/manifest.ts
 var OPENAI_KNOWN_MODELS = [
   {
@@ -442,6 +504,12 @@ var openAIManifest = {
 };
 
 // src/adapters/openai/adapter.ts
+var OPENAI_METADATA_EXTRACTOR = makeMetadataExtractor([
+  { header: "x-request-id", key: "request_id" },
+  { header: "openai-processing-ms", key: "processing_ms", numeric: true },
+  { header: "x-ratelimit-remaining-requests", key: "requests_remaining", numeric: true },
+  { header: "x-ratelimit-remaining-tokens", key: "tokens_remaining", numeric: true }
+]);
 var OpenAIAdapter = class {
   vendor = openAIManifest.vendor;
   acceptedAuth = ["apiKey"];
@@ -459,7 +527,8 @@ var OpenAIAdapter = class {
   }
   stream(req) {
     return streamFromOpenAI(this.ensureClient(), req, {
-      knownModels: openAIManifest.knownModels
+      knownModels: openAIManifest.knownModels,
+      vendorMetadataExtractor: OPENAI_METADATA_EXTRACTOR
     });
   }
   appendAssistantToolCall(history, toolCalls) {
@@ -691,6 +760,12 @@ var anthropicManifest = {
 };
 
 // src/adapters/anthropic/adapter.ts
+var ANTHROPIC_METADATA_EXTRACTOR = makeMetadataExtractor([
+  { header: "request-id", key: "request_id" },
+  { header: "anthropic-ratelimit-requests-remaining", key: "requests_remaining", numeric: true },
+  { header: "anthropic-ratelimit-input-tokens-remaining", key: "input_tokens_remaining", numeric: true },
+  { header: "anthropic-ratelimit-output-tokens-remaining", key: "output_tokens_remaining", numeric: true }
+]);
 var AnthropicAdapter = class {
   vendor = anthropicManifest.vendor;
   acceptedAuth = ["apiKey"];
@@ -725,11 +800,20 @@ var AnthropicAdapter = class {
     if (tools) body.tools = tools;
     if (req.temperature !== void 0) body.temperature = req.temperature;
     let stream;
+    let responseHeaders;
     try {
-      stream = await client.messages.create(
+      const apiPromise = client.messages.create(
         body,
         { signal: req.signal }
       );
+      const maybeWithResponse = apiPromise.withResponse;
+      if (typeof maybeWithResponse === "function") {
+        const result = await maybeWithResponse.call(apiPromise);
+        stream = result.data;
+        responseHeaders = result.response?.headers;
+      } else {
+        stream = await apiPromise;
+      }
     } catch (err) {
       if (isAbortError2(err)) {
         yield { type: "end", finishReason: "aborted" };
@@ -840,6 +924,12 @@ var AnthropicAdapter = class {
       ...usage ? { usage } : {},
       ...stopReason ? { rawReason: stopReason } : {}
     };
+    if (responseHeaders) {
+      const metadata = ANTHROPIC_METADATA_EXTRACTOR(responseHeaders);
+      if (Object.keys(metadata).length > 0) {
+        endEvent.vendorMetadata = metadata;
+      }
+    }
     yield endEvent;
   }
   appendAssistantToolCall(history, toolCalls) {
@@ -1632,6 +1722,9 @@ var vertexOpenAIManifest = {
 };
 
 // src/adapters/vertex-openai/adapter.ts
+var VERTEX_OPENAI_METADATA_EXTRACTOR = makeMetadataExtractor([
+  { header: "x-request-id", key: "request_id" }
+]);
 var VertexOpenAIAdapter = class {
   vendor = vertexOpenAIManifest.vendor;
   acceptedAuth = ["vertexServiceAccount"];
@@ -1684,7 +1777,8 @@ var VertexOpenAIAdapter = class {
       // endpoint predates `max_completion_tokens` and may not recognize it.
       // The model-id heuristic wouldn't catch a partner model named in the
       // OpenAI style (unlikely today, but defensive).
-      tokensParam: "max_tokens"
+      tokensParam: "max_tokens",
+      vendorMetadataExtractor: VERTEX_OPENAI_METADATA_EXTRACTOR
     });
   }
   appendAssistantToolCall(history, toolCalls) {
@@ -2142,6 +2236,21 @@ var BedrockAdapter = class {
       ...usage ? { usage } : {},
       ...stopReason ? { rawReason: stopReason } : {}
     };
+    const httpHeaders = response.$metadata?.httpHeaders;
+    if (httpHeaders) {
+      const metadata = extractMetadata(
+        [
+          { header: "x-amzn-bedrock-invocation-latency", key: "invocation_latency_ms", numeric: true },
+          { header: "x-amzn-bedrock-cache-read-input-token-count", key: "cache_read_input_tokens", numeric: true },
+          { header: "x-amzn-bedrock-cache-write-input-token-count", key: "cache_write_input_tokens", numeric: true },
+          { header: "x-amzn-bedrock-trace-id", key: "trace_id" }
+        ],
+        httpHeaders
+      );
+      if (Object.keys(metadata).length > 0) {
+        endEvent.vendorMetadata = metadata;
+      }
+    }
     yield endEvent;
   }
   appendAssistantToolCall(history, toolCalls) {
@@ -2393,6 +2502,14 @@ var azureOpenAIManifest = {
 };
 
 // src/adapters/azure-openai/adapter.ts
+var AZURE_OPENAI_METADATA_EXTRACTOR = makeMetadataExtractor([
+  { header: "x-ms-region", key: "region" },
+  { header: "x-ms-deployment-name", key: "deployment" },
+  { header: "apim-request-id", key: "request_id" },
+  { header: "azureml-model-session", key: "model_session" },
+  { header: "x-ratelimit-remaining-requests", key: "requests_remaining", numeric: true },
+  { header: "x-ratelimit-remaining-tokens", key: "tokens_remaining", numeric: true }
+]);
 var AzureOpenAIAdapter = class {
   vendor = azureOpenAIManifest.vendor;
   acceptedAuth = ["azureOpenAIApiKey"];
@@ -2427,7 +2544,8 @@ var AzureOpenAIAdapter = class {
   stream(req) {
     return streamFromOpenAI(this.ensureClient(), req, {
       knownModels: azureOpenAIManifest.knownModels,
-      tokensParam: "max_completion_tokens"
+      tokensParam: "max_completion_tokens",
+      vendorMetadataExtractor: AZURE_OPENAI_METADATA_EXTRACTOR
     });
   }
   appendAssistantToolCall(history, toolCalls) {
@@ -2552,6 +2670,12 @@ var groqManifest = {
 };
 
 // src/adapters/groq/adapter.ts
+var GROQ_METADATA_EXTRACTOR = makeMetadataExtractor([
+  { header: "x-request-id", key: "request_id" },
+  { header: "x-groq-region", key: "region" },
+  { header: "x-ratelimit-remaining-requests", key: "requests_remaining", numeric: true },
+  { header: "x-ratelimit-remaining-tokens", key: "tokens_remaining", numeric: true }
+]);
 var GroqAdapter = class extends OpenAIAdapter {
   vendor = groqManifest.vendor;
   init(auth, client) {
@@ -2563,6 +2687,12 @@ var GroqAdapter = class extends OpenAIAdapter {
       client
     );
   }
+  stream(req) {
+    return streamFromOpenAI(this.ensureClient(), req, {
+      knownModels: groqManifest.knownModels,
+      vendorMetadataExtractor: GROQ_METADATA_EXTRACTOR
+    });
+  }
 };
 
 // src/adapters/groq/factory.ts
@@ -2663,6 +2793,10 @@ var huggingfaceManifest = {
 };
 
 // src/adapters/huggingface/adapter.ts
+var HUGGINGFACE_METADATA_EXTRACTOR = makeMetadataExtractor([
+  { header: "x-inference-provider", key: "provider" },
+  { header: "x-request-id", key: "request_id" }
+]);
 var HuggingfaceAdapter = class extends OpenAIAdapter {
   vendor = huggingfaceManifest.vendor;
   init(auth, client) {
@@ -2674,6 +2808,12 @@ var HuggingfaceAdapter = class extends OpenAIAdapter {
       client
     );
   }
+  stream(req) {
+    return streamFromOpenAI(this.ensureClient(), req, {
+      knownModels: huggingfaceManifest.knownModels,
+      vendorMetadataExtractor: HUGGINGFACE_METADATA_EXTRACTOR
+    });
+  }
 };
 
 // src/adapters/huggingface/factory.ts