@inflexa-ai/harness 0.14.2 → 0.15.1

package/dist/providers/ai-sdk.js

@@ -1,4 +1,4 @@
-import { generateText, streamText } from "ai";
+import { generateText, streamText, wrapLanguageModel } from "ai";
 import { createAnthropic } from "@ai-sdk/anthropic";
 import { createOpenAICompatible } from "@ai-sdk/openai-compatible";
 import { APICallError } from "@ai-sdk/provider";
@@ -222,6 +222,10 @@ function createRetry(signal, logger) {
  * vendor's own snake_case.
  *
  * `inputTokens` is the total billed prefix, cache reads included.
+ *
+ * `reasoningTokens` rides the neutral `outputTokenDetails` beside the text
+ * count and is copied verbatim: providers disagree on whether reasoning tokens
+ * are already inside the output total, so the harness reconciles nothing.
  */
 function toChatUsage(usage) {
     if (usage === undefined)
@@ -231,22 +235,72 @@ function toChatUsage(usage) {
         outputTokens: usage.outputTokens,
         cacheCreationInputTokens: usage.inputTokenDetails?.cacheWriteTokens,
         cacheReadInputTokens: usage.inputTokenDetails?.cacheReadTokens,
+        reasoningTokens: usage.outputTokenDetails?.reasoningTokens,
     };
 }
-function responseFromMessages(messages, fallbackText, finishReason, rawFinishReason, usage) {
+/** The id of the model bound into this provider — a bare id string is its own id. */
+function requestedModelIdOf(model) {
+    return typeof model === "string" ? model : model.modelId;
+}
+/**
+ * Capture the served model id at the raw model boundary, because that is the
+ * only place it is still distinguishable from the requested one: `generateText`
+ * and `streamText` both coalesce an unreported response model id to the bound
+ * model's own id, so reading `result.response.modelId` would silently republish
+ * the requested id as an endpoint's claim. A middleware sees the provider's
+ * result before that fallback applies, so "reported nothing" stays absent.
+ *
+ * Built per call, never per provider: one provider instance serves concurrent
+ * calls, and a shared mutable slot would hand one call another's answer. A bare
+ * model-id string has no wire result to observe, so it captures nothing.
+ */
+function captureServedModelId(model) {
+    if (typeof model === "string")
+        return { model, servedModelId: () => undefined };
+    let served;
+    const wrapped = wrapLanguageModel({
+        model,
+        middleware: {
+            wrapGenerate: async ({ doGenerate }) => {
+                const result = await doGenerate();
+                served = result.response?.modelId;
+                return result;
+            },
+            wrapStream: async ({ doStream }) => {
+                const result = await doStream();
+                return {
+                    ...result,
+                    stream: result.stream.pipeThrough(new TransformStream({
+                        transform(chunk, controller) {
+                            if (chunk.type === "response-metadata" && chunk.modelId !== undefined)
+                                served = chunk.modelId;
+                            controller.enqueue(chunk);
+                        },
+                    })),
+                };
+            },
+        },
+    });
+    return { model: wrapped, servedModelId: () => served };
+}
+function responseFromMessages(input) {
+    const { messages, fallbackText, finishReason, rawFinishReason, requestedModelId, servedModelId } = input;
+    const usage = toChatUsage(input.usage);
     const message = [...messages].reverse().find((m) => m.role === "assistant");
     if (message === undefined) {
-        return {
-            message: { role: "assistant", content: fallbackText },
-            finishReason,
-            rawFinishReason,
-            usage: toChatUsage(usage),
-        };
+        return { message: { role: "assistant", content: fallbackText }, finishReason, rawFinishReason, usage, requestedModelId, servedModelId };
     }
-    return { message, finishReason, rawFinishReason, usage: toChatUsage(usage) };
+    return { message, finishReason, rawFinishReason, usage, requestedModelId, servedModelId };
 }
-function responseFromGenerate(result) {
-    return responseFromMessages(result.responseMessages, result.text, result.finishReason, result.rawFinishReason, result.usage);
+function responseFromGenerate(result, identity) {
+    return responseFromMessages({
+        messages: result.responseMessages,
+        fallbackText: result.text,
+        finishReason: result.finishReason,
+        rawFinishReason: result.rawFinishReason,
+        usage: result.usage,
+        ...identity,
+    });
 }
 /**
  * Drop empty text parts — and messages left with no content at all — before
@@ -279,17 +333,19 @@ export function createAiSdkProvider(deps) {
     const capabilities = { toolCalling: deps.capabilities?.toolCalling ?? true };
     const logger = (deps.logger ?? createNoopLogger()).named("providers.ai-sdk");
     const maxOutputTokens = deps.maxOutputTokens ?? DEFAULT_MAX_OUTPUT_TOKENS;
+    const requestedModelId = requestedModelIdOf(deps.model);
     routeSdkWarningsTo(logger);
     function chat(req, session, signal) {
         const run = async () => {
             const retry = createRetry(signal, logger);
+            const capture = captureServedModelId(deps.model);
             try {
                 const result = await retry(async () => {
                     // Attribution headers are time-limited; resolving them inside the
                     // retried closure keeps them fresh across a multi-minute window.
                     const headers = await resolveBillingHeaders(deps.resolveBilling, session);
                     return generateText({
-                        model: deps.model,
+                        model: capture.model,
                         system: req.system,
                         messages: sanitizeMessages(req.messages),
                         tools: req.tools,
@@ -304,7 +360,7 @@ export function createAiSdkProvider(deps) {
                         providerOptions: req.providerOptions,
                     });
                 });
-                return ok(responseFromGenerate(result));
+                return ok(responseFromGenerate(result, { requestedModelId, servedModelId: capture.servedModelId() }));
             }
             catch (e) {
                 if (isAbortError(e) || signal?.aborted)
@@ -316,6 +372,7 @@ export function createAiSdkProvider(deps) {
     }
     async function* chatStream(req, session, signal) {
         const retry = createRetry(signal, logger);
+        const capture = captureServedModelId(deps.model);
         try {
             // Retry covers only stream establishment: streamText defers wire errors
             // to consumption, so a failure is not visible until the stream is read.
@@ -328,7 +385,7 @@ export function createAiSdkProvider(deps) {
             const opened = await retry(async () => {
                 const headers = await resolveBillingHeaders(deps.resolveBilling, session);
                 const result = streamText({
-                    model: deps.model,
+                    model: capture.model,
                     system: req.system,
                     messages: sanitizeMessages(req.messages),
                     tools: req.tools,
@@ -364,7 +421,15 @@ export function createAiSdkProvider(deps) {
                 return { kind: "streaming", result, iterator, firstDelta: first.value };
             });
             if (opened.kind === "completed") {
-                const response = responseFromMessages(opened.messages, "", opened.finishReason, opened.rawFinishReason, opened.usage);
+                const response = responseFromMessages({
+                    messages: opened.messages,
+                    fallbackText: "",
+                    finishReason: opened.finishReason,
+                    rawFinishReason: opened.rawFinishReason,
+                    usage: opened.usage,
+                    requestedModelId,
+                    servedModelId: capture.servedModelId(),
+                });
                 yield { type: "done", response };
                 return;
             }
@@ -377,7 +442,17 @@ export function createAiSdkProvider(deps) {
                 text += next.value;
                 yield { type: "text-delta", text: next.value };
             }
-            const response = responseFromMessages(await result.responseMessages, text, await result.finishReason, await result.rawFinishReason, await result.usage);
+            const response = responseFromMessages({
+                messages: await result.responseMessages,
+                fallbackText: text,
+                finishReason: await result.finishReason,
+                rawFinishReason: await result.rawFinishReason,
+                usage: await result.usage,
+                requestedModelId,
+                // Read after the drain: the metadata chunk carrying the served id
+                // reaches the capture only as the stream is consumed.
+                servedModelId: capture.servedModelId(),
+            });
             yield { type: "done", response };
         }
         catch (e) {

package/dist/providers/streaming-chat.js

@@ -67,6 +67,11 @@ export function createStreamingChat(provider, onText) {
                         // response is authoritative, so return it rather than the assembled deltas.
                         if (final !== undefined)
                             return ok(final);
+                        // The partial carries no model identity and no usage: it is
+                        // assembled from deltas rather than from a provider response, so
+                        // nothing here reported which model answered or what it cost.
+                        // Stamping the requested id would manufacture a claim the
+                        // endpoint never made.
                         return ok({ message: { role: "assistant", content: partial }, finishReason: "aborted" });
                     }
                     return err(toProviderError(e, workload));

package/dist/providers/types.d.ts

@@ -46,6 +46,11 @@ export type PromptCachePolicy = {
  * seeds the cache (billed at a premium; it only pays for itself once a later
  * call reads it back).
  *
+ * `reasoningTokens` is exactly what the provider reported — never derived from,
+ * nor reconciled against, `outputTokens`. Whether reasoning tokens are counted
+ * inside the output total varies by provider, so any arithmetic between the two
+ * would be a guess dressed as a figure.
+ *
  * Every field is optional: a provider that reports no usage at all, or reports
  * totals without a cache breakdown, is legitimate. Absent means "not reported",
  * never "zero".
@@ -55,6 +60,7 @@ export interface ChatUsage {
     readonly outputTokens?: number;
     readonly cacheCreationInputTokens?: number;
     readonly cacheReadInputTokens?: number;
+    readonly reasoningTokens?: number;
 }
 export interface ChatResponse {
     readonly message: Extract<ModelMessage, {
@@ -72,6 +78,23 @@ export interface ChatResponse {
     readonly finishReason: FinishReason | "aborted";
     readonly rawFinishReason?: string;
     readonly usage?: ChatUsage;
+    /**
+     * The id of the model this provider instance is bound to — what the caller
+     * asked for. Absent when the binding names no id.
+     */
+    readonly requestedModelId?: string;
+    /**
+     * The model id the provider itself reported as having answered. Absent
+     * whenever the provider reported none — it is NEVER filled in from
+     * `requestedModelId`, because a claim about which model answered is only
+     * worth anything when the endpoint actually made it.
+     *
+     * A value differing from `requestedModelId` is not an error: the pair exists
+     * so a consumer can observe an endpoint or proxy serving a different model
+     * version than the one configured. Treat the comparison as best-effort
+     * diagnostics, never as an invariant.
+     */
+    readonly servedModelId?: string;
 }
 export type ChatStreamEvent = {
     readonly type: "text-delta";

package/dist/providers/types.js

@@ -5,4 +5,5 @@
  * vendor-neutral interfaces. Every call carries an `AgentSession`; billing
  * attribution is resolved lazily at the provider wire boundary.
  */
+const _assertChatUsageFields = true;
 export {};