@lenne.tech/nest-server 11.40.0 → 11.41.1

package/src/core/modules/ai/providers/openai-compatible.provider.ts

@@ -9,6 +9,7 @@ import {
   LlmMessage,
   LlmResponse,
   LlmToolSchema,
+  LlmUsage,
 } from '../interfaces/llm-provider.interface';
 import { ResolvedAiConnection } from '../interfaces/resolved-ai-connection.interface';
 
@@ -29,6 +30,22 @@ export class OpenAiCompatibleProvider implements ILlmProvider {
   readonly capabilities: LlmCapabilities;
   readonly name = 'openai-compatible';
 
+  /**
+   * Floor below which the reasoning retry is not attempted at all.
+   *
+   * The retry shares the ORIGINAL call's timeout budget (see {@link chat}), so a
+   * first call that nearly exhausted it leaves too little for a second. Starting one
+   * anyway would spend the remainder waiting for a request that cannot finish, and
+   * then report the timeout as "the model rejects reasoning_effort". Measured retries
+   * answer in well under a second once the thinking phase is off.
+   *
+   * Consequence worth stating: a connection whose whole `timeoutMs` is below this floor
+   * never retries at all. That is the right trade for a value this far below any usable
+   * LLM timeout (the default is 120 s), but it is a behaviour the number decides, so it
+   * belongs here rather than in a reader's head.
+   */
+  protected static readonly MIN_REASONING_RETRY_MS = 1_000;
+
   private readonly logger = new Logger(OpenAiCompatibleProvider.name);
   private readonly defaultTimeoutMs: number;
 
@@ -77,11 +94,109 @@ export class OpenAiCompatibleProvider implements ILlmProvider {
         type: 'function',
       }));
     }
-    if (this.capabilities.jsonResponse) {
+    // A per-request `model` overrides the connection's — but the capability flags
+    // were probed against `connection.model` and persisted per CONNECTION, never
+    // per model. Applying them to a different model asserts something that was
+    // never measured: the endpoint may reject `response_format` for it, and the
+    // caller sees a transport error where it expected an answer. Fall back to the
+    // safe subset (prompt-driven JSON + defensive parsing) whenever the model the
+    // request actually targets is not the one the probe ran against.
+    // `options.jsonResponse === false` narrows this off for a single call — the way a
+    // caller whose PROMPT asks for prose says so. It can only ever narrow: the flag
+    // was measured against this connection, so no option may assert it where the
+    // probe never ran.
+    if (this.capabilities.jsonResponse && options?.jsonResponse !== false && body.model === this.connection.model) {
       body.response_format = { type: 'json_object' };
     }
 
     const timeoutMs = options?.timeoutMs ?? this.defaultTimeoutMs;
+    const startedAt = Date.now();
+    const answer = await this.postCompletion(url, body, timeoutMs);
+    if (!this.isReasoningStarved(answer)) {
+      return answer;
+    }
+
+    // The thinking phase ate the whole budget before a single character of the
+    // answer. Say so with the numbers — the failure is otherwise indistinguishable
+    // in a log from "the model had nothing to say", which is what made it so
+    // expensive to diagnose.
+    this.logger.warn(
+      `AI completion for model "${body.model}" returned NO content: finish_reason=${answer.finishReason}, ` +
+        `${answer.usage?.reasoningTokens ?? 0} of ${answer.usage?.completionTokens ?? 0} output tokens were spent ` +
+        `thinking against a budget of ${body.max_tokens}. Retrying once without the thinking phase.`,
+    );
+    // The retry shares the ORIGINAL call's timeout budget instead of starting a fresh
+    // one. `CoreAiService` checks `ai.maxRunMs` BETWEEN agent-loop iterations and never
+    // mid-call, so a second full timeout here would silently double the ceiling the
+    // framework documents as `maxIterations` x the per-call timeout — 20 minutes instead
+    // of 10 at the defaults. On the SSE path that is time the client spends in total
+    // silence: `promptStream()` yields nothing but tool actions until the run settles,
+    // and a starved completion produces no tool call to report.
+    const remainingMs = timeoutMs - (Date.now() - startedAt);
+    if (remainingMs < OpenAiCompatibleProvider.MIN_REASONING_RETRY_MS) {
+      this.logger.warn(
+        `Not retrying model "${body.model}" without the thinking phase: only ${Math.max(0, remainingMs)}ms of the ` +
+          `${timeoutMs}ms budget remain. Keeping the original, empty completion.`,
+      );
+      return answer;
+    }
+
+    try {
+      const retried = await this.postCompletion(url, { ...body, reasoning_effort: 'none' }, remainingMs);
+      // BOTH calls were billed upstream, so both must reach the caller. `CoreAiService`
+      // accumulates only what `chat()` returns, that total lands in the audit record's
+      // `totalTokens`, and `CoreAiBudgetService` enforces `ai.budget` from exactly that
+      // field. Returning the retry's usage alone would hide the STARVED call — the one
+      // that by definition burned the entire `max_tokens` allowance — from the limit
+      // that exists to bound it.
+      return { ...retried, usage: this.mergeUsage(answer.usage, retried.usage) };
+    } catch (err) {
+      // Usually a 400: the backend does not accept `reasoning_effort` for this model.
+      // But the same catch also sees timeouts and transport failures, and naming the
+      // 400 for one of those would send the reader after a cause that is not there —
+      // the precise kind of misdirection this whole change exists to remove. Report
+      // what actually happened and let the original answer stand: it keeps its
+      // `finishReason` and the usage the caller already paid for.
+      this.logger.warn(
+        `Retry without the thinking phase failed for model "${body.model}" ` +
+          `(${(err as Error)?.message ?? 'unknown error'}) — keeping the original, empty completion. ` +
+          'Raise the token budget or configure a model that answers within it.',
+      );
+      return answer;
+    }
+  }
+
+  /**
+   * Add up the usage of the two calls a retried completion actually made.
+   *
+   * Kept separate from {@link chat} because "what did this run cost" is a question the
+   * budget, the audit record and the client's usage summary all read from one number,
+   * and a provider that reports only half of it under-enforces every limit built on it.
+   *
+   * A field absent from BOTH sides stays absent — a backend that reports no breakdown
+   * must not be made to look like it reported zero.
+   */
+  protected mergeUsage(first?: LlmUsage, second?: LlmUsage): LlmUsage | undefined {
+    if (!first) {
+      return second;
+    }
+    if (!second) {
+      return first;
+    }
+    const sum = (a?: number, b?: number) => (a === undefined && b === undefined ? undefined : (a ?? 0) + (b ?? 0));
+    return {
+      completionTokens: sum(first.completionTokens, second.completionTokens),
+      promptTokens: sum(first.promptTokens, second.promptTokens),
+      reasoningTokens: sum(first.reasoningTokens, second.reasoningTokens),
+      totalTokens: sum(first.totalTokens, second.totalTokens),
+    };
+  }
+
+  /**
+   * POST one completion and map it to {@link LlmResponse}. Transport failures and
+   * non-2xx responses throw, exactly as a single-shot `chat()` always did.
+   */
+  protected async postCompletion(url: string, body: Record<string, any>, timeoutMs: number): Promise<LlmResponse> {
     let response: Response;
     try {
       response = await fetch(url, {
@@ -106,26 +221,64 @@ export class OpenAiCompatibleProvider implements ILlmProvider {
     }
 
     const result = (await response.json()) as {
-      choices?: { message?: { content?: string; tool_calls?: any[] } }[];
-      usage?: { completion_tokens?: number; prompt_tokens?: number; total_tokens?: number };
+      choices?: { finish_reason?: string; message?: { content?: string; tool_calls?: any[] } }[];
+      usage?: {
+        completion_tokens?: number;
+        completion_tokens_details?: { reasoning_tokens?: number };
+        prompt_tokens?: number;
+        total_tokens?: number;
+      };
     };
 
-    const choice = result.choices?.[0]?.message;
-    const text = choice?.content ?? '';
-    const nativeToolCalls = this.capabilities.nativeTools ? this.mapNativeToolCalls(choice?.tool_calls) : undefined;
+    const choice = result.choices?.[0];
+    const text = choice?.message?.content ?? '';
+    const nativeToolCalls = this.capabilities.nativeTools
+      ? this.mapNativeToolCalls(choice?.message?.tool_calls)
+      : undefined;
 
     return {
+      finishReason: choice?.finish_reason,
       raw: result,
       text,
       toolCalls: nativeToolCalls,
       usage: {
         completionTokens: result.usage?.completion_tokens,
         promptTokens: result.usage?.prompt_tokens,
+        reasoningTokens: result.usage?.completion_tokens_details?.reasoning_tokens,
         totalTokens: result.usage?.total_tokens,
       },
     };
   }
 
+  /**
+   * True when the output budget was exhausted before the model produced ANY
+   * answer — a reasoning model that spent every token on its thinking phase.
+   *
+   * The backend answers `200` with `finish_reason: 'length'`, empty content and
+   * (where it reports the breakdown) `reasoning_tokens == completion_tokens`.
+   * Measured against an OpenAI-compatible hosting endpoint on 2026-09-07 with a 900-token budget:
+   * `Mistral-Medium-3.5-128B` and `Qwen3.6-35B-A3B-FP8` both return nothing,
+   * while `gpt-oss-120b` and `Ministral-3-14B-Instruct` answer normally.
+   *
+   * The narrowness is deliberate, in both directions:
+   *
+   * - **Content present** → the answer merely got cut short. That is a budget
+   *   question the caller can now see via `finishReason`, and discarding the
+   *   partial text to re-ask would lose something usable.
+   * - **`finish_reason: 'stop'` with empty content** → the model chose to say
+   *   nothing. No budget ran out, so removing the thinking phase addresses
+   *   nothing and would cost a second upstream call on every such answer.
+   * - **Tool calls present** → the model DID answer, in the tool channel.
+   *
+   * `reasoning_tokens` is treated as corroborating, not required: backends that
+   * omit the breakdown produce exactly the same symptom, and the retry is
+   * harmless where the diagnosis is wrong (one extra call on a request that
+   * returned nothing either way).
+   */
+  protected isReasoningStarved(response: LlmResponse): boolean {
+    return response.finishReason === 'length' && !response.text && !response.toolCalls?.length;
+  }
+
   /**
    * Optional SSRF hardening: when `ai.allowedBaseUrlHosts` is configured (non-empty),
    * only allow requests to those hosts (matched by `host` incl. port, or bare
@@ -162,6 +315,50 @@ export class OpenAiCompatibleProvider implements ILlmProvider {
     }
   }
 
+  /**
+   * Output budget for the native-tool probe.
+   *
+   * A reasoning model spends output tokens on its thinking phase BEFORE it emits
+   * `tool_calls`. With a budget of a few tokens the endpoint answers `200` with
+   * `finish_reason: 'length'` and no tool call at all — which the probe used to read
+   * as "native tools unsupported", persisting a false negative that is never
+   * re-probed and degrades the assistant to emulated tool calling for good.
+   *
+   * Measured against an OpenAI-compatible hosting endpoint on 2026-07-25
+   * (`tool_choice: 'required'` plus a trivial `ping` tool, varying only
+   * `max_tokens`):
+   *
+   * | model                    |  8 | 64 | 256 |
+   * |--------------------------|----|----|-----|
+   * | Ministral-3-14B-Instruct | ✅ | ✅ | ✅  |
+   * | Mistral-Medium-3.5-128B  | ✅ | ✅ | ✅  |
+   * | gpt-oss-120b             | ❌ | ✅ | ✅  |
+   * | Qwen3.5-122B-A10B-FP8    | ❌ | ✅ | ✅  |
+   * | Qwen3.6-35B-A3B-FP8      | ❌ | ❌ | ✅  |
+   *
+   * 256 covers every model tested and costs a few hundred tokens ONCE per
+   * connection, which is nothing against the cost of the wrong flag.
+   */
+  protected static readonly NATIVE_TOOL_PROBE_MAX_TOKENS = 256;
+
+  /**
+   * Second and FINAL budget for the native-tool probe, used only when the first
+   * attempt came back truncated.
+   *
+   * The retry has to be bounded, and the bound has to live here. An inconclusive
+   * result leaves the capability `undefined`, `detectAndPersistCapabilities` only
+   * persists booleans, and the orchestrator re-runs detection whenever a flag is
+   * undefined — so "just leave it undetected and try again later" would fire one
+   * extra upstream completion before EVERY user prompt, ahead of the rate limiter
+   * and outside any budget accounting. Two attempts, then a definite answer.
+   *
+   * Returning `false` after a model failed to emit a tool call within 1024 output
+   * tokens is not the false negative this fix exists to prevent: a model that needs
+   * more than that before its first tool call cannot drive an agent loop usefully
+   * anyway, and emulated tool calling is the correct fallback for it.
+   */
+  protected static readonly NATIVE_TOOL_PROBE_MAX_TOKENS_RETRY = 1024;
+
   /**
    * The configured egress allowlist as a lowercase array, whatever shape it has.
    *
@@ -221,27 +418,162 @@ export class OpenAiCompatibleProvider implements ILlmProvider {
   /**
    * Probe the backend to auto-detect capabilities for flags the connection left
    * undefined. Explicit flags are authoritative and are NOT probed. Best effort:
-   * - JSON: send `response_format: json_object`; 2xx → true, 4xx → false.
+   * - JSON: send `response_format: json_object`; 2xx AND content that actually
+   *   parses as JSON → true. A 2xx alone is not evidence — a backend that does not
+   *   implement `response_format` simply ignores the field and answers normally.
    * - Native tools: send a trivial tool with `tool_choice: 'required'`; 2xx WITH a
-   *   `tool_calls` result → true, otherwise → false (a backend that silently ignores
-   *   tools returns no tool_calls and is treated as unsupported).
+   *   `tool_calls` result → true. A backend that answers fully but silently ignores
+   *   the tools returns no tool_calls and IS a real negative. A response truncated
+   *   by the token budget (`finish_reason: 'length'` without tool_calls) proves
+   *   nothing, so it is retried ONCE with
+   *   {@link NATIVE_TOOL_PROBE_MAX_TOKENS_RETRY}; a second truncation resolves to
+   *   `false`.
+   *
+   * This method always returns a definite boolean for a flag it probed. That is
+   * deliberate: an `undefined` result is not persisted by
+   * `detectAndPersistCapabilities`, and the orchestrator re-runs detection whenever
+   * a flag is undefined — so an endpoint that keeps truncating would trigger one
+   * extra upstream completion before every user prompt, ahead of the rate limiter
+   * and outside budget accounting.
    *
    * Throws on a transport error so callers can treat the connection as undetected
    * (and retry later) rather than persisting a wrong value.
    */
   async detectCapabilities(): Promise<{ jsonResponse?: boolean; nativeTools?: boolean }> {
+    const needsJson = this.connection.supportsJsonResponse === undefined;
+    const needsTools = this.connection.supportsNativeTools === undefined;
+    // Independent upstream calls, so run them together. This matters more since both
+    // probes gained a truncation retry: sequentially, a fresh connection can now cost
+    // FOUR round trips before the user's own completion starts — and lazy detection
+    // sits inline on the interactive prompt path.
+    const [jsonResponse, nativeTools] = await Promise.all([
+      needsJson ? this.probeJsonResponse() : Promise.resolve(undefined),
+      needsTools ? this.probeNativeTools() : Promise.resolve(undefined),
+    ]);
     const result: { jsonResponse?: boolean; nativeTools?: boolean } = {};
-    if (this.connection.supportsJsonResponse === undefined) {
+    if (needsJson) {
+      result.jsonResponse = jsonResponse;
+    }
+    if (needsTools) {
+      result.nativeTools = nativeTools;
+    }
+    return result;
+  }
+
+  /**
+   * Probe structured-JSON support.
+   *
+   * A 2xx alone is NOT evidence: a backend that does not implement
+   * `response_format` typically ignores the unknown field and answers normally, so
+   * trusting the status code alone persists `supportsJsonResponse: true` for an
+   * endpoint that never honours it — after which `chat()` sends `response_format`
+   * on every single call. Like the native-tool flag this is written once and never
+   * re-probed, so the wrong value is permanent.
+   *
+   * The response content must therefore actually parse as JSON. The budget matches
+   * the tool probe for the same reason: a reasoning model needs room to get past
+   * its thinking phase before it emits anything parseable — INCLUDING the tool
+   * probe's retry on truncation, which this probe needs just as badly.
+   *
+   * Measured against an OpenAI-compatible hosting endpoint on 2026-09-03 (this
+   * same probe, varying only `max_tokens`; the number is the completion tokens the
+   * model actually spent):
+   *
+   * | model                         | 256          | 1024   |
+   * |-------------------------------|--------------|--------|
+   * | Ministral-3-14B-Instruct-2512 | ✅ 9         | —      |
+   * | gpt-oss-120b                  | ✅ 88        | —      |
+   * | Qwen3.6-35B-A3B-FP8           | ✅ 202       | —      |
+   * | Mistral-Medium-3.5-128B       | ❌ truncated | ✅ 878 |
+   * | Qwen3.5-122B-A10B-FP8         | ❌ truncated | ✅ 365 |
+   *
+   * TWO of five need the retry — so without it the probe records "structured JSON
+   * unsupported" for an endpoint that demonstrably supports it.
+   *
+   * That false negative costs on two different paths, and the expensive one is the
+   * quiet one. `detectAndPersistCapabilities` WRITES what the probe returns, and a
+   * written flag is authoritative and never re-probed — so a fresh connection whose
+   * flags are unset is pinned to the wrong `false` for good, silently falling back
+   * to prompt-driven JSON. The loud path is the `ai.capabilityDriftCheck` boot
+   * warning (`supportsJsonResponse declared true but the endpoint reports false`),
+   * which merely reads as endpoint drift and sends the next reader hunting for one
+   * — which is how this was found. Note the warning fires only where that
+   * opt-in check is enabled, while the persisted flag is wrong everywhere.
+   *
+   * A COMPLETE answer that merely is not JSON stays a real negative: the endpoint
+   * ignored `response_format`, and a larger budget cannot change that.
+   */
+  protected async probeJsonResponse(): Promise<boolean> {
+    const budgets = [
+      OpenAiCompatibleProvider.NATIVE_TOOL_PROBE_MAX_TOKENS,
+      OpenAiCompatibleProvider.NATIVE_TOOL_PROBE_MAX_TOKENS_RETRY,
+    ];
+
+    for (const [attempt, maxTokens] of budgets.entries()) {
       const res = await this.probe({
-        max_tokens: 8,
+        max_tokens: maxTokens,
         messages: [{ content: 'Reply with the JSON object {"ok":true}.', role: 'user' }],
         response_format: { type: 'json_object' },
       });
-      result.jsonResponse = res.ok;
+      if (!res.ok) {
+        return false;
+      }
+      const choice = res.json?.choices?.[0];
+      const content = choice?.message?.content;
+      const truncated = choice?.finish_reason === 'length';
+      if (typeof content === 'string' && content.trim()) {
+        try {
+          JSON.parse(content);
+          return true;
+        } catch {
+          // A parse failure is only a REAL negative when the model finished on its
+          // own terms. Truncation is the other reason JSON does not parse, and it
+          // arrives in two shapes depending on how the model emits: a reasoning
+          // model buffers behind its thinking phase and returns EMPTY content, while
+          // one that streams directly returns a partial body like `{"ok":tr`. Both
+          // are `finish_reason: 'length'`, and classifying the partial one here
+          // instead of retrying reaches the exact false negative this ladder exists
+          // to remove — just through the other door.
+          if (!truncated) {
+            this.logger.warn(
+              `JSON-response probe for model "${this.connection.model}" returned non-JSON content — ` +
+                'recording structured JSON as unsupported',
+            );
+            return false;
+          }
+        }
+      } else if (!truncated) {
+        // Empty content that finished on its own terms is a real negative.
+        return false;
+      }
+
+      const isLastAttempt = attempt === budgets.length - 1;
+      this.logger.warn(
+        `JSON-response probe for model "${this.connection.model}" was truncated (finish_reason=length) at ` +
+          `max_tokens=${maxTokens}` +
+          (isLastAttempt
+            ? ' on the final attempt — recording structured JSON as unsupported; a model that cannot emit a ' +
+              'trivial JSON object within a usable output budget is better served by the prompt-driven fallback'
+            : ' — retrying once with a larger budget'),
+      );
     }
-    if (this.connection.supportsNativeTools === undefined) {
+
+    return false;
+  }
+
+  /**
+   * Run the native-tool probe, escalating the output budget once on truncation.
+   * Extracted so the retry policy is overridable and testable on its own.
+   */
+  protected async probeNativeTools(): Promise<boolean> {
+    const budgets = [
+      OpenAiCompatibleProvider.NATIVE_TOOL_PROBE_MAX_TOKENS,
+      OpenAiCompatibleProvider.NATIVE_TOOL_PROBE_MAX_TOKENS_RETRY,
+    ];
+
+    for (const [attempt, maxTokens] of budgets.entries()) {
       const res = await this.probe({
-        max_tokens: 8,
+        max_tokens: maxTokens,
         messages: [{ content: 'Call the ping tool.', role: 'user' }],
         tool_choice: 'required',
         tools: [
@@ -255,9 +587,29 @@ export class OpenAiCompatibleProvider implements ILlmProvider {
           },
         ],
       });
-      result.nativeTools = res.ok && !!res.json?.choices?.[0]?.message?.tool_calls?.length;
+      const choice = res.json?.choices?.[0];
+
+      if (res.ok && choice?.message?.tool_calls?.length) {
+        return true;
+      }
+      // A non-2xx, or a complete answer without tool calls, is a REAL negative —
+      // retrying with a bigger budget would not change it.
+      if (!res.ok || choice?.finish_reason !== 'length') {
+        return false;
+      }
+
+      const isLastAttempt = attempt === budgets.length - 1;
+      this.logger.warn(
+        `Native-tool probe for model "${this.connection.model}" was truncated (finish_reason=length) at ` +
+          `max_tokens=${maxTokens}` +
+          (isLastAttempt
+            ? ' on the final attempt — recording native tools as unsupported; the model does not reach a tool ' +
+              'call within a usable output budget, so emulated tool calling is the correct fallback'
+            : ' — retrying once with a larger budget'),
+      );
     }
-    return result;
+
+    return false;
   }
 
   /**
@@ -306,7 +658,18 @@ export class OpenAiCompatibleProvider implements ILlmProvider {
         128_000,
         ['gpt-4o', 'gpt-4.1', 'gpt-4-turbo', 'o1', 'o3', 'gpt-oss', 'mistral-large', 'mistral-small3', 'command-r'],
       ],
-      [131_072, ['qwen2.5', 'qwen3', 'llama-3.1', 'llama3.1', 'llama-3.3', 'llama3.3']],
+      // Two DIFFERENT gaps, both closed here (measured 2026-07-25 against an
+      // OpenAI-compatible hosting endpoint; both models verified to accept >=163k
+      // prompt tokens):
+      //   - `mistral-medium` DID match the generic `mistral` -> 32768 entry below,
+      //     capping a 256k model at an eighth of its window. It must therefore be
+      //     matched before it -- this bucket is evaluated first.
+      //   - `ministral` matched NOTHING at all: "ministral" does not contain the
+      //     substring "mistral" (m-i-n-i-s-t-r-a-l), so it fell through the whole
+      //     table to the conservative 8192 default.
+      // Pinned one power of two below the verified capacity so the orchestrator
+      // trims before the endpoint rejects.
+      [131_072, ['qwen2.5', 'qwen3', 'llama-3.1', 'llama3.1', 'llama-3.3', 'llama3.3', 'ministral', 'mistral-medium']],
       [65_536, ['mixtral']],
       [32_768, ['qwen2', 'mistral', 'gemma2', 'gemma-2']],
       [16_385, ['gpt-3.5']],