@carlonicora/nestjs-neo4jsonapi 2.3.0 → 2.4.0

package/dist/core/llm/services/llm.service.js

@@ -65,6 +65,7 @@ const llm_cache_service_1 = require("./llm-cache.service");
 const model_service_1 = require("../../llm/services/model.service");
 const schema_utils_1 = require("../../llm/utils/schema.utils");
 const mock_from_zod_1 = require("../utils/mock-from-zod");
+const repair_truncated_json_1 = require("../utils/repair-truncated-json");
 const llm_raw_response_1 = require("../common/llm-raw-response");
 const llm_call_dumper_service_1 = require("./llm-call-dumper.service");
 const openrouter_fetch_1 = require("./openrouter-fetch");
@@ -86,6 +87,40 @@ const DEFAULT_REQUEST_WATCHDOG_MS = 30_000;
 const DEFAULT_REQUEST_DEADLINE_ATTEMPTS = 3;
 /** Grace on top of the budgeted attempts, covering backoff between retries. */
 const DEADLINE_SLACK_MS = 15_000;
+/**
+ * How long a FAILED stream may wait for its `usage` promise before giving up on
+ * it. Unlike `call()`, a stream's `result` promise runs under no outer deadline,
+ * so an unbounded await in its catch would replace a settled rejection with a
+ * caller that hangs forever. Two seconds is generous for a promise the SDK has
+ * normally already settled, and irrelevant to the happy path.
+ */
+const USAGE_SETTLE_TIMEOUT_MS = 2_000;
+/**
+ * Waits before each EXTRA attempt of the transient-network retry, in ms — two
+ * extras, so three attempts in total.
+ *
+ * Deliberately LONG. LangChain's `AsyncCaller` already retries roughly six times
+ * FAST underneath every `.invoke(...)`, so by the time a failure surfaces here
+ * the quick retries are spent and the fault has lasted seconds, not
+ * milliseconds — a DNS outage, a provider brown-out, a saturated egress NAT.
+ * Retrying fast again would burn both extra attempts inside the same bad second
+ * and change nothing. (Evidence: a crashed worker run with 28 `ENOTFOUND` DNS
+ * failures under load, every one of them already through LangChain's fast
+ * retries.)
+ */
+const TRANSIENT_RETRY_WAITS_MS = [5_000, 15_000];
+/**
+ * Network-level error codes that say "the request never reached a working
+ * provider" — the class of failure another attempt can actually fix.
+ */
+const TRANSIENT_ERROR_CODES = new Set([
+    "ENOTFOUND",
+    "EAI_AGAIN",
+    "ECONNRESET",
+    "ETIMEDOUT",
+    "ECONNREFUSED",
+    "EPIPE",
+]);
 /**
  * True for the abort a request timeout raises, whichever layer raised it — the
  * OpenAI SDK's `APIConnectionTimeoutError`, undici's `TimeoutError`/
@@ -245,6 +280,37 @@ let LLMService = LLMService_1 = class LLMService {
     attemptTimeoutMs(explicit) {
         return explicit ?? this.config.get("ai")?.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
     }
+    /**
+     * Reads a stream's `usage` promise WITHOUT ever hanging on it.
+     *
+     * Used only from the streaming error paths. `streamObject`/`streamText`
+     * usually settle `usage` even when `object`/`text` reject — a schema-invalid
+     * or aborted generation is still billed — so it is worth awaiting. But the
+     * `result` promise those catches belong to has no outer deadline (unlike
+     * `call()`, which runs under `runBounded`), so if the SDK ever rejected the
+     * content promise while leaving `usage` pending, an unbounded await would turn
+     * a prompt rejection into a caller that waits forever. Rejection is handled by
+     * `.catch`; PENDENCY is handled by the race. The loser's timer is cleared, so
+     * a settled call leaves no timer behind.
+     *
+     * @returns the usage object, or undefined if it rejected or did not settle in time
+     */
+    async readUsageBounded(usage, timeoutMs = USAGE_SETTLE_TIMEOUT_MS) {
+        let timer;
+        try {
+            return await Promise.race([
+                Promise.resolve(usage).catch(() => undefined),
+                new Promise((resolve) => {
+                    timer = setTimeout(() => resolve(undefined), timeoutMs);
+                    timer.unref?.();
+                }),
+            ]);
+        }
+        finally {
+            if (timer)
+                clearTimeout(timer);
+        }
+    }
     /**
      * Runs one provider call under a WATCHDOG and an ABSOLUTE DEADLINE.
      *
@@ -305,6 +371,102 @@ let LLMService = LLMService_1 = class LLMService {
                 clearTimeout(deadline);
         }
     }
+    /**
+     * True for a failure that means the request never reached a working provider,
+     * and is therefore worth another attempt: a DNS/socket-level error code, an
+     * HTTP 429 or 5xx, or the message text either of those arrives as.
+     *
+     * All three are checked because the same failure wears different clothes per
+     * transport: undici puts the code on `error.cause.code` behind a bare
+     * `TypeError: fetch failed`, the OpenAI SDK puts the status on the error and
+     * the code in the message, and LangChain re-wraps both in a plain `Error`.
+     *
+     * Deliberately NOT transient: a stall that burned its whole deadline
+     * ({@link LLMTimeoutError}), a refusal (402/403), a malformed request (400) or
+     * a parse failure. Those either already had their retry (`call()` re-issues a
+     * timed-out attempt once, escalating the OpenRouter pin) or will fail
+     * identically forever. The 429 vocabulary matches the one
+     * `VisionLLMService.isRateLimitError` retries on, so the two agree on what a
+     * rate limit looks like.
+     */
+    isTransientNetworkError(err) {
+        if (err === undefined || err === null)
+            return false;
+        const candidate = err;
+        const cause = candidate.cause;
+        const codes = [candidate.code, cause?.code];
+        if (codes.some((code) => typeof code === "string" && TRANSIENT_ERROR_CODES.has(code)))
+            return true;
+        const statuses = [candidate.status, candidate.statusCode, candidate.response?.status, cause?.status];
+        if (statuses.some((status) => typeof status === "number" && (status === 429 || (status >= 500 && status <= 599))))
+            return true;
+        const own = err instanceof Error ? err.message : String(err);
+        const causeMessage = cause instanceof Error ? cause.message : "";
+        const haystack = `${own} ${causeMessage}`.toLowerCase();
+        for (const code of TRANSIENT_ERROR_CODES)
+            if (haystack.includes(code.toLowerCase()))
+                return true;
+        return (haystack.includes("socket hang up") ||
+            haystack.includes("fetch failed") ||
+            haystack.includes("network error") ||
+            haystack.includes("econnaborted") ||
+            // 429 / rate limiting — matched as a whole word so a token count never
+            // reads as a status code.
+            /\b429\b/.test(haystack) ||
+            haystack.includes("rate limit") ||
+            haystack.includes("resource exhausted") ||
+            haystack.includes("too many requests") ||
+            // 5xx. Only the codes providers actually emit, again whole-word: a blanket
+            // /5\d\d/ would retry "context length 512 exceeded" forever.
+            /\b(500|502|503|504|529)\b/.test(haystack) ||
+            haystack.includes("internal server error") ||
+            haystack.includes("bad gateway") ||
+            haystack.includes("service unavailable") ||
+            haystack.includes("gateway timeout") ||
+            haystack.includes("overloaded"));
+    }
+    /**
+     * Sleeps the jittered backoff for one transient retry and says so in the log.
+     * ±20% jitter so a fleet of workers knocked out by the same DNS blip does not
+     * come back in lockstep and knock it out again.
+     */
+    async waitBeforeTransientRetry(label, attempt, error) {
+        const base = TRANSIENT_RETRY_WAITS_MS[attempt];
+        const waitMs = Math.round(base * (0.8 + Math.random() * 0.4));
+        this.logger.warn(`[${label}] transient network failure on attempt ${attempt + 1}/${TRANSIENT_RETRY_WAITS_MS.length + 1} — ` +
+            `retrying in ${Math.round(waitMs / 1000)}s: ${error instanceof Error ? error.message : String(error)}`);
+        await new Promise((resolve) => {
+            const timer = setTimeout(resolve, waitMs);
+            timer.unref?.();
+        });
+    }
+    /**
+     * Runs a provider call under {@link runBounded}, retrying it when — and only
+     * when — it failed for a transient network reason
+     * ({@link isTransientNetworkError}). Two extra attempts, with the long
+     * jittered waits {@link TRANSIENT_RETRY_WAITS_MS} explains.
+     *
+     * Owns a FRESH AbortController per attempt: a controller that has already
+     * aborted stays aborted forever, so re-using one would make every retry abort
+     * before it sent a byte. `work` therefore receives the signal rather than
+     * capturing one from the caller.
+     */
+    async runWithTransientRetry(label, attemptTimeoutMs, work) {
+        for (let attempt = 0;; attempt++) {
+            const controller = new AbortController();
+            try {
+                return await this.runBounded(label, attemptTimeoutMs, controller, () => work(controller.signal));
+            }
+            catch (error) {
+                if (attempt >= TRANSIENT_RETRY_WAITS_MS.length || !this.isTransientNetworkError(error))
+                    throw error;
+                // The attempt is over; release whatever socket it may still hold before
+                // waiting out the backoff.
+                controller.abort();
+                await this.waitBeforeTransientRetry(label, attempt, error);
+            }
+        }
+    }
     /**
      * Records token usage for cost/observability attribution. Never throws —
      * a persistence failure logs a warning and the LLM call continues, so
@@ -315,6 +477,18 @@ let LLMService = LLMService_1 = class LLMService {
      * is the ONLY token-usage write inside the package; any future package caller
      * MUST use the same token rather than injecting `TokenUsageService` directly
      * (see the token's docblock for why).
+     *
+     * ZERO-TOKEN SUCCESS RULE: a call that SUCCEEDED but reported no usage at all
+     * (the provider omitted `usage_metadata`) is still recorded — the call really
+     * happened and must stay visible — but with `applyMinimum: false`, so it
+     * costs 0 credits instead of being floored to `minCreditsPerRecord`. Flooring
+     * exists to stop sub-cent REAL usage rounding to nothing, not to invent a
+     * charge for tokens nobody measured. A success carrying real counts keeps the
+     * floor exactly as before.
+     *
+     * This is deliberately NOT the same rule as the zero-token FAILURE rule in
+     * {@link persistUsageOnFailure}, which writes nothing at all: there the
+     * provider was never reached, so there is no call to make visible.
      */
     async persistUsage(params, tokens) {
         // Attribution is opt-in: the caller decides which entity this usage is
@@ -322,6 +496,7 @@ let LLMService = LLMService_1 = class LLMService {
         // so we skip — the package stays domain-agnostic.
         if (!params.relationshipId || !params.relationshipType)
             return;
+        const measured = tokens.input + tokens.output + (tokens.cached ?? 0) > 0;
         try {
             await (this.tokenUsageRecorder ?? this.tokenUsageService).recordTokenUsage({
                 tokens,
@@ -329,12 +504,35 @@ let LLMService = LLMService_1 = class LLMService {
                 relationshipId: params.relationshipId,
                 relationshipType: params.relationshipType,
                 modelWeight: params.modelWeight,
+                ...(measured ? {} : { applyMinimum: false }),
             });
         }
         catch (err) {
             this.logger.warn(`TokenUsage persistence failed — continuing: ${String(err)}`);
         }
     }
+    /**
+     * Records what a FAILED call already burned. A failure is not a free call:
+     * the provider bills every round it served, so a tool loop that dies on its
+     * final structured invocation has already been charged for six figures of
+     * input tokens. Billing only successful calls understates real spend.
+     *
+     * ZERO-TOKEN RULE: a failure that consumed nothing (the provider was never
+     * reached — input validation, an unreachable host, an immediate abort) is NOT
+     * recorded. `recordTokenUsage` floors every record at `minCreditsPerRecord`,
+     * so writing a 0/0 row would invent a charge for tokens nobody spent; the
+     * floor exists to stop sub-cent REAL usage rounding to nothing, not to price
+     * a call that never happened. Such failures remain fully visible through the
+     * dump session, which closes with `finalStatus: "error"`.
+     *
+     * Never throws (delegates to {@link persistUsage}), so it can sit in a catch
+     * block without masking the original error.
+     */
+    async persistUsageOnFailure(params, tokens) {
+        if (tokens.input + tokens.output + (tokens.cached ?? 0) === 0)
+            return;
+        await this.persistUsage(params, tokens);
+    }
     /**
      * Converts AgentMessageType to LangChain BaseMessage
      */
@@ -606,14 +804,15 @@ let LLMService = LLMService_1 = class LLMService {
         // reports it while it is still open, and the deadline guarantees this
         // promise settles even if the provider never answers.
         const attemptTimeoutMs = this.attemptTimeoutMs(params.timeout);
-        const controller = new AbortController();
         const label = `${params.metadata?.nodeName ?? "llm.call"}:${aiConfig.model}`;
         try {
-            const result = await this.runBounded(label, attemptTimeoutMs, controller, () => this._invokeOriginal(params, session, (i, o, c) => {
+            // The abort signal now comes from the retry wrapper, which owns a fresh
+            // controller per attempt — a reused one would abort every retry instantly.
+            const result = await this.runWithTransientRetry(label, attemptTimeoutMs, (signal) => this._invokeOriginal(params, session, (i, o, c) => {
                 totalInput += i;
                 totalOutput += o;
                 totalCached += c;
-            }, (kind) => parseFallbacks.push(kind), (w) => warnings.push(w), { attemptTimeoutMs, signal: controller.signal, label }));
+            }, (kind) => parseFallbacks.push(kind), (w) => warnings.push(w), { attemptTimeoutMs, signal, label }));
             session.close({
                 finalStatus: "success",
                 totalTokens: { input: totalInput, output: totalOutput, cached: totalCached },
@@ -643,8 +842,26 @@ let LLMService = LLMService_1 = class LLMService {
                 warnings,
                 parseFallbacks,
             });
+            // The provider charged for everything spent up to the failure — a timeout
+            // mid tool-loop can burn six figures of input tokens, all of them already
+            // reported through `addTokens` by the time we get here. Never throws, so
+            // this cannot mask the original error.
+            await this.persistUsageOnFailure({
+                tokenUsageType: params.tokenUsageType,
+                relationshipId: params.relationshipId,
+                relationshipType: params.relationshipType,
+                modelWeight,
+            }, { input: totalInput, output: totalOutput, cached: totalCached });
             console.error("[LLMService] Error calling LLM:", error);
-            throw new Error(`LLM service error: ${message}`);
+            // The message text is load-bearing — callers match on "LLM service error:"
+            // — so it stays byte-for-byte identical, and the original error rides along
+            // as `cause`. Without it every network diagnosis stopped at this wrapper:
+            // the ENOTFOUND / status / stack that explained the failure was thrown
+            // away here. Assigned rather than passed as `new Error(msg, { cause })`
+            // because this package targets ES2021, whose Error takes one argument.
+            const wrapped = new Error(`LLM service error: ${message}`);
+            wrapped.cause = error;
+            throw wrapped;
         }
     }
     async _invokeOriginal(params, session, addTokens, addParseFallback, addWarning, 
@@ -785,12 +1002,21 @@ let LLMService = LLMService_1 = class LLMService {
                     `in=${iterationUsage?.input_tokens ?? 0} out=${iterationUsage?.output_tokens ?? 0} ` +
                     `reasoning=${iterationUsage?.output_token_details?.reasoning ?? 0} ` +
                     `tools=[${requestedTools.join(",") || "none"}]`);
-                // Track token usage
+                // Track token usage. Reported to the caller AS IT IS SPENT, not at the
+                // end: this iteration is already billed by the provider, and a later
+                // failure (a timeout on the final structured call, an unparseable
+                // answer) must not make those tokens disappear from `call()`'s totals.
+                // The final return therefore adds only ITS OWN delta — see `addTokens`
+                // at the return sites below.
                 const responseUsage = toolResponse.usage_metadata;
                 if (responseUsage) {
-                    totalInputTokens += responseUsage.input_tokens ?? 0;
-                    totalOutputTokens += responseUsage.output_tokens ?? 0;
-                    totalCachedTokens += responseUsage.input_token_details?.cache_read ?? 0;
+                    const iterationInput = responseUsage.input_tokens ?? 0;
+                    const iterationOutput = responseUsage.output_tokens ?? 0;
+                    const iterationCached = responseUsage.input_token_details?.cache_read ?? 0;
+                    totalInputTokens += iterationInput;
+                    totalOutputTokens += iterationOutput;
+                    totalCachedTokens += iterationCached;
+                    addTokens(iterationInput, iterationOutput, iterationCached);
                 }
                 // Check for tool calls
                 const toolCalls = toolResponse.tool_calls ?? [];
@@ -998,9 +1224,15 @@ let LLMService = LLMService_1 = class LLMService {
             },
             finishReason: raw?.response_metadata?.finish_reason,
         });
-        const input = totalInputTokens + (raw?.usage_metadata?.input_tokens ?? 0);
-        const output = totalOutputTokens + (raw?.usage_metadata?.output_tokens ?? 0);
-        const cached = totalCachedTokens + (raw?.usage_metadata?.input_token_details?.cache_read ?? 0);
+        // The final structured response's OWN usage. Reported separately from the
+        // totals because the tool loop has already handed its share to `addTokens`;
+        // re-reporting the sum would bill every tool iteration twice.
+        const finalInput = raw?.usage_metadata?.input_tokens ?? 0;
+        const finalOutput = raw?.usage_metadata?.output_tokens ?? 0;
+        const finalCached = raw?.usage_metadata?.input_token_details?.cache_read ?? 0;
+        const input = totalInputTokens + finalInput;
+        const output = totalOutputTokens + finalOutput;
+        const cached = totalCachedTokens + finalCached;
         /**
          * One summary line per completed call, emitted at EVERY return site — the
          * degraded runs are the ones worth diagnosing, so a fallback-parsed call must
@@ -1047,7 +1279,7 @@ let LLMService = LLMService_1 = class LLMService {
                     const validated = params.outputSchema.parse(normaliseStrictOutput(toolCallArgs));
                     console.warn("[LLMService] Fallback tool_calls parsing succeeded");
                     logCallSummary("fallback:tool_calls");
-                    addTokens(input, output, cached);
+                    addTokens(finalInput, finalOutput, finalCached);
                     return {
                         ...validated,
                         tokenUsage: { input, output, cached },
@@ -1084,7 +1316,7 @@ let LLMService = LLMService_1 = class LLMService {
                         const validated = params.outputSchema.parse(cleanedArgs);
                         console.warn("[LLMService] Lenient tool_calls parsing succeeded");
                         logCallSummary("fallback:lenient");
-                        addTokens(input, output, cached);
+                        addTokens(finalInput, finalOutput, finalCached);
                         return {
                             ...validated,
                             tokenUsage: { input, output, cached },
@@ -1103,13 +1335,41 @@ let LLMService = LLMService_1 = class LLMService {
                 const validated = params.outputSchema.parse(normaliseStrictOutput(manualParse));
                 console.warn("[LLMService] Fallback parsing succeeded");
                 logCallSummary("fallback:raw");
-                addTokens(input, output, cached);
+                addTokens(finalInput, finalOutput, finalCached);
                 return {
                     ...validated,
                     tokenUsage: { input, output, cached },
                 };
             }
             catch (fallbackError) {
+                // Last rung. A MAX_TOKENS truncation (`finish_reason: "length"`) stops
+                // the payload mid-value, so every rung above — all of which need the
+                // whole document to parse — rejects a response whose completed elements
+                // were perfectly good. Trim to the last complete value, close the open
+                // containers, and validate exactly as the raw rung does.
+                const repaired = (0, repair_truncated_json_1.repairTruncatedJson)(rawContent);
+                if (repaired !== null) {
+                    try {
+                        const validated = params.outputSchema.parse(normaliseStrictOutput(JSON.parse(repaired)));
+                        this.logger.warn(`[${label}] parseFallback: "truncation-repair" — recovered a truncated payload ` +
+                            `(finishReason=${finishReason}, ${rawContent.length}→${repaired.length} chars)`);
+                        logCallSummary("fallback:truncation-repair");
+                        addTokens(finalInput, finalOutput, finalCached);
+                        return {
+                            ...validated,
+                            tokenUsage: { input, output, cached },
+                        };
+                    }
+                    catch {
+                        // Repaired text still does not satisfy the schema — fall through to
+                        // the diagnostic below, which reports the ORIGINAL failure.
+                    }
+                }
+                // Every salvage attempt failed, so this call is about to throw — but the
+                // unparseable answer was generated and billed like any other. Report it
+                // before unwinding, so `call()`'s catch can record it. No return site
+                // follows this one, so nothing is counted twice.
+                addTokens(finalInput, finalOutput, finalCached);
                 throw new Error(`LLM failed to return structured output. ` +
                     `Finish reason: ${finishReason}. ` +
                     `Raw content preview: ${rawContent.substring(0, 200)}...` +
@@ -1138,7 +1398,7 @@ let LLMService = LLMService_1 = class LLMService {
             return salvageParse();
         }
         logCallSummary("clean");
-        addTokens(input, output, cached);
+        addTokens(finalInput, finalOutput, finalCached);
         return {
             ...result,
             tokenUsage: {
@@ -1261,77 +1521,132 @@ let LLMService = LLMService_1 = class LLMService {
         // final-structured response, just streamed instead of awaited atomically.
         // Avoids widening the dumper's union type for a single call site.
         session.startIteration("final-structured", []);
-        // Abort the stream if the provider stalls, so a hung connection can't pin
-        // the session open indefinitely. The AbortError surfaces as a rejection on
-        // the awaited promises below (caught + logged).
-        const controller = new AbortController();
-        const timeoutId = setTimeout(() => controller.abort(), this.attemptTimeoutMs(params.timeout));
+        const attemptTimeoutMs = this.attemptTimeoutMs(params.timeout);
+        const label = `${params.metadata?.nodeName ?? "llm.streamCall"}:${aiConfig.model}`;
+        // One attempt = one abort controller + one whole-stream timeout, so a hung
+        // connection can't pin the session open indefinitely. The bound is NOT
+        // `runWithTransientRetry`'s per-attempt deadline: a stream outlives its
+        // first byte, so its timer must stay armed until the stream finishes.
+        //
         // Schema cast: `streamObject`'s typing is a conditional union over the
         // output mode (`object` / `enum` / `array` / `no-schema`). Our T is always
         // a Zod object schema; the runtime call is correct.
-        const streamResult = streamObject({
-            model,
-            schema: params.outputSchema,
-            system,
-            prompt: finalInstructions,
-            temperature: params.temperature,
-            maxOutputTokens: params.maxTokens,
-            maxRetries: 2,
-            abortSignal: controller.signal,
-        });
+        const startAttempt = () => {
+            const controller = new AbortController();
+            const timeoutId = setTimeout(() => controller.abort(), attemptTimeoutMs);
+            timeoutId.unref?.();
+            return {
+                controller,
+                timeoutId,
+                handle: streamObject({
+                    model,
+                    schema: params.outputSchema,
+                    system,
+                    prompt: finalInstructions,
+                    temperature: params.temperature,
+                    maxOutputTokens: params.maxTokens,
+                    maxRetries: 2,
+                    abortSignal: controller.signal,
+                }),
+            };
+        };
+        let attempt = startAttempt();
+        // Set the instant the caller starts reading the stream. Past that point a
+        // restart would REPLAY output the consumer has already seen, so the
+        // transient retry below only ever fires while the consumer is still idle —
+        // which is precisely the window in which a DNS/connect failure lands.
+        let consumerStarted = false;
         // Build the result Promise that closes the session once the stream finishes.
         // This is awaitable independently of consuming the streams — `streamObject`
         // internally tees the source, so consuming `textStream` (or not) doesn't
         // affect `result` resolution.
         const resultPromise = (async () => {
-            try {
-                const finalObject = (await streamResult.object);
-                const usage = await streamResult.usage;
-                const input = usage?.inputTokens ?? 0;
-                const output = usage?.outputTokens ?? 0;
-                const cached = usage?.inputTokenDetails?.cacheReadTokens ?? 0;
-                session.recordResponse({
-                    content: JSON.stringify(finalObject),
-                    tokenUsage: { input, output },
-                    finishReason: String(await streamResult.finishReason),
-                });
-                session.close({
-                    finalStatus: "success",
-                    totalTokens: { input, output, cached },
-                    warnings: [],
-                    parseFallbacks: [],
-                });
-                await this.persistUsage({
-                    tokenUsageType: params.tokenUsageType,
-                    relationshipId: params.relationshipId,
-                    relationshipType: params.relationshipType,
-                    modelWeight,
-                }, { input, output, cached });
-                return { ...finalObject, tokenUsage: { input, output }, modelWeight };
-            }
-            catch (error) {
-                const message = error instanceof Error ? error.message : String(error);
-                const stack = error instanceof Error ? (error.stack ?? "").split("\n").slice(0, 10).join("\n") : undefined;
-                session.close({
-                    finalStatus: "error",
-                    errorMessage: message,
-                    errorStack: stack,
-                    totalTokens: { input: 0, output: 0 },
-                    warnings: [],
-                    parseFallbacks: [],
-                });
-                console.error("[LLMService.streamCall] Error:", error);
-                throw new Error(`LLM streamCall error: ${message}`);
-            }
-            finally {
-                clearTimeout(timeoutId);
+            for (let retry = 0;; retry++) {
+                try {
+                    const finalObject = (await attempt.handle.object);
+                    const usage = await attempt.handle.usage;
+                    const input = usage?.inputTokens ?? 0;
+                    const output = usage?.outputTokens ?? 0;
+                    const cached = usage?.inputTokenDetails?.cacheReadTokens ?? 0;
+                    session.recordResponse({
+                        content: JSON.stringify(finalObject),
+                        tokenUsage: { input, output },
+                        finishReason: String(await attempt.handle.finishReason),
+                    });
+                    session.close({
+                        finalStatus: "success",
+                        totalTokens: { input, output, cached },
+                        warnings: [],
+                        parseFallbacks: [],
+                    });
+                    clearTimeout(attempt.timeoutId);
+                    await this.persistUsage({
+                        tokenUsageType: params.tokenUsageType,
+                        relationshipId: params.relationshipId,
+                        relationshipType: params.relationshipType,
+                        modelWeight,
+                    }, { input, output, cached });
+                    return { ...finalObject, tokenUsage: { input, output }, modelWeight };
+                }
+                catch (error) {
+                    // Restartable only while nothing has been delivered — see
+                    // `consumerStarted`.
+                    if (!consumerStarted && retry < TRANSIENT_RETRY_WAITS_MS.length && this.isTransientNetworkError(error)) {
+                        clearTimeout(attempt.timeoutId);
+                        attempt.controller.abort();
+                        await this.waitBeforeTransientRetry(label, retry, error);
+                        attempt = startAttempt();
+                        continue;
+                    }
+                    clearTimeout(attempt.timeoutId);
+                    const message = error instanceof Error ? error.message : String(error);
+                    const stack = error instanceof Error ? (error.stack ?? "").split("\n").slice(0, 10).join("\n") : undefined;
+                    // `streamObject` may have settled `usage` even though `object`
+                    // rejected (a schema-invalid payload is still a billed generation).
+                    // Read it defensively so the session and the ledger report the real
+                    // figures instead of the hard-coded 0/0 this used to close with.
+                    // Bounded: this promise has no outer deadline, so a `usage` that never
+                    // settles must not stop `result` from rejecting (see readUsageBounded).
+                    const usage = await this.readUsageBounded(attempt.handle.usage);
+                    const input = usage?.inputTokens ?? 0;
+                    const output = usage?.outputTokens ?? 0;
+                    const cached = usage?.inputTokenDetails?.cacheReadTokens ?? 0;
+                    session.close({
+                        finalStatus: "error",
+                        errorMessage: message,
+                        errorStack: stack,
+                        totalTokens: { input, output, cached },
+                        warnings: [],
+                        parseFallbacks: [],
+                    });
+                    await this.persistUsageOnFailure({
+                        tokenUsageType: params.tokenUsageType,
+                        relationshipId: params.relationshipId,
+                        relationshipType: params.relationshipType,
+                        modelWeight,
+                    }, { input, output, cached });
+                    console.error("[LLMService.streamCall] Error:", error);
+                    const wrapped = new Error(`LLM streamCall error: ${message}`);
+                    wrapped.cause = error;
+                    throw wrapped;
+                }
             }
         })();
         // Surface (don't swallow) a rejected result even when the caller never
         // awaits `result` — e.g. on abort/timeout or an unreachable provider.
         resultPromise.catch((err) => this.logger.warn(`streamCall result rejected: ${String(err)}`));
+        // Wrapped rather than handed over directly so that (a) the consumer's first
+        // pull marks the stream unrestartable, and (b) the `partialObjectStream`
+        // getter — one of three getters that lock the source on first access — is
+        // touched only if the consumer actually reads, and on whichever attempt
+        // finally connected.
+        async function* partialObjects() {
+            consumerStarted = true;
+            for await (const partial of attempt.handle.partialObjectStream)
+                yield partial;
+        }
         return {
-            partialObjectStream: streamResult.partialObjectStream,
+            partialObjectStream: partialObjects(),
             result: resultPromise,
         };
     }
@@ -1407,62 +1722,101 @@ let LLMService = LLMService_1 = class LLMService {
         });
         const model = provider.chatModel(aiConfig.model);
         session.startIteration("final-structured", []);
-        // Abort the stream if the provider stalls (see streamCall). The AbortError
-        // surfaces as a rejection on the awaited promises below (caught + logged).
-        const controller = new AbortController();
-        const timeoutId = setTimeout(() => controller.abort(), this.attemptTimeoutMs(params.timeout));
-        const streamResult = streamText({
-            model,
-            system,
-            prompt: params.prompt,
-            temperature: params.temperature,
-            maxOutputTokens: params.maxTokens,
-            maxRetries: 2,
-            abortSignal: controller.signal,
-        });
+        const attemptTimeoutMs = this.attemptTimeoutMs(params.timeout);
+        const label = `${params.metadata?.nodeName ?? "llm.streamText"}:${aiConfig.model}`;
+        // One attempt = one abort controller + one whole-stream timeout (see
+        // streamCall). The AbortError surfaces as a rejection on the awaited
+        // promises below (caught + logged).
+        const startAttempt = () => {
+            const controller = new AbortController();
+            const timeoutId = setTimeout(() => controller.abort(), attemptTimeoutMs);
+            timeoutId.unref?.();
+            return {
+                controller,
+                timeoutId,
+                handle: streamText({
+                    model,
+                    system,
+                    prompt: params.prompt,
+                    temperature: params.temperature,
+                    maxOutputTokens: params.maxTokens,
+                    maxRetries: 2,
+                    abortSignal: controller.signal,
+                }),
+            };
+        };
+        let attempt = startAttempt();
+        // See streamCall: a stream may only be restarted while the consumer has not
+        // yet read a single part.
+        let consumerStarted = false;
         const resultPromise = (async () => {
-            try {
-                const text = await streamResult.text;
-                const reasoning = (await streamResult.reasoningText) ?? "";
-                const usage = await streamResult.usage;
-                const input = usage?.inputTokens ?? 0;
-                const output = usage?.outputTokens ?? 0;
-                const cached = usage?.inputTokenDetails?.cacheReadTokens ?? 0;
-                session.recordResponse({
-                    content: text,
-                    tokenUsage: { input, output },
-                    finishReason: String(await streamResult.finishReason),
-                });
-                session.close({
-                    finalStatus: "success",
-                    totalTokens: { input, output, cached },
-                    warnings: [],
-                    parseFallbacks: [],
-                });
-                await this.persistUsage({
-                    tokenUsageType: params.tokenUsageType,
-                    relationshipId: params.relationshipId,
-                    relationshipType: params.relationshipType,
-                    modelWeight,
-                }, { input, output, cached });
-                return { text, reasoning, tokenUsage: { input, output }, modelWeight };
-            }
-            catch (error) {
-                const message = error instanceof Error ? error.message : String(error);
-                const stack = error instanceof Error ? (error.stack ?? "").split("\n").slice(0, 10).join("\n") : undefined;
-                session.close({
-                    finalStatus: "error",
-                    errorMessage: message,
-                    errorStack: stack,
-                    totalTokens: { input: 0, output: 0 },
-                    warnings: [],
-                    parseFallbacks: [],
-                });
-                console.error("[LLMService.streamText] Error:", error);
-                throw new Error(`LLM streamText error: ${message}`);
-            }
-            finally {
-                clearTimeout(timeoutId);
+            for (let retry = 0;; retry++) {
+                try {
+                    const text = await attempt.handle.text;
+                    const reasoning = (await attempt.handle.reasoningText) ?? "";
+                    const usage = await attempt.handle.usage;
+                    const input = usage?.inputTokens ?? 0;
+                    const output = usage?.outputTokens ?? 0;
+                    const cached = usage?.inputTokenDetails?.cacheReadTokens ?? 0;
+                    session.recordResponse({
+                        content: text,
+                        tokenUsage: { input, output },
+                        finishReason: String(await attempt.handle.finishReason),
+                    });
+                    session.close({
+                        finalStatus: "success",
+                        totalTokens: { input, output, cached },
+                        warnings: [],
+                        parseFallbacks: [],
+                    });
+                    clearTimeout(attempt.timeoutId);
+                    await this.persistUsage({
+                        tokenUsageType: params.tokenUsageType,
+                        relationshipId: params.relationshipId,
+                        relationshipType: params.relationshipType,
+                        modelWeight,
+                    }, { input, output, cached });
+                    return { text, reasoning, tokenUsage: { input, output }, modelWeight };
+                }
+                catch (error) {
+                    // Restartable only while nothing has been delivered — see
+                    // `consumerStarted`.
+                    if (!consumerStarted && retry < TRANSIENT_RETRY_WAITS_MS.length && this.isTransientNetworkError(error)) {
+                        clearTimeout(attempt.timeoutId);
+                        attempt.controller.abort();
+                        await this.waitBeforeTransientRetry(label, retry, error);
+                        attempt = startAttempt();
+                        continue;
+                    }
+                    clearTimeout(attempt.timeoutId);
+                    const message = error instanceof Error ? error.message : String(error);
+                    const stack = error instanceof Error ? (error.stack ?? "").split("\n").slice(0, 10).join("\n") : undefined;
+                    // A narration that streamed for 20s and then broke was still generated
+                    // and still billed — read whatever usage settled (see streamCall), under
+                    // the same bound so `result` always settles.
+                    const usage = await this.readUsageBounded(attempt.handle.usage);
+                    const input = usage?.inputTokens ?? 0;
+                    const output = usage?.outputTokens ?? 0;
+                    const cached = usage?.inputTokenDetails?.cacheReadTokens ?? 0;
+                    session.close({
+                        finalStatus: "error",
+                        errorMessage: message,
+                        errorStack: stack,
+                        totalTokens: { input, output, cached },
+                        warnings: [],
+                        parseFallbacks: [],
+                    });
+                    await this.persistUsageOnFailure({
+                        tokenUsageType: params.tokenUsageType,
+                        relationshipId: params.relationshipId,
+                        relationshipType: params.relationshipType,
+                        modelWeight,
+                    }, { input, output, cached });
+                    console.error("[LLMService.streamText] Error:", error);
+                    const wrapped = new Error(`LLM streamText error: ${message}`);
+                    wrapped.cause = error;
+                    throw wrapped;
+                }
             }
         })();
         // Normalize the AI SDK `fullStream` to text/reasoning deltas. Consuming this
@@ -1470,7 +1824,11 @@ let LLMService = LLMService_1 = class LLMService {
         // resolve. Reasoning-capable models interleave `reasoning-delta` parts (e.g.
         // Ollama emits the full thinking trace before answer content).
         async function* normalizedStream() {
-            for await (const part of streamResult.fullStream) {
+            // The first pull marks the stream unrestartable (see `consumerStarted`),
+            // and reads `attempt` late so a consumer that starts after a transient
+            // retry gets the stream that actually connected.
+            consumerStarted = true;
+            for await (const part of attempt.handle.fullStream) {
                 if (part.type === "text-delta") {
                     yield { type: "text", delta: part.text };
                 }
@@ -1531,6 +1889,12 @@ let LLMService = LLMService_1 = class LLMService {
             outputSchemaName: params.tool.name,
         });
         const attemptTimeoutMs = this.attemptTimeoutMs(params.timeout);
+        // Hoisted above the try so the catch can bill the attempts the provider
+        // already served before the failure. Accumulated (not overwritten) across
+        // attempts: the nudge retry is a SECOND request and is charged as one.
+        let totalInputTokens = 0;
+        let totalOutputTokens = 0;
+        let totalCachedTokens = 0;
         try {
             const model = this.modelService.getLLM({
                 modelWeight,
@@ -1610,10 +1974,18 @@ let LLMService = LLMService_1 = class LLMService {
                 return null;
             };
             // Each provider invocation is bounded independently — the nudge retry is a
-            // second request and gets its own budget, not the leftovers of the first.
+            // second request and gets its own budget, not the leftovers of the first —
+            // and each is retried on a transient network failure, with its own fresh
+            // abort controller per attempt.
             const invokeBounded = async (messages, attempt) => {
-                const controller = new AbortController();
-                return (await this.runBounded(`extractViaTool:${params.tool.name}:${attempt}`, attemptTimeoutMs, controller, () => bound.invoke(messages, { signal: controller.signal })));
+                const response = (await this.runWithTransientRetry(`extractViaTool:${params.tool.name}:${attempt}`, attemptTimeoutMs, (signal) => bound.invoke(messages, { signal })));
+                // Bill as we go: an attempt that answered is charged whether or not its
+                // payload turns out to be usable.
+                const usage = response.usage_metadata;
+                totalInputTokens += usage?.input_tokens ?? 0;
+                totalOutputTokens += usage?.output_tokens ?? 0;
+                totalCachedTokens += usage?.input_token_details?.cache_read ?? 0;
+                return response;
             };
             let response = await invokeBounded(baseMessages, "attempt-1");
             let parsed = tryExtract(response);
@@ -1629,9 +2001,11 @@ let LLMService = LLMService_1 = class LLMService {
             }
             if (parsed === null)
                 throw new Error("extractViaTool: model did not call the tool");
+            // The winning response's OWN usage — what the dump's response entry
+            // describes. The session total and the ledger use the accumulated figures
+            // instead, so a nudge retry bills both requests rather than only the last.
             const inputTokens = response.usage_metadata?.input_tokens ?? 0;
             const outputTokens = response.usage_metadata?.output_tokens ?? 0;
-            const cachedTokens = response.usage_metadata?.input_token_details?.cache_read ?? 0;
             session.recordResponse({
                 content: JSON.stringify(parsed),
                 tokenUsage: { input: inputTokens, output: outputTokens },
@@ -1639,7 +2013,7 @@ let LLMService = LLMService_1 = class LLMService {
             });
             session.close({
                 finalStatus: "success",
-                totalTokens: { input: inputTokens, output: outputTokens, cached: cachedTokens },
+                totalTokens: { input: totalInputTokens, output: totalOutputTokens, cached: totalCachedTokens },
                 warnings: [],
                 parseFallbacks: [],
             });
@@ -1648,7 +2022,7 @@ let LLMService = LLMService_1 = class LLMService {
                 relationshipId: params.relationshipId,
                 relationshipType: params.relationshipType,
                 modelWeight,
-            }, { input: inputTokens, output: outputTokens, cached: cachedTokens });
+            }, { input: totalInputTokens, output: totalOutputTokens, cached: totalCachedTokens });
             // Write-through on a miss so the next identical cacheable call hits.
             if (cacheKey && this.cache)
                 await this.cache.set(cacheKey, parsed);
@@ -1659,10 +2033,18 @@ let LLMService = LLMService_1 = class LLMService {
             session.close({
                 finalStatus: "error",
                 errorMessage: message,
-                totalTokens: { input: 0, output: 0 },
+                totalTokens: { input: totalInputTokens, output: totalOutputTokens, cached: totalCachedTokens },
                 warnings: [],
                 parseFallbacks: [],
             });
+            // Two refusals still cost two generations — bill what was served before
+            // giving up. Never throws, so the original error survives untouched.
+            await this.persistUsageOnFailure({
+                tokenUsageType: params.tokenUsageType,
+                relationshipId: params.relationshipId,
+                relationshipType: params.relationshipType,
+                modelWeight,
+            }, { input: totalInputTokens, output: totalOutputTokens, cached: totalCachedTokens });
             console.error("[LLMService.extractViaTool] Error:", error);
             throw error instanceof Error ? error : new Error(message);
         }
@@ -1670,14 +2052,22 @@ let LLMService = LLMService_1 = class LLMService {
     /**
      * Single-step model invocation with tools bound — the durable-checkpointing
      * counterpart to {@link call}'s internal tool loop. Performs exactly ONE
-     * model invocation (no tool execution, no loop, no retries, no structured
-     * output) and returns the raw AIMessage with any `tool_calls` untouched, so
-     * the caller (e.g. the operator agent) can checkpoint state and execute the
-     * tool calls itself.
+     * model STEP (no tool execution, no loop, no structured output) and returns
+     * the raw AIMessage with any `tool_calls` untouched, so the caller (e.g. the
+     * operator agent) can checkpoint state and execute the tool calls itself.
+     *
+     * "One step" is not "one socket": the step is bounded like every other call
+     * (per-attempt budget, watchdog, deadline) and re-issued on a transient
+     * network failure. What it never does is re-run the AGENT — the caller's
+     * checkpointed state is untouched either way.
      *
      * Reuses {@link call}'s model construction (`modelService.getLLM` +
-     * `bindTools`) and `LLMCallDumper` hooks. Token usage for the step is
-     * returned to the caller (which aggregates it) rather than tracked here.
+     * `bindTools`) and `LLMCallDumper` hooks. The step's token usage is still
+     * RETURNED to the caller (so an agent can keep its own running total), and —
+     * when `relationshipId`/`relationshipType` are supplied — is now also
+     * persisted here, exactly like every other provider call in this service.
+     * A caller that omits the attribution gets the previous behaviour: nothing is
+     * written. Each step is billed once, by whichever path completes it.
      *
      * @param params.systemPrompts - System prompts, prepended (in order) as
      *                               SystemMessages before `messages`
@@ -1685,6 +2075,9 @@ let LLMService = LLMService_1 = class LLMService {
      * @param params.tools - Tools to bind (NOT executed by this method)
      * @param params.temperature - Optional temperature override
      * @param params.metadata - Optional metadata for dump-session tracking
+     * @param params.tokenUsageType - Optional usage type for the recorded row
+     * @param params.relationshipId - Optional entity this usage is attributed to
+     * @param params.relationshipType - Optional entity type for the attribution
      *
      * @returns The raw AIMessage (tool_calls intact) plus this call's token usage
      */
@@ -1709,10 +2102,28 @@ let LLMService = LLMService_1 = class LLMService {
             })),
             outputSchemaName: "callStep",
         });
+        // This method used to be the ONE provider call in the service with no bound
+        // of any kind: no per-attempt budget on the model, no watchdog, no deadline.
+        // A stalled operator step therefore hung its durable run forever, which is
+        // exactly the failure `runBounded` exists to end — so it gets the same
+        // treatment as `call()`, plus the transient-network retry.
+        const attemptTimeoutMs = this.attemptTimeoutMs();
+        const label = `llm.callStep:${aiConfig.model}`;
+        // Hoisted above the try so the catch reports and bills whatever the step
+        // managed to consume before it threw.
+        let input = 0;
+        let output = 0;
+        // Prompt-cache hits, priced at `cachedInputCostPer1MTokens` by `computeCost`.
+        // The operator re-sends its whole conversation every step, so a cache hit is
+        // the NORM here: dropping this would price the entire prompt at the uncached
+        // rate and over-bill every operator row. Credited exactly as `call()` and
+        // `streamCall` credit theirs.
+        let cached = 0;
         try {
             const baseModel = this.modelService.getLLM({
                 temperature: params.temperature,
                 modelWeight,
+                timeoutMs: attemptTimeoutMs,
             });
             const modelWithTools = params.tools.length > 0 ? baseModel.bindTools(params.tools) : baseModel;
             const conversationMessages = [
@@ -1720,12 +2131,11 @@ let LLMService = LLMService_1 = class LLMService {
                 ...params.messages,
             ];
             session.startIteration("tool-loop", conversationMessages);
-            const response = (await (params.metadata
-                ? modelWithTools.invoke(conversationMessages, { metadata: params.metadata })
-                : modelWithTools.invoke(conversationMessages)));
+            const response = (await this.runWithTransientRetry(label, attemptTimeoutMs, (signal) => modelWithTools.invoke(conversationMessages, params.metadata ? { metadata: params.metadata, signal } : { signal })));
             const raw = response;
-            const input = raw.usage_metadata?.input_tokens ?? 0;
-            const output = raw.usage_metadata?.output_tokens ?? 0;
+            input = raw.usage_metadata?.input_tokens ?? 0;
+            output = raw.usage_metadata?.output_tokens ?? 0;
+            cached = raw.usage_metadata?.input_token_details?.cache_read ?? 0;
             session.recordResponse({
                 content: typeof response.content === "string" ? response.content : "",
                 toolCalls: (response.tool_calls ?? []).map((c) => ({
@@ -1738,10 +2148,16 @@ let LLMService = LLMService_1 = class LLMService {
             });
             session.close({
                 finalStatus: "success",
-                totalTokens: { input, output },
+                totalTokens: { input, output, cached },
                 warnings: [],
                 parseFallbacks: [],
             });
+            await this.persistUsage({
+                tokenUsageType: params.tokenUsageType,
+                relationshipId: params.relationshipId,
+                relationshipType: params.relationshipType,
+                modelWeight,
+            }, { input, output, cached });
             return { message: response, tokenUsage: { input, output } };
         }
         catch (error) {
@@ -1751,10 +2167,16 @@ let LLMService = LLMService_1 = class LLMService {
                 finalStatus: "error",
                 errorMessage: message,
                 errorStack: stack,
-                totalTokens: { input: 0, output: 0 },
+                totalTokens: { input, output, cached },
                 warnings: [],
                 parseFallbacks: [],
             });
+            await this.persistUsageOnFailure({
+                tokenUsageType: params.tokenUsageType,
+                relationshipId: params.relationshipId,
+                relationshipType: params.relationshipType,
+                modelWeight,
+            }, { input, output, cached });
             console.error("[LLMService.callStep] Error:", error);
             throw error instanceof Error ? error : new Error(message);
         }