akm-cli 0.9.15-beta.1 → 0.9.15-beta.3

package/dist/llm/embedders/remote.js

@@ -7,9 +7,10 @@
  * Calls the configured `/embeddings` endpoint and L2-normalizes the returned
  * vectors so the scoring pipeline's L2-to-cosine conversion is correct.
  */
-import { fetchWithTimeout, isHttpUrl, readBodyWithByteCap } from "../../core/common.js";
+import { abortableDelay, backoffDelay, fetchWithTimeout, isHttpUrl, readBodyWithByteCap } from "../../core/common.js";
 import { concurrentMap } from "../../core/concurrent.js";
 import { resolveSecret } from "../../core/config/config.js";
+import { ENV_REFERENCE_PATTERN, SECRET_STORE_REFERENCE_PATTERN } from "../../core/config/schema/primitives.js";
 import { defaultConcurrencyForEndpoint } from "../../core/loopback.js";
 import { redactErrorBody, redactSensitiveText } from "../../core/redaction.js";
 import { warnVerbose } from "../../core/warn.js";
@@ -24,18 +25,130 @@ import { resolveSecretFromStore } from "../../sources/snapshot-fetchers/secret-s
 export const DEFAULT_REMOTE_BATCH_SIZE = 100;
 /**
  * Conservative default token budget per HTTP request when the config gives
- * no better number (`maxTokens` or `contextLength`). #874's measurements:
+ * no better number (`maxTokens` — see #956 for why `contextLength`
+ * no longer feeds this). #874's measurements:
  * a batch of 100 small docs (~400 KB, ~100K tokens) took 14.8s against a
  * healthy local endpoint — half the 30s request timeout — and a single
  * 128 KB (~24K token) document alone was rejected by the endpoint as
- * exceeding its context size. 8000 tokens keeps a batch's estimated size
- * comfortably under both the timeout and common local-model context windows.
+ * exceeding its context size.
+ *
+ * Lowered from 8000 to 6000 (#954, field report on beta.1): the 4-chars-
+ * per-token estimator undercounts dense technical text by 7-55%, so 8000
+ * against an 8192-token llama.cpp embedder regularly landed real requests
+ * over the endpoint's context window. 6000 is the value the field confirmed
+ * stops that steady trickle of rejections; `embedBatch`'s run-scoped
+ * adaptive budget below still shrinks further, for an endpoint where even
+ * this is not enough.
  */
-export const DEFAULT_TOKEN_BUDGET = 8000;
+export const DEFAULT_TOKEN_BUDGET = 6000;
 /** Cheap token estimator: 4 chars ≈ 1 token. Used in verbose logging and error messages. */
 export function estimateTokenCount(text) {
     return Math.round(text.length / 4);
 }
+/**
+ * Default per-document embedding cap (`embedding.maxInputTokens`, #956)
+ * — the materializer truncates a document's embedded text to
+ * this cap (head only) instead of skipping it outright, so one oversized
+ * entry can no longer fail a whole batch. Fragments are not embedded at all
+ * (only the entry's own search text is), so this is the only lever on how
+ * much of a large document contributes to its vector.
+ */
+export const DEFAULT_MAX_INPUT_TOKENS = 512;
+/**
+ * Truncate `text` to at most `maxTokens` (estimated via
+ * {@link estimateTokenCount}, the same 4-chars≈1-token rule the batching
+ * budget uses), keeping only its head. The cut never splits a UTF-16
+ * surrogate pair. Text already at or under the cap is returned unchanged
+ * (`truncated: false`) — including empty text, which is never itself
+ * "truncated".
+ */
+export function capEmbeddingText(text, maxTokens) {
+    if (estimateTokenCount(text) <= maxTokens)
+        return { text, truncated: false };
+    const charBudget = Math.max(0, maxTokens * 4);
+    let cut = Math.min(charBudget, text.length);
+    if (cut > 0 && cut < text.length) {
+        const code = text.charCodeAt(cut);
+        // A low surrogate (0xDC00-0xDFFF) at the cut point means its high
+        // surrogate is the character just before it — back off one position so
+        // the pair stays together rather than yielding a lone surrogate.
+        if (code >= 0xdc00 && code <= 0xdfff)
+            cut -= 1;
+    }
+    return { text: text.slice(0, cut), truncated: true };
+}
+/**
+ * Default per-request timeout when `embedding.timeoutMs` is unset (#954).
+ * The prior fixed 30s cut off exactly the field-report case: a
+ * local model server on a full-budget (`DEFAULT_TOKEN_BUDGET`) batch
+ * legitimately takes longer than that, and the timeout fired mid-response
+ * with no retry — every batch it hit was silently dropped for the rest of
+ * an hours-long run. 120s comfortably covers a slow local batch while still
+ * bounding a genuinely dead endpoint to a few minutes, not forever.
+ */
+export const DEFAULT_EMBEDDING_TIMEOUT_MS = 120_000;
+/** Resolve the effective per-request timeout: `embedding.timeoutMs` when set, else the default above. */
+export function resolveEmbeddingTimeoutMs(config) {
+    return config.timeoutMs ?? DEFAULT_EMBEDDING_TIMEOUT_MS;
+}
+/**
+ * Scale the per-request timeout down for a smaller-than-budget request
+ * (#954, field-report follow-up): `embedding.timeoutMs` /
+ * {@link resolveEmbeddingTimeoutMs} is the budget for a request at the FULL
+ * token budget; a batch using only a fraction of it gets a proportionally
+ * smaller timeout, floored at 30s and never above the configured
+ * `timeoutMs` itself, so a dead server is detected in seconds on the common
+ * case of small documents instead of always waiting out the full configured
+ * budget.
+ */
+export function scaleEmbeddingTimeoutMs(timeoutMs, requestTokens, tokenBudget) {
+    const scaled = tokenBudget > 0 ? timeoutMs * (requestTokens / tokenBudget) : timeoutMs;
+    return Math.min(Math.max(scaled, 30_000), timeoutMs);
+}
+/**
+ * True when `err` is a request- or body-read timeout (#954) —
+ * `fetchWithTimeout`'s connection/header timeout ("Request timed out
+ * after...") or `readBodyWithByteCap`'s body-phase `BodyReadTimeoutError`.
+ * Only this failure mode gets the back-off-and-retry treatment:
+ * the field evidence was specifically that a timed-out request keeps
+ * computing server-side, so abandoning it immediately (the prior
+ * behavior) just grows the provider's queue further. A genuine network/HTTP
+ * failure (connection refused, malformed response, a real error response)
+ * has no such still-in-flight hazard and keeps the original
+ * skip-immediately behavior.
+ */
+export function isEmbeddingTimeoutError(err) {
+    if (!(err instanceof Error))
+        return false;
+    if (err.name === "BodyReadTimeoutError")
+        return true;
+    return err.message.startsWith("Request timed out after ");
+}
+/** TEST-ONLY seam: override the backoff base/max so retry-backoff tests run fast without waiting real seconds. */
+let embeddingTimeoutBackoffOverrideForTests;
+/** TEST-ONLY. Pass undefined to restore the real 5s/60s backoff. */
+export function _setEmbeddingTimeoutBackoffForTests(config) {
+    embeddingTimeoutBackoffOverrideForTests = config;
+}
+/**
+ * Backoff before the single same-size retry on a request timeout
+ * (#954) — reuses the same jittered
+ * exponential formula {@link backoffDelay} uses for the rest of the
+ * codebase's retry paths, at a base of "5s, doubling, capped at 60s".
+ *
+ * `timeoutAttempt` (#954, field-report follow-up) is how many times the
+ * SAME-SIZE-retry-then-split chain has already split before reaching this
+ * size — 0 at the top level. The first-timeout backoff for each successive,
+ * smaller size after a split grows with it (5s, then doubling, capped at
+ * 60s) instead of every split resetting to a flat ~5s: the field evidence
+ * was that an abandoned request keeps computing server-side, so a server
+ * already draining a whole chain of abandoned requests needs progressively
+ * more room, not the same fixed pause at every size.
+ */
+export function embeddingTimeoutRetryBackoffMs(timeoutAttempt = 0) {
+    const { baseMs, maxMs } = embeddingTimeoutBackoffOverrideForTests ?? { baseMs: 5_000, maxMs: 60_000 };
+    return backoffDelay(timeoutAttempt, baseMs, maxMs);
+}
 /**
  * Distinguishes a batch rejected because it exceeded the endpoint's context
  * window from every other failure mode (network error, 5xx, malformed
@@ -50,8 +163,15 @@ export class ContextExceededError extends Error {
         this.name = "ContextExceededError";
     }
 }
-/** Patterns providers use to report a request too large for the model's context window. */
-const CONTEXT_EXCEEDED_PATTERN = /exceed_context_size_error|context size|context length|too many tokens/i;
+/**
+ * Patterns providers use to report a request too large for the model's
+ * context window. `input is too large to process`/`physical batch size`/
+ * `ubatch` (#954) cover llama.cpp's own physical-batch
+ * rejection (HTTP 500, e.g. "input is too large to process. increase the
+ * physical batch size"), which was previously an unrecognized generic
+ * failure — the whole batch was dropped instead of split and retried.
+ */
+const CONTEXT_EXCEEDED_PATTERN = /exceed_context_size_error|context size|context length|too many tokens|input is too large to process|physical batch size|ubatch/i;
 /**
  * True when an HTTP failure means "this request's input is too large for the
  * endpoint's context window" rather than some other failure. HTTP 413
@@ -65,16 +185,25 @@ export function isContextExceededResponse(status, body) {
 }
 /**
  * Resolve the effective in-flight request window for `RemoteEmbedder.embedBatch`.
- * FIXED — no config override: 1 for a loopback endpoint, 2 for a remote one,
- * via the shared `defaultConcurrencyForEndpoint` (`src/core/loopback.ts`),
- * the same lowest-common-denominator rule `getDefaultLlmConcurrency`
- * (`src/indexer/indexer.ts`) uses. The actual throughput knob is request
- * SIZE, not request count: `embedding.batchSize` (document cap) and
- * `embedding.maxTokens`/`contextLength` (token budget) reach a larger batch
- * per request, which is where most of the win is — a 32-input batch takes
- * about the same wall time as one input against a healthy endpoint.
+ * Default (unset `embedding.concurrency`): 1 for a loopback endpoint, 2 for a
+ * remote one, via the shared `defaultConcurrencyForEndpoint`
+ * (`src/core/loopback.ts`), the same lowest-common-denominator rule
+ * `getDefaultLlmConcurrency` (`src/indexer/indexer.ts`) uses.
+ *
+ * `embedding.concurrency` (#954) overrides this default in
+ * either direction, bounded 1-16 at the config schema — added after field
+ * evidence that a multi-slot local server (llama.cpp `--parallel N`, vLLM)
+ * genuinely serves parallel requests and was left idle by the fixed default.
+ * Request SIZE remains the first throughput lever regardless:
+ * `embedding.batchSize` (document cap) and `embedding.maxTokens` (request
+ * token budget — see #956; `contextLength` no longer feeds it)
+ * reach a larger batch per request, which is where most of the win is for a
+ * single-slot server — a 32-input batch takes about the same wall time as
+ * one input against a healthy endpoint.
  */
 export function resolveEmbeddingConcurrency(config) {
+    if (typeof config.concurrency === "number")
+        return config.concurrency;
     return defaultConcurrencyForEndpoint(config.endpoint);
 }
 /**
@@ -113,6 +242,21 @@ export function buildTokenBoundedBatches(texts, tokenBudget, maxCount) {
     flush();
     return batches;
 }
+/**
+ * Shrink factor applied to the effective request budget on the first
+ * context-size rejection of an `embedBatch` run (#954, field report on
+ * beta.1): one 25% cut absorbs the estimator's measured undercount without
+ * repeatedly re-shrinking mid-run — see the "shrink at most once" rule on
+ * {@link RemoteEmbedder.embedBatch}.
+ */
+const ADAPTIVE_BUDGET_SHRINK_FACTOR = 0.75;
+/**
+ * Floor on the adaptive-budget shrink above, as a multiple of
+ * `maxInputTokens` (#954): a request budget below twice the per-document cap
+ * could no longer batch more than one document per request, defeating the
+ * point of batching at all.
+ */
+const ADAPTIVE_BUDGET_FLOOR_MULTIPLIER = 2;
 export class RemoteEmbedder {
     config;
     endpoint;
@@ -138,6 +282,7 @@ export class RemoteEmbedder {
         if (ollamaOpts) {
             body.options = ollamaOpts;
         }
+        const timeoutMs = resolveEmbeddingTimeoutMs(this.config);
         // `signal` MUST go through fetchWithTimeout's dedicated 4th parameter, not
         // the RequestInit: fetchWithTimeout replaces `opts.signal` with its own
         // controller (`{ ...opts, signal: controller.signal }`), so a signal passed
@@ -146,16 +291,16 @@ export class RemoteEmbedder {
             method: "POST",
             headers,
             body: JSON.stringify(body),
-        }, 30_000, signal);
+        }, timeoutMs, signal);
         if (!response.ok) {
-            const errBody = await readBodyWithByteCap(response, undefined, { bodyTimeoutMs: 30_000, signal }).catch((err) => {
+            const errBody = await readBodyWithByteCap(response, undefined, { bodyTimeoutMs: timeoutMs, signal }).catch((err) => {
                 if (signal?.aborted)
                     throw err;
                 return "";
             });
             throw new Error(`Embedding request failed (${response.status}): ${this.safeErrorBody(errBody)}`);
         }
-        const json = JSON.parse(await readBodyWithByteCap(response, undefined, { bodyTimeoutMs: 30_000, signal }));
+        const json = JSON.parse(await readBodyWithByteCap(response, undefined, { bodyTimeoutMs: timeoutMs, signal }));
         if (!json.data?.[0]?.embedding) {
             throw new Error(`Unexpected embedding response format: missing data[0].embedding.${embeddingEndpointPathHint(this.endpoint)}`);
         }
@@ -197,8 +342,37 @@ export class RemoteEmbedder {
      * {@link isContextExceededResponse}) is split in half and retried
      * recursively rather than skipped outright, down to individual documents; a
      * single document that still fails this way becomes a genuine
-     * `context-window-exceeded` skip. Every other failure (network error, 5xx,
-     * malformed response) keeps the original skip-the-whole-batch behavior.
+     * `context-window-exceeded` skip.
+     *
+     * A request TIMEOUT (see {@link isEmbeddingTimeoutError}) never drops the
+     * batch outright (#954): the field evidence
+     * was that akm abandoning a timed-out request does not stop the server
+     * from still computing it, so immediately skipping (or immediately
+     * splitting, the prior behavior) just let the provider's queue grow
+     * while every following batch died the same way. Instead, on a timeout,
+     * this backs off ({@link embeddingTimeoutRetryBackoffMs}) and retries the
+     * SAME request once; a second timeout splits it in half (like a
+     * context-size rejection) and retries each half the same way, down to
+     * single documents — a single document that times out twice is finally
+     * skipped. Every other failure (network error, malformed response, a
+     * non-timeout HTTP failure) keeps the original skip-the-whole-batch-
+     * immediately behavior, at any size. The per-request timeout itself also
+     * scales down with the request's estimated size via
+     * {@link scaleEmbeddingTimeoutMs}, so a dead server is detected in seconds
+     * on a small batch rather than always waiting out the full configured
+     * `embedding.timeoutMs`.
+     *
+     * Run-scoped adaptive budget (#954, field report on beta.1): the FIRST
+     * context-size rejection of the run shrinks the effective request budget
+     * by {@link ADAPTIVE_BUDGET_SHRINK_FACTOR} (floored at
+     * {@link ADAPTIVE_BUDGET_FLOOR_MULTIPLIER} times `maxInputTokens`) for
+     * every batch not yet dispatched — the still-planned tail of `texts` is
+     * re-batched with `buildTokenBoundedBatches` at the smaller budget, and a
+     * `budget-lowered` `onBatch` event reports it once. This never touches the
+     * split-and-retry of the rejected batch itself (above), and never fires a
+     * second time in the same run even if a later batch is also rejected — a
+     * static configured budget that is simply too big for the endpoint should
+     * self-correct once, not ratchet down forever.
      */
     async embedBatch(texts, signal, onSkip, onBatch) {
         if (texts.length === 0)
@@ -206,9 +380,64 @@ export class RemoteEmbedder {
         const results = new Array(texts.length).fill(undefined);
         const headers = this.buildHeaders();
         const ollamaOpts = resolveOllamaOptions(this.config);
-        const tokenBudget = this.config.maxTokens ?? this.config.contextLength ?? DEFAULT_TOKEN_BUDGET;
+        // #956: `contextLength` is Ollama's `num_ctx` ONLY (see
+        // resolveOllamaOptions below) — it used to double as this client-side
+        // request budget too, so a config author setting it for one purpose
+        // silently changed the other. `maxTokens` is the sole knob for the
+        // request budget now; unset falls back to DEFAULT_TOKEN_BUDGET.
+        //
+        // `effectiveTokenBudget` (#954) starts at the configured/default value
+        // and MAY shrink once, on the run's first context-size rejection — see
+        // `maybeShrinkBudget` below. `textBatches` is mutated in place (spliced)
+        // by that shrink rather than reassigned, so the in-flight
+        // `concurrentMap` pool below (which reads this same array by reference)
+        // picks up the re-planned tail without restarting.
+        let effectiveTokenBudget = this.config.maxTokens ?? DEFAULT_TOKEN_BUDGET;
         const maxCount = this.config.batchSize ?? DEFAULT_REMOTE_BATCH_SIZE;
-        const textBatches = buildTokenBoundedBatches(texts, tokenBudget, maxCount);
+        const maxInputTokens = this.config.maxInputTokens ?? DEFAULT_MAX_INPUT_TOKENS;
+        const textBatches = buildTokenBoundedBatches(texts, effectiveTokenBudget, maxCount);
+        const configuredTimeoutMs = resolveEmbeddingTimeoutMs(this.config);
+        // How many of `textBatches` concurrentMap has already claimed (its own
+        // `nextIndex`, mirrored here so a budget shrink knows where the
+        // not-yet-dispatched tail begins). Assigned, not incremented, at the top
+        // of `runProviderBatch` — batches are claimed in strictly increasing
+        // order, so the highest `batchIndex` seen so far IS the claimed count.
+        let dispatchedBatchCount = 0;
+        // Set once the run's first context-size rejection has shrunk the budget
+        // (#954) — guards `maybeShrinkBudget` so it never fires twice.
+        let budgetShrunk = false;
+        // On the FIRST context-size rejection of this `embedBatch` call, shrink
+        // `effectiveTokenBudget` and re-plan every batch `concurrentMap` has not
+        // yet claimed from the smaller budget. Never touches `rejectedIndices`
+        // itself — the caller's own split-and-retry handles that batch — and is
+        // a no-op after the first call (`budgetShrunk`).
+        const maybeShrinkBudget = (rejectedIndices, rejectedBatchIndex, rejectedRequestTokens) => {
+            if (budgetShrunk)
+                return;
+            budgetShrunk = true;
+            const floor = ADAPTIVE_BUDGET_FLOOR_MULTIPLIER * maxInputTokens;
+            effectiveTokenBudget = Math.max(Math.round(effectiveTokenBudget * ADAPTIVE_BUDGET_SHRINK_FACTOR), floor);
+            const notYetDispatched = textBatches.slice(dispatchedBatchCount);
+            const remainingIndices = notYetDispatched.flatMap((batch) => batch.indices);
+            if (remainingIndices.length > 0) {
+                const remainingTexts = remainingIndices.map((i) => texts[i]);
+                const replanned = buildTokenBoundedBatches(remainingTexts, effectiveTokenBudget, maxCount).map((batch) => ({
+                    indices: batch.indices.map((localIndex) => remainingIndices[localIndex]),
+                    oversized: batch.oversized,
+                }));
+                textBatches.splice(dispatchedBatchCount, textBatches.length - dispatchedBatchCount, ...replanned);
+            }
+            warnVerbose(`[embed] provider rejected a ${rejectedRequestTokens}-token request as over its context; request budget lowered to ${effectiveTokenBudget.toLocaleString()} for the rest of this run`);
+            commitBatch(rejectedIndices, rejectedIndices.map(() => undefined), undefined, {
+                batchIndex: rejectedBatchIndex,
+                batchCount: textBatches.length,
+                docCount: rejectedIndices.length,
+                requestTokens: rejectedRequestTokens,
+                elapsedMs: 0,
+                outcome: "budget-lowered",
+                reason: `provider rejected ${rejectedRequestTokens.toLocaleString()} tokens as over its context; request budget lowered to ${effectiveTokenBudget.toLocaleString()} for the rest of this run`,
+            });
+        };
         // Stops the pool from claiming any FURTHER provider batch once the
         // caller's onBatch has failed once (the materializer's transaction
         // failed, so a subsequent commit would just fail again) — dispatching
@@ -242,7 +471,7 @@ export class RemoteEmbedder {
         // fabricated "batch-request-failed" skip (#954). Checked (and rethrown)
         // once the pool drains, the same way `signal?.aborted` is today.
         let firstOnBatchError;
-        const commitBatch = (indices, embeddings, model) => {
+        const commitBatch = (indices, embeddings, model, outcome) => {
             if (!onBatch)
                 return;
             // Once persistence has failed once, an already in-flight batch that
@@ -252,7 +481,7 @@ export class RemoteEmbedder {
             if (firstOnBatchError !== undefined)
                 return;
             try {
-                onBatch(indices, embeddings, model);
+                onBatch(indices, embeddings, model, outcome);
             }
             catch (err) {
                 firstOnBatchError = err;
@@ -260,65 +489,207 @@ export class RemoteEmbedder {
             }
         };
         // Requests a single provider batch (by index list), recursing on a
-        // context-size rejection. Never throws except to propagate a genuine
-        // caller abort — every other outcome (success or a non-abort failure)
-        // resolves normally after reporting via onSkip/onBatch. `onBatch` fires
-        // only once this try/catch has already settled success vs. failure, so a
-        // throw from it is never caught and reclassified by this block.
-        const requestAndCommit = async (indices) => {
+        // context-size rejection OR a repeated timeout (#954). Never
+        // throws except to propagate a genuine caller abort — every other
+        // outcome (success or a non-abort failure) resolves normally after
+        // reporting via onSkip/onBatch. `onBatch` fires only once this
+        // try/catch has already settled success vs. failure, so a throw from it
+        // is never caught and reclassified by this block.
+        //
+        // `isTimeoutRetry` marks the SECOND attempt at this exact `indices`
+        // (after the one same-size backoff-and-retry) — a second
+        // timeout at that point splits or terminally skips rather than backing
+        // off again.
+        //
+        // `timeoutAttempt` (#954, field-report follow-up) counts how many splits
+        // down the timeout-retry chain this call is: 0 at the top level, then
+        // +1 each time a second timeout at one size splits into two smaller
+        // requests. It is the `attempt` fed to {@link embeddingTimeoutRetryBackoffMs}
+        // so each successive size's first-timeout backoff is longer than the
+        // last (5s, doubling, capped at 60s) instead of every split restarting
+        // at the same ~5s delay — a server still draining a whole run of
+        // abandoned requests needs more room the deeper the chain goes, not the
+        // same fixed pause every time.
+        const requestAndCommit = async (indices, batchIndex, isTimeoutRetry = false, timeoutAttempt = 0) => {
+            // A concurrent batch may have tripped the circuit breaker (or failed
+            // onBatch) while this call was queued behind a split or a backoff —
+            // never let a deeper recursive call make a request that can no longer
+            // be reported, mirroring how the pool below never claims a
+            // not-yet-started textBatch once dispatch has stopped.
+            if (dispatchAbort.signal.aborted)
+                return;
             const batch = indices.map((i) => texts[i]);
+            const requestTokens = batch.reduce((sum, text) => sum + estimateTokenCount(text), 0);
+            const requestTimeoutMs = scaleEmbeddingTimeoutMs(configuredTimeoutMs, requestTokens, effectiveTokenBudget);
+            const requestStart = Date.now();
             let batchEmbeddings;
             let responseModel;
+            let outcome;
+            let failureReason;
             try {
-                const { vectors, model } = await this.requestBatch(batch, headers, ollamaOpts, signal);
+                const { vectors, model } = await this.requestBatch(batch, headers, ollamaOpts, requestTimeoutMs, signal);
                 for (let k = 0; k < indices.length; k++) {
                     results[indices[k]] = vectors[k];
                 }
                 batchEmbeddings = indices.map((i) => results[i]);
                 responseModel = model;
+                outcome = "stored";
             }
             catch (err) {
                 // A caller abort must still propagate — it is not a "this batch
                 // failed" condition, it means stop entirely.
                 if (signal?.aborted)
                     throw err;
+                if (err instanceof ContextExceededError) {
+                    // #954: the run's FIRST context-size rejection (any size) shrinks
+                    // the budget for everything not yet dispatched; a no-op after the
+                    // first call. Deliberately BEFORE the split below — it must fire
+                    // for a single-document rejection too (which never reaches the
+                    // `indices.length > 1` split branch), and it never touches this
+                    // batch's own split-and-retry.
+                    maybeShrinkBudget(indices, batchIndex, requestTokens);
+                }
                 if (err instanceof ContextExceededError && indices.length > 1) {
                     const mid = Math.ceil(indices.length / 2);
-                    await requestAndCommit(indices.slice(0, mid));
-                    await requestAndCommit(indices.slice(mid));
+                    await requestAndCommit(indices.slice(0, mid), batchIndex, false, timeoutAttempt);
+                    await requestAndCommit(indices.slice(mid), batchIndex, false, timeoutAttempt);
+                    return;
+                }
+                const timedOut = isEmbeddingTimeoutError(err);
+                if (timedOut && !isTimeoutRetry) {
+                    // First timeout at this size: back off so the provider can drain
+                    // the abandoned request, then retry the SAME request once before
+                    // ever splitting or skipping (#954). The backoff grows with
+                    // `timeoutAttempt`, not a flat ~5s every time, so a chain of
+                    // splits down to smaller and smaller requests gives the provider
+                    // proportionally more room to drain each time.
+                    const backoffMs = embeddingTimeoutRetryBackoffMs(timeoutAttempt);
+                    warnVerbose(`[embed] batch of ${batch.length} document(s) timed out after ${requestTimeoutMs}ms; retrying once after a ${Math.round(backoffMs)}ms backoff`);
+                    // Default-level notice (#954 field-report follow-up), not just the
+                    // verbose line above — a run silently waiting out a multi-minute
+                    // back-off looked identical to a hang otherwise. Nothing has
+                    // failed or succeeded yet, so there is nothing to persist:
+                    // `embeddings` are all `undefined` and the materializer's onBatch
+                    // must not touch storage for this event.
+                    commitBatch(indices, indices.map(() => undefined), undefined, {
+                        batchIndex,
+                        batchCount: textBatches.length,
+                        docCount: indices.length,
+                        requestTokens,
+                        elapsedMs: backoffMs,
+                        outcome: "retrying",
+                        reason: "timed out",
+                    });
+                    await abortableDelay(backoffMs, signal, "embedding interrupted during retry backoff");
+                    if (!dispatchAbort.signal.aborted) {
+                        return requestAndCommit(indices, batchIndex, true, timeoutAttempt);
+                    }
+                    // Dispatch was stopped (by another batch's circuit-breaker trip)
+                    // while this one was backing off — fall through and skip below
+                    // instead of issuing a request that can no longer be reported.
+                }
+                else if (timedOut && indices.length > 1) {
+                    // Timed out again on the retry: split rather than skip outright —
+                    // the provider may still fit it once it is smaller (#954),
+                    // the same treatment a context-size rejection gets.
+                    // Each half's OWN first-timeout backoff (#954 follow-up) starts
+                    // one attempt further down the chain than this size's did.
+                    const mid = Math.ceil(indices.length / 2);
+                    await requestAndCommit(indices.slice(0, mid), batchIndex, false, timeoutAttempt + 1);
+                    await requestAndCommit(indices.slice(mid), batchIndex, false, timeoutAttempt + 1);
                     return;
                 }
                 const message = err instanceof Error ? err.message : String(err);
                 const reason = err instanceof ContextExceededError ? "context-window-exceeded" : "batch-request-failed";
-                warnVerbose(`[embed] batch of ${batch.length} document(s) failed and was skipped: ${message}`);
-                for (const idx of indices) {
-                    onSkip?.({ index: idx, reason, message });
+                const failureKind = reason === "batch-request-failed" ? (timedOut ? "timeout" : "network-error") : undefined;
+                // Default-level visibility for a failed batch (not verbose-only) is
+                // still guaranteed here — just not via warn(). The `commitBatch` call
+                // below carries `outcome: "failed"` and this `message` as `reason`
+                // through `onBatch`, and materialize-embeddings.ts's per-batch line
+                // (also default-level) prints it from there. A warn() call here used
+                // to print the identical event a second time on stderr — the same
+                // class of double-print bug fixed for the truncation/re-embed-reason
+                // lines in materialize-embeddings.ts (#954, field-report follow-up).
+                // Per-entry batch-mapping detail stays verbose-only
+                // (materialize-embeddings.ts).
+                let stopRequested = false;
+                for (const [k, idx] of indices.entries()) {
+                    if (onSkip?.({
+                        index: idx,
+                        reason,
+                        message,
+                        batchStart: k === 0,
+                        batchSize: indices.length,
+                        failureKind,
+                    }) === false)
+                        stopRequested = true;
                 }
+                // #954: the caller's circuit breaker asked to stop —
+                // gate further dispatch through the SAME dispatchAbort controller
+                // the onBatch-throw path above uses, but resolve this call normally
+                // (never reject) with whatever results already landed, since this is
+                // a policy decision, not a persistence failure.
+                if (stopRequested)
+                    stopDispatch();
                 batchEmbeddings = indices.map(() => undefined);
+                outcome = "failed";
+                failureReason = message;
             }
-            commitBatch(indices, batchEmbeddings, responseModel);
+            commitBatch(indices, batchEmbeddings, responseModel, {
+                batchIndex,
+                batchCount: textBatches.length,
+                docCount: indices.length,
+                requestTokens,
+                elapsedMs: Date.now() - requestStart,
+                outcome,
+                reason: failureReason,
+            });
         };
-        const runProviderBatch = async (textBatch) => {
+        const runProviderBatch = async (textBatch, batchIndex) => {
+            // Claimed in strictly increasing order by `concurrentMap` below, so
+            // the highest `batchIndex` seen so far is exactly how many batches it
+            // has claimed (#954) — see `maybeShrinkBudget`'s doc comment above.
+            // Assigned synchronously at entry, before any `await`, so this always
+            // matches `concurrentMap`'s own `nextIndex` at the moment of claim.
+            dispatchedBatchCount = batchIndex;
             if (textBatch.oversized) {
                 const idx = textBatch.indices[0];
                 const estTokens = estimateTokenCount(texts[idx]);
                 onSkip?.({
                     index: idx,
                     reason: "context-window-exceeded",
-                    message: `Document estimated at ${estTokens} tokens exceeds the ${tokenBudget}-token embedding budget; skipped.`,
+                    message: `Document estimated at ${estTokens} tokens exceeds the ${effectiveTokenBudget}-token embedding budget; skipped.`,
+                    batchStart: true,
+                    batchSize: 1,
+                });
+                // Never made a request — excluded from the default-level per-batch
+                // line (there is no request outcome to report), but still counted
+                // in the run's oversized-skip total via `onSkip` above.
+                commitBatch([idx], [undefined], undefined, {
+                    batchIndex,
+                    batchCount: textBatches.length,
+                    docCount: 1,
+                    requestTokens: estTokens,
+                    elapsedMs: 0,
+                    outcome: "failed",
+                    reason: "oversized",
                 });
-                commitBatch([idx], [undefined]);
                 return;
             }
-            await requestAndCommit(textBatch.indices);
+            await requestAndCommit(textBatch.indices, batchIndex);
         };
         const concurrency = resolveEmbeddingConcurrency(this.config);
         // concurrentMap swallows a thrown fn (per-item, results discarded here —
         // requestAndCommit only throws to signal a caller abort), so abort must
         // be re-checked once the pool has drained rather than relying on the
         // throw itself to escape. Dispatch is gated on `dispatchAbort`, not the
-        // caller's `signal` directly — see the comment above.
-        await concurrentMap(textBatches, runProviderBatch, concurrency, { signal: dispatchAbort.signal });
+        // caller's `signal` directly — see the comment above. `batchIndex` is
+        // 1-based (matches the human-readable "batch N/Total" line) and comes
+        // from `concurrentMap`'s own 0-based item index, not a separately
+        // tracked counter, so it stays correct under concurrency.
+        await concurrentMap(textBatches, (textBatch, i) => runProviderBatch(textBatch, i + 1), concurrency, {
+            signal: dispatchAbort.signal,
+        });
         if (callerAbortListener)
             signal?.removeEventListener("abort", callerAbortListener);
         if (signal?.aborted) {
@@ -335,8 +706,14 @@ export class RemoteEmbedder {
      * used by the embedding-fingerprint canary to verify a config-string
      * rename against what the endpoint actually served, not just re-assert the
      * configured string. Throws on any failure.
+     *
+     * `timeoutMs` is the caller's ALREADY-SCALED per-request timeout (#954
+     * — see {@link scaleEmbeddingTimeoutMs}), not re-resolved here:
+     * `embedBatch` computes it per request from that request's own size so a
+     * split-down retry gets a smaller, size-appropriate timeout rather than
+     * always the full configured `embedding.timeoutMs`.
      */
-    async requestBatch(batch, headers, ollamaOpts, signal) {
+    async requestBatch(batch, headers, ollamaOpts, timeoutMs, signal) {
         const body = {
             input: batch,
             model: this.model,
@@ -353,9 +730,9 @@ export class RemoteEmbedder {
             method: "POST",
             headers,
             body: JSON.stringify(body),
-        }, 30_000, signal);
+        }, timeoutMs, signal);
         if (!response.ok) {
-            const respBody = await readBodyWithByteCap(response, undefined, { bodyTimeoutMs: 30_000, signal }).catch((err) => {
+            const respBody = await readBodyWithByteCap(response, undefined, { bodyTimeoutMs: timeoutMs, signal }).catch((err) => {
                 if (signal?.aborted)
                     throw err;
                 return "";
@@ -366,7 +743,7 @@ export class RemoteEmbedder {
             }
             throw new Error(message);
         }
-        const json = JSON.parse(await readBodyWithByteCap(response, undefined, { bodyTimeoutMs: 30_000, signal }));
+        const json = JSON.parse(await readBodyWithByteCap(response, undefined, { bodyTimeoutMs: timeoutMs, signal }));
         if (!json.data || json.data.length !== batch.length) {
             throw new Error(`Unexpected embedding batch response: expected ${batch.length} embeddings, got ${json.data?.length ?? 0}.${embeddingEndpointPathHint(this.endpoint)}`);
         }
@@ -471,3 +848,22 @@ function resolveOllamaOptions(config) {
 export function hasRemoteEndpoint(config) {
     return isHttpUrl(config.endpoint);
 }
+/**
+ * Describe WHERE an `embedding.apiKey` came from, never its value — the
+ * actionable outcome of the #953 field gap: `resolveSecret` throws on an
+ * unresolvable `secret://` reference, so a keyless request can only mean
+ * `embedding.apiKey` was absent from the config the run actually loaded (a
+ * different config root, scope, or a config edited after the run started).
+ * A default-level progress line naming the credential's SOURCE (this
+ * helper), printed once before the first provider request, lets a field run
+ * self-diagnose that without ever surfacing the secret itself.
+ */
+export function describeEmbeddingCredential(apiKey) {
+    if (!apiKey)
+        return "none configured";
+    if (SECRET_STORE_REFERENCE_PATTERN.test(apiKey))
+        return `${apiKey} (store)`;
+    if (ENV_REFERENCE_PATTERN.test(apiKey))
+        return `${apiKey} (env)`;
+    return "literal apiKey";
+}