akm-cli 0.9.15-beta.1 → 0.9.15-beta.3

package/dist/indexer/materialize-embeddings.js

@@ -1,12 +1,14 @@
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
+import { getConfigPath } from "../core/paths.js";
 import { isVerbose, warn, warnVerbose } from "../core/warn.js";
 import { embedBatch } from "../llm/embedder.js";
 import { DETERMINISTIC_EMBED_MODEL_ID, isDeterministicEmbedEnabled } from "../llm/embedders/deterministic.js";
 import { DEFAULT_LOCAL_MODEL } from "../llm/embedders/local.js";
-import { buildTokenBoundedBatches, DEFAULT_REMOTE_BATCH_SIZE, DEFAULT_TOKEN_BUDGET, estimateTokenCount, hasRemoteEndpoint, } from "../llm/embedders/remote.js";
+import { buildTokenBoundedBatches, capEmbeddingText, DEFAULT_MAX_INPUT_TOKENS, DEFAULT_REMOTE_BATCH_SIZE, DEFAULT_TOKEN_BUDGET, describeEmbeddingCredential, estimateTokenCount, hasRemoteEndpoint, normalizeEmbeddingEndpoint, } from "../llm/embedders/remote.js";
 import { cosineSimilarity } from "../llm/embedders/types.js";
+import { purgeEmbeddingSalvage, relabelEmbeddingSalvageFingerprint, reuseSalvagedEmbeddings, } from "../storage/repositories/embedding-salvage-repository.js";
 import { getEmbeddableEntryCount } from "../storage/repositories/index-entries-repository.js";
 import { deleteMeta, getMeta, setMeta } from "../storage/repositories/index-meta-repository.js";
 import { getAllEntriesForEmbedding, getEmbeddingCount, isVecFastPathComplete, isVecFastPathReady, purgeEmbeddings, sampleEmbeddedEntriesForCanary, setVecFastPathReady, upsertEmbedding, } from "../storage/repositories/index-vec-repository.js";
@@ -23,8 +25,14 @@ export function deriveSemanticProviderFingerprint(embedding) {
     }
     return `local:${embedding?.localModel ?? DEFAULT_LOCAL_MODEL}`;
 }
-/** How often (in stored entries) to emit a progress line during a large embedding run (#954). */
-const PROGRESS_INTERVAL = 500;
+/**
+ * The heartbeat text emitted every 15s while a provider request is in
+ * flight, and by default (not `--verbose`-only) since silence indistinguishable
+ * from a hang was the field report's own symptom (#954).
+ */
+export function formatEmbeddingHeartbeat(storedCount, total, failedCount) {
+    return `Still generating embeddings: ${storedCount}/${total} stored, ${failedCount} failed; waiting on embedding provider.`;
+}
 /**
  * Number of already-embedded entries sampled for the fingerprint-rename
  * canary (#955) — small and cheap even against a slow local server; a
@@ -38,6 +46,23 @@ const CANARY_SAMPLE_SIZE = 8;
  * same-model rename rather than a real model change (#955).
  */
 const CANARY_SIMILARITY_THRESHOLD = 0.999;
+/**
+ * Consecutive transport failures after which the embedding pass stops
+ * dispatching further requests and ends the run as a failure rather than
+ * grinding through every remaining batch against a dead endpoint (#954).
+ * Two independent trip conditions
+ * share this threshold — see `onSkip` below: 3 consecutive failures at
+ * single-document size (timeout OR network error — a multi-document
+ * timeout is not by itself evidence the endpoint is dead, since
+ * `RemoteEmbedder.embedBatch` already retries and splits it smaller before
+ * ever reporting it as failed at single-document size), or 3 consecutive
+ * network errors at ANY size (a network error is never retried, so it is
+ * trusted immediately regardless of how large the request was).
+ * `context-window-exceeded` never counts — that reason proves the provider
+ * IS reachable, and split-and-retry already handles it; it resets both
+ * streaks instead.
+ */
+const CIRCUIT_BREAKER_THRESHOLD = 3;
 /**
  * Pure decision: do stored vectors remain valid against freshly re-embedded
  * canary samples? The ONE place that computes the canary's similarity
@@ -89,7 +114,7 @@ function medianOf(values) {
  * on what the server (or local model) actually reported plus the observed
  * vector width, so a gateway/transport change that keeps returning the same
  * underlying model can be told apart from a genuine model change without
- * relying on the operator's config string (#955 field-review addendum).
+ * relying on the operator's config string (#955).
  * Returns undefined when nothing was actually observed this call (no vector
  * to measure yet).
  */
@@ -111,8 +136,14 @@ function deriveObservedEmbeddingIdentity(embedding, observedModel, observedVecto
  * direct `RemoteEmbedder`) so every embedder branch — remote, local,
  * deterministic, and test overrides via `_setEmbedderForTests` — is
  * exercised identically to the main embedding pass.
+ *
+ * `maxInputTokens` must be the SAME cap the main pass below applies via
+ * {@link capEmbeddingText} — the stored vector for each sampled entry was
+ * produced from its capped text, so comparing against a fresh vector of the
+ * uncapped text would compare unlike inputs for any entry over the cap
+ * (#955).
  */
-async function runEmbeddingCanary(db, config, signal) {
+async function runEmbeddingCanary(db, config, signal, maxInputTokens) {
     const samples = sampleEmbeddedEntriesForCanary(db, CANARY_SAMPLE_SIZE);
     if (samples.length === 0) {
         return { outcome: "keep", verified: false, viaIdentityMatch: false };
@@ -121,7 +152,15 @@ async function runEmbeddingCanary(db, config, signal) {
     const skips = [];
     let canaryVectors;
     try {
-        canaryVectors = await embedBatch(samples.map((sample) => sample.searchText), config.embedding, signal, (skip) => skips.push(skip), (_indices, _embeddings, model) => {
+        canaryVectors = await embedBatch(
+        // #955: the stored vector for each sample was produced from
+        // capEmbeddingText(searchText, maxInputTokens) — the main pass below
+        // caps every document before embedding it. The canary must re-embed
+        // the SAME capped text, or an entry over the cap compares a fresh
+        // vector of a different input against a stored vector of the capped
+        // one, and a genuine model match can read as a rebuild-worthy
+        // mismatch for reasons unrelated to the model.
+        samples.map((sample) => capEmbeddingText(sample.searchText, maxInputTokens).text), config.embedding, signal, (skip) => skips.push(skip), (_indices, _embeddings, model) => {
             if (model)
                 observedModel = model;
         });
@@ -176,11 +215,47 @@ function throwIfAborted(signal) {
     }
 }
 export async function generateEmbeddingsForDb(db, config, onProgress, signal, entryIds, opts) {
+    // Drift guard (#954): refuse an ambient transaction. Every
+    // per-batch `db.transaction()` below is meant to be its own durable commit
+    // (#954) — inside an already-open outer transaction it would nest as an
+    // unobservable SAVEPOINT instead, so an interruption (competing-process
+    // collision, SIGKILL) could lose the whole pass rather than only the batch
+    // in flight. This is an internal contract error (a caller bug), not a
+    // user-facing failure class: callers with their own transaction (e.g. `akm
+    // bundle update`'s unified update transaction) must run the embedding
+    // phase on a separate connection AFTER their own transaction commits — see
+    // `runEmbeddingPass` in `src/indexer/indexer.ts`.
+    if (db.inTransaction) {
+        throw new Error("generateEmbeddingsForDb was called with an ambient transaction already open on `db`: per-batch commits " +
+            "would become SAVEPOINTs inside it, losing the crash-durability contract per-batch commit exists for. " +
+            "Run the embedding phase on a connection with no open transaction.");
+    }
     throwIfAborted(signal);
     if (config.semanticSearchMode === "off") {
+        // #955: salvage is self-emptying only if every path that skips reuse
+        // also drains it — otherwise a full rebuild performed with semantic
+        // search disabled leaves permanent orphaned rows behind (nothing will
+        // ever consume them, since this path never reaches the reuse step).
+        purgeEmbeddingSalvage(db);
         onProgress({ phase: "embeddings", message: "Semantic search disabled; skipping embeddings." });
         return { success: false, message: "Semantic search is disabled." };
     }
+    // #953 field gap: the actionable outcome is a self-diagnosing run, not a
+    // fix (every RemoteEmbedder path already resolves secret:// through one
+    // boundary — a keyless request can only mean embedding.apiKey was absent
+    // from the config THIS run loaded). One default-level line, before the
+    // first provider request of the phase (the canary probe or the main
+    // pass, whichever runs first below), naming the endpoint/model/credential
+    // SOURCE — never the credential value.
+    if (hasRemoteEndpoint(config.embedding ?? {})) {
+        const endpoint = normalizeEmbeddingEndpoint(config.embedding?.endpoint ?? "");
+        const credential = describeEmbeddingCredential(config.embedding?.apiKey);
+        const configFileSuffix = isVerbose() ? `; config: ${getConfigPath()}` : "";
+        onProgress({
+            phase: "embeddings",
+            message: `[embed] endpoint ${endpoint}, model ${config.embedding?.model ?? "unknown"}; credential: ${credential}${configFileSuffix}`,
+        });
+    }
     // A targeted call starts from an already-published generation. Preserve its
     // trust decision in O(1): successful writes for the changed IDs keep a
     // healthy fast path healthy, but can never promote a generation already
@@ -191,6 +266,10 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
     let targetEntryIds = entryIds;
     /** Set only on an actual rebuild, so the up-front "Re-embedding N entries" line names why. */
     let rebuildReason;
+    // Resolved once and reused by both the canary (below) and the main pass's
+    // cap loop (further down) — the same cap must apply to both, or the canary
+    // compares a differently-capped text against the stored vector (#955).
+    const maxInputTokens = config.embedding?.maxInputTokens ?? DEFAULT_MAX_INPUT_TOKENS;
     if (opts?.forceReembed) {
         // `akm index --reembed`: an explicit operator override, skips the canary
         // entirely. The new fingerprint (and identity, now stale/unknown until
@@ -200,6 +279,9 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
         // instead of purging again from zero (#955/#956).
         db.transaction(() => {
             purgeEmbeddings(db, { dropVecTable: true });
+            // #955: an explicit forced rebuild must re-embed everything, not
+            // quietly satisfy some of it from stale salvage.
+            purgeEmbeddingSalvage(db);
             deleteMeta(db, "embeddingDim");
             setMeta(db, "embeddingFingerprint", currentFingerprint);
             deleteMeta(db, "embeddingIdentity");
@@ -208,7 +290,7 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
         rebuildReason = "forced by --reembed";
     }
     else if (storedFingerprint && storedFingerprint !== currentFingerprint) {
-        const decision = await runEmbeddingCanary(db, config, signal);
+        const decision = await runEmbeddingCanary(db, config, signal, maxInputTokens);
         if (decision.outcome === "unverifiable") {
             // Destroying a good index because the server happens to be down right
             // now is worse than leaving a rename unverified until the next run —
@@ -221,6 +303,9 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
         if (decision.outcome === "rebuild") {
             db.transaction(() => {
                 purgeEmbeddings(db, { dropVecTable: true });
+                // #955: the stored vectors AND any leftover salvage both belong to
+                // a different model now — neither is reusable, so both go.
+                purgeEmbeddingSalvage(db);
                 deleteMeta(db, "embeddingDim");
                 setMeta(db, "embeddingFingerprint", currentFingerprint);
                 if (decision.identity)
@@ -238,6 +323,11 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
             // immediate write means a crash right after this decision does not
             // re-run the canary needlessly on the next attempt.
             setMeta(db, "embeddingFingerprint", currentFingerprint);
+            // #955: the model did not actually change, only the fingerprint
+            // STRING did (e.g. a gateway rename) — any leftover salvage rows
+            // tagged with the OLD string are still valid vectors. Relabel them so
+            // the reuse step below (and any later pass) can still find them.
+            relabelEmbeddingSalvageFingerprint(db, storedFingerprint, currentFingerprint);
             if (decision.identity)
                 setMeta(db, "embeddingIdentity", decision.identity);
             if (decision.verified) {
@@ -253,24 +343,94 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
             // lose or verify against — adopt the label silently, no purge line.
         }
     }
+    else {
+        // No rename to verify (either this is the very first
+        // pass ever for this db, or the fingerprint already matches the last
+        // successful one) — still record it NOW rather than deferring to a
+        // fully successful pass, mirroring the rebuild/keep branches above
+        // (#955/#956). Without this, an interrupted FIRST-EVER pass left
+        // `embeddingFingerprint` unset despite a per-batch commit below (#954)
+        // already having durably written real vectors — a later `akm index
+        // --full`'s salvage-before-discard step tags rows by this meta
+        // (`salvageEmbeddingsBeforeDiscard`) and treats an unset fingerprint as
+        // "nothing was ever verified", silently turning genuinely-embedded
+        // vectors into a full re-embed instead of a salvage-and-reuse.
+        setMeta(db, "embeddingFingerprint", currentFingerprint);
+    }
     try {
         throwIfAborted(signal);
         const allEntries = getAllEntriesForEmbedding(db, targetEntryIds);
-        if (allEntries.length === 0) {
+        let vecFailedCount = 0;
+        let vecUnavailableCount = 0;
+        // #955: before any provider call, hand back vectors salvaged from a
+        // full rebuild or a generation bump for entries whose search_text is
+        // byte-identical to what was salvaged under the SAME fingerprint — a
+        // fingerprint mismatch or a single-byte content change both correctly
+        // fall through to the provider below instead.
+        const { reusedCount, remaining: candidateEntries } = reuseSalvagedEmbeddings(db, allEntries, currentFingerprint, (entry, embedding) => {
+            const result = upsertEmbedding(db, entry.id, embedding);
+            if (result.vec === "failed")
+                vecFailedCount++;
+            if (result.vec === "unavailable")
+                vecUnavailableCount++;
+            return result.stored;
+        });
+        if (reusedCount > 0) {
+            onProgress({
+                phase: "embeddings",
+                message: `Reused ${reusedCount} embedding${reusedCount === 1 ? "" : "s"} from the previous generation; embedding ${candidateEntries.length} new.`,
+            });
+        }
+        if (candidateEntries.length === 0) {
             onProgress({ phase: "embeddings", message: "Embeddings already up to date." });
             setMeta(db, "embeddingFingerprint", currentFingerprint);
-            return { success: true };
+            if (reusedCount > 0) {
+                const vecGenerationComplete = targetEntryIds === undefined ? isVecFastPathComplete(db) : vecFastPathWasReady;
+                setVecFastPathReady(db, vecFailedCount === 0 && vecUnavailableCount === 0 && vecGenerationComplete);
+            }
+            // A pass that completes (even one that did nothing but reuse) purges
+            // whatever is left — salvage is consumed by the NEXT pass, never kept
+            // around as a second cache.
+            purgeEmbeddingSalvage(db);
+            return reusedCount > 0 ? { success: true, vecInsertFailures: vecFailedCount } : { success: true };
+        }
+        // Cap each document's embedded text at
+        // embedding.maxInputTokens (default DEFAULT_MAX_INPUT_TOKENS, resolved
+        // once above so the canary uses the identical cap) instead of ever
+        // failing a whole batch over one oversized entry — truncation keeps the
+        // head of the text, unicode-safe. A document is skipped only when its
+        // head is empty (the impossible case: nothing left to embed), never
+        // merely for being long.
+        let truncatedCount = 0;
+        const texts = [];
+        const pendingEntries = [];
+        for (const entry of candidateEntries) {
+            const capped = capEmbeddingText(entry.searchText, maxInputTokens);
+            if (capped.text.length === 0)
+                continue;
+            if (capped.truncated)
+                truncatedCount++;
+            pendingEntries.push(entry);
+            texts.push(capped.text);
+        }
+        if (truncatedCount > 0) {
+            // Through onProgress ONLY, not warn() too — onProgress already reaches
+            // stderr at the default level in every output mode (#954), and the
+            // index CLI's progress handler writes it through info() (log-file
+            // aware), so calling warn() as well printed the identical sentence
+            // twice in text mode.
+            const message = `[embed] ${truncatedCount} entr${truncatedCount === 1 ? "y" : "ies"} truncated to the ${maxInputTokens}-token embedding cap (embedding.maxInputTokens); rerun with a higher cap to embed the full text.`;
+            onProgress({ phase: "embeddings", message });
         }
         if (rebuildReason) {
-            const message = `[embed] Re-embedding ${allEntries.length} entr${allEntries.length === 1 ? "y" : "ies"} because ${rebuildReason}`;
-            warn(message);
+            // See the truncation notice above: onProgress ONLY.
+            const message = `[embed] Re-embedding ${pendingEntries.length} entr${pendingEntries.length === 1 ? "y" : "ies"} because ${rebuildReason}`;
             onProgress({ phase: "embeddings", message });
         }
         onProgress({
             phase: "embeddings",
-            message: `Generating embeddings for ${allEntries.length} entr${allEntries.length === 1 ? "y" : "ies"}.`,
+            message: `Generating embeddings for ${pendingEntries.length} entr${pendingEntries.length === 1 ? "y" : "ies"}.`,
         });
-        const texts = allEntries.map((entry) => entry.searchText);
         if (isVerbose()) {
             // Mirror RemoteEmbedder's actual token-bounded batching (#874) so this
             // log reflects the real request grouping rather than a fixed count of
@@ -279,7 +439,9 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
             // for inference throughput only, never fails/skips), so there's
             // nothing meaningful to report per-batch for them.
             if (hasRemoteEndpoint(config.embedding ?? {})) {
-                const tokenBudget = config.embedding?.maxTokens ?? config.embedding?.contextLength ?? DEFAULT_TOKEN_BUDGET;
+                // Mirrors RemoteEmbedder.embedBatch's own tokenBudget resolution
+                // (#956: contextLength no longer feeds this).
+                const tokenBudget = config.embedding?.maxTokens ?? DEFAULT_TOKEN_BUDGET;
                 const maxCount = config.embedding?.batchSize ?? DEFAULT_REMOTE_BATCH_SIZE;
                 const batches = buildTokenBoundedBatches(texts, tokenBudget, maxCount);
                 const batchNumberByIndex = new Map();
@@ -287,7 +449,7 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
                     for (const i of batch.indices)
                         batchNumberByIndex.set(i, batchIdx + 1);
                 });
-                for (const [i, entry] of allEntries.entries()) {
+                for (const [i, entry] of pendingEntries.entries()) {
                     const chars = entry.searchText.length;
                     const tokens = estimateTokenCount(entry.searchText);
                     const batch = batches[batchNumberByIndex.get(i) - 1];
@@ -298,7 +460,7 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
                 }
             }
             else {
-                for (const entry of allEntries) {
+                for (const entry of pendingEntries) {
                     warnVerbose(`[embed] ${entry.itemRef} (${entry.searchText.length} chars, est. ${estimateTokenCount(entry.searchText)} tokens)`);
                 }
             }
@@ -307,15 +469,12 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
         let storedCount = 0;
         let skippedCount = 0;
         let embedFailedCount = 0;
-        let vecFailedCount = 0;
-        let vecUnavailableCount = 0;
         let storedTokens = 0;
-        let lastProgressBucket = 0;
         try {
             heartbeatTimer = setInterval(() => {
                 onProgress({
                     phase: "embeddings",
-                    message: `Still generating embeddings: ${storedCount}/${allEntries.length} stored; waiting on embedding provider.`,
+                    message: formatEmbeddingHeartbeat(storedCount, pendingEntries.length, embedFailedCount),
                 });
             }, 15000);
             // A failing sub-batch or an oversized document is SKIPPED by embedBatch,
@@ -323,6 +482,42 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
             // few bad documents don't discard every other entry's embedding.
             const skips = [];
             const embedStart = Date.now();
+            // Circuit breaker (#954): stop
+            // dispatching further batches once either consecutive-failure streak
+            // below reaches CIRCUIT_BREAKER_THRESHOLD — a dead/hung provider used
+            // to grind through every remaining batch for hours, one 30s (now
+            // configurable, and now backed off/retried/split first — see
+            // RemoteEmbedder.embedBatch) timeout at a time, ending in one
+            // aggregate warning and `ok: true`. Counted per BATCH
+            // (`skip.batchStart`), not per document: a single failed 100-document
+            // batch must not look like 100 consecutive failures.
+            let consecutiveSingleDocFailures = 0;
+            let consecutiveNetworkErrorFailures = 0;
+            let circuitBreakerReason;
+            const onSkip = (skip) => {
+                skips.push(skip);
+                if (!skip.batchStart)
+                    return undefined;
+                if (skip.reason === "context-window-exceeded") {
+                    consecutiveSingleDocFailures = 0;
+                    consecutiveNetworkErrorFailures = 0;
+                    return undefined;
+                }
+                // "batch-request-failed": a timeout only counts once retries have
+                // already narrowed it down to a single document (embedBatch backs
+                // off, retries, and splits a multi-document timeout before ever
+                // reporting it here); a network error counts immediately at any
+                // size — it was never retried, so it is trusted right away.
+                consecutiveSingleDocFailures = skip.batchSize === 1 ? consecutiveSingleDocFailures + 1 : 0;
+                consecutiveNetworkErrorFailures =
+                    skip.failureKind === "network-error" ? consecutiveNetworkErrorFailures + 1 : 0;
+                if (consecutiveSingleDocFailures >= CIRCUIT_BREAKER_THRESHOLD ||
+                    consecutiveNetworkErrorFailures >= CIRCUIT_BREAKER_THRESHOLD) {
+                    circuitBreakerReason = skip.message;
+                    return false;
+                }
+                return undefined;
+            };
             // Commit each provider batch in its own short transaction as it lands,
             // rather than buffering the whole run in memory for one transaction at
             // the very end (#954) — a competing-process lock error or any other
@@ -334,12 +529,56 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
             // is kept; every batch from one run shares the same provider/model.
             let observedModel;
             let observedVectorLen;
-            const onBatch = (indices, batchEmbeddings, model) => {
+            // Whether the remote provider's endpoint/model/token language is
+            // meaningful for this run — the per-batch diagnostic line below is
+            // remote-only, same gate the credential diagnostic (#953) above uses.
+            const reportPerBatchLine = hasRemoteEndpoint(config.embedding ?? {});
+            const onBatch = (indices, batchEmbeddings, model, outcome) => {
+                // #954 field-report follow-up: a "retrying" event carries nothing to
+                // commit — the request hasn't settled yet — only the notice that a
+                // back-off is about to be waited out, default-level so a run is
+                // never silently stalled indistinguishably from a hang.
+                if (outcome?.outcome === "retrying") {
+                    if (reportPerBatchLine) {
+                        onProgress({
+                            phase: "embeddings",
+                            message: `[embed] batch ${outcome.batchIndex}/${outcome.batchCount}: ${outcome.docCount} docs, ${outcome.requestTokens.toLocaleString()} tokens → retrying after ${(outcome.elapsedMs / 1000).toFixed(1)} s`,
+                        });
+                    }
+                    return;
+                }
+                // #954: a "budget-lowered" event is the same kind of notice as
+                // "retrying" above — the run's first context-size rejection just
+                // shrank the request budget for everything not yet dispatched, but
+                // THIS rejected batch's own indices are still being split and
+                // retried by the embedder (their real stored/failed outcome lands in
+                // a later onBatch call). Nothing here has settled, so it must never
+                // touch storage, only report the notice — one line, at most once per
+                // run.
+                if (outcome?.outcome === "budget-lowered") {
+                    if (reportPerBatchLine) {
+                        onProgress({
+                            phase: "embeddings",
+                            message: `[embed] batch ${outcome.batchIndex}/${outcome.batchCount}: ${outcome.docCount} docs, ${outcome.requestTokens.toLocaleString()} tokens → ${outcome.reason}`,
+                        });
+                    }
+                    return;
+                }
                 if (model)
                     observedModel = model;
+                // A batch that delivered at least one real embedding proves the
+                // provider is currently answering — reset both circuit-breaker
+                // streaks. (A wholly failed batch's `batchEmbeddings` are all
+                // `undefined`, per commitBatch's skip path, so this never
+                // re-triggers what onSkip just counted moments earlier.)
+                if (batchEmbeddings.some((embedding) => embedding !== undefined)) {
+                    consecutiveSingleDocFailures = 0;
+                    consecutiveNetworkErrorFailures = 0;
+                }
                 db.transaction(() => {
                     for (let k = 0; k < indices.length; k++) {
-                        const entry = allEntries[indices[k]];
+                        const index = indices[k];
+                        const entry = pendingEntries[index];
                         if (!entry)
                             continue;
                         const embedding = batchEmbeddings[k];
@@ -352,7 +591,13 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
                         const result = upsertEmbedding(db, entry.id, embedding);
                         if (result.stored) {
                             storedCount++;
-                            storedTokens += estimateTokenCount(entry.searchText);
+                            // #954: sum the estimate of the text actually sent —
+                            // `texts[index]` is the capped string `embedBatch` was handed,
+                            // parallel to `pendingEntries` by construction above (the
+                            // `entry` guard covers both) — not `entry.searchText`, which is
+                            // the pre-cap original and overstates throughput for every
+                            // entry over the cap.
+                            storedTokens += estimateTokenCount(texts[index]);
                         }
                         else {
                             skippedCount++;
@@ -363,29 +608,36 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
                             vecUnavailableCount++;
                     }
                 })();
-                const bucket = Math.floor(storedCount / PROGRESS_INTERVAL);
-                if (bucket > lastProgressBucket) {
-                    lastProgressBucket = bucket;
+                // Default level, one line per provider batch (#954, field-report
+                // follow-up): oversized documents never made a request
+                // (`reason === "oversized"`), so there is no batch outcome to
+                // report — they are covered by the run's final oversized-skip
+                // count and list instead.
+                if (outcome && outcome.reason !== "oversized" && reportPerBatchLine) {
+                    const elapsedSeconds = (outcome.elapsedMs / 1000).toFixed(1);
+                    const outcomeLabel = outcome.outcome === "stored"
+                        ? `${outcome.docCount} stored (${elapsedSeconds} s)`
+                        : `failed: ${outcome.reason}`;
                     onProgress({
                         phase: "embeddings",
-                        message: `Embedded ${storedCount}/${allEntries.length} entries.`,
+                        message: `[embed] batch ${outcome.batchIndex}/${outcome.batchCount}: ${outcome.docCount} docs, ${outcome.requestTokens.toLocaleString()} tokens → ${outcomeLabel}`,
                     });
                 }
+                // Every committed batch, not just every 500 stored entries (#954)
+                // — the prior bucketing left a non-verbose run silent
+                // for the entire embedding phase on anything smaller than 500
+                // entries, indistinguishable from a hang.
+                onProgress({
+                    phase: "embeddings",
+                    message: `Embedded ${storedCount}/${pendingEntries.length} entries.`,
+                });
             };
-            await embedBatch(texts, config.embedding, signal, (skip) => skips.push(skip), onBatch);
+            await embedBatch(texts, config.embedding, signal, onSkip, onBatch);
             throwIfAborted(signal);
             const elapsedSeconds = Math.max((Date.now() - embedStart) / 1000, 0.001);
             if (skippedCount > 0) {
                 warn(`[embed] ${skippedCount} embedding${skippedCount === 1 ? "" : "s"} skipped (entry deleted between queue and write)`);
             }
-            if (embedFailedCount > 0) {
-                const detail = skips
-                    .slice(0, 20)
-                    .map((skip) => `  - ${allEntries[skip.index]?.itemRef ?? skip.index} (${skip.reason}): ${skip.message}`)
-                    .join("\n");
-                const more = skips.length > 20 ? `\n  ...and ${skips.length - 20} more` : "";
-                warn(`[embed] ${embedFailedCount} embedding${embedFailedCount === 1 ? "" : "s"} could not be generated and ${embedFailedCount === 1 ? "was" : "were"} skipped:\n${detail}${more}`);
-            }
             const vecGenerationComplete = targetEntryIds === undefined ? isVecFastPathComplete(db) : vecFastPathWasReady;
             setVecFastPathReady(db, vecFailedCount === 0 && vecUnavailableCount === 0 && vecGenerationComplete);
             if (vecFailedCount > 0) {
@@ -395,14 +647,66 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
             }
             const entriesPerSec = storedCount / elapsedSeconds;
             const tokensPerSec = storedTokens / elapsedSeconds;
+            const totalStored = storedCount + reusedCount;
+            // #954, field-report follow-up: the final line
+            // reports every outcome, not just what was stored — counts come from
+            // the same collected `skips` the circuit breaker already uses,
+            // categorized by `reason`/`failureKind`. "oversized skipped" =
+            // context-window-exceeded (never fit any request, at any size);
+            // "timed out" = a batch-request-failed skip whose last attempt timed
+            // out (retries/splits already exhausted before this counted); "failed"
+            // = every other batch-request-failed skip (a genuine, never-retried
+            // network/HTTP failure).
+            const oversizedSkips = skips.filter((skip) => skip.reason === "context-window-exceeded");
+            const timedOutSkips = skips.filter((skip) => skip.reason === "batch-request-failed" && skip.failureKind === "timeout");
+            const failedSkips = skips.filter((skip) => skip.reason === "batch-request-failed" && skip.failureKind !== "timeout");
+            const throughputLine = reusedCount > 0
+                ? // #955: report reused and newly-embedded counts separately — the
+                    // rate figures below are provider throughput only (reuse is a
+                    // plain DB write, not provider work) and would be misleadingly
+                    // inflated if reused entries were folded into them.
+                    `Stored ${totalStored} embedding${totalStored === 1 ? "" : "s"} (${reusedCount} reused, ${storedCount} newly embedded) in ${elapsedSeconds.toFixed(1)}s (${entriesPerSec.toFixed(1)} entries/s, ~${Math.round(tokensPerSec)} tokens/s)`
+                : `Stored ${storedCount} embedding${storedCount === 1 ? "" : "s"} in ${elapsedSeconds.toFixed(1)}s (${entriesPerSec.toFixed(1)} entries/s, ~${Math.round(tokensPerSec)} tokens/s)`;
             onProgress({
                 phase: "embeddings",
-                message: `Stored ${storedCount} embedding${storedCount === 1 ? "" : "s"} in ${elapsedSeconds.toFixed(1)}s (${entriesPerSec.toFixed(1)} entries/s, ~${Math.round(tokensPerSec)} tokens/s).`,
+                message: `${throughputLine}; ${oversizedSkips.length} oversized skipped, ${timedOutSkips.length} timed out, ${failedSkips.length} failed.`,
             });
+            // Bounded itemRef-level detail for every skip category, not just
+            // oversized — the aggregate counts above say HOW MANY documents timed
+            // out or failed, but give the operator no way to find out WHICH ones
+            // short of rerunning with --verbose and re-reading the whole log.
+            // Default level caps each list (there is nothing actionable about the
+            // 21st identical failure); --verbose prints every one, matching the
+            // per-document mapping lines' own verbosity gate above.
+            const printSkipList = (label, skipList) => {
+                if (skipList.length === 0)
+                    return;
+                const limit = isVerbose() ? skipList.length : 20;
+                const listed = skipList
+                    .slice(0, limit)
+                    .map((skip) => `  - ${pendingEntries[skip.index]?.itemRef ?? skip.index}: ${skip.message}`)
+                    .join("\n");
+                const more = skipList.length > limit ? `\n  ...and ${skipList.length - limit} more` : "";
+                onProgress({ phase: "embeddings", message: `[embed] ${label} skipped:\n${listed}${more}` });
+            };
+            printSkipList("oversized documents", oversizedSkips);
+            printSkipList("timed-out documents", timedOutSkips);
+            printSkipList("failed documents", failedSkips);
             setMeta(db, "embeddingFingerprint", currentFingerprint);
             const observedIdentity = deriveObservedEmbeddingIdentity(config.embedding, observedModel, observedVectorLen);
             if (observedIdentity)
                 setMeta(db, "embeddingIdentity", observedIdentity);
+            // Circuit breaker tripped (#954): committed batches are
+            // kept (nothing above discards them), but the pass is not a success —
+            // the provider looks dead, not just occasionally flaky.
+            if (circuitBreakerReason !== undefined) {
+                const message = `embedding provider failed ${CIRCUIT_BREAKER_THRESHOLD} consecutive batches ` +
+                    `(last: ${circuitBreakerReason}); stopped after ${storedCount} embedding${storedCount === 1 ? "" : "s"} ` +
+                    "were stored — rerun akm index when the endpoint is healthy";
+                warn(`[embed] ${message}`);
+                onProgress({ phase: "embeddings", message });
+                return { success: false, message, vecInsertFailures: vecFailedCount };
+            }
             // Only a total failure (nothing at all embedded, despite having entries
             // to embed) turns into a phase failure. Any partial success — the vast
             // majority of a large bundle embedding fine around a handful of skips —
@@ -416,6 +720,10 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
                     message: `All ${embedFailedCount} embedding batch(es) failed: ${firstMessage}`,
                 };
             }
+            // A pass that completes without abort or circuit-break purges
+            // whatever salvage is left — consumed by this pass's reuse step
+            // above, or superseded by what it just embedded.
+            purgeEmbeddingSalvage(db);
             return { success: true, vecInsertFailures: vecFailedCount };
         }
         finally {

package/dist/indexer/search/search-source.js

@@ -238,6 +238,7 @@ export async function ensureSourceCaches(config, options) {
     const cfg = config ?? loadConfig();
     const force = options?.force === true;
     const materialize = options?.materialize !== false;
+    const onProgress = options?.onProgress ?? (() => { });
     // Polymorphic refresh: walk every enabled source through its registered
     // provider and call `sync()`. Every cache-backed kind (git, website, npm)
     // refreshes the same way — a bad source warns and is skipped without
@@ -249,6 +250,12 @@ export async function ensureSourceCaches(config, options) {
     // content ENDS UP; the lock's `localRoot` merely records the result. So this
     // path correctly uses the provider, not the shared `lockContentRootFor`
     // resolver that reads/writes use to agree on where content already IS.
+    //
+    // Two passes: first resolve which sources will actually sync (constructing
+    // each provider once, exactly as before), so the progress count ("i/n")
+    // reflects real syncs rather than every configured entry including managed/
+    // unsyncable ones; then sync them in order, reporting progress per source.
+    const toSync = [];
     for (const entry of getSources(cfg)) {
         if (entry.enabled === false)
             continue;
@@ -282,12 +289,27 @@ export async function ensureSourceCaches(config, options) {
             warnIfSourceUnavailableForRead(entry, provider.name);
             continue;
         }
+        toSync.push(provider);
+    }
+    for (const [i, provider] of toSync.entries()) {
+        onProgress(`Hydrating source ${i + 1}/${toSync.length}: ${provider.name}`);
+        let heartbeat;
         try {
-            await provider.sync({ force, secrets: options?.secrets, ensureWebsiteMirror });
+            heartbeat = setInterval(() => {
+                onProgress(`Still hydrating source ${i + 1}/${toSync.length}: ${provider.name}...`);
+            }, 15000);
+            // `toSync` only ever holds providers whose `.sync` passed the check
+            // above; the optional-chain here is just to satisfy the type (the
+            // narrowing does not survive the array round-trip).
+            await provider.sync?.({ force, secrets: options?.secrets, ensureWebsiteMirror });
         }
         catch (err) {
             warn(`Warning: failed to refresh ${provider.kind} source "${provider.name}": ${err instanceof Error ? err.message : String(err)}`);
         }
+        finally {
+            if (heartbeat)
+                clearInterval(heartbeat);
+        }
     }
 }
 /**