akm-cli 0.9.16 → 0.9.17-alpha.10

package/dist/indexer/materialize-embeddings.js

@@ -6,26 +6,29 @@ import { isVerbose, warn, warnVerbose } from "../core/warn.js";
 import { embedBatch } from "../llm/embedder.js";
 import { DETERMINISTIC_EMBED_MODEL_ID, isDeterministicEmbedEnabled } from "../llm/embedders/deterministic.js";
 import { DEFAULT_LOCAL_MODEL } from "../llm/embedders/local.js";
-import { buildTokenBoundedBatches, capEmbeddingText, DEFAULT_MAX_INPUT_TOKENS, DEFAULT_REMOTE_BATCH_SIZE, DEFAULT_TOKEN_BUDGET, describeEmbeddingCredential, estimateTokenCount, hasRemoteEndpoint, normalizeEmbeddingEndpoint, } from "../llm/embedders/remote.js";
-import { cosineSimilarity } from "../llm/embedders/types.js";
-import { purgeEmbeddingSalvage, relabelEmbeddingSalvageFingerprint, reuseSalvagedEmbeddings, } from "../storage/repositories/embedding-salvage-repository.js";
+import { applyEmbeddingTemplate, documentTemplateFingerprint, resolveEmbeddingProfile } from "../llm/embedders/profile.js";
+import { capEmbeddingText, DEFAULT_MAX_INPUT_TOKENS, describeEmbeddingCredential, estimateTokenCount, hasRemoteEndpoint, normalizeEmbeddingEndpoint, } from "../llm/embedders/remote.js";
 import { getEmbeddableEntryCount } from "../storage/repositories/index-entries-repository.js";
-import { deleteMeta, getMeta, setMeta } from "../storage/repositories/index-meta-repository.js";
-import { EMBEDDING_DIM } from "../storage/repositories/index-schema.js";
-import { getAllEntriesForEmbedding, getEmbeddingCount, isVecFastPathComplete, isVecFastPathReady, purgeEmbeddings, repairVecFastPath, sampleEmbeddedEntriesForCanary, setVecFastPathReady, upsertEmbedding, } from "../storage/repositories/index-vec-repository.js";
+import { getMeta, setMeta } from "../storage/repositories/index-meta-repository.js";
+import { getAllEntriesForEmbedding, getEmbeddingCount, purgeEmbeddings, upsertEmbedding, } from "../storage/repositories/index-vec-repository.js";
 import { reclassifyIndexDbContention } from "./index-db-contention.js";
-/** Identifies the embedding provider+model+dimension a stored vector was generated with. */
+/**
+ * Identifies what a stored vector was generated with: provider, model and
+ * dimension, plus the document template when the embedding profile has one
+ * (so changing it re-embeds the index).
+ */
 export function deriveSemanticProviderFingerprint(embedding) {
     if (isDeterministicEmbedEnabled()) {
         return `deterministic:${DETERMINISTIC_EMBED_MODEL_ID}`;
     }
+    const template = documentTemplateFingerprint(embedding);
     if (embedding?.endpoint) {
         // Fingerprint keys on vector identity only (model + dimension). The endpoint
         // is transport/routing and has no bearing on vector compatibility, so moving
-        // the same model+dimension to a different host must not force a full re-embed.
-        return `remote:${embedding.model}|${embedding.dimension ?? "default"}`;
+        // the same model+dimension to a different host must not force a re-embed.
+        return `remote:${embedding.model}|${embedding.dimension ?? "default"}${template}`;
     }
-    return `local:${embedding?.localModel ?? DEFAULT_LOCAL_MODEL}`;
+    return `local:${embedding?.localModel ?? DEFAULT_LOCAL_MODEL}${template}`;
 }
 /**
  * The heartbeat text emitted every 15s while a provider request is in
@@ -35,19 +38,6 @@ export function deriveSemanticProviderFingerprint(embedding) {
 export function formatEmbeddingHeartbeat(storedCount, total, failedCount) {
     return `Still generating embeddings: ${storedCount}/${total} stored, ${failedCount} failed; waiting on embedding provider.`;
 }
-/**
- * Number of already-embedded entries sampled for the fingerprint-rename
- * canary (#955) — small and cheap even against a slow local server; a
- * handful of chunks is a strong compatibility signal (a different model
- * cannot plausibly land near-identical vectors by chance).
- */
-const CANARY_SAMPLE_SIZE = 8;
-/**
- * Minimum median cosine similarity between stored and freshly re-embedded
- * canary vectors for a fingerprint-string change to be treated as a
- * same-model rename rather than a real model change (#955).
- */
-const CANARY_SIMILARITY_THRESHOLD = 0.999;
 /**
  * Consecutive transport failures after which the embedding pass stops
  * dispatching further requests and ends the run as a failure rather than
@@ -65,152 +55,6 @@ const CANARY_SIMILARITY_THRESHOLD = 0.999;
  * streaks instead.
  */
 const CIRCUIT_BREAKER_THRESHOLD = 3;
-/**
- * Pure decision: do stored vectors remain valid against freshly re-embedded
- * canary samples? The ONE place that computes the canary's similarity
- * numbers — callers must use this result rather than recomputing it (#955).
- *
- * An empty sample means nothing is stored to lose or verify against, so
- * there is nothing to decide — keep.
- *
- * A sample whose re-embed FAILED (`fresh === undefined`, e.g. a provider
- * sub-batch that was skipped) is EXCLUDED from the similarity computation
- * entirely, not scored as zero: a partial provider failure is not evidence
- * of a different model (#955). A dimension mismatch on a successful
- * re-embed still counts as zero similarity via {@link cosineSimilarity}'s
- * own dimension-mismatch guard — that IS evidence. When half or fewer of
- * the sampled entries re-embedded successfully, the sample is too thin to
- * trust either verdict — the outcome is `unverifiable`, the same outcome a
- * total canary failure already produces.
- *
- * Otherwise the MEDIAN pairwise cosine similarity of the verified samples
- * must clear {@link CANARY_SIMILARITY_THRESHOLD}; the median (not the
- * minimum or mean) tolerates one stale or lightly-edited sample without
- * either discarding a real match or being fooled by it.
- */
-export function decideEmbeddingCompatibility(pairs) {
-    if (pairs.length === 0)
-        return { outcome: "keep", medianSimilarity: undefined, verifiedSamples: 0 };
-    const verified = pairs.filter((pair) => pair.fresh !== undefined);
-    if (verified.length * 2 <= pairs.length) {
-        return { outcome: "unverifiable", medianSimilarity: undefined, verifiedSamples: verified.length };
-    }
-    const similarities = verified.map((pair) => cosineSimilarity(pair.stored, pair.fresh));
-    const medianSimilarity = medianOf(similarities);
-    return {
-        outcome: medianSimilarity >= CANARY_SIMILARITY_THRESHOLD ? "keep" : "rebuild",
-        medianSimilarity,
-        verifiedSamples: verified.length,
-    };
-}
-function medianOf(values) {
-    const sorted = [...values].sort((a, b) => a - b);
-    const mid = Math.floor(sorted.length / 2);
-    return sorted.length % 2 === 0
-        ? (sorted[mid - 1] + sorted[mid]) / 2
-        : sorted[mid];
-}
-/**
- * Identity of the embedding vectors actually observed on a run — as opposed
- * to {@link deriveSemanticProviderFingerprint}'s CONFIG-derived string. Keys
- * on what the server (or local model) actually reported plus the observed
- * vector width, so a gateway/transport change that keeps returning the same
- * underlying model can be told apart from a genuine model change without
- * relying on the operator's config string (#955).
- * Returns undefined when nothing was actually observed this call (no vector
- * to measure yet).
- */
-function deriveObservedEmbeddingIdentity(embedding, observedModel, observedVectorLen) {
-    if (isDeterministicEmbedEnabled()) {
-        return `deterministic:${DETERMINISTIC_EMBED_MODEL_ID}`;
-    }
-    if (observedVectorLen === undefined)
-        return undefined;
-    if (embedding?.endpoint) {
-        return `remote:${observedModel ?? embedding.model ?? "unknown"}|${observedVectorLen}`;
-    }
-    return `local:${embedding?.localModel ?? DEFAULT_LOCAL_MODEL}|${observedVectorLen}`;
-}
-/**
- * Run the fingerprint-rename canary: re-embed a small sample of already-
- * stored entries with the CURRENT config and decide whether the stored
- * index survives. Goes through the standard {@link embedBatch} facade (not a
- * direct `RemoteEmbedder`) so every embedder branch — remote, local,
- * deterministic, and test overrides via `_setEmbedderForTests` — is
- * exercised identically to the main embedding pass.
- *
- * `maxInputTokens` must be the SAME cap the main pass below applies via
- * {@link capEmbeddingText} — the stored vector for each sampled entry was
- * produced from its capped text, so comparing against a fresh vector of the
- * uncapped text would compare unlike inputs for any entry over the cap
- * (#955).
- */
-async function runEmbeddingCanary(db, config, signal, maxInputTokens) {
-    const samples = sampleEmbeddedEntriesForCanary(db, CANARY_SAMPLE_SIZE);
-    if (samples.length === 0) {
-        return { outcome: "keep", verified: false, viaIdentityMatch: false };
-    }
-    let observedModel;
-    const skips = [];
-    let canaryVectors;
-    try {
-        canaryVectors = await embedBatch(
-        // #955: the stored vector for each sample was produced from
-        // capEmbeddingText(searchText, maxInputTokens) — the main pass below
-        // caps every document before embedding it. The canary must re-embed
-        // the SAME capped text, or an entry over the cap compares a fresh
-        // vector of a different input against a stored vector of the capped
-        // one, and a genuine model match can read as a rebuild-worthy
-        // mismatch for reasons unrelated to the model.
-        samples.map((sample) => capEmbeddingText(sample.searchText, maxInputTokens).text), config.embedding, signal, (skip) => skips.push(skip), (_indices, _embeddings, model) => {
-            if (model)
-                observedModel = model;
-        });
-    }
-    catch (error) {
-        const message = error instanceof Error ? error.message : String(error);
-        return {
-            outcome: "unverifiable",
-            message: `could not verify embedding compatibility (${message}); keeping existing vectors — rerun akm index when the endpoint is reachable`,
-        };
-    }
-    const observedVectorLen = canaryVectors.find((vector) => vector !== undefined)?.length;
-    const observedIdentity = deriveObservedEmbeddingIdentity(config.embedding, observedModel, observedVectorLen);
-    const storedIdentity = getMeta(db, "embeddingIdentity");
-    if (storedIdentity && observedIdentity && storedIdentity === observedIdentity) {
-        // The server reports the same model identity as last time — no need to
-        // even look at the cosines; the config string alone was misleading.
-        return { outcome: "keep", verified: true, identity: observedIdentity, viaIdentityMatch: true };
-    }
-    const pairs = samples.map((sample, i) => ({ stored: sample.vector, fresh: canaryVectors[i] }));
-    const decision = decideEmbeddingCompatibility(pairs);
-    if (decision.outcome === "unverifiable") {
-        // Covers both a total provider failure (RemoteEmbedder skips a failing
-        // request rather than throwing, #874, so an unreachable endpoint
-        // surfaces here as an all-`undefined` canary result, not a caught
-        // exception) and a partial one thin enough that neither verdict can be
-        // trusted (#955) — same message path either way.
-        const message = skips[0]?.message ?? "embedding provider returned no vectors for the canary sample";
-        return {
-            outcome: "unverifiable",
-            message: `could not verify embedding compatibility (${message}); keeping existing vectors — rerun akm index when the endpoint is reachable`,
-        };
-    }
-    if (decision.outcome === "keep") {
-        return {
-            outcome: "keep",
-            verified: true,
-            identity: observedIdentity,
-            viaIdentityMatch: false,
-            medianSimilarity: decision.medianSimilarity,
-        };
-    }
-    return {
-        outcome: "rebuild",
-        identity: observedIdentity,
-        reason: `vectors differ (median similarity ${decision.medianSimilarity?.toFixed(3)})`,
-    };
-}
 function throwIfAborted(signal) {
     if (signal?.aborted) {
         throw signal.reason instanceof Error ? signal.reason : new Error("index interrupted");
@@ -223,10 +67,9 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
     // unobservable SAVEPOINT instead, so an interruption (competing-process
     // collision, SIGKILL) could lose the whole pass rather than only the batch
     // in flight. This is an internal contract error (a caller bug), not a
-    // user-facing failure class: callers with their own transaction (e.g. `akm
-    // bundle update`'s unified update transaction) must run the embedding
-    // phase on a separate connection AFTER their own transaction commits — see
-    // `runEmbeddingPass` in `src/indexer/indexer.ts`.
+    // user-facing failure class: a caller with its own transaction must run the
+    // embedding phase on a separate connection AFTER that transaction commits —
+    // see `runEmbeddingPass` in `src/indexer/indexer.ts`.
     if (db.inTransaction) {
         throw new Error("generateEmbeddingsForDb was called with an ambient transaction already open on `db`: per-batch commits " +
             "would become SAVEPOINTs inside it, losing the crash-durability contract per-batch commit exists for. " +
@@ -234,11 +77,6 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
     }
     throwIfAborted(signal);
     if (config.semanticSearchMode === "off") {
-        // #955: salvage is self-emptying only if every path that skips reuse
-        // also drains it — otherwise a full rebuild performed with semantic
-        // search disabled leaves permanent orphaned rows behind (nothing will
-        // ever consume them, since this path never reaches the reuse step).
-        purgeEmbeddingSalvage(db);
         onProgress({ phase: "embeddings", message: "Semantic search disabled; skipping embeddings." });
         return { success: false, message: "Semantic search is disabled." };
     }
@@ -246,8 +84,7 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
     // fix (every RemoteEmbedder path already resolves secret:// through one
     // boundary — a keyless request can only mean embedding.apiKey was absent
     // from the config THIS run loaded). One default-level line, before the
-    // first provider request of the phase (the canary probe or the main
-    // pass, whichever runs first below), naming the endpoint/model/credential
+    // first provider request of the phase, naming the endpoint/model/credential
     // SOURCE — never the credential value.
     if (hasRemoteEndpoint(config.embedding ?? {})) {
         const endpoint = normalizeEmbeddingEndpoint(config.embedding?.endpoint ?? "");
@@ -258,167 +95,56 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
             message: `[embed] endpoint ${endpoint}, model ${config.embedding?.model ?? "unknown"}; credential: ${credential}${configFileSuffix}`,
         });
     }
-    // A targeted call starts from an already-published generation. Preserve its
-    // trust decision in O(1): successful writes for the changed IDs keep a
-    // healthy fast path healthy, but can never promote a generation already
-    // marked degraded. Global runs can afford to verify the entire derived set.
-    const vecFastPathWasReady = isVecFastPathReady(db);
-    const currentFingerprint = deriveSemanticProviderFingerprint(config.embedding);
-    const storedFingerprint = getMeta(db, "embeddingFingerprint");
+    const model = deriveSemanticProviderFingerprint(config.embedding);
+    const storedModel = getMeta(db, "embeddingFingerprint");
     let targetEntryIds = entryIds;
-    /** Set only on an actual rebuild, so the up-front "Re-embedding N entries" line names why. */
+    /** Set only when every entry is (re)embedded, so the up-front "Re-embedding N entries" line names why. */
     let rebuildReason;
-    // Resolved once and reused by both the canary (below) and the main pass's
-    // cap loop (further down) — the same cap must apply to both, or the canary
-    // compares a differently-capped text against the stored vector (#955).
     const maxInputTokens = config.embedding?.maxInputTokens ?? DEFAULT_MAX_INPUT_TOKENS;
     if (opts?.forceReembed) {
-        // `akm index --reembed`: an explicit operator override, skips the canary
-        // entirely. The new fingerprint (and identity, now stale/unknown until
-        // the next successful pass observes it) is written in the SAME
-        // transaction as the purge, before any embedding request — a restart
-        // then sees a matching fingerprint and only heals what is still missing
-        // instead of purging again from zero (#955/#956).
+        // The explicit operator override: purge and re-embed everything. The model
+        // is recorded in the SAME transaction as the purge, before any embedding
+        // request, so a restart heals only what is still missing instead of
+        // purging again from zero (#955/#956).
         db.transaction(() => {
-            purgeEmbeddings(db, { dropVecTable: true });
-            // #955: an explicit forced rebuild must re-embed everything, not
-            // quietly satisfy some of it from stale salvage.
-            purgeEmbeddingSalvage(db);
-            deleteMeta(db, "embeddingDim");
-            setMeta(db, "embeddingFingerprint", currentFingerprint);
-            deleteMeta(db, "embeddingIdentity");
+            purgeEmbeddings(db);
+            setMeta(db, "embeddingFingerprint", model);
         })();
         targetEntryIds = undefined;
         rebuildReason = "forced by --reembed";
     }
-    else if (storedFingerprint && storedFingerprint !== currentFingerprint) {
-        const decision = await runEmbeddingCanary(db, config, signal, maxInputTokens);
-        if (decision.outcome === "unverifiable") {
-            // Destroying a good index because the server happens to be down right
-            // now is worse than leaving a rename unverified until the next run —
-            // keep the vectors AND the old fingerprint so the next `akm index`
-            // retries the canary instead of silently treating this as resolved.
-            warn(`[embed] ${decision.message}`);
-            onProgress({ phase: "embeddings", message: decision.message });
-            return { success: false, message: decision.message };
-        }
-        if (decision.outcome === "rebuild") {
-            db.transaction(() => {
-                purgeEmbeddings(db, { dropVecTable: true });
-                // #955: the stored vectors AND any leftover salvage both belong to
-                // a different model now — neither is reusable, so both go.
-                purgeEmbeddingSalvage(db);
-                deleteMeta(db, "embeddingDim");
-                setMeta(db, "embeddingFingerprint", currentFingerprint);
-                if (decision.identity)
-                    setMeta(db, "embeddingIdentity", decision.identity);
-                else
-                    deleteMeta(db, "embeddingIdentity");
-            })();
-            targetEntryIds = undefined;
-            rebuildReason = decision.reason;
-        }
-        else {
-            // Keep: adopt the new fingerprint (and identity, when observed)
-            // immediately rather than deferring to end-of-run — nothing was
-            // purged, so there is nothing an interruption could lose, and an
-            // immediate write means a crash right after this decision does not
-            // re-run the canary needlessly on the next attempt.
-            setMeta(db, "embeddingFingerprint", currentFingerprint);
-            // #955: the model did not actually change, only the fingerprint
-            // STRING did (e.g. a gateway rename) — any leftover salvage rows
-            // tagged with the OLD string are still valid vectors. Relabel them so
-            // the reuse step below (and any later pass) can still find them.
-            relabelEmbeddingSalvageFingerprint(db, storedFingerprint, currentFingerprint);
-            if (decision.identity)
-                setMeta(db, "embeddingIdentity", decision.identity);
-            if (decision.verified) {
-                const keptCount = getEmbeddingCount(db);
-                const detail = decision.viaIdentityMatch
-                    ? "server-reported model unchanged"
-                    : `stored vectors are compatible (median similarity ${decision.medianSimilarity?.toFixed(3)})`;
-                const message = `[embed] embedding model renamed (${storedFingerprint} → ${currentFingerprint}); ${detail}, keeping ${keptCount} embedding${keptCount === 1 ? "" : "s"}.`;
-                warn(message);
-                onProgress({ phase: "embeddings", message });
-            }
-            // Empty-sample case (decision.verified === false): nothing stored to
-            // lose or verify against — adopt the label silently, no purge line.
-        }
+    else if (storedModel && storedModel !== model) {
+        // The configured model changed. Nothing is purged: each stored row keeps
+        // the model that produced it until the pass below replaces it, so an
+        // interrupted pass resumes with only the rows still on the old model.
+        // Readers serve the current model's rows only, so nothing mixes vectors
+        // from two models.
+        setMeta(db, "embeddingFingerprint", model);
+        targetEntryIds = undefined;
+        rebuildReason = `the embedding model changed (${storedModel} → ${model}); stored vectors are kept until each entry is re-embedded`;
     }
     else {
-        // No rename to verify (either this is the very first
-        // pass ever for this db, or the fingerprint already matches the last
-        // successful one) — still record it NOW rather than deferring to a
-        // fully successful pass, mirroring the rebuild/keep branches above
-        // (#955/#956). Without this, an interrupted FIRST-EVER pass left
-        // `embeddingFingerprint` unset despite a per-batch commit below (#954)
-        // already having durably written real vectors — a later `akm index
-        // --full`'s salvage-before-discard step tags rows by this meta
-        // (`salvageEmbeddingsBeforeDiscard`) and treats an unset fingerprint as
-        // "nothing was ever verified", silently turning genuinely-embedded
-        // vectors into a full re-embed instead of a salvage-and-reuse.
-        setMeta(db, "embeddingFingerprint", currentFingerprint);
+        // First pass ever for this index, or the model already matches — record
+        // it NOW rather than after a fully successful pass (#955/#956), so an
+        // interrupted first pass still labels the vectors it durably committed.
+        setMeta(db, "embeddingFingerprint", model);
     }
     try {
         throwIfAborted(signal);
-        if (entryIds === undefined && (!vecFastPathWasReady || !isVecFastPathComplete(db))) {
-            const storedDim = Number(getMeta(db, "embeddingDim"));
-            const expectedDim = Number.isInteger(storedDim) && storedDim > 0 ? storedDim : (config.embedding?.dimension ?? EMBEDDING_DIM);
-            const repair = repairVecFastPath(db, expectedDim);
-            if (repair.available &&
-                (repair.repaired > 0 || repair.removedOrphans > 0 || repair.rejected > 0 || repair.error !== undefined)) {
-                const detail = repair.error ? `; repair stopped: ${repair.error}` : "";
-                onProgress({
-                    phase: "embeddings",
-                    message: `[embed] Repaired ${repair.repaired} missing sqlite-vec row${repair.repaired === 1 ? "" : "s"}; removed ${repair.removedOrphans} orphan${repair.removedOrphans === 1 ? "" : "s"}; ${repair.rejected} rejected${detail}.`,
-                });
-            }
-        }
-        const allEntries = getAllEntriesForEmbedding(db, targetEntryIds);
-        let vecFailedCount = 0;
-        let vecUnavailableCount = 0;
-        // #955: before any provider call, hand back vectors salvaged from a
-        // full rebuild or a generation bump for entries whose search_text is
-        // byte-identical to what was salvaged under the SAME fingerprint — a
-        // fingerprint mismatch or a single-byte content change both correctly
-        // fall through to the provider below instead.
-        const { reusedCount, remaining: candidateEntries } = reuseSalvagedEmbeddings(db, allEntries, currentFingerprint, (entry, embedding) => {
-            const result = upsertEmbedding(db, entry.id, embedding);
-            if (result.vec === "failed")
-                vecFailedCount++;
-            if (result.vec === "unavailable")
-                vecUnavailableCount++;
-            return result.stored;
-        });
-        if (reusedCount > 0) {
-            onProgress({
-                phase: "embeddings",
-                message: `Reused ${reusedCount} embedding${reusedCount === 1 ? "" : "s"} from the previous generation; embedding ${candidateEntries.length} new.`,
-            });
-        }
+        const candidateEntries = getAllEntriesForEmbedding(db, targetEntryIds, model);
         if (candidateEntries.length === 0) {
             onProgress({ phase: "embeddings", message: "Embeddings already up to date." });
-            setMeta(db, "embeddingFingerprint", currentFingerprint);
-            if (reusedCount > 0) {
-                const vecGenerationComplete = targetEntryIds === undefined ? isVecFastPathComplete(db) : vecFastPathWasReady;
-                setVecFastPathReady(db, vecFailedCount === 0 && vecUnavailableCount === 0 && vecGenerationComplete);
-            }
-            // A pass that completes (even one that did nothing but reuse) purges
-            // whatever is left — salvage is consumed by the NEXT pass, never kept
-            // around as a second cache.
-            purgeEmbeddingSalvage(db);
-            return reusedCount > 0 ? { success: true, vecInsertFailures: vecFailedCount } : { success: true };
+            return { success: true };
         }
-        // Cap each document's embedded text at
-        // embedding.maxInputTokens (default DEFAULT_MAX_INPUT_TOKENS, resolved
-        // once above so the canary uses the identical cap) instead of ever
-        // failing a whole batch over one oversized entry — truncation keeps the
-        // head of the text, unicode-safe. A document is skipped only when its
-        // head is empty (the impossible case: nothing left to embed), never
-        // merely for being long.
+        // Cap each document's embedded text at embedding.maxInputTokens (default
+        // DEFAULT_MAX_INPUT_TOKENS) instead of ever failing a whole batch over one
+        // oversized entry — truncation keeps the head of the text, unicode-safe. A
+        // document is skipped only when its head is empty (the impossible case:
+        // nothing left to embed), never merely for being long.
         let truncatedCount = 0;
         const texts = [];
         const pendingEntries = [];
+        const { documentTemplate } = resolveEmbeddingProfile(config.embedding);
         for (const entry of candidateEntries) {
             const capped = capEmbeddingText(entry.searchText, maxInputTokens);
             if (capped.text.length === 0)
@@ -426,7 +152,7 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
             if (capped.truncated)
                 truncatedCount++;
             pendingEntries.push(entry);
-            texts.push(capped.text);
+            texts.push(applyEmbeddingTemplate(documentTemplate, capped.text));
         }
         if (truncatedCount > 0) {
             // Through onProgress ONLY, not warn() too — onProgress already reaches
@@ -446,38 +172,11 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
             phase: "embeddings",
             message: `Generating embeddings for ${pendingEntries.length} entr${pendingEntries.length === 1 ? "y" : "ies"}.`,
         });
+        // Per-document sizes; the provider's actual request grouping is reported
+        // per batch below (and each skipped document is listed at the end).
         if (isVerbose()) {
-            // Mirror RemoteEmbedder's actual token-bounded batching (#874) so this
-            // log reflects the real request grouping rather than a fixed count of
-            // 100 that no longer matches what gets sent over the wire. Local runs
-            // don't batch by size at all (LocalEmbedder chunks by a fixed count
-            // for inference throughput only, never fails/skips), so there's
-            // nothing meaningful to report per-batch for them.
-            if (hasRemoteEndpoint(config.embedding ?? {})) {
-                // Mirrors RemoteEmbedder.embedBatch's own tokenBudget resolution
-                // (#956: contextLength no longer feeds this).
-                const tokenBudget = config.embedding?.maxTokens ?? DEFAULT_TOKEN_BUDGET;
-                const maxCount = config.embedding?.batchSize ?? DEFAULT_REMOTE_BATCH_SIZE;
-                const batches = buildTokenBoundedBatches(texts, tokenBudget, maxCount);
-                const batchNumberByIndex = new Map();
-                batches.forEach((batch, batchIdx) => {
-                    for (const i of batch.indices)
-                        batchNumberByIndex.set(i, batchIdx + 1);
-                });
-                for (const [i, entry] of pendingEntries.entries()) {
-                    const chars = entry.searchText.length;
-                    const tokens = estimateTokenCount(entry.searchText);
-                    const batch = batches[batchNumberByIndex.get(i) - 1];
-                    const label = batch?.oversized
-                        ? "oversized (skipped)"
-                        : `batch ${batchNumberByIndex.get(i)}/${batches.length}`;
-                    warnVerbose(`[embed] ${entry.itemRef} (${chars} chars, est. ${tokens} tokens) → ${label}`);
-                }
-            }
-            else {
-                for (const entry of pendingEntries) {
-                    warnVerbose(`[embed] ${entry.itemRef} (${entry.searchText.length} chars, est. ${estimateTokenCount(entry.searchText)} tokens)`);
-                }
+            for (const entry of pendingEntries) {
+                warnVerbose(`[embed] ${entry.itemRef} (${entry.searchText.length} chars, est. ${estimateTokenCount(entry.searchText)} tokens)`);
             }
         }
         let heartbeatTimer;
@@ -538,17 +237,11 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
             // the very end (#954) — a competing-process lock error or any other
             // interruption partway through now keeps whatever already committed
             // instead of losing the entire pass.
-            // Tracks what this run actually observed, so a successful pass can
-            // record `embeddingIdentity` from real data rather than the config
-            // string alone (#955) — only the first non-empty batch's vector width
-            // is kept; every batch from one run shares the same provider/model.
-            let observedModel;
-            let observedVectorLen;
             // Whether the remote provider's endpoint/model/token language is
             // meaningful for this run — the per-batch diagnostic line below is
             // remote-only, same gate the credential diagnostic (#953) above uses.
             const reportPerBatchLine = hasRemoteEndpoint(config.embedding ?? {});
-            const onBatch = (indices, batchEmbeddings, model, outcome) => {
+            const onBatch = (indices, batchEmbeddings, outcome) => {
                 // #954 field-report follow-up: a "retrying" event carries nothing to
                 // commit — the request hasn't settled yet — only the notice that a
                 // back-off is about to be waited out, default-level so a run is
@@ -579,8 +272,6 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
                     }
                     return;
                 }
-                if (model)
-                    observedModel = model;
                 // A batch that delivered at least one real embedding proves the
                 // provider is currently answering — reset both circuit-breaker
                 // streaks. (A wholly failed batch's `batchEmbeddings` are all
@@ -601,10 +292,7 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
                             embedFailedCount++;
                             continue;
                         }
-                        if (observedVectorLen === undefined)
-                            observedVectorLen = embedding.length;
-                        const result = upsertEmbedding(db, entry.id, embedding);
-                        if (result.stored) {
+                        if (upsertEmbedding(db, entry.id, embedding, model)) {
                             storedCount++;
                             // #954: sum the estimate of the text actually sent —
                             // `texts[index]` is the capped string `embedBatch` was handed,
@@ -617,10 +305,6 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
                         else {
                             skippedCount++;
                         }
-                        if (result.vec === "failed")
-                            vecFailedCount++;
-                        if (result.vec === "unavailable")
-                            vecUnavailableCount++;
                     }
                 })();
                 // Default level, one line per provider batch (#954, field-report
@@ -653,16 +337,8 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
             if (skippedCount > 0) {
                 warn(`[embed] ${skippedCount} embedding${skippedCount === 1 ? "" : "s"} skipped (entry deleted between queue and write)`);
             }
-            const vecGenerationComplete = targetEntryIds === undefined ? isVecFastPathComplete(db) : vecFastPathWasReady;
-            setVecFastPathReady(db, vecFailedCount === 0 && vecUnavailableCount === 0 && vecGenerationComplete);
-            if (vecFailedCount > 0) {
-                warn(`[embed] ${vecFailedCount} sqlite-vec fast-path insert${vecFailedCount === 1 ? "" : "s"} failed — ` +
-                    "semantic search will use the slower JS-cosine fallback over stored embeddings. " +
-                    "Rebuild with 'akm index --full' after resolving the vec table (often a vector-dimension mismatch).");
-            }
             const entriesPerSec = storedCount / elapsedSeconds;
             const tokensPerSec = storedTokens / elapsedSeconds;
-            const totalStored = storedCount + reusedCount;
             // #954, field-report follow-up: the final line
             // reports every outcome, not just what was stored — counts come from
             // the same collected `skips` the circuit breaker already uses,
@@ -675,16 +351,11 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
             const oversizedSkips = skips.filter((skip) => skip.reason === "context-window-exceeded");
             const timedOutSkips = skips.filter((skip) => skip.reason === "batch-request-failed" && skip.failureKind === "timeout");
             const failedSkips = skips.filter((skip) => skip.reason === "batch-request-failed" && skip.failureKind !== "timeout");
-            const throughputLine = reusedCount > 0
-                ? // #955: report reused and newly-embedded counts separately — the
-                    // rate figures below are provider throughput only (reuse is a
-                    // plain DB write, not provider work) and would be misleadingly
-                    // inflated if reused entries were folded into them.
-                    `Stored ${totalStored} embedding${totalStored === 1 ? "" : "s"} (${reusedCount} reused, ${storedCount} newly embedded) in ${elapsedSeconds.toFixed(1)}s (${entriesPerSec.toFixed(1)} entries/s, ~${Math.round(tokensPerSec)} tokens/s)`
-                : `Stored ${storedCount} embedding${storedCount === 1 ? "" : "s"} in ${elapsedSeconds.toFixed(1)}s (${entriesPerSec.toFixed(1)} entries/s, ~${Math.round(tokensPerSec)} tokens/s)`;
             onProgress({
                 phase: "embeddings",
-                message: `${throughputLine}; ${oversizedSkips.length} oversized skipped, ${timedOutSkips.length} timed out, ${failedSkips.length} failed.`,
+                message: `Stored ${storedCount} embedding${storedCount === 1 ? "" : "s"} in ${elapsedSeconds.toFixed(1)}s ` +
+                    `(${entriesPerSec.toFixed(1)} entries/s, ~${Math.round(tokensPerSec)} tokens/s); ` +
+                    `${oversizedSkips.length} oversized skipped, ${timedOutSkips.length} timed out, ${failedSkips.length} failed.`,
             });
             // Bounded itemRef-level detail for every skip category, not just
             // oversized — the aggregate counts above say HOW MANY documents timed
@@ -707,10 +378,6 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
             printSkipList("oversized documents", oversizedSkips);
             printSkipList("timed-out documents", timedOutSkips);
             printSkipList("failed documents", failedSkips);
-            setMeta(db, "embeddingFingerprint", currentFingerprint);
-            const observedIdentity = deriveObservedEmbeddingIdentity(config.embedding, observedModel, observedVectorLen);
-            if (observedIdentity)
-                setMeta(db, "embeddingIdentity", observedIdentity);
             // Circuit breaker tripped (#954): committed batches are
             // kept (nothing above discards them), but the pass is not a success —
             // the provider looks dead, not just occasionally flaky.
@@ -720,7 +387,7 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
                     "were stored — rerun akm index when the endpoint is healthy";
                 warn(`[embed] ${message}`);
                 onProgress({ phase: "embeddings", message });
-                return { success: false, message, vecInsertFailures: vecFailedCount };
+                return { success: false, message };
             }
             // Only a total failure (nothing at all embedded, despite having entries
             // to embed) turns into a phase failure. Any partial success — the vast
@@ -735,11 +402,7 @@ export async function generateEmbeddingsForDb(db, config, onProgress, signal, en
                     message: `All ${embedFailedCount} embedding batch(es) failed: ${firstMessage}`,
                 };
             }
-            // A pass that completes without abort or circuit-break purges
-            // whatever salvage is left — consumed by this pass's reuse step
-            // above, or superseded by what it just embedded.
-            purgeEmbeddingSalvage(db);
-            return { success: true, vecInsertFailures: vecFailedCount };
+            return { success: true };
         }
         finally {
             if (heartbeatTimer)
@@ -779,7 +442,7 @@ export function publishTargetedEmbeddingMeta(db, config) {
         return;
     }
     const entryCount = getEmbeddableEntryCount(db);
-    const embeddingCount = getEmbeddingCount(db);
+    const embeddingCount = getEmbeddingCount(db, deriveSemanticProviderFingerprint(config.embedding));
     const ready = entryCount > 0 && embeddingCount >= entryCount;
     setMeta(db, "hasEmbeddings", ready ? "1" : "0");
 }