hippo-memory 1.69.0 → 1.71.0

package/dist/{embeddings.js → store/embeddings/index.js}

@@ -1,183 +1,109 @@
-/**
- * Optional embedding-based semantic search for Hippo.
- * Uses @huggingface/transformers (local, zero API keys, ~22MB model).
- * Falls back silently if the library is not installed.
- */
-import { envByName } from './env.js';
+/** Optional embedding-based semantic search for Hippo.
+ * Uses @huggingface/transformers (local, no API keys, ~22MB model); falls back silently if not installed. */
+import { envByName } from '../../util/env.js';
 import * as fs from 'fs';
 import * as path from 'path';
 import { randomUUID } from 'crypto';
-import { loadAllEntries } from './store/entry-reads.js';
-import { openHippoDb, closeHippoDb, getMeta, rethrowIfSqliteBlocked, setMeta } from './db.js';
-import { EMBEDDING_MODEL_META_KEY, deleteOrphanVectors, hasStoredVectors, loadVectors, replaceAllVectors, storedVectorIds, upsertVectors, } from './vector-store.js';
-import { initializeParticle, savePhysicsState, loadPhysicsState, resetAllPhysicsState } from './physics-state.js';
-import { loadConfig } from './config.js';
-import { resolveEmbeddingProvider } from './embedding-provider.js';
-import { DEFAULT_EMBEDDING_MODEL } from './local-embedding.js';
-import { redactSecretsStrict } from './secret-detect.js';
-import { log } from './log.js';
-import { StoreNotPortedError } from './db/sqlite-blocked.js';
+import { chunked, loadAllEntryIds, loadEntriesByIds } from '../entry-reads.js';
+import { rethrowIfSqliteBlocked } from '../../db/index.js';
+import { EMBEDDING_MODEL_META_KEY } from '../../db/vector-store.js';
+import { initializeParticle } from '../../db/physics-state.js';
+import { indexedModel, pruneStoredVectors, replacesIndex, resetStoredParticles, saveEmbeddingIndex, saveIndexIdentity, saveStoredVectors, seedStoredParticle, storedIndexState, } from '../vector-index.js';
+import { loadConfig } from '../../core/config.js';
+import { resolveEmbeddingProvider } from './provider.js';
+import { redactSecretsStrict } from '../../util/secret-detect.js';
+import { errorMessage, log } from '../../util/log.js';
+import { StoreNotPortedError } from '../../util/sqlite-blocked.js';
 export { EMBEDDING_MODEL_META_KEY };
-/**
- * Bump whenever `embeddingInputText`'s composition changes in a way that
- * changes the resulting vectors for existing entries. Folded into the stored
- * index identity (see `embeddingIndexIdentity`) so a text-format change is
- * treated exactly like an embedding-model change: the next embed-touching
- * operation detects the mismatch and reindexes automatically. Format 2 =
- * `path:*` tags excluded (see `embeddingInputText`); format 1 (implicit, no
- * suffix) = `${content} ${tags.join(' ')}` including path tags.
- */
+/** Bump when `embeddingInputText` changes existing vectors; folded into the stored index identity (`embeddingIndexIdentity`) so it reindexes like a model
+ * change. Format 2 = `path:*` tags excluded; format 1 (implicit, no suffix) included them. */
 export const EMBED_TEXT_FORMAT = 2;
-/**
- * The stored-index identity for a given embedding provider id: folds
- * `EMBED_TEXT_FORMAT` into the provider id so index-identity comparisons
- * automatically invalidate on either a model change OR a text-format change.
- * This is the ONE choke point both `embeddingModelRequiresReindex` (compare
- * side) and `saveStoredEmbeddingModel` (save side) go through — they MUST
- * version identically, or every call reindexes in a loop (or none ever do).
- */
+/** Stored-index identity for a provider id: folds `EMBED_TEXT_FORMAT` in so a model or text-format change invalidates the index. The ONE choke point for
+ * compare (`embeddingModelRequiresReindex`) and save (`saveStoredEmbeddingModel`); they MUST version identically or every call reindexes. */
 export function embeddingIndexIdentity(providerId) {
     return `${providerId}#t${EMBED_TEXT_FORMAT}`;
 }
-/**
- * Build the text embedded for a memory entry: content plus its tags, joined
- * by a space and trimmed — the same shape as the legacy
- * `` `${e.content} ${e.tags.join(' ')}`.trim() `` composition, minus `path:*`
- * tags.
- *
- * `path:*` tags are excluded because they are auto-derived from
- * `process.cwd()` (see `extractPathTags` in cli.ts) and carry every path
- * component of the store's location, INCLUDING the store directory name
- * itself. That means identical content embeds to a DIFFERENT vector
- * depending on WHERE the store happens to live — e.g. a fresh benchmark run
- * under `tempfile.mkdtemp()` gets a new directory name (hence new path
- * tokens, hence a new vector) every single run, even with byte-identical
- * content ingested in byte-identical order. This was diagnosed as the
- * DOMINANT root cause of cross-fresh-ingest recall-rank variance measured on
- * LoCoMo (mean evidence-recall@5 stdev 0.0175 across 4 fresh re-ingests of
- * identical data; see `benchmarks/LOCOMO_INVESTIGATION.md`, "Determinism
- * characterization"). It is a real product defect beyond benchmarks too:
- * retrieval semantics should not depend on a project directory's name.
- *
- * Only `path:*` is excluded. Other tags (`conv:`, `session:`, `speaker:`,
- * `dia:`, `error`, `scope:`, etc.) remain embedded — they carry semantic
- * meaning. Path relevance at recall time is already handled explicitly by
- * the v39 scope-isolation layer (`origin_project`, `pathOverlapScore`), so
- * embedding-level path tokens are redundant with a dedicated mechanism
- * rather than a feature.
- */
+/** Text embedded for an entry: content plus tags, space-joined and trimmed, minus `path:*` tags.
+ * `path:*` tags derive from cwd, so identical content would embed differently per store location; path relevance is handled by `pathOverlapScore`. */
 export function embeddingInputText(entry) {
     const tags = entry.tags.filter((t) => !t.startsWith('path:'));
     return `${entry.content} ${tags.join(' ')}`.trim();
 }
 function loadStoredEmbeddingModel(hippoRoot) {
     try {
-        const db = openHippoDb(hippoRoot);
-        try {
-            const model = getMeta(db, EMBEDDING_MODEL_META_KEY, '').trim();
-            return model || null;
-        }
-        finally {
-            closeHippoDb(db);
-        }
+        return storedIndexState(hippoRoot).storedModel;
     }
     catch (err) {
         rethrowIfSqliteBlocked(err);
-        log.debug(`stored embedding model unreadable: ${err instanceof Error ? err.message : String(err)}`);
+        log.debug(`stored embedding model unreadable: ${errorMessage(err)}`);
         return null;
     }
 }
-/**
- * Persist the stored-index identity for `model` (see `embeddingIndexIdentity`).
- * Versioning happens INSIDE this function, not at call sites, so every caller
- * — current and future — gets the identity format for free. Must stay
- * consistent with the compare side in `embeddingModelRequiresReindex`.
- */
+/** Persist the stored-index identity for `model` (see `embeddingIndexIdentity`); versioning happens here, not at call sites,
+ * and must stay consistent with the compare side in `embeddingModelRequiresReindex`. */
 export function saveStoredEmbeddingModel(hippoRoot, model) {
-    const db = openHippoDb(hippoRoot);
-    try {
-        setMeta(db, EMBEDDING_MODEL_META_KEY, embeddingIndexIdentity(model));
-    }
-    finally {
-        closeHippoDb(db);
-    }
-}
-/** The model the index was built by: the stored identity, else the default model when vectors exist. */
-export function indexedModel(state) {
-    return state.storedModel ?? (state.hasVectors ? DEFAULT_EMBEDDING_MODEL : null);
+    saveIndexIdentity(hippoRoot, embeddingIndexIdentity(model));
 }
 /** The one rebuild rule every store applies to its indexed model. */
 export function indexNeedsRebuild(indexed, providerId) {
     return indexed !== null && indexed !== embeddingIndexIdentity(providerId);
 }
-/** Whether a vector write under the index identity `model` first drops the stored index, which another model built. */
-export function replacesIndex(state, model) {
-    const indexed = indexedModel(state);
-    return indexed !== null && indexed !== model;
-}
 export function resolveIndexedEmbeddingModel(hippoRoot, index) {
     const storedModel = loadStoredEmbeddingModel(hippoRoot);
     if (storedModel)
         return storedModel;
-    return indexedModel({ storedModel, hasVectors: index ? Object.keys(index).length > 0 : withVectorDb(hippoRoot, hasStoredVectors) });
+    return indexedModel({ storedModel, hasVectors: index ? Object.keys(index).length > 0 : storedIndexState(hippoRoot).hasVectors });
 }
 export function embeddingModelRequiresReindex(hippoRoot, model, index) {
     return indexNeedsRebuild(resolveIndexedEmbeddingModel(hippoRoot, index), model);
 }
-/** hippo.db's index state, the meta row and the EXISTS on one handle. */
-export function embeddingIndexStateAt(hippoRoot) {
-    return withVectorDb(hippoRoot, embeddingIndexStateOn);
-}
-export function embeddingIndexStateOn(db) {
-    return {
-        storedModel: getMeta(db, EMBEDDING_MODEL_META_KEY, '').trim() || null,
-        hasVectors: hasStoredVectors(db),
-    };
+const STORE_PAGE = 64;
+/** The rows for `ids`, a page at a time and in that order; an id deleted since it was listed is skipped. */
+function* entryPages(hippoRoot, ids) {
+    for (const page of chunked(ids, STORE_PAGE)) {
+        const byId = new Map(loadEntriesByIds(hippoRoot, page).map((e) => [e.id, e]));
+        const entries = page.flatMap((id) => byId.get(id) ?? []);
+        if (entries.length > 0)
+            yield entries;
+    }
 }
-async function rebuildEmbeddingIndex(entries, provider) {
+async function rebuildEmbeddingIndex(hippoRoot, ids, provider) {
     const rebuilt = {};
-    if (entries.length === 0)
-        return rebuilt;
-    const texts = entries.map((e) => embeddingInputText(e));
-    // provider.embed batches internally; on a hard transport/auth failure it
-    // throws, so the caller aborts WITHOUT saving a partial index (atomic
-    // reindex: the old index + stored identity are preserved).
-    const vectors = await provider.embed(texts, 'passage');
-    for (let i = 0; i < entries.length; i++) {
-        const vec = vectors[i];
-        if (vec && vec.length > 0) {
-            rebuilt[entries[i].id] = vec;
-        }
-        else {
-            noteSkippedEmbedding(entries[i].id);
+    for (const entries of entryPages(hippoRoot, ids)) {
+        // On a hard transport/auth failure provider.embed throws, so the caller aborts WITHOUT saving
+        // a partial index (atomic reindex: the old index + stored identity are preserved).
+        const vectors = await provider.embed(entries.map((e) => embeddingInputText(e)), 'passage');
+        for (let i = 0; i < entries.length; i++) {
+            const vec = vectors[i];
+            if (vec && vec.length > 0) {
+                rebuilt[entries[i].id] = vec;
+            }
+            else {
+                noteSkippedEmbedding(entries[i].id);
+            }
         }
     }
     return rebuilt;
 }
-function resetPhysicsFromIndex(hippoRoot, entries, index) {
+function resetPhysicsFromIndex(hippoRoot, ids, index) {
     try {
-        const db = openHippoDb(hippoRoot);
-        try {
-            resetAllPhysicsState(db, entries, index);
-        }
-        finally {
-            closeHippoDb(db);
-        }
+        resetStoredParticles(hippoRoot, ids, index);
     }
     catch (err) {
         // Best effort: retrieval still falls back without physics state.
-        log.warn(`physics reset after reindex failed: ${err instanceof Error ? err.message : String(err)}`);
+        log.warn(`physics reset after reindex failed: ${errorMessage(err)}`);
     }
 }
 /** A provider's `[]` row is a swallowed per-item failure; name the memory so the gap can be traced. */
 function noteSkippedEmbedding(id) {
     log.warn('memory not embedded; the next embed run retries it', { id });
 }
-/**
- * Cosine similarity between two vectors. Handles unnormalized vectors.
- * Returns 0 for empty or mismatched vectors.
- */
+/** Cosine similarity; handles unnormalized vectors, returns 0 for empty or mismatched vectors. */
 export function cosineSimilarity(a, b) {
+    return cosineOf(a, b);
+}
+/** `cosineSimilarity` over any indexable vectors; a Float32 view reads the same doubles as its number[] copy, so the score is the same. */
+export function cosineOf(a, b) {
     if (a.length === 0 || b.length === 0 || a.length !== b.length)
         return 0;
     let dot = 0;
@@ -194,31 +120,11 @@ export function cosineSimilarity(a, b) {
     // Clamp to [-1, 1] to handle floating point drift
     return Math.min(1, Math.max(-1, dot / denom));
 }
-function withVectorDb(hippoRoot, fn) {
-    const db = openHippoDb(hippoRoot);
-    try {
-        return fn(db);
-    }
-    finally {
-        closeHippoDb(db);
-    }
-}
-/** Every stored vector keyed by memory id; `{}` when none. Search reads only the rows it ranks via `loadStoredVectors`. */
-export function loadEmbeddingIndex(hippoRoot) {
-    return Object.fromEntries(withVectorDb(hippoRoot, (db) => loadVectors(db)));
-}
-/** Stored vectors for `ids` only. */
-export function loadStoredVectors(hippoRoot, ids) {
-    return ids.length === 0 ? new Map() : withVectorDb(hippoRoot, (db) => loadVectors(db, ids));
-}
-/** Replace every stored vector with `index`; `model` defaults to the stored index identity. */
-export function saveEmbeddingIndex(hippoRoot, index, model) {
-    withVectorDb(hippoRoot, (db) => replaceAllVectors(db, index, model ?? getMeta(db, EMBEDDING_MODEL_META_KEY, '')));
-}
 const EMBED_LOCK_FILE = 'embeddings.lock';
 const EMBED_LOCK_WAIT_MS = 10_000;
 const EMBED_LOCK_OWNER = `${process.pid}:${randomUUID()}`;
-// In-process mutex plus an O_EXCL "<pid>:<token>" lock file: our token is a lock we leaked; our PID with another token is a live worker thread, unless the lock predates this process (a reused PID).
+// In-process mutex plus an O_EXCL "<pid>:<token>" lock file: our token is a lock we leaked; our PID
+// with another token is a live worker thread, unless the lock predates this process (a reused PID).
 let _embedWriteLock = Promise.resolve();
 function embedLockHolderAlive(lockPath) {
     let raw;
@@ -316,7 +222,7 @@ function warnEmbedFailureOnce(source, rawMessage) {
     const message = redactSecretsStrict(rawMessage).replace(/\s+/g, ' ').replace(/\.+$/, '');
     log.warn(`embedding failed (${source}): ${message}. Memories are stored without embeddings until this is fixed.`);
 }
-// store-port.ts imports this module, so its requireGroup would close an import cycle.
+// requireGroup (src/store/port.ts) would close an import cycle with this module.
 function vectorGroups(store) {
     if (store.vectors === undefined)
         throw new StoreNotPortedError(store.kind, 'vectors');
@@ -343,7 +249,6 @@ async function writeVectorPage(writes, write, page, vectors, withParticles) {
     }
     return { written, modelMismatch: false };
 }
-const STORE_PAGE = 64;
 const OTHER_MODEL_INDEX = "the vector index was built by another embedding model; run 'hippo embed' to rebuild it";
 /** Embeds every memory with no vector under `model`, page by page, so a provider failure keeps the pages written; a rebuild reseeds the particles it drops. */
 async function backfillInStore(writes, provider, model, rebuild) {
@@ -381,10 +286,10 @@ function embedMemoryInStore(store, provider, entry) {
                 warnEmbedFailureOnce('index', OTHER_MODEL_INDEX);
         }
         catch (err) {
-            warnEmbedFailureOnce(provider.kind, err instanceof Error ? err.message : String(err));
+            warnEmbedFailureOnce(provider.kind, errorMessage(err));
         }
     }).catch((err) => {
-        log.warn(`skipped embedding ${entry.id} (${err instanceof Error ? err.message : String(err)})`);
+        log.warn(`skipped embedding ${entry.id} (${errorMessage(err)})`);
     });
 }
 /** Embed a single memory entry and cache the result in the embedding index; with `store`, through its vectorWrites, never opening hippo.db. */
@@ -395,7 +300,7 @@ export async function embedMemory(hippoRoot, entry, model, store) {
     }
     catch (err) {
         // Callers fire and forget, so this must resolve: a bad config warns once instead of rejecting.
-        warnEmbedFailureOnce('config', err instanceof Error ? err.message : String(err));
+        warnEmbedFailureOnce('config', errorMessage(err));
         return;
     }
     if (!provider.isAvailable())
@@ -403,62 +308,56 @@ export async function embedMemory(hippoRoot, entry, model, store) {
     if (store)
         return embedMemoryInStore(store, provider, entry);
     return withEmbedLock(hippoRoot, async () => {
-        // embedMemory is best-effort: an embedding failure (API down / bad key / 5xx)
-        // must not reject the caller's write — `getEmbedding` historically swallowed
-        // failures and returned []. The explicit `hippo embed` / `embedAll` path is
-        // where failures surface. On any failure we leave the existing index as-is.
+        // Best-effort: an embedding failure (API down, bad key, 5xx) must not reject the caller's write; the existing index is left as-is.
+        // Failures surface on the explicit `hippo embed` / `embedAll` path.
         try {
             const identity = provider.id;
             if (embeddingModelRequiresReindex(hippoRoot, identity)) {
-                // Host-wide rebuild. The embedding index is keyed by entry.id
-                // (which is tenant-scoped) but the index itself is one per hippoRoot.
-                // Cross-tenant content equivalence is visible at the vector level.
-                // Per-tenant indices would be a larger architecture change.
-                const entries = loadAllEntries(hippoRoot);
-                const rebuiltIndex = await rebuildEmbeddingIndex(entries, provider);
-                saveEmbeddingIndex(hippoRoot, rebuiltIndex, embeddingIndexIdentity(identity));
-                saveStoredEmbeddingModel(hippoRoot, identity);
-                resetPhysicsFromIndex(hippoRoot, entries, rebuiltIndex);
+                await rebuildIndexForProvider(hippoRoot, provider);
                 return;
             }
             const text = embeddingInputText(entry);
             const [vector] = await provider.embed([text], 'passage');
             if (!vector || vector.length === 0)
                 return;
-            withVectorDb(hippoRoot, (db) => upsertVectors(db, [[entry.id, vector]], embeddingIndexIdentity(identity)));
+            saveStoredVectors(hippoRoot, [[entry.id, vector]], embeddingIndexIdentity(identity));
             saveStoredEmbeddingModel(hippoRoot, identity);
             // Initialize physics state for this memory
-            try {
-                const db = openHippoDb(hippoRoot);
-                try {
-                    const existing = loadPhysicsState(db, [entry.id]);
-                    if (!existing.has(entry.id)) {
-                        const particle = initializeParticle(entry, vector);
-                        savePhysicsState(db, [particle]);
-                    }
-                }
-                finally {
-                    closeHippoDb(db);
-                }
-            }
-            catch {
-                // Physics init is best-effort — don't break embedding
-            }
+            initializePhysicsIfMissing(hippoRoot, entry, vector);
         }
         catch (err) {
             // Provider failure (API down / bad key). Best-effort: leave the index as-is, but say so once.
-            warnEmbedFailureOnce(provider.kind, err instanceof Error ? err.message : String(err));
+            warnEmbedFailureOnce(provider.kind, errorMessage(err));
         }
     }).catch((err) => {
-        log.warn(`skipped embedding ${entry.id} (${err instanceof Error ? err.message : String(err)}); run 'hippo embed' to backfill`);
+        log.warn(`skipped embedding ${entry.id} (${errorMessage(err)}); run 'hippo embed' to backfill`);
     });
 }
+// Every page is embedded before one transaction replaces the index, and the identity is saved only after that: a provider failure or a crash
+// at any earlier point leaves the old identity, so embeddingModelRequiresReindex has the next run rebuild from the start.
+async function rebuildIndexForProvider(hippoRoot, provider) {
+    const identity = provider.id;
+    // Host-wide rebuild: one embedding index per hippoRoot, keyed by tenant-scoped entry.id; per-tenant indices would be a larger architecture change.
+    const ids = loadAllEntryIds(hippoRoot);
+    const rebuiltIndex = await rebuildEmbeddingIndex(hippoRoot, ids, provider);
+    saveEmbeddingIndex(hippoRoot, rebuiltIndex, embeddingIndexIdentity(identity));
+    saveStoredEmbeddingModel(hippoRoot, identity);
+    resetPhysicsFromIndex(hippoRoot, ids, rebuiltIndex);
+    return Object.keys(rebuiltIndex).length;
+}
+function initializePhysicsIfMissing(hippoRoot, entry, vector) {
+    try {
+        seedStoredParticle(hippoRoot, entry, vector);
+    }
+    catch (err) {
+        // Physics init is best-effort and must not fail the embedding that just landed.
+        log.debug(`physics state not initialised for ${entry.id}: ${errorMessage(err)}`);
+    }
+}
 /** Throws when an unavailable provider is a misconfiguration rather than an intentional no-op. */
 function throwIfProviderKeyMissing(hippoRoot, provider) {
-    // A configured (non-disabled) API provider with a missing key is a
-    // misconfiguration, not a no-op: surface it so programmatic callers of the
-    // exported embedAll() learn nothing was written. Local-not-installed and an
-    // explicit enabled=false stay silent no-ops (best-effort / intentional).
+    // A configured API provider with a missing key is a misconfiguration: surface it so callers of the exported embedAll() learn nothing was written.
+    // Local-not-installed and an explicit enabled=false stay silent no-ops.
     const cfg = loadConfig(hippoRoot).embeddings;
     if (provider.kind !== 'local' &&
         cfg.enabled !== false &&
@@ -467,21 +366,13 @@ function throwIfProviderKeyMissing(hippoRoot, provider) {
         throw new Error(`Embedding provider '${provider.kind}' is configured but ${provider.keyEnv} is not set.`);
     }
 }
-// Embed entries without a cached vector in save-checkpointed chunks.
-// provider.embed batches internally (one HTTP request per batchSize for API
-// providers; sequential for local). A `[]` row means that single item could
-// not be embedded and is left for a later run (resumable). On a hard provider
-// failure mid-backfill we persist the chunks already embedded this run rather
-// than discarding paid progress, then stop and resume on the next run.
+// Embeds entries without a cached vector in save-checkpointed chunks; a `[]` row means that item could not be embedded and is retried next run.
+// On a hard provider failure mid-backfill the chunks already embedded are persisted (paid progress), then it stops and resumes next run.
 async function backfillPending(hippoRoot, provider, pending, model) {
     let count = 0;
-    // Initialized to `undefined` (not a known-evidence literal like `null`) so
-    // it stays a plain `unknown` binding for the arbitrary caught value below;
-    // falsy either way, so `if (backfillError)` behaves identically.
+    // Initialized to `undefined` so it stays a plain `unknown` binding for the arbitrary caught value; falsy either way.
     let backfillError = undefined;
-    const SAVE_CHUNK = 64;
-    for (let i = 0; i < pending.length; i += SAVE_CHUNK) {
-        const chunk = pending.slice(i, i + SAVE_CHUNK);
+    for (const chunk of entryPages(hippoRoot, pending)) {
         let vectors;
         try {
             vectors = await provider.embed(chunk.map((e) => embeddingInputText(e)), 'passage');
@@ -500,7 +391,7 @@ async function backfillPending(hippoRoot, provider, pending, model) {
             else
                 noteSkippedEmbedding(chunk[j].id);
         }
-        count += withVectorDb(hippoRoot, (db) => upsertVectors(db, rows, model));
+        count += saveStoredVectors(hippoRoot, rows, model);
     }
     return { count, backfillError };
 }
@@ -515,24 +406,13 @@ export async function embedAll(hippoRoot, model, provider = resolveEmbeddingProv
         return withProcessEmbedLock(() => embedAllInStore(store, provider));
     return withEmbedLock(hippoRoot, async () => {
         const identity = provider.id;
-        // Host-wide by design. embedAll backfills vectors for all tenants'
-        // entries into the per-host embedding index. Per-tenant filtering would
-        // produce partial indices and break recall.
-        const entries = loadAllEntries(hippoRoot);
-        const model = embeddingIndexIdentity(identity);
-        if (embeddingModelRequiresReindex(hippoRoot, identity)) {
-            const rebuiltIndex = await rebuildEmbeddingIndex(entries, provider);
-            saveEmbeddingIndex(hippoRoot, rebuiltIndex, model);
-            saveStoredEmbeddingModel(hippoRoot, identity);
-            resetPhysicsFromIndex(hippoRoot, entries, rebuiltIndex);
-            return Object.keys(rebuiltIndex).length;
-        }
-        const embedded = withVectorDb(hippoRoot, (db) => {
-            deleteOrphanVectors(db);
-            return storedVectorIds(db);
-        });
-        const pending = entries.filter((e) => !embedded.has(e.id));
-        const { count, backfillError } = await backfillPending(hippoRoot, provider, pending, model);
+        if (embeddingModelRequiresReindex(hippoRoot, identity))
+            return rebuildIndexForProvider(hippoRoot, provider);
+        // Host-wide by design: embedAll backfills all tenants' entries into the per-host index; per-tenant filtering would produce partial indices and break
+        // recall.
+        const embedded = pruneStoredVectors(hippoRoot);
+        const pending = loadAllEntryIds(hippoRoot).filter((id) => !embedded.has(id));
+        const { count, backfillError } = await backfillPending(hippoRoot, provider, pending, embeddingIndexIdentity(identity));
         saveStoredEmbeddingModel(hippoRoot, identity);
         // Partial progress is now persisted; surface a hard backfill failure so the
         // explicit embed path reports it (best-effort callers go via embedMemory).
@@ -541,4 +421,4 @@ export async function embedAll(hippoRoot, model, provider = resolveEmbeddingProv
         return count;
     });
 }
-//# sourceMappingURL=embeddings.js.map
+//# sourceMappingURL=index.js.map

package/dist/{local-embedding.d.ts → store/embeddings/local.d.ts}

@@ -12,4 +12,4 @@ export declare function requireLocalPipeline(model: string): Promise<void>;
 export declare function resolveEmbeddingModel(hippoRoot: string, explicitModel?: string): string;
 /** Embeds `text` with the local model, or returns `[]` when Transformers.js is missing or fails; `role` adds e5-style prefixes. */
 export declare function getEmbedding(text: string, model?: string, role?: EmbeddingRole): Promise<number[]>;
-//# sourceMappingURL=local-embedding.d.ts.map
+//# sourceMappingURL=local.d.ts.map