hippo-memory 1.70.0 → 1.71.0

package/dist/store/embeddings/index.d.ts

@@ -1,102 +1,33 @@
-/**
- * Optional embedding-based semantic search for Hippo.
- * Uses @huggingface/transformers (local, zero API keys, ~22MB model).
- * Falls back silently if the library is not installed.
- */
+/** Optional embedding-based semantic search for Hippo.
+ * Uses @huggingface/transformers (local, no API keys, ~22MB model); falls back silently if not installed. */
 import { MemoryEntry } from '../../core/memory.js';
-import { type DatabaseSyncLike } from '../../db/index.js';
 import { EMBEDDING_MODEL_META_KEY } from '../../db/vector-store.js';
 import { type EmbeddingProvider } from './provider.js';
 import type { HippoStore } from '../index.js';
 export { EMBEDDING_MODEL_META_KEY };
-/**
- * Bump whenever `embeddingInputText`'s composition changes in a way that
- * changes the resulting vectors for existing entries. Folded into the stored
- * index identity (see `embeddingIndexIdentity`) so a text-format change is
- * treated exactly like an embedding-model change: the next embed-touching
- * operation detects the mismatch and reindexes automatically. Format 2 =
- * `path:*` tags excluded (see `embeddingInputText`); format 1 (implicit, no
- * suffix) = `${content} ${tags.join(' ')}` including path tags.
- */
+/** Bump when `embeddingInputText` changes existing vectors; folded into the stored index identity (`embeddingIndexIdentity`) so it reindexes like a model
+ * change. Format 2 = `path:*` tags excluded; format 1 (implicit, no suffix) included them. */
 export declare const EMBED_TEXT_FORMAT = 2;
-/**
- * The stored-index identity for a given embedding provider id: folds
- * `EMBED_TEXT_FORMAT` into the provider id so index-identity comparisons
- * automatically invalidate on either a model change OR a text-format change.
- * This is the ONE choke point both `embeddingModelRequiresReindex` (compare
- * side) and `saveStoredEmbeddingModel` (save side) go through — they MUST
- * version identically, or every call reindexes in a loop (or none ever do).
- */
+/** Stored-index identity for a provider id: folds `EMBED_TEXT_FORMAT` in so a model or text-format change invalidates the index. The ONE choke point for
+ * compare (`embeddingModelRequiresReindex`) and save (`saveStoredEmbeddingModel`); they MUST version identically or every call reindexes. */
 export declare function embeddingIndexIdentity(providerId: string): string;
-/**
- * Build the text embedded for a memory entry: content plus its tags, joined
- * by a space and trimmed — the same shape as the legacy
- * `` `${e.content} ${e.tags.join(' ')}`.trim() `` composition, minus `path:*`
- * tags.
- *
- * `path:*` tags are excluded because they are auto-derived from
- * `process.cwd()` (see `extractPathTags` in cli.ts) and carry every path
- * component of the store's location, INCLUDING the store directory name
- * itself. That means identical content embeds to a DIFFERENT vector
- * depending on WHERE the store happens to live — e.g. a fresh benchmark run
- * under `tempfile.mkdtemp()` gets a new directory name (hence new path
- * tokens, hence a new vector) every single run, even with byte-identical
- * content ingested in byte-identical order. This was diagnosed as the
- * DOMINANT root cause of cross-fresh-ingest recall-rank variance measured on
- * LoCoMo (mean evidence-recall@5 stdev 0.0175 across 4 fresh re-ingests of
- * identical data; see `benchmarks/LOCOMO_INVESTIGATION.md`, "Determinism
- * characterization"). It is a real product defect beyond benchmarks too:
- * retrieval semantics should not depend on a project directory's name.
- *
- * Only `path:*` is excluded. Other tags (`conv:`, `session:`, `speaker:`,
- * `dia:`, `error`, `scope:`, etc.) remain embedded — they carry semantic
- * meaning. Path relevance at recall time is already handled explicitly by
- * the v39 scope-isolation layer (`origin_project`, `pathOverlapScore`), so
- * embedding-level path tokens are redundant with a dedicated mechanism
- * rather than a feature.
- */
+/** Text embedded for an entry: content plus tags, space-joined and trimmed, minus `path:*` tags.
+ * `path:*` tags derive from cwd, so identical content would embed differently per store location; path relevance is handled by `pathOverlapScore`. */
 export declare function embeddingInputText(entry: {
     content: string;
     tags: string[];
 }): string;
-/**
- * Persist the stored-index identity for `model` (see `embeddingIndexIdentity`).
- * Versioning happens INSIDE this function, not at call sites, so every caller
- * — current and future — gets the identity format for free. Must stay
- * consistent with the compare side in `embeddingModelRequiresReindex`.
- */
+/** Persist the stored-index identity for `model` (see `embeddingIndexIdentity`); versioning happens here, not at call sites,
+ * and must stay consistent with the compare side in `embeddingModelRequiresReindex`. */
 export declare function saveStoredEmbeddingModel(hippoRoot: string, model: string): void;
-/** What a store's vector index was built by: the stored identity, and whether any vector exists. */
-export interface EmbeddingIndexState {
-    readonly storedModel: string | null;
-    readonly hasVectors: boolean;
-}
-/** The model the index was built by: the stored identity, else the default model when vectors exist. */
-export declare function indexedModel(state: EmbeddingIndexState): string | null;
 /** The one rebuild rule every store applies to its indexed model. */
 export declare function indexNeedsRebuild(indexed: string | null, providerId: string): boolean;
-/** Whether a vector write under the index identity `model` first drops the stored index, which another model built. */
-export declare function replacesIndex(state: EmbeddingIndexState, model: string): boolean;
 export declare function resolveIndexedEmbeddingModel(hippoRoot: string, index?: Record<string, number[]>): string | null;
 export declare function embeddingModelRequiresReindex(hippoRoot: string, model: string, index?: Record<string, number[]>): boolean;
-/** hippo.db's index state, the meta row and the EXISTS on one handle. */
-export declare function embeddingIndexStateAt(hippoRoot: string): EmbeddingIndexState;
-export declare function embeddingIndexStateOn(db: DatabaseSyncLike): EmbeddingIndexState;
-/**
- * Cosine similarity between two vectors. Handles unnormalized vectors.
- * Returns 0 for empty or mismatched vectors.
- */
+/** Cosine similarity; handles unnormalized vectors, returns 0 for empty or mismatched vectors. */
 export declare function cosineSimilarity(a: number[], b: number[]): number;
 /** `cosineSimilarity` over any indexable vectors; a Float32 view reads the same doubles as its number[] copy, so the score is the same. */
 export declare function cosineOf(a: ArrayLike<number>, b: ArrayLike<number>): number;
-/** Every stored vector keyed by memory id; `{}` when none. Search reads only the rows it ranks via `loadStoredVectors`. */
-export declare function loadEmbeddingIndex(hippoRoot: string): Record<string, number[]>;
-/** Stored vectors for `ids` only. */
-export declare function loadStoredVectors(hippoRoot: string, ids: readonly string[]): Map<string, number[]>;
-/** `loadStoredVectors` as Float32 views, for a caller that only scores them. */
-export declare function loadStoredVectorViews(hippoRoot: string, ids: readonly string[]): Map<string, Float32Array>;
-/** Replace every stored vector with `index`; `model` defaults to the stored index identity. */
-export declare function saveEmbeddingIndex(hippoRoot: string, index: Record<string, number[]>, model?: string): void;
 /** Embed a single memory entry and cache the result in the embedding index; with `store`, through its vectorWrites, never opening hippo.db. */
 export declare function embedMemory(hippoRoot: string, entry: MemoryEntry, model?: string, store?: HippoStore): Promise<void>;
 /** Embeds every entry with no cached vector, prunes vectors of deleted memories, and returns how many it embedded; `provider` defaults to the

package/dist/store/embeddings/index.js

@@ -1,85 +1,37 @@
-/**
- * Optional embedding-based semantic search for Hippo.
- * Uses @huggingface/transformers (local, zero API keys, ~22MB model).
- * Falls back silently if the library is not installed.
- */
+/** Optional embedding-based semantic search for Hippo.
+ * Uses @huggingface/transformers (local, no API keys, ~22MB model); falls back silently if not installed. */
 import { envByName } from '../../util/env.js';
 import * as fs from 'fs';
 import * as path from 'path';
 import { randomUUID } from 'crypto';
-import { loadAllEntries } from '../entry-reads.js';
-import { openHippoDb, closeHippoDb, getMeta, rethrowIfSqliteBlocked, setMeta } from '../../db/index.js';
-import { EMBEDDING_MODEL_META_KEY, deleteOrphanVectors, hasStoredVectors, loadVectors, loadVectorViews, replaceAllVectors, storedVectorIds, upsertVectors, } from '../../db/vector-store.js';
-import { initializeParticle, savePhysicsState, loadPhysicsState, resetAllPhysicsState } from '../../db/physics-state.js';
+import { chunked, loadAllEntryIds, loadEntriesByIds } from '../entry-reads.js';
+import { rethrowIfSqliteBlocked } from '../../db/index.js';
+import { EMBEDDING_MODEL_META_KEY } from '../../db/vector-store.js';
+import { initializeParticle } from '../../db/physics-state.js';
+import { indexedModel, pruneStoredVectors, replacesIndex, resetStoredParticles, saveEmbeddingIndex, saveIndexIdentity, saveStoredVectors, seedStoredParticle, storedIndexState, } from '../vector-index.js';
 import { loadConfig } from '../../core/config.js';
 import { resolveEmbeddingProvider } from './provider.js';
-import { DEFAULT_EMBEDDING_MODEL } from './local.js';
 import { redactSecretsStrict } from '../../util/secret-detect.js';
 import { errorMessage, log } from '../../util/log.js';
 import { StoreNotPortedError } from '../../util/sqlite-blocked.js';
 export { EMBEDDING_MODEL_META_KEY };
-/**
- * Bump whenever `embeddingInputText`'s composition changes in a way that
- * changes the resulting vectors for existing entries. Folded into the stored
- * index identity (see `embeddingIndexIdentity`) so a text-format change is
- * treated exactly like an embedding-model change: the next embed-touching
- * operation detects the mismatch and reindexes automatically. Format 2 =
- * `path:*` tags excluded (see `embeddingInputText`); format 1 (implicit, no
- * suffix) = `${content} ${tags.join(' ')}` including path tags.
- */
+/** Bump when `embeddingInputText` changes existing vectors; folded into the stored index identity (`embeddingIndexIdentity`) so it reindexes like a model
+ * change. Format 2 = `path:*` tags excluded; format 1 (implicit, no suffix) included them. */
 export const EMBED_TEXT_FORMAT = 2;
-/**
- * The stored-index identity for a given embedding provider id: folds
- * `EMBED_TEXT_FORMAT` into the provider id so index-identity comparisons
- * automatically invalidate on either a model change OR a text-format change.
- * This is the ONE choke point both `embeddingModelRequiresReindex` (compare
- * side) and `saveStoredEmbeddingModel` (save side) go through — they MUST
- * version identically, or every call reindexes in a loop (or none ever do).
- */
+/** Stored-index identity for a provider id: folds `EMBED_TEXT_FORMAT` in so a model or text-format change invalidates the index. The ONE choke point for
+ * compare (`embeddingModelRequiresReindex`) and save (`saveStoredEmbeddingModel`); they MUST version identically or every call reindexes. */
 export function embeddingIndexIdentity(providerId) {
     return `${providerId}#t${EMBED_TEXT_FORMAT}`;
 }
-/**
- * Build the text embedded for a memory entry: content plus its tags, joined
- * by a space and trimmed — the same shape as the legacy
- * `` `${e.content} ${e.tags.join(' ')}`.trim() `` composition, minus `path:*`
- * tags.
- *
- * `path:*` tags are excluded because they are auto-derived from
- * `process.cwd()` (see `extractPathTags` in cli.ts) and carry every path
- * component of the store's location, INCLUDING the store directory name
- * itself. That means identical content embeds to a DIFFERENT vector
- * depending on WHERE the store happens to live — e.g. a fresh benchmark run
- * under `tempfile.mkdtemp()` gets a new directory name (hence new path
- * tokens, hence a new vector) every single run, even with byte-identical
- * content ingested in byte-identical order. This was diagnosed as the
- * DOMINANT root cause of cross-fresh-ingest recall-rank variance measured on
- * LoCoMo (mean evidence-recall@5 stdev 0.0175 across 4 fresh re-ingests of
- * identical data; see `benchmarks/LOCOMO_INVESTIGATION.md`, "Determinism
- * characterization"). It is a real product defect beyond benchmarks too:
- * retrieval semantics should not depend on a project directory's name.
- *
- * Only `path:*` is excluded. Other tags (`conv:`, `session:`, `speaker:`,
- * `dia:`, `error`, `scope:`, etc.) remain embedded — they carry semantic
- * meaning. Path relevance at recall time is already handled explicitly by
- * the v39 scope-isolation layer (`origin_project`, `pathOverlapScore`), so
- * embedding-level path tokens are redundant with a dedicated mechanism
- * rather than a feature.
- */
+/** Text embedded for an entry: content plus tags, space-joined and trimmed, minus `path:*` tags.
+ * `path:*` tags derive from cwd, so identical content would embed differently per store location; path relevance is handled by `pathOverlapScore`. */
 export function embeddingInputText(entry) {
     const tags = entry.tags.filter((t) => !t.startsWith('path:'));
     return `${entry.content} ${tags.join(' ')}`.trim();
 }
 function loadStoredEmbeddingModel(hippoRoot) {
     try {
-        const db = openHippoDb(hippoRoot);
-        try {
-            const model = getMeta(db, EMBEDDING_MODEL_META_KEY, '').trim();
-            return model || null;
-        }
-        finally {
-            closeHippoDb(db);
-        }
+        return storedIndexState(hippoRoot).storedModel;
     }
     catch (err) {
         rethrowIfSqliteBlocked(err);
@@ -87,82 +39,55 @@ function loadStoredEmbeddingModel(hippoRoot) {
         return null;
     }
 }
-/**
- * Persist the stored-index identity for `model` (see `embeddingIndexIdentity`).
- * Versioning happens INSIDE this function, not at call sites, so every caller
- * — current and future — gets the identity format for free. Must stay
- * consistent with the compare side in `embeddingModelRequiresReindex`.
- */
+/** Persist the stored-index identity for `model` (see `embeddingIndexIdentity`); versioning happens here, not at call sites,
+ * and must stay consistent with the compare side in `embeddingModelRequiresReindex`. */
 export function saveStoredEmbeddingModel(hippoRoot, model) {
-    const db = openHippoDb(hippoRoot);
-    try {
-        setMeta(db, EMBEDDING_MODEL_META_KEY, embeddingIndexIdentity(model));
-    }
-    finally {
-        closeHippoDb(db);
-    }
-}
-/** The model the index was built by: the stored identity, else the default model when vectors exist. */
-export function indexedModel(state) {
-    return state.storedModel ?? (state.hasVectors ? DEFAULT_EMBEDDING_MODEL : null);
+    saveIndexIdentity(hippoRoot, embeddingIndexIdentity(model));
 }
 /** The one rebuild rule every store applies to its indexed model. */
 export function indexNeedsRebuild(indexed, providerId) {
     return indexed !== null && indexed !== embeddingIndexIdentity(providerId);
 }
-/** Whether a vector write under the index identity `model` first drops the stored index, which another model built. */
-export function replacesIndex(state, model) {
-    const indexed = indexedModel(state);
-    return indexed !== null && indexed !== model;
-}
 export function resolveIndexedEmbeddingModel(hippoRoot, index) {
     const storedModel = loadStoredEmbeddingModel(hippoRoot);
     if (storedModel)
         return storedModel;
-    return indexedModel({ storedModel, hasVectors: index ? Object.keys(index).length > 0 : withVectorDb(hippoRoot, hasStoredVectors) });
+    return indexedModel({ storedModel, hasVectors: index ? Object.keys(index).length > 0 : storedIndexState(hippoRoot).hasVectors });
 }
 export function embeddingModelRequiresReindex(hippoRoot, model, index) {
     return indexNeedsRebuild(resolveIndexedEmbeddingModel(hippoRoot, index), model);
 }
-/** hippo.db's index state, the meta row and the EXISTS on one handle. */
-export function embeddingIndexStateAt(hippoRoot) {
-    return withVectorDb(hippoRoot, embeddingIndexStateOn);
-}
-export function embeddingIndexStateOn(db) {
-    return {
-        storedModel: getMeta(db, EMBEDDING_MODEL_META_KEY, '').trim() || null,
-        hasVectors: hasStoredVectors(db),
-    };
+const STORE_PAGE = 64;
+/** The rows for `ids`, a page at a time and in that order; an id deleted since it was listed is skipped. */
+function* entryPages(hippoRoot, ids) {
+    for (const page of chunked(ids, STORE_PAGE)) {
+        const byId = new Map(loadEntriesByIds(hippoRoot, page).map((e) => [e.id, e]));
+        const entries = page.flatMap((id) => byId.get(id) ?? []);
+        if (entries.length > 0)
+            yield entries;
+    }
 }
-async function rebuildEmbeddingIndex(entries, provider) {
+async function rebuildEmbeddingIndex(hippoRoot, ids, provider) {
     const rebuilt = {};
-    if (entries.length === 0)
-        return rebuilt;
-    const texts = entries.map((e) => embeddingInputText(e));
-    // provider.embed batches internally; on a hard transport/auth failure it
-    // throws, so the caller aborts WITHOUT saving a partial index (atomic
-    // reindex: the old index + stored identity are preserved).
-    const vectors = await provider.embed(texts, 'passage');
-    for (let i = 0; i < entries.length; i++) {
-        const vec = vectors[i];
-        if (vec && vec.length > 0) {
-            rebuilt[entries[i].id] = vec;
-        }
-        else {
-            noteSkippedEmbedding(entries[i].id);
+    for (const entries of entryPages(hippoRoot, ids)) {
+        // On a hard transport/auth failure provider.embed throws, so the caller aborts WITHOUT saving
+        // a partial index (atomic reindex: the old index + stored identity are preserved).
+        const vectors = await provider.embed(entries.map((e) => embeddingInputText(e)), 'passage');
+        for (let i = 0; i < entries.length; i++) {
+            const vec = vectors[i];
+            if (vec && vec.length > 0) {
+                rebuilt[entries[i].id] = vec;
+            }
+            else {
+                noteSkippedEmbedding(entries[i].id);
+            }
         }
     }
     return rebuilt;
 }
-function resetPhysicsFromIndex(hippoRoot, entries, index) {
+function resetPhysicsFromIndex(hippoRoot, ids, index) {
     try {
-        const db = openHippoDb(hippoRoot);
-        try {
-            resetAllPhysicsState(db, entries, index);
-        }
-        finally {
-            closeHippoDb(db);
-        }
+        resetStoredParticles(hippoRoot, ids, index);
     }
     catch (err) {
         // Best effort: retrieval still falls back without physics state.
@@ -173,10 +98,7 @@ function resetPhysicsFromIndex(hippoRoot, entries, index) {
 function noteSkippedEmbedding(id) {
     log.warn('memory not embedded; the next embed run retries it', { id });
 }
-/**
- * Cosine similarity between two vectors. Handles unnormalized vectors.
- * Returns 0 for empty or mismatched vectors.
- */
+/** Cosine similarity; handles unnormalized vectors, returns 0 for empty or mismatched vectors. */
 export function cosineSimilarity(a, b) {
     return cosineOf(a, b);
 }
@@ -198,35 +120,11 @@ export function cosineOf(a, b) {
     // Clamp to [-1, 1] to handle floating point drift
     return Math.min(1, Math.max(-1, dot / denom));
 }
-function withVectorDb(hippoRoot, fn) {
-    const db = openHippoDb(hippoRoot);
-    try {
-        return fn(db);
-    }
-    finally {
-        closeHippoDb(db);
-    }
-}
-/** Every stored vector keyed by memory id; `{}` when none. Search reads only the rows it ranks via `loadStoredVectors`. */
-export function loadEmbeddingIndex(hippoRoot) {
-    return Object.fromEntries(withVectorDb(hippoRoot, (db) => loadVectors(db)));
-}
-/** Stored vectors for `ids` only. */
-export function loadStoredVectors(hippoRoot, ids) {
-    return ids.length === 0 ? new Map() : withVectorDb(hippoRoot, (db) => loadVectors(db, ids));
-}
-/** `loadStoredVectors` as Float32 views, for a caller that only scores them. */
-export function loadStoredVectorViews(hippoRoot, ids) {
-    return ids.length === 0 ? new Map() : withVectorDb(hippoRoot, (db) => loadVectorViews(db, ids));
-}
-/** Replace every stored vector with `index`; `model` defaults to the stored index identity. */
-export function saveEmbeddingIndex(hippoRoot, index, model) {
-    withVectorDb(hippoRoot, (db) => replaceAllVectors(db, index, model ?? getMeta(db, EMBEDDING_MODEL_META_KEY, '')));
-}
 const EMBED_LOCK_FILE = 'embeddings.lock';
 const EMBED_LOCK_WAIT_MS = 10_000;
 const EMBED_LOCK_OWNER = `${process.pid}:${randomUUID()}`;
-// In-process mutex plus an O_EXCL "<pid>:<token>" lock file: our token is a lock we leaked; our PID with another token is a live worker thread, unless the lock predates this process (a reused PID).
+// In-process mutex plus an O_EXCL "<pid>:<token>" lock file: our token is a lock we leaked; our PID
+// with another token is a live worker thread, unless the lock predates this process (a reused PID).
 let _embedWriteLock = Promise.resolve();
 function embedLockHolderAlive(lockPath) {
     let raw;
@@ -324,7 +222,7 @@ function warnEmbedFailureOnce(source, rawMessage) {
     const message = redactSecretsStrict(rawMessage).replace(/\s+/g, ' ').replace(/\.+$/, '');
     log.warn(`embedding failed (${source}): ${message}. Memories are stored without embeddings until this is fixed.`);
 }
-// store-port.ts imports this module, so its requireGroup would close an import cycle.
+// requireGroup (src/store/port.ts) would close an import cycle with this module.
 function vectorGroups(store) {
     if (store.vectors === undefined)
         throw new StoreNotPortedError(store.kind, 'vectors');
@@ -351,7 +249,6 @@ async function writeVectorPage(writes, write, page, vectors, withParticles) {
     }
     return { written, modelMismatch: false };
 }
-const STORE_PAGE = 64;
 const OTHER_MODEL_INDEX = "the vector index was built by another embedding model; run 'hippo embed' to rebuild it";
 /** Embeds every memory with no vector under `model`, page by page, so a provider failure keeps the pages written; a rebuild reseeds the particles it drops. */
 async function backfillInStore(writes, provider, model, rebuild) {
@@ -411,10 +308,8 @@ export async function embedMemory(hippoRoot, entry, model, store) {
     if (store)
         return embedMemoryInStore(store, provider, entry);
     return withEmbedLock(hippoRoot, async () => {
-        // embedMemory is best-effort: an embedding failure (API down / bad key / 5xx)
-        // must not reject the caller's write — `getEmbedding` historically swallowed
-        // failures and returned []. The explicit `hippo embed` / `embedAll` path is
-        // where failures surface. On any failure we leave the existing index as-is.
+        // Best-effort: an embedding failure (API down, bad key, 5xx) must not reject the caller's write; the existing index is left as-is.
+        // Failures surface on the explicit `hippo embed` / `embedAll` path.
         try {
             const identity = provider.id;
             if (embeddingModelRequiresReindex(hippoRoot, identity)) {
@@ -425,7 +320,7 @@ export async function embedMemory(hippoRoot, entry, model, store) {
             const [vector] = await provider.embed([text], 'passage');
             if (!vector || vector.length === 0)
                 return;
-            withVectorDb(hippoRoot, (db) => upsertVectors(db, [[entry.id, vector]], embeddingIndexIdentity(identity)));
+            saveStoredVectors(hippoRoot, [[entry.id, vector]], embeddingIndexIdentity(identity));
             saveStoredEmbeddingModel(hippoRoot, identity);
             // Initialize physics state for this memory
             initializePhysicsIfMissing(hippoRoot, entry, vector);
@@ -438,31 +333,21 @@ export async function embedMemory(hippoRoot, entry, model, store) {
         log.warn(`skipped embedding ${entry.id} (${errorMessage(err)}); run 'hippo embed' to backfill`);
     });
 }
+// Every page is embedded before one transaction replaces the index, and the identity is saved only after that: a provider failure or a crash
+// at any earlier point leaves the old identity, so embeddingModelRequiresReindex has the next run rebuild from the start.
 async function rebuildIndexForProvider(hippoRoot, provider) {
     const identity = provider.id;
-    // Host-wide rebuild. The embedding index is keyed by entry.id
-    // (which is tenant-scoped) but the index itself is one per hippoRoot.
-    // Cross-tenant content equivalence is visible at the vector level.
-    // Per-tenant indices would be a larger architecture change.
-    const entries = loadAllEntries(hippoRoot);
-    const rebuiltIndex = await rebuildEmbeddingIndex(entries, provider);
+    // Host-wide rebuild: one embedding index per hippoRoot, keyed by tenant-scoped entry.id; per-tenant indices would be a larger architecture change.
+    const ids = loadAllEntryIds(hippoRoot);
+    const rebuiltIndex = await rebuildEmbeddingIndex(hippoRoot, ids, provider);
     saveEmbeddingIndex(hippoRoot, rebuiltIndex, embeddingIndexIdentity(identity));
     saveStoredEmbeddingModel(hippoRoot, identity);
-    resetPhysicsFromIndex(hippoRoot, entries, rebuiltIndex);
+    resetPhysicsFromIndex(hippoRoot, ids, rebuiltIndex);
+    return Object.keys(rebuiltIndex).length;
 }
 function initializePhysicsIfMissing(hippoRoot, entry, vector) {
     try {
-        const db = openHippoDb(hippoRoot);
-        try {
-            const existing = loadPhysicsState(db, [entry.id]);
-            if (!existing.has(entry.id)) {
-                const particle = initializeParticle(entry, vector);
-                savePhysicsState(db, [particle]);
-            }
-        }
-        finally {
-            closeHippoDb(db);
-        }
+        seedStoredParticle(hippoRoot, entry, vector);
     }
     catch (err) {
         // Physics init is best-effort and must not fail the embedding that just landed.
@@ -471,10 +356,8 @@ function initializePhysicsIfMissing(hippoRoot, entry, vector) {
 }
 /** Throws when an unavailable provider is a misconfiguration rather than an intentional no-op. */
 function throwIfProviderKeyMissing(hippoRoot, provider) {
-    // A configured (non-disabled) API provider with a missing key is a
-    // misconfiguration, not a no-op: surface it so programmatic callers of the
-    // exported embedAll() learn nothing was written. Local-not-installed and an
-    // explicit enabled=false stay silent no-ops (best-effort / intentional).
+    // A configured API provider with a missing key is a misconfiguration: surface it so callers of the exported embedAll() learn nothing was written.
+    // Local-not-installed and an explicit enabled=false stay silent no-ops.
     const cfg = loadConfig(hippoRoot).embeddings;
     if (provider.kind !== 'local' &&
         cfg.enabled !== false &&
@@ -483,21 +366,13 @@ function throwIfProviderKeyMissing(hippoRoot, provider) {
         throw new Error(`Embedding provider '${provider.kind}' is configured but ${provider.keyEnv} is not set.`);
     }
 }
-// Embed entries without a cached vector in save-checkpointed chunks.
-// provider.embed batches internally (one HTTP request per batchSize for API
-// providers; sequential for local). A `[]` row means that single item could
-// not be embedded and is left for a later run (resumable). On a hard provider
-// failure mid-backfill we persist the chunks already embedded this run rather
-// than discarding paid progress, then stop and resume on the next run.
+// Embeds entries without a cached vector in save-checkpointed chunks; a `[]` row means that item could not be embedded and is retried next run.
+// On a hard provider failure mid-backfill the chunks already embedded are persisted (paid progress), then it stops and resumes next run.
 async function backfillPending(hippoRoot, provider, pending, model) {
     let count = 0;
-    // Initialized to `undefined` (not a known-evidence literal like `null`) so
-    // it stays a plain `unknown` binding for the arbitrary caught value below;
-    // falsy either way, so `if (backfillError)` behaves identically.
+    // Initialized to `undefined` so it stays a plain `unknown` binding for the arbitrary caught value; falsy either way.
     let backfillError = undefined;
-    const SAVE_CHUNK = 64;
-    for (let i = 0; i < pending.length; i += SAVE_CHUNK) {
-        const chunk = pending.slice(i, i + SAVE_CHUNK);
+    for (const chunk of entryPages(hippoRoot, pending)) {
         let vectors;
         try {
             vectors = await provider.embed(chunk.map((e) => embeddingInputText(e)), 'passage');
@@ -516,7 +391,7 @@ async function backfillPending(hippoRoot, provider, pending, model) {
             else
                 noteSkippedEmbedding(chunk[j].id);
         }
-        count += withVectorDb(hippoRoot, (db) => upsertVectors(db, rows, model));
+        count += saveStoredVectors(hippoRoot, rows, model);
     }
     return { count, backfillError };
 }
@@ -531,24 +406,13 @@ export async function embedAll(hippoRoot, model, provider = resolveEmbeddingProv
         return withProcessEmbedLock(() => embedAllInStore(store, provider));
     return withEmbedLock(hippoRoot, async () => {
         const identity = provider.id;
-        // Host-wide by design. embedAll backfills vectors for all tenants'
-        // entries into the per-host embedding index. Per-tenant filtering would
-        // produce partial indices and break recall.
-        const entries = loadAllEntries(hippoRoot);
-        const model = embeddingIndexIdentity(identity);
-        if (embeddingModelRequiresReindex(hippoRoot, identity)) {
-            const rebuiltIndex = await rebuildEmbeddingIndex(entries, provider);
-            saveEmbeddingIndex(hippoRoot, rebuiltIndex, model);
-            saveStoredEmbeddingModel(hippoRoot, identity);
-            resetPhysicsFromIndex(hippoRoot, entries, rebuiltIndex);
-            return Object.keys(rebuiltIndex).length;
-        }
-        const embedded = withVectorDb(hippoRoot, (db) => {
-            deleteOrphanVectors(db);
-            return storedVectorIds(db);
-        });
-        const pending = entries.filter((e) => !embedded.has(e.id));
-        const { count, backfillError } = await backfillPending(hippoRoot, provider, pending, model);
+        if (embeddingModelRequiresReindex(hippoRoot, identity))
+            return rebuildIndexForProvider(hippoRoot, provider);
+        // Host-wide by design: embedAll backfills all tenants' entries into the per-host index; per-tenant filtering would produce partial indices and break
+        // recall.
+        const embedded = pruneStoredVectors(hippoRoot);
+        const pending = loadAllEntryIds(hippoRoot).filter((id) => !embedded.has(id));
+        const { count, backfillError } = await backfillPending(hippoRoot, provider, pending, embeddingIndexIdentity(identity));
         saveStoredEmbeddingModel(hippoRoot, identity);
         // Partial progress is now persisted; surface a hard backfill failure so the
         // explicit embed path reports it (best-effort callers go via embedMemory).

package/dist/store/embeddings/local.js

@@ -1,4 +1,4 @@
-// The local Transformers.js embedder, a leaf so embedding-provider.ts can wrap it without importing embeddings.ts.
+// The local Transformers.js embedder, a leaf so provider.ts can wrap it without importing index.ts.
 import { envModelCache } from '../../util/env.js';
 import * as fs from 'fs';
 import * as path from 'path';
@@ -25,11 +25,6 @@ export function prefixFor(model, role) {
     }
     return '';
 }
-// Use Function constructor to bypass TypeScript static module resolution
-// for optional peer dependencies that may not be installed.
-// SAFETY: `import(s)` always resolves to a module namespace object (or rejects);
-// Promise<object> names that honestly without claiming a specific module shape.
-const _dynImport = new Function('s', 'return import(s)');
 /** Check (synchronously) if @xenova/transformers or @huggingface/transformers is installed. */
 export function isEmbeddingAvailable() {
     if (_embeddingAvailable !== null)
@@ -71,20 +66,19 @@ function resolveTransformersPackage() {
         return null; // neither optional package is installed; callers fall back to no local embeddings
     }
 }
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
 async function loadPipeline(model) {
-    if (_pipelineInstances.has(model))
-        return _pipelineInstances.get(model);
-    if (_pipelineLoading.has(model))
-        return _pipelineLoading.get(model);
+    const loaded = _pipelineInstances.get(model);
+    if (loaded)
+        return loaded;
+    const inFlight = _pipelineLoading.get(model);
+    if (inFlight)
+        return inFlight;
     const loading = createPipeline(model);
     _pipelineLoading.set(model, loading);
     return loading;
 }
 /** The model's pipeline, or null with the reason recorded in `_pipelineErrors`. */
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
 async function createPipeline(model) {
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
     const pkg = resolveTransformersPackage();
     if (!pkg) {
         _pipelineErrors.set(model, 'no transformers package is installed');
@@ -113,23 +107,18 @@ async function createPipeline(model) {
     }
 }
 /** The package's `pipeline` function, or null with the reason recorded in `_pipelineErrors`. */
-// eslint-disable-next-line @typescript-eslint/no-explicit-any
 async function importPipelineFactory(pkg, model) {
     let pipelineFn = null;
     try {
-        // SAFETY: the resolved module's shape is untyped by design (optional peer
-        // dependency); pipelineFn/mod.env are read defensively below and any
-        // failure to find a usable pipeline falls through to `return null`.
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
-        const mod = await _dynImport(pkg);
-        if (envModelCache()) {
-            if (mod.env) {
-                mod.env.cacheDir = envModelCache();
-                mod.env.localModelPath = envModelCache();
-                mod.env.allowRemoteModels = false;
-            }
+        // The package is an optional peer, so the dynamic import is typed by what this file reads; a missing export falls to the null below.
+        const mod = await import(/* @vite-ignore */ pkg);
+        const cache = envModelCache();
+        if (cache && mod.env) {
+            mod.env.cacheDir = cache;
+            mod.env.localModelPath = cache;
+            mod.env.allowRemoteModels = false;
         }
-        pipelineFn = mod.pipeline ?? mod.default?.pipeline;
+        pipelineFn = mod.pipeline ?? mod.default?.pipeline ?? null;
     }
     catch (err) {
         // String(err) keeps a Node error's [ERR_...] code, which callers match on.
@@ -174,13 +163,7 @@ export async function getEmbedding(text, model = DEFAULT_EMBEDDING_MODEL, role)
             return [];
         const prefix = prefixFor(model, role);
         const input = prefix ? `${prefix}${text}` : text;
-        // SAFETY: pipe() is a Transformers.js feature-extraction pipeline call;
-        // its untyped output is read defensively below (only `.data`, cast on
-        // the return line to the documented Float32Array tensor shape).
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
         const output = await pipe(input, { pooling: poolingFor(model), normalize: true });
-        // SAFETY: output.data is a Float32Array per the feature-extraction
-        // pipeline's documented tensor output shape.
         return Array.from(output.data);
     }
     catch (err) {