hippo-memory 1.71.0 → 2.0.0

package/dist/rerankers/index.js

@@ -1,13 +1,14 @@
 import { createClefReranker } from './clef.js';
-import { crossEncoderReranker } from './cross-encoder.js';
+import { createCrossEncoderReranker } from './cross-encoder.js';
 import { createJevReranker } from './jev.js';
 import { createLlmReranker } from './llm.js';
-// Each hosted reranker is built once here, so its outage warning lasts the process.
+// Each reranker is built once here, so its outage warning lasts the process; jev falls back to the same cross-encoder, so one model loads.
+const crossEncoder = createCrossEncoderReranker();
 const REGISTRY = {
     clef: createClefReranker('clef'),
     'clef-flash': createClefReranker('clef-flash'),
-    'cross-encoder': crossEncoderReranker,
-    jev: createJevReranker(crossEncoderReranker),
+    'cross-encoder': crossEncoder,
+    jev: createJevReranker(crossEncoder),
     llm: createLlmReranker(),
 };
 function isRegisteredRerankerName(name) {

package/dist/rerankers/jev.js

@@ -1,3 +1,4 @@
+import { truncateWithEllipsis } from '../util/ellipsize.js';
 import { envJevModel, envJevTimeoutMs, envTypesafeApiKey } from '../util/env.js';
 import { redactSecretsStrict } from '../util/secret-detect.js';
 import { createOutageWarning } from './outage-warning.js';
@@ -15,9 +16,6 @@ const TRUNCATE_CHARS = 1200;
 const DEFAULT_MODEL = 'jev-1.13.0';
 // The pool size the eval numbers were measured at.
 export const JEV_DEFAULT_TOP_K = 40;
-function truncate(s, n) {
-    return s.length <= n ? s : `${s.slice(0, n)}...`;
-}
 // Number.isFinite rejects a string or null without coercing it, which the
 // declared type cannot promise about a third-party payload.
 function isProbability(v) {
@@ -40,7 +38,7 @@ function parseScores(body, n) {
 /** Redacted query plus numbered, redacted, truncated candidates. Shared with CLEF so both arms see matched input. */
 export function buildRelevanceRequest(query, head) {
     // Strict: this text leaves the machine, so Bearer, Basic-auth and JWT shapes go too.
-    const lines = head.map((r, i) => `[${i + 1}] ${truncate(redactSecretsStrict(r.entry.content), TRUNCATE_CHARS)}`);
+    const lines = head.map((r, i) => `[${i + 1}] ${truncateWithEllipsis(redactSecretsStrict(r.entry.content), TRUNCATE_CHARS)}`);
     const state = `Query: ${redactSecretsStrict(query)}\n\nNumbered candidate memories from an AI coding agent's project store:\n\n${lines.join('\n\n')}`;
     const questions = {};
     for (let i = 1; i <= head.length; i++) {

package/dist/rerankers/outage-warning.d.ts

@@ -1,5 +1,7 @@
+import { type LogFields } from '../util/log.js';
 export interface OutageWarning {
-    failed(reason: string): void;
+    /** `fields` carries the error's class and stack onto the warning line. */
+    failed(reason: string, fields?: LogFields): void;
     answered(): void;
 }
 /** Warns when `name` starts failing, at most every five minutes while it keeps failing,

package/dist/rerankers/outage-warning.js

@@ -1,6 +1,7 @@
 // One way for a reranker to report that it stopped working, without a line per recall.
+import { MINUTE_MS } from '../util/time.js';
 import { log } from '../util/log.js';
-const REPEAT_MS = 5 * 60_000;
+const REPEAT_MS = 5 * MINUTE_MS;
 /** Warns when `name` starts failing, at most every five minutes while it keeps failing,
  * and once when it answers again. `fallback` says what recall does meanwhile. */
 export function createOutageWarning(name, fallback) {
@@ -8,14 +9,15 @@ export function createOutageWarning(name, fallback) {
     let unreported = 0;
     let warnedAt = 0;
     return {
-        failed(reason) {
+        failed(reason, fields) {
             failures++;
             if (failures > 1 && Date.now() - warnedAt < REPEAT_MS) {
                 unreported++;
                 return;
             }
             const skipped = unreported > 0 ? ` ${unreported} more calls failed since the last warning.` : '';
-            log.warn(`${name} reranker unavailable (${reason}); ${fallback}.${skipped} While it fails this repeats at most every ${REPEAT_MS / 60_000} minutes.`);
+            const repeat = `While it fails this repeats at most every ${REPEAT_MS / MINUTE_MS} minutes.`;
+            log.warn(`${name} reranker unavailable (${reason}); ${fallback}.${skipped} ${repeat}`, fields);
             warnedAt = Date.now();
             unreported = 0;
         },

package/dist/search/bm25-search.js

@@ -2,7 +2,7 @@ import { estimateTokens } from '../util/token-text.js';
 import { tokenize } from '../util/tokenize.js';
 import { evalNow } from '../core/ablation.js';
 import { extractPathTags } from './path-context.js';
-import { detectScope } from '../sharing/scope.js';
+import { detectScope } from '../core/active-scope.js';
 import { compareScoredResults } from '../core/compare.js';
 import { bm25Score, buildCorpus, entryText } from './bm25.js';
 import { currentEntries } from './as-of.js';

package/dist/search/boosts.js

@@ -2,7 +2,7 @@ import { envSummaryDeboost } from '../util/env.js';
 import { calculateStrength, CHURN_STALE_TAG } from '../core/memory.js';
 import { isOutcomeFastAblated, isRecencyAblated, evalRecencyScaleDays } from '../core/ablation.js';
 import { pathBoostMultiplier } from './path-context.js';
-import { scopeMatch } from '../sharing/scope.js';
+import { scopeMatch } from '../core/active-scope.js';
 import { temporalBoost } from './temporal.js';
 import { DAY_MS } from '../util/time.js';
 const CHURN_STALE_RANK_MULTIPLIER = 0.5; // SHORTCUT: untuned; measure on real recall before any default.

package/dist/search/embed-memories.d.ts

@@ -0,0 +1,12 @@
+import type { MemoryEntry } from '../core/memory.js';
+import { type EmbeddingProvider } from '../embeddings/provider.js';
+import { type HippoStore } from '../store/port.js';
+/** `off` is a provider that is disabled or not installed, which is a choice; `failed` is a memory that stays out of the vector index. */
+export type EmbedOutcome = 'indexed' | 'off' | 'failed';
+/** Embed a single memory entry and cache the result in the embedding index; with `store`, through its vectorWrites, never opening hippo.db.
+ *  Resolves whatever fails, so a caller that fires and forgets stays safe; the outcome says whether the memory reached the index. */
+export declare function embedMemory(hippoRoot: string, entry: MemoryEntry, model?: string, store?: HippoStore): Promise<EmbedOutcome>;
+/** Embeds every entry with no cached vector, prunes vectors of deleted memories, and returns how many it embedded; `provider` defaults to the
+ *  configured one. With `store`, every tenant's memories with no vector for the provider's model go through its `vectorWrites`, never hippo.db. */
+export declare function embedAll(hippoRoot: string, model?: string, provider?: EmbeddingProvider, store?: HippoStore): Promise<number>;
+//# sourceMappingURL=embed-memories.d.ts.map

package/dist/search/embed-memories.js

@@ -0,0 +1,389 @@
+// Embeds memories into the vector index: picks the configured provider, computes each vector and hands it to the store,
+// which only keeps it (src/store/port.ts VectorWrites).
+import * as fs from 'fs';
+import * as path from 'path';
+import { randomUUID } from 'crypto';
+import { loadConfig } from '../core/config.js';
+import { initializeParticle } from '../core/physics.js';
+import { resolveEmbeddingProvider } from '../embeddings/provider.js';
+import { embeddingIndexIdentity, embeddingInputText, embeddingModelRequiresReindex, indexNeedsRebuild, saveStoredEmbeddingModel, } from '../store/embeddings/index.js';
+import { loadAllEntryIds, loadEntriesByIds } from '../store/entry-reads.js';
+import { requireGroup, } from '../store/port.js';
+import { indexedModel, pruneStoredVectors, replacesIndex, resetStoredParticles, saveEmbeddingIndex, saveStoredVectors, seedStoredParticle, } from '../store/vector-index.js';
+import { chunked } from '../util/chunked.js';
+import { envByName } from '../util/env.js';
+import { errorCode, errorMessage, log } from '../util/log.js';
+import { redactSecretsStrict } from '../util/secret-detect.js';
+const STORE_PAGE = 64;
+/** The rows for `ids`, a page at a time and in that order; an id deleted since it was listed is skipped. */
+function* entryPages(hippoRoot, ids) {
+    for (const page of chunked(ids, STORE_PAGE)) {
+        const byId = new Map(loadEntriesByIds(hippoRoot, page).map((e) => [e.id, e]));
+        const entries = page.flatMap((id) => byId.get(id) ?? []);
+        if (entries.length > 0)
+            yield entries;
+    }
+}
+async function rebuildEmbeddingIndex(hippoRoot, ids, provider) {
+    const rebuilt = {};
+    for (const entries of entryPages(hippoRoot, ids)) {
+        // On a hard transport/auth failure provider.embed throws, so the caller aborts WITHOUT saving
+        // a partial index (atomic reindex: the old index + stored identity are preserved).
+        const vectors = await provider.embed(entries.map((e) => embeddingInputText(e)), 'passage');
+        for (let i = 0; i < entries.length; i++) {
+            const vec = vectors[i];
+            if (vec && vec.length > 0) {
+                rebuilt[entries[i].id] = vec;
+            }
+            else {
+                noteSkippedEmbedding(entries[i].id);
+            }
+        }
+    }
+    return rebuilt;
+}
+function resetPhysicsFromIndex(hippoRoot, ids, index) {
+    try {
+        resetStoredParticles(hippoRoot, ids, index);
+    }
+    catch (err) {
+        // Best effort: retrieval still falls back without physics state.
+        log.warn(`physics reset after reindex failed: ${errorMessage(err)}`);
+    }
+}
+/** A provider's `[]` row is a swallowed per-item failure; name the memory so the gap can be traced. */
+function noteSkippedEmbedding(id) {
+    log.warn('memory not embedded; the next embed run retries it', { id });
+}
+const EMBED_LOCK_FILE = 'embeddings.lock';
+const EMBED_LOCK_WAIT_MS = 10_000;
+const EMBED_LOCK_OWNER = `${process.pid}:${randomUUID()}`;
+// In-process mutex plus an O_EXCL "<pid>:<token>" lock file: our token is a lock we leaked; our PID
+// with another token is a live worker thread, unless the lock predates this process (a reused PID).
+let _embedWriteLock = Promise.resolve();
+function embedLockHolderAlive(lockPath) {
+    let raw;
+    try {
+        raw = fs.readFileSync(lockPath, 'utf8');
+    }
+    catch (err) {
+        return !(err instanceof Error && 'code' in err && err.code === 'ENOENT');
+    }
+    const mtimeMs = fs.statSync(lockPath, { throwIfNoEntry: false })?.mtimeMs ?? 0;
+    const pid = Number(raw.split(':')[0]);
+    // An empty lock is a holder between create and write; after 5 s it is a crashed one.
+    if (!Number.isInteger(pid) || pid <= 0)
+        return Date.now() - mtimeMs < 5_000;
+    if (raw === EMBED_LOCK_OWNER)
+        return false;
+    if (pid === process.pid)
+        return mtimeMs >= Date.now() - process.uptime() * 1000;
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (err) {
+        return err instanceof Error && 'code' in err && err.code === 'EPERM';
+    }
+}
+// After the double-break race below the file can be another writer's lock, so remove only our own.
+function releaseEmbedFileLock(lockPath) {
+    let raw;
+    try {
+        raw = fs.readFileSync(lockPath, 'utf8');
+    }
+    catch (err) {
+        if (err instanceof Error && 'code' in err && err.code === 'ENOENT')
+            return;
+        throw err;
+    }
+    if (raw === EMBED_LOCK_OWNER)
+        fs.rmSync(lockPath, { force: true });
+}
+async function acquireEmbedFileLock(hippoRoot) {
+    const lockPath = path.join(hippoRoot, EMBED_LOCK_FILE);
+    const deadline = Date.now() + EMBED_LOCK_WAIT_MS;
+    for (;;) {
+        try {
+            fs.writeFileSync(lockPath, EMBED_LOCK_OWNER, { flag: 'wx' });
+            return () => releaseEmbedFileLock(lockPath);
+        }
+        catch (err) {
+            if (!(err instanceof Error && 'code' in err && err.code === 'EEXIST'))
+                throw err;
+        }
+        // SHORTCUT: two waiters can both break one dead holder's lock; a vector lost that way is backfilled by the next `hippo embed`.
+        if (!embedLockHolderAlive(lockPath)) {
+            fs.rmSync(lockPath, { force: true });
+            continue;
+        }
+        if (Date.now() >= deadline)
+            throw new Error(`the embedding index is busy: another hippo process holds ${lockPath}`);
+        await new Promise((r) => setTimeout(r, 50));
+    }
+}
+async function withEmbedLock(hippoRoot, fn) {
+    return withProcessEmbedLock(async () => {
+        const release = await acquireEmbedFileLock(hippoRoot);
+        try {
+            return await fn();
+        }
+        finally {
+            release();
+        }
+    });
+}
+// A store keeps each write whole in its own transaction, and a lock file in hippoRoot would not reach a worker on another host.
+async function withProcessEmbedLock(fn) {
+    let resolve;
+    const next = new Promise(r => { resolve = r; });
+    const prev = _embedWriteLock;
+    _embedWriteLock = next;
+    await prev;
+    try {
+        return await fn();
+    }
+    finally {
+        resolve();
+    }
+}
+// A bad key fails every write; one full warning per distinct failure tells the user, and a count of the repeats follows at most once per interval.
+const EMBED_WARN_KEY_CAP = 64;
+const EMBED_SUPPRESSED_SUMMARY_INTERVAL_MS = 60_000;
+const embedFailureSeen = new Map();
+function embedFailureKey(source, cause) {
+    const kind = errorCode(cause) || (cause instanceof Error ? cause.constructor.name : 'NonError');
+    return `${source}:${kind}`;
+}
+function warnEmbedFailureOnce(source, rawMessage, cause) {
+    const key = embedFailureKey(source, cause);
+    const now = Date.now();
+    const seen = embedFailureSeen.get(key);
+    if (seen) {
+        seen.suppressed += 1;
+        if (now - seen.lastNoticeAt >= EMBED_SUPPRESSED_SUMMARY_INTERVAL_MS) {
+            log.warn(`${seen.suppressed} more embedding errors from ${source} since the last notice; memories are still stored without embeddings.`);
+            seen.suppressed = 0;
+            seen.lastNoticeAt = now;
+        }
+        return;
+    }
+    // Oldest key out, so a full map still lets a new failure warn.
+    if (embedFailureSeen.size >= EMBED_WARN_KEY_CAP) {
+        for (const oldest of embedFailureSeen.keys()) {
+            embedFailureSeen.delete(oldest);
+            break;
+        }
+    }
+    embedFailureSeen.set(key, { lastNoticeAt: now, suppressed: 0 });
+    // Strict scrub: this line can land in a hook log file, and an API may echo the key back in its error body.
+    const message = redactSecretsStrict(rawMessage).replace(/\s+/g, ' ').replace(/\.+$/, '');
+    log.warn(`embedding failed (${source}): ${message}. Memories are stored without embeddings until this is fixed.`);
+}
+function vectorGroups(store) {
+    return [requireGroup(store, 'vectors'), requireGroup(store, 'vectorWrites')];
+}
+/** Writes one page's vectors, one writeVectors per tenant, and stops at the first refused write; `withParticles` seeds each memory's particle. */
+async function writeVectorPage(writes, write, page, vectors, withParticles) {
+    const byTenant = new Map();
+    page.forEach((entry, i) => {
+        const vector = vectors[i];
+        if (!vector || vector.length === 0)
+            return noteSkippedEmbedding(entry.id);
+        const row = withParticles ? { memoryId: entry.id, vector, particle: initializeParticle(entry, vector) } : { memoryId: entry.id, vector };
+        byTenant.set(entry.tenant_id, [...(byTenant.get(entry.tenant_id) ?? []), row]);
+    });
+    let written = 0;
+    for (const [tenantId, rows] of byTenant) {
+        const result = await writes.writeVectors({ ...write, tenantId, rows });
+        if (result.modelMismatch)
+            return { written, modelMismatch: true };
+        written += result.written;
+    }
+    return { written, modelMismatch: false };
+}
+const OTHER_MODEL_INDEX = "the vector index was built by another embedding model; run 'hippo embed' to rebuild it";
+/** Embeds every memory with no vector under `model`, page by page, so a provider failure keeps the pages written; a rebuild reseeds the particles it drops. */
+async function backfillInStore(writes, provider, model, rebuild) {
+    let count = 0;
+    let afterId;
+    for (;;) {
+        const page = await writes.entriesWithoutVector({ model, afterId, limit: STORE_PAGE });
+        if (page.length === 0)
+            return count;
+        afterId = page[page.length - 1].id;
+        const vectors = await provider.embed(page.map((e) => embeddingInputText(e)), 'passage');
+        const result = await writeVectorPage(writes, { model, replaceIndex: rebuild }, page, vectors, rebuild);
+        // Another process rebuilt the index mid-run; every later page would be refused too, after paying the provider for it.
+        if (result.modelMismatch)
+            throw new Error(OTHER_MODEL_INDEX);
+        count += result.written;
+    }
+}
+/** embedAll on a store: a model change re-embeds every memory, and the first page written drops the old index. */
+async function embedAllInStore(store, provider) {
+    const [reads, writes] = vectorGroups(store);
+    const rebuild = indexNeedsRebuild(indexedModel(await reads.embeddingIndexState()), provider.id);
+    return backfillInStore(writes, provider, embeddingIndexIdentity(provider.id), rebuild);
+}
+/** embedMemory on a store; like the hippo.db path it resolves whatever fails. Unlike it, it leaves a model change to embedAll, so a stale
+ *  read here never drops another process's rebuild; the read only saves a paid embed call, as the write refuses another model itself. */
+function embedMemoryInStore(store, provider, entry) {
+    return withProcessEmbedLock(async () => {
+        const [reads, writes] = vectorGroups(store);
+        const model = embeddingIndexIdentity(provider.id);
+        try {
+            const refused = replacesIndex(await reads.embeddingIndexState(), model)
+                || (await writeVectorPage(writes, { model, replaceIndex: false }, [entry], await provider.embed([embeddingInputText(entry)], 'passage'), true)).modelMismatch;
+            if (refused)
+                warnEmbedFailureOnce('index', OTHER_MODEL_INDEX);
+            return refused ? 'failed' : 'indexed';
+        }
+        catch (err) {
+            warnEmbedFailureOnce(provider.kind, errorMessage(err), err);
+            return 'failed';
+        }
+    }).catch((err) => {
+        log.warn(`skipped embedding ${entry.id} (${errorMessage(err)})`);
+        return 'failed';
+    });
+}
+const INLINE_REINDEX_MAX_ROWS = 256;
+/** Embed a single memory entry and cache the result in the embedding index; with `store`, through its vectorWrites, never opening hippo.db.
+ *  Resolves whatever fails, so a caller that fires and forgets stays safe; the outcome says whether the memory reached the index. */
+export async function embedMemory(hippoRoot, entry, model, store) {
+    let provider;
+    try {
+        provider = resolveEmbeddingProvider(hippoRoot, { model });
+    }
+    catch (err) {
+        // Callers fire and forget, so this must resolve: a bad config warns once instead of rejecting.
+        warnEmbedFailureOnce('config', errorMessage(err), err);
+        return 'failed';
+    }
+    if (!provider.isAvailable())
+        return 'off';
+    if (store)
+        return embedMemoryInStore(store, provider, entry);
+    return withEmbedLock(hippoRoot, () => embedMemoryLocked(hippoRoot, provider, entry)).catch((err) => {
+        log.warn(`skipped embedding ${entry.id} (${errorMessage(err)}); run 'hippo embed' to backfill`);
+        return 'failed';
+    });
+}
+/** Best-effort: an embedding failure (API down, bad key, 5xx) must not reject the caller's write; the existing index is left as-is.
+ *  Failures surface on the explicit `hippo embed` / `embedAll` path. */
+async function embedMemoryLocked(hippoRoot, provider, entry) {
+    try {
+        const identity = provider.id;
+        if (embeddingModelRequiresReindex(hippoRoot, identity)) {
+            // A fire-and-forget write heals a small index inline; a large one is left to `hippo embed`, so no write holds the lock for a whole-store embed.
+            if (loadAllEntryIds(hippoRoot).length > INLINE_REINDEX_MAX_ROWS) {
+                warnEmbedFailureOnce('index', OTHER_MODEL_INDEX);
+                return 'failed';
+            }
+            await rebuildIndexForProvider(hippoRoot, provider);
+            return 'indexed';
+        }
+        const [vector] = await provider.embed([embeddingInputText(entry)], 'passage');
+        if (!vector || vector.length === 0)
+            return 'failed';
+        saveStoredVectors(hippoRoot, [[entry.id, vector]], embeddingIndexIdentity(identity));
+        saveStoredEmbeddingModel(hippoRoot, identity);
+        initializePhysicsIfMissing(hippoRoot, entry, vector);
+        return 'indexed';
+    }
+    catch (err) {
+        // Provider failure (API down / bad key). Best-effort: leave the index as-is, but say so once.
+        warnEmbedFailureOnce(provider.kind, errorMessage(err), err);
+        return 'failed';
+    }
+}
+// Every page is embedded before one transaction replaces the index, and the identity is saved only after that: a provider failure or a crash
+// at any earlier point leaves the old identity, so embeddingModelRequiresReindex has the next run rebuild from the start.
+async function rebuildIndexForProvider(hippoRoot, provider) {
+    const identity = provider.id;
+    // Host-wide rebuild: one embedding index per hippoRoot, keyed by tenant-scoped entry.id; per-tenant indices would be a larger architecture change.
+    const ids = loadAllEntryIds(hippoRoot);
+    const rebuiltIndex = await rebuildEmbeddingIndex(hippoRoot, ids, provider);
+    saveEmbeddingIndex(hippoRoot, rebuiltIndex, embeddingIndexIdentity(identity));
+    saveStoredEmbeddingModel(hippoRoot, identity);
+    resetPhysicsFromIndex(hippoRoot, ids, rebuiltIndex);
+    return Object.keys(rebuiltIndex).length;
+}
+function initializePhysicsIfMissing(hippoRoot, entry, vector) {
+    try {
+        seedStoredParticle(hippoRoot, entry, vector);
+    }
+    catch (err) {
+        // Physics init is best-effort and must not fail the embedding that just landed.
+        log.debug(`physics state not initialised for ${entry.id}: ${errorMessage(err)}`);
+    }
+}
+/** Throws when an unavailable provider is a misconfiguration rather than an intentional no-op. */
+function throwIfProviderKeyMissing(hippoRoot, provider) {
+    // A configured API provider with a missing key is a misconfiguration: surface it so callers of the exported embedAll() learn nothing was written.
+    // Local-not-installed and an explicit enabled=false stay silent no-ops.
+    const cfg = loadConfig(hippoRoot).embeddings;
+    if (provider.kind !== 'local' &&
+        cfg.enabled !== false &&
+        provider.keyEnv &&
+        !envByName(provider.keyEnv)?.trim()) {
+        throw new Error(`Embedding provider '${provider.kind}' is configured but ${provider.keyEnv} is not set.`);
+    }
+}
+// Embeds entries without a cached vector in save-checkpointed chunks; a `[]` row means that item could not be embedded and is retried next run.
+// On a hard provider failure mid-backfill the chunks already embedded are persisted (paid progress), then it stops and resumes next run.
+async function backfillPending(hippoRoot, provider, pending, model) {
+    let count = 0;
+    // Initialized to `undefined` so it stays a plain `unknown` binding for the arbitrary caught value; falsy either way.
+    let backfillError = undefined;
+    for (const chunk of entryPages(hippoRoot, pending)) {
+        let vectors;
+        try {
+            vectors = await provider.embed(chunk.map((e) => embeddingInputText(e)), 'passage');
+        }
+        catch (err) {
+            // Preserve the chunks already saved this run, then surface the failure
+            // below so the explicit `hippo embed` path never reports a false success.
+            backfillError = err;
+            break;
+        }
+        const rows = [];
+        for (let j = 0; j < chunk.length; j++) {
+            const vec = vectors[j];
+            if (vec && vec.length > 0)
+                rows.push([chunk[j].id, vec]);
+            else
+                noteSkippedEmbedding(chunk[j].id);
+        }
+        count += saveStoredVectors(hippoRoot, rows, model);
+    }
+    return { count, backfillError };
+}
+/** Embeds every entry with no cached vector, prunes vectors of deleted memories, and returns how many it embedded; `provider` defaults to the
+ *  configured one. With `store`, every tenant's memories with no vector for the provider's model go through its `vectorWrites`, never hippo.db. */
+export async function embedAll(hippoRoot, model, provider = resolveEmbeddingProvider(hippoRoot, { model }), store) {
+    if (!provider.isAvailable()) {
+        throwIfProviderKeyMissing(hippoRoot, provider);
+        return 0;
+    }
+    if (store)
+        return withProcessEmbedLock(() => embedAllInStore(store, provider));
+    return withEmbedLock(hippoRoot, async () => {
+        const identity = provider.id;
+        if (embeddingModelRequiresReindex(hippoRoot, identity))
+            return rebuildIndexForProvider(hippoRoot, provider);
+        // Host-wide by design: embedAll backfills all tenants' entries into the per-host index; per-tenant filtering would produce partial indices and break
+        // recall.
+        const embedded = pruneStoredVectors(hippoRoot);
+        const pending = loadAllEntryIds(hippoRoot).filter((id) => !embedded.has(id));
+        const { count, backfillError } = await backfillPending(hippoRoot, provider, pending, embeddingIndexIdentity(identity));
+        saveStoredEmbeddingModel(hippoRoot, identity);
+        // Partial progress is now persisted; surface a hard backfill failure so the
+        // explicit embed path reports it (best-effort callers go via embedMemory).
+        if (backfillError)
+            throw backfillError;
+        return count;
+    });
+}
+//# sourceMappingURL=embed-memories.js.map

package/dist/search/engine.d.ts

@@ -0,0 +1,8 @@
+import type { MemoryEntry } from '../core/memory.js';
+import type { PhysicsConfig } from '../core/physics-config.js';
+import type { SearchResult } from '../core/search-types.js';
+import { type HybridSearchOptions } from './hybrid.js';
+import type { VectorSource } from './vector.js';
+/** Physics scoring when `physics` holds its config, hybrid otherwise, both on the store `source` names; each caller keeps its own options. */
+export declare function searchByEngine(source: VectorSource, query: string, pool: MemoryEntry[], physics: PhysicsConfig | null, options: Omit<HybridSearchOptions, 'hippoRoot' | 'store'>): Promise<SearchResult[]>;
+//# sourceMappingURL=engine.d.ts.map

package/dist/search/engine.js

@@ -0,0 +1,10 @@
+// The one search-engine choice every ranker over a candidate pool makes: recall's band, MCP's wide list, the CLI core and context.
+import { hybridSearch } from './hybrid.js';
+import { physicsSearchIn } from './physics-search.js';
+/** Physics scoring when `physics` holds its config, hybrid otherwise, both on the store `source` names; each caller keeps its own options. */
+export function searchByEngine(source, query, pool, physics, options) {
+    return physics
+        ? physicsSearchIn(source, query, pool, { ...options, physicsConfig: physics })
+        : hybridSearch(query, pool, { ...options, hippoRoot: source.root, store: source.store });
+}
+//# sourceMappingURL=engine.js.map

package/dist/search/goal-boost.d.ts

@@ -1,4 +1,4 @@
-import type { MemoryEntry } from '../core/memory.js';
+import { type MemoryEntry } from '../core/memory.js';
 import type { RerankStep } from '../core/search-types.js';
 import type { ActiveGoals, GoalRecallLogRow } from '../store/goals.js';
 /** Options of {@link boostByGoals}. */

package/dist/search/goal-boost.js

@@ -1,10 +1,13 @@
 // src/search/goal-boost.ts: the goal-stack boost over goals already read; it ranks rows and touches no store.
 import { compareScoresDesc } from '../core/compare.js';
+import { DEFAULT_SCHEMA_FIT } from '../core/memory.js';
 const MAX_FINAL_MULTIPLIER = 3.0;
+const FIRST_MATCH_BOOST = 2.0;
+const EXTRA_MATCH_BOOST = 0.5;
 /** The capped boost for one row whose tags match `matches` active goals. */
 function goalBoostMultiplier(entry, tags, matches, goalsByTag, policiesByGoalId) {
-    // Base 2.0x for first match, +0.5x per additional, capped at 3.0x.
-    let multiplier = Math.min(2.0 + 0.5 * (matches.length - 1), MAX_FINAL_MULTIPLIER);
+    // First match boosts by FIRST_MATCH_BOOST, each further one adds EXTRA_MATCH_BOOST, capped at MAX_FINAL_MULTIPLIER.
+    let multiplier = Math.min(FIRST_MATCH_BOOST + EXTRA_MATCH_BOOST * (matches.length - 1), MAX_FINAL_MULTIPLIER);
     // Compose per-policy multipliers per matched tag.
     for (const tag of matches) {
         const goal = goalsByTag.get(tag);
@@ -20,7 +23,7 @@ function goalBoostMultiplier(entry, tags, matches, goalsByTag, policiesByGoalId)
             multiplier *=
                 1.0 +
                     Math.max(0, policy.weightSchemaFit - 1.0) *
-                        (entry.schema_fit ?? 0.5);
+                        (entry.schema_fit ?? DEFAULT_SCHEMA_FIT);
         }
         else if (policy.policyType === 'recency-first') {
             multiplier *= policy.weightRecency;

package/dist/search/hybrid.d.ts

@@ -1,10 +1,10 @@
 import type { MemoryEntry } from '../core/memory.js';
 import type { RerankerFn, RerankerOptions } from '../rerankers/types.js';
 import { type BM25Corpus } from './bm25.js';
-import { type HybridVectorCandidates } from './vector.js';
+import { type HybridVectorCandidates, type VectorSource } from './vector.js';
 import { type GraphStreamOptions } from './fusion.js';
 import { type ResultCost, type SearchResult } from '../core/search-types.js';
-import type { HippoStore } from '../store/index.js';
+import { type HippoStore } from '../store/index.js';
 export interface HybridSearchOptions {
     budget?: number;
     now?: Date;
@@ -46,6 +46,13 @@ export interface HybridSearchOptions {
     /** The deadline a caller already started for this recall's query embedding; a fresh budget when unset. */
     queryEmbedDeadline?: AbortSignal;
 }
+export declare const DEFAULT_EMBEDDING_WEIGHT = 0.6;
+export declare const DEFAULT_MMR_LAMBDA = 0.7;
 /** BM25 blended with cosine similarity when stored vectors and a provider are available, BM25 * strength * recency otherwise. */
 export declare function hybridSearch(query: string, entries: MemoryEntry[], options?: HybridSearchOptions): Promise<SearchResult[]>;
+/** hippo.db under `hippoRoot` when the caller passed no store: the default lives on this exported entry, never inside the vector arm. */
+export declare function vectorSourceOf({ hippoRoot, store }: {
+    hippoRoot?: string;
+    store?: HippoStore;
+}): VectorSource | undefined;
 //# sourceMappingURL=hybrid.d.ts.map

package/dist/search/hybrid.js

@@ -1,7 +1,7 @@
 import { tokenize } from '../util/tokenize.js';
 import { evalNow } from '../core/ablation.js';
 import { extractPathTags } from './path-context.js';
-import { detectScope } from '../sharing/scope.js';
+import { detectScope } from '../core/active-scope.js';
 import { compareScoredResults } from '../core/compare.js';
 import { bm25Score, buildCorpus, entryText } from './bm25.js';
 import { currentEntries } from './as-of.js';
@@ -13,9 +13,10 @@ import { scoreHybridPool } from './hybrid-score.js';
 import { applyMmrWindow, applyReranker } from './rerank.js';
 import { dedupeExtracted, fitBudget, withDagChildren } from './finalize.js';
 import { DEFAULT_RECALL_BUDGET } from '../core/search-types.js';
+import { sqliteStore } from '../store/index.js';
 // Share of the blended score that cosine takes, and the relevance side of the MMR relevance-versus-diversity trade.
-const DEFAULT_EMBEDDING_WEIGHT = 0.6;
-const DEFAULT_MMR_LAMBDA = 0.7;
+export const DEFAULT_EMBEDDING_WEIGHT = 0.6;
+export const DEFAULT_MMR_LAMBDA = 0.7;
 /** BM25 blended with cosine similarity when stored vectors and a provider are available, BM25 * strength * recency otherwise. */
 export async function hybridSearch(query, entries, options = {}) {
     const now = options.now ?? evalNow(); // honors HIPPO_FAKE_NOW (eval-only; see ablation.ts)
@@ -26,7 +27,7 @@ export async function hybridSearch(query, entries, options = {}) {
     const queryTerms = tokenize(query);
     if (queryTerms.length === 0)
         return [];
-    const arm = await resolveVectorArm(query, pool, options);
+    const arm = await resolveVectorArm(query, pool, vectorSourceOf(options), options);
     // A prepared corpus covers the caller's entries only, so rows the vector arm added force a rebuild.
     const corpus = (arm.addedRows ? undefined : options.preparedCorpus) ?? buildCorpus(arm.entries.map(entryText));
     const bm25Scores = arm.entries.map((_, i) => bm25Score(corpus, i, queryTerms));
@@ -48,6 +49,10 @@ export async function hybridSearch(query, entries, options = {}) {
     const ordered = await orderHybrid(query, withDagChildren(dedupeExtracted(scored), arm.entries), arm, options);
     return fitBudget(ordered, options.budget ?? DEFAULT_RECALL_BUDGET, options.minResults ?? 1, options.cost);
 }
+/** hippo.db under `hippoRoot` when the caller passed no store: the default lives on this exported entry, never inside the vector arm. */
+export function vectorSourceOf({ hippoRoot, store }) {
+    return hippoRoot ? { root: hippoRoot, store: store ?? sqliteStore(hippoRoot) } : undefined;
+}
 function hybridBoostContext(query, pool, now, options) {
     return {
         now,

package/dist/search/physics-search.d.ts

@@ -1,7 +1,7 @@
 import { type MemoryEntry } from '../core/memory.js';
 import { type PhysicsConfig } from '../core/physics-config.js';
-import { type HippoStore } from '../store/index.js';
-import { type HybridVectorCandidates } from './vector.js';
+import type { HippoStore } from '../store/index.js';
+import { type HybridVectorCandidates, type VectorSource } from './vector.js';
 import { type ResultCost, type SearchResult } from '../core/search-types.js';
 export interface PhysicsSearchOptions {
     budget?: number;
@@ -27,6 +27,9 @@ export interface PhysicsSearchOptions {
     /** Where vectors and particles are read; hippo.db under `hippoRoot` when unset. */
     store?: HippoStore;
 }
-/** Scores memories by gravity, momentum and cluster amplification; rows without physics state rank through hybridSearch. */
+/** Scores memories by gravity, momentum and cluster amplification; rows without physics state rank through hybridSearch.
+ *  Reads hippo.db under `hippoRoot` when no store is passed. */
 export declare function physicsSearch(query: string, entries: MemoryEntry[], options?: PhysicsSearchOptions): Promise<SearchResult[]>;
+/** physicsSearch on the store the caller names, so a caller holding its request's store never falls back to hippo.db. */
+export declare function physicsSearchIn(source: VectorSource, query: string, entries: MemoryEntry[], options?: PhysicsSearchOptions): Promise<SearchResult[]>;
 //# sourceMappingURL=physics-search.d.ts.map