hippo-memory 1.71.0 → 1.72.0

package/dist/search/embed-memories.js

@@ -0,0 +1,377 @@
+// Embeds memories into the vector index: picks the configured provider, computes each vector and hands it to the store,
+// which only keeps it (src/store/port.ts VectorWrites).
+import * as fs from 'fs';
+import * as path from 'path';
+import { randomUUID } from 'crypto';
+import { loadConfig } from '../core/config.js';
+import { initializeParticle } from '../db/physics-state.js';
+import { resolveEmbeddingProvider } from '../embeddings/provider.js';
+import { embeddingIndexIdentity, embeddingInputText, embeddingModelRequiresReindex, indexNeedsRebuild, saveStoredEmbeddingModel, } from '../store/embeddings/index.js';
+import { loadAllEntryIds, loadEntriesByIds } from '../store/entry-reads.js';
+import { requireGroup, } from '../store/port.js';
+import { indexedModel, pruneStoredVectors, replacesIndex, resetStoredParticles, saveEmbeddingIndex, saveStoredVectors, seedStoredParticle, } from '../store/vector-index.js';
+import { chunked } from '../util/chunked.js';
+import { envByName } from '../util/env.js';
+import { errorCode, errorMessage, log } from '../util/log.js';
+import { redactSecretsStrict } from '../util/secret-detect.js';
+const STORE_PAGE = 64;
+/** The rows for `ids`, a page at a time and in that order; an id deleted since it was listed is skipped. */
+function* entryPages(hippoRoot, ids) {
+    for (const page of chunked(ids, STORE_PAGE)) {
+        const byId = new Map(loadEntriesByIds(hippoRoot, page).map((e) => [e.id, e]));
+        const entries = page.flatMap((id) => byId.get(id) ?? []);
+        if (entries.length > 0)
+            yield entries;
+    }
+}
+async function rebuildEmbeddingIndex(hippoRoot, ids, provider) {
+    const rebuilt = {};
+    for (const entries of entryPages(hippoRoot, ids)) {
+        // On a hard transport/auth failure provider.embed throws, so the caller aborts WITHOUT saving
+        // a partial index (atomic reindex: the old index + stored identity are preserved).
+        const vectors = await provider.embed(entries.map((e) => embeddingInputText(e)), 'passage');
+        for (let i = 0; i < entries.length; i++) {
+            const vec = vectors[i];
+            if (vec && vec.length > 0) {
+                rebuilt[entries[i].id] = vec;
+            }
+            else {
+                noteSkippedEmbedding(entries[i].id);
+            }
+        }
+    }
+    return rebuilt;
+}
+function resetPhysicsFromIndex(hippoRoot, ids, index) {
+    try {
+        resetStoredParticles(hippoRoot, ids, index);
+    }
+    catch (err) {
+        // Best effort: retrieval still falls back without physics state.
+        log.warn(`physics reset after reindex failed: ${errorMessage(err)}`);
+    }
+}
+/** A provider's `[]` row is a swallowed per-item failure; name the memory so the gap can be traced. */
+function noteSkippedEmbedding(id) {
+    log.warn('memory not embedded; the next embed run retries it', { id });
+}
+const EMBED_LOCK_FILE = 'embeddings.lock';
+const EMBED_LOCK_WAIT_MS = 10_000;
+const EMBED_LOCK_OWNER = `${process.pid}:${randomUUID()}`;
+// In-process mutex plus an O_EXCL "<pid>:<token>" lock file: our token is a lock we leaked; our PID
+// with another token is a live worker thread, unless the lock predates this process (a reused PID).
+let _embedWriteLock = Promise.resolve();
+function embedLockHolderAlive(lockPath) {
+    let raw;
+    try {
+        raw = fs.readFileSync(lockPath, 'utf8');
+    }
+    catch (err) {
+        return !(err instanceof Error && 'code' in err && err.code === 'ENOENT');
+    }
+    const mtimeMs = fs.statSync(lockPath, { throwIfNoEntry: false })?.mtimeMs ?? 0;
+    const pid = Number(raw.split(':')[0]);
+    // An empty lock is a holder between create and write; after 5 s it is a crashed one.
+    if (!Number.isInteger(pid) || pid <= 0)
+        return Date.now() - mtimeMs < 5_000;
+    if (raw === EMBED_LOCK_OWNER)
+        return false;
+    if (pid === process.pid)
+        return mtimeMs >= Date.now() - process.uptime() * 1000;
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (err) {
+        return err instanceof Error && 'code' in err && err.code === 'EPERM';
+    }
+}
+// After the double-break race below the file can be another writer's lock, so remove only our own.
+function releaseEmbedFileLock(lockPath) {
+    let raw;
+    try {
+        raw = fs.readFileSync(lockPath, 'utf8');
+    }
+    catch (err) {
+        if (err instanceof Error && 'code' in err && err.code === 'ENOENT')
+            return;
+        throw err;
+    }
+    if (raw === EMBED_LOCK_OWNER)
+        fs.rmSync(lockPath, { force: true });
+}
+async function acquireEmbedFileLock(hippoRoot) {
+    const lockPath = path.join(hippoRoot, EMBED_LOCK_FILE);
+    const deadline = Date.now() + EMBED_LOCK_WAIT_MS;
+    for (;;) {
+        try {
+            fs.writeFileSync(lockPath, EMBED_LOCK_OWNER, { flag: 'wx' });
+            return () => releaseEmbedFileLock(lockPath);
+        }
+        catch (err) {
+            if (!(err instanceof Error && 'code' in err && err.code === 'EEXIST'))
+                throw err;
+        }
+        // SHORTCUT: two waiters can both break one dead holder's lock; a vector lost that way is backfilled by the next `hippo embed`.
+        if (!embedLockHolderAlive(lockPath)) {
+            fs.rmSync(lockPath, { force: true });
+            continue;
+        }
+        if (Date.now() >= deadline)
+            throw new Error(`the embedding index is busy: another hippo process holds ${lockPath}`);
+        await new Promise((r) => setTimeout(r, 50));
+    }
+}
+async function withEmbedLock(hippoRoot, fn) {
+    return withProcessEmbedLock(async () => {
+        const release = await acquireEmbedFileLock(hippoRoot);
+        try {
+            return await fn();
+        }
+        finally {
+            release();
+        }
+    });
+}
+// A store keeps each write whole in its own transaction, and a lock file in hippoRoot would not reach a worker on another host.
+async function withProcessEmbedLock(fn) {
+    let resolve;
+    const next = new Promise(r => { resolve = r; });
+    const prev = _embedWriteLock;
+    _embedWriteLock = next;
+    await prev;
+    try {
+        return await fn();
+    }
+    finally {
+        resolve();
+    }
+}
+// A bad key fails every write; one full warning per distinct failure tells the user, and a count of the repeats follows at most once per interval.
+const EMBED_WARN_KEY_CAP = 64;
+const EMBED_SUPPRESSED_SUMMARY_INTERVAL_MS = 60_000;
+const embedFailureSeen = new Map();
+function embedFailureKey(source, cause) {
+    const kind = errorCode(cause) || (cause instanceof Error ? cause.constructor.name : 'NonError');
+    return `${source}:${kind}`;
+}
+function warnEmbedFailureOnce(source, rawMessage, cause) {
+    const key = embedFailureKey(source, cause);
+    const now = Date.now();
+    const seen = embedFailureSeen.get(key);
+    if (seen) {
+        seen.suppressed += 1;
+        if (now - seen.lastNoticeAt >= EMBED_SUPPRESSED_SUMMARY_INTERVAL_MS) {
+            log.warn(`${seen.suppressed} more embedding errors from ${source} since the last notice; memories are still stored without embeddings.`);
+            seen.suppressed = 0;
+            seen.lastNoticeAt = now;
+        }
+        return;
+    }
+    // Oldest key out, so a full map still lets a new failure warn.
+    if (embedFailureSeen.size >= EMBED_WARN_KEY_CAP) {
+        for (const oldest of embedFailureSeen.keys()) {
+            embedFailureSeen.delete(oldest);
+            break;
+        }
+    }
+    embedFailureSeen.set(key, { lastNoticeAt: now, suppressed: 0 });
+    // Strict scrub: this line can land in a hook log file, and an API may echo the key back in its error body.
+    const message = redactSecretsStrict(rawMessage).replace(/\s+/g, ' ').replace(/\.+$/, '');
+    log.warn(`embedding failed (${source}): ${message}. Memories are stored without embeddings until this is fixed.`);
+}
+function vectorGroups(store) {
+    return [requireGroup(store, 'vectors'), requireGroup(store, 'vectorWrites')];
+}
+/** Writes one page's vectors, one writeVectors per tenant, and stops at the first refused write; `withParticles` seeds each memory's particle. */
+async function writeVectorPage(writes, write, page, vectors, withParticles) {
+    const byTenant = new Map();
+    page.forEach((entry, i) => {
+        const vector = vectors[i];
+        if (!vector || vector.length === 0)
+            return noteSkippedEmbedding(entry.id);
+        const row = withParticles ? { memoryId: entry.id, vector, particle: initializeParticle(entry, vector) } : { memoryId: entry.id, vector };
+        byTenant.set(entry.tenantId, [...(byTenant.get(entry.tenantId) ?? []), row]);
+    });
+    let written = 0;
+    for (const [tenantId, rows] of byTenant) {
+        const result = await writes.writeVectors({ ...write, tenantId, rows });
+        if (result.modelMismatch)
+            return { written, modelMismatch: true };
+        written += result.written;
+    }
+    return { written, modelMismatch: false };
+}
+const OTHER_MODEL_INDEX = "the vector index was built by another embedding model; run 'hippo embed' to rebuild it";
+/** Embeds every memory with no vector under `model`, page by page, so a provider failure keeps the pages written; a rebuild reseeds the particles it drops. */
+async function backfillInStore(writes, provider, model, rebuild) {
+    let count = 0;
+    let afterId;
+    for (;;) {
+        const page = await writes.entriesWithoutVector({ model, afterId, limit: STORE_PAGE });
+        if (page.length === 0)
+            return count;
+        afterId = page[page.length - 1].id;
+        const vectors = await provider.embed(page.map((e) => embeddingInputText(e)), 'passage');
+        const result = await writeVectorPage(writes, { model, replaceIndex: rebuild }, page, vectors, rebuild);
+        // Another process rebuilt the index mid-run; every later page would be refused too, after paying the provider for it.
+        if (result.modelMismatch)
+            throw new Error(OTHER_MODEL_INDEX);
+        count += result.written;
+    }
+}
+/** embedAll on a store: a model change re-embeds every memory, and the first page written drops the old index. */
+async function embedAllInStore(store, provider) {
+    const [reads, writes] = vectorGroups(store);
+    const rebuild = indexNeedsRebuild(indexedModel(await reads.embeddingIndexState()), provider.id);
+    return backfillInStore(writes, provider, embeddingIndexIdentity(provider.id), rebuild);
+}
+/** embedMemory on a store; like the hippo.db path it resolves whatever fails. Unlike it, it leaves a model change to embedAll, so a stale
+ *  read here never drops another process's rebuild; the read only saves a paid embed call, as the write refuses another model itself. */
+function embedMemoryInStore(store, provider, entry) {
+    return withProcessEmbedLock(async () => {
+        const [reads, writes] = vectorGroups(store);
+        const model = embeddingIndexIdentity(provider.id);
+        try {
+            const refused = replacesIndex(await reads.embeddingIndexState(), model)
+                || (await writeVectorPage(writes, { model, replaceIndex: false }, [entry], await provider.embed([embeddingInputText(entry)], 'passage'), true)).modelMismatch;
+            if (refused)
+                warnEmbedFailureOnce('index', OTHER_MODEL_INDEX);
+        }
+        catch (err) {
+            warnEmbedFailureOnce(provider.kind, errorMessage(err), err);
+        }
+    }).catch((err) => {
+        log.warn(`skipped embedding ${entry.id} (${errorMessage(err)})`);
+    });
+}
+/** Embed a single memory entry and cache the result in the embedding index; with `store`, through its vectorWrites, never opening hippo.db. */
+export async function embedMemory(hippoRoot, entry, model, store) {
+    let provider;
+    try {
+        provider = resolveEmbeddingProvider(hippoRoot, { model });
+    }
+    catch (err) {
+        // Callers fire and forget, so this must resolve: a bad config warns once instead of rejecting.
+        warnEmbedFailureOnce('config', errorMessage(err), err);
+        return;
+    }
+    if (!provider.isAvailable())
+        return;
+    if (store)
+        return embedMemoryInStore(store, provider, entry);
+    return withEmbedLock(hippoRoot, async () => {
+        // Best-effort: an embedding failure (API down, bad key, 5xx) must not reject the caller's write; the existing index is left as-is.
+        // Failures surface on the explicit `hippo embed` / `embedAll` path.
+        try {
+            const identity = provider.id;
+            if (embeddingModelRequiresReindex(hippoRoot, identity)) {
+                await rebuildIndexForProvider(hippoRoot, provider);
+                return;
+            }
+            const text = embeddingInputText(entry);
+            const [vector] = await provider.embed([text], 'passage');
+            if (!vector || vector.length === 0)
+                return;
+            saveStoredVectors(hippoRoot, [[entry.id, vector]], embeddingIndexIdentity(identity));
+            saveStoredEmbeddingModel(hippoRoot, identity);
+            // Initialize physics state for this memory
+            initializePhysicsIfMissing(hippoRoot, entry, vector);
+        }
+        catch (err) {
+            // Provider failure (API down / bad key). Best-effort: leave the index as-is, but say so once.
+            warnEmbedFailureOnce(provider.kind, errorMessage(err), err);
+        }
+    }).catch((err) => {
+        log.warn(`skipped embedding ${entry.id} (${errorMessage(err)}); run 'hippo embed' to backfill`);
+    });
+}
+// Every page is embedded before one transaction replaces the index, and the identity is saved only after that: a provider failure or a crash
+// at any earlier point leaves the old identity, so embeddingModelRequiresReindex has the next run rebuild from the start.
+async function rebuildIndexForProvider(hippoRoot, provider) {
+    const identity = provider.id;
+    // Host-wide rebuild: one embedding index per hippoRoot, keyed by tenant-scoped entry.id; per-tenant indices would be a larger architecture change.
+    const ids = loadAllEntryIds(hippoRoot);
+    const rebuiltIndex = await rebuildEmbeddingIndex(hippoRoot, ids, provider);
+    saveEmbeddingIndex(hippoRoot, rebuiltIndex, embeddingIndexIdentity(identity));
+    saveStoredEmbeddingModel(hippoRoot, identity);
+    resetPhysicsFromIndex(hippoRoot, ids, rebuiltIndex);
+    return Object.keys(rebuiltIndex).length;
+}
+function initializePhysicsIfMissing(hippoRoot, entry, vector) {
+    try {
+        seedStoredParticle(hippoRoot, entry, vector);
+    }
+    catch (err) {
+        // Physics init is best-effort and must not fail the embedding that just landed.
+        log.debug(`physics state not initialised for ${entry.id}: ${errorMessage(err)}`);
+    }
+}
+/** Throws when an unavailable provider is a misconfiguration rather than an intentional no-op. */
+function throwIfProviderKeyMissing(hippoRoot, provider) {
+    // A configured API provider with a missing key is a misconfiguration: surface it so callers of the exported embedAll() learn nothing was written.
+    // Local-not-installed and an explicit enabled=false stay silent no-ops.
+    const cfg = loadConfig(hippoRoot).embeddings;
+    if (provider.kind !== 'local' &&
+        cfg.enabled !== false &&
+        provider.keyEnv &&
+        !envByName(provider.keyEnv)?.trim()) {
+        throw new Error(`Embedding provider '${provider.kind}' is configured but ${provider.keyEnv} is not set.`);
+    }
+}
+// Embeds entries without a cached vector in save-checkpointed chunks; a `[]` row means that item could not be embedded and is retried next run.
+// On a hard provider failure mid-backfill the chunks already embedded are persisted (paid progress), then it stops and resumes next run.
+async function backfillPending(hippoRoot, provider, pending, model) {
+    let count = 0;
+    // Initialized to `undefined` so it stays a plain `unknown` binding for the arbitrary caught value; falsy either way.
+    let backfillError = undefined;
+    for (const chunk of entryPages(hippoRoot, pending)) {
+        let vectors;
+        try {
+            vectors = await provider.embed(chunk.map((e) => embeddingInputText(e)), 'passage');
+        }
+        catch (err) {
+            // Preserve the chunks already saved this run, then surface the failure
+            // below so the explicit `hippo embed` path never reports a false success.
+            backfillError = err;
+            break;
+        }
+        const rows = [];
+        for (let j = 0; j < chunk.length; j++) {
+            const vec = vectors[j];
+            if (vec && vec.length > 0)
+                rows.push([chunk[j].id, vec]);
+            else
+                noteSkippedEmbedding(chunk[j].id);
+        }
+        count += saveStoredVectors(hippoRoot, rows, model);
+    }
+    return { count, backfillError };
+}
+/** Embeds every entry with no cached vector, prunes vectors of deleted memories, and returns how many it embedded; `provider` defaults to the
+ *  configured one. With `store`, every tenant's memories with no vector for the provider's model go through its `vectorWrites`, never hippo.db. */
+export async function embedAll(hippoRoot, model, provider = resolveEmbeddingProvider(hippoRoot, { model }), store) {
+    if (!provider.isAvailable()) {
+        throwIfProviderKeyMissing(hippoRoot, provider);
+        return 0;
+    }
+    if (store)
+        return withProcessEmbedLock(() => embedAllInStore(store, provider));
+    return withEmbedLock(hippoRoot, async () => {
+        const identity = provider.id;
+        if (embeddingModelRequiresReindex(hippoRoot, identity))
+            return rebuildIndexForProvider(hippoRoot, provider);
+        // Host-wide by design: embedAll backfills all tenants' entries into the per-host index; per-tenant filtering would produce partial indices and break
+        // recall.
+        const embedded = pruneStoredVectors(hippoRoot);
+        const pending = loadAllEntryIds(hippoRoot).filter((id) => !embedded.has(id));
+        const { count, backfillError } = await backfillPending(hippoRoot, provider, pending, embeddingIndexIdentity(identity));
+        saveStoredEmbeddingModel(hippoRoot, identity);
+        // Partial progress is now persisted; surface a hard backfill failure so the
+        // explicit embed path reports it (best-effort callers go via embedMemory).
+        if (backfillError)
+            throw backfillError;
+        return count;
+    });
+}
+//# sourceMappingURL=embed-memories.js.map

package/dist/search/goal-boost.d.ts

@@ -1,4 +1,4 @@
-import type { MemoryEntry } from '../core/memory.js';
+import { type MemoryEntry } from '../core/memory.js';
 import type { RerankStep } from '../core/search-types.js';
 import type { ActiveGoals, GoalRecallLogRow } from '../store/goals.js';
 /** Options of {@link boostByGoals}. */

package/dist/search/goal-boost.js

@@ -1,10 +1,13 @@
 // src/search/goal-boost.ts: the goal-stack boost over goals already read; it ranks rows and touches no store.
 import { compareScoresDesc } from '../core/compare.js';
+import { DEFAULT_SCHEMA_FIT } from '../core/memory.js';
 const MAX_FINAL_MULTIPLIER = 3.0;
+const FIRST_MATCH_BOOST = 2.0;
+const EXTRA_MATCH_BOOST = 0.5;
 /** The capped boost for one row whose tags match `matches` active goals. */
 function goalBoostMultiplier(entry, tags, matches, goalsByTag, policiesByGoalId) {
-    // Base 2.0x for first match, +0.5x per additional, capped at 3.0x.
-    let multiplier = Math.min(2.0 + 0.5 * (matches.length - 1), MAX_FINAL_MULTIPLIER);
+    // First match boosts by FIRST_MATCH_BOOST, each further one adds EXTRA_MATCH_BOOST, capped at MAX_FINAL_MULTIPLIER.
+    let multiplier = Math.min(FIRST_MATCH_BOOST + EXTRA_MATCH_BOOST * (matches.length - 1), MAX_FINAL_MULTIPLIER);
     // Compose per-policy multipliers per matched tag.
     for (const tag of matches) {
         const goal = goalsByTag.get(tag);
@@ -20,7 +23,7 @@ function goalBoostMultiplier(entry, tags, matches, goalsByTag, policiesByGoalId)
             multiplier *=
                 1.0 +
                     Math.max(0, policy.weightSchemaFit - 1.0) *
-                        (entry.schema_fit ?? 0.5);
+                        (entry.schema_fit ?? DEFAULT_SCHEMA_FIT);
         }
         else if (policy.policyType === 'recency-first') {
             multiplier *= policy.weightRecency;

package/dist/search/hybrid.d.ts

@@ -1,10 +1,10 @@
 import type { MemoryEntry } from '../core/memory.js';
 import type { RerankerFn, RerankerOptions } from '../rerankers/types.js';
 import { type BM25Corpus } from './bm25.js';
-import { type HybridVectorCandidates } from './vector.js';
+import { type HybridVectorCandidates, type VectorSource } from './vector.js';
 import { type GraphStreamOptions } from './fusion.js';
 import { type ResultCost, type SearchResult } from '../core/search-types.js';
-import type { HippoStore } from '../store/index.js';
+import { type HippoStore } from '../store/index.js';
 export interface HybridSearchOptions {
     budget?: number;
     now?: Date;
@@ -46,6 +46,13 @@ export interface HybridSearchOptions {
     /** The deadline a caller already started for this recall's query embedding; a fresh budget when unset. */
     queryEmbedDeadline?: AbortSignal;
 }
+export declare const DEFAULT_EMBEDDING_WEIGHT = 0.6;
+export declare const DEFAULT_MMR_LAMBDA = 0.7;
 /** BM25 blended with cosine similarity when stored vectors and a provider are available, BM25 * strength * recency otherwise. */
 export declare function hybridSearch(query: string, entries: MemoryEntry[], options?: HybridSearchOptions): Promise<SearchResult[]>;
+/** hippo.db under `hippoRoot` when the caller passed no store: the default lives on this exported entry, never inside the vector arm. */
+export declare function vectorSourceOf({ hippoRoot, store }: {
+    hippoRoot?: string;
+    store?: HippoStore;
+}): VectorSource | undefined;
 //# sourceMappingURL=hybrid.d.ts.map

package/dist/search/hybrid.js

@@ -1,7 +1,7 @@
 import { tokenize } from '../util/tokenize.js';
 import { evalNow } from '../core/ablation.js';
 import { extractPathTags } from './path-context.js';
-import { detectScope } from '../sharing/scope.js';
+import { detectScope } from '../core/active-scope.js';
 import { compareScoredResults } from '../core/compare.js';
 import { bm25Score, buildCorpus, entryText } from './bm25.js';
 import { currentEntries } from './as-of.js';
@@ -13,9 +13,10 @@ import { scoreHybridPool } from './hybrid-score.js';
 import { applyMmrWindow, applyReranker } from './rerank.js';
 import { dedupeExtracted, fitBudget, withDagChildren } from './finalize.js';
 import { DEFAULT_RECALL_BUDGET } from '../core/search-types.js';
+import { sqliteStore } from '../store/index.js';
 // Share of the blended score that cosine takes, and the relevance side of the MMR relevance-versus-diversity trade.
-const DEFAULT_EMBEDDING_WEIGHT = 0.6;
-const DEFAULT_MMR_LAMBDA = 0.7;
+export const DEFAULT_EMBEDDING_WEIGHT = 0.6;
+export const DEFAULT_MMR_LAMBDA = 0.7;
 /** BM25 blended with cosine similarity when stored vectors and a provider are available, BM25 * strength * recency otherwise. */
 export async function hybridSearch(query, entries, options = {}) {
     const now = options.now ?? evalNow(); // honors HIPPO_FAKE_NOW (eval-only; see ablation.ts)
@@ -26,7 +27,7 @@ export async function hybridSearch(query, entries, options = {}) {
     const queryTerms = tokenize(query);
     if (queryTerms.length === 0)
         return [];
-    const arm = await resolveVectorArm(query, pool, options);
+    const arm = await resolveVectorArm(query, pool, vectorSourceOf(options), options);
     // A prepared corpus covers the caller's entries only, so rows the vector arm added force a rebuild.
     const corpus = (arm.addedRows ? undefined : options.preparedCorpus) ?? buildCorpus(arm.entries.map(entryText));
     const bm25Scores = arm.entries.map((_, i) => bm25Score(corpus, i, queryTerms));
@@ -48,6 +49,10 @@ export async function hybridSearch(query, entries, options = {}) {
     const ordered = await orderHybrid(query, withDagChildren(dedupeExtracted(scored), arm.entries), arm, options);
     return fitBudget(ordered, options.budget ?? DEFAULT_RECALL_BUDGET, options.minResults ?? 1, options.cost);
 }
+/** hippo.db under `hippoRoot` when the caller passed no store: the default lives on this exported entry, never inside the vector arm. */
+export function vectorSourceOf({ hippoRoot, store }) {
+    return hippoRoot ? { root: hippoRoot, store: store ?? sqliteStore(hippoRoot) } : undefined;
+}
 function hybridBoostContext(query, pool, now, options) {
     return {
         now,

package/dist/search/physics-search.d.ts

@@ -1,7 +1,7 @@
 import { type MemoryEntry } from '../core/memory.js';
 import { type PhysicsConfig } from '../core/physics-config.js';
-import { type HippoStore } from '../store/index.js';
-import { type HybridVectorCandidates } from './vector.js';
+import type { HippoStore } from '../store/index.js';
+import { type HybridVectorCandidates, type VectorSource } from './vector.js';
 import { type ResultCost, type SearchResult } from '../core/search-types.js';
 export interface PhysicsSearchOptions {
     budget?: number;
@@ -27,6 +27,9 @@ export interface PhysicsSearchOptions {
     /** Where vectors and particles are read; hippo.db under `hippoRoot` when unset. */
     store?: HippoStore;
 }
-/** Scores memories by gravity, momentum and cluster amplification; rows without physics state rank through hybridSearch. */
+/** Scores memories by gravity, momentum and cluster amplification; rows without physics state rank through hybridSearch.
+ *  Reads hippo.db under `hippoRoot` when no store is passed. */
 export declare function physicsSearch(query: string, entries: MemoryEntry[], options?: PhysicsSearchOptions): Promise<SearchResult[]>;
+/** physicsSearch on the store the caller names, so a caller holding its request's store never falls back to hippo.db. */
+export declare function physicsSearchIn(source: VectorSource, query: string, entries: MemoryEntry[], options?: PhysicsSearchOptions): Promise<SearchResult[]>;
 //# sourceMappingURL=physics-search.d.ts.map

package/dist/search/physics-search.js

@@ -1,50 +1,55 @@
 import { calculateStrength } from '../core/memory.js';
 import { estimateTokens } from '../util/token-text.js';
-import { evalNow, isRecallBoostAblated } from '../core/ablation.js';
+import { evalNow, isRecallBoostAblated, strengthAblation } from '../core/ablation.js';
 import { indexNeedsRebuild } from '../store/embeddings/index.js';
 import { indexedModel } from '../store/vector-index.js';
-import { resolveEmbeddingProvider } from '../store/embeddings/provider.js';
+import { resolveEmbeddingProvider } from '../embeddings/provider.js';
 import { physicsScore as computePhysicsScores, computeMass } from '../core/physics.js';
 import { DEFAULT_PHYSICS_CONFIG } from '../core/physics-config.js';
-import { rethrowIfSqliteBlocked } from '../db/index.js';
+import { rethrowIfSqliteBlocked } from '../util/sqlite-blocked.js';
 import { compareScoredResults } from '../core/compare.js';
 import { errorMessage, log } from '../util/log.js';
-import { sqliteStore } from '../store/index.js';
 import { churnStaleFactor, summaryMultipliers, summaryScoring } from './boosts.js';
 import { addDagFields, ageInDays } from './breakdown.js';
 import { fitBudget } from './finalize.js';
-import { hybridSearch } from './hybrid.js';
+import { hybridSearch, vectorSourceOf } from './hybrid.js';
 import { currentEntries } from './as-of.js';
-import { embedQueryBy, requireVectorReads, startQueryEmbedDeadline, vectorCandidatesOutside } from './vector.js';
+import { embedQueryBy, requireVectorReads, startQueryEmbedDeadline, vectorCandidatesOutside, } from './vector.js';
 import { DEFAULT_RECALL_BUDGET } from '../core/search-types.js';
-/** Scores memories by gravity, momentum and cluster amplification; rows without physics state rank through hybridSearch. */
+/** Scores memories by gravity, momentum and cluster amplification; rows without physics state rank through hybridSearch.
+ *  Reads hippo.db under `hippoRoot` when no store is passed. */
 export async function physicsSearch(query, entries, options = {}) {
+    const source = entries.length > 0 ? vectorSourceOf(options) : undefined;
+    return source ? physicsSearchIn(source, query, entries, options) : [];
+}
+/** physicsSearch on the store the caller names, so a caller holding its request's store never falls back to hippo.db. */
+export async function physicsSearchIn(source, query, entries, options = {}) {
     const now = options.now ?? evalNow(); // honors HIPPO_FAKE_NOW (eval-only; see ablation.ts)
     const explain = options.explain ?? false;
     const scoring = { now, explain, summary: summaryScoring(options) };
-    if (entries.length === 0 || !options.hippoRoot)
+    if (entries.length === 0)
         return [];
+    const { root, store } = source;
+    const hybrid = { ...options, hippoRoot: root, store };
     // memory_physics keeps only current positions and masses, so a past-dated query must rank without them.
     if (options.asOf)
-        return hybridSearch(query, entries, options);
-    const root = options.hippoRoot;
-    const store = options.store ?? sqliteStore(root);
+        return hybridSearch(query, entries, hybrid);
     // Shared with every hybrid ranking below, so a provider that stalls here is not waited on a second time.
     const queryEmbedDeadline = startQueryEmbedDeadline();
     const queryVector = await physicsQueryVector(query, root, store, queryEmbedDeadline, options.queryEmbedding);
     if (!queryVector)
-        return hybridSearch(query, entries, { ...options, queryEmbedDeadline });
+        return hybridSearch(query, entries, { ...hybrid, queryEmbedDeadline });
     // Checked here too, since a caller's own query vector skips the check inside physicsQueryVector.
     const reads = requireVectorReads(store);
     const pool = currentEntries(await withVectorCandidates(reads, entries, queryVector, options.vectorCandidates), options);
     const physicsMap = await loadCandidateParticles(reads, pool);
     if (!physicsMap)
-        return hybridSearch(query, pool, { ...options, queryEmbedDeadline });
+        return hybridSearch(query, pool, { ...hybrid, queryEmbedDeadline });
     const pools = splitByParticle(pool, physicsMap, queryVector, now);
     const config = options.physicsConfig ?? DEFAULT_PHYSICS_CONFIG;
     const physicsResults = scorePhysicsPool(pools, queryVector, config, scoring);
     const classicResults = pools.classic.length > 0
-        ? await hybridSearch(query, pools.classic, { ...options, queryEmbedDeadline, vectorCandidates: undefined, budget: Infinity, explain })
+        ? await hybridSearch(query, pools.classic, { ...hybrid, queryEmbedDeadline, vectorCandidates: undefined, budget: Infinity, explain })
         : [];
     const merged = mergeScorePools(physicsResults, classicResults);
     merged.sort(compareScoredResults);
@@ -107,7 +112,9 @@ function splitByParticle(pool, physicsMap, queryVector, now) {
         }
         pools.entries.push(entry);
         // EVAL-ONLY ablation (see ablation.ts): persisted masses bake in recall history twice, so the flag recomputes mass live.
-        pools.particles.push(isRecallBoostAblated() ? { ...particle, mass: computeMass(calculateStrength(entry, now), entry.retrieval_count) } : particle);
+        pools.particles.push(isRecallBoostAblated()
+            ? { ...particle, mass: computeMass(calculateStrength(entry, now, { ablation: strengthAblation() }), entry.retrieval_count) }
+            : particle);
     }
     return pools;
 }

package/dist/search/rerank.js

@@ -1,4 +1,4 @@
-import { cosineOf } from '../store/embeddings/index.js';
+import { cosineOf } from '../core/cosine.js';
 const DEFAULT_RERANK_TOP_K = 50;
 // MMR is O(K^2) cosine ops; candidates below this window never survive budget filtering anyway.
 const MMR_CANDIDATE_CAP = 100;

package/dist/search/vector.d.ts

@@ -1,6 +1,6 @@
 import type { MemoryEntry } from '../core/memory.js';
 import type { VectorCandidateSpec } from '../store/search-rows.js';
-import { type EmbeddingProvider } from '../store/embeddings/provider.js';
+import { type EmbeddingProvider } from '../embeddings/provider.js';
 import { type HippoStore, type VectorReads } from '../store/index.js';
 import { type CurrentnessOptions } from './as-of.js';
 /** hybridSearch's vector arm: which rows it may add, plus the caller's JS admission rules (exact private regex, entry filters). */
@@ -15,24 +15,26 @@ export interface VectorArm {
     embeddingIndex: Record<string, ArrayLike<number>>;
     queryVector: number[];
 }
+/** Where the vector arm reads: the embedding provider configured under `root`, and the vectors `store` holds. */
+export interface VectorSource {
+    readonly root: string;
+    readonly store: HippoStore;
+}
 export interface VectorArmOptions extends CurrentnessOptions {
-    hippoRoot?: string;
     vectorCandidates?: HybridVectorCandidates;
-    /** Where vectors are read; hippo.db under `hippoRoot` when unset. */
-    store?: HippoStore;
     /** The deadline a caller already started for this recall's query embedding; a fresh budget when unset. */
     queryEmbedDeadline?: AbortSignal;
 }
 /** One deadline for every query embedding a recall makes, so two stores or a physics fallback share the budget. */
 export declare function startQueryEmbedDeadline(): AbortSignal;
-/** The query's vector, or null when an API provider outlasts `deadline`; any other provider failure throws as before. */
+/** The query's vector, or null when the provider outlasts `deadline`: an API call, or a local model still downloading. Any other failure throws. */
 export declare function embedQueryBy(deadline: AbortSignal, provider: EmbeddingProvider, query: string): Promise<number[] | null>;
 /** The store's vector reads; a store without them answers 501, as any unported path does. */
 export declare function requireVectorReads(store: HippoStore): VectorReads;
 /** The nearest admitted rows not already in `entries`. */
 export declare function vectorCandidatesOutside(reads: VectorReads, entries: readonly MemoryEntry[], queryVector: readonly number[], spec: HybridVectorCandidates): Promise<MemoryEntry[]>;
 /** Embeds the query and loads stored vectors; any failure leaves BM25 to rank alone. */
-export declare function resolveVectorArm(query: string, entries: MemoryEntry[], options: VectorArmOptions): Promise<VectorArm>;
+export declare function resolveVectorArm(query: string, entries: MemoryEntry[], source: VectorSource | undefined, options: VectorArmOptions): Promise<VectorArm>;
 export interface DenseScores {
     cosine: number[];
     /** Whether each entry had a cached doc vector to compare against. */