hippo-memory 1.71.0 → 1.72.0

package/dist/store/embeddings/index.js

@@ -1,19 +1,10 @@
-/** Optional embedding-based semantic search for Hippo.
- * Uses @huggingface/transformers (local, no API keys, ~22MB model); falls back silently if not installed. */
-import { envByName } from '../../util/env.js';
-import * as fs from 'fs';
-import * as path from 'path';
-import { randomUUID } from 'crypto';
-import { chunked, loadAllEntryIds, loadEntriesByIds } from '../entry-reads.js';
-import { rethrowIfSqliteBlocked } from '../../db/index.js';
+/** The stored vector index's identity: which model and text format built it, and when a provider change means a rebuild.
+ * Picking a provider and computing vectors sit in src/search/embed-memories.ts; this module only reads and writes what is stored. */
+import { cosineOf } from '../../core/cosine.js';
 import { EMBEDDING_MODEL_META_KEY } from '../../db/vector-store.js';
-import { initializeParticle } from '../../db/physics-state.js';
-import { indexedModel, pruneStoredVectors, replacesIndex, resetStoredParticles, saveEmbeddingIndex, saveIndexIdentity, saveStoredVectors, seedStoredParticle, storedIndexState, } from '../vector-index.js';
-import { loadConfig } from '../../core/config.js';
-import { resolveEmbeddingProvider } from './provider.js';
-import { redactSecretsStrict } from '../../util/secret-detect.js';
+import { indexedModel, saveIndexIdentity, storedIndexState } from '../vector-index.js';
 import { errorMessage, log } from '../../util/log.js';
-import { StoreNotPortedError } from '../../util/sqlite-blocked.js';
+import { rethrowIfSqliteBlocked } from '../../util/sqlite-blocked.js';
 export { EMBEDDING_MODEL_META_KEY };
 /** Bump when `embeddingInputText` changes existing vectors; folded into the stored index identity (`embeddingIndexIdentity`) so it reindexes like a model
  * change. Format 2 = `path:*` tags excluded; format 1 (implicit, no suffix) included them. */
@@ -57,368 +48,8 @@ export function resolveIndexedEmbeddingModel(hippoRoot, index) {
 export function embeddingModelRequiresReindex(hippoRoot, model, index) {
     return indexNeedsRebuild(resolveIndexedEmbeddingModel(hippoRoot, index), model);
 }
-const STORE_PAGE = 64;
-/** The rows for `ids`, a page at a time and in that order; an id deleted since it was listed is skipped. */
-function* entryPages(hippoRoot, ids) {
-    for (const page of chunked(ids, STORE_PAGE)) {
-        const byId = new Map(loadEntriesByIds(hippoRoot, page).map((e) => [e.id, e]));
-        const entries = page.flatMap((id) => byId.get(id) ?? []);
-        if (entries.length > 0)
-            yield entries;
-    }
-}
-async function rebuildEmbeddingIndex(hippoRoot, ids, provider) {
-    const rebuilt = {};
-    for (const entries of entryPages(hippoRoot, ids)) {
-        // On a hard transport/auth failure provider.embed throws, so the caller aborts WITHOUT saving
-        // a partial index (atomic reindex: the old index + stored identity are preserved).
-        const vectors = await provider.embed(entries.map((e) => embeddingInputText(e)), 'passage');
-        for (let i = 0; i < entries.length; i++) {
-            const vec = vectors[i];
-            if (vec && vec.length > 0) {
-                rebuilt[entries[i].id] = vec;
-            }
-            else {
-                noteSkippedEmbedding(entries[i].id);
-            }
-        }
-    }
-    return rebuilt;
-}
-function resetPhysicsFromIndex(hippoRoot, ids, index) {
-    try {
-        resetStoredParticles(hippoRoot, ids, index);
-    }
-    catch (err) {
-        // Best effort: retrieval still falls back without physics state.
-        log.warn(`physics reset after reindex failed: ${errorMessage(err)}`);
-    }
-}
-/** A provider's `[]` row is a swallowed per-item failure; name the memory so the gap can be traced. */
-function noteSkippedEmbedding(id) {
-    log.warn('memory not embedded; the next embed run retries it', { id });
-}
 /** Cosine similarity; handles unnormalized vectors, returns 0 for empty or mismatched vectors. */
 export function cosineSimilarity(a, b) {
     return cosineOf(a, b);
 }
-/** `cosineSimilarity` over any indexable vectors; a Float32 view reads the same doubles as its number[] copy, so the score is the same. */
-export function cosineOf(a, b) {
-    if (a.length === 0 || b.length === 0 || a.length !== b.length)
-        return 0;
-    let dot = 0;
-    let normA = 0;
-    let normB = 0;
-    for (let i = 0; i < a.length; i++) {
-        dot += a[i] * b[i];
-        normA += a[i] * a[i];
-        normB += b[i] * b[i];
-    }
-    const denom = Math.sqrt(normA) * Math.sqrt(normB);
-    if (denom < 1e-10)
-        return 0;
-    // Clamp to [-1, 1] to handle floating point drift
-    return Math.min(1, Math.max(-1, dot / denom));
-}
-const EMBED_LOCK_FILE = 'embeddings.lock';
-const EMBED_LOCK_WAIT_MS = 10_000;
-const EMBED_LOCK_OWNER = `${process.pid}:${randomUUID()}`;
-// In-process mutex plus an O_EXCL "<pid>:<token>" lock file: our token is a lock we leaked; our PID
-// with another token is a live worker thread, unless the lock predates this process (a reused PID).
-let _embedWriteLock = Promise.resolve();
-function embedLockHolderAlive(lockPath) {
-    let raw;
-    try {
-        raw = fs.readFileSync(lockPath, 'utf8');
-    }
-    catch (err) {
-        return !(err instanceof Error && 'code' in err && err.code === 'ENOENT');
-    }
-    const mtimeMs = fs.statSync(lockPath, { throwIfNoEntry: false })?.mtimeMs ?? 0;
-    const pid = Number(raw.split(':')[0]);
-    // An empty lock is a holder between create and write; after 5 s it is a crashed one.
-    if (!Number.isInteger(pid) || pid <= 0)
-        return Date.now() - mtimeMs < 5_000;
-    if (raw === EMBED_LOCK_OWNER)
-        return false;
-    if (pid === process.pid)
-        return mtimeMs >= Date.now() - process.uptime() * 1000;
-    try {
-        process.kill(pid, 0);
-        return true;
-    }
-    catch (err) {
-        return err instanceof Error && 'code' in err && err.code === 'EPERM';
-    }
-}
-// After the double-break race below the file can be another writer's lock, so remove only our own.
-function releaseEmbedFileLock(lockPath) {
-    let raw;
-    try {
-        raw = fs.readFileSync(lockPath, 'utf8');
-    }
-    catch (err) {
-        if (err instanceof Error && 'code' in err && err.code === 'ENOENT')
-            return;
-        throw err;
-    }
-    if (raw === EMBED_LOCK_OWNER)
-        fs.rmSync(lockPath, { force: true });
-}
-async function acquireEmbedFileLock(hippoRoot) {
-    const lockPath = path.join(hippoRoot, EMBED_LOCK_FILE);
-    const deadline = Date.now() + EMBED_LOCK_WAIT_MS;
-    for (;;) {
-        try {
-            fs.writeFileSync(lockPath, EMBED_LOCK_OWNER, { flag: 'wx' });
-            return () => releaseEmbedFileLock(lockPath);
-        }
-        catch (err) {
-            if (!(err instanceof Error && 'code' in err && err.code === 'EEXIST'))
-                throw err;
-        }
-        // SHORTCUT: two waiters can both break one dead holder's lock; a vector lost that way is backfilled by the next `hippo embed`.
-        if (!embedLockHolderAlive(lockPath)) {
-            fs.rmSync(lockPath, { force: true });
-            continue;
-        }
-        if (Date.now() >= deadline)
-            throw new Error(`the embedding index is busy: another hippo process holds ${lockPath}`);
-        await new Promise((r) => setTimeout(r, 50));
-    }
-}
-async function withEmbedLock(hippoRoot, fn) {
-    return withProcessEmbedLock(async () => {
-        const release = await acquireEmbedFileLock(hippoRoot);
-        try {
-            return await fn();
-        }
-        finally {
-            release();
-        }
-    });
-}
-// A store keeps each write whole in its own transaction, and a lock file in hippoRoot would not reach a worker on another host.
-async function withProcessEmbedLock(fn) {
-    let resolve;
-    const next = new Promise(r => { resolve = r; });
-    const prev = _embedWriteLock;
-    _embedWriteLock = next;
-    await prev;
-    try {
-        return await fn();
-    }
-    finally {
-        resolve();
-    }
-}
-// A bad key fails every write; one warning tells the user, N would bury the command's own output.
-let _embedFailureWarned = false;
-function warnEmbedFailureOnce(source, rawMessage) {
-    if (_embedFailureWarned)
-        return;
-    _embedFailureWarned = true;
-    // Strict scrub: this line can land in a hook log file, and an API may echo the key back in its error body.
-    const message = redactSecretsStrict(rawMessage).replace(/\s+/g, ' ').replace(/\.+$/, '');
-    log.warn(`embedding failed (${source}): ${message}. Memories are stored without embeddings until this is fixed.`);
-}
-// requireGroup (src/store/port.ts) would close an import cycle with this module.
-function vectorGroups(store) {
-    if (store.vectors === undefined)
-        throw new StoreNotPortedError(store.kind, 'vectors');
-    if (store.vectorWrites === undefined)
-        throw new StoreNotPortedError(store.kind, 'vectorWrites');
-    return [store.vectors, store.vectorWrites];
-}
-/** Writes one page's vectors, one writeVectors per tenant, and stops at the first refused write; `withParticles` seeds each memory's particle. */
-async function writeVectorPage(writes, write, page, vectors, withParticles) {
-    const byTenant = new Map();
-    page.forEach((entry, i) => {
-        const vector = vectors[i];
-        if (!vector || vector.length === 0)
-            return noteSkippedEmbedding(entry.id);
-        const row = withParticles ? { memoryId: entry.id, vector, particle: initializeParticle(entry, vector) } : { memoryId: entry.id, vector };
-        byTenant.set(entry.tenantId, [...(byTenant.get(entry.tenantId) ?? []), row]);
-    });
-    let written = 0;
-    for (const [tenantId, rows] of byTenant) {
-        const result = await writes.writeVectors({ ...write, tenantId, rows });
-        if (result.modelMismatch)
-            return { written, modelMismatch: true };
-        written += result.written;
-    }
-    return { written, modelMismatch: false };
-}
-const OTHER_MODEL_INDEX = "the vector index was built by another embedding model; run 'hippo embed' to rebuild it";
-/** Embeds every memory with no vector under `model`, page by page, so a provider failure keeps the pages written; a rebuild reseeds the particles it drops. */
-async function backfillInStore(writes, provider, model, rebuild) {
-    let count = 0;
-    let afterId;
-    for (;;) {
-        const page = await writes.entriesWithoutVector({ model, afterId, limit: STORE_PAGE });
-        if (page.length === 0)
-            return count;
-        afterId = page[page.length - 1].id;
-        const vectors = await provider.embed(page.map((e) => embeddingInputText(e)), 'passage');
-        const result = await writeVectorPage(writes, { model, replaceIndex: rebuild }, page, vectors, rebuild);
-        // Another process rebuilt the index mid-run; every later page would be refused too, after paying the provider for it.
-        if (result.modelMismatch)
-            throw new Error(OTHER_MODEL_INDEX);
-        count += result.written;
-    }
-}
-/** embedAll on a store: a model change re-embeds every memory, and the first page written drops the old index. */
-async function embedAllInStore(store, provider) {
-    const [reads, writes] = vectorGroups(store);
-    const rebuild = indexNeedsRebuild(indexedModel(await reads.embeddingIndexState()), provider.id);
-    return backfillInStore(writes, provider, embeddingIndexIdentity(provider.id), rebuild);
-}
-/** embedMemory on a store; like the hippo.db path it resolves whatever fails. Unlike it, it leaves a model change to embedAll, so a stale
- *  read here never drops another process's rebuild; the read only saves a paid embed call, as the write refuses another model itself. */
-function embedMemoryInStore(store, provider, entry) {
-    return withProcessEmbedLock(async () => {
-        const [reads, writes] = vectorGroups(store);
-        const model = embeddingIndexIdentity(provider.id);
-        try {
-            const refused = replacesIndex(await reads.embeddingIndexState(), model)
-                || (await writeVectorPage(writes, { model, replaceIndex: false }, [entry], await provider.embed([embeddingInputText(entry)], 'passage'), true)).modelMismatch;
-            if (refused)
-                warnEmbedFailureOnce('index', OTHER_MODEL_INDEX);
-        }
-        catch (err) {
-            warnEmbedFailureOnce(provider.kind, errorMessage(err));
-        }
-    }).catch((err) => {
-        log.warn(`skipped embedding ${entry.id} (${errorMessage(err)})`);
-    });
-}
-/** Embed a single memory entry and cache the result in the embedding index; with `store`, through its vectorWrites, never opening hippo.db. */
-export async function embedMemory(hippoRoot, entry, model, store) {
-    let provider;
-    try {
-        provider = resolveEmbeddingProvider(hippoRoot, { model });
-    }
-    catch (err) {
-        // Callers fire and forget, so this must resolve: a bad config warns once instead of rejecting.
-        warnEmbedFailureOnce('config', errorMessage(err));
-        return;
-    }
-    if (!provider.isAvailable())
-        return;
-    if (store)
-        return embedMemoryInStore(store, provider, entry);
-    return withEmbedLock(hippoRoot, async () => {
-        // Best-effort: an embedding failure (API down, bad key, 5xx) must not reject the caller's write; the existing index is left as-is.
-        // Failures surface on the explicit `hippo embed` / `embedAll` path.
-        try {
-            const identity = provider.id;
-            if (embeddingModelRequiresReindex(hippoRoot, identity)) {
-                await rebuildIndexForProvider(hippoRoot, provider);
-                return;
-            }
-            const text = embeddingInputText(entry);
-            const [vector] = await provider.embed([text], 'passage');
-            if (!vector || vector.length === 0)
-                return;
-            saveStoredVectors(hippoRoot, [[entry.id, vector]], embeddingIndexIdentity(identity));
-            saveStoredEmbeddingModel(hippoRoot, identity);
-            // Initialize physics state for this memory
-            initializePhysicsIfMissing(hippoRoot, entry, vector);
-        }
-        catch (err) {
-            // Provider failure (API down / bad key). Best-effort: leave the index as-is, but say so once.
-            warnEmbedFailureOnce(provider.kind, errorMessage(err));
-        }
-    }).catch((err) => {
-        log.warn(`skipped embedding ${entry.id} (${errorMessage(err)}); run 'hippo embed' to backfill`);
-    });
-}
-// Every page is embedded before one transaction replaces the index, and the identity is saved only after that: a provider failure or a crash
-// at any earlier point leaves the old identity, so embeddingModelRequiresReindex has the next run rebuild from the start.
-async function rebuildIndexForProvider(hippoRoot, provider) {
-    const identity = provider.id;
-    // Host-wide rebuild: one embedding index per hippoRoot, keyed by tenant-scoped entry.id; per-tenant indices would be a larger architecture change.
-    const ids = loadAllEntryIds(hippoRoot);
-    const rebuiltIndex = await rebuildEmbeddingIndex(hippoRoot, ids, provider);
-    saveEmbeddingIndex(hippoRoot, rebuiltIndex, embeddingIndexIdentity(identity));
-    saveStoredEmbeddingModel(hippoRoot, identity);
-    resetPhysicsFromIndex(hippoRoot, ids, rebuiltIndex);
-    return Object.keys(rebuiltIndex).length;
-}
-function initializePhysicsIfMissing(hippoRoot, entry, vector) {
-    try {
-        seedStoredParticle(hippoRoot, entry, vector);
-    }
-    catch (err) {
-        // Physics init is best-effort and must not fail the embedding that just landed.
-        log.debug(`physics state not initialised for ${entry.id}: ${errorMessage(err)}`);
-    }
-}
-/** Throws when an unavailable provider is a misconfiguration rather than an intentional no-op. */
-function throwIfProviderKeyMissing(hippoRoot, provider) {
-    // A configured API provider with a missing key is a misconfiguration: surface it so callers of the exported embedAll() learn nothing was written.
-    // Local-not-installed and an explicit enabled=false stay silent no-ops.
-    const cfg = loadConfig(hippoRoot).embeddings;
-    if (provider.kind !== 'local' &&
-        cfg.enabled !== false &&
-        provider.keyEnv &&
-        !envByName(provider.keyEnv)?.trim()) {
-        throw new Error(`Embedding provider '${provider.kind}' is configured but ${provider.keyEnv} is not set.`);
-    }
-}
-// Embeds entries without a cached vector in save-checkpointed chunks; a `[]` row means that item could not be embedded and is retried next run.
-// On a hard provider failure mid-backfill the chunks already embedded are persisted (paid progress), then it stops and resumes next run.
-async function backfillPending(hippoRoot, provider, pending, model) {
-    let count = 0;
-    // Initialized to `undefined` so it stays a plain `unknown` binding for the arbitrary caught value; falsy either way.
-    let backfillError = undefined;
-    for (const chunk of entryPages(hippoRoot, pending)) {
-        let vectors;
-        try {
-            vectors = await provider.embed(chunk.map((e) => embeddingInputText(e)), 'passage');
-        }
-        catch (err) {
-            // Preserve the chunks already saved this run, then surface the failure
-            // below so the explicit `hippo embed` path never reports a false success.
-            backfillError = err;
-            break;
-        }
-        const rows = [];
-        for (let j = 0; j < chunk.length; j++) {
-            const vec = vectors[j];
-            if (vec && vec.length > 0)
-                rows.push([chunk[j].id, vec]);
-            else
-                noteSkippedEmbedding(chunk[j].id);
-        }
-        count += saveStoredVectors(hippoRoot, rows, model);
-    }
-    return { count, backfillError };
-}
-/** Embeds every entry with no cached vector, prunes vectors of deleted memories, and returns how many it embedded; `provider` defaults to the
- *  configured one. With `store`, every tenant's memories with no vector for the provider's model go through its `vectorWrites`, never hippo.db. */
-export async function embedAll(hippoRoot, model, provider = resolveEmbeddingProvider(hippoRoot, { model }), store) {
-    if (!provider.isAvailable()) {
-        throwIfProviderKeyMissing(hippoRoot, provider);
-        return 0;
-    }
-    if (store)
-        return withProcessEmbedLock(() => embedAllInStore(store, provider));
-    return withEmbedLock(hippoRoot, async () => {
-        const identity = provider.id;
-        if (embeddingModelRequiresReindex(hippoRoot, identity))
-            return rebuildIndexForProvider(hippoRoot, provider);
-        // Host-wide by design: embedAll backfills all tenants' entries into the per-host index; per-tenant filtering would produce partial indices and break
-        // recall.
-        const embedded = pruneStoredVectors(hippoRoot);
-        const pending = loadAllEntryIds(hippoRoot).filter((id) => !embedded.has(id));
-        const { count, backfillError } = await backfillPending(hippoRoot, provider, pending, embeddingIndexIdentity(identity));
-        saveStoredEmbeddingModel(hippoRoot, identity);
-        // Partial progress is now persisted; surface a hard backfill failure so the
-        // explicit embed path reports it (best-effort callers go via embedMemory).
-        if (backfillError)
-            throw backfillError;
-        return count;
-    });
-}
 //# sourceMappingURL=index.js.map

package/dist/store/entry-reads.d.ts

@@ -3,10 +3,6 @@ import { type DatabaseSyncLike } from '../db/index.js';
 export declare const TENANT_IS = "+tenant_id = ?";
 /** Read a memory entry by ID. With `tenantId` the read is tenant-scoped (cross-tenant returns null); omitted, no tenant filter is applied. */
 export declare function readEntry(hippoRoot: string, id: string, tenantId?: string): MemoryEntry | null;
-/** Ids per `IN (...)` list: far under SQLite's bound-parameter limit, with room for the tenant filter. */
-export declare const ID_CHUNK = 500;
-/** `items` in consecutive slices of at most `size`. */
-export declare function chunked<T>(items: readonly T[], size?: number): T[][];
 /** Answers whether a tenant's rows in this store hold an id: `ids` are looked up now, one query per chunk, and any other id on its first ask. */
 export declare function heldIdLookup(hippoRoot: string, tenantId: string, ids: readonly string[]): (id: string) => boolean;
 /** Rows by id on the caller's handle, one query per chunk; an id missing or in another tenant is absent from the map. */
@@ -28,6 +24,19 @@ export declare function loadFreshRawMemories(hippoRoot: string, count: number, t
 export declare function loadAllEntries(hippoRoot: string, tenantId?: string): MemoryEntry[];
 /** Every tenant's memory ids in loadAllEntries' order, for a caller that reads the rows a page at a time instead of holding them all. */
 export declare function loadAllEntryIds(hippoRoot: string): string[];
+/** How many memory rows the store holds across every tenant. */
+export declare function countAllEntries(hippoRoot: string): number;
+/** A superset, in loadAllEntries' order, of the rows whose lowercased content holds any of `tokens` or whose lowercased tags hold `tag`.
+ * The caller re-checks each row exactly; this only spares decoding rows that cannot match. */
+export declare function loadEntriesMentioning(hippoRoot: string, tokens: readonly string[], tag: string, tenantId?: string): MemoryEntry[];
+export interface EntryTextKey {
+    id: string;
+    tenantId: string;
+    content: string;
+    source: string;
+}
+/** Every tenant's rows in loadAllEntries' order, with only the columns that say whether another store already holds each. */
+export declare function loadEntryTextKeys(hippoRoot: string): EntryTextKey[];
 export interface EntriesWithBase {
     entries: MemoryEntry[];
     base: {

package/dist/store/entry-reads.js

@@ -3,7 +3,9 @@ import { MEMORY_SELECT_COLUMNS, rowToEntry, parseJsonArray } from './rows.js';
 import { onHandle, openStore } from './open.js';
 import { escapeLike } from '../util/escape.js';
 import { originInSql } from '../core/project-identity.js';
-import { scopeAdmitSql } from './recall-scope.js';
+import { scopeAdmitSql } from './rule-sql.js';
+import { chunked } from '../util/chunked.js';
+import { DEFAULT_TENANT_ID } from '../util/env.js';
 const MAX_IDS_PER_READ = 500;
 // The plus keeps SQLite on the primary key for an id list: with a bare tenant_id it walks every row of the tenant instead.
 export const TENANT_IS = '+tenant_id = ?';
@@ -18,15 +20,6 @@ export function readEntry(hippoRoot, id, tenantId) {
         return row ? rowToEntry(row) : null;
     }, openStore);
 }
-/** Ids per `IN (...)` list: far under SQLite's bound-parameter limit, with room for the tenant filter. */
-export const ID_CHUNK = 500;
-/** `items` in consecutive slices of at most `size`. */
-export function chunked(items, size = ID_CHUNK) {
-    const out = [];
-    for (let i = 0; i < items.length; i += size)
-        out.push(items.slice(i, i + size));
-    return out;
-}
 /** Answers whether a tenant's rows in this store hold an id: `ids` are looked up now, one query per chunk, and any other id on its first ask. */
 export function heldIdLookup(hippoRoot, tenantId, ids) {
     const held = new Map();
@@ -201,6 +194,48 @@ export function loadAllEntryIds(hippoRoot) {
         return db.prepare('SELECT id FROM memories ORDER BY created ASC, id ASC').all().map((row) => row.id);
     }, openStore);
 }
+/** How many memory rows the store holds across every tenant. */
+export function countAllEntries(hippoRoot) {
+    return onHandle(hippoRoot, (db) => {
+        // SAFETY: row's shape matches the single `COUNT(*) AS c` column.
+        const row = db.prepare('SELECT COUNT(*) AS c FROM memories').get();
+        return Number(row?.c ?? 0);
+    }, openStore);
+}
+// The only code points whose JS toLowerCase yields ASCII, which SQLite's ASCII-only LIKE folding cannot see.
+const FOLDS_TO_ASCII = ['\u0130', '\u212A'];
+/** A superset, in loadAllEntries' order, of the rows whose lowercased content holds any of `tokens` or whose lowercased tags hold `tag`.
+ * The caller re-checks each row exactly; this only spares decoding rows that cannot match. */
+export function loadEntriesMentioning(hippoRoot, tokens, tag, tenantId) {
+    const likes = [...new Set(tokens)].map((t) => `%${escapeLike(t)}%`);
+    const clauses = likes.map(() => `content LIKE ? ESCAPE '\\'`);
+    const params = [...likes];
+    // A tag needing JSON escapes or case folding past ASCII is not found by a substring of tags_json, so every tagged row stays in.
+    if (/^[\x20-\x7e]*$/.test(tag) && !/["\\]/.test(tag)) {
+        clauses.push(`tags_json LIKE ? ESCAPE '\\'`);
+        params.push(`%${escapeLike(tag)}%`);
+    }
+    else
+        clauses.push(`COALESCE(tags_json, '[]') != '[]'`);
+    for (const ch of FOLDS_TO_ASCII) {
+        clauses.push('instr(content, ?) > 0', 'instr(tags_json, ?) > 0');
+        params.push(ch, ch);
+    }
+    const tenantClause = tenantId !== undefined ? 'tenant_id = ? AND ' : '';
+    return onHandle(hippoRoot, (db) => {
+        // SAFETY: selects exactly MEMORY_SELECT_COLUMNS, matching MemoryRow's field set.
+        const rows = db.prepare(`SELECT ${MEMORY_SELECT_COLUMNS} FROM memories WHERE ${tenantClause}(${clauses.join(' OR ')}) ORDER BY created ASC, id ASC`).all(...(tenantId !== undefined ? [tenantId] : []), ...params);
+        return rows.map(rowToEntry);
+    }, openStore);
+}
+/** Every tenant's rows in loadAllEntries' order, with only the columns that say whether another store already holds each. */
+export function loadEntryTextKeys(hippoRoot) {
+    return onHandle(hippoRoot, (db) => {
+        // SAFETY: rows' shape matches the four columns named in the SELECT.
+        const rows = db.prepare('SELECT id, tenant_id, content, source FROM memories ORDER BY created ASC, id ASC').all();
+        return rows.map((row) => ({ id: row.id, tenantId: row.tenant_id ?? DEFAULT_TENANT_ID, content: row.content, source: row.source }));
+    }, openStore);
+}
 /** Every entry plus a by-id lookup of each as loaded, rebuilt from its row on first use, so no deep copy of the store is kept. */
 export function loadAllEntriesWithBase(hippoRoot) {
     const db = openStore(hippoRoot);
@@ -265,7 +300,8 @@ export function loadVaultRawRows(hippoRoot, likeParam, tenantId) {
 export function selectPreviewRows(db, columns, tenantId) {
     const tenant = columns.has('tenant_id') ? ' WHERE tenant_id = ?' : '';
     const column = (name, fallback) => (columns.has(name) ? name : `${fallback} AS ${name}`);
-    const statement = db.prepare(`SELECT id, content, ${column('source', "''")}, ${column('confidence', 'NULL')}, ${column('extracted_from', 'NULL')}, ${column('dag_level', '0')}, ${column('tags_json', "'[]'")} FROM memories${tenant}`);
+    const statement = db.prepare(`SELECT id, content, ${column('source', "''")}, ${column('confidence', 'NULL')}, ${column('extracted_from', 'NULL')}, ` +
+        `${column('dag_level', '0')}, ${column('tags_json', "'[]'")} FROM memories${tenant}`);
     // SAFETY: the SELECT names every PreviewRow field, each a literal fallback when its column is missing.
     return (tenant ? statement.all(tenantId) : statement.all());
 }

package/dist/store/entry-writes.d.ts

@@ -10,11 +10,11 @@ export interface WriteEntryOptions {
  * `opts.actor` defaults to 'cli'; the api layer passes the resolved actor so the audit log gets one row per write. */
 export declare function writeEntry(hippoRoot: string, entry: MemoryEntry, opts?: WriteEntryOptions): void;
 /** writeEntry for each of `entries` in one transaction, so all land or none; each
- * keeps its audit row, and mirrors follow the commit. Returns how many were written. */
+ * keeps its audit row, and mirrors follow the commit. Returns how many were written; an empty list opens no store. */
 export declare function writeEntriesTogether(hippoRoot: string, entries: readonly MemoryEntry[]): number;
-/** writeEntry for each of `entries` on one open store, so the open and its lock wait are
- * paid once; each row commits alone, and none opens the store when the list is empty. */
-export declare function writeEntriesSeparately(hippoRoot: string, entries: readonly MemoryEntry[]): void;
+/** Runs `fill` in one transaction; each `put` writes one entry and returns false when the store rejects its value,
+ * leaving that row's refusal audit and the rest of the batch. Mirrors follow the commit. Returns how many were written. */
+export declare function writeEntriesSkippingRejected(hippoRoot: string, fill: (put: (entry: MemoryEntry) => boolean) => void, actor?: string): number;
 /** Adds `tag` to each of a tenant's rows that lacks it, in `ids` order, each read fresh
  * on one open store and committed alone; an id the tenant does not hold is skipped. */
 export declare function addTagToEntries(hippoRoot: string, tenantId: string, ids: readonly string[], tag: string): void;
@@ -36,7 +36,7 @@ export interface StrengthenOptions {
 /** Strengthen what a read returned: update only the four retrieval columns on the live row, never a stale copy.
  *  Best effort: a failure logs and never fails the read. Returns the ids found in this store. */
 export declare function strengthenRetrieved(hippoRoot: string, ids: readonly string[], opts: StrengthenOptions): Set<string>;
-/** strengthenRetrieved on an open handle that holds no transaction, so a recall's last writes share one handle. */
+/** strengthenRetrieved on an open handle in its own write scope, a savepoint inside a caller's transaction, so a recall's last writes share one handle. */
 export declare function strengthenRetrievedInOwnTx(db: DatabaseSyncLike, ids: readonly string[], opts: StrengthenOptions): Set<string>;
 /** strengthenRetrieved on the caller's handle, inside the caller's transaction. Throws; the caller decides. */
 export declare function strengthenRetrievedOn(db: DatabaseSyncLike, ids: readonly string[], opts: StrengthenOptions): Set<string>;

package/dist/store/entry-writes.js

@@ -1,6 +1,8 @@
+import { strengthAblation } from '../core/ablation.js';
 import { markRetrieved } from '../core/memory.js';
-import { closeHippoDb, openHippoDb, rethrowIfSqliteBlocked, withWriteScope } from '../db/index.js';
-import { RejectedValueError } from './rejection.js';
+import { closeHippoDb, openHippoDb, withWriteScope } from '../db/index.js';
+import { rethrowIfSqliteBlocked } from '../util/sqlite-blocked.js';
+import { RejectedValueError } from '../core/api-errors.js';
 import { markSummaryDirtyInTx } from './summary-dirty.js';
 import { errorMessage, log } from '../util/log.js';
 import { auditRejectionRefusal, audit } from './audit-event.js';
@@ -16,8 +18,10 @@ export function writeEntry(hippoRoot, entry, opts) {
     }, openStore);
 }
 /** writeEntry for each of `entries` in one transaction, so all land or none; each
- * keeps its audit row, and mirrors follow the commit. Returns how many were written. */
+ * keeps its audit row, and mirrors follow the commit. Returns how many were written; an empty list opens no store. */
 export function writeEntriesTogether(hippoRoot, entries) {
+    if (entries.length === 0)
+        return 0;
     const stamped = entries.map((entry) => stampOriginProject(hippoRoot, entry));
     const db = openStore(hippoRoot);
     try {
@@ -39,15 +43,31 @@ export function writeEntriesTogether(hippoRoot, entries) {
     }
     return stamped.length;
 }
-/** writeEntry for each of `entries` on one open store, so the open and its lock wait are
- * paid once; each row commits alone, and none opens the store when the list is empty. */
-export function writeEntriesSeparately(hippoRoot, entries) {
-    if (entries.length === 0)
-        return;
+/** Runs `fill` in one transaction; each `put` writes one entry and returns false when the store rejects its value,
+ * leaving that row's refusal audit and the rest of the batch. Mirrors follow the commit. Returns how many were written. */
+export function writeEntriesSkippingRejected(hippoRoot, fill, actor = 'cli') {
+    const written = [];
     onHandle(hippoRoot, (db) => {
-        for (const entry of entries)
-            writeEntryOn(db, hippoRoot, entry);
+        withWriteScope(db, 'write_entries_skipping_rejected', () => {
+            fill((entry) => {
+                const stamped = stampOriginProject(hippoRoot, entry);
+                try {
+                    writeEntryDbOnly(db, stamped, { actor });
+                }
+                catch (error) {
+                    if (!(error instanceof RejectedValueError))
+                        throw error;
+                    auditRejectionRefusal(db, error, actor);
+                    return false;
+                }
+                written.push(stamped);
+                return true;
+            });
+        });
     }, openStore);
+    for (const entry of written)
+        writeEntryMirrors(hippoRoot, entry);
+    return written.length;
 }
 /** Adds `tag` to each of a tenant's rows that lacks it, in `ids` order, each read fresh
  * on one open store and committed alone; an id the tenant does not hold is skipped. */
@@ -129,7 +149,7 @@ export function strengthenRetrieved(hippoRoot, ids, opts) {
         closeHippoDb(db);
     }
 }
-/** strengthenRetrieved on an open handle that holds no transaction, so a recall's last writes share one handle. */
+/** strengthenRetrieved on an open handle in its own write scope, a savepoint inside a caller's transaction, so a recall's last writes share one handle. */
 export function strengthenRetrievedInOwnTx(db, ids, opts) {
     if (ids.length === 0 || opts.recallBoostAblated)
         return new Set();
@@ -152,7 +172,7 @@ export function strengthenRetrievedOn(db, ids, opts) {
     const byId = selectEntriesByIds(db, ids, opts.tenantId);
     const live = [...new Set(ids)].flatMap((id) => byId.get(id) ?? []);
     const update = db.prepare('UPDATE memories SET retrieval_count = ?, last_retrieved = ?, half_life_days = ?, strength = ? WHERE id = ?');
-    for (const e of markRetrieved(live)) {
+    for (const e of markRetrieved(live, undefined, { ablation: strengthAblation() })) {
         update.run(e.retrieval_count, e.last_retrieved, e.half_life_days, e.strength, e.id);
         found.add(e.id);
     }

package/dist/store/failure-log-at.d.ts

@@ -0,0 +1,8 @@
+import { type FailureEvent, type FailureOutcome } from './failure-log.js';
+/** {@link recordFailure} on a handle of its own. */
+export declare function recordFailureAt(hippoRoot: string, event: FailureEvent): number;
+/** {@link requestOutcome} on a handle of its own. */
+export declare function requestOutcomeAt(hippoRoot: string, tenantId: string, requestId: string, sessionId: string): FailureOutcome | null;
+/** {@link settleFailureOutcome} on a handle of its own. */
+export declare function settleFailureOutcomeAt(hippoRoot: string, tenantId: string, requestId: string, outcome: FailureOutcome): void;
+//# sourceMappingURL=failure-log-at.d.ts.map

package/dist/store/failure-log-at.js

@@ -0,0 +1,16 @@
+// The failure log on a handle of its own, for code that has a folder and no connection.
+import { onHandle } from './open.js';
+import { recordFailure, requestOutcome, settleFailureOutcome } from './failure-log.js';
+/** {@link recordFailure} on a handle of its own. */
+export function recordFailureAt(hippoRoot, event) {
+    return onHandle(hippoRoot, (db) => recordFailure(db, event));
+}
+/** {@link requestOutcome} on a handle of its own. */
+export function requestOutcomeAt(hippoRoot, tenantId, requestId, sessionId) {
+    return onHandle(hippoRoot, (db) => requestOutcome(db, tenantId, requestId, sessionId));
+}
+/** {@link settleFailureOutcome} on a handle of its own. */
+export function settleFailureOutcomeAt(hippoRoot, tenantId, requestId, outcome) {
+    onHandle(hippoRoot, (db) => settleFailureOutcome(db, tenantId, requestId, outcome));
+}
+//# sourceMappingURL=failure-log-at.js.map

package/dist/store/failure-log.d.ts

@@ -28,8 +28,8 @@ export interface FailureEvent {
     /** Override the timestamp (tests). ISO string. */
     now?: string;
 }
-/** Append one failure row and prune rows past {@link FAILURE_LOG_RETENTION_DAYS}. */
-export declare function recordFailure(db: DatabaseSyncLike, event: FailureEvent): void;
+/** Prune rows past {@link FAILURE_LOG_RETENTION_DAYS}, then append one failure row; returns the new row's id. */
+export declare function recordFailure(db: DatabaseSyncLike, event: FailureEvent): number;
 /** The outcome logged for a caller's request id, or null, so a retried send finds the first one; another session's id is a ConflictError. */
 export declare function requestOutcome(db: DatabaseSyncLike, tenantId: string, requestId: string, sessionId: string): FailureOutcome | null;
 /** A retry that stored what the first try could not rewrites that row, since the request id allows one row per tenant. */