hippo-memory 1.69.0 → 1.70.0

package/dist/{physics-state.d.ts → db/physics-state.d.ts}

@@ -3,9 +3,9 @@
  * Stores and loads particle state (position, velocity, mass, charge, temperature)
  * in SQLite using BLOB columns for 384-dim vectors.
  */
-import type { DatabaseSyncLike } from './db.js';
-import type { MemoryEntry } from './memory.js';
-import type { PhysicsParticle } from './physics.js';
+import type { DatabaseSyncLike } from './index.js';
+import type { MemoryEntry } from '../core/memory.js';
+import type { PhysicsParticle } from '../core/physics.js';
 export declare function float32ToBuffer(arr: number[]): Buffer;
 export declare function bufferToFloat32(buf: Buffer | Uint8Array): number[];
 /**

package/dist/{physics-state.js → db/physics-state.js}

@@ -3,9 +3,11 @@
  * Stores and loads particle state (position, velocity, mass, charge, temperature)
  * in SQLite using BLOB columns for 384-dim vectors.
  */
-import { evalNow } from './ablation.js';
-import { computeMass, computeCharge, computeTemperature, vecZero } from './physics.js';
-import { calculateStrength } from './memory.js';
+import { evalNow } from '../core/ablation.js';
+import { withWriteScope } from './busy.js';
+import { computeMass, computeCharge, computeTemperature, vecZero } from '../core/physics.js';
+import { calculateStrength } from '../core/memory.js';
+import { DAY_MS } from '../util/time.js';
 // ---------------------------------------------------------------------------
 // Float32Array <-> Buffer serialization
 // ---------------------------------------------------------------------------
@@ -100,20 +102,11 @@ export function savePhysicsState(db, particles) {
       last_simulation = excluded.last_simulation,
       updated_at = datetime('now')
   `);
-    db.exec('BEGIN IMMEDIATE');
-    try {
+    withWriteScope(db, 'save_particles', () => {
         for (const p of particles) {
             stmt.run(p.memoryId, float32ToBuffer(p.position), float32ToBuffer(p.velocity), p.mass, p.charge, p.temperature, p.lastSimulation);
         }
-        db.exec('COMMIT');
-    }
-    catch (error) {
-        try {
-            db.exec('ROLLBACK');
-        }
-        catch { /* already rolled back; keep the original error */ }
-        throw error;
-    }
+    });
 }
 /**
  * Initialize physics state for a memory from its embedding vector.
@@ -121,7 +114,7 @@ export function savePhysicsState(db, particles) {
  */
 export function initializeParticle(entry, embedding, now = evalNow()) {
     const strength = calculateStrength(entry, now);
-    const ageDays = (now.getTime() - new Date(entry.created).getTime()) / (1000 * 60 * 60 * 24);
+    const ageDays = (now.getTime() - new Date(entry.created).getTime()) / DAY_MS;
     return {
         memoryId: entry.id,
         position: [...embedding], // copy — position will diverge from original embedding
@@ -160,7 +153,7 @@ export function refreshParticleProperties(particles, entries, now = evalNow()) {
         if (!entry)
             continue;
         const strength = calculateStrength(entry, now);
-        const ageDays = (now.getTime() - new Date(entry.created).getTime()) / (1000 * 60 * 60 * 24);
+        const ageDays = (now.getTime() - new Date(entry.created).getTime()) / DAY_MS;
         p.mass = computeMass(strength, entry.retrieval_count);
         p.charge = computeCharge(entry.emotional_valence);
         p.temperature = computeTemperature(ageDays, 1.0);

package/dist/{raw-archive-mirror-cleanup.d.ts → db/raw-archive-mirror-cleanup.d.ts}

@@ -1,4 +1,4 @@
-import type { DatabaseSyncLike } from './db.js';
+import type { DatabaseSyncLike } from './index.js';
 /**
  * Path A backfill cleanup: existing raw_archive rows that were archived BEFORE
  * commit 70180b5 left their markdown mirrors at <hippoRoot>/<layer>/<id>.md
@@ -11,10 +11,10 @@ import type { DatabaseSyncLike } from './db.js';
  *   - On success (every layer either deleted or didn't exist), UPDATE the row
  *     with the cleanup timestamp.
  *   - On any unlink failure, leave mirror_cleaned_at NULL so the next
- *     openHippoDb call retries. Warn (capped at MAX_WARN_LOGS rows per call).
+ *     connection retries. Warn (capped at MAX_WARN_LOGS rows per call).
  *
- * Steady-state cost is one SELECT returning empty after every row has been
- * cleaned successfully.
+ * Returns whether every pending row was cleaned; the SELECT has no index to
+ * use, so a connection runs it only when the archive has changed since.
  */
-export declare function cleanupArchivedMirrors(hippoRoot: string, db: DatabaseSyncLike): void;
+export declare function cleanupArchivedMirrors(hippoRoot: string, db: DatabaseSyncLike): boolean;
 //# sourceMappingURL=raw-archive-mirror-cleanup.d.ts.map

package/dist/{raw-archive-mirror-cleanup.js → db/raw-archive-mirror-cleanup.js}

@@ -1,6 +1,6 @@
 import * as fs from 'fs';
 import * as path from 'path';
-import { log } from './log.js';
+import { errorMessage, log } from '../util/log.js';
 const LAYERS = ['episodic', 'buffer', 'semantic'];
 const MAX_WARN_LOGS = 5;
 /**
@@ -15,10 +15,10 @@ const MAX_WARN_LOGS = 5;
  *   - On success (every layer either deleted or didn't exist), UPDATE the row
  *     with the cleanup timestamp.
  *   - On any unlink failure, leave mirror_cleaned_at NULL so the next
- *     openHippoDb call retries. Warn (capped at MAX_WARN_LOGS rows per call).
+ *     connection retries. Warn (capped at MAX_WARN_LOGS rows per call).
  *
- * Steady-state cost is one SELECT returning empty after every row has been
- * cleaned successfully.
+ * Returns whether every pending row was cleaned; the SELECT has no index to
+ * use, so a connection runs it only when the archive has changed since.
  */
 export function cleanupArchivedMirrors(hippoRoot, db) {
     // SAFETY: the SELECT above names exactly one column, memory_id (raw_archive.memory_id
@@ -27,10 +27,11 @@ export function cleanupArchivedMirrors(hippoRoot, db) {
         .prepare(`SELECT memory_id FROM raw_archive WHERE mirror_cleaned_at IS NULL`)
         .all();
     if (rows.length === 0)
-        return;
+        return true;
     const update = db.prepare(`UPDATE raw_archive SET mirror_cleaned_at = ? WHERE memory_id = ?`);
     const now = new Date().toISOString();
     let warnCount = 0;
+    let everyRowCleaned = true;
     for (const row of rows) {
         let allOk = true;
         for (const layer of LAYERS) {
@@ -43,7 +44,7 @@ export function cleanupArchivedMirrors(hippoRoot, db) {
             catch (err) {
                 allOk = false;
                 if (warnCount < MAX_WARN_LOGS) {
-                    log.warn(`cleanupArchivedMirrors: unlink failed for ${filePath} (will retry on next DB open): ${err instanceof Error ? err.message : String(err)}`);
+                    log.warn(`cleanupArchivedMirrors: unlink failed for ${filePath} (will retry on next DB open): ${errorMessage(err)}`);
                     warnCount += 1;
                 }
             }
@@ -51,6 +52,10 @@ export function cleanupArchivedMirrors(hippoRoot, db) {
         if (allOk) {
             update.run(now, row.memory_id);
         }
+        else {
+            everyRowCleaned = false;
+        }
     }
+    return everyRowCleaned;
 }
 //# sourceMappingURL=raw-archive-mirror-cleanup.js.map

package/dist/db/request-stores.d.ts

@@ -1,29 +1,42 @@
 import type { DatabaseSyncLike } from './sqlite.js';
+import { type OpenedDb } from './connect.js';
+import { type RequestScope } from '../util/request-scope.js';
 export interface RequestStoresOptions {
     /** Lock wait of every open in the scope that does not pass its own. */
     readonly busyWaitMs?: number;
     /** Set for hook scopes: after one lock wait runs out, {@link RequestStores.noteBusy} makes the scope's later writes skip at once. */
     readonly failFastWhenBusy?: boolean;
+    /** The store whose open-time setup another connection has already run, so an open in this scope skips it: a store worker's reader, which cannot write it. */
+    readonly setupDoneFor?: string;
 }
 /** The handles one request opened, keyed by resolved store path and lock wait. */
-export declare class RequestStores {
+export declare class RequestStores implements RequestScope {
     #private;
     readonly busyWaitMs: number | undefined;
+    readonly requestId: string | undefined;
+    readonly deadline: import("../util/request-scope.js").RequestDeadline | undefined;
+    readonly setupDoneFor: string | undefined;
     constructor(opts?: RequestStoresOptions);
     get closed(): boolean;
     /** The scope's handle on `hippoRoot`, opened on first use. `opts.busyWaitMs` overrides the scope's lock wait. */
     get(hippoRoot: string, opts?: {
         busyWaitMs?: number;
     }): DatabaseSyncLike;
+    /** `get`, with the facts of the open's probe when this call made the connection and null for a cached handle. */
+    getWithFacts(hippoRoot: string, opts?: {
+        busyWaitMs?: number;
+    }): OpenedDb;
     /** A lock held past one full wait belongs to a long transaction, so a hook scope's later writes skip at once. */
     noteBusy(): void;
     close(): void;
 }
 /** Runs `fn` in a request scope whose handles close when `fn` settles; inside an open scope `fn` joins it and `opts` is ignored. */
 export declare function runWithRequestStores<T>(fn: () => T | Promise<T>, opts?: RequestStoresOptions): Promise<T>;
+/** runWithRequestStores for a synchronous `fn`, so a batch called from outside any request still opens each store once. */
+export declare function withRequestStoresSync<T>(fn: () => T): T;
 /** The scope this code runs in, if any. A closed scope still answers: its `get` falls through to a plain open. */
 export declare function currentRequestStores(): RequestStores | undefined;
-/** Runs `fn` outside every scope, for a connection that must outlive the request that happens to open it. */
+/** Runs `fn` outside every store scope, for a connection that must outlive the request that happens to open it; its log lines keep the request's id. */
 export declare function outsideRequestStores<T>(fn: () => T): T;
 /** Whether a scope owns `db` and will close it itself. */
 export declare function isScopedHandle(db: DatabaseSyncLike): boolean;

package/dist/db/request-stores.js

@@ -1,7 +1,7 @@
 // One store handle per store for each request (HTTP call, MCP tool call, scoped CLI command), opened lazily and closed with it.
 import * as path from 'node:path';
-import { AsyncLocalStorage } from 'node:async_hooks';
-import { connectHippoDb, getHippoDbPath } from './connect.js';
+import { connectWithFacts, getHippoDbPath } from './connect.js';
+import { currentDeadline, currentRequestId, requestScopes } from '../util/request-scope.js';
 // Handles a scope owns, so closeHippoDb leaves them open for the rest of the request.
 const scopedHandles = new WeakSet();
 // process.exit skips every finally, so one exit listener closes the handles of scopes still open.
@@ -21,11 +21,16 @@ function trackLive(scope) {
 /** The handles one request opened, keyed by resolved store path and lock wait. */
 export class RequestStores {
     busyWaitMs;
+    // Taken from the scope this one opens inside, so the request's log lines keep their id.
+    requestId = currentRequestId();
+    deadline = currentDeadline();
+    setupDoneFor;
     #failFastWhenBusy;
     #handles = new Map();
     #closed = false;
     constructor(opts = {}) {
         this.busyWaitMs = opts.busyWaitMs;
+        this.setupDoneFor = opts.setupDoneFor;
         this.#failFastWhenBusy = opts.failFastWhenBusy === true;
     }
     get closed() {
@@ -33,22 +38,26 @@ export class RequestStores {
     }
     /** The scope's handle on `hippoRoot`, opened on first use. `opts.busyWaitMs` overrides the scope's lock wait. */
     get(hippoRoot, opts) {
+        return this.getWithFacts(hippoRoot, opts).db;
+    }
+    /** `get`, with the facts of the open's probe when this call made the connection and null for a cached handle. */
+    getWithFacts(hippoRoot, opts) {
         const busyWaitMs = opts?.busyWaitMs ?? this.busyWaitMs;
         // A listener or timer that outlives its request still sees this scope, so it gets a connection of its own.
         if (this.#closed)
-            return connectHippoDb(hippoRoot, busyWaitMs);
+            return connectWithFacts(hippoRoot, busyWaitMs);
         const key = `${path.resolve(getHippoDbPath(hippoRoot))}\0${busyWaitMs ?? ''}`;
         const cached = this.#handles.get(key);
         if (cached?.isOpen && !cached.isTransaction)
-            return cached;
-        const db = connectHippoDb(hippoRoot, busyWaitMs);
+            return { db: cached, facts: null };
+        const opened = connectWithFacts(hippoRoot, busyWaitMs);
         // An open nested inside a transaction gets its own connection, which its caller closes.
         if (cached?.isOpen)
-            return db;
-        this.#handles.set(key, db);
-        scopedHandles.add(db);
+            return opened;
+        this.#handles.set(key, opened.db);
+        scopedHandles.add(opened.db);
         trackLive(this);
-        return db;
+        return opened;
     }
     /** A lock held past one full wait belongs to a long transaction, so a hook scope's later writes skip at once. */
     noteBusy() {
@@ -69,14 +78,13 @@ export class RequestStores {
         this.#handles.clear();
     }
 }
-const scopes = new AsyncLocalStorage();
 /** Runs `fn` in a request scope whose handles close when `fn` settles; inside an open scope `fn` joins it and `opts` is ignored. */
 export async function runWithRequestStores(fn, opts) {
-    const outer = scopes.getStore();
+    const outer = currentRequestStores();
     if (outer && !outer.closed)
         return fn();
     const stores = new RequestStores(opts);
-    return scopes.run(stores, async () => {
+    return requestScopes.run(stores, async () => {
         try {
             return await fn();
         }
@@ -85,13 +93,27 @@ export async function runWithRequestStores(fn, opts) {
         }
     });
 }
+/** runWithRequestStores for a synchronous `fn`, so a batch called from outside any request still opens each store once. */
+export function withRequestStoresSync(fn) {
+    const outer = currentRequestStores();
+    if (outer && !outer.closed)
+        return fn();
+    const stores = new RequestStores();
+    try {
+        return requestScopes.run(stores, fn);
+    }
+    finally {
+        stores.close();
+    }
+}
 /** The scope this code runs in, if any. A closed scope still answers: its `get` falls through to a plain open. */
 export function currentRequestStores() {
-    return scopes.getStore();
+    const scope = requestScopes.getStore();
+    return scope instanceof RequestStores ? scope : undefined;
 }
-/** Runs `fn` outside every scope, for a connection that must outlive the request that happens to open it. */
+/** Runs `fn` outside every store scope, for a connection that must outlive the request that happens to open it; its log lines keep the request's id. */
 export function outsideRequestStores(fn) {
-    return scopes.exit(fn);
+    return requestScopes.run({ requestId: currentRequestId() }, fn);
 }
 /** Whether a scope owns `db` and will close it itself. */
 export function isScopedHandle(db) {

package/dist/db/tables.d.ts

@@ -3,6 +3,8 @@ import type { DatabaseSyncLike } from './sqlite.js';
 export declare function tableColumns(db: DatabaseSyncLike, tableName: string): Set<string>;
 export declare function tableHasColumn(db: DatabaseSyncLike, tableName: string, columnName: string): boolean;
 export declare function tableExists(db: DatabaseSyncLike, tableName: string): boolean;
+/** Names of the real tables, without sqlite internals and FTS virtual tables. */
+export declare function listTableNames(db: DatabaseSyncLike): string[];
 /** Row count of one table; null when the table is missing or unreadable, which callers show as unknown. */
 export declare function countTableRows(db: DatabaseSyncLike, table: string): number | null;
 export declare function pruneConsolidationRuns(db: DatabaseSyncLike, keep?: number): void;

package/dist/db/tables.js

@@ -17,6 +17,12 @@ export function tableExists(db, tableName) {
     const row = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name = ?`).get(tableName);
     return !!row?.name;
 }
+/** Names of the real tables, without sqlite internals and FTS virtual tables. */
+export function listTableNames(db) {
+    // SAFETY: each row's shape matches the single `name` column named in the SELECT below.
+    const rows = db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND sql NOT LIKE 'CREATE VIRTUAL TABLE%' ORDER BY name`).all();
+    return rows.map((r) => r.name);
+}
 /** Row count of one table; null when the table is missing or unreadable, which callers show as unknown. */
 export function countTableRows(db, table) {
     try {

package/dist/{vector-store.d.ts → db/vector-store.d.ts}

@@ -1,5 +1,5 @@
 /** Embedding vectors in the `memory_vectors` table: one Float32 BLOB per memory id, plus the one-time import of the old JSON index. */
-import type { DatabaseSyncLike } from './db.js';
+import type { DatabaseSyncLike } from './index.js';
 export declare const EMBEDDING_MODEL_META_KEY = "embedding_model";
 export declare const MEMORY_VECTORS_DDL = "\n  CREATE TABLE IF NOT EXISTS memory_vectors (\n    memory_id TEXT PRIMARY KEY,\n    model     TEXT NOT NULL,\n    dim       INTEGER NOT NULL,\n    vector    BLOB NOT NULL\n  );\n  CREATE TRIGGER IF NOT EXISTS trg_memory_vectors_delete AFTER DELETE ON memories BEGIN\n    DELETE FROM memory_vectors WHERE memory_id = old.id;\n  END;\n";
 export declare function encodeVector(vector: readonly number[]): Uint8Array;
@@ -12,6 +12,8 @@ export declare function replaceAllVectors(db: DatabaseSyncLike, index: Readonly<
 export declare function deleteOrphanVectors(db: DatabaseSyncLike): number;
 /** Vectors for `ids`, or every stored vector when `ids` is omitted. */
 export declare function loadVectors(db: DatabaseSyncLike, ids?: readonly string[]): Map<string, number[]>;
+/** `loadVectors` for `ids` with no copy: each value views the bytes the store returned for that row. */
+export declare function loadVectorViews(db: DatabaseSyncLike, ids: readonly string[]): Map<string, Float32Array>;
 export declare function storedVectorIds(db: DatabaseSyncLike): Set<string>;
 export declare function hasStoredVectors(db: DatabaseSyncLike): boolean;
 export interface VectorMatch {
@@ -28,7 +30,7 @@ export interface VectorRow {
 export declare function rankVectorRows(query: readonly number[], rows: Iterable<VectorRow>, k: number): VectorMatch[];
 /** The `k` stored vectors closest to `query` among `memories m` rows passing `where` (SQL starting with ` AND`).
  *  Filters run before the cut, so rows a caller may not see can never push admitted rows out of the top k. */
-export declare function topVectorMatches(db: DatabaseSyncLike, query: readonly number[], k: number, where: string, params: readonly (string | number)[]): VectorMatch[];
+export declare function topVectorMatches(db: DatabaseSyncLike, query: readonly number[], k: number, where: string, params: readonly (string | number)[]): Promise<VectorMatch[]>;
 /** Move a legacy `embeddings.json` into `memory_vectors`, then keep the file as `embeddings.json.imported-<stamp>`.
  *  Runs after the migration commits, so the rename never outlives rolled-back rows; a file an older binary writes later imports the same way. */
 export declare function importLegacyEmbeddingIndex(db: DatabaseSyncLike, hippoRoot: string): void;

package/dist/{vector-store.js → db/vector-store.js}

@@ -1,7 +1,8 @@
 /** Embedding vectors in the `memory_vectors` table: one Float32 BLOB per memory id, plus the one-time import of the old JSON index. */
 import * as fs from 'fs';
 import * as path from 'path';
-import { log } from './log.js';
+import { withWriteScope } from './busy.js';
+import { log } from '../util/log.js';
 /** Legacy whole-file index; imported once into `memory_vectors`, then kept beside the store as a renamed backup. */
 const LEGACY_EMBEDDINGS_FILE = 'embeddings.json';
 export const EMBEDDING_MODEL_META_KEY = 'embedding_model';
@@ -40,24 +41,9 @@ export function upsertVectors(db, rows, model) {
     }
     return written;
 }
-function inTransaction(db, fn) {
-    db.exec('BEGIN IMMEDIATE');
-    try {
-        const out = fn();
-        db.exec('COMMIT');
-        return out;
-    }
-    catch (err) {
-        try {
-            db.exec('ROLLBACK');
-        }
-        catch { /* keep the original error */ }
-        throw err;
-    }
-}
 /** Replace the whole table with `index` in one transaction, as a full rebuild does. */
 export function replaceAllVectors(db, index, model) {
-    inTransaction(db, () => {
+    withWriteScope(db, 'replace_all_vectors', () => {
         db.exec('DELETE FROM memory_vectors');
         upsertVectors(db, Object.entries(index), model);
     });
@@ -66,23 +52,33 @@ export function replaceAllVectors(db, index, model) {
 export function deleteOrphanVectors(db) {
     return Number(db.prepare('DELETE FROM memory_vectors WHERE memory_id NOT IN (SELECT id FROM memories)').run().changes ?? 0);
 }
-/** Vectors for `ids`, or every stored vector when `ids` is omitted. */
-export function loadVectors(db, ids) {
-    const out = new Map();
+// One read behind the copied and the viewed shape, so the two can never return different rows.
+function eachStoredVector(db, ids, take) {
     const read = (rows) => {
         // SAFETY: both SELECTs below name exactly these two columns; node:sqlite returns BLOBs as Uint8Array.
         for (const row of rows)
-            out.set(row.memory_id, Array.from(decodeVector(row.vector)));
+            take(row.memory_id, decodeVector(row.vector));
     };
     if (ids === undefined) {
         read(db.prepare('SELECT memory_id, vector FROM memory_vectors').iterate());
-        return out;
+        return;
     }
     const unique = [...new Set(ids)];
     for (let i = 0; i < unique.length; i += ID_CHUNK) {
         const chunk = unique.slice(i, i + ID_CHUNK);
         read(db.prepare(`SELECT memory_id, vector FROM memory_vectors WHERE memory_id IN (${chunk.map(() => '?').join(', ')})`).all(...chunk));
     }
+}
+/** Vectors for `ids`, or every stored vector when `ids` is omitted. */
+export function loadVectors(db, ids) {
+    const out = new Map();
+    eachStoredVector(db, ids, (id, vector) => out.set(id, Array.from(vector)));
+    return out;
+}
+/** `loadVectors` for `ids` with no copy: each value views the bytes the store returned for that row. */
+export function loadVectorViews(db, ids) {
+    const out = new Map();
+    eachStoredVector(db, ids, (id, vector) => out.set(id, vector));
     return out;
 }
 export function storedVectorIds(db) {
@@ -95,51 +91,80 @@ export function hasStoredVectors(db) {
 /** The `k` rows closest to `query` by cosine, best first; ties go to the smaller id, so row order never changes the top k.
  *  The query rounds to Float32 as stored vectors are, and scores add up in float64, so every store that feeds it the same bytes ranks alike. */
 export function rankVectorRows(query, rows, k) {
-    const top = [];
+    const best = bestMatches(query, k);
+    if (!best)
+        return [];
+    for (const { id, vector } of rows)
+        best.offer(id, vector);
+    return best.top;
+}
+// One scoring loop for rows in memory and rows streamed from the store, so the two can never rank apart.
+function bestMatches(query, k) {
     if (k <= 0 || query.length === 0)
-        return top;
+        return null;
     const q = Float32Array.from(query);
     let qNorm = 0;
     for (let i = 0; i < q.length; i++)
         qNorm += q[i] * q[i];
     qNorm = Math.sqrt(qNorm);
     if (qNorm < 1e-10)
-        return top;
-    for (const { id, vector: v } of rows) {
-        if (v.length !== q.length)
-            continue;
-        let dot = 0;
-        let norm = 0;
-        for (let i = 0; i < v.length; i++) {
-            dot += q[i] * v[i];
-            norm += v[i] * v[i];
-        }
-        if (norm < 1e-20)
-            continue;
-        const score = dot / (qNorm * Math.sqrt(norm));
-        if (top.length === k && !beats(score, id, top[k - 1]))
-            continue;
-        insertSorted(top, { id, score }, k);
-    }
-    return top;
+        return null;
+    const top = [];
+    return {
+        top,
+        offer(id, v) {
+            if (v.length !== q.length)
+                return;
+            let dot = 0;
+            let norm = 0;
+            for (let i = 0; i < v.length; i++) {
+                dot += q[i] * v[i];
+                norm += v[i] * v[i];
+            }
+            if (norm < 1e-20)
+                return;
+            const score = dot / (qNorm * Math.sqrt(norm));
+            if (top.length === k && !beats(score, id, top[k - 1]))
+                return;
+            insertSorted(top, { id, score }, k);
+        },
+    };
 }
-function* decodedRows(rows) {
-    for (const row of rows)
-        yield { id: row.id, vector: decodeVector(row.vector) };
+/** Rows the nearest-vector scan scores between two turns of the event loop. */
+const VECTOR_SCAN_CHUNK = 256;
+function nextTurn() {
+    return new Promise((resolve) => { setImmediate(resolve); });
 }
 /** The `k` stored vectors closest to `query` among `memories m` rows passing `where` (SQL starting with ` AND`).
  *  Filters run before the cut, so rows a caller may not see can never push admitted rows out of the top k. */
-export function topVectorMatches(db, query, k, where, params) {
-    if (k <= 0 || query.length === 0)
+export async function topVectorMatches(db, query, k, where, params) {
+    const best = bestMatches(query, k);
+    if (!best)
         return [];
-    // SHORTCUT: brute-force cosine over every vector of matching dim, fine to ~100k rows; an ANN index (sqlite-vec, HNSW) is the upgrade.
+    // SHORTCUT: brute-force cosine over every admitted vector, a chunk per event-loop turn; latency still grows with rows, fine to ~100k, and an ANN index (sqlite-vec, HNSW) is the upgrade.
+    // SAFETY: the SELECT names exactly these two columns; node:sqlite returns BLOBs as Uint8Array.
     const rows = db.prepare(`
     SELECT v.memory_id AS id, v.vector AS vector
     FROM memory_vectors v JOIN memories m ON m.id = v.memory_id
     WHERE v.dim = ?${where}
   `).iterate(query.length, ...params);
-    // SAFETY: the SELECT above names exactly these two columns; node:sqlite returns BLOBs as Uint8Array.
-    return rankVectorRows(query, decodedRows(rows), k);
+    const floats = new Float32Array(query.length);
+    const bytes = new Uint8Array(floats.buffer);
+    let scored = 0;
+    for (const row of rows) {
+        // The cursor stays open across the pause: under WAL a reader blocks no writer, and the scan keeps one snapshot.
+        if (scored > 0 && scored % VECTOR_SCAN_CHUNK === 0)
+            await nextTurn();
+        scored++;
+        if (row.vector.byteLength === bytes.byteLength) {
+            bytes.set(row.vector);
+            best.offer(row.id, floats);
+        }
+        else {
+            best.offer(row.id, decodeVector(row.vector));
+        }
+    }
+    return best.top;
 }
 // Ties break on id so the cut is the same on every run.
 function beats(score, id, other) {
@@ -213,7 +238,7 @@ export function importLegacyEmbeddingIndex(db, hippoRoot) {
     }
     // SAFETY: the SELECT names exactly the one column read.
     const model = db.prepare(`SELECT value FROM meta WHERE key = ?`).get(EMBEDDING_MODEL_META_KEY)?.value ?? '';
-    const written = inTransaction(db, () => upsertVectors(db, index, model));
+    const written = withWriteScope(db, 'import_legacy_vectors', () => upsertVectors(db, index, model));
     const backup = `${fp}.imported-${stamp()}`;
     if (moveAside(fp, backup)) {
         log.info(`moved ${written} vectors from ${LEGACY_EMBEDDINGS_FILE} into the store; the file is kept as ${path.basename(backup)}`, { hippoRoot });

package/dist/db/wal-checkpointer.d.ts

@@ -0,0 +1,11 @@
+export interface WalCheckpointer {
+    /** Counts one finished response and starts a checkpoint when enough have finished, unless one is still running. */
+    noteResponse(): void;
+    /** Resolves once the worker's connection is closed, so the caller's own close can be SQLite's last. */
+    stop(): Promise<void>;
+}
+/** The wal_autocheckpoint value for a new connection to `dbPath` in this process. */
+export declare function autoCheckpointPages(dbPath: string): number;
+/** Checkpoints `dbPath` off the calling thread until `stop()`; the worker starts with the first checkpoint that is due. */
+export declare function startWalCheckpointer(dbPath: string): WalCheckpointer;
+//# sourceMappingURL=wal-checkpointer.d.ts.map

package/dist/db/wal-checkpointer.js

@@ -0,0 +1,92 @@
+// A served store's WAL is checkpointed on a worker thread: the fsync calls of a checkpoint would otherwise stall every request on the event loop.
+import * as path from 'node:path';
+import { Worker } from 'node:worker_threads';
+import { errorMessage, log } from '../util/log.js';
+// WAL pages at which a connection checkpoints inside its own commit.
+const INLINE_CHECKPOINT_PAGES = 100;
+// The same limit while a worker checkpoints the store: SQLite still bounds the WAL when the worker falls behind.
+const BACKSTOP_CHECKPOINT_PAGES = 4000;
+// Sized so one background checkpoint covers about 1000 WAL pages of recalls, SQLite's own default.
+const RESPONSES_PER_CHECKPOINT = 32;
+// After a checkpoint that left frames behind, a sooner retry keeps what an inline checkpoint would have to flush small.
+const RESPONSES_PER_RETRY = 8;
+// A commit that lands during a pass leaves frames behind, and only a WAL with none left can be reused from its start.
+const MAX_PASSES = 4;
+// CommonJS in a string, so the worker starts the same way from dist, vitest and strip-types, with no second entry file.
+const WORKER_SOURCE = `
+const { parentPort, workerData } = require('node:worker_threads');
+const { DatabaseSync } = require('node:sqlite');
+const db = new DatabaseSync(workerData.dbPath);
+db.exec('PRAGMA busy_timeout = 5000');
+db.exec('PRAGMA synchronous = NORMAL');
+const pass = db.prepare('PRAGMA wal_checkpoint(PASSIVE)');
+parentPort.on('message', (message) => {
+  if (message === 'stop') {
+    db.close();
+    parentPort.close();
+    return;
+  }
+  let seen = -1;
+  let settled = false;
+  for (let n = 0; n < workerData.maxPasses && !settled; n++) {
+    const { busy, log, checkpointed } = pass.get();
+    if (busy) break;
+    settled = log === seen && checkpointed === log;
+    seen = log;
+  }
+  parentPort.postMessage(settled);
+});
+`;
+const backgrounded = new Set();
+/** The wal_autocheckpoint value for a new connection to `dbPath` in this process. */
+export function autoCheckpointPages(dbPath) {
+    return backgrounded.has(path.resolve(dbPath)) ? BACKSTOP_CHECKPOINT_PAGES : INLINE_CHECKPOINT_PAGES;
+}
+/** Checkpoints `dbPath` off the calling thread until `stop()`; the worker starts with the first checkpoint that is due. */
+export function startWalCheckpointer(dbPath) {
+    const key = path.resolve(dbPath);
+    let worker;
+    let exited;
+    let running = false;
+    let stopped = false;
+    let responses = 0;
+    let due = RESPONSES_PER_CHECKPOINT;
+    backgrounded.add(key);
+    const end = () => {
+        stopped = true;
+        backgrounded.delete(key);
+    };
+    const spawn = () => {
+        // An explicit execArgv keeps the parent's loaders out of the worker and silences its repeat of the node:sqlite warning.
+        const started = new Worker(WORKER_SOURCE, { eval: true, execArgv: ['--no-warnings'], workerData: { dbPath: key, maxPasses: MAX_PASSES } });
+        started.unref();
+        started.on('message', (settled) => {
+            running = false;
+            due = settled ? RESPONSES_PER_CHECKPOINT : RESPONSES_PER_RETRY;
+        });
+        started.on('error', (err) => {
+            end();
+            log.warn(`background checkpoint of ${key} stopped (${errorMessage(err)}); each connection checkpoints inside its own commit again`);
+        });
+        exited = new Promise((resolve) => { started.once('exit', () => resolve()); });
+        return started;
+    };
+    return {
+        noteResponse() {
+            if (stopped || ++responses < due || running)
+                return;
+            responses = 0;
+            running = true;
+            worker ??= spawn();
+            worker.postMessage('checkpoint');
+        },
+        async stop() {
+            const live = !stopped;
+            end();
+            if (live)
+                worker?.postMessage('stop');
+            await exited;
+        },
+    };
+}
+//# sourceMappingURL=wal-checkpointer.js.map