hippo-memory 1.69.0 → 1.71.0

package/dist/db/physics-state.d.ts

@@ -0,0 +1,23 @@
+/** Physics state persistence: particle state (position, velocity, mass, charge, temperature) in SQLite, with BLOB columns for 384-dim vectors. */
+import type { DatabaseSyncLike } from './index.js';
+import type { MemoryEntry } from '../core/memory.js';
+import type { PhysicsParticle } from '../core/physics.js';
+export declare function float32ToBuffer(arr: number[]): Buffer;
+export declare function bufferToFloat32(buf: Buffer | Uint8Array): number[];
+/** Create the memory_physics table (migration v8). */
+export declare function createPhysicsTable(db: DatabaseSyncLike): void;
+/**
+ * Load physics state for specific memory IDs (or all if no IDs given).
+ */
+export declare function loadPhysicsState(db: DatabaseSyncLike, memoryIds?: string[]): Map<string, PhysicsParticle>;
+/**
+ * Save physics state for multiple particles (batch upsert).
+ */
+export declare function savePhysicsState(db: DatabaseSyncLike, particles: PhysicsParticle[]): void;
+/** Initialize physics state for a memory from its embedding vector; returns the new particle without persisting (the caller must save). */
+export declare function initializeParticle(entry: MemoryEntry, embedding: number[], now?: Date): PhysicsParticle;
+/** Drops every particle, then seeds one per entry that has an embedding; `entries` may be read lazily, as each batch is written before the next is read. */
+export declare function resetAllPhysicsState(db: DatabaseSyncLike, entries: Iterable<MemoryEntry>, embeddingIndex: Record<string, number[]>, now?: Date): number;
+/** Refresh mass, charge and temperature for existing particles from current memory attributes (called during consolidation). */
+export declare function refreshParticleProperties(particles: PhysicsParticle[], entries: Map<string, MemoryEntry>, now?: Date): void;
+//# sourceMappingURL=physics-state.d.ts.map

package/dist/{physics-state.js → db/physics-state.js}

@@ -1,14 +1,10 @@
-/**
- * Physics state persistence for Hippo.
- * Stores and loads particle state (position, velocity, mass, charge, temperature)
- * in SQLite using BLOB columns for 384-dim vectors.
- */
-import { evalNow } from './ablation.js';
-import { computeMass, computeCharge, computeTemperature, vecZero } from './physics.js';
-import { calculateStrength } from './memory.js';
-// ---------------------------------------------------------------------------
+/** Physics state persistence: particle state (position, velocity, mass, charge, temperature) in SQLite, with BLOB columns for 384-dim vectors. */
+import { evalNow } from '../core/ablation.js';
+import { withWriteScope } from './busy.js';
+import { computeMass, computeCharge, computeTemperature, vecZero } from '../core/physics.js';
+import { calculateStrength } from '../core/memory.js';
+import { DAY_MS } from '../util/time.js';
 // Float32Array <-> Buffer serialization
-// ---------------------------------------------------------------------------
 export function float32ToBuffer(arr) {
     const f32 = new Float32Array(arr);
     return Buffer.from(f32.buffer);
@@ -22,13 +18,8 @@ export function bufferToFloat32(buf) {
     new Uint8Array(aligned).set(bytes);
     return Array.from(new Float32Array(aligned));
 }
-// ---------------------------------------------------------------------------
 // Schema migration
-// ---------------------------------------------------------------------------
-/**
- * Create the memory_physics table (migration v8).
- * Call this from db.ts MIGRATIONS array.
- */
+/** Create the memory_physics table (migration v8). */
 export function createPhysicsTable(db) {
     db.exec(`
     CREATE TABLE IF NOT EXISTS memory_physics (
@@ -100,28 +91,16 @@ export function savePhysicsState(db, particles) {
       last_simulation = excluded.last_simulation,
       updated_at = datetime('now')
   `);
-    db.exec('BEGIN IMMEDIATE');
-    try {
+    withWriteScope(db, 'save_particles', () => {
         for (const p of particles) {
             stmt.run(p.memoryId, float32ToBuffer(p.position), float32ToBuffer(p.velocity), p.mass, p.charge, p.temperature, p.lastSimulation);
         }
-        db.exec('COMMIT');
-    }
-    catch (error) {
-        try {
-            db.exec('ROLLBACK');
-        }
-        catch { /* already rolled back; keep the original error */ }
-        throw error;
-    }
+    });
 }
-/**
- * Initialize physics state for a memory from its embedding vector.
- * Returns the new particle (does not persist — caller must save).
- */
+/** Initialize physics state for a memory from its embedding vector; returns the new particle without persisting (the caller must save). */
 export function initializeParticle(entry, embedding, now = evalNow()) {
     const strength = calculateStrength(entry, now);
-    const ageDays = (now.getTime() - new Date(entry.created).getTime()) / (1000 * 60 * 60 * 24);
+    const ageDays = (now.getTime() - new Date(entry.created).getTime()) / DAY_MS;
     return {
         memoryId: entry.id,
         position: [...embedding], // copy — position will diverge from original embedding
@@ -132,35 +111,38 @@ export function initializeParticle(entry, embedding, now = evalNow()) {
         lastSimulation: now.toISOString(),
     };
 }
-/**
- * Reset all physics states from original embeddings.
- * Drops existing physics data and re-initializes from the embedding index.
- */
+const RESET_BATCH = 64;
+/** Drops every particle, then seeds one per entry that has an embedding; `entries` may be read lazily, as each batch is written before the next is read. */
 export function resetAllPhysicsState(db, entries, embeddingIndex, now = evalNow()) {
     db.exec('DELETE FROM memory_physics');
-    const particles = [];
-    for (const entry of entries) {
-        const embedding = embeddingIndex[entry.id];
-        if (!embedding || embedding.length === 0)
-            continue;
-        particles.push(initializeParticle(entry, embedding, now));
-    }
-    if (particles.length > 0) {
-        savePhysicsState(db, particles);
-    }
-    return particles.length;
+    // One transaction for every batch, so a failure part way leaves no particles rather than some.
+    return withWriteScope(db, 'reset_particles', () => {
+        let count = 0;
+        let batch = [];
+        const flush = () => {
+            savePhysicsState(db, batch);
+            count += batch.length;
+            batch = [];
+        };
+        for (const entry of entries) {
+            const embedding = embeddingIndex[entry.id];
+            if (!embedding || embedding.length === 0)
+                continue;
+            if (batch.push(initializeParticle(entry, embedding, now)) === RESET_BATCH)
+                flush();
+        }
+        flush();
+        return count;
+    });
 }
-/**
- * Refresh mass, charge, and temperature for existing particles
- * based on current memory attributes (called during consolidation).
- */
+/** Refresh mass, charge and temperature for existing particles from current memory attributes (called during consolidation). */
 export function refreshParticleProperties(particles, entries, now = evalNow()) {
     for (const p of particles) {
         const entry = entries.get(p.memoryId);
         if (!entry)
             continue;
         const strength = calculateStrength(entry, now);
-        const ageDays = (now.getTime() - new Date(entry.created).getTime()) / (1000 * 60 * 60 * 24);
+        const ageDays = (now.getTime() - new Date(entry.created).getTime()) / DAY_MS;
         p.mass = computeMass(strength, entry.retrieval_count);
         p.charge = computeCharge(entry.emotional_valence);
         p.temperature = computeTemperature(ageDays, 1.0);

package/dist/db/raw-archive-mirror-cleanup.d.ts

@@ -0,0 +1,5 @@
+import type { DatabaseSyncLike } from './index.js';
+/** Path A backfill cleanup: delete markdown mirrors (<hippoRoot>/<layer>/<id>.md) of rows archived before redaction, so RTBF holds for historical archives.
+ * Tracked by raw_archive.mirror_cleaned_at: set only on success, so a failed unlink retries next connection. Returns whether all pending rows were cleaned. */
+export declare function cleanupArchivedMirrors(hippoRoot: string, db: DatabaseSyncLike): boolean;
+//# sourceMappingURL=raw-archive-mirror-cleanup.d.ts.map

package/dist/{raw-archive-mirror-cleanup.js → db/raw-archive-mirror-cleanup.js}

@@ -1,25 +1,10 @@
 import * as fs from 'fs';
 import * as path from 'path';
-import { log } from './log.js';
+import { errorMessage, log } from '../util/log.js';
 const LAYERS = ['episodic', 'buffer', 'semantic'];
 const MAX_WARN_LOGS = 5;
-/**
- * Path A backfill cleanup: existing raw_archive rows that were archived BEFORE
- * commit 70180b5 left their markdown mirrors at <hippoRoot>/<layer>/<id>.md
- * with original content intact. The DB is already redacted via migration v20;
- * this function deletes those filesystem mirrors so RTBF holds for historical
- * archives too.
- *
- * Per-row tracking via raw_archive.mirror_cleaned_at (added in v21):
- *   - SELECT only rows WHERE mirror_cleaned_at IS NULL.
- *   - On success (every layer either deleted or didn't exist), UPDATE the row
- *     with the cleanup timestamp.
- *   - On any unlink failure, leave mirror_cleaned_at NULL so the next
- *     openHippoDb call retries. Warn (capped at MAX_WARN_LOGS rows per call).
- *
- * Steady-state cost is one SELECT returning empty after every row has been
- * cleaned successfully.
- */
+/** Path A backfill cleanup: delete markdown mirrors (<hippoRoot>/<layer>/<id>.md) of rows archived before redaction, so RTBF holds for historical archives.
+ * Tracked by raw_archive.mirror_cleaned_at: set only on success, so a failed unlink retries next connection. Returns whether all pending rows were cleaned. */
 export function cleanupArchivedMirrors(hippoRoot, db) {
     // SAFETY: the SELECT above names exactly one column, memory_id (raw_archive.memory_id
     // is NOT NULL TEXT), so the row shape matches this assertion.
@@ -27,10 +12,11 @@ export function cleanupArchivedMirrors(hippoRoot, db) {
         .prepare(`SELECT memory_id FROM raw_archive WHERE mirror_cleaned_at IS NULL`)
         .all();
     if (rows.length === 0)
-        return;
+        return true;
     const update = db.prepare(`UPDATE raw_archive SET mirror_cleaned_at = ? WHERE memory_id = ?`);
     const now = new Date().toISOString();
     let warnCount = 0;
+    let everyRowCleaned = true;
     for (const row of rows) {
         let allOk = true;
         for (const layer of LAYERS) {
@@ -43,7 +29,7 @@ export function cleanupArchivedMirrors(hippoRoot, db) {
             catch (err) {
                 allOk = false;
                 if (warnCount < MAX_WARN_LOGS) {
-                    log.warn(`cleanupArchivedMirrors: unlink failed for ${filePath} (will retry on next DB open): ${err instanceof Error ? err.message : String(err)}`);
+                    log.warn(`cleanupArchivedMirrors: unlink failed for ${filePath} (will retry on next DB open): ${errorMessage(err)}`);
                     warnCount += 1;
                 }
             }
@@ -51,6 +37,10 @@ export function cleanupArchivedMirrors(hippoRoot, db) {
         if (allOk) {
             update.run(now, row.memory_id);
         }
+        else {
+            everyRowCleaned = false;
+        }
     }
+    return everyRowCleaned;
 }
 //# sourceMappingURL=raw-archive-mirror-cleanup.js.map

package/dist/db/request-stores.d.ts

@@ -1,29 +1,42 @@
 import type { DatabaseSyncLike } from './sqlite.js';
+import { type OpenedDb } from './connect.js';
+import { type RequestScope } from '../util/request-scope.js';
 export interface RequestStoresOptions {
     /** Lock wait of every open in the scope that does not pass its own. */
     readonly busyWaitMs?: number;
     /** Set for hook scopes: after one lock wait runs out, {@link RequestStores.noteBusy} makes the scope's later writes skip at once. */
     readonly failFastWhenBusy?: boolean;
+    /** The store whose open-time setup another connection has already run, so an open in this scope skips it: a store worker's reader, which cannot write it. */
+    readonly setupDoneFor?: string;
 }
 /** The handles one request opened, keyed by resolved store path and lock wait. */
-export declare class RequestStores {
+export declare class RequestStores implements RequestScope {
     #private;
     readonly busyWaitMs: number | undefined;
+    readonly requestId: string | undefined;
+    readonly deadline: import("../util/request-scope.js").RequestDeadline | undefined;
+    readonly setupDoneFor: string | undefined;
     constructor(opts?: RequestStoresOptions);
     get closed(): boolean;
     /** The scope's handle on `hippoRoot`, opened on first use. `opts.busyWaitMs` overrides the scope's lock wait. */
     get(hippoRoot: string, opts?: {
         busyWaitMs?: number;
     }): DatabaseSyncLike;
+    /** `get`, with the facts of the open's probe when this call made the connection and null for a cached handle. */
+    getWithFacts(hippoRoot: string, opts?: {
+        busyWaitMs?: number;
+    }): OpenedDb;
     /** A lock held past one full wait belongs to a long transaction, so a hook scope's later writes skip at once. */
     noteBusy(): void;
     close(): void;
 }
 /** Runs `fn` in a request scope whose handles close when `fn` settles; inside an open scope `fn` joins it and `opts` is ignored. */
 export declare function runWithRequestStores<T>(fn: () => T | Promise<T>, opts?: RequestStoresOptions): Promise<T>;
+/** runWithRequestStores for a synchronous `fn`, so a batch called from outside any request still opens each store once. */
+export declare function withRequestStoresSync<T>(fn: () => T): T;
 /** The scope this code runs in, if any. A closed scope still answers: its `get` falls through to a plain open. */
 export declare function currentRequestStores(): RequestStores | undefined;
-/** Runs `fn` outside every scope, for a connection that must outlive the request that happens to open it. */
+/** Runs `fn` outside every store scope, for a connection that must outlive the request that happens to open it; its log lines keep the request's id. */
 export declare function outsideRequestStores<T>(fn: () => T): T;
 /** Whether a scope owns `db` and will close it itself. */
 export declare function isScopedHandle(db: DatabaseSyncLike): boolean;

package/dist/db/request-stores.js

@@ -1,7 +1,7 @@
 // One store handle per store for each request (HTTP call, MCP tool call, scoped CLI command), opened lazily and closed with it.
 import * as path from 'node:path';
-import { AsyncLocalStorage } from 'node:async_hooks';
-import { connectHippoDb, getHippoDbPath } from './connect.js';
+import { connectWithFacts, getHippoDbPath } from './connect.js';
+import { currentDeadline, currentRequestId, requestScopes } from '../util/request-scope.js';
 // Handles a scope owns, so closeHippoDb leaves them open for the rest of the request.
 const scopedHandles = new WeakSet();
 // process.exit skips every finally, so one exit listener closes the handles of scopes still open.
@@ -21,11 +21,16 @@ function trackLive(scope) {
 /** The handles one request opened, keyed by resolved store path and lock wait. */
 export class RequestStores {
     busyWaitMs;
+    // Taken from the scope this one opens inside, so the request's log lines keep their id.
+    requestId = currentRequestId();
+    deadline = currentDeadline();
+    setupDoneFor;
     #failFastWhenBusy;
     #handles = new Map();
     #closed = false;
     constructor(opts = {}) {
         this.busyWaitMs = opts.busyWaitMs;
+        this.setupDoneFor = opts.setupDoneFor;
         this.#failFastWhenBusy = opts.failFastWhenBusy === true;
     }
     get closed() {
@@ -33,22 +38,26 @@ export class RequestStores {
     }
     /** The scope's handle on `hippoRoot`, opened on first use. `opts.busyWaitMs` overrides the scope's lock wait. */
     get(hippoRoot, opts) {
+        return this.getWithFacts(hippoRoot, opts).db;
+    }
+    /** `get`, with the facts of the open's probe when this call made the connection and null for a cached handle. */
+    getWithFacts(hippoRoot, opts) {
         const busyWaitMs = opts?.busyWaitMs ?? this.busyWaitMs;
         // A listener or timer that outlives its request still sees this scope, so it gets a connection of its own.
         if (this.#closed)
-            return connectHippoDb(hippoRoot, busyWaitMs);
+            return connectWithFacts(hippoRoot, busyWaitMs);
         const key = `${path.resolve(getHippoDbPath(hippoRoot))}\0${busyWaitMs ?? ''}`;
         const cached = this.#handles.get(key);
         if (cached?.isOpen && !cached.isTransaction)
-            return cached;
-        const db = connectHippoDb(hippoRoot, busyWaitMs);
+            return { db: cached, facts: null };
+        const opened = connectWithFacts(hippoRoot, busyWaitMs);
         // An open nested inside a transaction gets its own connection, which its caller closes.
         if (cached?.isOpen)
-            return db;
-        this.#handles.set(key, db);
-        scopedHandles.add(db);
+            return opened;
+        this.#handles.set(key, opened.db);
+        scopedHandles.add(opened.db);
         trackLive(this);
-        return db;
+        return opened;
     }
     /** A lock held past one full wait belongs to a long transaction, so a hook scope's later writes skip at once. */
     noteBusy() {
@@ -69,14 +78,13 @@ export class RequestStores {
         this.#handles.clear();
     }
 }
-const scopes = new AsyncLocalStorage();
 /** Runs `fn` in a request scope whose handles close when `fn` settles; inside an open scope `fn` joins it and `opts` is ignored. */
 export async function runWithRequestStores(fn, opts) {
-    const outer = scopes.getStore();
+    const outer = currentRequestStores();
     if (outer && !outer.closed)
         return fn();
     const stores = new RequestStores(opts);
-    return scopes.run(stores, async () => {
+    return requestScopes.run(stores, async () => {
         try {
             return await fn();
         }
@@ -85,13 +93,27 @@ export async function runWithRequestStores(fn, opts) {
         }
     });
 }
+/** runWithRequestStores for a synchronous `fn`, so a batch called from outside any request still opens each store once. */
+export function withRequestStoresSync(fn) {
+    const outer = currentRequestStores();
+    if (outer && !outer.closed)
+        return fn();
+    const stores = new RequestStores();
+    try {
+        return requestScopes.run(stores, fn);
+    }
+    finally {
+        stores.close();
+    }
+}
 /** The scope this code runs in, if any. A closed scope still answers: its `get` falls through to a plain open. */
 export function currentRequestStores() {
-    return scopes.getStore();
+    const scope = requestScopes.getStore();
+    return scope instanceof RequestStores ? scope : undefined;
 }
-/** Runs `fn` outside every scope, for a connection that must outlive the request that happens to open it. */
+/** Runs `fn` outside every store scope, for a connection that must outlive the request that happens to open it; its log lines keep the request's id. */
 export function outsideRequestStores(fn) {
-    return scopes.exit(fn);
+    return requestScopes.run({ requestId: currentRequestId() }, fn);
 }
 /** Whether a scope owns `db` and will close it itself. */
 export function isScopedHandle(db) {

package/dist/db/tables.d.ts

@@ -3,6 +3,8 @@ import type { DatabaseSyncLike } from './sqlite.js';
 export declare function tableColumns(db: DatabaseSyncLike, tableName: string): Set<string>;
 export declare function tableHasColumn(db: DatabaseSyncLike, tableName: string, columnName: string): boolean;
 export declare function tableExists(db: DatabaseSyncLike, tableName: string): boolean;
+/** Names of the real tables, without sqlite internals and FTS virtual tables. */
+export declare function listTableNames(db: DatabaseSyncLike): string[];
 /** Row count of one table; null when the table is missing or unreadable, which callers show as unknown. */
 export declare function countTableRows(db: DatabaseSyncLike, table: string): number | null;
 export declare function pruneConsolidationRuns(db: DatabaseSyncLike, keep?: number): void;

package/dist/db/tables.js

@@ -17,6 +17,12 @@ export function tableExists(db, tableName) {
     const row = db.prepare(`SELECT name FROM sqlite_master WHERE type='table' AND name = ?`).get(tableName);
     return !!row?.name;
 }
+/** Names of the real tables, without sqlite internals and FTS virtual tables. */
+export function listTableNames(db) {
+    // SAFETY: each row's shape matches the single `name` column named in the SELECT below.
+    const rows = db.prepare(`SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' AND sql NOT LIKE 'CREATE VIRTUAL TABLE%' ORDER BY name`).all();
+    return rows.map((r) => r.name);
+}
 /** Row count of one table; null when the table is missing or unreadable, which callers show as unknown. */
 export function countTableRows(db, table) {
     try {

package/dist/{vector-store.d.ts → db/vector-store.d.ts}

@@ -1,5 +1,5 @@
 /** Embedding vectors in the `memory_vectors` table: one Float32 BLOB per memory id, plus the one-time import of the old JSON index. */
-import type { DatabaseSyncLike } from './db.js';
+import type { DatabaseSyncLike } from './index.js';
 export declare const EMBEDDING_MODEL_META_KEY = "embedding_model";
 export declare const MEMORY_VECTORS_DDL = "\n  CREATE TABLE IF NOT EXISTS memory_vectors (\n    memory_id TEXT PRIMARY KEY,\n    model     TEXT NOT NULL,\n    dim       INTEGER NOT NULL,\n    vector    BLOB NOT NULL\n  );\n  CREATE TRIGGER IF NOT EXISTS trg_memory_vectors_delete AFTER DELETE ON memories BEGIN\n    DELETE FROM memory_vectors WHERE memory_id = old.id;\n  END;\n";
 export declare function encodeVector(vector: readonly number[]): Uint8Array;
@@ -12,7 +12,11 @@ export declare function replaceAllVectors(db: DatabaseSyncLike, index: Readonly<
 export declare function deleteOrphanVectors(db: DatabaseSyncLike): number;
 /** Vectors for `ids`, or every stored vector when `ids` is omitted. */
 export declare function loadVectors(db: DatabaseSyncLike, ids?: readonly string[]): Map<string, number[]>;
+/** `loadVectors` for `ids` with no copy: each value views the bytes the store returned for that row. */
+export declare function loadVectorViews(db: DatabaseSyncLike, ids: readonly string[]): Map<string, Float32Array>;
 export declare function storedVectorIds(db: DatabaseSyncLike): Set<string>;
+/** Float count of the first stored row, from its blob length; the vector is never read. */
+export declare function storedVectorDims(db: DatabaseSyncLike): number | undefined;
 export declare function hasStoredVectors(db: DatabaseSyncLike): boolean;
 export interface VectorMatch {
     id: string;
@@ -28,7 +32,7 @@ export interface VectorRow {
 export declare function rankVectorRows(query: readonly number[], rows: Iterable<VectorRow>, k: number): VectorMatch[];
 /** The `k` stored vectors closest to `query` among `memories m` rows passing `where` (SQL starting with ` AND`).
  *  Filters run before the cut, so rows a caller may not see can never push admitted rows out of the top k. */
-export declare function topVectorMatches(db: DatabaseSyncLike, query: readonly number[], k: number, where: string, params: readonly (string | number)[]): VectorMatch[];
+export declare function topVectorMatches(db: DatabaseSyncLike, query: readonly number[], k: number, where: string, params: readonly (string | number)[]): Promise<VectorMatch[]>;
 /** Move a legacy `embeddings.json` into `memory_vectors`, then keep the file as `embeddings.json.imported-<stamp>`.
  *  Runs after the migration commits, so the rename never outlives rolled-back rows; a file an older binary writes later imports the same way. */
 export declare function importLegacyEmbeddingIndex(db: DatabaseSyncLike, hippoRoot: string): void;

package/dist/{vector-store.js → db/vector-store.js}

@@ -1,7 +1,8 @@
 /** Embedding vectors in the `memory_vectors` table: one Float32 BLOB per memory id, plus the one-time import of the old JSON index. */
 import * as fs from 'fs';
 import * as path from 'path';
-import { log } from './log.js';
+import { withWriteScope } from './busy.js';
+import { log } from '../util/log.js';
 /** Legacy whole-file index; imported once into `memory_vectors`, then kept beside the store as a renamed backup. */
 const LEGACY_EMBEDDINGS_FILE = 'embeddings.json';
 export const EMBEDDING_MODEL_META_KEY = 'embedding_model';
@@ -40,24 +41,9 @@ export function upsertVectors(db, rows, model) {
     }
     return written;
 }
-function inTransaction(db, fn) {
-    db.exec('BEGIN IMMEDIATE');
-    try {
-        const out = fn();
-        db.exec('COMMIT');
-        return out;
-    }
-    catch (err) {
-        try {
-            db.exec('ROLLBACK');
-        }
-        catch { /* keep the original error */ }
-        throw err;
-    }
-}
 /** Replace the whole table with `index` in one transaction, as a full rebuild does. */
 export function replaceAllVectors(db, index, model) {
-    inTransaction(db, () => {
+    withWriteScope(db, 'replace_all_vectors', () => {
         db.exec('DELETE FROM memory_vectors');
         upsertVectors(db, Object.entries(index), model);
     });
@@ -66,80 +52,126 @@ export function replaceAllVectors(db, index, model) {
 export function deleteOrphanVectors(db) {
     return Number(db.prepare('DELETE FROM memory_vectors WHERE memory_id NOT IN (SELECT id FROM memories)').run().changes ?? 0);
 }
-/** Vectors for `ids`, or every stored vector when `ids` is omitted. */
-export function loadVectors(db, ids) {
-    const out = new Map();
+// One read behind the copied and the viewed shape, so the two can never return different rows.
+function eachStoredVector(db, ids, take) {
     const read = (rows) => {
         // SAFETY: both SELECTs below name exactly these two columns; node:sqlite returns BLOBs as Uint8Array.
         for (const row of rows)
-            out.set(row.memory_id, Array.from(decodeVector(row.vector)));
+            take(row.memory_id, decodeVector(row.vector));
     };
     if (ids === undefined) {
         read(db.prepare('SELECT memory_id, vector FROM memory_vectors').iterate());
-        return out;
+        return;
     }
     const unique = [...new Set(ids)];
     for (let i = 0; i < unique.length; i += ID_CHUNK) {
         const chunk = unique.slice(i, i + ID_CHUNK);
         read(db.prepare(`SELECT memory_id, vector FROM memory_vectors WHERE memory_id IN (${chunk.map(() => '?').join(', ')})`).all(...chunk));
     }
+}
+/** Vectors for `ids`, or every stored vector when `ids` is omitted. */
+export function loadVectors(db, ids) {
+    const out = new Map();
+    eachStoredVector(db, ids, (id, vector) => out.set(id, Array.from(vector)));
+    return out;
+}
+/** `loadVectors` for `ids` with no copy: each value views the bytes the store returned for that row. */
+export function loadVectorViews(db, ids) {
+    const out = new Map();
+    eachStoredVector(db, ids, (id, vector) => out.set(id, vector));
     return out;
 }
 export function storedVectorIds(db) {
     // SAFETY: the SELECT names exactly the one column read.
     return new Set(db.prepare('SELECT memory_id FROM memory_vectors').all().map((r) => r.memory_id));
 }
+/** Float count of the first stored row, from its blob length; the vector is never read. */
+export function storedVectorDims(db) {
+    // SAFETY: the SELECT names exactly the one column read.
+    const row = db.prepare('SELECT length(vector) AS bytes FROM memory_vectors LIMIT 1').get();
+    return row === undefined ? undefined : Math.floor(row.bytes / 4);
+}
 export function hasStoredVectors(db) {
     return db.prepare('SELECT 1 FROM memory_vectors LIMIT 1').get() !== undefined;
 }
 /** The `k` rows closest to `query` by cosine, best first; ties go to the smaller id, so row order never changes the top k.
  *  The query rounds to Float32 as stored vectors are, and scores add up in float64, so every store that feeds it the same bytes ranks alike. */
 export function rankVectorRows(query, rows, k) {
-    const top = [];
+    const best = bestMatches(query, k);
+    if (!best)
+        return [];
+    for (const { id, vector } of rows)
+        best.offer(id, vector);
+    return best.top;
+}
+// One scoring loop for rows in memory and rows streamed from the store, so the two can never rank apart.
+function bestMatches(query, k) {
     if (k <= 0 || query.length === 0)
-        return top;
+        return null;
     const q = Float32Array.from(query);
     let qNorm = 0;
     for (let i = 0; i < q.length; i++)
         qNorm += q[i] * q[i];
     qNorm = Math.sqrt(qNorm);
     if (qNorm < 1e-10)
-        return top;
-    for (const { id, vector: v } of rows) {
-        if (v.length !== q.length)
-            continue;
-        let dot = 0;
-        let norm = 0;
-        for (let i = 0; i < v.length; i++) {
-            dot += q[i] * v[i];
-            norm += v[i] * v[i];
-        }
-        if (norm < 1e-20)
-            continue;
-        const score = dot / (qNorm * Math.sqrt(norm));
-        if (top.length === k && !beats(score, id, top[k - 1]))
-            continue;
-        insertSorted(top, { id, score }, k);
-    }
-    return top;
+        return null;
+    const top = [];
+    return {
+        top,
+        offer(id, v) {
+            if (v.length !== q.length)
+                return;
+            let dot = 0;
+            let norm = 0;
+            for (let i = 0; i < v.length; i++) {
+                dot += q[i] * v[i];
+                norm += v[i] * v[i];
+            }
+            if (norm < 1e-20)
+                return;
+            const score = dot / (qNorm * Math.sqrt(norm));
+            if (top.length === k && !beats(score, id, top[k - 1]))
+                return;
+            insertSorted(top, { id, score }, k);
+        },
+    };
 }
-function* decodedRows(rows) {
-    for (const row of rows)
-        yield { id: row.id, vector: decodeVector(row.vector) };
+/** Rows the nearest-vector scan scores between two turns of the event loop. */
+const VECTOR_SCAN_CHUNK = 256;
+function nextTurn() {
+    return new Promise((resolve) => { setImmediate(resolve); });
 }
 /** The `k` stored vectors closest to `query` among `memories m` rows passing `where` (SQL starting with ` AND`).
  *  Filters run before the cut, so rows a caller may not see can never push admitted rows out of the top k. */
-export function topVectorMatches(db, query, k, where, params) {
-    if (k <= 0 || query.length === 0)
+export async function topVectorMatches(db, query, k, where, params) {
+    const best = bestMatches(query, k);
+    if (!best)
         return [];
-    // SHORTCUT: brute-force cosine over every vector of matching dim, fine to ~100k rows; an ANN index (sqlite-vec, HNSW) is the upgrade.
+    // SHORTCUT: brute-force cosine over every admitted vector, a chunk per event-loop turn; latency
+    // still grows with rows, fine to ~100k, and an ANN index (sqlite-vec, HNSW) is the upgrade.
+    // SAFETY: the SELECT names exactly these two columns; node:sqlite returns BLOBs as Uint8Array.
     const rows = db.prepare(`
     SELECT v.memory_id AS id, v.vector AS vector
     FROM memory_vectors v JOIN memories m ON m.id = v.memory_id
     WHERE v.dim = ?${where}
   `).iterate(query.length, ...params);
-    // SAFETY: the SELECT above names exactly these two columns; node:sqlite returns BLOBs as Uint8Array.
-    return rankVectorRows(query, decodedRows(rows), k);
+    const floats = new Float32Array(query.length);
+    const bytes = new Uint8Array(floats.buffer);
+    let scored = 0;
+    for (const row of rows) {
+        // The cursor stays open across the pause: under WAL a reader blocks no writer, and the scan keeps one snapshot.
+        if (scored > 0 && scored % VECTOR_SCAN_CHUNK === 0)
+            await nextTurn();
+        scored++;
+        if (row.vector.byteLength === bytes.byteLength) {
+            bytes.set(row.vector);
+            best.offer(row.id, floats);
+        }
+        else {
+            best.offer(row.id, decodeVector(row.vector));
+        }
+    }
+    return best.top;
 }
 // Ties break on id so the cut is the same on every run.
 function beats(score, id, other) {
@@ -213,7 +245,7 @@ export function importLegacyEmbeddingIndex(db, hippoRoot) {
     }
     // SAFETY: the SELECT names exactly the one column read.
     const model = db.prepare(`SELECT value FROM meta WHERE key = ?`).get(EMBEDDING_MODEL_META_KEY)?.value ?? '';
-    const written = inTransaction(db, () => upsertVectors(db, index, model));
+    const written = withWriteScope(db, 'import_legacy_vectors', () => upsertVectors(db, index, model));
     const backup = `${fp}.imported-${stamp()}`;
     if (moveAside(fp, backup)) {
         log.info(`moved ${written} vectors from ${LEGACY_EMBEDDINGS_FILE} into the store; the file is kept as ${path.basename(backup)}`, { hippoRoot });

package/dist/db/wal-checkpointer.d.ts

@@ -0,0 +1,11 @@
+export interface WalCheckpointer {
+    /** Counts one finished response and starts a checkpoint when enough have finished, unless one is still running. */
+    noteResponse(): void;
+    /** Resolves once the worker's connection is closed, so the caller's own close can be SQLite's last. */
+    stop(): Promise<void>;
+}
+/** The wal_autocheckpoint value for a new connection to `dbPath` in this process. */
+export declare function autoCheckpointPages(dbPath: string): number;
+/** Checkpoints `dbPath` off the calling thread until `stop()`; the worker starts with the first checkpoint that is due. */
+export declare function startWalCheckpointer(dbPath: string): WalCheckpointer;
+//# sourceMappingURL=wal-checkpointer.d.ts.map