hippo-memory 1.68.0 → 1.70.0

package/dist/{dashboard.js → dashboard/dashboard.js}

@@ -2,18 +2,19 @@
 import * as http from 'http';
 import * as path from 'path';
 import * as fs from 'fs';
-import { randomBytes, timingSafeEqual } from 'crypto';
-import { evalNow } from './ablation.js';
-import { readEntry } from './store/entry-reads.js';
-import { listCards } from './store-cards.js';
-import { resolveTenantId } from './tenant.js';
-import { loadCardDetail } from './card-detail.js';
-import { closeAfterReply, isCrossSite, LOOPBACK_HOST_HEADER } from './http-util.js';
-import { log } from './log.js';
+import { createHash, randomBytes, randomUUID, timingSafeEqual } from 'crypto';
+import { evalNow } from '../core/ablation.js';
+import { readEntry } from '../store/entry-reads.js';
+import { listCards } from '../store/cards.js';
+import { resolveTenantId } from '../store/tenant.js';
+import { loadCardDetail } from '../store/card-detail.js';
+import { bodyDeadlineMs, BodyTimeoutError, closeAfterReply, isCrossSite, LOOPBACK_HOST_HEADER } from '../util/http-util.js';
+import { errorMessage, log } from '../util/log.js';
 import { createSnapshotService, isLiveMemory } from './dashboard-snapshot.js';
 import { buildMemoryDetail, buildMemoryPage, buildOverview, buildProjectDetail, buildSearch, } from './dashboard-queries.js';
 import { forgetMemory, markWrong, pinMemory, resolveOpenConflict, } from './dashboard-actions.js';
 import { ParamError, parseActionBody, parseConflictId, parseMemoryId, parseMemoryQuery, parseSearchText, } from './dashboard-params.js';
+import { runWithRequestId } from '../util/request-scope.js';
 const MIME_TYPES = {
     '.html': 'text/html; charset=utf-8',
     '.js': 'application/javascript; charset=utf-8',
@@ -72,6 +73,10 @@ function readActionBody(req) {
         const chunks = [];
         let size = 0;
         let aborted = false;
+        const deadlineMs = bodyDeadlineMs();
+        const timer = setTimeout(() => reject(new BodyTimeoutError(`request body not received within ${deadlineMs} ms`)), deadlineMs);
+        // Unref: a request that already failed another way must not hold the process open for the rest of the deadline.
+        timer.unref();
         req.on('data', (chunk) => {
             size += chunk.length;
             if (size <= BODY_MAX_BYTES)
@@ -83,6 +88,7 @@ function readActionBody(req) {
         });
         req.on('error', reject);
         req.on('end', () => {
+            clearTimeout(timer);
             if (size > BODY_MAX_BYTES)
                 return reject(new ParamError(`Body must be at most ${BODY_MAX_BYTES} bytes`));
             try {
@@ -123,7 +129,7 @@ function handleRead(ctx, segments) {
             return sendOrNotFound(res, null);
         const snapshotId = snapshots.get(tenantId, fresh).id;
         const embedded = snapshots.embeddedIds()?.has(entry.id) ?? false;
-        return sendOrNotFound(res, buildMemoryDetail(hippoRoot, tenantId, entry, snapshotId, ctx.now(), embedded));
+        return sendOrNotFound(res, buildMemoryDetail(hippoRoot, tenantId, entry, { snapshotId, nowMs: ctx.now(), embedded }));
     }
     if (head === 'search' && segments.length === 2) {
         return sendOrNotFound(res, buildSearch(snapshots.get(tenantId, fresh), parseSearchText(url.searchParams)));
@@ -175,7 +181,7 @@ async function handlePost(ctx, segments) {
     if (result.entry) {
         // The write is committed, so a rebuild failure must not turn it into a 500: the next read rebuilds.
         const embedded = snapshots.embeddedIds()?.has(result.entry.id) ?? false;
-        const detail = buildMemoryDetail(hippoRoot, tenantId, result.entry, snapshots.currentId(), ctx.now(), embedded);
+        const detail = buildMemoryDetail(hippoRoot, tenantId, result.entry, { snapshotId: snapshots.currentId(), nowMs: ctx.now(), embedded });
         jsonResponse(res, detail, result.status);
     }
     else {
@@ -183,21 +189,43 @@ async function handlePost(ctx, segments) {
     }
     return true;
 }
+// One style block, not style attributes, so the policy below can name its hash and allow no other inline style.
+const NOT_BUILT_CSS = "body{font-family:Georgia,'Palatino Linotype',serif;max-width:640px;margin:60px auto;padding:24px;line-height:1.6;background:#f4efe6;color:#3a3228}" +
+    'h1{color:#c45c3c}' +
+    'pre{background:#faf7f2;padding:16px;border:1px solid #c4b9a8;border-radius:3px;font-family:Consolas,monospace}';
+// Memory text is written by other tools and people, so every reply tells the browser to load code only from this server and never to frame, sniff or name the page elsewhere.
+const SECURITY_HEADERS = {
+    'Content-Security-Policy': [
+        "default-src 'none'",
+        "script-src 'self'",
+        `style-src 'self' 'sha256-${createHash('sha256').update(NOT_BUILT_CSS).digest('base64')}'`,
+        "img-src 'self' data:",
+        "font-src 'self'",
+        "connect-src 'self'",
+        "base-uri 'none'",
+        "form-action 'none'",
+        "frame-ancestors 'none'",
+    ].join('; '),
+    'X-Content-Type-Options': 'nosniff',
+    'Referrer-Policy': 'no-referrer',
+    'X-Frame-Options': 'DENY',
+};
 function notBuilt(res) {
     res.writeHead(404, { 'Content-Type': 'text/html; charset=utf-8' });
     res.end(`<!doctype html>
-<html lang="en"><head><title>Hippo Dashboard</title><meta charset="utf-8"></head>
-<body style="font-family:Georgia,'Palatino Linotype',serif;max-width:640px;margin:60px auto;padding:24px;line-height:1.6;background:#f4efe6;color:#3a3228">
-<h1 style="color:#c45c3c">Hippo Dashboard</h1>
+<html lang="en"><head><title>Hippo Dashboard</title><meta charset="utf-8"><style>${NOT_BUILT_CSS}</style></head>
+<body>
+<h1>Hippo Dashboard</h1>
 <p>The React UI bundle is not built yet. Run:</p>
-<pre style="background:#faf7f2;padding:16px;border:1px solid #c4b9a8;border-radius:3px;font-family:Consolas,monospace">cd ui && npm install && npm run build</pre>
+<pre>cd ui && npm install && npm run build</pre>
 <p>Then refresh this page. The dashboard server will serve <code>dist-ui/index.html</code> automatically once present.</p>
 </body></html>`);
 }
 function serveSpa(res, distUiDir, pathname) {
     const safePath = path.normalize(pathname).replace(/^(\.\.[/\\])+/, '');
     const filePath = path.join(distUiDir, safePath);
-    if (!filePath.startsWith(distUiDir))
+    // The separator matters: a bare prefix also admits a sibling folder whose name starts with this one.
+    if (filePath !== distUiDir && !filePath.startsWith(distUiDir + path.sep))
         return forbidden(res);
     if (fs.existsSync(filePath) && fs.statSync(filePath).isFile() && serveStaticFile(res, filePath))
         return;
@@ -221,6 +249,78 @@ function cookieValue(header, name) {
     }
     return undefined;
 }
+/** Where a page load that carried the token goes next, without it; null for an API call or a write, which is answered in place. */
+function tokenFreeLocation(req, url) {
+    if ((req.method !== 'GET' && req.method !== 'HEAD') || url.pathname.startsWith('/api/'))
+        return null;
+    const query = new URLSearchParams(url.searchParams);
+    query.delete('token');
+    const rest = query.toString();
+    // One leading slash only: `//host` would send the browser to another site.
+    return `${url.pathname.replace(/^\/+/, '/')}${rest ? `?${rest}` : ''}`;
+}
+/** Moves the browser off the URL that carries the token, so the token stays out of history, bookmarks and copied links; false when the request is answered in place. */
+function leaveTokenUrl(req, res, url) {
+    const location = tokenFreeLocation(req, url);
+    if (location === null)
+        return false;
+    if (req.headers['sec-fetch-site'] !== 'cross-site') {
+        res.writeHead(303, { Location: location });
+        res.end();
+        return true;
+    }
+    // A browser withholds a SameSite=Strict cookie from a redirect another site started; a refresh from this page is same-site.
+    const target = location.replace(/&/g, '&amp;').replace(/"/g, '&quot;');
+    res.writeHead(200, { 'Content-Type': 'text/html; charset=utf-8' });
+    res.end(`<!doctype html>\n<html lang="en"><head><meta charset="utf-8"><meta http-equiv="refresh" content="0;url=${target}"><title>Hippo Dashboard</title></head><body></body></html>`);
+    return true;
+}
+/** True when the request may proceed; false when this already answered it (redirect off the token URL, or 401). */
+function admitDashboardRequest(req, res, url, access) {
+    const { token, port } = access;
+    // Cookies ignore the port, so the name carries it: two dashboards on one host keep separate tokens.
+    const cookieName = `hippo_dashboard_${req.socket.localPort ?? port}`;
+    if (sameToken(url.searchParams.get('token') ?? undefined, token)) {
+        res.setHeader('Set-Cookie', `${cookieName}=${token}; HttpOnly; SameSite=Strict; Path=/`);
+        if (leaveTokenUrl(req, res, url))
+            return false;
+    }
+    else if (!sameToken(cookieValue(req.headers.cookie, cookieName), token)) {
+        res.writeHead(401, { 'Content-Type': 'text/plain' });
+        res.end('Unauthorized: open the dashboard with the URL `hippo dashboard` printed; it carries the access token.');
+        return false;
+    }
+    return true;
+}
+function answerDashboardFailure(req, res, err) {
+    const clientFault = err instanceof ParamError || err instanceof URIError || err instanceof BodyTimeoutError;
+    // A cut-short response is logged even for a client fault; a bare 400 is not.
+    if (res.headersSent || !clientFault) {
+        log.error('dashboard request failed', { error: errorMessage(err), path: req.url });
+    }
+    if (res.headersSent) {
+        res.end();
+        return;
+    }
+    if (err instanceof BodyDrainExceeded || err instanceof BodyTimeoutError) {
+        // No `Connection: close` header: Node then destroys the socket as soon as the reply is written.
+        res.writeHead(err instanceof BodyTimeoutError ? 408 : 400, { 'Content-Type': 'application/json; charset=utf-8' });
+        res.end(JSON.stringify({ error: err.message }), () => closeAfterReply(req));
+        return;
+    }
+    if (err instanceof ParamError)
+        return jsonResponse(res, { error: err.message }, 400);
+    if (err instanceof URIError)
+        return jsonResponse(res, { error: 'Malformed URL path' }, 400);
+    jsonResponse(res, { error: 'Internal error' }, 500);
+}
+function printDashboardBanner(boundPort, token, distUiDir) {
+    // The banner is the `hippo dashboard` command's printed result, so it stays on stdout.
+    console.log(`Hippo Dashboard running at http://localhost:${boundPort}/?token=${token}`);
+    if (distUiDir !== null)
+        console.log(`Serving React UI from ${distUiDir}`);
+    console.log('Press Ctrl+C to stop.');
+}
 /** Serves the dashboard on 127.0.0.1 behind a per-start `token` (tests pass one), since loopback alone lets any local process read every memory; `opts` sets the projection and cache clocks. */
 export function serveDashboard(hippoRoot, port = 3333, token = randomBytes(32).toString('base64url'), opts) {
     const distUiDir = path.resolve(import.meta.dirname, '..', 'dist-ui');
@@ -232,16 +332,8 @@ export function serveDashboard(hippoRoot, port = 3333, token = randomBytes(32).t
         if (host !== undefined && !LOOPBACK_HOST_HEADER.test(host))
             return forbidden(res);
         const url = new URL(req.url ?? '/', `http://${req.headers.host ?? 'localhost'}`);
-        // Cookies ignore the port, so the name carries it: two dashboards on one host keep separate tokens.
-        const cookieName = `hippo_dashboard_${req.socket.localPort ?? port}`;
-        if (sameToken(url.searchParams.get('token') ?? undefined, token)) {
-            res.setHeader('Set-Cookie', `${cookieName}=${token}; HttpOnly; SameSite=Strict; Path=/`);
-        }
-        else if (!sameToken(cookieValue(req.headers.cookie, cookieName), token)) {
-            res.writeHead(401, { 'Content-Type': 'text/plain' });
-            res.end('Unauthorized: open the dashboard with the URL `hippo dashboard` printed; it carries the access token.');
+        if (!admitDashboardRequest(req, res, url, { token, port }))
             return;
-        }
         if (url.pathname.startsWith('/api/')) {
             // A malformed % sequence throws URIError here; the caller answers 400.
             const segments = url.pathname.split('/').filter((s) => s !== '').map((s) => decodeURIComponent(s));
@@ -257,39 +349,18 @@ export function serveDashboard(hippoRoot, port = 3333, token = randomBytes(32).t
             return serveSpa(res, distUiDir, url.pathname);
         notBuilt(res);
     };
-    const server = http.createServer((req, res) => {
-        handleRequest(req, res).catch((err) => {
-            const clientFault = err instanceof ParamError || err instanceof URIError;
-            // A cut-short response is logged even for a client fault; a bare 400 is not.
-            if (res.headersSent || !clientFault) {
-                log.error('dashboard request failed', { error: err instanceof Error ? err.message : String(err), path: req.url });
-            }
-            if (res.headersSent) {
-                res.end();
-                return;
-            }
-            if (err instanceof BodyDrainExceeded) {
-                // No `Connection: close` header: Node then destroys the socket as soon as the reply is written.
-                res.writeHead(400, { 'Content-Type': 'application/json; charset=utf-8' });
-                res.end(JSON.stringify({ error: err.message }), () => closeAfterReply(req));
-                return;
-            }
-            if (err instanceof ParamError)
-                return jsonResponse(res, { error: err.message }, 400);
-            if (err instanceof URIError)
-                return jsonResponse(res, { error: 'Malformed URL path' }, 400);
-            jsonResponse(res, { error: 'Internal error' }, 500);
-        });
-    });
+    const onRequest = (req, res) => {
+        for (const [name, value] of Object.entries(SECURITY_HEADERS))
+            res.setHeader(name, value);
+        handleRequest(req, res).catch((err) => answerDashboardFailure(req, res, err));
+    };
+    // One id per request, so the lines it logs can be told apart from a concurrent request's.
+    const server = http.createServer((req, res) => runWithRequestId(randomUUID(), () => onRequest(req, res)));
     server.on('close', () => snapshots.close());
     server.listen(port, '127.0.0.1', () => {
         // SAFETY: listen() was given a TCP port, so address() is AddressInfo, never a pipe name.
         const boundPort = server.address().port;
-        // The banner is the `hippo dashboard` command's printed result, so it stays on stdout.
-        console.log(`Hippo Dashboard running at http://localhost:${boundPort}/?token=${token}`);
-        if (hasDistUi)
-            console.log(`Serving React UI from ${distUiDir}`);
-        console.log('Press Ctrl+C to stop.');
+        printDashboardBanner(boundPort, token, hasDistUi ? distUiDir : null);
     });
     return server;
 }

package/dist/db/backup.js

@@ -1,6 +1,6 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
-import { log } from '../log.js';
+import { log } from '../util/log.js';
 /** Copies the database before a repair writes, so the audit ids plus this file are the way back. */
 export function backupStore(db, hippoRoot, label, now = new Date()) {
     const dir = path.join(hippoRoot, 'backups');

package/dist/db/busy.d.ts

@@ -8,7 +8,24 @@ export declare class StoreBusyError extends Error {
 /** A held lock in any store: SQLite's busy codes or a port's StoreBusyError. */
 export declare function isStoreBusy<E>(error: E): boolean;
 export declare function execWithBusyRetry(db: DatabaseSyncLike, sql: string, timeoutMs?: number): void;
+/** Waits for the lock past busy_timeout when given; ignored inside a caller's transaction, which already holds it. */
+export interface WriteScopeOptions {
+    readonly busyWaitMs?: number;
+}
 /** Runs `fn` as one write: BEGIN IMMEDIATE on an idle handle, else SAVEPOINT `name` inside the caller's transaction.
  *  A deferred scope that reads first cannot wait out another writer, while BEGIN IMMEDIATE waits the open's busy_timeout. */
-export declare function withWriteScope<T>(db: DatabaseSyncLike, name: string, fn: () => T): T;
+export declare function withWriteScope<T>(db: DatabaseSyncLike, name: string, fn: () => T, opts?: WriteScopeOptions): T;
+/** Runs the reads in `fn` against one snapshot, so two statements cannot straddle another process's write. */
+export declare function withReadSnapshot<T>(db: DatabaseSyncLike, fn: () => T): T;
+/** Runs `fn` and undoes every write it made, throw or not: a dry run that reports what a real run would do. Takes no write lock up front. */
+export declare function withTrialScope<T>(db: DatabaseSyncLike, name: string, fn: () => T): T;
+/** What `withWriteScopeOr` hands its callback's `rollback(value)`; a class so no stored value can pass for one. */
+declare class RolledBack<V> {
+    readonly value: V;
+    constructor(value: V);
+}
+/** `withWriteScope` for a write that can refuse: a callback that returns `rollback(value)` gets the scope's writes undone
+ *  and `value` back, with no throw. Any other return commits. */
+export declare function withWriteScopeOr<T, R>(db: DatabaseSyncLike, name: string, fn: (rollback: <V>(value: V) => RolledBack<V>) => T | RolledBack<R>): T | R;
+export {};
 //# sourceMappingURL=busy.d.ts.map

package/dist/db/busy.js

@@ -32,11 +32,23 @@ export function execWithBusyRetry(db, sql, timeoutMs = 30000) {
         }
     }
 }
+function undoScope(db, top, name) {
+    if (top) {
+        db.exec('ROLLBACK');
+    }
+    else {
+        db.exec(`ROLLBACK TO SAVEPOINT ${name}`);
+        db.exec(`RELEASE SAVEPOINT ${name}`);
+    }
+}
 /** Runs `fn` as one write: BEGIN IMMEDIATE on an idle handle, else SAVEPOINT `name` inside the caller's transaction.
  *  A deferred scope that reads first cannot wait out another writer, while BEGIN IMMEDIATE waits the open's busy_timeout. */
-export function withWriteScope(db, name, fn) {
+export function withWriteScope(db, name, fn, opts) {
     const top = db.isTransaction === false;
-    db.exec(top ? 'BEGIN IMMEDIATE' : `SAVEPOINT ${name}`);
+    if (top && opts)
+        execWithBusyRetry(db, 'BEGIN IMMEDIATE', opts.busyWaitMs);
+    else
+        db.exec(top ? 'BEGIN IMMEDIATE' : `SAVEPOINT ${name}`);
     try {
         const result = fn();
         db.exec(top ? 'COMMIT' : `RELEASE SAVEPOINT ${name}`);
@@ -44,13 +56,66 @@ export function withWriteScope(db, name, fn) {
     }
     catch (error) {
         try {
-            if (top) {
-                db.exec('ROLLBACK');
-            }
-            else {
-                db.exec(`ROLLBACK TO SAVEPOINT ${name}`);
-                db.exec(`RELEASE SAVEPOINT ${name}`);
-            }
+            undoScope(db, top, name);
+        }
+        catch { /* already rolled back; keep the original error */ }
+        throw error;
+    }
+}
+/** Runs the reads in `fn` against one snapshot, so two statements cannot straddle another process's write. */
+export function withReadSnapshot(db, fn) {
+    if (db.isTransaction !== false)
+        return fn();
+    db.exec('BEGIN');
+    try {
+        return fn();
+    }
+    finally {
+        db.exec('COMMIT');
+    }
+}
+/** Runs `fn` and undoes every write it made, throw or not: a dry run that reports what a real run would do. Takes no write lock up front. */
+export function withTrialScope(db, name, fn) {
+    const top = db.isTransaction === false;
+    db.exec(top ? 'BEGIN' : `SAVEPOINT ${name}`);
+    let result;
+    try {
+        result = fn();
+    }
+    catch (error) {
+        try {
+            undoScope(db, top, name);
+        }
+        catch { /* already rolled back; keep the original error */ }
+        throw error;
+    }
+    undoScope(db, top, name);
+    return result;
+}
+/** What `withWriteScopeOr` hands its callback's `rollback(value)`; a class so no stored value can pass for one. */
+class RolledBack {
+    value;
+    constructor(value) {
+        this.value = value;
+    }
+}
+/** `withWriteScope` for a write that can refuse: a callback that returns `rollback(value)` gets the scope's writes undone
+ *  and `value` back, with no throw. Any other return commits. */
+export function withWriteScopeOr(db, name, fn) {
+    const top = db.isTransaction === false;
+    db.exec(top ? 'BEGIN IMMEDIATE' : `SAVEPOINT ${name}`);
+    try {
+        const result = fn((value) => new RolledBack(value));
+        if (result instanceof RolledBack) {
+            undoScope(db, top, name);
+            return result.value;
+        }
+        db.exec(top ? 'COMMIT' : `RELEASE SAVEPOINT ${name}`);
+        return result;
+    }
+    catch (error) {
+        try {
+            undoScope(db, top, name);
         }
         catch { /* already rolled back; keep the original error */ }
         throw error;

package/dist/db/connect.d.ts

@@ -1,5 +1,13 @@
 import { type DatabaseSyncLike } from './sqlite.js';
+import { type OpenFacts } from './migrate.js';
 export declare function getHippoDbPath(hippoRoot: string): string;
-/** A new connection with the store's pragmas, migrations and mirror cleanup applied; the caller owns and closes it. */
-export declare function connectHippoDb(hippoRoot: string, busyWaitMs?: number): DatabaseSyncLike;
+/** Runs `step` on this store unless it already ran to the end in this process; a throw leaves it due. */
+export declare function oncePerStore(hippoRoot: string, name: string, step: () => void): void;
+/** A connection with what its own open's probe read; facts go stale with the next statement, so a handle made by an earlier call carries null. */
+export interface OpenedDb {
+    readonly db: DatabaseSyncLike;
+    readonly facts: OpenFacts | null;
+}
+/** A new connection with the store's pragmas and migrations applied and the mirror cleanup run when it is due; the caller owns and closes it. */
+export declare function connectWithFacts(hippoRoot: string, busyWaitMs?: number): OpenedDb;
 //# sourceMappingURL=connect.d.ts.map

package/dist/db/connect.js

@@ -1,43 +1,76 @@
 import * as fs from 'fs';
 import * as path from 'path';
-import { cleanupArchivedMirrors } from '../raw-archive-mirror-cleanup.js';
-import { log } from '../log.js';
+import { cleanupArchivedMirrors } from './raw-archive-mirror-cleanup.js';
+import { errorMessage, log } from '../util/log.js';
 import { DatabaseSync } from './sqlite.js';
 import { execWithBusyRetry } from './busy.js';
 import { runMigrations } from './migrate.js';
+import { autoCheckpointPages } from './wal-checkpointer.js';
+const DEFAULT_BUSY_WAIT_MS = 5000;
 export function getHippoDbPath(hippoRoot) {
     return path.join(hippoRoot, 'hippo.db');
 }
+// Per store file, what this process has already done to it; a store file created anew starts over.
+const settledSteps = new Map();
+const sweptThrough = new Map();
+function storeKey(hippoRoot) {
+    return path.resolve(getHippoDbPath(hippoRoot));
+}
+/** Runs `step` on this store unless it already ran to the end in this process; a throw leaves it due. */
+export function oncePerStore(hippoRoot, name, step) {
+    const key = storeKey(hippoRoot);
+    const settled = settledSteps.get(key) ?? new Set();
+    if (settled.has(name))
+        return;
+    step();
+    settledSteps.set(key, settled.add(name));
+}
 // Owner-only on create; SQLite gives the WAL and SHM files the db's mode. Existing paths keep theirs.
 function createStoreFilesOwnerOnly(hippoRoot) {
     fs.mkdirSync(hippoRoot, { recursive: true, mode: 0o700 });
     try {
         fs.closeSync(fs.openSync(getHippoDbPath(hippoRoot), 'wx', 0o600));
+        settledSteps.delete(storeKey(hippoRoot));
+        sweptThrough.delete(storeKey(hippoRoot));
     }
     catch (err) {
         if (!(err instanceof Error && 'code' in err && err.code === 'EEXIST'))
             throw err;
     }
 }
-/** A new connection with the store's pragmas, migrations and mirror cleanup applied; the caller owns and closes it. */
-export function connectHippoDb(hippoRoot, busyWaitMs) {
+// The archive only grows and its ids never repeat, so an unchanged top id means no row this process has not swept; a failed sweep stays due.
+function sweepArchivedMirrorsIfDue(hippoRoot, db, probedTop) {
+    const top = probedTop ?? archiveTop(db);
+    const key = storeKey(hippoRoot);
+    if (sweptThrough.get(key) === top)
+        return;
+    if (cleanupArchivedMirrors(hippoRoot, db))
+        sweptThrough.set(key, top);
+}
+function archiveTop(db) {
+    // SAFETY: MAX over the integer key returns one row with one column, null for an empty table.
+    const row = db.prepare('SELECT MAX(id) AS top FROM raw_archive').get();
+    return Number(row?.top ?? 0);
+}
+/** A new connection with the store's pragmas and migrations applied and the mirror cleanup run when it is due; the caller owns and closes it. */
+export function connectWithFacts(hippoRoot, busyWaitMs) {
     createStoreFilesOwnerOnly(hippoRoot);
     const db = new DatabaseSync(getHippoDbPath(hippoRoot));
     try {
-        db.exec(`PRAGMA busy_timeout = ${busyWaitMs ?? 5000}`);
+        db.exec(`PRAGMA busy_timeout = ${busyWaitMs ?? DEFAULT_BUSY_WAIT_MS}`);
         execWithBusyRetry(db, 'PRAGMA journal_mode = WAL', busyWaitMs);
         db.exec('PRAGMA synchronous = NORMAL');
-        db.exec('PRAGMA wal_autocheckpoint = 100');
+        db.exec(`PRAGMA wal_autocheckpoint = ${autoCheckpointPages(getHippoDbPath(hippoRoot))}`);
         db.exec('PRAGMA foreign_keys = ON');
-        runMigrations(db, hippoRoot, busyWaitMs);
+        const facts = runMigrations(db, hippoRoot, busyWaitMs);
         // Orphan mirrors of archived raw rows go here; a filesystem failure must not block the open.
         try {
-            cleanupArchivedMirrors(hippoRoot, db);
+            sweepArchivedMirrorsIfDue(hippoRoot, db, facts?.archiveTop);
         }
         catch (cleanupErr) {
-            log.error(`openHippoDb: cleanupArchivedMirrors failed (non-fatal): ${cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)}`);
+            log.error(`openHippoDb: cleanupArchivedMirrors failed (non-fatal): ${errorMessage(cleanupErr)}`);
         }
-        return db;
+        return { db, facts };
     }
     catch (error) {
         try {

package/dist/db/index.d.ts

@@ -0,0 +1,9 @@
+export type { DatabaseSyncLike } from './sqlite.js';
+export { isSqliteBusy, isStoreBusy, StoreBusyError, STORE_BUSY_MESSAGE, execWithBusyRetry, withReadSnapshot, withTrialScope, withWriteScope, withWriteScopeOr } from './busy.js';
+export { getSchemaVersion, getMeta, setMeta, isFtsAvailable } from './meta.js';
+export { countTableRows, pruneConsolidationRuns } from './tables.js';
+export { getCurrentSchemaVersion, IncompatibleBinaryError, ftsRowCounts, repairFtsDrift } from './migrate.js';
+export { getHippoDbPath, HOOK_DB_WAIT_MS, noteStoreBusy, withSharedStoreHandles, SERVER_DB_WAIT_MS, SLEEP_DB_WAIT_MS, scopedBusyWait, withSqliteBlocked, withSqliteAllowed, withSqliteOffLoop, outsideSqliteOffLoop, rethrowIfSqliteBlocked, OTHER_STORE_MARKER, openHippoDb, openHippoDbReadOnly, closeHippoDb, } from './open.js';
+export { RequestStores, runWithRequestStores, currentRequestStores, outsideRequestStores } from './request-stores.js';
+export { OtherStoreFolderError, SqliteBlockedError } from '../util/sqlite-blocked.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/db/index.js

@@ -0,0 +1,8 @@
+export { isSqliteBusy, isStoreBusy, StoreBusyError, STORE_BUSY_MESSAGE, execWithBusyRetry, withReadSnapshot, withTrialScope, withWriteScope, withWriteScopeOr } from './busy.js';
+export { getSchemaVersion, getMeta, setMeta, isFtsAvailable } from './meta.js';
+export { countTableRows, pruneConsolidationRuns } from './tables.js';
+export { getCurrentSchemaVersion, IncompatibleBinaryError, ftsRowCounts, repairFtsDrift } from './migrate.js';
+export { getHippoDbPath, HOOK_DB_WAIT_MS, noteStoreBusy, withSharedStoreHandles, SERVER_DB_WAIT_MS, SLEEP_DB_WAIT_MS, scopedBusyWait, withSqliteBlocked, withSqliteAllowed, withSqliteOffLoop, outsideSqliteOffLoop, rethrowIfSqliteBlocked, OTHER_STORE_MARKER, openHippoDb, openHippoDbReadOnly, closeHippoDb, } from './open.js';
+export { RequestStores, runWithRequestStores, currentRequestStores, outsideRequestStores } from './request-stores.js';
+export { OtherStoreFolderError, SqliteBlockedError } from '../util/sqlite-blocked.js';
+//# sourceMappingURL=index.js.map

package/dist/db/meta.d.ts

@@ -2,6 +2,10 @@ import type { DatabaseSyncLike } from './sqlite.js';
 export declare const META_TABLE_DDL = "\n    CREATE TABLE IF NOT EXISTS meta (\n      key TEXT PRIMARY KEY,\n      value TEXT NOT NULL\n    );\n  ";
 export declare function ensureMetaTable(db: DatabaseSyncLike): void;
 export declare function getSchemaVersion(db: DatabaseSyncLike): number;
+/** SQLite's own `user_version` stamp, which can differ from the `meta` schema_version row. */
+export declare function pragmaUserVersion(db: DatabaseSyncLike): number;
+/** The connection's commit counter: it moves when another connection commits. */
+export declare function pragmaDataVersion(db: DatabaseSyncLike): number;
 export declare function setSchemaVersion(db: DatabaseSyncLike, version: number): void;
 export declare function getMeta(db: DatabaseSyncLike, key: string, fallback?: string): string;
 export declare function setMeta(db: DatabaseSyncLike, key: string, value: string): void;

package/dist/db/meta.js

@@ -1,4 +1,4 @@
-import { compareSemver } from '../version.js';
+import { compareSemver } from '../util/version.js';
 import { tableExists } from './tables.js';
 export const META_TABLE_DDL = `
     CREATE TABLE IF NOT EXISTS meta (
@@ -18,6 +18,16 @@ export function getSchemaVersion(db) {
     const version = Number(row?.value ?? 0);
     return Number.isFinite(version) ? version : 0;
 }
+/** SQLite's own `user_version` stamp, which can differ from the `meta` schema_version row. */
+export function pragmaUserVersion(db) {
+    // SAFETY: PRAGMA user_version returns one row with one integer column.
+    return db.prepare('PRAGMA user_version').get().user_version;
+}
+/** The connection's commit counter: it moves when another connection commits. */
+export function pragmaDataVersion(db) {
+    // SAFETY: PRAGMA data_version returns one row with that single integer column.
+    return db.prepare('PRAGMA data_version').get().data_version;
+}
 export function setSchemaVersion(db, version) {
     db.prepare(`INSERT INTO meta(key, value) VALUES('schema_version', ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`).run(String(version));
     db.exec(`PRAGMA user_version = ${Math.max(0, Math.trunc(version))}`);

package/dist/db/migrate.d.ts

@@ -7,7 +7,14 @@ export declare class IncompatibleBinaryError extends Error {
 export declare function assertBinaryCompatible(db: DatabaseSyncLike): void;
 /** The fast path's single probe: versions, the binary minimum, the FTS flag, and counts of the meta defaults and required objects. */
 export declare const SCHEMA_PROBE_SQL: string;
-export declare function runMigrations(db: DatabaseSyncLike, hippoRoot?: string, busyWaitMs?: number): void;
+/** What the probe of a current store read beyond the schema proof, so the same open need not read them again. */
+export interface OpenFacts {
+    /** The largest raw_archive id, 0 for an empty archive. */
+    readonly archiveTop: number;
+    readonly hasMemories: boolean;
+}
+/** Brings the store to the current schema; returns the probe's facts when it was current already, null after any repair. */
+export declare function runMigrations(db: DatabaseSyncLike, hippoRoot?: string, busyWaitMs?: number): OpenFacts | null;
 /** Row counts of `memories` and its full-text index, which differ once the index drifts; null when either table is missing. */
 export declare function ftsRowCounts(db: DatabaseSyncLike): {
     memories: number;