hippo-memory 1.69.0 → 1.70.0

package/dist/db/backup.js

@@ -1,6 +1,6 @@
 import * as fs from 'node:fs';
 import * as path from 'node:path';
-import { log } from '../log.js';
+import { log } from '../util/log.js';
 /** Copies the database before a repair writes, so the audit ids plus this file are the way back. */
 export function backupStore(db, hippoRoot, label, now = new Date()) {
     const dir = path.join(hippoRoot, 'backups');

package/dist/db/busy.d.ts

@@ -8,7 +8,24 @@ export declare class StoreBusyError extends Error {
 /** A held lock in any store: SQLite's busy codes or a port's StoreBusyError. */
 export declare function isStoreBusy<E>(error: E): boolean;
 export declare function execWithBusyRetry(db: DatabaseSyncLike, sql: string, timeoutMs?: number): void;
+/** Waits for the lock past busy_timeout when given; ignored inside a caller's transaction, which already holds it. */
+export interface WriteScopeOptions {
+    readonly busyWaitMs?: number;
+}
 /** Runs `fn` as one write: BEGIN IMMEDIATE on an idle handle, else SAVEPOINT `name` inside the caller's transaction.
  *  A deferred scope that reads first cannot wait out another writer, while BEGIN IMMEDIATE waits the open's busy_timeout. */
-export declare function withWriteScope<T>(db: DatabaseSyncLike, name: string, fn: () => T): T;
+export declare function withWriteScope<T>(db: DatabaseSyncLike, name: string, fn: () => T, opts?: WriteScopeOptions): T;
+/** Runs the reads in `fn` against one snapshot, so two statements cannot straddle another process's write. */
+export declare function withReadSnapshot<T>(db: DatabaseSyncLike, fn: () => T): T;
+/** Runs `fn` and undoes every write it made, throw or not: a dry run that reports what a real run would do. Takes no write lock up front. */
+export declare function withTrialScope<T>(db: DatabaseSyncLike, name: string, fn: () => T): T;
+/** What `withWriteScopeOr` hands its callback's `rollback(value)`; a class so no stored value can pass for one. */
+declare class RolledBack<V> {
+    readonly value: V;
+    constructor(value: V);
+}
+/** `withWriteScope` for a write that can refuse: a callback that returns `rollback(value)` gets the scope's writes undone
+ *  and `value` back, with no throw. Any other return commits. */
+export declare function withWriteScopeOr<T, R>(db: DatabaseSyncLike, name: string, fn: (rollback: <V>(value: V) => RolledBack<V>) => T | RolledBack<R>): T | R;
+export {};
 //# sourceMappingURL=busy.d.ts.map

package/dist/db/busy.js

@@ -32,11 +32,23 @@ export function execWithBusyRetry(db, sql, timeoutMs = 30000) {
         }
     }
 }
+function undoScope(db, top, name) {
+    if (top) {
+        db.exec('ROLLBACK');
+    }
+    else {
+        db.exec(`ROLLBACK TO SAVEPOINT ${name}`);
+        db.exec(`RELEASE SAVEPOINT ${name}`);
+    }
+}
 /** Runs `fn` as one write: BEGIN IMMEDIATE on an idle handle, else SAVEPOINT `name` inside the caller's transaction.
  *  A deferred scope that reads first cannot wait out another writer, while BEGIN IMMEDIATE waits the open's busy_timeout. */
-export function withWriteScope(db, name, fn) {
+export function withWriteScope(db, name, fn, opts) {
     const top = db.isTransaction === false;
-    db.exec(top ? 'BEGIN IMMEDIATE' : `SAVEPOINT ${name}`);
+    if (top && opts)
+        execWithBusyRetry(db, 'BEGIN IMMEDIATE', opts.busyWaitMs);
+    else
+        db.exec(top ? 'BEGIN IMMEDIATE' : `SAVEPOINT ${name}`);
     try {
         const result = fn();
         db.exec(top ? 'COMMIT' : `RELEASE SAVEPOINT ${name}`);
@@ -44,13 +56,66 @@ export function withWriteScope(db, name, fn) {
     }
     catch (error) {
         try {
-            if (top) {
-                db.exec('ROLLBACK');
-            }
-            else {
-                db.exec(`ROLLBACK TO SAVEPOINT ${name}`);
-                db.exec(`RELEASE SAVEPOINT ${name}`);
-            }
+            undoScope(db, top, name);
+        }
+        catch { /* already rolled back; keep the original error */ }
+        throw error;
+    }
+}
+/** Runs the reads in `fn` against one snapshot, so two statements cannot straddle another process's write. */
+export function withReadSnapshot(db, fn) {
+    if (db.isTransaction !== false)
+        return fn();
+    db.exec('BEGIN');
+    try {
+        return fn();
+    }
+    finally {
+        db.exec('COMMIT');
+    }
+}
+/** Runs `fn` and undoes every write it made, throw or not: a dry run that reports what a real run would do. Takes no write lock up front. */
+export function withTrialScope(db, name, fn) {
+    const top = db.isTransaction === false;
+    db.exec(top ? 'BEGIN' : `SAVEPOINT ${name}`);
+    let result;
+    try {
+        result = fn();
+    }
+    catch (error) {
+        try {
+            undoScope(db, top, name);
+        }
+        catch { /* already rolled back; keep the original error */ }
+        throw error;
+    }
+    undoScope(db, top, name);
+    return result;
+}
+/** What `withWriteScopeOr` hands its callback's `rollback(value)`; a class so no stored value can pass for one. */
+class RolledBack {
+    value;
+    constructor(value) {
+        this.value = value;
+    }
+}
+/** `withWriteScope` for a write that can refuse: a callback that returns `rollback(value)` gets the scope's writes undone
+ *  and `value` back, with no throw. Any other return commits. */
+export function withWriteScopeOr(db, name, fn) {
+    const top = db.isTransaction === false;
+    db.exec(top ? 'BEGIN IMMEDIATE' : `SAVEPOINT ${name}`);
+    try {
+        const result = fn((value) => new RolledBack(value));
+        if (result instanceof RolledBack) {
+            undoScope(db, top, name);
+            return result.value;
+        }
+        db.exec(top ? 'COMMIT' : `RELEASE SAVEPOINT ${name}`);
+        return result;
+    }
+    catch (error) {
+        try {
+            undoScope(db, top, name);
         }
         catch { /* already rolled back; keep the original error */ }
         throw error;

package/dist/db/connect.d.ts

@@ -1,5 +1,13 @@
 import { type DatabaseSyncLike } from './sqlite.js';
+import { type OpenFacts } from './migrate.js';
 export declare function getHippoDbPath(hippoRoot: string): string;
-/** A new connection with the store's pragmas, migrations and mirror cleanup applied; the caller owns and closes it. */
-export declare function connectHippoDb(hippoRoot: string, busyWaitMs?: number): DatabaseSyncLike;
+/** Runs `step` on this store unless it already ran to the end in this process; a throw leaves it due. */
+export declare function oncePerStore(hippoRoot: string, name: string, step: () => void): void;
+/** A connection with what its own open's probe read; facts go stale with the next statement, so a handle made by an earlier call carries null. */
+export interface OpenedDb {
+    readonly db: DatabaseSyncLike;
+    readonly facts: OpenFacts | null;
+}
+/** A new connection with the store's pragmas and migrations applied and the mirror cleanup run when it is due; the caller owns and closes it. */
+export declare function connectWithFacts(hippoRoot: string, busyWaitMs?: number): OpenedDb;
 //# sourceMappingURL=connect.d.ts.map

package/dist/db/connect.js

@@ -1,43 +1,76 @@
 import * as fs from 'fs';
 import * as path from 'path';
-import { cleanupArchivedMirrors } from '../raw-archive-mirror-cleanup.js';
-import { log } from '../log.js';
+import { cleanupArchivedMirrors } from './raw-archive-mirror-cleanup.js';
+import { errorMessage, log } from '../util/log.js';
 import { DatabaseSync } from './sqlite.js';
 import { execWithBusyRetry } from './busy.js';
 import { runMigrations } from './migrate.js';
+import { autoCheckpointPages } from './wal-checkpointer.js';
+const DEFAULT_BUSY_WAIT_MS = 5000;
 export function getHippoDbPath(hippoRoot) {
     return path.join(hippoRoot, 'hippo.db');
 }
+// Per store file, what this process has already done to it; a store file created anew starts over.
+const settledSteps = new Map();
+const sweptThrough = new Map();
+function storeKey(hippoRoot) {
+    return path.resolve(getHippoDbPath(hippoRoot));
+}
+/** Runs `step` on this store unless it already ran to the end in this process; a throw leaves it due. */
+export function oncePerStore(hippoRoot, name, step) {
+    const key = storeKey(hippoRoot);
+    const settled = settledSteps.get(key) ?? new Set();
+    if (settled.has(name))
+        return;
+    step();
+    settledSteps.set(key, settled.add(name));
+}
 // Owner-only on create; SQLite gives the WAL and SHM files the db's mode. Existing paths keep theirs.
 function createStoreFilesOwnerOnly(hippoRoot) {
     fs.mkdirSync(hippoRoot, { recursive: true, mode: 0o700 });
     try {
         fs.closeSync(fs.openSync(getHippoDbPath(hippoRoot), 'wx', 0o600));
+        settledSteps.delete(storeKey(hippoRoot));
+        sweptThrough.delete(storeKey(hippoRoot));
     }
     catch (err) {
         if (!(err instanceof Error && 'code' in err && err.code === 'EEXIST'))
             throw err;
     }
 }
-/** A new connection with the store's pragmas, migrations and mirror cleanup applied; the caller owns and closes it. */
-export function connectHippoDb(hippoRoot, busyWaitMs) {
+// The archive only grows and its ids never repeat, so an unchanged top id means no row this process has not swept; a failed sweep stays due.
+function sweepArchivedMirrorsIfDue(hippoRoot, db, probedTop) {
+    const top = probedTop ?? archiveTop(db);
+    const key = storeKey(hippoRoot);
+    if (sweptThrough.get(key) === top)
+        return;
+    if (cleanupArchivedMirrors(hippoRoot, db))
+        sweptThrough.set(key, top);
+}
+function archiveTop(db) {
+    // SAFETY: MAX over the integer key returns one row with one column, null for an empty table.
+    const row = db.prepare('SELECT MAX(id) AS top FROM raw_archive').get();
+    return Number(row?.top ?? 0);
+}
+/** A new connection with the store's pragmas and migrations applied and the mirror cleanup run when it is due; the caller owns and closes it. */
+export function connectWithFacts(hippoRoot, busyWaitMs) {
     createStoreFilesOwnerOnly(hippoRoot);
     const db = new DatabaseSync(getHippoDbPath(hippoRoot));
     try {
-        db.exec(`PRAGMA busy_timeout = ${busyWaitMs ?? 5000}`);
+        db.exec(`PRAGMA busy_timeout = ${busyWaitMs ?? DEFAULT_BUSY_WAIT_MS}`);
         execWithBusyRetry(db, 'PRAGMA journal_mode = WAL', busyWaitMs);
         db.exec('PRAGMA synchronous = NORMAL');
-        db.exec('PRAGMA wal_autocheckpoint = 100');
+        db.exec(`PRAGMA wal_autocheckpoint = ${autoCheckpointPages(getHippoDbPath(hippoRoot))}`);
         db.exec('PRAGMA foreign_keys = ON');
-        runMigrations(db, hippoRoot, busyWaitMs);
+        const facts = runMigrations(db, hippoRoot, busyWaitMs);
         // Orphan mirrors of archived raw rows go here; a filesystem failure must not block the open.
         try {
-            cleanupArchivedMirrors(hippoRoot, db);
+            sweepArchivedMirrorsIfDue(hippoRoot, db, facts?.archiveTop);
         }
         catch (cleanupErr) {
-            log.error(`openHippoDb: cleanupArchivedMirrors failed (non-fatal): ${cleanupErr instanceof Error ? cleanupErr.message : String(cleanupErr)}`);
+            log.error(`openHippoDb: cleanupArchivedMirrors failed (non-fatal): ${errorMessage(cleanupErr)}`);
         }
-        return db;
+        return { db, facts };
     }
     catch (error) {
         try {

package/dist/db/index.d.ts

@@ -0,0 +1,9 @@
+export type { DatabaseSyncLike } from './sqlite.js';
+export { isSqliteBusy, isStoreBusy, StoreBusyError, STORE_BUSY_MESSAGE, execWithBusyRetry, withReadSnapshot, withTrialScope, withWriteScope, withWriteScopeOr } from './busy.js';
+export { getSchemaVersion, getMeta, setMeta, isFtsAvailable } from './meta.js';
+export { countTableRows, pruneConsolidationRuns } from './tables.js';
+export { getCurrentSchemaVersion, IncompatibleBinaryError, ftsRowCounts, repairFtsDrift } from './migrate.js';
+export { getHippoDbPath, HOOK_DB_WAIT_MS, noteStoreBusy, withSharedStoreHandles, SERVER_DB_WAIT_MS, SLEEP_DB_WAIT_MS, scopedBusyWait, withSqliteBlocked, withSqliteAllowed, withSqliteOffLoop, outsideSqliteOffLoop, rethrowIfSqliteBlocked, OTHER_STORE_MARKER, openHippoDb, openHippoDbReadOnly, closeHippoDb, } from './open.js';
+export { RequestStores, runWithRequestStores, currentRequestStores, outsideRequestStores } from './request-stores.js';
+export { OtherStoreFolderError, SqliteBlockedError } from '../util/sqlite-blocked.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/db/index.js

@@ -0,0 +1,8 @@
+export { isSqliteBusy, isStoreBusy, StoreBusyError, STORE_BUSY_MESSAGE, execWithBusyRetry, withReadSnapshot, withTrialScope, withWriteScope, withWriteScopeOr } from './busy.js';
+export { getSchemaVersion, getMeta, setMeta, isFtsAvailable } from './meta.js';
+export { countTableRows, pruneConsolidationRuns } from './tables.js';
+export { getCurrentSchemaVersion, IncompatibleBinaryError, ftsRowCounts, repairFtsDrift } from './migrate.js';
+export { getHippoDbPath, HOOK_DB_WAIT_MS, noteStoreBusy, withSharedStoreHandles, SERVER_DB_WAIT_MS, SLEEP_DB_WAIT_MS, scopedBusyWait, withSqliteBlocked, withSqliteAllowed, withSqliteOffLoop, outsideSqliteOffLoop, rethrowIfSqliteBlocked, OTHER_STORE_MARKER, openHippoDb, openHippoDbReadOnly, closeHippoDb, } from './open.js';
+export { RequestStores, runWithRequestStores, currentRequestStores, outsideRequestStores } from './request-stores.js';
+export { OtherStoreFolderError, SqliteBlockedError } from '../util/sqlite-blocked.js';
+//# sourceMappingURL=index.js.map

package/dist/db/meta.d.ts

@@ -2,6 +2,10 @@ import type { DatabaseSyncLike } from './sqlite.js';
 export declare const META_TABLE_DDL = "\n    CREATE TABLE IF NOT EXISTS meta (\n      key TEXT PRIMARY KEY,\n      value TEXT NOT NULL\n    );\n  ";
 export declare function ensureMetaTable(db: DatabaseSyncLike): void;
 export declare function getSchemaVersion(db: DatabaseSyncLike): number;
+/** SQLite's own `user_version` stamp, which can differ from the `meta` schema_version row. */
+export declare function pragmaUserVersion(db: DatabaseSyncLike): number;
+/** The connection's commit counter: it moves when another connection commits. */
+export declare function pragmaDataVersion(db: DatabaseSyncLike): number;
 export declare function setSchemaVersion(db: DatabaseSyncLike, version: number): void;
 export declare function getMeta(db: DatabaseSyncLike, key: string, fallback?: string): string;
 export declare function setMeta(db: DatabaseSyncLike, key: string, value: string): void;

package/dist/db/meta.js

@@ -1,4 +1,4 @@
-import { compareSemver } from '../version.js';
+import { compareSemver } from '../util/version.js';
 import { tableExists } from './tables.js';
 export const META_TABLE_DDL = `
     CREATE TABLE IF NOT EXISTS meta (
@@ -18,6 +18,16 @@ export function getSchemaVersion(db) {
     const version = Number(row?.value ?? 0);
     return Number.isFinite(version) ? version : 0;
 }
+/** SQLite's own `user_version` stamp, which can differ from the `meta` schema_version row. */
+export function pragmaUserVersion(db) {
+    // SAFETY: PRAGMA user_version returns one row with one integer column.
+    return db.prepare('PRAGMA user_version').get().user_version;
+}
+/** The connection's commit counter: it moves when another connection commits. */
+export function pragmaDataVersion(db) {
+    // SAFETY: PRAGMA data_version returns one row with that single integer column.
+    return db.prepare('PRAGMA data_version').get().data_version;
+}
 export function setSchemaVersion(db, version) {
     db.prepare(`INSERT INTO meta(key, value) VALUES('schema_version', ?) ON CONFLICT(key) DO UPDATE SET value=excluded.value`).run(String(version));
     db.exec(`PRAGMA user_version = ${Math.max(0, Math.trunc(version))}`);

package/dist/db/migrate.d.ts

@@ -7,7 +7,14 @@ export declare class IncompatibleBinaryError extends Error {
 export declare function assertBinaryCompatible(db: DatabaseSyncLike): void;
 /** The fast path's single probe: versions, the binary minimum, the FTS flag, and counts of the meta defaults and required objects. */
 export declare const SCHEMA_PROBE_SQL: string;
-export declare function runMigrations(db: DatabaseSyncLike, hippoRoot?: string, busyWaitMs?: number): void;
+/** What the probe of a current store read beyond the schema proof, so the same open need not read them again. */
+export interface OpenFacts {
+    /** The largest raw_archive id, 0 for an empty archive. */
+    readonly archiveTop: number;
+    readonly hasMemories: boolean;
+}
+/** Brings the store to the current schema; returns the probe's facts when it was current already, null after any repair. */
+export declare function runMigrations(db: DatabaseSyncLike, hippoRoot?: string, busyWaitMs?: number): OpenFacts | null;
 /** Row counts of `memories` and its full-text index, which differ once the index drifts; null when either table is missing. */
 export declare function ftsRowCounts(db: DatabaseSyncLike): {
     memories: number;

package/dist/db/migrate.js

@@ -1,7 +1,7 @@
-import { PACKAGE_VERSION, compareSemver } from '../version.js';
-import { log } from '../log.js';
-import { importLegacyEmbeddingIndex } from '../vector-store.js';
-import { execWithBusyRetry } from './busy.js';
+import { PACKAGE_VERSION, compareSemver } from '../util/version.js';
+import { errorMessage, log } from '../util/log.js';
+import { importLegacyEmbeddingIndex } from './vector-store.js';
+import { execWithBusyRetry, withWriteScope } from './busy.js';
 import { tableExists } from './tables.js';
 import { ensureMetaTable, getMeta, getSchemaVersion, setMeta, setSchemaVersion } from './meta.js';
 import { MEMORIES_FTS_DDL, REQUIRED_SCHEMA_OBJECTS, ensureContinuityIndexes, ensureContinuityTables } from './continuity.js';
@@ -40,34 +40,40 @@ export const SCHEMA_PROBE_SQL = `SELECT
   (SELECT value FROM meta WHERE key = 'min_compatible_binary') AS min_binary,
   (SELECT value FROM meta WHERE key = 'fts5_available') AS fts5,
   (SELECT COUNT(*) FROM meta WHERE key IN (${sqlList(META_DEFAULTS.map(([key]) => key))})) AS meta_keys,
-  (SELECT COUNT(*) FROM sqlite_master WHERE type IN ('table', 'index') AND name IN (${sqlList(REQUIRED_SCHEMA_OBJECTS)})) AS objects`;
-/** True when the store is current and whole, so the open can skip every DDL statement. Runs the binary check either way. */
-function schemaIsCurrent(db) {
+  (SELECT COUNT(*) FROM sqlite_master WHERE type IN ('table', 'index') AND name IN (${sqlList(REQUIRED_SCHEMA_OBJECTS)})) AS objects,
+  (SELECT MAX(id) FROM raw_archive) AS archive_top,
+  (SELECT 1 FROM memories LIMIT 1) AS has_memories`;
+/** The probe's facts when the store is current and whole, so the open can skip every DDL statement; else null. Runs the binary check either way. */
+function currentStoreFacts(db) {
     let probe;
     try {
-        // SAFETY: the SELECT names exactly these six columns and always returns one row.
+        // SAFETY: the SELECT names exactly these eight columns and always returns one row.
         probe = db.prepare(SCHEMA_PROBE_SQL).get();
     }
     catch (err) {
-        // A store with no meta table yet is the slow path's job; anything else is a real error.
-        if (err instanceof Error && /no such table: meta/.test(err.message))
-            return false;
+        // A store missing a table the probe reads is the slow path's job; anything else is a real error.
+        if (err instanceof Error && /no such table: (meta|raw_archive|memories)\b/.test(err.message))
+            return null;
         throw err;
     }
     if (!probe)
-        return false;
+        return null;
     assertMinBinary(probe.min_binary);
-    return Number(probe.user_version) === CURRENT_SCHEMA_VERSION
+    const current = Number(probe.user_version) === CURRENT_SCHEMA_VERSION
         && Number(probe.schema_version) === CURRENT_SCHEMA_VERSION
         && Number(probe.meta_keys) === META_DEFAULTS.length
         && Number(probe.objects) === REQUIRED_SCHEMA_OBJECTS.length
         && probe.fts5 === '1';
+    return current ? { archiveTop: Number(probe.archive_top ?? 0), hasMemories: probe.has_memories !== null } : null;
 }
+/** Brings the store to the current schema; returns the probe's facts when it was current already, null after any repair. */
 export function runMigrations(db, hippoRoot, busyWaitMs) {
-    if (!schemaIsCurrent(db))
+    const facts = currentStoreFacts(db);
+    if (!facts)
         migrateAndHeal(db, hippoRoot, busyWaitMs);
     if (hippoRoot)
         importLegacyVectors(db, hippoRoot);
+    return facts;
 }
 function migrateAndHeal(db, hippoRoot, busyWaitMs) {
     ensureMetaTable(db);
@@ -88,29 +94,18 @@ function migrateAndHeal(db, hippoRoot, busyWaitMs) {
 }
 /** Applies one migration under the write lock and returns the version the store is now at. */
 function applyMigration(db, migration, hippoRoot, busyWaitMs) {
-    execWithBusyRetry(db, 'BEGIN IMMEDIATE', busyWaitMs);
-    try {
+    return withWriteScope(db, 'apply_migration', () => {
         // A newer binary may have migrated and raised the minimum while we waited for the lock.
         assertBinaryCompatible(db);
         // Re-read under the write lock: another process may have applied this
         // migration while we waited, and re-running one is not idempotent.
         const applied = getSchemaVersion(db);
-        if (applied >= migration.version) {
-            db.exec('COMMIT');
+        if (applied >= migration.version)
             return applied;
-        }
         migration.up(db, { hippoRoot });
         setSchemaVersion(db, migration.version);
-        db.exec('COMMIT');
         return migration.version;
-    }
-    catch (error) {
-        try {
-            db.exec('ROLLBACK');
-        }
-        catch { /* already rolled back; keep the original error */ }
-        throw error;
-    }
+    }, { busyWaitMs });
 }
 // The fast path requires user_version as well, and a store stamped only in meta would otherwise never reach it.
 function healUserVersion(db, version, busyWaitMs) {
@@ -126,7 +121,7 @@ function importLegacyVectors(db, hippoRoot) {
         importLegacyEmbeddingIndex(db, hippoRoot);
     }
     catch (err) {
-        log.warn(`embeddings.json import failed; the next open retries it (${err instanceof Error ? err.message : String(err)})`, { hippoRoot });
+        log.warn(`embeddings.json import failed; the next open retries it (${errorMessage(err)})`, { hippoRoot });
     }
 }
 function ensureMetaDefaults(db) {
@@ -154,7 +149,7 @@ function ensureOptionalFts(db) {
         available = true;
     }
     catch (err) {
-        log.warn(`full-text index unavailable (${err instanceof Error ? err.message : String(err)}); search falls back to slower LIKE matching`);
+        log.warn(`full-text index unavailable (${errorMessage(err)}); search falls back to slower LIKE matching`);
     }
     // Read-first: only write when the flag actually changed, so a healthy
     // store's open doesn't take a write lock for a same-value upsert.

package/dist/db/migrations/v08.js

@@ -1,4 +1,4 @@
-import { createPhysicsTable } from '../../physics-state.js';
+import { createPhysicsTable } from '../physics-state.js';
 export const v08 = {
     version: 8,
     up: (db) => {

package/dist/db/migrations/v14.js

@@ -1,4 +1,38 @@
 import { tableHasColumn } from '../tables.js';
+const CREATE_TABLE_RAW_ARCHIVE_SQL = `
+        CREATE TABLE IF NOT EXISTS raw_archive (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          memory_id TEXT NOT NULL,
+          archived_at TEXT NOT NULL,
+          reason TEXT NOT NULL,
+          archived_by TEXT,
+          payload_json TEXT NOT NULL
+        )
+      `;
+const CREATE_TRIGGER_TRG_MEMORIES_RAW_APPEND_ONLY_SQL = `
+        CREATE TRIGGER IF NOT EXISTS trg_memories_raw_append_only
+        BEFORE DELETE ON memories
+        WHEN OLD.kind = 'raw'
+        BEGIN
+          SELECT RAISE(ABORT, 'raw is append-only');
+        END
+      `;
+const CREATE_TRIGGER_TRG_MEMORIES_KIND_CHECK_INSERT_SQL = `
+        CREATE TRIGGER IF NOT EXISTS trg_memories_kind_check_insert
+        BEFORE INSERT ON memories
+        WHEN NEW.kind IS NOT NULL AND NEW.kind NOT IN ('raw','distilled','superseded','archived')
+        BEGIN
+          SELECT RAISE(ABORT, 'invalid kind: must be raw|distilled|superseded|archived');
+        END
+      `;
+const CREATE_TRIGGER_TRG_MEMORIES_KIND_CHECK_UPDATE_SQL = `
+        CREATE TRIGGER IF NOT EXISTS trg_memories_kind_check_update
+        BEFORE UPDATE ON memories
+        WHEN NEW.kind IS NOT NULL AND NEW.kind NOT IN ('raw','distilled','superseded','archived')
+        BEGIN
+          SELECT RAISE(ABORT, 'invalid kind: must be raw|distilled|superseded|archived');
+        END
+      `;
 export const v14 = {
     version: 14,
     up: (db) => {
@@ -22,45 +56,15 @@ export const v14 = {
         db.exec(`UPDATE memories SET kind = 'superseded' WHERE kind IS NULL AND superseded_by IS NOT NULL`);
         db.exec(`UPDATE memories SET kind = 'distilled' WHERE kind IS NULL`);
         // raw_archive: legitimate path for kind='raw' removal (used by archiveRawMemory).
-        db.exec(`
-        CREATE TABLE IF NOT EXISTS raw_archive (
-          id INTEGER PRIMARY KEY AUTOINCREMENT,
-          memory_id TEXT NOT NULL,
-          archived_at TEXT NOT NULL,
-          reason TEXT NOT NULL,
-          archived_by TEXT,
-          payload_json TEXT NOT NULL
-        )
-      `);
+        db.exec(CREATE_TABLE_RAW_ARCHIVE_SQL);
         db.exec(`CREATE INDEX IF NOT EXISTS idx_raw_archive_memory_id ON raw_archive(memory_id)`);
         // Append-only invariant: kind='raw' rows cannot be deleted directly.
-        // Use raw_archive flow: archive-then-update-then-delete (see src/raw-archive.ts).
-        db.exec(`
-        CREATE TRIGGER IF NOT EXISTS trg_memories_raw_append_only
-        BEFORE DELETE ON memories
-        WHEN OLD.kind = 'raw'
-        BEGIN
-          SELECT RAISE(ABORT, 'raw is append-only');
-        END
-      `);
+        // Use raw_archive flow: archive-then-update-then-delete (see src/store/raw-archive.ts).
+        db.exec(CREATE_TRIGGER_TRG_MEMORIES_RAW_APPEND_ONLY_SQL);
         // CHECK substitute: ALTER TABLE cannot add CHECK, so enforce kind allowed-set
         // via INSERT/UPDATE triggers.
-        db.exec(`
-        CREATE TRIGGER IF NOT EXISTS trg_memories_kind_check_insert
-        BEFORE INSERT ON memories
-        WHEN NEW.kind IS NOT NULL AND NEW.kind NOT IN ('raw','distilled','superseded','archived')
-        BEGIN
-          SELECT RAISE(ABORT, 'invalid kind: must be raw|distilled|superseded|archived');
-        END
-      `);
-        db.exec(`
-        CREATE TRIGGER IF NOT EXISTS trg_memories_kind_check_update
-        BEFORE UPDATE ON memories
-        WHEN NEW.kind IS NOT NULL AND NEW.kind NOT IN ('raw','distilled','superseded','archived')
-        BEGIN
-          SELECT RAISE(ABORT, 'invalid kind: must be raw|distilled|superseded|archived');
-        END
-      `);
+        db.exec(CREATE_TRIGGER_TRG_MEMORIES_KIND_CHECK_INSERT_SQL);
+        db.exec(CREATE_TRIGGER_TRG_MEMORIES_KIND_CHECK_UPDATE_SQL);
     },
 };
 //# sourceMappingURL=v14.js.map

package/dist/db/migrations/v16.js

@@ -1,4 +1,30 @@
 import { tableHasColumn } from '../tables.js';
+const CREATE_TABLE_API_KEYS_SQL = `
+        CREATE TABLE IF NOT EXISTS api_keys (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          key_id TEXT UNIQUE NOT NULL,
+          key_hash TEXT NOT NULL,
+          tenant_id TEXT NOT NULL DEFAULT 'default',
+          label TEXT,
+          created_at TEXT NOT NULL,
+          revoked_at TEXT
+        )
+      `;
+const CREATE_INDEX_IDX_API_KEYS_TENANT_ACTIVE_SQL = `
+        CREATE INDEX IF NOT EXISTS idx_api_keys_tenant_active
+        ON api_keys(tenant_id) WHERE revoked_at IS NULL
+      `;
+const CREATE_TABLE_AUDIT_LOG_SQL = `
+        CREATE TABLE IF NOT EXISTS audit_log (
+          id INTEGER PRIMARY KEY AUTOINCREMENT,
+          ts TEXT NOT NULL,
+          tenant_id TEXT NOT NULL DEFAULT 'default',
+          actor TEXT NOT NULL,
+          op TEXT NOT NULL,
+          target_id TEXT,
+          metadata_json TEXT NOT NULL DEFAULT '{}'
+        )
+      `;
 export const v16 = {
     version: 16,
     up: (db) => {
@@ -28,32 +54,9 @@ export const v16 = {
         // Stub auth: api_keys (scrypt-hashed; plaintext returned to caller exactly once)
         // and audit_log (append-only mutation trail). Both carry tenant_id from day 1 so
         // future multi-tenant enforcement is a config flip, not a re-migration.
-        db.exec(`
-        CREATE TABLE IF NOT EXISTS api_keys (
-          id INTEGER PRIMARY KEY AUTOINCREMENT,
-          key_id TEXT UNIQUE NOT NULL,
-          key_hash TEXT NOT NULL,
-          tenant_id TEXT NOT NULL DEFAULT 'default',
-          label TEXT,
-          created_at TEXT NOT NULL,
-          revoked_at TEXT
-        )
-      `);
-        db.exec(`
-        CREATE INDEX IF NOT EXISTS idx_api_keys_tenant_active
-        ON api_keys(tenant_id) WHERE revoked_at IS NULL
-      `);
-        db.exec(`
-        CREATE TABLE IF NOT EXISTS audit_log (
-          id INTEGER PRIMARY KEY AUTOINCREMENT,
-          ts TEXT NOT NULL,
-          tenant_id TEXT NOT NULL DEFAULT 'default',
-          actor TEXT NOT NULL,
-          op TEXT NOT NULL,
-          target_id TEXT,
-          metadata_json TEXT NOT NULL DEFAULT '{}'
-        )
-      `);
+        db.exec(CREATE_TABLE_API_KEYS_SQL);
+        db.exec(CREATE_INDEX_IDX_API_KEYS_TENANT_ACTIVE_SQL);
+        db.exec(CREATE_TABLE_AUDIT_LOG_SQL);
         db.exec(`CREATE INDEX IF NOT EXISTS idx_audit_log_tenant_ts ON audit_log(tenant_id, ts DESC)`);
     },
 };

package/dist/db/migrations/v18.js

@@ -1,7 +1,4 @@
-export const v18 = {
-    version: 18,
-    up: (db) => {
-        db.exec(`
+const GOAL_STACK_SCHEMA_SQL = `
         CREATE TABLE IF NOT EXISTS goal_stack (
           id TEXT PRIMARY KEY,
           session_id TEXT NOT NULL,
@@ -54,7 +51,11 @@ export const v18 = {
           ON goal_recall_log(goal_id);
         CREATE UNIQUE INDEX IF NOT EXISTS uniq_goal_recall_log_memory_goal
           ON goal_recall_log(memory_id, goal_id);
-      `);
+      `;
+export const v18 = {
+    version: 18,
+    up: (db) => {
+        db.exec(GOAL_STACK_SCHEMA_SQL);
     },
 };
 //# sourceMappingURL=v18.js.map

package/dist/db/migrations/v20.js

@@ -1,9 +1,10 @@
+import { warnDamagedColumn } from '../../util/stored-json.js';
 export const v20 = {
     version: 20,
     up: (db) => {
         // GDPR Path A backfill: redact every existing
         // raw_archive.payload_json so historical archives match the new
-        // metadata-only contract from src/raw-archive.ts. Read each row, parse
+        // metadata-only contract from src/store/raw-archive.ts. Read each row, parse
         // the existing JSON to extract tenant_id and kind (best effort), then
         // UPDATE with the redacted shape. Rows with unparseable legacy JSON get
         // redacted with tenant_id='unknown', kind='unknown'. The audit_log
@@ -25,7 +26,8 @@ export const v20 = {
                 kind = parsed.kind ?? 'unknown';
             }
             catch {
-                // Unparseable legacy payload — redact with unknowns.
+                // The row is still redacted, with unknowns; the line says which one lost its tenant and kind.
+                warnDamagedColumn({ table: 'raw_archive', id: row.id, column: 'payload_json' }, 'not valid JSON');
             }
             const redacted = JSON.stringify({
                 redacted: true,