@pcircle/memesh 4.5.1 → 4.6.1

package/dist/db.js

@@ -6,176 +6,12 @@ import { runAutoDecay } from './core/lifecycle.js';
 import { resolveEmbeddingDimension } from './core/config.js';
 import { computeSignalScore } from './core/signal-scorer.js';
 import { getDbPath } from './core/paths.js';
-import { insertFtsRow } from './storage/fts-index.js';
+import { insertFtsRow, joinIndexedObservations, removeFromFts } from './storage/fts-index.js';
+import { SCHEMA_SQL, FTS_SQL, safeAlter, migrateEntitiesSchema, ensureTagsUniqueIndex, ensureHookRunsSince, ensureFtsSegmentation, rebuildFtsIndex, runOnceMigration, FTS_SEGMENTATION_VERSION, } from './storage/schema.js';
+export { runOnceMigration, FTS_SEGMENTATION_VERSION };
+import { truncateTitle, isBoilerplateObservation } from './core/title.js';
 let db = null;
-const SCHEMA_SQL = `
-CREATE TABLE IF NOT EXISTS entities (
-  id INTEGER PRIMARY KEY AUTOINCREMENT,
-  name TEXT NOT NULL UNIQUE,
-  type TEXT NOT NULL,
-  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-  metadata JSON
-);
-
-CREATE TABLE IF NOT EXISTS observations (
-  id INTEGER PRIMARY KEY AUTOINCREMENT,
-  entity_id INTEGER NOT NULL,
-  content TEXT NOT NULL,
-  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-  FOREIGN KEY (entity_id) REFERENCES entities(id) ON DELETE CASCADE
-);
-
-CREATE TABLE IF NOT EXISTS relations (
-  id INTEGER PRIMARY KEY AUTOINCREMENT,
-  from_entity_id INTEGER NOT NULL,
-  to_entity_id INTEGER NOT NULL,
-  relation_type TEXT NOT NULL,
-  metadata JSON,
-  created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-  FOREIGN KEY (from_entity_id) REFERENCES entities(id) ON DELETE CASCADE,
-  FOREIGN KEY (to_entity_id) REFERENCES entities(id) ON DELETE CASCADE,
-  UNIQUE(from_entity_id, to_entity_id, relation_type)
-);
-
-CREATE TABLE IF NOT EXISTS tags (
-  id INTEGER PRIMARY KEY AUTOINCREMENT,
-  entity_id INTEGER NOT NULL,
-  tag TEXT NOT NULL,
-  FOREIGN KEY (entity_id) REFERENCES entities(id) ON DELETE CASCADE
-);
-
-CREATE INDEX IF NOT EXISTS idx_tags_entity ON tags(entity_id);
-CREATE INDEX IF NOT EXISTS idx_tags_tag ON tags(tag);
--- The tags dedup DELETE + idx_tags_entity_tag_unique creation live in
--- ensureTagsUniqueIndex(), AFTER this exec — the DELETE is a one-time
--- migration, and a DML statement in this string made every open start a
--- write transaction even when it deleted nothing (same reader-breaking
--- pattern as the hook_runs_since note below).
-CREATE INDEX IF NOT EXISTS idx_observations_entity ON observations(entity_id);
-CREATE INDEX IF NOT EXISTS idx_relations_from ON relations(from_entity_id);
-CREATE INDEX IF NOT EXISTS idx_relations_to ON relations(to_entity_id);
-CREATE INDEX IF NOT EXISTS idx_entities_type_created ON entities(type, created_at);
-
--- Migration markers and small bits of persistent state (index segmentation
--- version, embedding dimension, pending-reindex flags, backfill markers).
---
--- This used to be created ad hoc by each helper that needed it — four inline
--- CREATE TABLE IF NOT EXISTS copies in src/db.ts, none of them visible to
--- scripts/check-schema-drift.mjs, which only extracts SCHEMA_SQL and FTS_SQL.
--- A column added to one copy would not have been caught. It also meant the
--- hook-side schema had no metadata table at all, so hooks could not
--- participate in migrations even in principle.
-CREATE TABLE IF NOT EXISTS memesh_metadata (
-  key   TEXT PRIMARY KEY,
-  value TEXT NOT NULL
-);
-
--- Proof that a capture hook actually RAN. Nothing else in this schema can
--- give it: every other signal is "a row was written", and "the hook ran and
--- found nothing worth saving" is the healthy case that produces no row at
--- all. So a quiet day and a dead capture loop were byte-identical in the
--- database, and the one doctor message that had to cover both cried wolf on
--- the first and stayed silent on the second.
---
--- Written by the three hooks that hold a read-write handle (Stop, PreCompact,
--- PostToolUse), each calling recordHookRun() at its own SUCCESSFUL exit —
--- after capture, not at open. Stamping at open certified the wrong thing: a
--- hook that opened the database and then died mid-capture looked alive for a
--- day. "Successful" is precise: a completed capture, or a well-formed
--- payload the hook correctly decided not to capture (dedup, low-signal
--- session). A malformed payload (schema-flip shapes) and a write that did
--- not land both leave no stamp — either would make the heartbeat mask the
--- exact dropout it exists to expose. The recall-side hooks open read-only
--- and deliberately do not appear here: their liveness answers a different
--- question, and giving them a write handle would put a lock acquisition on
--- the SessionStart hot path.
---
--- One row per hook, upserted. It does not grow.
-CREATE TABLE IF NOT EXISTS hook_runs (
-  hook        TEXT PRIMARY KEY,
-  last_run_at TIMESTAMP NOT NULL,
-  run_count   INTEGER NOT NULL DEFAULT 0
-);
-
--- The 'hook_runs_since' metadata key (when this database first became able
--- to record hook runs) is stamped by ensureHookRunsSince() AFTER this exec,
--- NOT here. It used to be an INSERT OR IGNORE in this string, and that made
--- every open — including opens that only ever read — start a write
--- transaction: an INSERT statement takes the WAL writer lock even when
--- OR IGNORE ends up changing nothing. Two states regressed from "reads work"
--- to "open throws": a peer holding the writer lock past busy_timeout, and a
--- read-only database FILE (measured: "attempt to write a readonly database"
--- killed recall along with capture). The helper SELECTs first and writes
--- only when the key is genuinely absent — once per database lifetime.
-`;
-const FTS_SQL = `
-CREATE VIRTUAL TABLE IF NOT EXISTS entities_fts USING fts5(
-  name, observations, content='',
-  tokenize='unicode61 remove_diacritics 1'
-);
-
--- Term -> document-count view over the index above. Stores nothing of its own;
--- it exists so search() can drop query terms that appear in most of the corpus,
--- which are the ones BM25 already scores near zero. See dropUbiquitousTerms().
-CREATE VIRTUAL TABLE IF NOT EXISTS fts_vocab USING fts5vocab(entities_fts, 'row');
-`;
-function ensureTagsUniqueIndex(handle) {
-    try {
-        const present = handle
-            .prepare("SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = 'idx_tags_entity_tag_unique'")
-            .get();
-        if (present)
-            return;
-        handle.exec('BEGIN IMMEDIATE; ' +
-            'DELETE FROM tags WHERE id NOT IN (SELECT MIN(id) FROM tags GROUP BY entity_id, tag); ' +
-            'CREATE UNIQUE INDEX IF NOT EXISTS idx_tags_entity_tag_unique ON tags(entity_id, tag); ' +
-            'COMMIT;');
-    }
-    catch (err) {
-        try {
-            handle.exec('ROLLBACK');
-        }
-        catch { }
-        try {
-            process.stderr.write(`MeMesh: could not create the tags unique index (${err instanceof Error ? err.message : String(err)}). ` +
-                `Reads are unaffected; the next open retries the dedup and index together.\n`);
-        }
-        catch { }
-    }
-}
-function ensureHookRunsSince(handle) {
-    try {
-        const row = handle
-            .prepare("SELECT value FROM memesh_metadata WHERE key = 'hook_runs_since'")
-            .get();
-        if (row) {
-            const m = /^(\d{4})-(\d{2})-(\d{2})[ T](\d{2}):(\d{2}):(\d{2})$/.exec(row.value ?? '');
-            if (m) {
-                const then = Date.UTC(+m[1], +m[2] - 1, +m[3], +m[4], +m[5], +m[6]);
-                const d = new Date(then);
-                const intact = Number.isFinite(then)
-                    && d.getUTCFullYear() === +m[1] && d.getUTCMonth() === +m[2] - 1 && d.getUTCDate() === +m[3]
-                    && d.getUTCHours() === +m[4] && d.getUTCMinutes() === +m[5] && d.getUTCSeconds() === +m[6];
-                if (intact && then <= Date.now() + 5 * 60 * 1000)
-                    return;
-            }
-            handle
-                .prepare("UPDATE memesh_metadata SET value = datetime('now') WHERE key = 'hook_runs_since'")
-                .run();
-            return;
-        }
-        handle
-            .prepare("INSERT OR IGNORE INTO memesh_metadata (key, value) VALUES ('hook_runs_since', datetime('now'))")
-            .run();
-    }
-    catch (err) {
-        try {
-            process.stderr.write(`MeMesh: could not stamp hook_runs_since (${err instanceof Error ? err.message : String(err)}). ` +
-                `Reads are unaffected; doctor's hook-activity tracking starts once the database is writable.\n`);
-        }
-        catch { }
-    }
-}
+let dimensionMismatchNoticed = false;
 export function openDatabase(dbPath) {
     if (db)
         return db;
@@ -236,41 +72,13 @@ function migrateToCurrentSchema(db, resolvedPath) {
         }
         catch { }
     }
-    const safeAlter = (sql) => {
-        try {
-            db.exec(sql);
-        }
-        catch (e) {
-            const msg = e instanceof Error ? e.message : String(e);
-            if (!/duplicate column name/i.test(msg))
-                throw e;
-        }
-    };
-    const columns = db.prepare("PRAGMA table_info(entities)").all();
-    if (!columns.some((c) => c.name === 'status')) {
-        safeAlter("ALTER TABLE entities ADD COLUMN status TEXT NOT NULL DEFAULT 'active'");
-        db.exec("CREATE INDEX IF NOT EXISTS idx_entities_status ON entities(status)");
-    }
-    const scoringCols = db.prepare("PRAGMA table_info(entities)").all();
-    if (!scoringCols.some((c) => c.name === 'access_count')) {
-        safeAlter("ALTER TABLE entities ADD COLUMN access_count INTEGER DEFAULT 0");
-        safeAlter("ALTER TABLE entities ADD COLUMN last_accessed_at TIMESTAMP");
-        safeAlter("ALTER TABLE entities ADD COLUMN confidence REAL DEFAULT 1.0");
-        safeAlter("ALTER TABLE entities ADD COLUMN valid_from TIMESTAMP");
-        safeAlter("ALTER TABLE entities ADD COLUMN valid_until TIMESTAMP");
-    }
-    if (!scoringCols.some((c) => c.name === 'namespace')) {
-        safeAlter("ALTER TABLE entities ADD COLUMN namespace TEXT DEFAULT 'personal'");
-        db.exec("CREATE INDEX IF NOT EXISTS idx_entities_namespace ON entities(namespace)");
-    }
-    const recallCols = db.prepare("PRAGMA table_info(entities)").all();
-    if (!recallCols.some((c) => c.name === 'recall_hits')) {
-        safeAlter("ALTER TABLE entities ADD COLUMN recall_hits INTEGER DEFAULT 0");
-        safeAlter("ALTER TABLE entities ADD COLUMN recall_misses INTEGER DEFAULT 0");
-    }
+    migrateEntitiesSchema(db);
     runAutoDecay(db);
     backfillSignalScores(db);
+    backfillTitles(db);
+    backfillAcceptedProposalTrust(db);
     ensureDreamProposalsTable(db);
+    ensureConflictJudgedPairsTable(db);
     ensureLlmTelemetryTable(db);
     runAutoTelemetryPrune(db);
     ensureFtsSegmentation(db);
@@ -293,82 +101,6 @@ function migrateToCurrentSchema(db, resolvedPath) {
         ensureVecTable(db, resolvedPath, targetDim, dimensionKnown);
     }
 }
-export const FTS_SEGMENTATION_VERSION = 3;
-const MIGRATION_RETRY_BACKOFF_MS = 24 * 60 * 60 * 1000;
-function isTransientDbError(err) {
-    const code = err?.code ?? '';
-    const msg = err?.message ?? '';
-    return (/SQLITE_BUSY|SQLITE_LOCKED|SQLITE_PROTOCOL/.test(code) ||
-        /database is locked|database table is locked|locking protocol/i.test(msg));
-}
-export function runOnceMigration(db, opts) {
-    const { key, version, describe, migrate } = opts;
-    const attemptKey = `${key}_last_attempt`;
-    const readMarker = (k) => db.prepare('SELECT value FROM memesh_metadata WHERE key = ?').get(k)?.value;
-    const stored = readMarker(key);
-    if (stored && parseInt(stored, 10) >= version)
-        return false;
-    const lastAttempt = readMarker(attemptKey);
-    if (lastAttempt && Date.now() - parseInt(lastAttempt, 10) < MIGRATION_RETRY_BACKOFF_MS) {
-        return false;
-    }
-    try {
-        db.transaction(() => {
-            const current = readMarker(key);
-            if (current && parseInt(current, 10) >= version)
-                return;
-            migrate(db, current ? parseInt(current, 10) : 0);
-            db.prepare('INSERT OR REPLACE INTO memesh_metadata (key, value) VALUES (?, ?)').run(key, String(version));
-            db.prepare('DELETE FROM memesh_metadata WHERE key = ?').run(attemptKey);
-        }).immediate();
-        return true;
-    }
-    catch (err) {
-        if (isTransientDbError(err)) {
-            process.stderr.write(`MeMesh: ${describe} deferred (${err instanceof Error ? err.message : String(err)}). ` +
-                `Another process holds the database; it will run on the next start.\n`);
-            return false;
-        }
-        try {
-            db.prepare('INSERT OR REPLACE INTO memesh_metadata (key, value) VALUES (?, ?)').run(attemptKey, String(Date.now()));
-        }
-        catch { }
-        process.stderr.write(`MeMesh: ${describe} failed (${err instanceof Error ? err.message : String(err)}). ` +
-            `Your memories are unaffected — this rebuilds a derived index. ` +
-            `It will retry in 24h, or run 'memesh reindex --fts' to retry now.\n`);
-        return false;
-    }
-}
-function ensureFtsSegmentation(db) {
-    runOnceMigration(db, {
-        key: 'fts_segmentation_version',
-        version: FTS_SEGMENTATION_VERSION,
-        describe: 'search index rebuild',
-        migrate: rebuildFtsIndex,
-    });
-}
-const FTS_REBUILD_PAGE_SIZE = 500;
-function rebuildFtsIndex(db) {
-    db.exec("INSERT INTO entities_fts (entities_fts) VALUES('delete-all')");
-    const page = db.prepare(`SELECT e.id, e.name, COALESCE(group_concat(o.content, ' '), '') AS obs
-       FROM entities e
-       LEFT JOIN observations o ON o.entity_id = e.id
-      WHERE e.status = 'active' AND e.id > ?
-      GROUP BY e.id
-      ORDER BY e.id
-      LIMIT ?`);
-    let afterId = 0;
-    for (;;) {
-        const rows = page.all(afterId, FTS_REBUILD_PAGE_SIZE);
-        if (rows.length === 0)
-            break;
-        for (const row of rows)
-            insertFtsRow(db, row.id, row.name, row.obs);
-        afterId = rows[rows.length - 1].id;
-        if (rows.length < FTS_REBUILD_PAGE_SIZE)
-            break;
-    }
-}
 export function reindexFts() {
     const database = getDatabase();
     database.transaction(() => {
@@ -383,20 +115,7 @@ export function reindexFts() {
         .get();
     return { entities: c };
 }
-let vectorRebuildConsentFor = null;
-export async function allowVectorIndexRebuild(dbPath, canRefill) {
-    if (!(await canRefill()))
-        return false;
-    vectorRebuildConsentFor = path.resolve(dbPath);
-    return true;
-}
-function consumeVectorRebuildConsent(resolvedPath) {
-    const granted = vectorRebuildConsentFor;
-    vectorRebuildConsentFor = null;
-    return granted !== null && granted === path.resolve(resolvedPath);
-}
 function ensureVecTable(db, resolvedPath, targetDim, dimensionKnown = true) {
-    const rebuildConsented = consumeVectorRebuildConsent(resolvedPath);
     const storedDim = db.prepare("SELECT value FROM memesh_metadata WHERE key = 'embedding_dimension'").get();
     const currentDim = storedDim ? parseInt(storedDim.value, 10) : 0;
     const vecExists = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='entities_vec'").get();
@@ -409,25 +128,21 @@ function ensureVecTable(db, resolvedPath, targetDim, dimensionKnown = true) {
             `Fix ~/.memesh/config.json to change embedders.\n`);
         return;
     }
-    if (vecExists &&
-        currentDim !== 0 &&
-        currentDim !== targetDim &&
-        !rebuildConsented) {
-        process.stderr.write(`MeMesh: this database records ${currentDim}-dim embeddings but the current ` +
-            `configuration asks for ${targetDim}. Keeping the existing vector index rather ` +
-            `than rebuilding it, because rebuilding deletes every stored vector. ` +
-            `If the configuration is wrong, fix it. If you meant to switch embedders, run ` +
-            `'memesh reindex --vectors' to rebuild the index at ${targetDim} and regenerate.\n`);
+    if (vecExists && currentDim !== 0 && currentDim !== targetDim) {
+        if (!dimensionMismatchNoticed) {
+            process.stderr.write(`MeMesh: this database records ${currentDim}-dim embeddings but the current ` +
+                `configuration asks for ${targetDim}. Nothing is deleted — the index is kept ` +
+                `so a rebuild can resume — but semantic search is OFF until the rebuild ` +
+                `finishes, because a ${targetDim}-dim query cannot be matched against a ` +
+                `${currentDim}-dim index; recall is on keyword search alone meanwhile. ` +
+                `Run 'memesh reindex' to build the ${targetDim}-dim index alongside it and ` +
+                `switch over once it is complete.\n`);
+            dimensionMismatchNoticed = true;
+        }
+        markReindexOwed(currentDim, targetDim, 'dimension-change', db);
         return;
     }
     db.transaction(() => {
-        if (vecExists) {
-            process.stderr.write(`MeMesh: Embedding dimension changed (${currentDim} → ${targetDim}). Rebuilding vector index.\n` +
-                `MeMesh: Old embeddings deleted. Run 'memesh reindex' to regenerate vectors for all entities.\n` +
-                `MeMesh: Without reindex, only newly accessed entities will be embedded.\n`);
-            db.exec('DROP TABLE entities_vec');
-            db.prepare("INSERT OR REPLACE INTO memesh_metadata (key, value) VALUES ('pending_reindex', ?)").run(JSON.stringify({ from: currentDim, to: targetDim, droppedAt: new Date().toISOString() }));
-        }
         db.exec(`
       CREATE VIRTUAL TABLE IF NOT EXISTS entities_vec USING vec0(
         embedding float[${targetDim}]
@@ -436,17 +151,164 @@ function ensureVecTable(db, resolvedPath, targetDim, dimensionKnown = true) {
         db.prepare("INSERT OR REPLACE INTO memesh_metadata (key, value) VALUES ('embedding_dimension', ?)").run(String(targetDim));
     }).immediate();
 }
-export function getPendingReindexInfo() {
+export const GENERATION_TABLE = 'entities_vec_next';
+const GENERATION_HASH_TABLE = 'entities_vec_next_source';
+const GENERATION_KEY = 'vector_generation';
+function tableExists(conn, name) {
+    return conn.prepare("SELECT COUNT(*) AS c FROM sqlite_master WHERE type IN ('table','view') AND name = ?").get(name).c > 0;
+}
+function assertVectorWidth(dimension) {
+    if (!Number.isInteger(dimension) || dimension <= 0 || dimension > 65536) {
+        throw new Error(`Refusing to build a vector index at width ${String(dimension)}.`);
+    }
+}
+export function readVectorGeneration() {
+    if (!db)
+        return { state: 'none' };
+    try {
+        const row = db.prepare('SELECT value FROM memesh_metadata WHERE key = ?').get(GENERATION_KEY);
+        if (!row)
+            return { state: 'none' };
+        const parsed = JSON.parse(row.value);
+        if (typeof parsed.dimension !== 'number' || typeof parsed.provider !== 'string') {
+            return { state: 'unreadable', detail: 'the marker is missing its dimension or provider' };
+        }
+        return {
+            state: 'open',
+            info: {
+                dimension: parsed.dimension,
+                provider: parsed.provider,
+                startedAt: String(parsed.startedAt ?? ''),
+            },
+        };
+    }
+    catch (err) {
+        return { state: 'unreadable', detail: err instanceof Error ? err.message : String(err) };
+    }
+}
+export function generationRowIds() {
+    const out = new Set();
     if (!db)
+        return out;
+    if (!tableExists(db, GENERATION_TABLE))
+        return out;
+    const rows = db.prepare(`SELECT rowid AS id FROM ${GENERATION_TABLE}`).all();
+    for (const r of rows)
+        out.add(Number(r.id));
+    return out;
+}
+export function beginVectorGeneration(dimension, provider) {
+    assertVectorWidth(dimension);
+    const conn = getDatabase();
+    const read = readVectorGeneration();
+    const stagingExists = tableExists(conn, GENERATION_TABLE);
+    if (read.state === 'unreadable' && stagingExists) {
+        throw new Error('A half-built vector index is present but its marker cannot be read '
+            + `(${read.detail}). Resuming it risks mixing vectors from two different `
+            + 'embedding spaces, and discarding it throws away embeddings a previous run '
+            + 'already produced, so neither is done automatically. Run '
+            + '`memesh reindex --discard-generation` to throw the half-built index away '
+            + 'and start clean.');
+    }
+    const existing = read.state === 'open' ? read.info : null;
+    const compatible = stagingExists && existing !== null
+        && existing.dimension === dimension && existing.provider === provider;
+    if (stagingExists && !compatible) {
+        discardVectorGeneration();
+    }
+    const startedAt = compatible && existing ? existing.startedAt : new Date().toISOString();
+    conn.transaction(() => {
+        conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS ${GENERATION_TABLE} USING vec0(embedding float[${dimension}])`);
+        conn.exec(`CREATE TABLE IF NOT EXISTS ${GENERATION_HASH_TABLE} (`
+            + 'rowid_ref INTEGER PRIMARY KEY, text_hash TEXT NOT NULL)');
+        conn.prepare('INSERT OR REPLACE INTO memesh_metadata (key, value) VALUES (?, ?)').run(GENERATION_KEY, JSON.stringify({ dimension, provider, startedAt }));
+    }).immediate();
+    return { resumed: compatible };
+}
+export function generationRowHashes() {
+    const out = new Map();
+    if (!db)
+        return out;
+    if (!tableExists(db, GENERATION_HASH_TABLE))
+        return out;
+    const rows = db.prepare(`SELECT rowid_ref AS id, text_hash AS h FROM ${GENERATION_HASH_TABLE}`).all();
+    for (const r of rows)
+        out.set(Number(r.id), r.h);
+    return out;
+}
+export function recordGenerationRow(entityId, textHash) {
+    const conn = getDatabase();
+    conn.prepare(`INSERT OR REPLACE INTO ${GENERATION_HASH_TABLE} (rowid_ref, text_hash) VALUES (?, ?)`).run(BigInt(entityId), textHash);
+}
+export function discardVectorGeneration() {
+    const conn = getDatabase();
+    conn.transaction(() => {
+        conn.exec(`DROP TABLE IF EXISTS ${GENERATION_TABLE}`);
+        conn.exec(`DROP TABLE IF EXISTS ${GENERATION_HASH_TABLE}`);
+        conn.prepare('DELETE FROM memesh_metadata WHERE key = ?').run(GENERATION_KEY);
+    }).immediate();
+}
+export function swapVectorGeneration(dimension) {
+    assertVectorWidth(dimension);
+    const conn = getDatabase();
+    conn.transaction(() => {
+        conn.exec(`DELETE FROM ${GENERATION_TABLE} WHERE rowid NOT IN `
+            + `(SELECT id FROM entities WHERE status = 'active')`);
+        const storedDim = Number(conn.prepare("SELECT value FROM memesh_metadata WHERE key = 'embedding_dimension'").get()?.value ?? 0);
+        if (storedDim === dimension) {
+            conn.exec(`INSERT INTO ${GENERATION_TABLE} (rowid, embedding) `
+                + `SELECT v.rowid, v.embedding FROM entities_vec v `
+                + `WHERE v.rowid IN (SELECT id FROM entities WHERE status = 'active') `
+                + `AND v.rowid NOT IN (SELECT rowid FROM ${GENERATION_TABLE})`);
+        }
+        const staged = conn.prepare(`SELECT COUNT(*) AS c FROM ${GENERATION_TABLE}`).get().c;
+        conn.exec('DROP TABLE IF EXISTS entities_vec');
+        conn.exec(`CREATE VIRTUAL TABLE entities_vec USING vec0(embedding float[${dimension}])`);
+        conn.exec(`INSERT INTO entities_vec (rowid, embedding) SELECT rowid, embedding FROM ${GENERATION_TABLE}`);
+        const installed = conn.prepare('SELECT COUNT(*) AS c FROM entities_vec').get().c;
+        if (installed !== staged) {
+            throw new Error(`Vector index swap copied ${installed} of ${staged} staged rows; refusing to publish a short index.`);
+        }
+        conn.exec(`DROP TABLE ${GENERATION_TABLE}`);
+        conn.exec(`DROP TABLE IF EXISTS ${GENERATION_HASH_TABLE}`);
+        conn.prepare('INSERT OR REPLACE INTO memesh_metadata (key, value) VALUES (?, ?)').run('embedding_dimension', String(dimension));
+        conn.prepare('DELETE FROM memesh_metadata WHERE key = ?').run(GENERATION_KEY);
+    }).immediate();
+}
+export function getStoredEmbeddingDimension() {
+    if (!db)
+        return 0;
+    const row = db.prepare("SELECT value FROM memesh_metadata WHERE key = 'embedding_dimension'").get();
+    return row ? Number(row.value) || 0 : 0;
+}
+export function getPendingReindexInfo() {
+    return readPendingReindex(db);
+}
+function readPendingReindex(conn) {
+    if (!conn)
         return null;
     try {
-        const row = db.prepare("SELECT value FROM memesh_metadata WHERE key = 'pending_reindex'").get();
+        const row = conn.prepare("SELECT value FROM memesh_metadata WHERE key = 'pending_reindex'").get();
         return row ? JSON.parse(row.value) : null;
     }
     catch {
         return null;
     }
 }
+export function markReindexOwed(from, to, reason, conn = db) {
+    if (!conn)
+        return;
+    const existing = readPendingReindex(conn);
+    if (existing && existing.from === from && existing.to === to && existing.reason === reason) {
+        return;
+    }
+    conn.prepare("INSERT OR REPLACE INTO memesh_metadata (key, value) VALUES ('pending_reindex', ?)").run(JSON.stringify({
+        from,
+        to,
+        reason,
+        noticedAt: existing?.noticedAt ?? new Date().toISOString(),
+    }));
+}
 export function clearPendingReindexFlag() {
     if (!db)
         return;
@@ -472,14 +334,21 @@ function ensureDreamProposalsTable(db) {
   `);
     const dpCols = db.prepare("PRAGMA table_info(dream_proposals)").all();
     if (!dpCols.some((c) => c.name === 'source_kind')) {
-        try {
-            db.exec("ALTER TABLE dream_proposals ADD COLUMN source_kind TEXT NOT NULL DEFAULT 'entities'");
-        }
-        catch (err) {
-            if (!String(err.message).includes('duplicate column name'))
-                throw err;
-        }
+        safeAlter(db, "ALTER TABLE dream_proposals ADD COLUMN source_kind TEXT NOT NULL DEFAULT 'entities'");
     }
+    if (!dpCols.some((c) => c.name === 'kind')) {
+        safeAlter(db, "ALTER TABLE dream_proposals ADD COLUMN kind TEXT NOT NULL DEFAULT 'digest'");
+    }
+}
+function ensureConflictJudgedPairsTable(db) {
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS conflict_judged_pairs (
+      pair_key TEXT PRIMARY KEY,
+      verdict TEXT NOT NULL,
+      proposal_id INTEGER,
+      judged_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
+    );
+  `);
 }
 function ensureLlmTelemetryTable(db) {
     db.exec(`
@@ -574,11 +443,134 @@ function backfillSignalScores(db) {
     });
     tx();
 }
+function deriveHeuristicTitle(type, observations) {
+    if (observations.length === 0)
+        return null;
+    if (type === 'lesson_learned' || type === 'lesson' || type === 'mistake') {
+        const errObs = observations.find((o) => /^Error:\s*/.test(o.trim()));
+        if (errObs) {
+            const firstLine = errObs.trim().replace(/^Error:\s*/, '').split('\n')[0].trim();
+            if (firstLine)
+                return truncateTitle(firstLine);
+        }
+    }
+    if (type === 'commit') {
+        const first = observations[0]?.split('\n')[0].trim();
+        if (first && !/^(Branch|Diff stats):/.test(first))
+            return truncateTitle(first);
+    }
+    const nonTrivial = observations.filter((o) => o.length > 30 && !isBoilerplateObservation(o));
+    const pool = nonTrivial.length > 0 ? nonTrivial : observations;
+    const best = pool.slice(0, 3).reduce((a, b) => (b.length > a.length ? b : a), pool[0]);
+    const firstLine = best?.split('\n')[0].trim();
+    return firstLine ? truncateTitle(firstLine) : null;
+}
+function backfillAcceptedProposalTrust(db) {
+    const MARKER = 'accepted_proposal_trust_v1';
+    if (db.prepare('SELECT value FROM memesh_metadata WHERE key = ?').get(MARKER))
+        return;
+    const stamp = (cleared, skipped) => db.prepare('INSERT OR REPLACE INTO memesh_metadata (key, value) VALUES (?, ?)')
+        .run(MARKER, JSON.stringify({ at: new Date().toISOString(), cleared, skipped }));
+    let rows;
+    try {
+        rows = db.prepare(`SELECT id, metadata FROM entities
+        WHERE metadata IS NOT NULL
+          AND json_valid(metadata)
+          AND json_extract(metadata, '$.trust') = 'untrusted'
+          AND json_extract(metadata, '$.proposal_id') IS NOT NULL`).all();
+    }
+    catch {
+        return;
+    }
+    if (rows.length === 0) {
+        stamp(0, 0);
+        return;
+    }
+    const updateStmt = db.prepare('UPDATE entities SET metadata = ? WHERE id = ?');
+    const tx = db.transaction(() => {
+        let cleared = 0;
+        let skipped = 0;
+        for (const row of rows) {
+            let metadata;
+            try {
+                const parsed = JSON.parse(row.metadata ?? '{}');
+                if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+                    skipped++;
+                    continue;
+                }
+                metadata = parsed;
+            }
+            catch {
+                skipped++;
+                continue;
+            }
+            delete metadata.trust;
+            updateStmt.run(JSON.stringify(metadata), row.id);
+            cleared++;
+        }
+        stamp(cleared, skipped);
+    });
+    tx();
+}
+function backfillTitles(db) {
+    const MARKER = 'title_backfill_v1';
+    const done = db.prepare('SELECT value FROM memesh_metadata WHERE key = ?').get(MARKER);
+    if (done)
+        return;
+    const rows = db.prepare('SELECT id, name, type, status, metadata FROM entities WHERE title IS NULL').all();
+    const stamp = (titled, skipped) => db.prepare('INSERT OR REPLACE INTO memesh_metadata (key, value) VALUES (?, ?)').run(MARKER, JSON.stringify({ at: new Date().toISOString(), titled, skipped }));
+    if (rows.length === 0) {
+        stamp(0, 0);
+        return;
+    }
+    const obsStmt = db.prepare('SELECT content FROM observations WHERE entity_id = ? ORDER BY id');
+    const updateStmt = db.prepare('UPDATE entities SET title = ?, metadata = ? WHERE id = ?');
+    const tx = db.transaction(() => {
+        let titled = 0;
+        let skipped = 0;
+        for (const row of rows) {
+            let metadata;
+            if (row.metadata) {
+                try {
+                    metadata = JSON.parse(row.metadata);
+                }
+                catch {
+                    skipped++;
+                    continue;
+                }
+                if (typeof metadata !== 'object' || metadata === null || Array.isArray(metadata)) {
+                    skipped++;
+                    continue;
+                }
+            }
+            else {
+                metadata = {};
+            }
+            const observations = obsStmt.all(row.id).map(o => o.content);
+            const title = deriveHeuristicTitle(row.type, observations);
+            if (!title) {
+                skipped++;
+                continue;
+            }
+            metadata.title_source = 'heuristic';
+            updateStmt.run(title, JSON.stringify(metadata), row.id);
+            if (row.status === 'active') {
+                const obsText = joinIndexedObservations(observations);
+                removeFromFts(db, row.id, row.name, obsText);
+                insertFtsRow(db, row.id, row.name, obsText, title);
+            }
+            titled++;
+        }
+        db.prepare('INSERT OR REPLACE INTO memesh_metadata (key, value) VALUES (?, ?)').run(MARKER, JSON.stringify({ at: new Date().toISOString(), titled, skipped }));
+    });
+    tx();
+}
 export function closeDatabase() {
     if (db) {
         db.close();
         db = null;
     }
+    dimensionMismatchNoticed = false;
 }
 export function getDatabase() {
     if (!db)