akm-cli 0.9.0-beta.5 → 0.9.0-beta.51

package/dist/indexer/db/db.js

@@ -7,29 +7,34 @@ import path from "node:path";
 import { parseAssetRef } from "../../core/asset/asset-ref.js";
 import { bestEffort } from "../../core/best-effort.js";
 import { getDbPath } from "../../core/paths.js";
-import { REGISTRY_INDEX_CACHE_DDL } from "../../core/state-db.js";
 import { warn } from "../../core/warn.js";
 import { cosineSimilarity } from "../../llm/embedders/types.js";
 import { sha256Hex } from "../../runtime.js";
-import { openDatabase as openSqlite } from "../../storage/database.js";
+import { openDatabase } from "../../storage/database.js";
+import { applyStandardPragmas } from "../../storage/sqlite-pragmas.js";
 import { buildSearchFields } from "../search/search-fields.js";
 import { ensureUsageEventsSchema } from "../usage/usage-events.js";
-import { backupDataDir, EMBEDDING_DIM_CHANGE_REASON } from "./db-backup.js";
 // ── Constants ───────────────────────────────────────────────────────────────
+// NOTE: schema changes are additive. DB_VERSION is a forensic stamp only — it
+// no longer gates any destructive path (the old nuclear drop-and-rebuild was
+// removed; index.db's idempotent CREATE … IF NOT EXISTS schema converges any
+// older/partial DB forward without dropping data). Graph re-keying uses a
+// TARGETED, graph-only migration (migrateGraphFilesSchema) — the model for any
+// incompatible change: migrate in place, never wipe the whole index.
 export const DB_VERSION = 17;
 export const EMBEDDING_DIM = 384;
-export const GRAPH_SCHEMA_VERSION = 3;
+// #624-P1: graph_files re-keyed to (stash_root, file_path, body_hash). Bumped 3→4
+// as a marker; the actual migration is the targeted drop in migrateGraphFilesSchema.
+export const GRAPH_SCHEMA_VERSION = 4;
 // ── Database lifecycle ──────────────────────────────────────────────────────
-export function openDatabase(dbPath, options) {
+export function openIndexDatabase(dbPath, options) {
     const resolvedPath = dbPath ?? getDbPath();
     const dir = path.dirname(resolvedPath);
     if (!fs.existsSync(dir)) {
         fs.mkdirSync(dir, { recursive: true });
     }
-    const db = openSqlite(resolvedPath);
-    db.exec("PRAGMA journal_mode = WAL");
-    db.exec("PRAGMA busy_timeout = 30000");
-    db.exec("PRAGMA foreign_keys = ON");
+    const db = openDatabase(resolvedPath);
+    applyStandardPragmas(db, { dataDir: dir });
     // Try to load sqlite-vec extension
     loadVecExtension(db);
     // Dim resolution: explicit option wins; otherwise consult the on-disk
@@ -38,7 +43,7 @@ export function openDatabase(dbPath, options) {
     // both are absent do we fall through to the no-clobber path, which keeps
     // ensureSchema from touching `index_meta.embeddingDim` at all.
     const resolvedDim = options?.embeddingDim ?? resolveConfiguredEmbeddingDim();
-    ensureSchema(db, resolvedDim, { dataDir: dir });
+    ensureSchema(db, resolvedDim);
     // Warn once at init if using JS fallback with many entries
     warnIfVecMissing(db, { once: true });
     return db;
@@ -67,10 +72,9 @@ function resolveConfiguredEmbeddingDim() {
 }
 export function openExistingDatabase(dbPath) {
     const resolvedPath = dbPath ?? getDbPath();
-    const db = openSqlite(resolvedPath);
-    db.exec("PRAGMA journal_mode = WAL");
-    db.exec("PRAGMA busy_timeout = 30000");
-    db.exec("PRAGMA foreign_keys = ON");
+    const dir = path.dirname(resolvedPath);
+    const db = openDatabase(resolvedPath);
+    applyStandardPragmas(db, { dataDir: dir });
     // Existing-DB callers must not mutate schema or embedding metadata on open,
     // but some paths still need write access to usage_events and other tables.
     loadVecExtension(db);
@@ -123,7 +127,44 @@ export function warnIfVecMissing(db, { once } = { once: false }) {
         }
     }, "embeddings table may not exist yet during init");
 }
-function ensureSchema(db, embeddingDim, options) {
+// ── Schema ──────────────────────────────────────────────────────────────────
+/**
+ * DDL for the `registry_index_cache` table. This table lives in index.db
+ * (managed by this module), so its DDL belongs here next to the `ensureSchema`
+ * that applies it — not in state-db.ts.
+ *
+ * Created with CREATE TABLE IF NOT EXISTS so it is safe to call inside
+ * `ensureSchema()`. Caches the result of resolving and fetching remote registry
+ * stash indexes so `akm search` does not hit the network on every invocation.
+ *
+ * Indexed (query) columns:
+ *   registry_url  TEXT PK   — canonical URL of the registry; cache key.
+ *   fetched_at    TEXT      — ISO-8601; used to detect stale entries (TTL).
+ *   etag          TEXT      — HTTP ETag for conditional GET (If-None-Match).
+ *   last_modified TEXT      — HTTP Last-Modified for conditional GET.
+ *
+ * Non-indexed payload:
+ *   index_json    TEXT      — JSON blob of the fetched registry index document.
+ *
+ * ADD COLUMN extension points (future migrations):
+ *   ALTER TABLE registry_index_cache ADD COLUMN schema_version INTEGER DEFAULT 1;
+ *   ALTER TABLE registry_index_cache ADD COLUMN kit_count INTEGER DEFAULT NULL;
+ *   ALTER TABLE registry_index_cache ADD COLUMN error_message TEXT DEFAULT NULL;
+ */
+const REGISTRY_INDEX_CACHE_DDL = `
+  CREATE TABLE IF NOT EXISTS registry_index_cache (
+    registry_url  TEXT    PRIMARY KEY,
+    fetched_at    TEXT    NOT NULL,
+    etag          TEXT,
+    last_modified TEXT,
+    index_json    TEXT    NOT NULL DEFAULT '{}'
+  );
+
+  CREATE INDEX IF NOT EXISTS idx_registry_cache_fetched
+    ON registry_index_cache(fetched_at);
+`;
+/** A row backed up out of the legacy `usage_events` table during a version upgrade. */
+function ensureSchema(db, embeddingDim) {
     // Create meta table first so we can check version
     db.exec(`
     CREATE TABLE IF NOT EXISTS index_meta (
@@ -131,46 +172,16 @@ function ensureSchema(db, embeddingDim, options) {
       value TEXT NOT NULL
     );
   `);
-    // MVP DB-backup hook (0.8.x): when the stored DB version differs from the
-    // running binary's DB_VERSION, snapshot the data directory BEFORE
-    // `handleVersionUpgrade()` drops tables. This is best-effort —
-    // `backupDataDir` returns null on opt-out, missing data dir, low free
-    // space, or copy errors, and we proceed with the upgrade in all cases.
-    // The proper migration framework lands in 0.9.0; until then this lets
-    // operators recover with `scripts/migrations/restore-data-dir.sh`.
-    if (options?.dataDir) {
-        const storedVersionRaw = getMeta(db, "version");
-        const storedVersion = storedVersionRaw !== undefined && storedVersionRaw !== "" ? Number.parseInt(storedVersionRaw, 10) : null;
-        const willUpgrade = storedVersionRaw !== undefined && storedVersionRaw !== "" && storedVersionRaw !== String(DB_VERSION);
-        if (willUpgrade) {
-            try {
-                // Pass env explicitly so tests can override AKM_DB_BACKUP / AKM_DB_BACKUP_RETAIN
-                // without mutating process.env. Production callers default to process.env.
-                const result = backupDataDir({
-                    dataDir: options.dataDir,
-                    sourceVersion: storedVersion !== null && !Number.isNaN(storedVersion) ? storedVersion : null,
-                    targetVersion: DB_VERSION,
-                    env: process.env,
-                });
-                if (result) {
-                    warn("[akm] data directory backed up to %s before v%s→v%d upgrade", result.path, storedVersionRaw, DB_VERSION);
-                }
-            }
-            catch (err) {
-                // Defensive — backupDataDir already swallows most errors, but if it
-                // throws for an unexpected reason we must still proceed with the
-                // upgrade so the user isn't locked out of their binary.
-                warn("[akm] pre-upgrade data dir backup raised an unexpected error — %s; upgrade will proceed without a snapshot", err instanceof Error ? err.message : String(err));
-            }
-        }
-    }
-    // Check stored version — if it differs from DB_VERSION, drop and recreate all tables.
-    // Usage events are preserved across version upgrades so that utility score
-    // history is not silently lost. The backup is captured here and threaded
-    // explicitly to `restoreUsageEventsBackup` below — the previous version
-    // attached `__usageBackup` to the Database instance via a typeless property
-    // injection, which was a source of fragile coupling.
-    const usageBackup = handleVersionUpgrade(db);
+    // index.db is a fully regenerable derived cache, so its schema is built
+    // idempotently below: every table is CREATE … IF NOT EXISTS and column
+    // additions go through guarded ALTERs (ensureDerivedFromColumn) and targeted
+    // migrations (migrateGraphFilesSchema / migrateGraphDataFromLegacy). Opening a
+    // database with an older or partial schema converges it forward WITHOUT ever
+    // dropping data — there is intentionally no "nuclear drop the whole index on a
+    // DB_VERSION mismatch" path (a destructive design the regenerable index never
+    // needed, and whose pre-drop data-dir backup it required). A genuinely
+    // incompatible change is handled by an additive/targeted migration; the few
+    // derived tables that ever must be rebuilt are regenerated by `akm index`.
     db.exec(`
     CREATE TABLE IF NOT EXISTS entries (
       id          INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -299,15 +310,27 @@ function ensureSchema(db, embeddingDim, options) {
      CREATE INDEX IF NOT EXISTS idx_llm_cache_updated
        ON llm_enrichment_cache(updated_at);
   `);
-    // Graph extraction tables — schema v2 (entry_id PK).
+    // Graph extraction tables — schema v4 ((stash_root, file_path, body_hash) PK).
     //
-    // graph_files is keyed on entries.id so child tables cascade-delete cleanly
-    // when an entry is removed, and so JOINs from graph rows to entries are a
-    // direct PK lookup. (stash_root, file_path) is retained as UNIQUE so the
-    // extractor's path-based upsert still works.
+    // graph_files is self-keyed on (stash_root, file_path, body_hash) and is NO
+    // LONGER tied to entries.id. This is the #624-P1 win: deleting and
+    // re-inserting an entries row during a reindex no longer cascade-wipes the
+    // extracted graph — as long as the file's body_hash is unchanged, the graph
+    // data survives. body_hash is part of the PK so a content change yields a
+    // distinct key; a UNIQUE index on (stash_root, file_path) still enforces
+    // exactly one graph_files row per path (delete-then-insert on a hash change).
     //
-    // graph_file_entities and graph_file_relations no longer duplicate file_path;
-    // they reference entry_id and inherit stash scoping via graph_files.
+    // graph_file_entities and graph_file_relations carry (stash_root, file_path,
+    // body_hash) and declare a composite FK -> graph_files ON DELETE CASCADE so
+    // child rows are removed when a graph_files row is replaced.
+    //
+    // #624-P1 targeted migration: an existing DB may still hold the OLD graph_files
+    // (entry_id PK). SQLite can't ALTER a primary key, so we RENAME the 3 graph
+    // tables aside (→ *_legacy) here — ONLY the graph tables, never the index/
+    // embeddings — then the CREATE block below builds the new shape, then
+    // migrateGraphDataFromLegacy() copies the data across so the graph is PRESERVED
+    // (not re-extracted).
+    migrateGraphFilesSchema(db);
     db.exec(`
     CREATE TABLE IF NOT EXISTS graph_meta (
       stash_root          TEXT PRIMARY KEY,
@@ -331,7 +354,6 @@ function ensureSchema(db, embeddingDim, options) {
     );
 
     CREATE TABLE IF NOT EXISTS graph_files (
-      entry_id          INTEGER PRIMARY KEY REFERENCES entries(id) ON DELETE CASCADE,
       stash_root        TEXT NOT NULL,
       file_path         TEXT NOT NULL,
       file_order        INTEGER NOT NULL,
@@ -341,26 +363,34 @@ function ensureSchema(db, embeddingDim, options) {
       status            TEXT NOT NULL DEFAULT 'extracted',
       reason            TEXT,
       extraction_run_id TEXT,
-      UNIQUE(stash_root, file_path)
+      PRIMARY KEY (stash_root, file_path, body_hash)
     );
 
+    CREATE UNIQUE INDEX IF NOT EXISTS idx_graph_files_path
+      ON graph_files(stash_root, file_path);
+
     CREATE INDEX IF NOT EXISTS idx_graph_files_stash_order
       ON graph_files(stash_root, file_order);
 
     CREATE TABLE IF NOT EXISTS graph_file_entities (
-      entry_id     INTEGER NOT NULL REFERENCES graph_files(entry_id) ON DELETE CASCADE,
-      entity_order INTEGER NOT NULL,
       stash_root   TEXT NOT NULL,
+      file_path    TEXT NOT NULL,
+      body_hash    TEXT NOT NULL,
+      entity_order INTEGER NOT NULL,
       entity_norm  TEXT NOT NULL,
       entity       TEXT NOT NULL,
-      PRIMARY KEY (entry_id, entity_order)
+      PRIMARY KEY (stash_root, file_path, body_hash, entity_order),
+      FOREIGN KEY (stash_root, file_path, body_hash)
+        REFERENCES graph_files(stash_root, file_path, body_hash) ON DELETE CASCADE
     );
 
     CREATE INDEX IF NOT EXISTS idx_graph_file_entities_entity_norm
       ON graph_file_entities(stash_root, entity_norm);
 
     CREATE TABLE IF NOT EXISTS graph_file_relations (
-      entry_id       INTEGER NOT NULL REFERENCES graph_files(entry_id) ON DELETE CASCADE,
+      stash_root     TEXT NOT NULL,
+      file_path      TEXT NOT NULL,
+      body_hash      TEXT NOT NULL,
       relation_order INTEGER NOT NULL,
       from_entity_norm TEXT NOT NULL,
       from_entity    TEXT NOT NULL,
@@ -368,9 +398,31 @@ function ensureSchema(db, embeddingDim, options) {
       to_entity      TEXT NOT NULL,
       relation_type  TEXT,
       confidence     REAL,
-      PRIMARY KEY (entry_id, relation_order)
+      PRIMARY KEY (stash_root, file_path, body_hash, relation_order),
+      FOREIGN KEY (stash_root, file_path, body_hash)
+        REFERENCES graph_files(stash_root, file_path, body_hash) ON DELETE CASCADE
+    );
+
+    -- #624-P3: lazy graph-extraction queue. Standalone table (NO FK to
+    -- graph_files — a queued file by definition has no graph row yet).
+    -- Idempotent on (stash_root, file_path); drained highest-priority-first.
+    -- CREATE TABLE IF NOT EXISTS is the forward migration (no DB_VERSION bump).
+    CREATE TABLE IF NOT EXISTS graph_extraction_queue (
+      stash_root TEXT NOT NULL,
+      file_path  TEXT NOT NULL,
+      body_hash  TEXT NOT NULL,
+      queued_at  TEXT NOT NULL DEFAULT (datetime('now')),
+      priority   INTEGER NOT NULL DEFAULT 0,
+      PRIMARY KEY (stash_root, file_path)
     );
+
+    CREATE INDEX IF NOT EXISTS idx_graph_extraction_queue_drain
+      ON graph_extraction_queue(stash_root, priority DESC, queued_at);
   `);
+    // #624-P1 migration step 2: copy any renamed-aside legacy graph data into the
+    // new-shape tables (just created above), then drop the legacy tables. No-op
+    // unless migrateGraphFilesSchema renamed a legacy graph_files this open.
+    migrateGraphDataFromLegacy(db);
     // FTS-dirty queue. Created here (not lazily on first upsert) so the
     // per-entry write path doesn't issue a CREATE TABLE IF NOT EXISTS on
     // every call — that DDL would fire thousands of times during a full
@@ -399,17 +451,10 @@ function ensureSchema(db, embeddingDim, options) {
         if (dimExplicit) {
             const storedDim = getMeta(db, "embeddingDim");
             if (storedDim && storedDim !== String(embeddingDim)) {
-                // Re-embedding the whole stash is expensive (LLM API calls + cache
-                // misses), so snapshot the data dir before we drop the vec table and
-                // wipe `embeddings`. This is the SAME hook the version-upgrade path
-                // uses earlier in this function, just gated on embedding-dim mismatch
-                // and tagged so operators can tell the two backup kinds apart.
-                backupBeforeEmbeddingDimChange(options?.dataDir, storedDim, String(embeddingDim));
-                bestEffort(() => db.exec("DROP TABLE IF EXISTS entries_vec"), "drop entries_vec on dim change");
-                // Delete stale BLOB embeddings so they don't produce silently wrong
-                // similarity scores against the new-dimension vec table.
-                bestEffort(() => db.exec("DELETE FROM embeddings"), "delete stale embeddings on dim change");
-                setMeta(db, "hasEmbeddings", "0");
+                // Stored vectors are incompatible with the new dimension. Drop the vec
+                // table so the block below recreates it at the new width; the BLOB rows
+                // go too. Regenerable from markdown — re-embedded by the next index.
+                purgeEmbeddings(db, { dropVecTable: true });
             }
         }
         const vecExists = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='entries_vec'").get();
@@ -436,9 +481,8 @@ function ensureSchema(db, embeddingDim, options) {
         if (dimExplicit) {
             const storedDim = getMeta(db, "embeddingDim");
             if (storedDim && storedDim !== String(embeddingDim)) {
-                backupBeforeEmbeddingDimChange(options?.dataDir, storedDim, String(embeddingDim));
-                bestEffort(() => db.exec("DELETE FROM embeddings"), "delete embeddings on explicit dim change");
-                setMeta(db, "hasEmbeddings", "0");
+                // JS-fallback path: no vec table, just clear the stale BLOB vectors.
+                purgeEmbeddings(db);
             }
             setMeta(db, "embeddingDim", String(embeddingDim));
         }
@@ -446,157 +490,28 @@ function ensureSchema(db, embeddingDim, options) {
     // Usage telemetry table
     ensureUsageEventsSchema(db);
     // Registry index cache table — caches remote registry index documents so
-    // `akm search` does not hit the network on every invocation. The DDL is
-    // defined in state-db.ts and shared here to avoid duplication.
+    // `akm search` does not hit the network on every invocation.
     db.exec(REGISTRY_INDEX_CACHE_DDL);
-    // Restore usage_events backed up by the version-upgrade path above.
-    restoreUsageEventsBackup(db, usageBackup);
-}
-/**
- * Detect a stored DB version that differs from {@link DB_VERSION}, drop the
- * old schema, and return a backup of the previous `usage_events` rows so the
- * rest of `ensureSchema()` can restore them once the new table exists.
- *
- * Returns an empty array when no upgrade is needed or when the previous
- * `usage_events` table is unreadable.
- */
-function handleVersionUpgrade(db) {
-    const storedVersion = getMeta(db, "version");
-    // BUG-L4: distinguish "missing" (undefined) from "present but empty" — both
-    // were previously coerced through `!storedVersion` and treated as "no
-    // upgrade needed", which caused fresh databases (with no version row) to
-    // skip the upgrade path correctly, but also caused the upgrade path to be
-    // taken when a corrupted/empty version string was persisted. The current
-    // tables get dropped only when the stored version exists AND differs from
-    // DB_VERSION; missing or empty version means a fresh DB and no upgrade.
-    if (storedVersion === undefined || storedVersion === "" || storedVersion === String(DB_VERSION))
-        return [];
-    let usageBackup = [];
-    bestEffort(() => {
-        usageBackup = db.prepare("SELECT * FROM usage_events").all();
-    }, "usage_events table may not exist in older versions");
-    db.exec("DROP TABLE IF EXISTS utility_scores");
-    db.exec("DROP TABLE IF EXISTS utility_scores_scoped");
-    db.exec("DROP INDEX IF EXISTS idx_utility_scores_scoped_entry_id");
-    db.exec("DROP TABLE IF EXISTS usage_events");
-    db.exec("DROP TABLE IF EXISTS embeddings");
-    db.exec("DROP TABLE IF EXISTS entries_vec");
-    db.exec("DROP TABLE IF EXISTS entries_fts");
-    db.exec("DROP TABLE IF EXISTS index_dir_state");
-    db.exec("DROP TABLE IF EXISTS llm_enrichment_cache");
-    db.exec("DROP INDEX IF EXISTS idx_llm_cache_updated");
-    db.exec("DROP TABLE IF EXISTS graph_file_relations");
-    db.exec("DROP TABLE IF EXISTS graph_file_entities");
-    db.exec("DROP TABLE IF EXISTS graph_files");
-    db.exec("DROP TABLE IF EXISTS graph_meta");
-    db.exec("DROP TABLE IF EXISTS graph_relations");
-    db.exec("DROP TABLE IF EXISTS graph_entities");
-    db.exec("DROP TABLE IF EXISTS graph_nodes");
-    db.exec("DROP TABLE IF EXISTS graph_stashes");
-    db.exec("DROP INDEX IF EXISTS idx_entries_dir");
-    db.exec("DROP INDEX IF EXISTS idx_entries_type");
-    db.exec("DROP TABLE IF EXISTS entries");
-    db.exec("DELETE FROM index_meta");
-    warn("[akm] Index rebuilt due to version upgrade. Run 'akm index' to repopulate.");
-    return usageBackup;
 }
 /**
- * Snapshot the data directory before the embedding-dimension drop path wipes
- * `embeddings` and recreates `entries_vec`. Re-embedding a real-world stash
- * is expensive (LLM calls + cache misses), so we capture the pre-drop state
- * here using the same MVP backup helper the version-upgrade hook uses
- * earlier in {@link ensureSchema}.
- *
- * The backup is tagged with the `embedding-dim-change` reason so it lands in
- * `<dataDir>/backups/<timestamp>-embedding-dim-change/` instead of the
- * version-upgrade-flavored `<timestamp>-pre-v<N>/` directory. Restoration
- * works identically via `scripts/migrations/restore-data-dir.sh`.
+ * Purge stored embeddings (BLOB rows in `embeddings`, plus the `entries_vec`
+ * virtual table) and mark the index as embedding-free. The single place that
+ * invalidates embeddings — used on a dimension change, a model/provider change,
+ * and a full rebuild.
  *
- * Failures are non-fatal — they downgrade to a warning and the destructive
- * ops run anyway, matching the version-upgrade hook's behavior so a broken
- * backup cannot brick a binary that bumped the configured dim. Likewise,
- * `AKM_DB_BACKUP=0` opts out via the same path.
- */
-function backupBeforeEmbeddingDimChange(dataDir, fromDim, toDim) {
-    if (!dataDir)
-        return;
-    try {
-        const result = backupDataDir({
-            dataDir,
-            // The DB version isn't changing here — pass the current DB_VERSION for
-            // both source and target so the metadata sidecar still records the
-            // running binary's version for forensic context.
-            sourceVersion: DB_VERSION,
-            targetVersion: DB_VERSION,
-            reason: EMBEDDING_DIM_CHANGE_REASON,
-            env: process.env,
-        });
-        if (result) {
-            warn("[akm] embedding dimension changed %s→%s; data directory backed up to %s; embeddings will be regenerated", fromDim, toDim, result.path);
-        }
-    }
-    catch (err) {
-        // Defensive — backupDataDir already swallows most errors, but if it
-        // throws for an unexpected reason we must still proceed with the drop
-        // so the user isn't locked out of their binary on a changed dim.
-        warn("[akm] pre-embedding-dim-change data dir backup raised an unexpected error — %s; embeddings will be regenerated without a snapshot", err instanceof Error ? err.message : String(err));
-    }
-}
-/**
- * Re-insert backed-up `usage_events` rows into the freshly-created table.
+ * No backup: embeddings are a derived cache, fully regenerable from the markdown
+ * by the next `akm index`. (Recovery model decided 2026-06-25.)
  *
- * Wrapped in an outer try/catch because schema changes across versions may
- * make the backup incompatible with the new table definition; in that case
- * the backup is discarded silently rather than blocking startup.
+ * `dropVecTable: true` DROPs `entries_vec` — used on a DIMENSION change, where
+ * the vec0 table must be recreated at the new width by the caller. The default
+ * clears its rows in place (same dimension, stale vectors).
  */
-function restoreUsageEventsBackup(db, backup) {
-    if (backup.length === 0)
-        return;
-    try {
-        // BUG-H4: introspect the *target* table's columns rather than relying on
-        // `row[0]`'s keys. The backup may carry columns the new schema dropped,
-        // and the new schema may have NOT-NULL columns without DEFAULT that the
-        // old backup never carried. Project the backup onto the intersection so
-        // we don't silently lose every row to per-row INSERT errors, and warn
-        // once if any backup column was dropped from the new schema.
-        const targetCols = db.prepare("PRAGMA table_info(usage_events)").all().map((c) => c.name);
-        if (targetCols.length === 0) {
-            warn("[db] restoreUsageEventsBackup: usage_events table missing — discarding %d backup row(s)", backup.length);
-            return;
-        }
-        const targetSet = new Set(targetCols);
-        const backupCols = Object.keys(backup[0] ?? {});
-        const projectedCols = backupCols.filter((c) => targetSet.has(c));
-        const droppedCols = backupCols.filter((c) => !targetSet.has(c));
-        if (projectedCols.length === 0) {
-            warn("[db] restoreUsageEventsBackup: no overlapping columns between backup and current schema — discarding %d row(s); dropped: %s", backup.length, droppedCols.join(", ") || "(none)");
-            return;
-        }
-        if (droppedCols.length > 0) {
-            warn("[db] restoreUsageEventsBackup: dropping columns no longer in usage_events schema: %s", droppedCols.join(", "));
-        }
-        let restored = 0;
-        let failed = 0;
-        db.transaction(() => {
-            const placeholders = projectedCols.map(() => "?").join(", ");
-            const insert = db.prepare(`INSERT INTO usage_events (${projectedCols.join(", ")}) VALUES (${placeholders})`);
-            for (const row of backup) {
-                try {
-                    insert.run(...projectedCols.map((c) => row[c]));
-                    restored++;
-                }
-                catch {
-                    failed++;
-                }
-            }
-        })();
-        if (failed > 0) {
-            warn("[db] restoreUsageEventsBackup: restored %d row(s); skipped %d incompatible row(s)", restored, failed);
-        }
-    }
-    catch (err) {
-        warn("[db] restoreUsageEventsBackup: discarded %d backup row(s) — %s", backup.length, err instanceof Error ? err.message : String(err));
+export function purgeEmbeddings(db, opts) {
+    bestEffort(() => db.exec("DELETE FROM embeddings"), "purge embeddings");
+    if (isVecAvailable(db)) {
+        bestEffort(() => db.exec(opts?.dropVecTable ? "DROP TABLE IF EXISTS entries_vec" : "DELETE FROM entries_vec"), "purge entries_vec");
     }
+    setMeta(db, "hasEmbeddings", "0");
 }
 // ── Meta helpers ────────────────────────────────────────────────────────────
 export function getMeta(db, key) {
@@ -700,9 +615,8 @@ function getUpsertStmts(db) {
  *
  * Ensures the `entries.derived_from` column + index exist on the open
  * connection. Called from `ensureSchema()` after the entries CREATE so that
- * legacy databases (created against a pre-v17 binary but reopened without
- * triggering `handleVersionUpgrade()`) still gain the new column without
- * data loss. Idempotent: a `PRAGMA table_info` lookup gates the ALTER.
+ * legacy databases (created against a pre-v17 binary) still gain the new column
+ * without data loss. Idempotent: a `PRAGMA table_info` lookup gates the ALTER.
  */
 function ensureDerivedFromColumn(db) {
     bestEffort(() => {
@@ -715,6 +629,110 @@ function ensureDerivedFromColumn(db) {
         db.exec("CREATE INDEX IF NOT EXISTS idx_entries_derived_from ON entries(derived_from)");
     }, "entries table may not exist on a brand-new DB before CREATE — caller is responsible");
 }
+/**
+ * Returns true when a table exists in the current database.
+ */
+function tableExists(db, name) {
+    const row = db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=? LIMIT 1").get(name);
+    return row !== undefined && row !== null;
+}
+/**
+ * #624-P1 targeted graph-schema migration — STEP 1 of 2 (rename).
+ *
+ * graph_files was re-keyed from `entry_id INTEGER PRIMARY KEY REFERENCES
+ * entries(id)` to a self-contained `(stash_root, file_path, body_hash)` PK.
+ * SQLite cannot ALTER a primary key, so an existing DB carrying the OLD shape
+ * has its 3 graph tables RENAMED to `*_legacy` here; ensureSchema's CREATE block
+ * then builds the new-shape tables, and {@link migrateGraphDataFromLegacy} COPIES
+ * the data across before dropping the legacy tables. The graph is preserved —
+ * NOT re-extracted (re-extraction is ~19s/file of LLM work).
+ *
+ * Crucially this is GRAPH-SCOPED: it touches ONLY the graph tables, never the
+ * index / embeddings / enrichment cache. So users keep their (expensive)
+ * embeddings instead of being forced into a full re-embed by a DB_VERSION bump.
+ *
+ * Detection: the old schema has an `entry_id` column on graph_files. Fresh DBs
+ * (no graph_files yet) and already-migrated DBs (no entry_id column) are no-ops.
+ * Idempotent.
+ */
+function migrateGraphFilesSchema(db) {
+    bestEffort(() => {
+        const cols = db.prepare("PRAGMA table_info(graph_files)").all();
+        const isLegacyShape = cols.some((c) => c.name === "entry_id");
+        if (!isLegacyShape)
+            return;
+        // A previous interrupted migration may have left *_legacy behind — drop those
+        // husks first so the rename below doesn't collide.
+        db.exec("DROP TABLE IF EXISTS graph_file_relations_legacy");
+        db.exec("DROP TABLE IF EXISTS graph_file_entities_legacy");
+        db.exec("DROP TABLE IF EXISTS graph_files_legacy");
+        // Rename the 3 entry_id-keyed tables aside. graph_meta is unchanged (stash_root
+        // key) so it is left in place. ALTER … RENAME auto-updates child FK refs in
+        // SQLite ≥3.25, which is fine — the legacy children are dropped after the copy.
+        db.exec("ALTER TABLE graph_files RENAME TO graph_files_legacy");
+        if (tableExists(db, "graph_file_entities")) {
+            db.exec("ALTER TABLE graph_file_entities RENAME TO graph_file_entities_legacy");
+        }
+        if (tableExists(db, "graph_file_relations")) {
+            db.exec("ALTER TABLE graph_file_relations RENAME TO graph_file_relations_legacy");
+        }
+    }, "graph_files may not exist on a brand-new DB before CREATE — caller is responsible");
+}
+/**
+ * #624-P1 targeted graph-schema migration — STEP 2 of 2 (copy + drop legacy).
+ *
+ * Runs AFTER the graph CREATE TABLE block, so the new-shape tables exist. Copies
+ * every legacy row into the re-keyed tables — the old tables already carry
+ * (stash_root, file_path, body_hash) next to entry_id, so the projection is a
+ * straight column copy (children JOIN back to graph_files_legacy to resolve the
+ * composite key from their entry_id). Then drops the `*_legacy` tables.
+ *
+ * Best-effort: a copy failure (e.g. a pre-body_hash legacy schema) is tolerated,
+ * and the legacy tables are dropped regardless so they never linger. Rows whose
+ * body_hash is null/empty can't form the new PK and are skipped (they re-extract).
+ */
+function migrateGraphDataFromLegacy(db) {
+    if (!tableExists(db, "graph_files_legacy"))
+        return;
+    let migratedFiles = 0;
+    bestEffort(() => {
+        db.transaction(() => {
+            const res = db
+                .prepare(`INSERT OR IGNORE INTO graph_files
+             (stash_root, file_path, body_hash, file_order, file_type, confidence, status, reason, extraction_run_id)
+           SELECT stash_root, file_path, body_hash, file_order, file_type, confidence, status, reason, extraction_run_id
+             FROM graph_files_legacy
+            WHERE body_hash IS NOT NULL AND body_hash != ''`)
+                .run();
+            migratedFiles = Number(res.changes);
+            if (tableExists(db, "graph_file_entities_legacy")) {
+                db.exec(`INSERT OR IGNORE INTO graph_file_entities
+             (stash_root, file_path, body_hash, entity_order, entity_norm, entity)
+           SELECT gf.stash_root, gf.file_path, gf.body_hash, e.entity_order, e.entity_norm, e.entity
+             FROM graph_file_entities_legacy e
+             JOIN graph_files_legacy gf ON gf.entry_id = e.entry_id
+            WHERE gf.body_hash IS NOT NULL AND gf.body_hash != ''`);
+            }
+            if (tableExists(db, "graph_file_relations_legacy")) {
+                db.exec(`INSERT OR IGNORE INTO graph_file_relations
+             (stash_root, file_path, body_hash, relation_order, from_entity_norm, from_entity, to_entity_norm, to_entity, relation_type, confidence)
+           SELECT gf.stash_root, gf.file_path, gf.body_hash, r.relation_order, r.from_entity_norm, r.from_entity, r.to_entity_norm, r.to_entity, r.relation_type, r.confidence
+             FROM graph_file_relations_legacy r
+             JOIN graph_files_legacy gf ON gf.entry_id = r.entry_id
+            WHERE gf.body_hash IS NOT NULL AND gf.body_hash != ''`);
+            }
+        })();
+    }, "graph data migration is best-effort; legacy tables are dropped regardless below");
+    // Always drop the legacy tables (children first), migrated or not.
+    bestEffort(() => {
+        db.exec("DROP TABLE IF EXISTS graph_file_relations_legacy");
+        db.exec("DROP TABLE IF EXISTS graph_file_entities_legacy");
+        db.exec("DROP TABLE IF EXISTS graph_files_legacy");
+    }, "drop legacy graph tables after migration");
+    if (migratedFiles > 0) {
+        warn(`[akm] graph index re-keyed (#624): migrated ${migratedFiles} extracted file(s) to the new schema — no re-extraction needed. Index + embeddings untouched.`);
+    }
+}
 /**
  * Phase 5A / Advantage D5: look up the derived-memory child row whose
  * `derived_from` column matches `parentRef` (e.g. `"memory:claude-prefs"`).
@@ -839,6 +857,38 @@ function deleteRelatedRows(db, ids) {
         // Clean up usage events before deleting entries
         bestEffort(() => db.prepare(`DELETE FROM usage_events WHERE entry_id IN (${placeholders})`).run(...chunk), "delete usage_events for entries");
     }
+    // #624-P1: graph_files is NO LONGER keyed on entries.id, so deleting an
+    // entries row must NOT wipe the extracted graph (that is the whole point —
+    // the graph survives a reindex when body_hash is unchanged). We therefore do
+    // NOT delete graph_files here. We DO, however, recompute graph_meta counts
+    // for the stash roots touched by the deleted entries so the summary numbers
+    // stay consistent with the live child rows (the counts are derived, and the
+    // entries delete may have changed which files are considered/indexed).
+    //
+    // Resolve the affected stash roots from the entries rows BEFORE deletion.
+    const affectedStashRoots = new Set();
+    for (let i = 0; i < numericIds.length; i += SQLITE_CHUNK_SIZE) {
+        const chunk = numericIds.slice(i, i + SQLITE_CHUNK_SIZE);
+        const placeholders = chunk.map(() => "?").join(",");
+        bestEffort(() => {
+            const rows = db
+                .prepare(`SELECT DISTINCT stash_dir FROM entries WHERE id IN (${placeholders})`)
+                .all(...chunk);
+            for (const row of rows) {
+                if (row.stash_dir)
+                    affectedStashRoots.add(row.stash_dir);
+            }
+        }, "resolve stash roots for graph_meta recompute");
+    }
+    for (const stashRoot of affectedStashRoots) {
+        bestEffort(() => db
+            .prepare(`UPDATE graph_meta
+             SET extracted_files = (SELECT COUNT(*) FROM graph_files WHERE stash_root = ?),
+                 entity_count    = (SELECT COUNT(*) FROM graph_file_entities WHERE stash_root = ?),
+                 relation_count  = (SELECT COUNT(*) FROM graph_file_relations WHERE stash_root = ?)
+             WHERE stash_root = ?`)
+            .run(stashRoot, stashRoot, stashRoot, stashRoot), "sync graph_meta counts after entries delete");
+    }
 }
 /**
  * Delete entries by their primary key IDs, along with all related rows
@@ -1048,12 +1098,12 @@ function searchBlobVec(db, queryEmbedding, k) {
     }
 }
 // ── FTS5 search ─────────────────────────────────────────────────────────────
-export function searchFts(db, query, limit, entryType) {
+export function searchFts(db, query, limit, entryType, excludeTypes) {
     const ftsQuery = sanitizeFtsQuery(query);
     if (!ftsQuery)
         return [];
     // Try the exact AND query first
-    const exactResults = runFtsQuery(db, ftsQuery, limit, entryType);
+    const exactResults = runFtsQuery(db, ftsQuery, limit, entryType, excludeTypes);
     if (exactResults.length > 0)
         return exactResults;
     // Exact match returned zero results — try prefix fallback.
@@ -1063,7 +1113,7 @@ export function searchFts(db, query, limit, entryType) {
     const prefixQuery = buildPrefixQuery(ftsQuery);
     if (!prefixQuery)
         return [];
-    return runFtsQuery(db, prefixQuery, limit, entryType);
+    return runFtsQuery(db, prefixQuery, limit, entryType, excludeTypes);
 }
 /**
  * Build a prefix query from an FTS5 query string by appending `*` to each
@@ -1086,9 +1136,14 @@ function buildPrefixQuery(ftsQuery) {
         return null;
     return prefixTokens.join(" ");
 }
-function runFtsQuery(db, ftsQuery, limit, entryType) {
+function runFtsQuery(db, ftsQuery, limit, entryType, excludeTypes) {
     let sql;
     let params;
+    // #627 — exclude-type clause. Only applies on the untyped ('any') path; an
+    // explicit include filter (entryType) already narrows to a single type, so
+    // exclusion is redundant there. An empty list skips the clause entirely
+    // (never emit `NOT IN ()`, which is a SQL error / always-false).
+    const excludes = excludeTypes && excludeTypes.length > 0 ? excludeTypes : [];
     // Join on integer entry_id directly (no CAST needed; we store integer)
     // Use bm25() with per-column weights: entry_id(0), name(10), description(5), tags(3), hints(2), content(1)
     if (entryType && entryType !== "any") {
@@ -1105,16 +1160,19 @@ function runFtsQuery(db, ftsQuery, limit, entryType) {
         params = [ftsQuery, entryType, limit];
     }
     else {
+        const excludeClause = excludes.length > 0 ? `AND e.entry_type NOT IN (${excludes.map(() => "?").join(", ")})` : "";
         sql = `
       SELECT e.id, e.file_path AS filePath, e.entry_json, e.search_text AS searchText,
              bm25(entries_fts, 0, 10.0, 5.0, 3.0, 2.0, 1.0) AS bm25Score
       FROM entries_fts f
       JOIN entries e ON e.id = f.entry_id
       WHERE entries_fts MATCH ?
+        ${excludeClause}
       ORDER BY bm25Score, e.id ASC
       LIMIT ?
     `;
-        params = [ftsQuery, limit];
+        // Param order: MATCH, then the NOT IN values, then LIMIT.
+        params = [ftsQuery, ...excludes, limit];
     }
     try {
         const rows = db.prepare(sql).all(...params);
@@ -1183,14 +1241,21 @@ function parseEntryRows(rows, context) {
     }
     return entries;
 }
-export function getAllEntries(db, entryType) {
+export function getAllEntries(db, entryType, excludeTypes) {
     let sql;
     let params;
+    // #627 — exclude-type clause applies only on the untyped ('any') path. Empty
+    // list skips the clause (never `NOT IN ()`).
+    const excludes = excludeTypes && excludeTypes.length > 0 ? excludeTypes : [];
     if (entryType && entryType !== "any") {
         sql =
             "SELECT id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text FROM entries WHERE entry_type = ?";
         params = [entryType];
     }
+    else if (excludes.length > 0) {
+        sql = `SELECT id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text FROM entries WHERE entry_type NOT IN (${excludes.map(() => "?").join(", ")})`;
+        params = [...excludes];
+    }
     else {
         sql = "SELECT id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text FROM entries";
         params = [];
@@ -1198,6 +1263,47 @@ export function getAllEntries(db, entryType) {
     const rows = db.prepare(sql).all(...params);
     return parseEntryRows(rows, "getAllEntries");
 }
+/**
+ * #609 — read graph entities (normalized) for a set of entry ids. Used by the
+ * recombine pass to cluster memories by shared graph entity ("graph"
+ * relatedness source). Returns a map of `entry_id -> entity_norm[]`. Entries
+ * with no graph entities (graph extraction has not run, or the file produced
+ * no entities) are simply absent from the map — callers must fail open
+ * (fall back to tag relatedness) when the map is empty.
+ */
+export function getEntitiesByEntryIds(db, entryIds) {
+    const result = new Map();
+    if (entryIds.length === 0)
+        return result;
+    // #624-P1: graph_file_entities no longer carries entry_id. Re-derive the
+    // entry_id -> entity_norm[] contract by JOINing through entries on
+    // (stash_dir, file_path) -> graph_files. Chunk the IN(?) list because the
+    // recombine pass can pass 10k+ entry ids (well over the SQLite param limit).
+    for (let i = 0; i < entryIds.length; i += SQLITE_CHUNK_SIZE) {
+        const chunk = entryIds.slice(i, i + SQLITE_CHUNK_SIZE);
+        const placeholders = chunk.map(() => "?").join(", ");
+        const rows = db
+            .prepare(`SELECT e.id AS entry_id, gfe.entity_norm AS entity_norm
+           FROM entries e
+           JOIN graph_files gf
+             ON gf.stash_root = e.stash_dir AND gf.file_path = e.file_path
+           JOIN graph_file_entities gfe
+             ON gfe.stash_root = gf.stash_root
+            AND gfe.file_path = gf.file_path
+            AND gfe.body_hash = gf.body_hash
+          WHERE e.id IN (${placeholders})
+          ORDER BY e.id, gfe.entity_order`)
+            .all(...chunk);
+        for (const row of rows) {
+            const list = result.get(row.entry_id);
+            if (list)
+                list.push(row.entity_norm);
+            else
+                result.set(row.entry_id, [row.entity_norm]);
+        }
+    }
+    return result;
+}
 export function findEntryIdByRef(db, ref) {
     const parsed = parseAssetRef(ref);
     const nameVariants = [parsed.name];
@@ -1262,6 +1368,19 @@ export function getEntryIdByFilePath(db, filePath) {
     const row = db.prepare("SELECT id FROM entries WHERE file_path = ? LIMIT 1").get(filePath);
     return row?.id;
 }
+/**
+ * Set of every non-empty `entries.file_path` currently indexed (across all
+ * stashes/sources). Used by staleness detection to spot files that exist on
+ * disk but were never indexed — a clock-independent signal for newly-added
+ * assets that an mtime-vs-builtAt comparison can miss when the two clocks
+ * (filesystem vs wall-clock) are skewed within the same millisecond.
+ */
+export function getIndexedFilePaths(db) {
+    const rows = db
+        .prepare("SELECT DISTINCT file_path FROM entries WHERE file_path IS NOT NULL AND file_path <> ''")
+        .all();
+    return new Set(rows.map((r) => r.file_path));
+}
 /**
  * Resolve a single `entries.file_path` by primary key, or `undefined` if no
  * row matches.
@@ -1477,25 +1596,98 @@ export function computeBodyHash(body) {
     return sha256Hex(body);
 }
 /**
- * Count search and show events for the given entry refs.
- * Returns a Map<ref, count> with only refs that have at least one event.
- * Used by the improve loop to find high-retrieval assets without feedback.
+ * Reduce a ref to its bare `type:name` form, dropping any `origin//` prefix.
+ *
+ * usage_events store entry_ref inconsistently: search/show writers persist
+ * whatever ref the result carried, which is sometimes stash-prefixed
+ * (`origin//type:name`) and sometimes bare (`type:name`). Retrieval counting
+ * keys on the bare form so both spellings of the same asset collapse together.
+ *
+ * Returns the bare form, or the original string when it cannot be parsed (best
+ * effort — never throws so a malformed stored ref can't break counting).
+ */
+function bareRef(ref) {
+    try {
+        const parsed = parseAssetRef(ref);
+        return `${parsed.type}:${parsed.name}`;
+    }
+    catch {
+        return ref;
+    }
+}
+/**
+ * Count retrieval events for the given entry refs.
+ *
+ * Counts `search`, `show`, and `curate` usage events. Returns a
+ * Map<inputRef, count> keyed by the *input* ref strings (only those with at
+ * least one matching event appear). Used by the improve loop to find
+ * high-retrieval assets without feedback.
+ *
+ * Matching is normalization-aware: each stored `entry_ref` is reduced to its
+ * bare `type:name` form before comparison, so a stash-prefixed stored ref
+ * (`origin//type:name`) still matches a bare input ref (`type:name`) and vice
+ * versa. Previously the raw `entry_ref IN (...)` comparison silently dropped
+ * roughly half the signal whenever the two spellings disagreed.
+ *
+ * `curate` events are included: their per-item rows are written with
+ * entry_ref populated (see logCurateEvent), so curation is a real retrieval
+ * signal here. Legacy summary-only curate rows with a NULL entry_ref simply
+ * contribute nothing.
  */
 export function getRetrievalCounts(db, refs) {
     if (refs.length === 0)
         return new Map();
-    const result = new Map();
+    // Map each distinct bare form back to the input ref(s) that produced it so we
+    // can re-key DB results (grouped by bare form) onto the caller's ref strings.
+    const bareToInputs = new Map();
+    for (const ref of refs) {
+        const bare = bareRef(ref);
+        const existing = bareToInputs.get(bare);
+        if (existing)
+            existing.push(ref);
+        else
+            bareToInputs.set(bare, [ref]);
+    }
+    const bareForms = [...bareToInputs.keys()];
+    // Accumulate counts per bare form across chunks before re-keying.
+    const countsByBare = new Map();
     // Chunk to stay within SQLITE_MAX_VARIABLE_NUMBER (same pattern as getUtilityScoresByIds).
-    for (let i = 0; i < refs.length; i += SQLITE_CHUNK_SIZE) {
-        const chunk = refs.slice(i, i + SQLITE_CHUNK_SIZE);
+    for (let i = 0; i < bareForms.length; i += SQLITE_CHUNK_SIZE) {
+        const chunk = bareForms.slice(i, i + SQLITE_CHUNK_SIZE);
         const placeholders = chunk.map(() => "?").join(", ");
+        // Normalize the stored entry_ref to its bare form inside SQL by stripping
+        // everything up to and including the last `//` separator. SQLite has no
+        // rfind, but stored origins never themselves contain `//`, so a stash ref
+        // has exactly one `//` and `substr(... instr ...)` is exact; bare refs have
+        // no `//` and pass through unchanged.
         const rows = db
-            .prepare(`SELECT entry_ref, COUNT(*) AS cnt FROM usage_events
-         WHERE event_type IN ('search','show') AND entry_ref IN (${placeholders})
-         GROUP BY entry_ref`)
+            .prepare(`SELECT
+           CASE
+             WHEN instr(entry_ref, '//') > 0
+               THEN substr(entry_ref, instr(entry_ref, '//') + 2)
+             ELSE entry_ref
+           END AS bare_ref,
+           COUNT(*) AS cnt
+         FROM usage_events
+         WHERE event_type IN ('search','show','curate')
+           AND entry_ref IS NOT NULL
+           AND CASE
+                 WHEN instr(entry_ref, '//') > 0
+                   THEN substr(entry_ref, instr(entry_ref, '//') + 2)
+                 ELSE entry_ref
+               END IN (${placeholders})
+         GROUP BY bare_ref`)
             .all(...chunk);
-        for (const r of rows)
-            result.set(r.entry_ref, r.cnt);
+        for (const r of rows) {
+            countsByBare.set(r.bare_ref, (countsByBare.get(r.bare_ref) ?? 0) + r.cnt);
+        }
+    }
+    // Re-key bare-form counts onto every input ref that maps to that bare form.
+    const result = new Map();
+    for (const [bare, count] of countsByBare) {
+        for (const input of bareToInputs.get(bare) ?? []) {
+            result.set(input, count);
+        }
     }
     return result;
 }