akm-cli 0.9.0-beta.6 → 0.9.0-rc.0

package/dist/indexer/db/db.js

@@ -1,47 +1,39 @@
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
-import fs from "node:fs";
 import { createRequire } from "node:module";
 import path from "node:path";
 import { parseAssetRef } from "../../core/asset/asset-ref.js";
 import { bestEffort } from "../../core/best-effort.js";
 import { getDbPath } from "../../core/paths.js";
-import { REGISTRY_INDEX_CACHE_DDL } from "../../core/state-db.js";
 import { warn } from "../../core/warn.js";
 import { cosineSimilarity } from "../../llm/embedders/types.js";
 import { sha256Hex } from "../../runtime.js";
-import { openDatabase as openSqlite } from "../../storage/database.js";
+import { openManagedDatabase } from "../../storage/managed-db.js";
+import { computeNextUtility, HIGH_UTILITY_THRESHOLD, UTILITY_REVIEW_THRESHOLD, } from "../feedback/utility-policy.js";
+import { buildPrefixQuery, sanitizeFtsQuery } from "../search/fts-query.js";
 import { buildSearchFields } from "../search/search-fields.js";
-import { ensureUsageEventsSchema } from "../usage/usage-events.js";
-import { backupDataDir, EMBEDDING_DIM_CHANGE_REASON } from "./db-backup.js";
-// ── Constants ───────────────────────────────────────────────────────────────
-export const DB_VERSION = 17;
-export const EMBEDDING_DIM = 384;
-export const GRAPH_SCHEMA_VERSION = 3;
+import { ENTRY_COLUMNS, rowToIndexedEntry } from "./entry-mapper.js";
+import { ensureSchema } from "./schema.js";
+export { HIGH_UTILITY_THRESHOLD, sanitizeFtsQuery, UTILITY_REVIEW_THRESHOLD };
 // ── Database lifecycle ──────────────────────────────────────────────────────
-export function openDatabase(dbPath, options) {
-    const resolvedPath = dbPath ?? getDbPath();
-    const dir = path.dirname(resolvedPath);
-    if (!fs.existsSync(dir)) {
-        fs.mkdirSync(dir, { recursive: true });
-    }
-    const db = openSqlite(resolvedPath);
-    db.exec("PRAGMA journal_mode = WAL");
-    db.exec("PRAGMA busy_timeout = 30000");
-    db.exec("PRAGMA foreign_keys = ON");
-    // Try to load sqlite-vec extension
-    loadVecExtension(db);
-    // Dim resolution: explicit option wins; otherwise consult the on-disk
-    // config so unparameterised opens (registry providers, graph helpers,
-    // ad-hoc CLI subcommands) honour the operator-declared dimension. Only if
-    // both are absent do we fall through to the no-clobber path, which keeps
-    // ensureSchema from touching `index_meta.embeddingDim` at all.
-    const resolvedDim = options?.embeddingDim ?? resolveConfiguredEmbeddingDim();
-    ensureSchema(db, resolvedDim, { dataDir: dir });
-    // Warn once at init if using JS fallback with many entries
-    warnIfVecMissing(db, { once: true });
-    return db;
+export function openIndexDatabase(dbPath, options) {
+    return openManagedDatabase({
+        path: dbPath ?? getDbPath(),
+        init: (db) => {
+            // Try to load sqlite-vec extension
+            loadVecExtension(db);
+            // Dim resolution: explicit option wins; otherwise consult the on-disk
+            // config so unparameterised opens (registry providers, graph helpers,
+            // ad-hoc CLI subcommands) honour the operator-declared dimension. Only if
+            // both are absent do we fall through to the no-clobber path, which keeps
+            // ensureSchema from touching `index_meta.embeddingDim` at all.
+            const resolvedDim = options?.embeddingDim ?? resolveConfiguredEmbeddingDim();
+            ensureSchema(db, resolvedDim);
+            // Warn once at init if using JS fallback with many entries
+            warnIfVecMissing(db, { once: true });
+        },
+    });
 }
 /**
  * Read the operator-configured embedding dimension from the on-disk config.
@@ -66,15 +58,10 @@ function resolveConfiguredEmbeddingDim() {
     }
 }
 export function openExistingDatabase(dbPath) {
-    const resolvedPath = dbPath ?? getDbPath();
-    const db = openSqlite(resolvedPath);
-    db.exec("PRAGMA journal_mode = WAL");
-    db.exec("PRAGMA busy_timeout = 30000");
-    db.exec("PRAGMA foreign_keys = ON");
     // Existing-DB callers must not mutate schema or embedding metadata on open,
-    // but some paths still need write access to usage_events and other tables.
-    loadVecExtension(db);
-    return db;
+    // but some paths still need write access to usage_events and other tables —
+    // so init only loads the vec extension, it does not run ensureSchema.
+    return openManagedDatabase({ path: dbPath ?? getDbPath(), init: loadVecExtension });
 }
 export function closeDatabase(db) {
     db.close();
@@ -123,480 +110,25 @@ export function warnIfVecMissing(db, { once } = { once: false }) {
         }
     }, "embeddings table may not exist yet during init");
 }
-function ensureSchema(db, embeddingDim, options) {
-    // Create meta table first so we can check version
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS index_meta (
-      key   TEXT PRIMARY KEY,
-      value TEXT NOT NULL
-    );
-  `);
-    // MVP DB-backup hook (0.8.x): when the stored DB version differs from the
-    // running binary's DB_VERSION, snapshot the data directory BEFORE
-    // `handleVersionUpgrade()` drops tables. This is best-effort —
-    // `backupDataDir` returns null on opt-out, missing data dir, low free
-    // space, or copy errors, and we proceed with the upgrade in all cases.
-    // The proper migration framework lands in 0.9.0; until then this lets
-    // operators recover with `scripts/migrations/restore-data-dir.sh`.
-    if (options?.dataDir) {
-        const storedVersionRaw = getMeta(db, "version");
-        const storedVersion = storedVersionRaw !== undefined && storedVersionRaw !== "" ? Number.parseInt(storedVersionRaw, 10) : null;
-        const willUpgrade = storedVersionRaw !== undefined && storedVersionRaw !== "" && storedVersionRaw !== String(DB_VERSION);
-        if (willUpgrade) {
-            try {
-                // Pass env explicitly so tests can override AKM_DB_BACKUP / AKM_DB_BACKUP_RETAIN
-                // without mutating process.env. Production callers default to process.env.
-                const result = backupDataDir({
-                    dataDir: options.dataDir,
-                    sourceVersion: storedVersion !== null && !Number.isNaN(storedVersion) ? storedVersion : null,
-                    targetVersion: DB_VERSION,
-                    env: process.env,
-                });
-                if (result) {
-                    warn("[akm] data directory backed up to %s before v%s→v%d upgrade", result.path, storedVersionRaw, DB_VERSION);
-                }
-            }
-            catch (err) {
-                // Defensive — backupDataDir already swallows most errors, but if it
-                // throws for an unexpected reason we must still proceed with the
-                // upgrade so the user isn't locked out of their binary.
-                warn("[akm] pre-upgrade data dir backup raised an unexpected error — %s; upgrade will proceed without a snapshot", err instanceof Error ? err.message : String(err));
-            }
-        }
-    }
-    // Check stored version — if it differs from DB_VERSION, drop and recreate all tables.
-    // Usage events are preserved across version upgrades so that utility score
-    // history is not silently lost. The backup is captured here and threaded
-    // explicitly to `restoreUsageEventsBackup` below — the previous version
-    // attached `__usageBackup` to the Database instance via a typeless property
-    // injection, which was a source of fragile coupling.
-    const usageBackup = handleVersionUpgrade(db);
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS entries (
-      id          INTEGER PRIMARY KEY AUTOINCREMENT,
-      entry_key   TEXT NOT NULL UNIQUE,
-      dir_path    TEXT NOT NULL,
-      file_path   TEXT NOT NULL,
-      stash_dir   TEXT NOT NULL,
-      entry_json  TEXT NOT NULL,
-      search_text TEXT NOT NULL,
-      entry_type  TEXT NOT NULL,
-      derived_from TEXT
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_entries_dir ON entries(dir_path);
-    CREATE INDEX IF NOT EXISTS idx_entries_type ON entries(entry_type);
-    CREATE INDEX IF NOT EXISTS idx_entries_file_path ON entries(file_path);
-  `);
-    // Phase 5A / DB v17: backfill `derived_from` column + index on databases
-    // that were created at v17 fresh OR carry a partial v17 schema (a DB whose
-    // `index_meta.version` was bumped to 17 but whose `entries` table still
-    // lacks the column — this happens when a previous v17 binary opened a
-    // pre-v17 DB without taking the upgrade path because no version mismatch
-    // was seen at boot). The PRAGMA-then-ALTER guard runs unconditionally so
-    // both fresh and partial schemas converge. The CREATE INDEX for
-    // `derived_from` MUST run after this helper so we never reference a
-    // column that has not yet been added on partial schemas.
-    ensureDerivedFromColumn(db);
-    // Validated WorkflowDocument JSON, one row per indexed workflow entry.
-    // Pure index data — fully rebuilt on each `akm index`. ON DELETE CASCADE
-    // means clearing entries (full rebuild or per-dir delete) drops these too.
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS workflow_documents (
-      entry_id        INTEGER PRIMARY KEY REFERENCES entries(id) ON DELETE CASCADE,
-      schema_version  INTEGER NOT NULL,
-      document_json   TEXT NOT NULL,
-      source_path     TEXT NOT NULL,
-      source_hash     TEXT NOT NULL,
-      updated_at      TEXT NOT NULL
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_workflow_documents_source_path
-      ON workflow_documents(source_path);
-  `);
-    // Set version immediately after table creation so a crash before the end of
-    // ensureSchema() does not leave the database in a versionless state on next open.
-    const versionAfterCreate = getMeta(db, "version");
-    if (!versionAfterCreate) {
-        setMeta(db, "version", String(DB_VERSION));
-    }
-    // BLOB-based embedding storage (always available, no sqlite-vec needed)
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS embeddings (
-      id        INTEGER PRIMARY KEY,
-      embedding BLOB NOT NULL,
-      FOREIGN KEY (id) REFERENCES entries(id)
-    );
-  `);
-    // FTS5 table — multi-column with per-field weighting via bm25()
-    const ftsExists = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='entries_fts'").get();
-    if (!ftsExists) {
-        db.exec(`
-      CREATE VIRTUAL TABLE entries_fts USING fts5(
-        entry_id UNINDEXED,
-        name,
-        description,
-        tags,
-        hints,
-        content,
-        tokenize='porter unicode61'
-      );
-    `);
-    }
-    // Usage events table — created by ensureUsageEventsSchema() at runtime.
-    // Utility scores table (aggregated per-entry utility metrics)
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS utility_scores (
-      entry_id     INTEGER PRIMARY KEY,
-      utility      REAL NOT NULL DEFAULT 0,
-      show_count   INTEGER NOT NULL DEFAULT 0,
-      search_count INTEGER NOT NULL DEFAULT 0,
-      select_rate  REAL NOT NULL DEFAULT 0,
-      last_used_at TEXT,
-      updated_at   TEXT NOT NULL DEFAULT (datetime('now')),
-      FOREIGN KEY (entry_id) REFERENCES entries(id) ON DELETE CASCADE
-    );
-  `);
-    // Per-project scoped utility scores — tracks usage per (entry, cwd-anchor)
-    // so assets useful in project A don't pollute rankings in project B.
-    // The global utility_scores table is preserved as a fallback / cold-start aid.
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS utility_scores_scoped (
-      entry_id     INTEGER NOT NULL,
-      scope_key    TEXT NOT NULL,
-      utility      REAL NOT NULL DEFAULT 0,
-      last_used_at INTEGER NOT NULL,
-      PRIMARY KEY (entry_id, scope_key)
-    );
-    CREATE INDEX IF NOT EXISTS idx_utility_scores_scoped_entry_id
-      ON utility_scores_scoped(entry_id);
-  `);
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS index_dir_state (
-      dir_path          TEXT PRIMARY KEY,
-      file_set_hash     TEXT NOT NULL,
-      file_mtime_max_ms REAL NOT NULL,
-      reason            TEXT NOT NULL,
-      updated_at        TEXT NOT NULL
-    );
-  `);
-    // LLM enrichment result cache. Stores a SHA-256 body hash and the JSON
-    // result for each asset so that subsequent `akm index --enrich` runs can
-    // skip the LLM call when the body hasn't changed. The cache is keyed by
-    // a stable asset_ref string (e.g. the absolute file path for graph/memory
-    // passes, or `entryKey:passId` for the metadata-enhance pass).
-    // Entries are cleaned up when assets are removed or --re-enrich is used.
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS llm_enrichment_cache (
-      asset_ref     TEXT NOT NULL,
-      cache_variant TEXT NOT NULL,
-      body_hash     TEXT NOT NULL,
-      result_json   TEXT NOT NULL,
-      updated_at    INTEGER NOT NULL,
-      PRIMARY KEY (asset_ref, cache_variant)
-    );
-
-     CREATE INDEX IF NOT EXISTS idx_llm_cache_updated
-       ON llm_enrichment_cache(updated_at);
-  `);
-    // Graph extraction tables — schema v2 (entry_id PK).
-    //
-    // graph_files is keyed on entries.id so child tables cascade-delete cleanly
-    // when an entry is removed, and so JOINs from graph rows to entries are a
-    // direct PK lookup. (stash_root, file_path) is retained as UNIQUE so the
-    // extractor's path-based upsert still works.
-    //
-    // graph_file_entities and graph_file_relations no longer duplicate file_path;
-    // they reference entry_id and inherit stash scoping via graph_files.
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS graph_meta (
-      stash_root          TEXT PRIMARY KEY,
-      schema_version      INTEGER NOT NULL,
-      generated_at        TEXT NOT NULL,
-      considered_files    INTEGER NOT NULL DEFAULT 0,
-      extracted_files     INTEGER NOT NULL DEFAULT 0,
-      entity_count        INTEGER NOT NULL DEFAULT 0,
-      relation_count      INTEGER NOT NULL DEFAULT 0,
-      extraction_coverage REAL NOT NULL DEFAULT 0,
-      density             REAL NOT NULL DEFAULT 0,
-      extractor_id        TEXT,
-      extraction_run_id   TEXT,
-      model               TEXT,
-      prompt_version      TEXT,
-      batch_size          INTEGER,
-      cache_hits          INTEGER NOT NULL DEFAULT 0,
-      cache_misses        INTEGER NOT NULL DEFAULT 0,
-      truncation_count    INTEGER NOT NULL DEFAULT 0,
-      failure_count       INTEGER NOT NULL DEFAULT 0
-    );
-
-    CREATE TABLE IF NOT EXISTS graph_files (
-      entry_id          INTEGER PRIMARY KEY REFERENCES entries(id) ON DELETE CASCADE,
-      stash_root        TEXT NOT NULL,
-      file_path         TEXT NOT NULL,
-      file_order        INTEGER NOT NULL,
-      file_type         TEXT NOT NULL,
-      body_hash         TEXT NOT NULL,
-      confidence        REAL,
-      status            TEXT NOT NULL DEFAULT 'extracted',
-      reason            TEXT,
-      extraction_run_id TEXT,
-      UNIQUE(stash_root, file_path)
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_graph_files_stash_order
-      ON graph_files(stash_root, file_order);
-
-    CREATE TABLE IF NOT EXISTS graph_file_entities (
-      entry_id     INTEGER NOT NULL REFERENCES graph_files(entry_id) ON DELETE CASCADE,
-      entity_order INTEGER NOT NULL,
-      stash_root   TEXT NOT NULL,
-      entity_norm  TEXT NOT NULL,
-      entity       TEXT NOT NULL,
-      PRIMARY KEY (entry_id, entity_order)
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_graph_file_entities_entity_norm
-      ON graph_file_entities(stash_root, entity_norm);
-
-    CREATE TABLE IF NOT EXISTS graph_file_relations (
-      entry_id       INTEGER NOT NULL REFERENCES graph_files(entry_id) ON DELETE CASCADE,
-      relation_order INTEGER NOT NULL,
-      from_entity_norm TEXT NOT NULL,
-      from_entity    TEXT NOT NULL,
-      to_entity_norm TEXT NOT NULL,
-      to_entity      TEXT NOT NULL,
-      relation_type  TEXT,
-      confidence     REAL,
-      PRIMARY KEY (entry_id, relation_order)
-    );
-  `);
-    // FTS-dirty queue. Created here (not lazily on first upsert) so the
-    // per-entry write path doesn't issue a CREATE TABLE IF NOT EXISTS on
-    // every call — that DDL would fire thousands of times during a full
-    // index. See `markFtsDirty` and `rebuildFts({ incremental: true })`.
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS entries_fts_dirty (
-      entry_id INTEGER PRIMARY KEY
-    );
-  `);
-    // sqlite-vec table
-    //
-    // Dimension contract:
-    //   - When `embeddingDim` is `undefined`, the caller did NOT request a
-    //     specific dim. Do not touch `index_meta.embeddingDim` and do not run
-    //     the dim-change wipe — fall back to the stored dim (or the static
-    //     default) only when we have to materialise the vec table for the
-    //     first time. Without this guard, registry-side and other dim-unaware
-    //     `openDatabase()` callers would silently overwrite the dim-aware
-    //     improve/index value and oscillate the stored dim.
-    //   - When `embeddingDim` is a number, the caller explicitly asked for
-    //     that dim and owns the dim-change/backup/wipe semantics.
-    const dimExplicit = embeddingDim !== undefined;
-    const effectiveDim = embeddingDim ?? (Number(getMeta(db, "embeddingDim")) || EMBEDDING_DIM);
-    if (isVecAvailable(db)) {
-        // Check if stored embedding dimension differs from configured one
-        if (dimExplicit) {
-            const storedDim = getMeta(db, "embeddingDim");
-            if (storedDim && storedDim !== String(embeddingDim)) {
-                // Re-embedding the whole stash is expensive (LLM API calls + cache
-                // misses), so snapshot the data dir before we drop the vec table and
-                // wipe `embeddings`. This is the SAME hook the version-upgrade path
-                // uses earlier in this function, just gated on embedding-dim mismatch
-                // and tagged so operators can tell the two backup kinds apart.
-                backupBeforeEmbeddingDimChange(options?.dataDir, storedDim, String(embeddingDim));
-                bestEffort(() => db.exec("DROP TABLE IF EXISTS entries_vec"), "drop entries_vec on dim change");
-                // Delete stale BLOB embeddings so they don't produce silently wrong
-                // similarity scores against the new-dimension vec table.
-                bestEffort(() => db.exec("DELETE FROM embeddings"), "delete stale embeddings on dim change");
-                setMeta(db, "hasEmbeddings", "0");
-            }
-        }
-        const vecExists = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='entries_vec'").get();
-        if (!vecExists) {
-            if (!Number.isInteger(effectiveDim) || effectiveDim <= 0 || effectiveDim > 4096) {
-                throw new Error(`Invalid embedding dimension: ${effectiveDim}`);
-            }
-            db.exec(`
-        CREATE VIRTUAL TABLE entries_vec USING vec0(
-          id       INTEGER PRIMARY KEY,
-          embedding FLOAT[${effectiveDim}]
-        );
-      `);
-        }
-        if (dimExplicit) {
-            setMeta(db, "embeddingDim", String(embeddingDim));
-        }
-    }
-    else {
-        // Also purge BLOB embeddings on dimension change (JS fallback path).
-        // When sqlite-vec is unavailable, entries_vec doesn't exist but the BLOB
-        // embeddings table still stores vectors. If the configured dimension
-        // changes, those stored BLOBs become silently incompatible.
-        if (dimExplicit) {
-            const storedDim = getMeta(db, "embeddingDim");
-            if (storedDim && storedDim !== String(embeddingDim)) {
-                backupBeforeEmbeddingDimChange(options?.dataDir, storedDim, String(embeddingDim));
-                bestEffort(() => db.exec("DELETE FROM embeddings"), "delete embeddings on explicit dim change");
-                setMeta(db, "hasEmbeddings", "0");
-            }
-            setMeta(db, "embeddingDim", String(embeddingDim));
-        }
-    }
-    // Usage telemetry table
-    ensureUsageEventsSchema(db);
-    // Registry index cache table — caches remote registry index documents so
-    // `akm search` does not hit the network on every invocation. The DDL is
-    // defined in state-db.ts and shared here to avoid duplication.
-    db.exec(REGISTRY_INDEX_CACHE_DDL);
-    // Restore usage_events backed up by the version-upgrade path above.
-    restoreUsageEventsBackup(db, usageBackup);
-}
-/**
- * Detect a stored DB version that differs from {@link DB_VERSION}, drop the
- * old schema, and return a backup of the previous `usage_events` rows so the
- * rest of `ensureSchema()` can restore them once the new table exists.
- *
- * Returns an empty array when no upgrade is needed or when the previous
- * `usage_events` table is unreadable.
- */
-function handleVersionUpgrade(db) {
-    const storedVersion = getMeta(db, "version");
-    // BUG-L4: distinguish "missing" (undefined) from "present but empty" — both
-    // were previously coerced through `!storedVersion` and treated as "no
-    // upgrade needed", which caused fresh databases (with no version row) to
-    // skip the upgrade path correctly, but also caused the upgrade path to be
-    // taken when a corrupted/empty version string was persisted. The current
-    // tables get dropped only when the stored version exists AND differs from
-    // DB_VERSION; missing or empty version means a fresh DB and no upgrade.
-    if (storedVersion === undefined || storedVersion === "" || storedVersion === String(DB_VERSION))
-        return [];
-    let usageBackup = [];
-    bestEffort(() => {
-        usageBackup = db.prepare("SELECT * FROM usage_events").all();
-    }, "usage_events table may not exist in older versions");
-    db.exec("DROP TABLE IF EXISTS utility_scores");
-    db.exec("DROP TABLE IF EXISTS utility_scores_scoped");
-    db.exec("DROP INDEX IF EXISTS idx_utility_scores_scoped_entry_id");
-    db.exec("DROP TABLE IF EXISTS usage_events");
-    db.exec("DROP TABLE IF EXISTS embeddings");
-    db.exec("DROP TABLE IF EXISTS entries_vec");
-    db.exec("DROP TABLE IF EXISTS entries_fts");
-    db.exec("DROP TABLE IF EXISTS index_dir_state");
-    db.exec("DROP TABLE IF EXISTS llm_enrichment_cache");
-    db.exec("DROP INDEX IF EXISTS idx_llm_cache_updated");
-    db.exec("DROP TABLE IF EXISTS graph_file_relations");
-    db.exec("DROP TABLE IF EXISTS graph_file_entities");
-    db.exec("DROP TABLE IF EXISTS graph_files");
-    db.exec("DROP TABLE IF EXISTS graph_meta");
-    db.exec("DROP TABLE IF EXISTS graph_relations");
-    db.exec("DROP TABLE IF EXISTS graph_entities");
-    db.exec("DROP TABLE IF EXISTS graph_nodes");
-    db.exec("DROP TABLE IF EXISTS graph_stashes");
-    db.exec("DROP INDEX IF EXISTS idx_entries_dir");
-    db.exec("DROP INDEX IF EXISTS idx_entries_type");
-    db.exec("DROP TABLE IF EXISTS entries");
-    db.exec("DELETE FROM index_meta");
-    warn("[akm] Index rebuilt due to version upgrade. Run 'akm index' to repopulate.");
-    return usageBackup;
-}
 /**
- * Snapshot the data directory before the embedding-dimension drop path wipes
- * `embeddings` and recreates `entries_vec`. Re-embedding a real-world stash
- * is expensive (LLM calls + cache misses), so we capture the pre-drop state
- * here using the same MVP backup helper the version-upgrade hook uses
- * earlier in {@link ensureSchema}.
+ * Purge stored embeddings (BLOB rows in `embeddings`, plus the `entries_vec`
+ * virtual table) and mark the index as embedding-free. The single place that
+ * invalidates embeddings — used on a dimension change, a model/provider change,
+ * and a full rebuild.
  *
- * The backup is tagged with the `embedding-dim-change` reason so it lands in
- * `<dataDir>/backups/<timestamp>-embedding-dim-change/` instead of the
- * version-upgrade-flavored `<timestamp>-pre-v<N>/` directory. Restoration
- * works identically via `scripts/migrations/restore-data-dir.sh`.
- *
- * Failures are non-fatal — they downgrade to a warning and the destructive
- * ops run anyway, matching the version-upgrade hook's behavior so a broken
- * backup cannot brick a binary that bumped the configured dim. Likewise,
- * `AKM_DB_BACKUP=0` opts out via the same path.
- */
-function backupBeforeEmbeddingDimChange(dataDir, fromDim, toDim) {
-    if (!dataDir)
-        return;
-    try {
-        const result = backupDataDir({
-            dataDir,
-            // The DB version isn't changing here — pass the current DB_VERSION for
-            // both source and target so the metadata sidecar still records the
-            // running binary's version for forensic context.
-            sourceVersion: DB_VERSION,
-            targetVersion: DB_VERSION,
-            reason: EMBEDDING_DIM_CHANGE_REASON,
-            env: process.env,
-        });
-        if (result) {
-            warn("[akm] embedding dimension changed %s→%s; data directory backed up to %s; embeddings will be regenerated", fromDim, toDim, result.path);
-        }
-    }
-    catch (err) {
-        // Defensive — backupDataDir already swallows most errors, but if it
-        // throws for an unexpected reason we must still proceed with the drop
-        // so the user isn't locked out of their binary on a changed dim.
-        warn("[akm] pre-embedding-dim-change data dir backup raised an unexpected error — %s; embeddings will be regenerated without a snapshot", err instanceof Error ? err.message : String(err));
-    }
-}
-/**
- * Re-insert backed-up `usage_events` rows into the freshly-created table.
+ * No backup: embeddings are a derived cache, fully regenerable from the markdown
+ * by the next `akm index`. (Recovery model decided 2026-06-25.)
  *
- * Wrapped in an outer try/catch because schema changes across versions may
- * make the backup incompatible with the new table definition; in that case
- * the backup is discarded silently rather than blocking startup.
+ * `dropVecTable: true` DROPs `entries_vec` — used on a DIMENSION change, where
+ * the vec0 table must be recreated at the new width by the caller. The default
+ * clears its rows in place (same dimension, stale vectors).
  */
-function restoreUsageEventsBackup(db, backup) {
-    if (backup.length === 0)
-        return;
-    try {
-        // BUG-H4: introspect the *target* table's columns rather than relying on
-        // `row[0]`'s keys. The backup may carry columns the new schema dropped,
-        // and the new schema may have NOT-NULL columns without DEFAULT that the
-        // old backup never carried. Project the backup onto the intersection so
-        // we don't silently lose every row to per-row INSERT errors, and warn
-        // once if any backup column was dropped from the new schema.
-        const targetCols = db.prepare("PRAGMA table_info(usage_events)").all().map((c) => c.name);
-        if (targetCols.length === 0) {
-            warn("[db] restoreUsageEventsBackup: usage_events table missing — discarding %d backup row(s)", backup.length);
-            return;
-        }
-        const targetSet = new Set(targetCols);
-        const backupCols = Object.keys(backup[0] ?? {});
-        const projectedCols = backupCols.filter((c) => targetSet.has(c));
-        const droppedCols = backupCols.filter((c) => !targetSet.has(c));
-        if (projectedCols.length === 0) {
-            warn("[db] restoreUsageEventsBackup: no overlapping columns between backup and current schema — discarding %d row(s); dropped: %s", backup.length, droppedCols.join(", ") || "(none)");
-            return;
-        }
-        if (droppedCols.length > 0) {
-            warn("[db] restoreUsageEventsBackup: dropping columns no longer in usage_events schema: %s", droppedCols.join(", "));
-        }
-        let restored = 0;
-        let failed = 0;
-        db.transaction(() => {
-            const placeholders = projectedCols.map(() => "?").join(", ");
-            const insert = db.prepare(`INSERT INTO usage_events (${projectedCols.join(", ")}) VALUES (${placeholders})`);
-            for (const row of backup) {
-                try {
-                    insert.run(...projectedCols.map((c) => row[c]));
-                    restored++;
-                }
-                catch {
-                    failed++;
-                }
-            }
-        })();
-        if (failed > 0) {
-            warn("[db] restoreUsageEventsBackup: restored %d row(s); skipped %d incompatible row(s)", restored, failed);
-        }
-    }
-    catch (err) {
-        warn("[db] restoreUsageEventsBackup: discarded %d backup row(s) — %s", backup.length, err instanceof Error ? err.message : String(err));
+export function purgeEmbeddings(db, opts) {
+    bestEffort(() => db.exec("DELETE FROM embeddings"), "purge embeddings");
+    if (isVecAvailable(db)) {
+        bestEffort(() => db.exec(opts?.dropVecTable ? "DROP TABLE IF EXISTS entries_vec" : "DELETE FROM entries_vec"), "purge entries_vec");
     }
+    setMeta(db, "hasEmbeddings", "0");
 }
 // ── Meta helpers ────────────────────────────────────────────────────────────
 export function getMeta(db, key) {
@@ -695,26 +227,6 @@ function getUpsertStmts(db) {
     upsertStmtsByDb.set(db, stmts);
     return stmts;
 }
-/**
- * Phase 5A / DB v17 schema guard.
- *
- * Ensures the `entries.derived_from` column + index exist on the open
- * connection. Called from `ensureSchema()` after the entries CREATE so that
- * legacy databases (created against a pre-v17 binary but reopened without
- * triggering `handleVersionUpgrade()`) still gain the new column without
- * data loss. Idempotent: a `PRAGMA table_info` lookup gates the ALTER.
- */
-function ensureDerivedFromColumn(db) {
-    bestEffort(() => {
-        const cols = db.prepare("PRAGMA table_info(entries)").all();
-        const hasColumn = cols.some((c) => c.name === "derived_from");
-        if (!hasColumn) {
-            db.exec("ALTER TABLE entries ADD COLUMN derived_from TEXT");
-        }
-        // Index creation is idempotent on its own; safe to call unconditionally.
-        db.exec("CREATE INDEX IF NOT EXISTS idx_entries_derived_from ON entries(derived_from)");
-    }, "entries table may not exist on a brand-new DB before CREATE — caller is responsible");
-}
 /**
  * Phase 5A / Advantage D5: look up the derived-memory child row whose
  * `derived_from` column matches `parentRef` (e.g. `"memory:claude-prefs"`).
@@ -729,7 +241,7 @@ export function getDerivedForParent(db, parentRef) {
         return null;
     try {
         const row = db
-            .prepare(`SELECT id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text
+            .prepare(`SELECT ${ENTRY_COLUMNS}
          FROM entries
          WHERE derived_from = ?
          ORDER BY id DESC
@@ -737,23 +249,7 @@ export function getDerivedForParent(db, parentRef) {
             .get(parentRef);
         if (!row)
             return null;
-        let entry;
-        try {
-            entry = JSON.parse(row.entry_json);
-        }
-        catch {
-            warn(`[db] getDerivedForParent: skipping entry id=${row.id} — corrupt entry_json`);
-            return null;
-        }
-        return {
-            id: row.id,
-            entryKey: row.entry_key,
-            dirPath: row.dir_path,
-            filePath: row.file_path,
-            stashDir: row.stash_dir,
-            entry,
-            searchText: row.search_text,
-        };
+        return rowToIndexedEntry(row, "getDerivedForParent");
     }
     catch {
         /* `derived_from` column may not exist on legacy DBs that haven't been
@@ -761,6 +257,49 @@ export function getDerivedForParent(db, parentRef) {
         return null;
     }
 }
+/**
+ * 03-R3: for the given derived-twin row ids, fetch each twin's BASE memory
+ * `beliefState`, keyed by twin id.
+ *
+ * Used by the derived-twin belief inheritance in search ranking: a `.derived`
+ * twin has no belief state of its own, so it inherits its base memory's
+ * demoting state (contradicted/superseded/…) at search time. A twin's
+ * `entry_key` is exactly its base's `entry_key` plus the `.derived` suffix
+ * (same stash + type prefix, `<name>` vs `<name>.derived`), so the base is
+ * found by stripping that suffix — no ref/prefix reconstruction. Returns a map
+ * of twin id → base beliefState for bases that carry a non-empty state.
+ * Best-effort: any query error (e.g. legacy DB) yields no inheritance rather
+ * than failing the search.
+ */
+export function getBaseBeliefStatesForDerivedTwins(db, twinIds) {
+    const out = new Map();
+    if (twinIds.length === 0)
+        return out;
+    // Chunk at SQLITE_CHUNK_SIZE like the sibling bulk-by-id helpers, so a large
+    // `--limit` candidate set never trips SQLITE_MAX_VARIABLE_NUMBER (which would
+    // otherwise fall into the best-effort catch and silently disable the feature).
+    for (let i = 0; i < twinIds.length; i += SQLITE_CHUNK_SIZE) {
+        const chunk = twinIds.slice(i, i + SQLITE_CHUNK_SIZE);
+        const placeholders = chunk.map(() => "?").join(",");
+        bestEffort(() => {
+            const rows = db
+                .prepare(`SELECT twin.id AS twin_id, json_extract(base.entry_json, '$.beliefState') AS belief
+           FROM entries twin
+           JOIN entries base
+             ON base.entry_type = 'memory'
+            AND base.entry_key = substr(twin.entry_key, 1, length(twin.entry_key) - length('.derived'))
+           WHERE twin.id IN (${placeholders})
+             AND twin.entry_key LIKE '%.derived'
+             AND json_extract(base.entry_json, '$.beliefState') IS NOT NULL`)
+                .all(...chunk);
+            for (const r of rows) {
+                if (typeof r.belief === "string" && r.belief.trim().length > 0)
+                    out.set(r.twin_id, r.belief.trim());
+            }
+        }, "legacy DB / entry_json without beliefState — treat as no twin inheritance");
+    }
+    return out;
+}
 /**
  * Phase 2A / Rec 5: bulk-load positive feedback event counts for the given
  * entry ids. Used by the utility-decay forgetting curve to stabilize
@@ -839,6 +378,38 @@ function deleteRelatedRows(db, ids) {
         // Clean up usage events before deleting entries
         bestEffort(() => db.prepare(`DELETE FROM usage_events WHERE entry_id IN (${placeholders})`).run(...chunk), "delete usage_events for entries");
     }
+    // #624-P1: graph_files is NO LONGER keyed on entries.id, so deleting an
+    // entries row must NOT wipe the extracted graph (that is the whole point —
+    // the graph survives a reindex when body_hash is unchanged). We therefore do
+    // NOT delete graph_files here. We DO, however, recompute graph_meta counts
+    // for the stash roots touched by the deleted entries so the summary numbers
+    // stay consistent with the live child rows (the counts are derived, and the
+    // entries delete may have changed which files are considered/indexed).
+    //
+    // Resolve the affected stash roots from the entries rows BEFORE deletion.
+    const affectedStashRoots = new Set();
+    for (let i = 0; i < numericIds.length; i += SQLITE_CHUNK_SIZE) {
+        const chunk = numericIds.slice(i, i + SQLITE_CHUNK_SIZE);
+        const placeholders = chunk.map(() => "?").join(",");
+        bestEffort(() => {
+            const rows = db
+                .prepare(`SELECT DISTINCT stash_dir FROM entries WHERE id IN (${placeholders})`)
+                .all(...chunk);
+            for (const row of rows) {
+                if (row.stash_dir)
+                    affectedStashRoots.add(row.stash_dir);
+            }
+        }, "resolve stash roots for graph_meta recompute");
+    }
+    for (const stashRoot of affectedStashRoots) {
+        bestEffort(() => db
+            .prepare(`UPDATE graph_meta
+             SET extracted_files = (SELECT COUNT(*) FROM graph_files WHERE stash_root = ?),
+                 entity_count    = (SELECT COUNT(*) FROM graph_file_entities WHERE stash_root = ?),
+                 relation_count  = (SELECT COUNT(*) FROM graph_file_relations WHERE stash_root = ?)
+             WHERE stash_root = ?`)
+            .run(stashRoot, stashRoot, stashRoot, stashRoot), "sync graph_meta counts after entries delete");
+    }
 }
 /**
  * Delete entries by their primary key IDs, along with all related rows
@@ -1048,12 +619,12 @@ function searchBlobVec(db, queryEmbedding, k) {
     }
 }
 // ── FTS5 search ─────────────────────────────────────────────────────────────
-export function searchFts(db, query, limit, entryType) {
+export function searchFts(db, query, limit, entryType, excludeTypes) {
     const ftsQuery = sanitizeFtsQuery(query);
     if (!ftsQuery)
         return [];
     // Try the exact AND query first
-    const exactResults = runFtsQuery(db, ftsQuery, limit, entryType);
+    const exactResults = runFtsQuery(db, ftsQuery, limit, entryType, excludeTypes);
     if (exactResults.length > 0)
         return exactResults;
     // Exact match returned zero results — try prefix fallback.
@@ -1063,59 +634,40 @@ export function searchFts(db, query, limit, entryType) {
     const prefixQuery = buildPrefixQuery(ftsQuery);
     if (!prefixQuery)
         return [];
-    return runFtsQuery(db, prefixQuery, limit, entryType);
-}
-/**
- * Build a prefix query from an FTS5 query string by appending `*` to each
- * token that is 3+ characters long. Tokens shorter than 3 characters are
- * kept as-is (no prefix expansion) to avoid overly broad matches.
- *
- * Returns null if no tokens qualify for prefix expansion.
- */
-function buildPrefixQuery(ftsQuery) {
-    const tokens = ftsQuery.split(/\s+/).filter(Boolean);
-    let hasPrefix = false;
-    const prefixTokens = tokens.map((t) => {
-        if (t.length >= 3) {
-            hasPrefix = true;
-            return `${t}*`;
-        }
-        return t;
-    });
-    if (!hasPrefix)
-        return null;
-    return prefixTokens.join(" ");
-}
-function runFtsQuery(db, ftsQuery, limit, entryType) {
-    let sql;
+    return runFtsQuery(db, prefixQuery, limit, entryType, excludeTypes);
+}
+function runFtsQuery(db, ftsQuery, limit, entryType, excludeTypes) {
+    // #627 — exclude-type clause. Only applies on the untyped ('any') path; an
+    // explicit include filter (entryType) already narrows to a single type, so
+    // exclusion is redundant there. An empty list skips the clause entirely
+    // (never emit `NOT IN ()`, which is a SQL error / always-false).
+    const excludes = excludeTypes && excludeTypes.length > 0 ? excludeTypes : [];
+    // The typed and untyped paths differ ONLY by one WHERE clause (an entry_type
+    // equality vs. an optional NOT IN exclusion) and their param order — the
+    // SELECT/JOIN/ORDER/LIMIT is shared, so build it once. Join on integer
+    // entry_id directly (no CAST; we store integer). bm25() per-column weights:
+    // entry_id(0), name(10), description(5), tags(3), hints(2), content(1).
+    let filterClause;
     let params;
-    // Join on integer entry_id directly (no CAST needed; we store integer)
-    // Use bm25() with per-column weights: entry_id(0), name(10), description(5), tags(3), hints(2), content(1)
     if (entryType && entryType !== "any") {
-        sql = `
-      SELECT e.id, e.file_path AS filePath, e.entry_json, e.search_text AS searchText,
-             bm25(entries_fts, 0, 10.0, 5.0, 3.0, 2.0, 1.0) AS bm25Score
-      FROM entries_fts f
-      JOIN entries e ON e.id = f.entry_id
-      WHERE entries_fts MATCH ?
-        AND e.entry_type = ?
-      ORDER BY bm25Score, e.id ASC
-      LIMIT ?
-    `;
+        filterClause = "AND e.entry_type = ?";
         params = [ftsQuery, entryType, limit];
     }
     else {
-        sql = `
-      SELECT e.id, e.file_path AS filePath, e.entry_json, e.search_text AS searchText,
-             bm25(entries_fts, 0, 10.0, 5.0, 3.0, 2.0, 1.0) AS bm25Score
-      FROM entries_fts f
-      JOIN entries e ON e.id = f.entry_id
-      WHERE entries_fts MATCH ?
-      ORDER BY bm25Score, e.id ASC
-      LIMIT ?
-    `;
-        params = [ftsQuery, limit];
+        filterClause = excludes.length > 0 ? `AND e.entry_type NOT IN (${excludes.map(() => "?").join(", ")})` : "";
+        // Param order: MATCH, then the NOT IN values, then LIMIT.
+        params = [ftsQuery, ...excludes, limit];
     }
+    const sql = `
+    SELECT e.id, e.file_path AS filePath, e.entry_json, e.search_text AS searchText,
+           bm25(entries_fts, 0, 10.0, 5.0, 3.0, 2.0, 1.0) AS bm25Score
+    FROM entries_fts f
+    JOIN entries e ON e.id = f.entry_id
+    WHERE entries_fts MATCH ?
+      ${filterClause}
+    ORDER BY bm25Score, e.id ASC
+    LIMIT ?
+  `;
     try {
         const rows = db.prepare(sql).all(...params);
         // Guard against corrupt JSON — skip the row rather than crashing
@@ -1139,65 +691,83 @@ function runFtsQuery(db, ftsQuery, limit, entryType) {
         }
         return results;
     }
-    catch {
+    catch (err) {
+        warn("[db] runFtsQuery failed:", err instanceof Error ? err.message : String(err));
         return [];
     }
 }
-export function sanitizeFtsQuery(query) {
-    // Allow only characters safe in FTS5 queries: letters, digits, underscores,
-    // and whitespace. Everything else (hyphens, dots, quotes, parens, asterisks,
-    // colons, carets, @, !, etc.) is replaced with a space so that compound
-    // identifiers like "code-review" or "k8s.setup" become AND-joined tokens
-    // ("code review", "k8s setup") rather than triggering FTS5 syntax errors.
-    let sanitized = query.replace(/[^a-zA-Z0-9_\s]/g, " ");
-    // Neutralize the NEAR operator (FTS5 proximity syntax)
-    sanitized = sanitized.replace(/\bNEAR\b/g, " ");
-    const tokens = sanitized.split(/\s+/).filter((t) => t.length >= 1);
-    if (tokens.length === 0)
-        return "";
-    // Use implicit AND (space-separated tokens) for precision. FTS5 treats
-    // space-separated tokens as an implicit AND, matching only rows that
-    // contain ALL terms.
-    return tokens.join(" ");
-}
+// ── All entries ─────────────────────────────────────────────────────────────
 function parseEntryRows(rows, context) {
     const entries = [];
     for (const row of rows) {
-        let entry;
-        try {
-            entry = JSON.parse(row.entry_json);
-        }
-        catch {
-            warn(`[db] ${context}: skipping entry id=${row.id} — corrupt entry_json`);
-            continue;
-        }
-        entries.push({
-            id: row.id,
-            entryKey: row.entry_key,
-            dirPath: row.dir_path,
-            filePath: row.file_path,
-            stashDir: row.stash_dir,
-            entry,
-            searchText: row.search_text,
-        });
+        const mapped = rowToIndexedEntry(row, context);
+        if (mapped)
+            entries.push(mapped);
     }
     return entries;
 }
-export function getAllEntries(db, entryType) {
+export function getAllEntries(db, entryType, excludeTypes) {
     let sql;
     let params;
+    // #627 — exclude-type clause applies only on the untyped ('any') path. Empty
+    // list skips the clause (never `NOT IN ()`).
+    const excludes = excludeTypes && excludeTypes.length > 0 ? excludeTypes : [];
     if (entryType && entryType !== "any") {
-        sql =
-            "SELECT id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text FROM entries WHERE entry_type = ?";
+        sql = `SELECT ${ENTRY_COLUMNS} FROM entries WHERE entry_type = ?`;
         params = [entryType];
     }
+    else if (excludes.length > 0) {
+        sql = `SELECT ${ENTRY_COLUMNS} FROM entries WHERE entry_type NOT IN (${excludes.map(() => "?").join(", ")})`;
+        params = [...excludes];
+    }
     else {
-        sql = "SELECT id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text FROM entries";
+        sql = `SELECT ${ENTRY_COLUMNS} FROM entries`;
         params = [];
     }
     const rows = db.prepare(sql).all(...params);
     return parseEntryRows(rows, "getAllEntries");
 }
+/**
+ * #609 — read graph entities (normalized) for a set of entry ids. Used by the
+ * recombine pass to cluster memories by shared graph entity ("graph"
+ * relatedness source). Returns a map of `entry_id -> entity_norm[]`. Entries
+ * with no graph entities (graph extraction has not run, or the file produced
+ * no entities) are simply absent from the map — callers must fail open
+ * (fall back to tag relatedness) when the map is empty.
+ */
+export function getEntitiesByEntryIds(db, entryIds) {
+    const result = new Map();
+    if (entryIds.length === 0)
+        return result;
+    // #624-P1: graph_file_entities no longer carries entry_id. Re-derive the
+    // entry_id -> entity_norm[] contract by JOINing through entries on
+    // (stash_dir, file_path) -> graph_files. Chunk the IN(?) list because the
+    // recombine pass can pass 10k+ entry ids (well over the SQLite param limit).
+    for (let i = 0; i < entryIds.length; i += SQLITE_CHUNK_SIZE) {
+        const chunk = entryIds.slice(i, i + SQLITE_CHUNK_SIZE);
+        const placeholders = chunk.map(() => "?").join(", ");
+        const rows = db
+            .prepare(`SELECT e.id AS entry_id, gfe.entity_norm AS entity_norm
+           FROM entries e
+           JOIN graph_files gf
+             ON gf.stash_root = e.stash_dir AND gf.file_path = e.file_path
+           JOIN graph_file_entities gfe
+             ON gfe.stash_root = gf.stash_root
+            AND gfe.file_path = gf.file_path
+            AND gfe.body_hash = gf.body_hash
+          WHERE e.id IN (${placeholders})
+          ORDER BY e.id, gfe.entity_order`)
+            .all(...chunk);
+        for (const row of rows) {
+            const list = result.get(row.entry_id);
+            if (list)
+                list.push(row.entity_norm);
+            else
+                result.set(row.entry_id, [row.entity_norm]);
+        }
+    }
+    return result;
+}
 export function findEntryIdByRef(db, ref) {
     const parsed = parseAssetRef(ref);
     const nameVariants = [parsed.name];
@@ -1221,8 +791,7 @@ export function getEntryCount(db) {
     return row.cnt;
 }
 export function getEmbeddableEntryCount(db) {
-    const row = db.prepare("SELECT COUNT(*) AS cnt FROM entries").get();
-    return row.cnt;
+    return getEntryCount(db);
 }
 export function getEmbeddingCount(db) {
     const row = db.prepare("SELECT COUNT(*) AS cnt FROM embeddings").get();
@@ -1244,9 +813,7 @@ export function getEntryById(db, id) {
     return { filePath: row.file_path, entry };
 }
 export function getEntriesByDir(db, dirPath) {
-    const rows = db
-        .prepare("SELECT id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text FROM entries WHERE dir_path = ?")
-        .all(dirPath);
+    const rows = db.prepare(`SELECT ${ENTRY_COLUMNS} FROM entries WHERE dir_path = ?`).all(dirPath);
     return parseEntryRows(rows, "getEntriesByDir");
 }
 /**
@@ -1262,6 +829,19 @@ export function getEntryIdByFilePath(db, filePath) {
     const row = db.prepare("SELECT id FROM entries WHERE file_path = ? LIMIT 1").get(filePath);
     return row?.id;
 }
+/**
+ * Set of every non-empty `entries.file_path` currently indexed (across all
+ * stashes/sources). Used by staleness detection to spot files that exist on
+ * disk but were never indexed — a clock-independent signal for newly-added
+ * assets that an mtime-vs-builtAt comparison can miss when the two clocks
+ * (filesystem vs wall-clock) are skewed within the same millisecond.
+ */
+export function getIndexedFilePaths(db) {
+    const rows = db
+        .prepare("SELECT DISTINCT file_path FROM entries WHERE file_path IS NOT NULL AND file_path <> ''")
+        .all();
+    return new Set(rows.map((r) => r.file_path));
+}
 /**
  * Resolve a single `entries.file_path` by primary key, or `undefined` if no
  * row matches.
@@ -1477,25 +1057,104 @@ export function computeBodyHash(body) {
     return sha256Hex(body);
 }
 /**
- * Count search and show events for the given entry refs.
- * Returns a Map<ref, count> with only refs that have at least one event.
- * Used by the improve loop to find high-retrieval assets without feedback.
+ * Reduce a ref to its bare `type:name` form, dropping any `origin//` prefix.
+ *
+ * usage_events store entry_ref inconsistently: search/show writers persist
+ * whatever ref the result carried, which is sometimes stash-prefixed
+ * (`origin//type:name`) and sometimes bare (`type:name`). Retrieval counting
+ * keys on the bare form so both spellings of the same asset collapse together.
+ *
+ * Returns the bare form, or the original string when it cannot be parsed (best
+ * effort — never throws so a malformed stored ref can't break counting).
+ */
+function bareRef(ref) {
+    try {
+        const parsed = parseAssetRef(ref);
+        return `${parsed.type}:${parsed.name}`;
+    }
+    catch {
+        return ref;
+    }
+}
+/**
+ * Count retrieval events for the given entry refs.
+ *
+ * Counts `search`, `show`, and `curate` usage events. Returns a
+ * Map<inputRef, count> keyed by the *input* ref strings (only those with at
+ * least one matching event appear). Used by the improve loop to find
+ * high-retrieval assets without feedback.
+ *
+ * Matching is normalization-aware: each stored `entry_ref` is reduced to its
+ * bare `type:name` form before comparison, so a stash-prefixed stored ref
+ * (`origin//type:name`) still matches a bare input ref (`type:name`) and vice
+ * versa. Previously the raw `entry_ref IN (...)` comparison silently dropped
+ * roughly half the signal whenever the two spellings disagreed.
+ *
+ * `curate` events are included: their per-item rows are written with
+ * entry_ref populated (see logCurateEvent), so curation is a real retrieval
+ * signal here. Legacy summary-only curate rows with a NULL entry_ref simply
+ * contribute nothing.
+ *
+ * Machine-sourced events (`source` = 'improve' or 'task') are EXCLUDED: this
+ * count feeds salience/ranking, and pipeline probe traffic counting as demand
+ * creates a self-reinforcing loop (meta-review 05 DRIFT-6). NULL sources
+ * (pre-column rows) count as user demand.
  */
 export function getRetrievalCounts(db, refs) {
     if (refs.length === 0)
         return new Map();
-    const result = new Map();
+    // Map each distinct bare form back to the input ref(s) that produced it so we
+    // can re-key DB results (grouped by bare form) onto the caller's ref strings.
+    const bareToInputs = new Map();
+    for (const ref of refs) {
+        const bare = bareRef(ref);
+        const existing = bareToInputs.get(bare);
+        if (existing)
+            existing.push(ref);
+        else
+            bareToInputs.set(bare, [ref]);
+    }
+    const bareForms = [...bareToInputs.keys()];
+    // Accumulate counts per bare form across chunks before re-keying.
+    const countsByBare = new Map();
     // Chunk to stay within SQLITE_MAX_VARIABLE_NUMBER (same pattern as getUtilityScoresByIds).
-    for (let i = 0; i < refs.length; i += SQLITE_CHUNK_SIZE) {
-        const chunk = refs.slice(i, i + SQLITE_CHUNK_SIZE);
+    for (let i = 0; i < bareForms.length; i += SQLITE_CHUNK_SIZE) {
+        const chunk = bareForms.slice(i, i + SQLITE_CHUNK_SIZE);
         const placeholders = chunk.map(() => "?").join(", ");
+        // Normalize the stored entry_ref to its bare form inside SQL by stripping
+        // everything up to and including the last `//` separator. SQLite has no
+        // rfind, but stored origins never themselves contain `//`, so a stash ref
+        // has exactly one `//` and `substr(... instr ...)` is exact; bare refs have
+        // no `//` and pass through unchanged.
         const rows = db
-            .prepare(`SELECT entry_ref, COUNT(*) AS cnt FROM usage_events
-         WHERE event_type IN ('search','show') AND entry_ref IN (${placeholders})
-         GROUP BY entry_ref`)
+            .prepare(`SELECT
+           CASE
+             WHEN instr(entry_ref, '//') > 0
+               THEN substr(entry_ref, instr(entry_ref, '//') + 2)
+             ELSE entry_ref
+           END AS bare_ref,
+           COUNT(*) AS cnt
+         FROM usage_events
+         WHERE event_type IN ('search','show','curate')
+           AND entry_ref IS NOT NULL
+           AND (source IS NULL OR source NOT IN ('improve','task'))
+           AND CASE
+                 WHEN instr(entry_ref, '//') > 0
+                   THEN substr(entry_ref, instr(entry_ref, '//') + 2)
+                 ELSE entry_ref
+               END IN (${placeholders})
+         GROUP BY bare_ref`)
             .all(...chunk);
-        for (const r of rows)
-            result.set(r.entry_ref, r.cnt);
+        for (const r of rows) {
+            countsByBare.set(r.bare_ref, (countsByBare.get(r.bare_ref) ?? 0) + r.cnt);
+        }
+    }
+    // Re-key bare-form counts onto every input ref that maps to that bare form.
+    const result = new Map();
+    for (const [bare, count] of countsByBare) {
+        for (const input of bareToInputs.get(bare) ?? []) {
+            result.set(input, count);
+        }
     }
     return result;
 }
@@ -1631,84 +1290,24 @@ export function getEntryByRef(db, type, name) {
     return db.prepare("SELECT id FROM entries WHERE entry_type = ? AND entry_key = ?").get(type, `${type}:${name}`);
 }
 /**
- * MemRL learning rate for feedback-driven utility updates (F-5 / #386).
- *
- * Follows the bounded-step formula from MemRL (arXiv:2601.03192):
- *   next = clamp(current + lr × (reward − current), 0, 1)
- *
- * This replaces the unbounded `-0.03 × negativeCount` delta that could
- * silently remove high-utility assets from the improvement loop.
- */
-const FEEDBACK_LR = 0.1;
-/**
- * Positive reward signal for a single positive feedback event.
- * Reward 1.0 means "fully correct / helpful".
- */
-const FEEDBACK_REWARD_POSITIVE = 1.0;
-/**
- * Negative reward signal for a single negative feedback event.
- * Reward 0.0 means "not helpful" (lowest MemRL signal).
- */
-const FEEDBACK_REWARD_NEGATIVE = 0.0;
-/**
- * Maximum total negative utility delta allowed in a single
- * `applyFeedbackToUtilityScore` call regardless of negativeCount.
- *
- * This caps the per-day negative impact (the function is called once per
- * feedback event — spamming 10 negatives in one session can move utility
- * at most `MAX_NEG_DELTA_PER_CALL`). The cap prevents a noisy negative-
- * feedback stream from silently destroying a high-utility asset's ranking.
- */
-const MAX_NEG_DELTA_PER_CALL = 0.15;
-/**
- * Utility threshold below which a review-needed escalation is triggered.
- * When a previously high-utility asset (≥ HIGH_UTILITY_THRESHOLD) drops
- * below this value, the caller should create an escalation proposal.
- */
-export const UTILITY_REVIEW_THRESHOLD = 0.5;
-/**
- * Utility level considered "high" — assets above this are tracked for
- * threshold-crossing escalation.
- */
-export const HIGH_UTILITY_THRESHOLD = 0.5;
-/**
- * Apply accumulated feedback counts to the utility score of an entry using the
- * MemRL bounded-step EMA formula (F-5 / #386, arXiv:2601.03192).
- *
- * Replaces the previous unbounded `-0.03 × negativeCount` formula with:
- *
- *   reward   = weighted average of positive and negative signals
- *   nextUtil = clamp(currentUtil + lr × (reward − currentUtil), 0, 1)
- *
- * The negative impact is additionally capped at {@link MAX_NEG_DELTA_PER_CALL}
- * to prevent a noisy feedback stream from silently erasing a high-utility asset.
+ * Apply accumulated feedback counts to the utility score of an entry, persisting
+ * the result. The bounded-step EMA policy itself (MemRL, F-5 / #386,
+ * arXiv:2601.03192) lives in {@link computeNextUtility} (feedback/utility-policy);
+ * this function only reads the current utility, applies the policy, and writes
+ * the new value.
  *
  * A new entry starts at 0.5 (neutral midpoint) before the EMA step is applied.
- *
- * Returns a {@link FeedbackUtilityResult} so the caller can detect when a
- * previously high-utility asset crosses below the review threshold and create
- * an escalation proposal.
+ * When there is no feedback (both counts zero) the score is left untouched — no
+ * DB write. Returns a {@link FeedbackUtilityResult} so the caller can detect a
+ * previously high-utility asset crossing below the review threshold and escalate.
  */
 export function applyFeedbackToUtilityScore(db, entryId, positiveCount, negativeCount) {
     const existing = getUtilityScore(db, entryId);
     const previousUtility = existing?.utility ?? 0.5;
+    const result = computeNextUtility(previousUtility, positiveCount, negativeCount);
     if (positiveCount === 0 && negativeCount === 0) {
-        return { previousUtility, nextUtility: previousUtility, crossedReviewThreshold: false };
-    }
-    const total = positiveCount + negativeCount;
-    // Weighted reward: proportion of positive signals.
-    const reward = positiveCount > 0 && negativeCount === 0
-        ? FEEDBACK_REWARD_POSITIVE
-        : negativeCount > 0 && positiveCount === 0
-            ? FEEDBACK_REWARD_NEGATIVE
-            : (positiveCount * FEEDBACK_REWARD_POSITIVE + negativeCount * FEEDBACK_REWARD_NEGATIVE) / total;
-    // MemRL bounded-step EMA: lr × (reward − current)
-    let delta = FEEDBACK_LR * (reward - previousUtility);
-    // Per-call negative cap: if delta is negative (net negative feedback), cap it.
-    if (delta < 0) {
-        delta = Math.max(delta, -MAX_NEG_DELTA_PER_CALL);
+        return result;
     }
-    const nextUtility = Math.max(0, Math.min(1, previousUtility + delta));
     const now = new Date().toISOString();
     db.prepare(`
     INSERT INTO utility_scores (entry_id, utility, show_count, search_count, select_rate, last_used_at, updated_at)
@@ -1716,16 +1315,14 @@ export function applyFeedbackToUtilityScore(db, entryId, positiveCount, negative
     ON CONFLICT(entry_id) DO UPDATE SET
       utility    = ?,
       updated_at = ?
-  `).run(entryId, nextUtility, now, now, nextUtility, now);
-    const crossedReviewThreshold = previousUtility >= HIGH_UTILITY_THRESHOLD && nextUtility < UTILITY_REVIEW_THRESHOLD;
-    return { previousUtility, nextUtility, crossedReviewThreshold };
+  `).run(entryId, result.nextUtility, now, now, result.nextUtility, now);
+    return result;
 }
 /**
  * Re-link detached usage_events to their current entry_ids via entry_ref.
  *
- * After a full rebuild, entry IDs change. This query matches events to their
- * new entry rows using the stable `entry_ref` ("type:name") column so usage
- * history survives a full reindex.
+ * After a full rebuild, entry IDs change. This restores each event's link
+ * using the stable `entry_ref` column so usage history survives a reindex.
  */
 export function relinkUsageEvents(db) {
     bestEffort(() => {
@@ -1742,63 +1339,42 @@ export function relinkUsageEvents(db) {
       WHERE entry_id IS NOT NULL
         AND entry_id NOT IN (SELECT id FROM entries)
     `);
-        // Step 2: re-resolve any null entry_id from entry_ref against the
-        // current entries table. Picks up entries that were re-created with
-        // the same ref (e.g. an asset moved between sources).
-        db.exec(`
-      UPDATE usage_events SET entry_id = (
-        SELECT e.id FROM entries e
-        WHERE substr(e.entry_key, length(e.entry_key) - length(usage_events.entry_ref)) = ':' || usage_events.entry_ref
-        LIMIT 1
-      )
-      WHERE entry_id IS NULL AND entry_ref IS NOT NULL
-    `);
+        // Step 2: re-resolve any null entry_id from entry_ref against the current
+        // entries table, reusing the SAME canonical resolver the read path uses at
+        // insert time (`findEntryIdByRef` → `parseAssetRef`). Resolving per DISTINCT
+        // ref keeps this O(distinct-refs) indexed lookups instead of the previous
+        // O(events × entries) non-indexable `substr(entry_key, …)` scan. It also
+        // fixes a silent correctness bug: the old suffix match compared the RAW
+        // `entry_ref`, so origin-qualified refs ("source//type:name") never matched
+        // an `entry_key` and lost their usage history on every full rebuild.
+        const refs = db
+            .prepare("SELECT DISTINCT entry_ref AS ref FROM usage_events WHERE entry_id IS NULL AND entry_ref IS NOT NULL")
+            .all();
+        const update = db.prepare("UPDATE usage_events SET entry_id = ? WHERE entry_ref = ? AND entry_id IS NULL");
+        const relinkTx = db.transaction(() => {
+            for (const { ref } of refs) {
+                let id;
+                try {
+                    id = findEntryIdByRef(db, ref);
+                }
+                catch (err) {
+                    if (err instanceof Error && err.name === "UsageError")
+                        continue;
+                    throw err;
+                }
+                if (id !== undefined)
+                    update.run(id, ref);
+            }
+        });
+        relinkTx();
     }, "usage_events table may not exist yet during entry_id re-resolution");
 }
 // ── registry_index_cache helpers ─────────────────────────────────────────────
-/**
- * Upsert a registry index cache entry in index.db.
- *
- * @param db          - Open index.db connection (from openDatabase / openExistingDatabase).
- * @param registryUrl - Canonical URL of the registry (used as primary key).
- * @param indexJson   - Serialised registry index document (JSON string).
- * @param opts.etag        - HTTP ETag from the response (optional).
- * @param opts.lastModified - HTTP Last-Modified from the response (optional).
- */
-export function upsertRegistryIndexCache(db, registryUrl, indexJson, opts) {
-    db.prepare(`
-    INSERT INTO registry_index_cache (registry_url, fetched_at, etag, last_modified, index_json)
-    VALUES (?, ?, ?, ?, ?)
-    ON CONFLICT(registry_url) DO UPDATE SET
-      fetched_at    = excluded.fetched_at,
-      etag          = excluded.etag,
-      last_modified = excluded.last_modified,
-      index_json    = excluded.index_json
-  `).run(registryUrl, new Date().toISOString(), opts?.etag ?? null, opts?.lastModified ?? null, indexJson);
-}
-/**
- * Look up a cached registry index entry from index.db.
- * Returns undefined when not found or when the entry is older than `maxAgeMs`.
- *
- * TTL check: if `Date.now() - new Date(fetched_at).getTime() > maxAgeMs` the
- * entry is considered a cache miss and undefined is returned.
- *
- * @param db          - Open index.db connection.
- * @param registryUrl - Canonical URL of the registry (primary key).
- * @param maxAgeMs    - Maximum age in milliseconds before the entry is stale (default: 1 hour).
- */
-export function getRegistryIndexCache(db, registryUrl, maxAgeMs = 3_600_000 /* 1 hour */) {
-    const row = db
-        .prepare(`SELECT fetched_at, etag, last_modified, index_json
-       FROM registry_index_cache WHERE registry_url = ?`)
-        .get(registryUrl);
-    if (!row)
-        return undefined;
-    const fetchedAt = Date.parse(row.fetched_at);
-    if (Number.isNaN(fetchedAt) || Date.now() - fetchedAt > maxAgeMs)
-        return undefined;
-    return { indexJson: row.index_json, etag: row.etag, lastModified: row.last_modified };
-}
+// The raw SQL for the `registry_index_cache` table now lives in the storage
+// layer (`src/storage/repositories/registry-index-cache-repository.ts`) so the
+// dependency arrow points indexer → storage. These thin re-exports preserve the
+// previously-public symbols for any importer of this module.
+export { getRegistryIndexCache, upsertRegistryIndexCache, } from "../../storage/repositories/registry-index-cache-repository.js";
 /**
  * Walk indexed entries and collect a deduplicated set of tags. When
  * `entryType` is provided, only entries of that type contribute tags.