akm-cli 0.9.0-beta.53 → 0.9.0-beta.55

package/dist/indexer/db/db.js

@@ -1,7 +1,6 @@
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
-import fs from "node:fs";
 import { createRequire } from "node:module";
 import path from "node:path";
 import { parseAssetRef } from "../../core/asset/asset-ref.js";
@@ -10,43 +9,31 @@ import { getDbPath } from "../../core/paths.js";
 import { warn } from "../../core/warn.js";
 import { cosineSimilarity } from "../../llm/embedders/types.js";
 import { sha256Hex } from "../../runtime.js";
-import { openDatabase } from "../../storage/database.js";
-import { applyStandardPragmas } from "../../storage/sqlite-pragmas.js";
+import { openManagedDatabase } from "../../storage/managed-db.js";
+import { computeNextUtility, HIGH_UTILITY_THRESHOLD, UTILITY_REVIEW_THRESHOLD, } from "../feedback/utility-policy.js";
+import { buildPrefixQuery, sanitizeFtsQuery } from "../search/fts-query.js";
 import { buildSearchFields } from "../search/search-fields.js";
-import { ensureUsageEventsSchema } from "../usage/usage-events.js";
-// ── Constants ───────────────────────────────────────────────────────────────
-// NOTE: schema changes are additive. DB_VERSION is a forensic stamp only — it
-// no longer gates any destructive path (the old nuclear drop-and-rebuild was
-// removed; index.db's idempotent CREATE … IF NOT EXISTS schema converges any
-// older/partial DB forward without dropping data). Graph re-keying uses a
-// TARGETED, graph-only migration (migrateGraphFilesSchema) — the model for any
-// incompatible change: migrate in place, never wipe the whole index.
-export const DB_VERSION = 17;
-export const EMBEDDING_DIM = 384;
-// #624-P1: graph_files re-keyed to (stash_root, file_path, body_hash). Bumped 3→4
-// as a marker; the actual migration is the targeted drop in migrateGraphFilesSchema.
-export const GRAPH_SCHEMA_VERSION = 4;
+import { ENTRY_COLUMNS, rowToIndexedEntry } from "./entry-mapper.js";
+import { ensureSchema } from "./schema.js";
+export { HIGH_UTILITY_THRESHOLD, sanitizeFtsQuery, UTILITY_REVIEW_THRESHOLD };
 // ── Database lifecycle ──────────────────────────────────────────────────────
 export function openIndexDatabase(dbPath, options) {
-    const resolvedPath = dbPath ?? getDbPath();
-    const dir = path.dirname(resolvedPath);
-    if (!fs.existsSync(dir)) {
-        fs.mkdirSync(dir, { recursive: true });
-    }
-    const db = openDatabase(resolvedPath);
-    applyStandardPragmas(db, { dataDir: dir });
-    // Try to load sqlite-vec extension
-    loadVecExtension(db);
-    // Dim resolution: explicit option wins; otherwise consult the on-disk
-    // config so unparameterised opens (registry providers, graph helpers,
-    // ad-hoc CLI subcommands) honour the operator-declared dimension. Only if
-    // both are absent do we fall through to the no-clobber path, which keeps
-    // ensureSchema from touching `index_meta.embeddingDim` at all.
-    const resolvedDim = options?.embeddingDim ?? resolveConfiguredEmbeddingDim();
-    ensureSchema(db, resolvedDim);
-    // Warn once at init if using JS fallback with many entries
-    warnIfVecMissing(db, { once: true });
-    return db;
+    return openManagedDatabase({
+        path: dbPath ?? getDbPath(),
+        init: (db) => {
+            // Try to load sqlite-vec extension
+            loadVecExtension(db);
+            // Dim resolution: explicit option wins; otherwise consult the on-disk
+            // config so unparameterised opens (registry providers, graph helpers,
+            // ad-hoc CLI subcommands) honour the operator-declared dimension. Only if
+            // both are absent do we fall through to the no-clobber path, which keeps
+            // ensureSchema from touching `index_meta.embeddingDim` at all.
+            const resolvedDim = options?.embeddingDim ?? resolveConfiguredEmbeddingDim();
+            ensureSchema(db, resolvedDim);
+            // Warn once at init if using JS fallback with many entries
+            warnIfVecMissing(db, { once: true });
+        },
+    });
 }
 /**
  * Read the operator-configured embedding dimension from the on-disk config.
@@ -71,14 +58,10 @@ function resolveConfiguredEmbeddingDim() {
     }
 }
 export function openExistingDatabase(dbPath) {
-    const resolvedPath = dbPath ?? getDbPath();
-    const dir = path.dirname(resolvedPath);
-    const db = openDatabase(resolvedPath);
-    applyStandardPragmas(db, { dataDir: dir });
     // Existing-DB callers must not mutate schema or embedding metadata on open,
-    // but some paths still need write access to usage_events and other tables.
-    loadVecExtension(db);
-    return db;
+    // but some paths still need write access to usage_events and other tables —
+    // so init only loads the vec extension, it does not run ensureSchema.
+    return openManagedDatabase({ path: dbPath ?? getDbPath(), init: loadVecExtension });
 }
 export function closeDatabase(db) {
     db.close();
@@ -127,372 +110,6 @@ export function warnIfVecMissing(db, { once } = { once: false }) {
         }
     }, "embeddings table may not exist yet during init");
 }
-// ── Schema ──────────────────────────────────────────────────────────────────
-/**
- * DDL for the `registry_index_cache` table. This table lives in index.db
- * (managed by this module), so its DDL belongs here next to the `ensureSchema`
- * that applies it — not in state-db.ts.
- *
- * Created with CREATE TABLE IF NOT EXISTS so it is safe to call inside
- * `ensureSchema()`. Caches the result of resolving and fetching remote registry
- * stash indexes so `akm search` does not hit the network on every invocation.
- *
- * Indexed (query) columns:
- *   registry_url  TEXT PK   — canonical URL of the registry; cache key.
- *   fetched_at    TEXT      — ISO-8601; used to detect stale entries (TTL).
- *   etag          TEXT      — HTTP ETag for conditional GET (If-None-Match).
- *   last_modified TEXT      — HTTP Last-Modified for conditional GET.
- *
- * Non-indexed payload:
- *   index_json    TEXT      — JSON blob of the fetched registry index document.
- *
- * ADD COLUMN extension points (future migrations):
- *   ALTER TABLE registry_index_cache ADD COLUMN schema_version INTEGER DEFAULT 1;
- *   ALTER TABLE registry_index_cache ADD COLUMN kit_count INTEGER DEFAULT NULL;
- *   ALTER TABLE registry_index_cache ADD COLUMN error_message TEXT DEFAULT NULL;
- */
-const REGISTRY_INDEX_CACHE_DDL = `
-  CREATE TABLE IF NOT EXISTS registry_index_cache (
-    registry_url  TEXT    PRIMARY KEY,
-    fetched_at    TEXT    NOT NULL,
-    etag          TEXT,
-    last_modified TEXT,
-    index_json    TEXT    NOT NULL DEFAULT '{}'
-  );
-
-  CREATE INDEX IF NOT EXISTS idx_registry_cache_fetched
-    ON registry_index_cache(fetched_at);
-`;
-/** A row backed up out of the legacy `usage_events` table during a version upgrade. */
-function ensureSchema(db, embeddingDim) {
-    // Create meta table first so we can check version
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS index_meta (
-      key   TEXT PRIMARY KEY,
-      value TEXT NOT NULL
-    );
-  `);
-    // index.db is a fully regenerable derived cache, so its schema is built
-    // idempotently below: every table is CREATE … IF NOT EXISTS and column
-    // additions go through guarded ALTERs (ensureDerivedFromColumn) and targeted
-    // migrations (migrateGraphFilesSchema / migrateGraphDataFromLegacy). Opening a
-    // database with an older or partial schema converges it forward WITHOUT ever
-    // dropping data — there is intentionally no "nuclear drop the whole index on a
-    // DB_VERSION mismatch" path (a destructive design the regenerable index never
-    // needed, and whose pre-drop data-dir backup it required). A genuinely
-    // incompatible change is handled by an additive/targeted migration; the few
-    // derived tables that ever must be rebuilt are regenerated by `akm index`.
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS entries (
-      id          INTEGER PRIMARY KEY AUTOINCREMENT,
-      entry_key   TEXT NOT NULL UNIQUE,
-      dir_path    TEXT NOT NULL,
-      file_path   TEXT NOT NULL,
-      stash_dir   TEXT NOT NULL,
-      entry_json  TEXT NOT NULL,
-      search_text TEXT NOT NULL,
-      entry_type  TEXT NOT NULL,
-      derived_from TEXT
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_entries_dir ON entries(dir_path);
-    CREATE INDEX IF NOT EXISTS idx_entries_type ON entries(entry_type);
-    CREATE INDEX IF NOT EXISTS idx_entries_file_path ON entries(file_path);
-  `);
-    // Phase 5A / DB v17: backfill `derived_from` column + index on databases
-    // that were created at v17 fresh OR carry a partial v17 schema (a DB whose
-    // `index_meta.version` was bumped to 17 but whose `entries` table still
-    // lacks the column — this happens when a previous v17 binary opened a
-    // pre-v17 DB without taking the upgrade path because no version mismatch
-    // was seen at boot). The PRAGMA-then-ALTER guard runs unconditionally so
-    // both fresh and partial schemas converge. The CREATE INDEX for
-    // `derived_from` MUST run after this helper so we never reference a
-    // column that has not yet been added on partial schemas.
-    ensureDerivedFromColumn(db);
-    // Validated WorkflowDocument JSON, one row per indexed workflow entry.
-    // Pure index data — fully rebuilt on each `akm index`. ON DELETE CASCADE
-    // means clearing entries (full rebuild or per-dir delete) drops these too.
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS workflow_documents (
-      entry_id        INTEGER PRIMARY KEY REFERENCES entries(id) ON DELETE CASCADE,
-      schema_version  INTEGER NOT NULL,
-      document_json   TEXT NOT NULL,
-      source_path     TEXT NOT NULL,
-      source_hash     TEXT NOT NULL,
-      updated_at      TEXT NOT NULL
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_workflow_documents_source_path
-      ON workflow_documents(source_path);
-  `);
-    // Set version immediately after table creation so a crash before the end of
-    // ensureSchema() does not leave the database in a versionless state on next open.
-    const versionAfterCreate = getMeta(db, "version");
-    if (!versionAfterCreate) {
-        setMeta(db, "version", String(DB_VERSION));
-    }
-    // BLOB-based embedding storage (always available, no sqlite-vec needed)
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS embeddings (
-      id        INTEGER PRIMARY KEY,
-      embedding BLOB NOT NULL,
-      FOREIGN KEY (id) REFERENCES entries(id)
-    );
-  `);
-    // FTS5 table — multi-column with per-field weighting via bm25()
-    const ftsExists = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='entries_fts'").get();
-    if (!ftsExists) {
-        db.exec(`
-      CREATE VIRTUAL TABLE entries_fts USING fts5(
-        entry_id UNINDEXED,
-        name,
-        description,
-        tags,
-        hints,
-        content,
-        tokenize='porter unicode61'
-      );
-    `);
-    }
-    // Usage events table — created by ensureUsageEventsSchema() at runtime.
-    // Utility scores table (aggregated per-entry utility metrics)
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS utility_scores (
-      entry_id     INTEGER PRIMARY KEY,
-      utility      REAL NOT NULL DEFAULT 0,
-      show_count   INTEGER NOT NULL DEFAULT 0,
-      search_count INTEGER NOT NULL DEFAULT 0,
-      select_rate  REAL NOT NULL DEFAULT 0,
-      last_used_at TEXT,
-      updated_at   TEXT NOT NULL DEFAULT (datetime('now')),
-      FOREIGN KEY (entry_id) REFERENCES entries(id) ON DELETE CASCADE
-    );
-  `);
-    // Per-project scoped utility scores — tracks usage per (entry, cwd-anchor)
-    // so assets useful in project A don't pollute rankings in project B.
-    // The global utility_scores table is preserved as a fallback / cold-start aid.
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS utility_scores_scoped (
-      entry_id     INTEGER NOT NULL,
-      scope_key    TEXT NOT NULL,
-      utility      REAL NOT NULL DEFAULT 0,
-      last_used_at INTEGER NOT NULL,
-      PRIMARY KEY (entry_id, scope_key)
-    );
-    CREATE INDEX IF NOT EXISTS idx_utility_scores_scoped_entry_id
-      ON utility_scores_scoped(entry_id);
-  `);
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS index_dir_state (
-      dir_path          TEXT PRIMARY KEY,
-      file_set_hash     TEXT NOT NULL,
-      file_mtime_max_ms REAL NOT NULL,
-      reason            TEXT NOT NULL,
-      updated_at        TEXT NOT NULL
-    );
-  `);
-    // LLM enrichment result cache. Stores a SHA-256 body hash and the JSON
-    // result for each asset so that subsequent `akm index --enrich` runs can
-    // skip the LLM call when the body hasn't changed. The cache is keyed by
-    // a stable asset_ref string (e.g. the absolute file path for graph/memory
-    // passes, or `entryKey:passId` for the metadata-enhance pass).
-    // Entries are cleaned up when assets are removed or --re-enrich is used.
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS llm_enrichment_cache (
-      asset_ref     TEXT NOT NULL,
-      cache_variant TEXT NOT NULL,
-      body_hash     TEXT NOT NULL,
-      result_json   TEXT NOT NULL,
-      updated_at    INTEGER NOT NULL,
-      PRIMARY KEY (asset_ref, cache_variant)
-    );
-
-     CREATE INDEX IF NOT EXISTS idx_llm_cache_updated
-       ON llm_enrichment_cache(updated_at);
-  `);
-    // Graph extraction tables — schema v4 ((stash_root, file_path, body_hash) PK).
-    //
-    // graph_files is self-keyed on (stash_root, file_path, body_hash) and is NO
-    // LONGER tied to entries.id. This is the #624-P1 win: deleting and
-    // re-inserting an entries row during a reindex no longer cascade-wipes the
-    // extracted graph — as long as the file's body_hash is unchanged, the graph
-    // data survives. body_hash is part of the PK so a content change yields a
-    // distinct key; a UNIQUE index on (stash_root, file_path) still enforces
-    // exactly one graph_files row per path (delete-then-insert on a hash change).
-    //
-    // graph_file_entities and graph_file_relations carry (stash_root, file_path,
-    // body_hash) and declare a composite FK -> graph_files ON DELETE CASCADE so
-    // child rows are removed when a graph_files row is replaced.
-    //
-    // #624-P1 targeted migration: an existing DB may still hold the OLD graph_files
-    // (entry_id PK). SQLite can't ALTER a primary key, so we RENAME the 3 graph
-    // tables aside (→ *_legacy) here — ONLY the graph tables, never the index/
-    // embeddings — then the CREATE block below builds the new shape, then
-    // migrateGraphDataFromLegacy() copies the data across so the graph is PRESERVED
-    // (not re-extracted).
-    migrateGraphFilesSchema(db);
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS graph_meta (
-      stash_root          TEXT PRIMARY KEY,
-      schema_version      INTEGER NOT NULL,
-      generated_at        TEXT NOT NULL,
-      considered_files    INTEGER NOT NULL DEFAULT 0,
-      extracted_files     INTEGER NOT NULL DEFAULT 0,
-      entity_count        INTEGER NOT NULL DEFAULT 0,
-      relation_count      INTEGER NOT NULL DEFAULT 0,
-      extraction_coverage REAL NOT NULL DEFAULT 0,
-      density             REAL NOT NULL DEFAULT 0,
-      extractor_id        TEXT,
-      extraction_run_id   TEXT,
-      model               TEXT,
-      prompt_version      TEXT,
-      batch_size          INTEGER,
-      cache_hits          INTEGER NOT NULL DEFAULT 0,
-      cache_misses        INTEGER NOT NULL DEFAULT 0,
-      truncation_count    INTEGER NOT NULL DEFAULT 0,
-      failure_count       INTEGER NOT NULL DEFAULT 0
-    );
-
-    CREATE TABLE IF NOT EXISTS graph_files (
-      stash_root        TEXT NOT NULL,
-      file_path         TEXT NOT NULL,
-      file_order        INTEGER NOT NULL,
-      file_type         TEXT NOT NULL,
-      body_hash         TEXT NOT NULL,
-      confidence        REAL,
-      status            TEXT NOT NULL DEFAULT 'extracted',
-      reason            TEXT,
-      extraction_run_id TEXT,
-      PRIMARY KEY (stash_root, file_path, body_hash)
-    );
-
-    CREATE UNIQUE INDEX IF NOT EXISTS idx_graph_files_path
-      ON graph_files(stash_root, file_path);
-
-    CREATE INDEX IF NOT EXISTS idx_graph_files_stash_order
-      ON graph_files(stash_root, file_order);
-
-    CREATE TABLE IF NOT EXISTS graph_file_entities (
-      stash_root   TEXT NOT NULL,
-      file_path    TEXT NOT NULL,
-      body_hash    TEXT NOT NULL,
-      entity_order INTEGER NOT NULL,
-      entity_norm  TEXT NOT NULL,
-      entity       TEXT NOT NULL,
-      PRIMARY KEY (stash_root, file_path, body_hash, entity_order),
-      FOREIGN KEY (stash_root, file_path, body_hash)
-        REFERENCES graph_files(stash_root, file_path, body_hash) ON DELETE CASCADE
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_graph_file_entities_entity_norm
-      ON graph_file_entities(stash_root, entity_norm);
-
-    CREATE TABLE IF NOT EXISTS graph_file_relations (
-      stash_root     TEXT NOT NULL,
-      file_path      TEXT NOT NULL,
-      body_hash      TEXT NOT NULL,
-      relation_order INTEGER NOT NULL,
-      from_entity_norm TEXT NOT NULL,
-      from_entity    TEXT NOT NULL,
-      to_entity_norm TEXT NOT NULL,
-      to_entity      TEXT NOT NULL,
-      relation_type  TEXT,
-      confidence     REAL,
-      PRIMARY KEY (stash_root, file_path, body_hash, relation_order),
-      FOREIGN KEY (stash_root, file_path, body_hash)
-        REFERENCES graph_files(stash_root, file_path, body_hash) ON DELETE CASCADE
-    );
-
-    -- #624-P3: lazy graph-extraction queue. Standalone table (NO FK to
-    -- graph_files — a queued file by definition has no graph row yet).
-    -- Idempotent on (stash_root, file_path); drained highest-priority-first.
-    -- CREATE TABLE IF NOT EXISTS is the forward migration (no DB_VERSION bump).
-    CREATE TABLE IF NOT EXISTS graph_extraction_queue (
-      stash_root TEXT NOT NULL,
-      file_path  TEXT NOT NULL,
-      body_hash  TEXT NOT NULL,
-      queued_at  TEXT NOT NULL DEFAULT (datetime('now')),
-      priority   INTEGER NOT NULL DEFAULT 0,
-      PRIMARY KEY (stash_root, file_path)
-    );
-
-    CREATE INDEX IF NOT EXISTS idx_graph_extraction_queue_drain
-      ON graph_extraction_queue(stash_root, priority DESC, queued_at);
-  `);
-    // #624-P1 migration step 2: copy any renamed-aside legacy graph data into the
-    // new-shape tables (just created above), then drop the legacy tables. No-op
-    // unless migrateGraphFilesSchema renamed a legacy graph_files this open.
-    migrateGraphDataFromLegacy(db);
-    // FTS-dirty queue. Created here (not lazily on first upsert) so the
-    // per-entry write path doesn't issue a CREATE TABLE IF NOT EXISTS on
-    // every call — that DDL would fire thousands of times during a full
-    // index. See `markFtsDirty` and `rebuildFts({ incremental: true })`.
-    db.exec(`
-    CREATE TABLE IF NOT EXISTS entries_fts_dirty (
-      entry_id INTEGER PRIMARY KEY
-    );
-  `);
-    // sqlite-vec table
-    //
-    // Dimension contract:
-    //   - When `embeddingDim` is `undefined`, the caller did NOT request a
-    //     specific dim. Do not touch `index_meta.embeddingDim` and do not run
-    //     the dim-change wipe — fall back to the stored dim (or the static
-    //     default) only when we have to materialise the vec table for the
-    //     first time. Without this guard, registry-side and other dim-unaware
-    //     `openDatabase()` callers would silently overwrite the dim-aware
-    //     improve/index value and oscillate the stored dim.
-    //   - When `embeddingDim` is a number, the caller explicitly asked for
-    //     that dim and owns the dim-change/backup/wipe semantics.
-    const dimExplicit = embeddingDim !== undefined;
-    const effectiveDim = embeddingDim ?? (Number(getMeta(db, "embeddingDim")) || EMBEDDING_DIM);
-    if (isVecAvailable(db)) {
-        // Check if stored embedding dimension differs from configured one
-        if (dimExplicit) {
-            const storedDim = getMeta(db, "embeddingDim");
-            if (storedDim && storedDim !== String(embeddingDim)) {
-                // Stored vectors are incompatible with the new dimension. Drop the vec
-                // table so the block below recreates it at the new width; the BLOB rows
-                // go too. Regenerable from markdown — re-embedded by the next index.
-                purgeEmbeddings(db, { dropVecTable: true });
-            }
-        }
-        const vecExists = db.prepare("SELECT name FROM sqlite_master WHERE type='table' AND name='entries_vec'").get();
-        if (!vecExists) {
-            if (!Number.isInteger(effectiveDim) || effectiveDim <= 0 || effectiveDim > 4096) {
-                throw new Error(`Invalid embedding dimension: ${effectiveDim}`);
-            }
-            db.exec(`
-        CREATE VIRTUAL TABLE entries_vec USING vec0(
-          id       INTEGER PRIMARY KEY,
-          embedding FLOAT[${effectiveDim}]
-        );
-      `);
-        }
-        if (dimExplicit) {
-            setMeta(db, "embeddingDim", String(embeddingDim));
-        }
-    }
-    else {
-        // Also purge BLOB embeddings on dimension change (JS fallback path).
-        // When sqlite-vec is unavailable, entries_vec doesn't exist but the BLOB
-        // embeddings table still stores vectors. If the configured dimension
-        // changes, those stored BLOBs become silently incompatible.
-        if (dimExplicit) {
-            const storedDim = getMeta(db, "embeddingDim");
-            if (storedDim && storedDim !== String(embeddingDim)) {
-                // JS-fallback path: no vec table, just clear the stale BLOB vectors.
-                purgeEmbeddings(db);
-            }
-            setMeta(db, "embeddingDim", String(embeddingDim));
-        }
-    }
-    // Usage telemetry table
-    ensureUsageEventsSchema(db);
-    // Registry index cache table — caches remote registry index documents so
-    // `akm search` does not hit the network on every invocation.
-    db.exec(REGISTRY_INDEX_CACHE_DDL);
-}
 /**
  * Purge stored embeddings (BLOB rows in `embeddings`, plus the `entries_vec`
  * virtual table) and mark the index as embedding-free. The single place that
@@ -610,129 +227,6 @@ function getUpsertStmts(db) {
     upsertStmtsByDb.set(db, stmts);
     return stmts;
 }
-/**
- * Phase 5A / DB v17 schema guard.
- *
- * Ensures the `entries.derived_from` column + index exist on the open
- * connection. Called from `ensureSchema()` after the entries CREATE so that
- * legacy databases (created against a pre-v17 binary) still gain the new column
- * without data loss. Idempotent: a `PRAGMA table_info` lookup gates the ALTER.
- */
-function ensureDerivedFromColumn(db) {
-    bestEffort(() => {
-        const cols = db.prepare("PRAGMA table_info(entries)").all();
-        const hasColumn = cols.some((c) => c.name === "derived_from");
-        if (!hasColumn) {
-            db.exec("ALTER TABLE entries ADD COLUMN derived_from TEXT");
-        }
-        // Index creation is idempotent on its own; safe to call unconditionally.
-        db.exec("CREATE INDEX IF NOT EXISTS idx_entries_derived_from ON entries(derived_from)");
-    }, "entries table may not exist on a brand-new DB before CREATE — caller is responsible");
-}
-/**
- * Returns true when a table exists in the current database.
- */
-function tableExists(db, name) {
-    const row = db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=? LIMIT 1").get(name);
-    return row !== undefined && row !== null;
-}
-/**
- * #624-P1 targeted graph-schema migration — STEP 1 of 2 (rename).
- *
- * graph_files was re-keyed from `entry_id INTEGER PRIMARY KEY REFERENCES
- * entries(id)` to a self-contained `(stash_root, file_path, body_hash)` PK.
- * SQLite cannot ALTER a primary key, so an existing DB carrying the OLD shape
- * has its 3 graph tables RENAMED to `*_legacy` here; ensureSchema's CREATE block
- * then builds the new-shape tables, and {@link migrateGraphDataFromLegacy} COPIES
- * the data across before dropping the legacy tables. The graph is preserved —
- * NOT re-extracted (re-extraction is ~19s/file of LLM work).
- *
- * Crucially this is GRAPH-SCOPED: it touches ONLY the graph tables, never the
- * index / embeddings / enrichment cache. So users keep their (expensive)
- * embeddings instead of being forced into a full re-embed by a DB_VERSION bump.
- *
- * Detection: the old schema has an `entry_id` column on graph_files. Fresh DBs
- * (no graph_files yet) and already-migrated DBs (no entry_id column) are no-ops.
- * Idempotent.
- */
-function migrateGraphFilesSchema(db) {
-    bestEffort(() => {
-        const cols = db.prepare("PRAGMA table_info(graph_files)").all();
-        const isLegacyShape = cols.some((c) => c.name === "entry_id");
-        if (!isLegacyShape)
-            return;
-        // A previous interrupted migration may have left *_legacy behind — drop those
-        // husks first so the rename below doesn't collide.
-        db.exec("DROP TABLE IF EXISTS graph_file_relations_legacy");
-        db.exec("DROP TABLE IF EXISTS graph_file_entities_legacy");
-        db.exec("DROP TABLE IF EXISTS graph_files_legacy");
-        // Rename the 3 entry_id-keyed tables aside. graph_meta is unchanged (stash_root
-        // key) so it is left in place. ALTER … RENAME auto-updates child FK refs in
-        // SQLite ≥3.25, which is fine — the legacy children are dropped after the copy.
-        db.exec("ALTER TABLE graph_files RENAME TO graph_files_legacy");
-        if (tableExists(db, "graph_file_entities")) {
-            db.exec("ALTER TABLE graph_file_entities RENAME TO graph_file_entities_legacy");
-        }
-        if (tableExists(db, "graph_file_relations")) {
-            db.exec("ALTER TABLE graph_file_relations RENAME TO graph_file_relations_legacy");
-        }
-    }, "graph_files may not exist on a brand-new DB before CREATE — caller is responsible");
-}
-/**
- * #624-P1 targeted graph-schema migration — STEP 2 of 2 (copy + drop legacy).
- *
- * Runs AFTER the graph CREATE TABLE block, so the new-shape tables exist. Copies
- * every legacy row into the re-keyed tables — the old tables already carry
- * (stash_root, file_path, body_hash) next to entry_id, so the projection is a
- * straight column copy (children JOIN back to graph_files_legacy to resolve the
- * composite key from their entry_id). Then drops the `*_legacy` tables.
- *
- * Best-effort: a copy failure (e.g. a pre-body_hash legacy schema) is tolerated,
- * and the legacy tables are dropped regardless so they never linger. Rows whose
- * body_hash is null/empty can't form the new PK and are skipped (they re-extract).
- */
-function migrateGraphDataFromLegacy(db) {
-    if (!tableExists(db, "graph_files_legacy"))
-        return;
-    let migratedFiles = 0;
-    bestEffort(() => {
-        db.transaction(() => {
-            const res = db
-                .prepare(`INSERT OR IGNORE INTO graph_files
-             (stash_root, file_path, body_hash, file_order, file_type, confidence, status, reason, extraction_run_id)
-           SELECT stash_root, file_path, body_hash, file_order, file_type, confidence, status, reason, extraction_run_id
-             FROM graph_files_legacy
-            WHERE body_hash IS NOT NULL AND body_hash != ''`)
-                .run();
-            migratedFiles = Number(res.changes);
-            if (tableExists(db, "graph_file_entities_legacy")) {
-                db.exec(`INSERT OR IGNORE INTO graph_file_entities
-             (stash_root, file_path, body_hash, entity_order, entity_norm, entity)
-           SELECT gf.stash_root, gf.file_path, gf.body_hash, e.entity_order, e.entity_norm, e.entity
-             FROM graph_file_entities_legacy e
-             JOIN graph_files_legacy gf ON gf.entry_id = e.entry_id
-            WHERE gf.body_hash IS NOT NULL AND gf.body_hash != ''`);
-            }
-            if (tableExists(db, "graph_file_relations_legacy")) {
-                db.exec(`INSERT OR IGNORE INTO graph_file_relations
-             (stash_root, file_path, body_hash, relation_order, from_entity_norm, from_entity, to_entity_norm, to_entity, relation_type, confidence)
-           SELECT gf.stash_root, gf.file_path, gf.body_hash, r.relation_order, r.from_entity_norm, r.from_entity, r.to_entity_norm, r.to_entity, r.relation_type, r.confidence
-             FROM graph_file_relations_legacy r
-             JOIN graph_files_legacy gf ON gf.entry_id = r.entry_id
-            WHERE gf.body_hash IS NOT NULL AND gf.body_hash != ''`);
-            }
-        })();
-    }, "graph data migration is best-effort; legacy tables are dropped regardless below");
-    // Always drop the legacy tables (children first), migrated or not.
-    bestEffort(() => {
-        db.exec("DROP TABLE IF EXISTS graph_file_relations_legacy");
-        db.exec("DROP TABLE IF EXISTS graph_file_entities_legacy");
-        db.exec("DROP TABLE IF EXISTS graph_files_legacy");
-    }, "drop legacy graph tables after migration");
-    if (migratedFiles > 0) {
-        warn(`[akm] graph index re-keyed (#624): migrated ${migratedFiles} extracted file(s) to the new schema — no re-extraction needed. Index + embeddings untouched.`);
-    }
-}
 /**
  * Phase 5A / Advantage D5: look up the derived-memory child row whose
  * `derived_from` column matches `parentRef` (e.g. `"memory:claude-prefs"`).
@@ -747,7 +241,7 @@ export function getDerivedForParent(db, parentRef) {
         return null;
     try {
         const row = db
-            .prepare(`SELECT id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text
+            .prepare(`SELECT ${ENTRY_COLUMNS}
          FROM entries
          WHERE derived_from = ?
          ORDER BY id DESC
@@ -755,23 +249,7 @@ export function getDerivedForParent(db, parentRef) {
             .get(parentRef);
         if (!row)
             return null;
-        let entry;
-        try {
-            entry = JSON.parse(row.entry_json);
-        }
-        catch {
-            warn(`[db] getDerivedForParent: skipping entry id=${row.id} — corrupt entry_json`);
-            return null;
-        }
-        return {
-            id: row.id,
-            entryKey: row.entry_key,
-            dirPath: row.dir_path,
-            filePath: row.file_path,
-            stashDir: row.stash_dir,
-            entry,
-            searchText: row.search_text,
-        };
+        return rowToIndexedEntry(row, "getDerivedForParent");
     }
     catch {
         /* `derived_from` column may not exist on legacy DBs that haven't been
@@ -1115,65 +593,38 @@ export function searchFts(db, query, limit, entryType, excludeTypes) {
         return [];
     return runFtsQuery(db, prefixQuery, limit, entryType, excludeTypes);
 }
-/**
- * Build a prefix query from an FTS5 query string by appending `*` to each
- * token that is 3+ characters long. Tokens shorter than 3 characters are
- * kept as-is (no prefix expansion) to avoid overly broad matches.
- *
- * Returns null if no tokens qualify for prefix expansion.
- */
-function buildPrefixQuery(ftsQuery) {
-    const tokens = ftsQuery.split(/\s+/).filter(Boolean);
-    let hasPrefix = false;
-    const prefixTokens = tokens.map((t) => {
-        if (t.length >= 3) {
-            hasPrefix = true;
-            return `${t}*`;
-        }
-        return t;
-    });
-    if (!hasPrefix)
-        return null;
-    return prefixTokens.join(" ");
-}
 function runFtsQuery(db, ftsQuery, limit, entryType, excludeTypes) {
-    let sql;
-    let params;
     // #627 — exclude-type clause. Only applies on the untyped ('any') path; an
     // explicit include filter (entryType) already narrows to a single type, so
     // exclusion is redundant there. An empty list skips the clause entirely
     // (never emit `NOT IN ()`, which is a SQL error / always-false).
     const excludes = excludeTypes && excludeTypes.length > 0 ? excludeTypes : [];
-    // Join on integer entry_id directly (no CAST needed; we store integer)
-    // Use bm25() with per-column weights: entry_id(0), name(10), description(5), tags(3), hints(2), content(1)
+    // The typed and untyped paths differ ONLY by one WHERE clause (an entry_type
+    // equality vs. an optional NOT IN exclusion) and their param order — the
+    // SELECT/JOIN/ORDER/LIMIT is shared, so build it once. Join on integer
+    // entry_id directly (no CAST; we store integer). bm25() per-column weights:
+    // entry_id(0), name(10), description(5), tags(3), hints(2), content(1).
+    let filterClause;
+    let params;
     if (entryType && entryType !== "any") {
-        sql = `
-      SELECT e.id, e.file_path AS filePath, e.entry_json, e.search_text AS searchText,
-             bm25(entries_fts, 0, 10.0, 5.0, 3.0, 2.0, 1.0) AS bm25Score
-      FROM entries_fts f
-      JOIN entries e ON e.id = f.entry_id
-      WHERE entries_fts MATCH ?
-        AND e.entry_type = ?
-      ORDER BY bm25Score, e.id ASC
-      LIMIT ?
-    `;
+        filterClause = "AND e.entry_type = ?";
         params = [ftsQuery, entryType, limit];
     }
     else {
-        const excludeClause = excludes.length > 0 ? `AND e.entry_type NOT IN (${excludes.map(() => "?").join(", ")})` : "";
-        sql = `
-      SELECT e.id, e.file_path AS filePath, e.entry_json, e.search_text AS searchText,
-             bm25(entries_fts, 0, 10.0, 5.0, 3.0, 2.0, 1.0) AS bm25Score
-      FROM entries_fts f
-      JOIN entries e ON e.id = f.entry_id
-      WHERE entries_fts MATCH ?
-        ${excludeClause}
-      ORDER BY bm25Score, e.id ASC
-      LIMIT ?
-    `;
+        filterClause = excludes.length > 0 ? `AND e.entry_type NOT IN (${excludes.map(() => "?").join(", ")})` : "";
         // Param order: MATCH, then the NOT IN values, then LIMIT.
         params = [ftsQuery, ...excludes, limit];
     }
+    const sql = `
+    SELECT e.id, e.file_path AS filePath, e.entry_json, e.search_text AS searchText,
+           bm25(entries_fts, 0, 10.0, 5.0, 3.0, 2.0, 1.0) AS bm25Score
+    FROM entries_fts f
+    JOIN entries e ON e.id = f.entry_id
+    WHERE entries_fts MATCH ?
+      ${filterClause}
+    ORDER BY bm25Score, e.id ASC
+    LIMIT ?
+  `;
     try {
         const rows = db.prepare(sql).all(...params);
         // Guard against corrupt JSON — skip the row rather than crashing
@@ -1201,43 +652,13 @@ function runFtsQuery(db, ftsQuery, limit, entryType, excludeTypes) {
         return [];
     }
 }
-export function sanitizeFtsQuery(query) {
-    // Allow only characters safe in FTS5 queries: letters, digits, underscores,
-    // and whitespace. Everything else (hyphens, dots, quotes, parens, asterisks,
-    // colons, carets, @, !, etc.) is replaced with a space so that compound
-    // identifiers like "code-review" or "k8s.setup" become AND-joined tokens
-    // ("code review", "k8s setup") rather than triggering FTS5 syntax errors.
-    let sanitized = query.replace(/[^a-zA-Z0-9_\s]/g, " ");
-    // Neutralize the NEAR operator (FTS5 proximity syntax)
-    sanitized = sanitized.replace(/\bNEAR\b/g, " ");
-    const tokens = sanitized.split(/\s+/).filter((t) => t.length >= 1);
-    if (tokens.length === 0)
-        return "";
-    // Use implicit AND (space-separated tokens) for precision. FTS5 treats
-    // space-separated tokens as an implicit AND, matching only rows that
-    // contain ALL terms.
-    return tokens.join(" ");
-}
+// ── All entries ─────────────────────────────────────────────────────────────
 function parseEntryRows(rows, context) {
     const entries = [];
     for (const row of rows) {
-        let entry;
-        try {
-            entry = JSON.parse(row.entry_json);
-        }
-        catch {
-            warn(`[db] ${context}: skipping entry id=${row.id} — corrupt entry_json`);
-            continue;
-        }
-        entries.push({
-            id: row.id,
-            entryKey: row.entry_key,
-            dirPath: row.dir_path,
-            filePath: row.file_path,
-            stashDir: row.stash_dir,
-            entry,
-            searchText: row.search_text,
-        });
+        const mapped = rowToIndexedEntry(row, context);
+        if (mapped)
+            entries.push(mapped);
     }
     return entries;
 }
@@ -1248,16 +669,15 @@ export function getAllEntries(db, entryType, excludeTypes) {
     // list skips the clause (never `NOT IN ()`).
     const excludes = excludeTypes && excludeTypes.length > 0 ? excludeTypes : [];
     if (entryType && entryType !== "any") {
-        sql =
-            "SELECT id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text FROM entries WHERE entry_type = ?";
+        sql = `SELECT ${ENTRY_COLUMNS} FROM entries WHERE entry_type = ?`;
         params = [entryType];
     }
     else if (excludes.length > 0) {
-        sql = `SELECT id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text FROM entries WHERE entry_type NOT IN (${excludes.map(() => "?").join(", ")})`;
+        sql = `SELECT ${ENTRY_COLUMNS} FROM entries WHERE entry_type NOT IN (${excludes.map(() => "?").join(", ")})`;
         params = [...excludes];
     }
     else {
-        sql = "SELECT id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text FROM entries";
+        sql = `SELECT ${ENTRY_COLUMNS} FROM entries`;
         params = [];
     }
     const rows = db.prepare(sql).all(...params);
@@ -1350,9 +770,7 @@ export function getEntryById(db, id) {
     return { filePath: row.file_path, entry };
 }
 export function getEntriesByDir(db, dirPath) {
-    const rows = db
-        .prepare("SELECT id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text FROM entries WHERE dir_path = ?")
-        .all(dirPath);
+    const rows = db.prepare(`SELECT ${ENTRY_COLUMNS} FROM entries WHERE dir_path = ?`).all(dirPath);
     return parseEntryRows(rows, "getEntriesByDir");
 }
 /**
@@ -1633,6 +1051,11 @@ function bareRef(ref) {
  * entry_ref populated (see logCurateEvent), so curation is a real retrieval
  * signal here. Legacy summary-only curate rows with a NULL entry_ref simply
  * contribute nothing.
+ *
+ * Machine-sourced events (`source` = 'improve' or 'task') are EXCLUDED: this
+ * count feeds salience/ranking, and pipeline probe traffic counting as demand
+ * creates a self-reinforcing loop (meta-review 05 DRIFT-6). NULL sources
+ * (pre-column rows) count as user demand.
  */
 export function getRetrievalCounts(db, refs) {
     if (refs.length === 0)
@@ -1671,6 +1094,7 @@ export function getRetrievalCounts(db, refs) {
          FROM usage_events
          WHERE event_type IN ('search','show','curate')
            AND entry_ref IS NOT NULL
+           AND (source IS NULL OR source NOT IN ('improve','task'))
            AND CASE
                  WHEN instr(entry_ref, '//') > 0
                    THEN substr(entry_ref, instr(entry_ref, '//') + 2)
@@ -1823,84 +1247,24 @@ export function getEntryByRef(db, type, name) {
     return db.prepare("SELECT id FROM entries WHERE entry_type = ? AND entry_key = ?").get(type, `${type}:${name}`);
 }
 /**
- * MemRL learning rate for feedback-driven utility updates (F-5 / #386).
- *
- * Follows the bounded-step formula from MemRL (arXiv:2601.03192):
- *   next = clamp(current + lr × (reward − current), 0, 1)
- *
- * This replaces the unbounded `-0.03 × negativeCount` delta that could
- * silently remove high-utility assets from the improvement loop.
- */
-const FEEDBACK_LR = 0.1;
-/**
- * Positive reward signal for a single positive feedback event.
- * Reward 1.0 means "fully correct / helpful".
- */
-const FEEDBACK_REWARD_POSITIVE = 1.0;
-/**
- * Negative reward signal for a single negative feedback event.
- * Reward 0.0 means "not helpful" (lowest MemRL signal).
- */
-const FEEDBACK_REWARD_NEGATIVE = 0.0;
-/**
- * Maximum total negative utility delta allowed in a single
- * `applyFeedbackToUtilityScore` call regardless of negativeCount.
- *
- * This caps the per-day negative impact (the function is called once per
- * feedback event — spamming 10 negatives in one session can move utility
- * at most `MAX_NEG_DELTA_PER_CALL`). The cap prevents a noisy negative-
- * feedback stream from silently destroying a high-utility asset's ranking.
- */
-const MAX_NEG_DELTA_PER_CALL = 0.15;
-/**
- * Utility threshold below which a review-needed escalation is triggered.
- * When a previously high-utility asset (≥ HIGH_UTILITY_THRESHOLD) drops
- * below this value, the caller should create an escalation proposal.
- */
-export const UTILITY_REVIEW_THRESHOLD = 0.5;
-/**
- * Utility level considered "high" — assets above this are tracked for
- * threshold-crossing escalation.
- */
-export const HIGH_UTILITY_THRESHOLD = 0.5;
-/**
- * Apply accumulated feedback counts to the utility score of an entry using the
- * MemRL bounded-step EMA formula (F-5 / #386, arXiv:2601.03192).
- *
- * Replaces the previous unbounded `-0.03 × negativeCount` formula with:
- *
- *   reward   = weighted average of positive and negative signals
- *   nextUtil = clamp(currentUtil + lr × (reward − currentUtil), 0, 1)
- *
- * The negative impact is additionally capped at {@link MAX_NEG_DELTA_PER_CALL}
- * to prevent a noisy feedback stream from silently erasing a high-utility asset.
+ * Apply accumulated feedback counts to the utility score of an entry, persisting
+ * the result. The bounded-step EMA policy itself (MemRL, F-5 / #386,
+ * arXiv:2601.03192) lives in {@link computeNextUtility} (feedback/utility-policy);
+ * this function only reads the current utility, applies the policy, and writes
+ * the new value.
  *
  * A new entry starts at 0.5 (neutral midpoint) before the EMA step is applied.
- *
- * Returns a {@link FeedbackUtilityResult} so the caller can detect when a
- * previously high-utility asset crosses below the review threshold and create
- * an escalation proposal.
+ * When there is no feedback (both counts zero) the score is left untouched — no
+ * DB write. Returns a {@link FeedbackUtilityResult} so the caller can detect a
+ * previously high-utility asset crossing below the review threshold and escalate.
  */
 export function applyFeedbackToUtilityScore(db, entryId, positiveCount, negativeCount) {
     const existing = getUtilityScore(db, entryId);
     const previousUtility = existing?.utility ?? 0.5;
+    const result = computeNextUtility(previousUtility, positiveCount, negativeCount);
     if (positiveCount === 0 && negativeCount === 0) {
-        return { previousUtility, nextUtility: previousUtility, crossedReviewThreshold: false };
-    }
-    const total = positiveCount + negativeCount;
-    // Weighted reward: proportion of positive signals.
-    const reward = positiveCount > 0 && negativeCount === 0
-        ? FEEDBACK_REWARD_POSITIVE
-        : negativeCount > 0 && positiveCount === 0
-            ? FEEDBACK_REWARD_NEGATIVE
-            : (positiveCount * FEEDBACK_REWARD_POSITIVE + negativeCount * FEEDBACK_REWARD_NEGATIVE) / total;
-    // MemRL bounded-step EMA: lr × (reward − current)
-    let delta = FEEDBACK_LR * (reward - previousUtility);
-    // Per-call negative cap: if delta is negative (net negative feedback), cap it.
-    if (delta < 0) {
-        delta = Math.max(delta, -MAX_NEG_DELTA_PER_CALL);
+        return result;
     }
-    const nextUtility = Math.max(0, Math.min(1, previousUtility + delta));
     const now = new Date().toISOString();
     db.prepare(`
     INSERT INTO utility_scores (entry_id, utility, show_count, search_count, select_rate, last_used_at, updated_at)
@@ -1908,9 +1272,8 @@ export function applyFeedbackToUtilityScore(db, entryId, positiveCount, negative
     ON CONFLICT(entry_id) DO UPDATE SET
       utility    = ?,
       updated_at = ?
-  `).run(entryId, nextUtility, now, now, nextUtility, now);
-    const crossedReviewThreshold = previousUtility >= HIGH_UTILITY_THRESHOLD && nextUtility < UTILITY_REVIEW_THRESHOLD;
-    return { previousUtility, nextUtility, crossedReviewThreshold };
+  `).run(entryId, result.nextUtility, now, now, result.nextUtility, now);
+    return result;
 }
 /**
  * Re-link detached usage_events to their current entry_ids via entry_ref.