akm-cli 0.9.0-beta.11 → 0.9.0-beta.26

package/dist/indexer/db/db.js

@@ -12,13 +12,22 @@ import { warn } from "../../core/warn.js";
 import { cosineSimilarity } from "../../llm/embedders/types.js";
 import { sha256Hex } from "../../runtime.js";
 import { openDatabase as openSqlite } from "../../storage/database.js";
+import { applyStandardPragmas } from "../../storage/sqlite-pragmas.js";
 import { buildSearchFields } from "../search/search-fields.js";
 import { ensureUsageEventsSchema } from "../usage/usage-events.js";
 import { backupDataDir, EMBEDDING_DIM_CHANGE_REASON } from "./db-backup.js";
 // ── Constants ───────────────────────────────────────────────────────────────
+// NOTE: do NOT bump DB_VERSION for graph-schema changes. A DB_VERSION mismatch
+// triggers handleVersionUpgrade()'s NUCLEAR drop of the ENTIRE index — entries,
+// embeddings, FTS, and the llm_enrichment_cache — forcing every user to re-embed
+// their whole corpus on upgrade. The graph tables are derived and cheap to
+// rebuild, so graph re-keying is migrated in a TARGETED, graph-only path
+// (migrateGraphFilesSchema) that leaves entries + embeddings untouched.
 export const DB_VERSION = 17;
 export const EMBEDDING_DIM = 384;
-export const GRAPH_SCHEMA_VERSION = 3;
+// #624-P1: graph_files re-keyed to (stash_root, file_path, body_hash). Bumped 3→4
+// as a marker; the actual migration is the targeted drop in migrateGraphFilesSchema.
+export const GRAPH_SCHEMA_VERSION = 4;
 // ── Database lifecycle ──────────────────────────────────────────────────────
 export function openDatabase(dbPath, options) {
     const resolvedPath = dbPath ?? getDbPath();
@@ -27,9 +36,7 @@ export function openDatabase(dbPath, options) {
         fs.mkdirSync(dir, { recursive: true });
     }
     const db = openSqlite(resolvedPath);
-    db.exec("PRAGMA journal_mode = WAL");
-    db.exec("PRAGMA busy_timeout = 30000");
-    db.exec("PRAGMA foreign_keys = ON");
+    applyStandardPragmas(db, { dataDir: dir });
     // Try to load sqlite-vec extension
     loadVecExtension(db);
     // Dim resolution: explicit option wins; otherwise consult the on-disk
@@ -67,10 +74,9 @@ function resolveConfiguredEmbeddingDim() {
 }
 export function openExistingDatabase(dbPath) {
     const resolvedPath = dbPath ?? getDbPath();
+    const dir = path.dirname(resolvedPath);
     const db = openSqlite(resolvedPath);
-    db.exec("PRAGMA journal_mode = WAL");
-    db.exec("PRAGMA busy_timeout = 30000");
-    db.exec("PRAGMA foreign_keys = ON");
+    applyStandardPragmas(db, { dataDir: dir });
     // Existing-DB callers must not mutate schema or embedding metadata on open,
     // but some paths still need write access to usage_events and other tables.
     loadVecExtension(db);
@@ -299,15 +305,27 @@ function ensureSchema(db, embeddingDim, options) {
      CREATE INDEX IF NOT EXISTS idx_llm_cache_updated
        ON llm_enrichment_cache(updated_at);
   `);
-    // Graph extraction tables — schema v2 (entry_id PK).
+    // Graph extraction tables — schema v4 ((stash_root, file_path, body_hash) PK).
+    //
+    // graph_files is self-keyed on (stash_root, file_path, body_hash) and is NO
+    // LONGER tied to entries.id. This is the #624-P1 win: deleting and
+    // re-inserting an entries row during a reindex no longer cascade-wipes the
+    // extracted graph — as long as the file's body_hash is unchanged, the graph
+    // data survives. body_hash is part of the PK so a content change yields a
+    // distinct key; a UNIQUE index on (stash_root, file_path) still enforces
+    // exactly one graph_files row per path (delete-then-insert on a hash change).
     //
-    // graph_files is keyed on entries.id so child tables cascade-delete cleanly
-    // when an entry is removed, and so JOINs from graph rows to entries are a
-    // direct PK lookup. (stash_root, file_path) is retained as UNIQUE so the
-    // extractor's path-based upsert still works.
+    // graph_file_entities and graph_file_relations carry (stash_root, file_path,
+    // body_hash) and declare a composite FK -> graph_files ON DELETE CASCADE so
+    // child rows are removed when a graph_files row is replaced.
     //
-    // graph_file_entities and graph_file_relations no longer duplicate file_path;
-    // they reference entry_id and inherit stash scoping via graph_files.
+    // #624-P1 targeted migration: an existing DB may still hold the OLD graph_files
+    // (entry_id PK). SQLite can't ALTER a primary key, so we RENAME the 3 graph
+    // tables aside (→ *_legacy) here — ONLY the graph tables, never the index/
+    // embeddings — then the CREATE block below builds the new shape, then
+    // migrateGraphDataFromLegacy() copies the data across so the graph is PRESERVED
+    // (not re-extracted).
+    migrateGraphFilesSchema(db);
     db.exec(`
     CREATE TABLE IF NOT EXISTS graph_meta (
       stash_root          TEXT PRIMARY KEY,
@@ -331,7 +349,6 @@ function ensureSchema(db, embeddingDim, options) {
     );
 
     CREATE TABLE IF NOT EXISTS graph_files (
-      entry_id          INTEGER PRIMARY KEY REFERENCES entries(id) ON DELETE CASCADE,
       stash_root        TEXT NOT NULL,
       file_path         TEXT NOT NULL,
       file_order        INTEGER NOT NULL,
@@ -341,26 +358,34 @@ function ensureSchema(db, embeddingDim, options) {
       status            TEXT NOT NULL DEFAULT 'extracted',
       reason            TEXT,
       extraction_run_id TEXT,
-      UNIQUE(stash_root, file_path)
+      PRIMARY KEY (stash_root, file_path, body_hash)
     );
 
+    CREATE UNIQUE INDEX IF NOT EXISTS idx_graph_files_path
+      ON graph_files(stash_root, file_path);
+
     CREATE INDEX IF NOT EXISTS idx_graph_files_stash_order
       ON graph_files(stash_root, file_order);
 
     CREATE TABLE IF NOT EXISTS graph_file_entities (
-      entry_id     INTEGER NOT NULL REFERENCES graph_files(entry_id) ON DELETE CASCADE,
-      entity_order INTEGER NOT NULL,
       stash_root   TEXT NOT NULL,
+      file_path    TEXT NOT NULL,
+      body_hash    TEXT NOT NULL,
+      entity_order INTEGER NOT NULL,
       entity_norm  TEXT NOT NULL,
       entity       TEXT NOT NULL,
-      PRIMARY KEY (entry_id, entity_order)
+      PRIMARY KEY (stash_root, file_path, body_hash, entity_order),
+      FOREIGN KEY (stash_root, file_path, body_hash)
+        REFERENCES graph_files(stash_root, file_path, body_hash) ON DELETE CASCADE
     );
 
     CREATE INDEX IF NOT EXISTS idx_graph_file_entities_entity_norm
       ON graph_file_entities(stash_root, entity_norm);
 
     CREATE TABLE IF NOT EXISTS graph_file_relations (
-      entry_id       INTEGER NOT NULL REFERENCES graph_files(entry_id) ON DELETE CASCADE,
+      stash_root     TEXT NOT NULL,
+      file_path      TEXT NOT NULL,
+      body_hash      TEXT NOT NULL,
       relation_order INTEGER NOT NULL,
       from_entity_norm TEXT NOT NULL,
       from_entity    TEXT NOT NULL,
@@ -368,9 +393,15 @@ function ensureSchema(db, embeddingDim, options) {
       to_entity      TEXT NOT NULL,
       relation_type  TEXT,
       confidence     REAL,
-      PRIMARY KEY (entry_id, relation_order)
+      PRIMARY KEY (stash_root, file_path, body_hash, relation_order),
+      FOREIGN KEY (stash_root, file_path, body_hash)
+        REFERENCES graph_files(stash_root, file_path, body_hash) ON DELETE CASCADE
     );
   `);
+    // #624-P1 migration step 2: copy any renamed-aside legacy graph data into the
+    // new-shape tables (just created above), then drop the legacy tables. No-op
+    // unless migrateGraphFilesSchema renamed a legacy graph_files this open.
+    migrateGraphDataFromLegacy(db);
     // FTS-dirty queue. Created here (not lazily on first upsert) so the
     // per-entry write path doesn't issue a CREATE TABLE IF NOT EXISTS on
     // every call — that DDL would fire thousands of times during a full
@@ -715,6 +746,110 @@ function ensureDerivedFromColumn(db) {
         db.exec("CREATE INDEX IF NOT EXISTS idx_entries_derived_from ON entries(derived_from)");
     }, "entries table may not exist on a brand-new DB before CREATE — caller is responsible");
 }
+/**
+ * Returns true when a table exists in the current database.
+ */
+function tableExists(db, name) {
+    const row = db.prepare("SELECT 1 FROM sqlite_master WHERE type='table' AND name=? LIMIT 1").get(name);
+    return row !== undefined && row !== null;
+}
+/**
+ * #624-P1 targeted graph-schema migration — STEP 1 of 2 (rename).
+ *
+ * graph_files was re-keyed from `entry_id INTEGER PRIMARY KEY REFERENCES
+ * entries(id)` to a self-contained `(stash_root, file_path, body_hash)` PK.
+ * SQLite cannot ALTER a primary key, so an existing DB carrying the OLD shape
+ * has its 3 graph tables RENAMED to `*_legacy` here; ensureSchema's CREATE block
+ * then builds the new-shape tables, and {@link migrateGraphDataFromLegacy} COPIES
+ * the data across before dropping the legacy tables. The graph is preserved —
+ * NOT re-extracted (re-extraction is ~19s/file of LLM work).
+ *
+ * Crucially this is GRAPH-SCOPED: it touches ONLY the graph tables, never the
+ * index / embeddings / enrichment cache. So users keep their (expensive)
+ * embeddings instead of being forced into a full re-embed by a DB_VERSION bump.
+ *
+ * Detection: the old schema has an `entry_id` column on graph_files. Fresh DBs
+ * (no graph_files yet) and already-migrated DBs (no entry_id column) are no-ops.
+ * Idempotent.
+ */
+function migrateGraphFilesSchema(db) {
+    bestEffort(() => {
+        const cols = db.prepare("PRAGMA table_info(graph_files)").all();
+        const isLegacyShape = cols.some((c) => c.name === "entry_id");
+        if (!isLegacyShape)
+            return;
+        // A previous interrupted migration may have left *_legacy behind — drop those
+        // husks first so the rename below doesn't collide.
+        db.exec("DROP TABLE IF EXISTS graph_file_relations_legacy");
+        db.exec("DROP TABLE IF EXISTS graph_file_entities_legacy");
+        db.exec("DROP TABLE IF EXISTS graph_files_legacy");
+        // Rename the 3 entry_id-keyed tables aside. graph_meta is unchanged (stash_root
+        // key) so it is left in place. ALTER … RENAME auto-updates child FK refs in
+        // SQLite ≥3.25, which is fine — the legacy children are dropped after the copy.
+        db.exec("ALTER TABLE graph_files RENAME TO graph_files_legacy");
+        if (tableExists(db, "graph_file_entities")) {
+            db.exec("ALTER TABLE graph_file_entities RENAME TO graph_file_entities_legacy");
+        }
+        if (tableExists(db, "graph_file_relations")) {
+            db.exec("ALTER TABLE graph_file_relations RENAME TO graph_file_relations_legacy");
+        }
+    }, "graph_files may not exist on a brand-new DB before CREATE — caller is responsible");
+}
+/**
+ * #624-P1 targeted graph-schema migration — STEP 2 of 2 (copy + drop legacy).
+ *
+ * Runs AFTER the graph CREATE TABLE block, so the new-shape tables exist. Copies
+ * every legacy row into the re-keyed tables — the old tables already carry
+ * (stash_root, file_path, body_hash) next to entry_id, so the projection is a
+ * straight column copy (children JOIN back to graph_files_legacy to resolve the
+ * composite key from their entry_id). Then drops the `*_legacy` tables.
+ *
+ * Best-effort: a copy failure (e.g. a pre-body_hash legacy schema) is tolerated,
+ * and the legacy tables are dropped regardless so they never linger. Rows whose
+ * body_hash is null/empty can't form the new PK and are skipped (they re-extract).
+ */
+function migrateGraphDataFromLegacy(db) {
+    if (!tableExists(db, "graph_files_legacy"))
+        return;
+    let migratedFiles = 0;
+    bestEffort(() => {
+        db.transaction(() => {
+            const res = db
+                .prepare(`INSERT OR IGNORE INTO graph_files
+             (stash_root, file_path, body_hash, file_order, file_type, confidence, status, reason, extraction_run_id)
+           SELECT stash_root, file_path, body_hash, file_order, file_type, confidence, status, reason, extraction_run_id
+             FROM graph_files_legacy
+            WHERE body_hash IS NOT NULL AND body_hash != ''`)
+                .run();
+            migratedFiles = Number(res.changes);
+            if (tableExists(db, "graph_file_entities_legacy")) {
+                db.exec(`INSERT OR IGNORE INTO graph_file_entities
+             (stash_root, file_path, body_hash, entity_order, entity_norm, entity)
+           SELECT gf.stash_root, gf.file_path, gf.body_hash, e.entity_order, e.entity_norm, e.entity
+             FROM graph_file_entities_legacy e
+             JOIN graph_files_legacy gf ON gf.entry_id = e.entry_id
+            WHERE gf.body_hash IS NOT NULL AND gf.body_hash != ''`);
+            }
+            if (tableExists(db, "graph_file_relations_legacy")) {
+                db.exec(`INSERT OR IGNORE INTO graph_file_relations
+             (stash_root, file_path, body_hash, relation_order, from_entity_norm, from_entity, to_entity_norm, to_entity, relation_type, confidence)
+           SELECT gf.stash_root, gf.file_path, gf.body_hash, r.relation_order, r.from_entity_norm, r.from_entity, r.to_entity_norm, r.to_entity, r.relation_type, r.confidence
+             FROM graph_file_relations_legacy r
+             JOIN graph_files_legacy gf ON gf.entry_id = r.entry_id
+            WHERE gf.body_hash IS NOT NULL AND gf.body_hash != ''`);
+            }
+        })();
+    }, "graph data migration is best-effort; legacy tables are dropped regardless below");
+    // Always drop the legacy tables (children first), migrated or not.
+    bestEffort(() => {
+        db.exec("DROP TABLE IF EXISTS graph_file_relations_legacy");
+        db.exec("DROP TABLE IF EXISTS graph_file_entities_legacy");
+        db.exec("DROP TABLE IF EXISTS graph_files_legacy");
+    }, "drop legacy graph tables after migration");
+    if (migratedFiles > 0) {
+        warn(`[akm] graph index re-keyed (#624): migrated ${migratedFiles} extracted file(s) to the new schema — no re-extraction needed. Index + embeddings untouched.`);
+    }
+}
 /**
  * Phase 5A / Advantage D5: look up the derived-memory child row whose
  * `derived_from` column matches `parentRef` (e.g. `"memory:claude-prefs"`).
@@ -839,6 +974,38 @@ function deleteRelatedRows(db, ids) {
         // Clean up usage events before deleting entries
         bestEffort(() => db.prepare(`DELETE FROM usage_events WHERE entry_id IN (${placeholders})`).run(...chunk), "delete usage_events for entries");
     }
+    // #624-P1: graph_files is NO LONGER keyed on entries.id, so deleting an
+    // entries row must NOT wipe the extracted graph (that is the whole point —
+    // the graph survives a reindex when body_hash is unchanged). We therefore do
+    // NOT delete graph_files here. We DO, however, recompute graph_meta counts
+    // for the stash roots touched by the deleted entries so the summary numbers
+    // stay consistent with the live child rows (the counts are derived, and the
+    // entries delete may have changed which files are considered/indexed).
+    //
+    // Resolve the affected stash roots from the entries rows BEFORE deletion.
+    const affectedStashRoots = new Set();
+    for (let i = 0; i < numericIds.length; i += SQLITE_CHUNK_SIZE) {
+        const chunk = numericIds.slice(i, i + SQLITE_CHUNK_SIZE);
+        const placeholders = chunk.map(() => "?").join(",");
+        bestEffort(() => {
+            const rows = db
+                .prepare(`SELECT DISTINCT stash_dir FROM entries WHERE id IN (${placeholders})`)
+                .all(...chunk);
+            for (const row of rows) {
+                if (row.stash_dir)
+                    affectedStashRoots.add(row.stash_dir);
+            }
+        }, "resolve stash roots for graph_meta recompute");
+    }
+    for (const stashRoot of affectedStashRoots) {
+        bestEffort(() => db
+            .prepare(`UPDATE graph_meta
+             SET extracted_files = (SELECT COUNT(*) FROM graph_files WHERE stash_root = ?),
+                 entity_count    = (SELECT COUNT(*) FROM graph_file_entities WHERE stash_root = ?),
+                 relation_count  = (SELECT COUNT(*) FROM graph_file_relations WHERE stash_root = ?)
+             WHERE stash_root = ?`)
+            .run(stashRoot, stashRoot, stashRoot, stashRoot), "sync graph_meta counts after entries delete");
+    }
 }
 /**
  * Delete entries by their primary key IDs, along with all related rows
@@ -1048,12 +1215,12 @@ function searchBlobVec(db, queryEmbedding, k) {
     }
 }
 // ── FTS5 search ─────────────────────────────────────────────────────────────
-export function searchFts(db, query, limit, entryType) {
+export function searchFts(db, query, limit, entryType, excludeTypes) {
     const ftsQuery = sanitizeFtsQuery(query);
     if (!ftsQuery)
         return [];
     // Try the exact AND query first
-    const exactResults = runFtsQuery(db, ftsQuery, limit, entryType);
+    const exactResults = runFtsQuery(db, ftsQuery, limit, entryType, excludeTypes);
     if (exactResults.length > 0)
         return exactResults;
     // Exact match returned zero results — try prefix fallback.
@@ -1063,7 +1230,7 @@ export function searchFts(db, query, limit, entryType) {
     const prefixQuery = buildPrefixQuery(ftsQuery);
     if (!prefixQuery)
         return [];
-    return runFtsQuery(db, prefixQuery, limit, entryType);
+    return runFtsQuery(db, prefixQuery, limit, entryType, excludeTypes);
 }
 /**
  * Build a prefix query from an FTS5 query string by appending `*` to each
@@ -1086,9 +1253,14 @@ function buildPrefixQuery(ftsQuery) {
         return null;
     return prefixTokens.join(" ");
 }
-function runFtsQuery(db, ftsQuery, limit, entryType) {
+function runFtsQuery(db, ftsQuery, limit, entryType, excludeTypes) {
     let sql;
     let params;
+    // #627 — exclude-type clause. Only applies on the untyped ('any') path; an
+    // explicit include filter (entryType) already narrows to a single type, so
+    // exclusion is redundant there. An empty list skips the clause entirely
+    // (never emit `NOT IN ()`, which is a SQL error / always-false).
+    const excludes = excludeTypes && excludeTypes.length > 0 ? excludeTypes : [];
     // Join on integer entry_id directly (no CAST needed; we store integer)
     // Use bm25() with per-column weights: entry_id(0), name(10), description(5), tags(3), hints(2), content(1)
     if (entryType && entryType !== "any") {
@@ -1105,16 +1277,19 @@ function runFtsQuery(db, ftsQuery, limit, entryType) {
         params = [ftsQuery, entryType, limit];
     }
     else {
+        const excludeClause = excludes.length > 0 ? `AND e.entry_type NOT IN (${excludes.map(() => "?").join(", ")})` : "";
         sql = `
       SELECT e.id, e.file_path AS filePath, e.entry_json, e.search_text AS searchText,
              bm25(entries_fts, 0, 10.0, 5.0, 3.0, 2.0, 1.0) AS bm25Score
       FROM entries_fts f
       JOIN entries e ON e.id = f.entry_id
       WHERE entries_fts MATCH ?
+        ${excludeClause}
       ORDER BY bm25Score, e.id ASC
       LIMIT ?
     `;
-        params = [ftsQuery, limit];
+        // Param order: MATCH, then the NOT IN values, then LIMIT.
+        params = [ftsQuery, ...excludes, limit];
     }
     try {
         const rows = db.prepare(sql).all(...params);
@@ -1183,14 +1358,21 @@ function parseEntryRows(rows, context) {
     }
     return entries;
 }
-export function getAllEntries(db, entryType) {
+export function getAllEntries(db, entryType, excludeTypes) {
     let sql;
     let params;
+    // #627 — exclude-type clause applies only on the untyped ('any') path. Empty
+    // list skips the clause (never `NOT IN ()`).
+    const excludes = excludeTypes && excludeTypes.length > 0 ? excludeTypes : [];
     if (entryType && entryType !== "any") {
         sql =
             "SELECT id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text FROM entries WHERE entry_type = ?";
         params = [entryType];
     }
+    else if (excludes.length > 0) {
+        sql = `SELECT id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text FROM entries WHERE entry_type NOT IN (${excludes.map(() => "?").join(", ")})`;
+        params = [...excludes];
+    }
     else {
         sql = "SELECT id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text FROM entries";
         params = [];
@@ -1198,6 +1380,47 @@ export function getAllEntries(db, entryType) {
     const rows = db.prepare(sql).all(...params);
     return parseEntryRows(rows, "getAllEntries");
 }
+/**
+ * #609 — read graph entities (normalized) for a set of entry ids. Used by the
+ * recombine pass to cluster memories by shared graph entity ("graph"
+ * relatedness source). Returns a map of `entry_id -> entity_norm[]`. Entries
+ * with no graph entities (graph extraction has not run, or the file produced
+ * no entities) are simply absent from the map — callers must fail open
+ * (fall back to tag relatedness) when the map is empty.
+ */
+export function getEntitiesByEntryIds(db, entryIds) {
+    const result = new Map();
+    if (entryIds.length === 0)
+        return result;
+    // #624-P1: graph_file_entities no longer carries entry_id. Re-derive the
+    // entry_id -> entity_norm[] contract by JOINing through entries on
+    // (stash_dir, file_path) -> graph_files. Chunk the IN(?) list because the
+    // recombine pass can pass 10k+ entry ids (well over the SQLite param limit).
+    for (let i = 0; i < entryIds.length; i += SQLITE_CHUNK_SIZE) {
+        const chunk = entryIds.slice(i, i + SQLITE_CHUNK_SIZE);
+        const placeholders = chunk.map(() => "?").join(", ");
+        const rows = db
+            .prepare(`SELECT e.id AS entry_id, gfe.entity_norm AS entity_norm
+           FROM entries e
+           JOIN graph_files gf
+             ON gf.stash_root = e.stash_dir AND gf.file_path = e.file_path
+           JOIN graph_file_entities gfe
+             ON gfe.stash_root = gf.stash_root
+            AND gfe.file_path = gf.file_path
+            AND gfe.body_hash = gf.body_hash
+          WHERE e.id IN (${placeholders})
+          ORDER BY e.id, gfe.entity_order`)
+            .all(...chunk);
+        for (const row of rows) {
+            const list = result.get(row.entry_id);
+            if (list)
+                list.push(row.entity_norm);
+            else
+                result.set(row.entry_id, [row.entity_norm]);
+        }
+    }
+    return result;
+}
 export function findEntryIdByRef(db, ref) {
     const parsed = parseAssetRef(ref);
     const nameVariants = [parsed.name];