akm-cli 0.5.0 → 0.6.0-rc1

package/dist/db.js

@@ -2,7 +2,7 @@ import { Database } from "bun:sqlite";
 import fs from "node:fs";
 import { createRequire } from "node:module";
 import path from "node:path";
-import { cosineSimilarity } from "./embedder";
+import { cosineSimilarity } from "./embedders/types";
 import { getDbPath } from "./paths";
 import { buildSearchFields } from "./search-fields";
 import { ensureUsageEventsSchema } from "./usage-events";
@@ -73,7 +73,6 @@ export function warnIfVecMissing(db, { once } = { once: false }) {
         /* embeddings table may not exist yet during init */
     }
 }
-// ── Schema ──────────────────────────────────────────────────────────────────
 function ensureSchema(db, embeddingDim) {
     // Create meta table first so we can check version
     db.exec(`
@@ -84,30 +83,11 @@ function ensureSchema(db, embeddingDim) {
   `);
     // Check stored version — if it differs from DB_VERSION, drop and recreate all tables.
     // Usage events are preserved across version upgrades so that utility score
-    // history is not silently lost.
-    const storedVersion = getMeta(db, "version");
-    if (storedVersion && storedVersion !== String(DB_VERSION)) {
-        // Back up usage_events before dropping tables
-        let usageBackup = [];
-        try {
-            usageBackup = db.prepare("SELECT * FROM usage_events").all();
-        }
-        catch {
-            /* table may not exist in older versions */
-        }
-        db.exec("DROP TABLE IF EXISTS utility_scores");
-        db.exec("DROP TABLE IF EXISTS usage_events");
-        db.exec("DROP TABLE IF EXISTS embeddings");
-        db.exec("DROP TABLE IF EXISTS entries_vec");
-        db.exec("DROP TABLE IF EXISTS entries_fts");
-        db.exec("DROP INDEX IF EXISTS idx_entries_dir");
-        db.exec("DROP INDEX IF EXISTS idx_entries_type");
-        db.exec("DROP TABLE IF EXISTS entries");
-        db.exec("DELETE FROM index_meta");
-        // Store backup for restoration after ensureUsageEventsSchema runs
-        db.__usageBackup = usageBackup;
-        console.warn("[akm] Index rebuilt due to version upgrade. Run 'akm index' to repopulate.");
-    }
+    // history is not silently lost. The backup is captured here and threaded
+    // explicitly to `restoreUsageEventsBackup` below — the previous version
+    // attached `__usageBackup` to the Database instance via a typeless property
+    // injection, which was a source of fragile coupling.
+    const usageBackup = handleVersionUpgrade(db);
     db.exec(`
     CREATE TABLE IF NOT EXISTS entries (
       id          INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -220,31 +200,67 @@ function ensureSchema(db, embeddingDim) {
     }
     // Usage telemetry table
     ensureUsageEventsSchema(db);
-    // Restore usage_events that were backed up during a version upgrade.
-    // Wrapped in outer try/catch because schema changes across versions may
-    // make the backup incompatible with the new table definition.
-    const dbAny = db;
-    const backup = dbAny.__usageBackup;
-    if (backup && backup.length > 0) {
-        try {
-            db.transaction(() => {
-                const cols = Object.keys(backup[0]);
-                const placeholders = cols.map(() => "?").join(", ");
-                const insert = db.prepare(`INSERT INTO usage_events (${cols.join(", ")}) VALUES (${placeholders})`);
-                for (const row of backup) {
-                    try {
-                        insert.run(...cols.map((c) => row[c]));
-                    }
-                    catch {
-                        /* skip rows that fail */
-                    }
+    // Restore usage_events backed up by the version-upgrade path above.
+    restoreUsageEventsBackup(db, usageBackup);
+}
+/**
+ * Detect a stored DB version that differs from {@link DB_VERSION}, drop the
+ * old schema, and return a backup of the previous `usage_events` rows so the
+ * rest of `ensureSchema()` can restore them once the new table exists.
+ *
+ * Returns an empty array when no upgrade is needed or when the previous
+ * `usage_events` table is unreadable.
+ */
+function handleVersionUpgrade(db) {
+    const storedVersion = getMeta(db, "version");
+    if (!storedVersion || storedVersion === String(DB_VERSION))
+        return [];
+    let usageBackup = [];
+    try {
+        usageBackup = db.prepare("SELECT * FROM usage_events").all();
+    }
+    catch {
+        /* table may not exist in older versions */
+    }
+    db.exec("DROP TABLE IF EXISTS utility_scores");
+    db.exec("DROP TABLE IF EXISTS usage_events");
+    db.exec("DROP TABLE IF EXISTS embeddings");
+    db.exec("DROP TABLE IF EXISTS entries_vec");
+    db.exec("DROP TABLE IF EXISTS entries_fts");
+    db.exec("DROP INDEX IF EXISTS idx_entries_dir");
+    db.exec("DROP INDEX IF EXISTS idx_entries_type");
+    db.exec("DROP TABLE IF EXISTS entries");
+    db.exec("DELETE FROM index_meta");
+    console.warn("[akm] Index rebuilt due to version upgrade. Run 'akm index' to repopulate.");
+    return usageBackup;
+}
+/**
+ * Re-insert backed-up `usage_events` rows into the freshly-created table.
+ *
+ * Wrapped in an outer try/catch because schema changes across versions may
+ * make the backup incompatible with the new table definition; in that case
+ * the backup is discarded silently rather than blocking startup.
+ */
+function restoreUsageEventsBackup(db, backup) {
+    if (backup.length === 0)
+        return;
+    try {
+        db.transaction(() => {
+            const cols = Object.keys(backup[0]);
+            const placeholders = cols.map(() => "?").join(", ");
+            const insert = db.prepare(`INSERT INTO usage_events (${cols.join(", ")}) VALUES (${placeholders})`);
+            for (const row of backup) {
+                try {
+                    insert.run(...cols.map((c) => row[c]));
                 }
-            })();
-        }
-        catch {
-            /* schema changed too much — discard backup gracefully */
-        }
-        delete dbAny.__usageBackup;
+                catch {
+                    /* skip rows that fail */
+                }
+            }
+        })();
+    }
+    catch {
+        /* schema changed too much — discard backup gracefully */
     }
 }
 // ── Meta helpers ────────────────────────────────────────────────────────────
@@ -280,8 +296,30 @@ export function upsertEntry(db, entryKey, dirPath, filePath, stashDir, entry, se
     const row = db.prepare("SELECT id FROM entries WHERE entry_key = ?").get(entryKey);
     if (!row)
         throw new Error("upsertEntry: entry_key not found after upsert");
+    // Mark this entry as FTS-dirty so an incremental rebuild only revisits the
+    // entries that actually changed. Without this, every `akm index` run had to
+    // re-scan and re-insert every FTS row, even if only one entry was touched.
+    markFtsDirty(db, row.id);
     return row.id;
 }
+/**
+ * Mark an entry as needing FTS re-indexing on the next `rebuildFts` call.
+ *
+ * The list lives in a small `entries_fts_dirty` table — a per-entry-id queue
+ * of work items. `rebuildFts({ incremental: true })` drains this list rather
+ * than scanning the entire `entries` table.
+ */
+function markFtsDirty(db, entryId) {
+    ensureFtsDirtyTable(db);
+    db.prepare("INSERT OR IGNORE INTO entries_fts_dirty (entry_id) VALUES (?)").run(entryId);
+}
+function ensureFtsDirtyTable(db) {
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS entries_fts_dirty (
+      entry_id INTEGER PRIMARY KEY
+    );
+  `);
+}
 export function deleteEntriesByDir(db, dirPath) {
     db.transaction(() => {
         const ids = db.prepare("SELECT id FROM entries WHERE dir_path = ?").all(dirPath);
@@ -302,6 +340,25 @@ function deleteRelatedRows(db, ids) {
         return;
     const numericIds = ids.map((r) => r.id);
     const vecAvail = isVecAvailable(db);
+    // Drop matching FTS rows + dirty markers immediately so an incremental
+    // rebuild after a deletion doesn't try to re-index entries that no longer
+    // exist (and so a full scan after deletion sees a consistent FTS).
+    for (let i = 0; i < numericIds.length; i += SQLITE_CHUNK_SIZE) {
+        const chunk = numericIds.slice(i, i + SQLITE_CHUNK_SIZE);
+        const placeholders = chunk.map(() => "?").join(",");
+        try {
+            db.prepare(`DELETE FROM entries_fts WHERE entry_id IN (${placeholders})`).run(...chunk);
+        }
+        catch {
+            /* fts table may not exist on a brand-new db */
+        }
+        try {
+            db.prepare(`DELETE FROM entries_fts_dirty WHERE entry_id IN (${placeholders})`).run(...chunk);
+        }
+        catch {
+            /* dirty table is created lazily by upsertEntry */
+        }
+    }
     // Process in chunks to stay within SQLITE_MAX_VARIABLE_NUMBER
     for (let i = 0; i < numericIds.length; i += SQLITE_CHUNK_SIZE) {
         const chunk = numericIds.slice(i, i + SQLITE_CHUNK_SIZE);
@@ -343,19 +400,52 @@ function deleteRelatedRows(db, ids) {
         }
     }
 }
-export function rebuildFts(db) {
-    // Wrap DELETE + INSERT in a single transaction so the FTS table is
-    // never left empty between the two statements if a crash occurs.
-    // Store the integer id directly (FTS5 stores all content as text
-    // internally; the join in searchFts compares numerically without CAST).
-    //
-    // Insert into separate FTS5 columns by extracting per-field text from
-    // the entry_json using buildSearchFields(). The entries.search_text column
-    // is kept as a concatenated fallback for embedding generation.
+/**
+ * Rebuild the FTS5 search index.
+ *
+ * `incremental` (default `false`): when true, only rebuild rows that
+ * `upsertEntry` marked dirty since the last `rebuildFts` call. The full path
+ * (default) wipes `entries_fts` and re-inserts every row from `entries` —
+ * appropriate for `akm index --full` and version-upgrade rebuilds.
+ *
+ * Both paths are wrapped in a single transaction so the FTS table is never
+ * left in a half-rebuilt state.
+ *
+ * Skipped corrupt-JSON rows are aggregated into one warning instead of
+ * spamming stderr per-entry.
+ */
+export function rebuildFts(db, options) {
+    const incremental = options?.incremental === true;
     db.transaction(() => {
-        db.exec("DELETE FROM entries_fts");
-        const rows = db.prepare("SELECT id, entry_json FROM entries").all();
+        let rows;
+        if (incremental) {
+            ensureFtsDirtyTable(db);
+            // Read the dirty queue and join against entries to get the JSON.
+            // Then drop the matching rows from entries_fts so the INSERT below
+            // doesn't double-up. The dirty list is drained at the end.
+            rows = db
+                .prepare(`SELECT e.id AS id, e.entry_json AS entry_json
+             FROM entries_fts_dirty d
+             JOIN entries e ON e.id = d.entry_id`)
+                .all();
+            if (rows.length === 0)
+                return;
+            const ids = rows.map((r) => r.id);
+            // Delete only the dirty FTS rows — chunk to stay under
+            // SQLITE_MAX_VARIABLE_NUMBER on large dirty queues.
+            for (let i = 0; i < ids.length; i += SQLITE_CHUNK_SIZE) {
+                const chunk = ids.slice(i, i + SQLITE_CHUNK_SIZE);
+                const placeholders = chunk.map(() => "?").join(",");
+                db.prepare(`DELETE FROM entries_fts WHERE entry_id IN (${placeholders})`).run(...chunk);
+            }
+        }
+        else {
+            // Full path: wipe and re-read every row.
+            db.exec("DELETE FROM entries_fts");
+            rows = db.prepare("SELECT id, entry_json FROM entries").all();
+        }
         const insertStmt = db.prepare("INSERT INTO entries_fts (entry_id, name, description, tags, hints, content) VALUES (?, ?, ?, ?, ?, ?)");
+        let skipped = 0;
         for (const row of rows) {
             let entry;
             let fields;
@@ -364,11 +454,27 @@ export function rebuildFts(db) {
                 fields = buildSearchFields(entry);
             }
             catch {
-                warn(`[db] rebuildFts: skipping entry id=${row.id} — invalid entry_json`);
+                skipped++;
                 continue;
             }
             insertStmt.run(row.id, fields.name, fields.description, fields.tags, fields.hints, fields.content);
         }
+        if (skipped > 0) {
+            warn(`[db] rebuildFts: skipped ${skipped} entr${skipped === 1 ? "y" : "ies"} with invalid entry_json`);
+        }
+        // Always drain the dirty queue — if it exists. A full rebuild also
+        // clears it because the full path covers everything the dirty list
+        // tracks.
+        if (incremental) {
+            db.exec("DELETE FROM entries_fts_dirty");
+        }
+        else {
+            // Full path: only drop the dirty table if it exists. Use
+            // `CREATE IF NOT EXISTS` then DELETE so we don't error on databases
+            // that haven't run any upserts yet (e.g. fresh schema).
+            ensureFtsDirtyTable(db);
+            db.exec("DELETE FROM entries_fts_dirty");
+        }
     })();
 }
 // ── Vector operations ───────────────────────────────────────────────────────