akm-cli 0.5.0 → 0.6.0-rc2

package/dist/{db.js → indexer/db.js}

@@ -2,13 +2,13 @@ import { Database } from "bun:sqlite";
 import fs from "node:fs";
 import { createRequire } from "node:module";
 import path from "node:path";
-import { cosineSimilarity } from "./embedder";
-import { getDbPath } from "./paths";
+import { getDbPath } from "../core/paths";
+import { warn } from "../core/warn";
+import { cosineSimilarity } from "../llm/embedders/types";
 import { buildSearchFields } from "./search-fields";
 import { ensureUsageEventsSchema } from "./usage-events";
-import { warn } from "./warn";
 // ── Constants ───────────────────────────────────────────────────────────────
-export const DB_VERSION = 8;
+export const DB_VERSION = 9;
 export const EMBEDDING_DIM = 384;
 // ── Database lifecycle ──────────────────────────────────────────────────────
 export function openDatabase(dbPath, options) {
@@ -19,6 +19,7 @@ export function openDatabase(dbPath, options) {
     }
     const db = new Database(resolvedPath);
     db.exec("PRAGMA journal_mode = WAL");
+    db.exec("PRAGMA busy_timeout = 5000");
     db.exec("PRAGMA foreign_keys = ON");
     // Try to load sqlite-vec extension
     loadVecExtension(db);
@@ -73,7 +74,6 @@ export function warnIfVecMissing(db, { once } = { once: false }) {
         /* embeddings table may not exist yet during init */
     }
 }
-// ── Schema ──────────────────────────────────────────────────────────────────
 function ensureSchema(db, embeddingDim) {
     // Create meta table first so we can check version
     db.exec(`
@@ -84,30 +84,11 @@ function ensureSchema(db, embeddingDim) {
   `);
     // Check stored version — if it differs from DB_VERSION, drop and recreate all tables.
     // Usage events are preserved across version upgrades so that utility score
-    // history is not silently lost.
-    const storedVersion = getMeta(db, "version");
-    if (storedVersion && storedVersion !== String(DB_VERSION)) {
-        // Back up usage_events before dropping tables
-        let usageBackup = [];
-        try {
-            usageBackup = db.prepare("SELECT * FROM usage_events").all();
-        }
-        catch {
-            /* table may not exist in older versions */
-        }
-        db.exec("DROP TABLE IF EXISTS utility_scores");
-        db.exec("DROP TABLE IF EXISTS usage_events");
-        db.exec("DROP TABLE IF EXISTS embeddings");
-        db.exec("DROP TABLE IF EXISTS entries_vec");
-        db.exec("DROP TABLE IF EXISTS entries_fts");
-        db.exec("DROP INDEX IF EXISTS idx_entries_dir");
-        db.exec("DROP INDEX IF EXISTS idx_entries_type");
-        db.exec("DROP TABLE IF EXISTS entries");
-        db.exec("DELETE FROM index_meta");
-        // Store backup for restoration after ensureUsageEventsSchema runs
-        db.__usageBackup = usageBackup;
-        console.warn("[akm] Index rebuilt due to version upgrade. Run 'akm index' to repopulate.");
-    }
+    // history is not silently lost. The backup is captured here and threaded
+    // explicitly to `restoreUsageEventsBackup` below — the previous version
+    // attached `__usageBackup` to the Database instance via a typeless property
+    // injection, which was a source of fragile coupling.
+    const usageBackup = handleVersionUpgrade(db);
     db.exec(`
     CREATE TABLE IF NOT EXISTS entries (
       id          INTEGER PRIMARY KEY AUTOINCREMENT,
@@ -122,6 +103,22 @@ function ensureSchema(db, embeddingDim) {
 
     CREATE INDEX IF NOT EXISTS idx_entries_dir ON entries(dir_path);
     CREATE INDEX IF NOT EXISTS idx_entries_type ON entries(entry_type);
+  `);
+    // Validated WorkflowDocument JSON, one row per indexed workflow entry.
+    // Pure index data — fully rebuilt on each `akm index`. ON DELETE CASCADE
+    // means clearing entries (full rebuild or per-dir delete) drops these too.
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS workflow_documents (
+      entry_id        INTEGER PRIMARY KEY REFERENCES entries(id) ON DELETE CASCADE,
+      schema_version  INTEGER NOT NULL,
+      document_json   TEXT NOT NULL,
+      source_path     TEXT NOT NULL,
+      source_hash     TEXT NOT NULL,
+      updated_at      TEXT NOT NULL
+    );
+
+    CREATE INDEX IF NOT EXISTS idx_workflow_documents_source_path
+      ON workflow_documents(source_path);
   `);
     // Set version immediately after table creation so a crash before the end of
     // ensureSchema() does not leave the database in a versionless state on next open.
@@ -165,6 +162,15 @@ function ensureSchema(db, embeddingDim) {
       updated_at   TEXT NOT NULL DEFAULT (datetime('now')),
       FOREIGN KEY (entry_id) REFERENCES entries(id) ON DELETE CASCADE
     );
+  `);
+    // FTS-dirty queue. Created here (not lazily on first upsert) so the
+    // per-entry write path doesn't issue a CREATE TABLE IF NOT EXISTS on
+    // every call — that DDL would fire thousands of times during a full
+    // index. See `markFtsDirty` and `rebuildFts({ incremental: true })`.
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS entries_fts_dirty (
+      entry_id INTEGER PRIMARY KEY
+    );
   `);
     // sqlite-vec table
     if (isVecAvailable(db)) {
@@ -220,31 +226,67 @@ function ensureSchema(db, embeddingDim) {
     }
     // Usage telemetry table
     ensureUsageEventsSchema(db);
-    // Restore usage_events that were backed up during a version upgrade.
-    // Wrapped in outer try/catch because schema changes across versions may
-    // make the backup incompatible with the new table definition.
-    const dbAny = db;
-    const backup = dbAny.__usageBackup;
-    if (backup && backup.length > 0) {
-        try {
-            db.transaction(() => {
-                const cols = Object.keys(backup[0]);
-                const placeholders = cols.map(() => "?").join(", ");
-                const insert = db.prepare(`INSERT INTO usage_events (${cols.join(", ")}) VALUES (${placeholders})`);
-                for (const row of backup) {
-                    try {
-                        insert.run(...cols.map((c) => row[c]));
-                    }
-                    catch {
-                        /* skip rows that fail */
-                    }
+    // Restore usage_events backed up by the version-upgrade path above.
+    restoreUsageEventsBackup(db, usageBackup);
+}
+/**
+ * Detect a stored DB version that differs from {@link DB_VERSION}, drop the
+ * old schema, and return a backup of the previous `usage_events` rows so the
+ * rest of `ensureSchema()` can restore them once the new table exists.
+ *
+ * Returns an empty array when no upgrade is needed or when the previous
+ * `usage_events` table is unreadable.
+ */
+function handleVersionUpgrade(db) {
+    const storedVersion = getMeta(db, "version");
+    if (!storedVersion || storedVersion === String(DB_VERSION))
+        return [];
+    let usageBackup = [];
+    try {
+        usageBackup = db.prepare("SELECT * FROM usage_events").all();
+    }
+    catch {
+        /* table may not exist in older versions */
+    }
+    db.exec("DROP TABLE IF EXISTS utility_scores");
+    db.exec("DROP TABLE IF EXISTS usage_events");
+    db.exec("DROP TABLE IF EXISTS embeddings");
+    db.exec("DROP TABLE IF EXISTS entries_vec");
+    db.exec("DROP TABLE IF EXISTS entries_fts");
+    db.exec("DROP INDEX IF EXISTS idx_entries_dir");
+    db.exec("DROP INDEX IF EXISTS idx_entries_type");
+    db.exec("DROP TABLE IF EXISTS entries");
+    db.exec("DELETE FROM index_meta");
+    console.warn("[akm] Index rebuilt due to version upgrade. Run 'akm index' to repopulate.");
+    return usageBackup;
+}
+/**
+ * Re-insert backed-up `usage_events` rows into the freshly-created table.
+ *
+ * Wrapped in an outer try/catch because schema changes across versions may
+ * make the backup incompatible with the new table definition; in that case
+ * the backup is discarded silently rather than blocking startup.
+ */
+function restoreUsageEventsBackup(db, backup) {
+    if (backup.length === 0)
+        return;
+    try {
+        db.transaction(() => {
+            const cols = Object.keys(backup[0]);
+            const placeholders = cols.map(() => "?").join(", ");
+            const insert = db.prepare(`INSERT INTO usage_events (${cols.join(", ")}) VALUES (${placeholders})`);
+            for (const row of backup) {
+                try {
+                    insert.run(...cols.map((c) => row[c]));
                 }
-            })();
-        }
-        catch {
-            /* schema changed too much — discard backup gracefully */
-        }
-        delete dbAny.__usageBackup;
+                catch {
+                    /* skip rows that fail */
+                }
+            }
+        })();
+    }
+    catch {
+        /* schema changed too much — discard backup gracefully */
     }
 }
 // ── Meta helpers ────────────────────────────────────────────────────────────
@@ -264,23 +306,45 @@ export function setMeta(db, key, value) {
  * reflect the changes.
  */
 export function upsertEntry(db, entryKey, dirPath, filePath, stashDir, entry, searchText) {
-    const stmt = db.prepare(`
-    INSERT INTO entries (entry_key, dir_path, file_path, stash_dir, entry_json, search_text, entry_type)
-    VALUES (?, ?, ?, ?, ?, ?, ?)
-    ON CONFLICT(entry_key) DO UPDATE SET
-      dir_path = excluded.dir_path,
-      file_path = excluded.file_path,
-      stash_dir = excluded.stash_dir,
-      entry_json = excluded.entry_json,
-      search_text = excluded.search_text,
-      entry_type = excluded.entry_type
-  `);
-    stmt.run(entryKey, dirPath, filePath, stashDir, JSON.stringify(entry), searchText, entry.type);
-    // Fetch the row id explicitly since last_insert_rowid() is unreliable for ON CONFLICT DO UPDATE
-    const row = db.prepare("SELECT id FROM entries WHERE entry_key = ?").get(entryKey);
-    if (!row)
+    // Hot path during indexing — cache the two prepared statements per
+    // database connection so we don't pay the SQL parse/compile cost on
+    // every call. The dirty-mark INSERT and the upsert-with-RETURNING
+    // share the same WeakMap so they live and die with the connection.
+    const stmts = getUpsertStmts(db);
+    const result = stmts.upsert.get(entryKey, dirPath, filePath, stashDir, JSON.stringify(entry), searchText, entry.type);
+    if (!result)
         throw new Error("upsertEntry: entry_key not found after upsert");
-    return row.id;
+    // Mark this entry as FTS-dirty so `rebuildFts({ incremental: true })`
+    // only revisits entries that actually changed. INSERT OR IGNORE is
+    // idempotent across multiple upserts of the same row.
+    stmts.markDirty.run(result.id);
+    return result.id;
+}
+const upsertStmtsByDb = new WeakMap();
+function getUpsertStmts(db) {
+    const existing = upsertStmtsByDb.get(db);
+    if (existing)
+        return existing;
+    const stmts = {
+        // RETURNING id handles ON CONFLICT DO UPDATE correctly — no second
+        // SELECT round-trip needed (last_insert_rowid() is unreliable for
+        // ON CONFLICT). Use `.get()` so a single row comes back.
+        upsert: db.prepare(`
+      INSERT INTO entries (entry_key, dir_path, file_path, stash_dir, entry_json, search_text, entry_type)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+      ON CONFLICT(entry_key) DO UPDATE SET
+        dir_path = excluded.dir_path,
+        file_path = excluded.file_path,
+        stash_dir = excluded.stash_dir,
+        entry_json = excluded.entry_json,
+        search_text = excluded.search_text,
+        entry_type = excluded.entry_type
+      RETURNING id
+    `),
+        markDirty: db.prepare("INSERT OR IGNORE INTO entries_fts_dirty (entry_id) VALUES (?)"),
+    };
+    upsertStmtsByDb.set(db, stmts);
+    return stmts;
 }
 export function deleteEntriesByDir(db, dirPath) {
     db.transaction(() => {
@@ -302,6 +366,25 @@ function deleteRelatedRows(db, ids) {
         return;
     const numericIds = ids.map((r) => r.id);
     const vecAvail = isVecAvailable(db);
+    // Drop matching FTS rows + dirty markers immediately so an incremental
+    // rebuild after a deletion doesn't try to re-index entries that no longer
+    // exist (and so a full scan after deletion sees a consistent FTS).
+    for (let i = 0; i < numericIds.length; i += SQLITE_CHUNK_SIZE) {
+        const chunk = numericIds.slice(i, i + SQLITE_CHUNK_SIZE);
+        const placeholders = chunk.map(() => "?").join(",");
+        try {
+            db.prepare(`DELETE FROM entries_fts WHERE entry_id IN (${placeholders})`).run(...chunk);
+        }
+        catch {
+            /* fts table may not exist on a brand-new db */
+        }
+        try {
+            db.prepare(`DELETE FROM entries_fts_dirty WHERE entry_id IN (${placeholders})`).run(...chunk);
+        }
+        catch {
+            /* dirty table is created lazily by upsertEntry */
+        }
+    }
     // Process in chunks to stay within SQLITE_MAX_VARIABLE_NUMBER
     for (let i = 0; i < numericIds.length; i += SQLITE_CHUNK_SIZE) {
         const chunk = numericIds.slice(i, i + SQLITE_CHUNK_SIZE);
@@ -343,19 +426,51 @@ function deleteRelatedRows(db, ids) {
         }
     }
 }
-export function rebuildFts(db) {
-    // Wrap DELETE + INSERT in a single transaction so the FTS table is
-    // never left empty between the two statements if a crash occurs.
-    // Store the integer id directly (FTS5 stores all content as text
-    // internally; the join in searchFts compares numerically without CAST).
-    //
-    // Insert into separate FTS5 columns by extracting per-field text from
-    // the entry_json using buildSearchFields(). The entries.search_text column
-    // is kept as a concatenated fallback for embedding generation.
+/**
+ * Rebuild the FTS5 search index.
+ *
+ * `incremental` (default `false`): when true, only rebuild rows that
+ * `upsertEntry` marked dirty since the last `rebuildFts` call. The full path
+ * (default) wipes `entries_fts` and re-inserts every row from `entries` —
+ * appropriate for `akm index --full` and version-upgrade rebuilds.
+ *
+ * Both paths are wrapped in a single transaction so the FTS table is never
+ * left in a half-rebuilt state.
+ *
+ * Skipped corrupt-JSON rows are aggregated into one warning instead of
+ * spamming stderr per-entry.
+ */
+export function rebuildFts(db, options) {
+    const incremental = options?.incremental === true;
     db.transaction(() => {
-        db.exec("DELETE FROM entries_fts");
-        const rows = db.prepare("SELECT id, entry_json FROM entries").all();
+        let rows;
+        if (incremental) {
+            // Read the dirty queue and join against entries to get the JSON.
+            // Then drop the matching rows from entries_fts so the INSERT below
+            // doesn't double-up. The dirty list is drained at the end.
+            rows = db
+                .prepare(`SELECT e.id AS id, e.entry_json AS entry_json
+             FROM entries_fts_dirty d
+             JOIN entries e ON e.id = d.entry_id`)
+                .all();
+            if (rows.length === 0)
+                return;
+            const ids = rows.map((r) => r.id);
+            // Delete only the dirty FTS rows — chunk to stay under
+            // SQLITE_MAX_VARIABLE_NUMBER on large dirty queues.
+            for (let i = 0; i < ids.length; i += SQLITE_CHUNK_SIZE) {
+                const chunk = ids.slice(i, i + SQLITE_CHUNK_SIZE);
+                const placeholders = chunk.map(() => "?").join(",");
+                db.prepare(`DELETE FROM entries_fts WHERE entry_id IN (${placeholders})`).run(...chunk);
+            }
+        }
+        else {
+            // Full path: wipe and re-read every row.
+            db.exec("DELETE FROM entries_fts");
+            rows = db.prepare("SELECT id, entry_json FROM entries").all();
+        }
         const insertStmt = db.prepare("INSERT INTO entries_fts (entry_id, name, description, tags, hints, content) VALUES (?, ?, ?, ?, ?, ?)");
+        let skipped = 0;
         for (const row of rows) {
             let entry;
             let fields;
@@ -364,11 +479,25 @@ export function rebuildFts(db) {
                 fields = buildSearchFields(entry);
             }
             catch {
-                warn(`[db] rebuildFts: skipping entry id=${row.id} — invalid entry_json`);
+                skipped++;
                 continue;
             }
             insertStmt.run(row.id, fields.name, fields.description, fields.tags, fields.hints, fields.content);
         }
+        if (skipped > 0) {
+            warn(`[db] rebuildFts: skipped ${skipped} entr${skipped === 1 ? "y" : "ies"} with invalid entry_json`);
+        }
+        // Always drain the dirty queue — if it exists. A full rebuild also
+        // clears it because the full path covers everything the dirty list
+        // tracks.
+        if (incremental) {
+            db.exec("DELETE FROM entries_fts_dirty");
+        }
+        else {
+            // Full path: drain the dirty queue too. The table is created by
+            // ensureSchema(), so it always exists at this point.
+            db.exec("DELETE FROM entries_fts_dirty");
+        }
     })();
 }
 // ── Vector operations ───────────────────────────────────────────────────────

package/dist/{file-context.js → indexer/file-context.js}

@@ -6,8 +6,8 @@
  */
 import fs from "node:fs";
 import path from "node:path";
-import { toPosix } from "./common";
-import { parseFrontmatter } from "./frontmatter";
+import { toPosix } from "../core/common";
+import { parseFrontmatter } from "../core/frontmatter";
 /**
  * Build a FileContext from a stash root and an absolute file path.
  *
@@ -81,7 +81,7 @@ async function ensureBuiltinsRegistered() {
     if (!builtinsPromise) {
         builtinsPromise = (async () => {
             const { registerBuiltinMatchers } = await import("./matchers.js");
-            const { registerBuiltinRenderers } = await import("./renderers.js");
+            const { registerBuiltinRenderers } = await import("../output/renderers.js");
             registerBuiltinMatchers();
             registerBuiltinRenderers();
         })();