akm-cli 0.9.0-rc.0 → 0.9.0-rc.13

package/dist/{indexer/db/entry-mapper.js → storage/repositories/index-entry-mapper.js}

@@ -2,19 +2,28 @@
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
 /**
- * Shared `entries`-row projection + mapper, extracted from indexer/db/db.ts.
+ * Shared `entries`-row projection + mapper for the `index.db` storage repos.
  *
  * Centralizes the one canonical `entries` SELECT column list and the
  * JSON-parse-guarded row → {@link DbIndexedEntry} mapping that several queries
  * used to reimplement. Corrupt `entry_json` rows are skipped (warn once) rather
  * than crashing the caller.
+ *
+ * Relocated from `src/indexer/db/entry-mapper.ts` (WI-5a): it now imports the
+ * shared shapes from the leaf types module `./index-entry-types.ts` rather
+ * than reaching across the storage↔indexer boundary into `db.ts`, so it no
+ * longer participates in the `db.ts` / `entry-mapper.ts` / `schema.ts` import
+ * cycle that used to pin them together.
  */
 import { warn } from "../../core/warn.js";
 /**
  * Canonical column list for reading a full indexed entry from the `entries`
  * table, in the order {@link rowToIndexedEntry} expects.
+ *
+ * Durable identity columns are surfaced here so every mapped entry carries
+ * current indexed provenance.
  */
-export const ENTRY_COLUMNS = "id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text";
+export const ENTRY_COLUMNS = "id, entry_key, dir_path, file_path, stash_dir, entry_json, search_text, item_ref, concept_id, bundle_id, adapter_id";
 /**
  * Map one raw `entries` row to a {@link DbIndexedEntry}, parsing `entry_json`.
  * Returns `null` (and warns, tagged with `context`) when the JSON is corrupt so
@@ -29,6 +38,10 @@ export function rowToIndexedEntry(row, context) {
         warn(`[db] ${context}: skipping entry id=${row.id} — corrupt entry_json`);
         return null;
     }
+    if (!row.item_ref || !row.concept_id || !row.bundle_id || !row.adapter_id) {
+        warn(`[db] ${context}: skipping entry id=${row.id} — missing indexed provenance`);
+        return null;
+    }
     return {
         id: row.id,
         entryKey: row.entry_key,
@@ -37,5 +50,9 @@ export function rowToIndexedEntry(row, context) {
         stashDir: row.stash_dir,
         entry,
         searchText: row.search_text,
+        itemRef: row.item_ref,
+        conceptId: row.concept_id,
+        bundleId: row.bundle_id,
+        adapterId: row.adapter_id,
     };
 }

package/dist/storage/repositories/index-entry-types.js

@@ -0,0 +1,4 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+export {};

package/dist/storage/repositories/index-fts-repository.js

@@ -0,0 +1,167 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * `index.db` FTS5 search + rebuild repository.
+ *
+ * Owns the `entries_fts` full-text query path and the incremental/full FTS
+ * rebuild.
+ */
+import { warn } from "../../core/warn.js";
+import { buildPrefixQuery, sanitizeFtsQuery } from "../../indexer/search/fts-query.js";
+import { buildSearchFields } from "../../indexer/search/search-fields.js";
+import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
+export function searchFts(db, query, limit, entryType, excludeTypes) {
+    const ftsQuery = sanitizeFtsQuery(query);
+    if (!ftsQuery)
+        return [];
+    // Try the exact AND query first
+    const exactResults = runFtsQuery(db, ftsQuery, limit, entryType, excludeTypes);
+    if (exactResults.length > 0)
+        return exactResults;
+    // Exact match returned zero results — try prefix fallback.
+    // Append FTS5 `*` suffix to each token that is >= 3 characters long.
+    // Short tokens (1-2 chars) are excluded from prefix expansion because
+    // they produce too many false positives.
+    const prefixQuery = buildPrefixQuery(ftsQuery);
+    if (!prefixQuery)
+        return [];
+    return runFtsQuery(db, prefixQuery, limit, entryType, excludeTypes);
+}
+function runFtsQuery(db, ftsQuery, limit, entryType, excludeTypes) {
+    // #627 — exclude-type clause. Only applies on the untyped ('any') path; an
+    // explicit include filter (entryType) already narrows to a single type, so
+    // exclusion is redundant there. An empty list skips the clause entirely
+    // (never emit `NOT IN ()`, which is a SQL error / always-false).
+    const excludes = excludeTypes && excludeTypes.length > 0 ? excludeTypes : [];
+    // The typed and untyped paths differ ONLY by one WHERE clause (an entry_type
+    // equality vs. an optional NOT IN exclusion) and their param order — the
+    // SELECT/JOIN/ORDER/LIMIT is shared, so build it once. Join on integer
+    // entry_id directly (no CAST; we store integer). bm25() per-column weights:
+    // entry_id(0), name(10), description(5), tags(3), hints(2), content(1).
+    let filterClause;
+    let params;
+    if (entryType && entryType !== "any") {
+        filterClause = "AND e.entry_type = ?";
+        params = [ftsQuery, entryType, limit];
+    }
+    else {
+        filterClause = excludes.length > 0 ? `AND e.entry_type NOT IN (${excludes.map(() => "?").join(", ")})` : "";
+        // Param order: MATCH, then the NOT IN values, then LIMIT.
+        params = [ftsQuery, ...excludes, limit];
+    }
+    const sql = `
+    SELECT e.id, e.file_path AS filePath, e.entry_json, e.search_text AS searchText,
+           e.item_ref AS itemRef, e.bundle_id AS bundleId, e.concept_id AS conceptId, e.adapter_id AS adapterId,
+           bm25(entries_fts, 0, 10.0, 5.0, 3.0, 2.0, 1.0) AS bm25Score
+    FROM entries_fts f
+    JOIN entries e ON e.id = f.entry_id
+    WHERE entries_fts MATCH ?
+      ${filterClause}
+    ORDER BY bm25Score, e.id ASC
+    LIMIT ?
+  `;
+    try {
+        const rows = db.prepare(sql).all(...params);
+        // Guard against corrupt JSON — skip the row rather than crashing
+        const results = [];
+        for (const row of rows) {
+            let entry;
+            try {
+                entry = JSON.parse(row.entry_json);
+            }
+            catch {
+                warn(`[db] searchFts: skipping entry id=${row.id} — corrupt entry_json`);
+                continue;
+            }
+            results.push({
+                id: row.id,
+                filePath: row.filePath,
+                entry,
+                searchText: row.searchText,
+                bm25Score: row.bm25Score,
+                itemRef: row.itemRef,
+                bundleId: row.bundleId,
+                conceptId: row.conceptId,
+                adapterId: row.adapterId,
+            });
+        }
+        return results;
+    }
+    catch (err) {
+        warn("[db] runFtsQuery failed:", err instanceof Error ? err.message : String(err));
+        return [];
+    }
+}
+/**
+ * Rebuild the FTS5 search index.
+ *
+ * `incremental` (default `false`): when true, only rebuild rows that
+ * `upsertEntry` marked dirty since the last `rebuildFts` call. The full path
+ * (default) wipes `entries_fts` and re-inserts every row from `entries` —
+ * appropriate for `akm index --full` and version-upgrade rebuilds.
+ *
+ * Both paths are wrapped in a single transaction so the FTS table is never
+ * left in a half-rebuilt state.
+ *
+ * Skipped corrupt-JSON rows are aggregated into one warning instead of
+ * spamming stderr per-entry.
+ */
+export function rebuildFts(db, options) {
+    const incremental = options?.incremental === true;
+    db.transaction(() => {
+        let rows;
+        if (incremental) {
+            // Read the dirty queue and join against entries to get the JSON.
+            // Then drop the matching rows from entries_fts so the INSERT below
+            // doesn't double-up. The dirty list is drained at the end.
+            rows = db
+                .prepare(`SELECT e.id AS id, e.entry_json AS entry_json
+             FROM entries_fts_dirty d
+             JOIN entries e ON e.id = d.entry_id`)
+                .all();
+            if (rows.length === 0)
+                return;
+            const ids = rows.map((r) => r.id);
+            // Delete only the dirty FTS rows — chunk to stay under
+            // SQLITE_MAX_VARIABLE_NUMBER on large dirty queues.
+            for (let i = 0; i < ids.length; i += SQLITE_CHUNK_SIZE) {
+                const chunk = ids.slice(i, i + SQLITE_CHUNK_SIZE);
+                const placeholders = chunk.map(() => "?").join(",");
+                db.prepare(`DELETE FROM entries_fts WHERE entry_id IN (${placeholders})`).run(...chunk);
+            }
+        }
+        else {
+            // Full path: wipe and re-read every row.
+            db.exec("DELETE FROM entries_fts");
+            rows = db.prepare("SELECT id, entry_json FROM entries").all();
+        }
+        const insertStmt = db.prepare("INSERT INTO entries_fts (entry_id, name, description, tags, hints, content) VALUES (?, ?, ?, ?, ?, ?)");
+        let skipped = 0;
+        for (const row of rows) {
+            let entry;
+            let fields;
+            try {
+                entry = JSON.parse(row.entry_json);
+                fields = buildSearchFields(entry);
+            }
+            catch {
+                skipped++;
+                continue;
+            }
+            insertStmt.run(row.id, fields.name, fields.description, fields.tags, fields.hints, fields.content);
+        }
+        if (skipped > 0) {
+            warn(`[db] rebuildFts: skipped ${skipped} entr${skipped === 1 ? "y" : "ies"} with invalid entry_json`);
+        }
+        // Always drain the dirty queue — both paths converge here. The
+        // incremental path drains it because we just consumed every dirty row;
+        // the full path drains it because a full rebuild covers everything the
+        // dirty list tracks. The table is guaranteed to exist (created by
+        // ensureSchema()).
+        //
+        // BUG-L1: previously the if/else arms ran identical statements — the
+        // duplication has been collapsed.
+        db.exec("DELETE FROM entries_fts_dirty");
+    })();
+}

package/dist/storage/repositories/index-llm-cache-repository.js

@@ -0,0 +1,108 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * `index.db` LLM enrichment-cache repository.
+ *
+ * Owns the raw SQL for `llm_enrichment_cache` — the body-hash-keyed cache that
+ * lets `akm index --enrich` skip the LLM call when a file's body is unchanged.
+ */
+import { bestEffort } from "../../core/best-effort.js";
+import { sha256Hex } from "../../runtime.js";
+import { SQLITE_CHUNK_SIZE } from "./index-sql.js";
+/**
+ * Look up a cached LLM result for the given asset_ref.
+ *
+ * Returns `undefined` when no entry exists OR when the stored body_hash
+ * doesn't match `currentBodyHash` (body has changed since the result was
+ * cached). In both cases the caller should invoke the LLM and write a new
+ * cache entry.
+ */
+export function getLlmCacheEntry(db, assetRef, currentBodyHash, cacheVariant = "") {
+    const row = db
+        .prepare("SELECT asset_ref, cache_variant, body_hash, result_json, updated_at FROM llm_enrichment_cache WHERE asset_ref = ? AND cache_variant = ?")
+        .get(assetRef, cacheVariant);
+    if (!row)
+        return undefined;
+    // Hash mismatch → body changed, treat as cache miss.
+    if (row.body_hash !== currentBodyHash)
+        return undefined;
+    return {
+        assetRef: row.asset_ref,
+        cacheVariant: row.cache_variant,
+        bodyHash: row.body_hash,
+        resultJson: row.result_json,
+        updatedAt: row.updated_at,
+    };
+}
+/**
+ * Batched variant of {@link getLlmCacheEntry}. Fetches every cache row whose
+ * `asset_ref` is in `refs` with a single `IN (...)` query (chunked to respect
+ * SQLITE_MAX_VARIABLE_NUMBER), returning a `Map<assetRef, LlmCacheEntry>`.
+ *
+ * Unlike `getLlmCacheEntry`, this does NOT filter by body hash — callers must
+ * compare `entry.bodyHash` against the current body hash themselves. This lets
+ * the batch path issue one DB query per chunk instead of one per file.
+ */
+export function getLlmCacheEntriesByRefs(db, refs, cacheVariant = "") {
+    const result = new Map();
+    if (refs.length === 0)
+        return result;
+    for (let i = 0; i < refs.length; i += SQLITE_CHUNK_SIZE) {
+        const chunk = refs.slice(i, i + SQLITE_CHUNK_SIZE);
+        const placeholders = chunk.map(() => "?").join(", ");
+        const rows = db
+            .prepare(`SELECT asset_ref, cache_variant, body_hash, result_json, updated_at FROM llm_enrichment_cache
+         WHERE cache_variant = ? AND asset_ref IN (${placeholders})`)
+            .all(cacheVariant, ...chunk);
+        for (const row of rows) {
+            result.set(row.asset_ref, {
+                assetRef: row.asset_ref,
+                cacheVariant: row.cache_variant,
+                bodyHash: row.body_hash,
+                resultJson: row.result_json,
+                updatedAt: row.updated_at,
+            });
+        }
+    }
+    return result;
+}
+/**
+ * Insert or update a cached LLM result for the given asset_ref.
+ */
+export function upsertLlmCacheEntry(db, assetRef, bodyHash, resultJson, cacheVariant = "") {
+    db.prepare(`INSERT INTO llm_enrichment_cache (asset_ref, cache_variant, body_hash, result_json, updated_at)
+     VALUES (?, ?, ?, ?, ?)
+     ON CONFLICT(asset_ref, cache_variant) DO UPDATE SET
+        body_hash   = excluded.body_hash,
+        result_json = excluded.result_json,
+        updated_at  = excluded.updated_at`).run(assetRef, cacheVariant, bodyHash, resultJson, Date.now());
+}
+/**
+ * Delete LLM cache entries whose asset_ref is no longer present in the
+ * `entries` table. Should be called during the cleanup phase of each index
+ * run to prevent the cache from growing unboundedly as assets are removed.
+ *
+ * The join uses a LIKE match against the entries `file_path` column because
+ * graph/memory cache refs are absolute file paths, while enrichment cache
+ * refs are entry_key strings — we preserve any entry that still has a
+ * corresponding row in either the entries table (by entry_key) or that
+ * matches a live file_path.
+ */
+export function clearStaleCacheEntries(db) {
+    bestEffort(() => {
+        db.exec(`
+      DELETE FROM llm_enrichment_cache
+      WHERE asset_ref NOT IN (SELECT file_path FROM entries)
+        AND asset_ref NOT IN (SELECT entry_key FROM entries)
+    `);
+    }, "llm_enrichment_cache may not exist in very old DBs opened without ensureSchema");
+}
+/**
+ * Compute a stable SHA-256 hex digest of a UTF-8 string. Used as the body_hash
+ * key in `llm_enrichment_cache`. Routed through the runtime boundary so the
+ * SQLite layer stays free of direct runtime-specific references.
+ */
+export function computeBodyHash(body) {
+    return sha256Hex(body);
+}

package/dist/storage/repositories/index-meta-repository.js

@@ -0,0 +1,49 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * `index.db` meta + per-directory index-state repository.
+ *
+ * Owns the raw SQL for `index_meta` (the key/value stamp table) and
+ * `index_dir_state` (incremental-index bookkeeping). Lives in the storage
+ * layer, not the indexer, so the storage layer owns these primitives.
+ */
+import path from "node:path";
+// ── Meta helpers ────────────────────────────────────────────────────────────
+export function getMeta(db, key) {
+    const row = db.prepare("SELECT value FROM index_meta WHERE key = ?").get(key);
+    return row?.value;
+}
+export function setMeta(db, key, value) {
+    db.prepare("INSERT OR REPLACE INTO index_meta (key, value) VALUES (?, ?)").run(key, value);
+}
+// ── Per-directory index state ───────────────────────────────────────────────
+export function getIndexDirState(db, dirPath) {
+    const row = db
+        .prepare("SELECT dir_path, file_set_hash, file_mtime_max_ms, reason, updated_at FROM index_dir_state WHERE dir_path = ?")
+        .get(dirPath);
+    if (!row)
+        return undefined;
+    return {
+        dirPath: row.dir_path,
+        fileSetHash: row.file_set_hash,
+        fileMtimeMaxMs: row.file_mtime_max_ms,
+        reason: row.reason,
+        updatedAt: row.updated_at,
+    };
+}
+export function upsertIndexDirState(db, state) {
+    db.prepare(`INSERT INTO index_dir_state (dir_path, file_set_hash, file_mtime_max_ms, reason, updated_at)
+     VALUES (?, ?, ?, ?, ?)
+     ON CONFLICT(dir_path) DO UPDATE SET
+       file_set_hash = excluded.file_set_hash,
+       file_mtime_max_ms = excluded.file_mtime_max_ms,
+       reason = excluded.reason,
+       updated_at = excluded.updated_at`).run(state.dirPath, state.fileSetHash, state.fileMtimeMaxMs, state.reason, new Date().toISOString());
+}
+export function deleteIndexDirState(db, dirPath) {
+    db.prepare("DELETE FROM index_dir_state WHERE dir_path = ?").run(dirPath);
+}
+export function deleteIndexDirStatesByStashDir(db, stashDir) {
+    db.prepare("DELETE FROM index_dir_state WHERE dir_path = ? OR dir_path LIKE ?").run(stashDir, `${stashDir}${path.sep}%`);
+}