akm-cli 0.1.3 → 0.2.1

package/dist/indexer.js

@@ -1,18 +1,24 @@
 import fs from "node:fs";
 import path from "node:path";
-import { resolveStashDir } from "./common";
-import { closeDatabase, DB_VERSION, deleteEntriesByDir, deleteEntriesByStashDir, getEntriesByDir, getEntryCount, getMeta, isVecAvailable, openDatabase, rebuildFts, setMeta, upsertEmbedding, upsertEntry, warnIfVecMissing, } from "./db";
+import { isHttpUrl, resolveStashDir } from "./common";
+import { closeDatabase, deleteEntriesByDir, deleteEntriesByStashDir, getEmbeddingCount, getEntriesByDir, getEntryCount, getMeta, isVecAvailable, openDatabase, rebuildFts, setMeta, upsertEmbedding, upsertEntry, upsertUtilityScore, warnIfVecMissing, } from "./db";
 import { generateMetadataFlat, loadStashFile } from "./metadata";
 import { getDbPath } from "./paths";
+import { buildSearchText } from "./search-fields";
+import { ensureUsageEventsSchema, purgeOldUsageEvents } from "./usage-events";
 import { walkStashFlat } from "./walker";
 import { warn } from "./warn";
 // ── Indexer ──────────────────────────────────────────────────────────────────
 export async function akmIndex(options) {
     const stashDir = options?.stashDir || resolveStashDir();
+    const onProgress = options?.onProgress ?? (() => { });
     // Load config and resolve all stash sources
     const { loadConfig } = await import("./config.js");
     const config = loadConfig();
-    const { resolveAllStashDirs } = await import("./search-source.js");
+    // Ensure git stash caches are extracted before resolving stash dirs,
+    // so their content directories exist on disk for the walker to discover.
+    const { ensureGitCaches, resolveAllStashDirs } = await import("./search-source.js");
+    await ensureGitCaches(config);
     const allStashDirs = resolveAllStashDirs(stashDir);
     const t0 = Date.now();
     // Open database — pass embedding dimension from config if available
@@ -25,8 +31,19 @@ export async function akmIndex(options) {
         const prevBuiltAt = getMeta(db, "builtAt");
         const isIncremental = !options?.full && prevStashDir === stashDir && !!prevBuiltAt;
         const builtAtMs = isIncremental && prevBuiltAt ? new Date(prevBuiltAt).getTime() : 0;
+        onProgress({
+            phase: "summary",
+            message: buildIndexSummaryMessage({
+                mode: isIncremental ? "incremental" : "full",
+                stashSources: allStashDirs.length,
+                semanticSearch: config.semanticSearch,
+                embeddingProvider: getEmbeddingProvider(config.embedding),
+                llmEnabled: !!config.llm,
+                vecAvailable: isVecAvailable(db),
+            }),
+        });
         if (options?.full || !isIncremental) {
-            // HI-5: the delete is now merged into the insert transaction inside
+            // The delete is now merged into the insert transaction inside
             // indexEntries() so that a reader never sees an empty database between
             // the wipe and the re-inserts.  The doFullDelete flag signals this path.
         }
@@ -59,21 +76,33 @@ export async function akmIndex(options) {
         const tWalkStart = Date.now();
         // Walk stash dirs and index entries.
         // doFullDelete=true merges the wipe into the same transaction as the
-        // inserts (HI-5) so readers never see an empty database mid-rebuild.
+        // inserts so readers never see an empty database mid-rebuild.
         const doFullDelete = options?.full || !isIncremental;
-        const { scannedDirs, skippedDirs, generatedCount, dirsNeedingLlm } = await indexEntries(db, allStashDirs, stashDir, isIncremental, builtAtMs, doFullDelete);
+        const { scannedDirs, skippedDirs, generatedCount, dirsNeedingLlm } = await indexEntries(db, allStashDirs, isIncremental, builtAtMs, doFullDelete);
+        onProgress({
+            phase: "scan",
+            message: `Scanned ${scannedDirs} ${scannedDirs === 1 ? "directory" : "directories"} and skipped ${skippedDirs}.`,
+        });
         const tWalkEnd = Date.now();
         // Enhance entries with LLM if configured
         await enhanceDirsWithLlm(db, config, dirsNeedingLlm);
+        onProgress({
+            phase: "llm",
+            message: config.llm
+                ? `LLM enhancement reviewed ${dirsNeedingLlm.length} ${dirsNeedingLlm.length === 1 ? "directory" : "directories"}.`
+                : "LLM enhancement disabled.",
+        });
         const tLlmEnd = Date.now();
         // Rebuild FTS after all inserts
         rebuildFts(db);
+        onProgress({ phase: "fts", message: "Rebuilt full-text search index." });
         const tFtsEnd = Date.now();
+        // Recompute utility scores from usage_events after FTS rebuild
+        recomputeUtilityScores(db);
         // Generate embeddings if semantic search is enabled
-        const hasEmbeddings = await generateEmbeddingsForDb(db, config);
+        const hasEmbeddings = await generateEmbeddingsForDb(db, config, onProgress);
         const tEmbedEnd = Date.now();
         // Update metadata
-        setMeta(db, "version", String(DB_VERSION));
         setMeta(db, "builtAt", new Date().toISOString());
         setMeta(db, "stashDir", stashDir);
         setMeta(db, "stashDirs", JSON.stringify(allStashDirs));
@@ -82,6 +111,8 @@ export async function akmIndex(options) {
         // Warn on every index run if using JS fallback with many entries
         warnIfVecMissing(db);
         const tEnd = Date.now();
+        const verification = verifyIndexState(db, config, totalEntries);
+        onProgress({ phase: "verify", message: verification.message });
         return {
             stashDir,
             totalEntries,
@@ -90,6 +121,7 @@ export async function akmIndex(options) {
             mode: isIncremental ? "incremental" : "full",
             directoriesScanned: scannedDirs,
             directoriesSkipped: skippedDirs,
+            verification,
             timing: {
                 totalMs: tEnd - t0,
                 walkMs: tWalkEnd - tWalkStart,
@@ -104,7 +136,7 @@ export async function akmIndex(options) {
     }
 }
 // ── Extracted helpers for indexing ────────────────────────────────────────────
-async function indexEntries(db, allStashDirs, _stashDir, isIncremental, builtAtMs, doFullDelete = false) {
+async function indexEntries(db, allStashDirs, isIncremental, builtAtMs, doFullDelete = false) {
     let scannedDirs = 0;
     let skippedDirs = 0;
     let generatedCount = 0;
@@ -170,7 +202,7 @@ async function indexEntries(db, allStashDirs, _stashDir, isIncremental, builtAtM
     // occurrence wins.
     const indexedAssetIdentities = new Set();
     const insertTransaction = db.transaction(() => {
-        // HI-5: Perform the full-rebuild wipe as the FIRST step of the insert
+        // Perform the full-rebuild wipe as the FIRST step of the insert
         // transaction so delete and re-insert are atomic — a concurrent reader
         // never observes an empty database between the two operations.
         if (doFullDelete) {
@@ -189,6 +221,8 @@ async function indexEntries(db, allStashDirs, _stashDir, isIncremental, builtAtM
                 }
             }
             db.exec("DELETE FROM entries_fts");
+            db.exec("DELETE FROM utility_scores");
+            db.exec("DELETE FROM usage_events");
             db.exec("DELETE FROM entries");
         }
         for (const { dirPath, currentStashDir, files, stash, skip } of dirRecords) {
@@ -235,8 +269,8 @@ async function enhanceDirsWithLlm(db, config, dirsNeedingLlm) {
         if (generatedEntries.length === 0)
             continue;
         const generatedStash = { entries: generatedEntries };
-        const enhanced = await enhanceStashWithLlm(config.llm, generatedStash, dirPath, files);
-        // HI-2: Re-upsert the enhanced entries in a single transaction so a crash
+        const enhanced = await enhanceStashWithLlm(config.llm, generatedStash, files);
+        // Re-upsert the enhanced entries in a single transaction so a crash
         // cannot leave half the entries updated and the rest stale.
         db.transaction(() => {
             for (const entry of enhanced.entries) {
@@ -248,27 +282,43 @@ async function enhanceDirsWithLlm(db, config, dirsNeedingLlm) {
         })();
     }
 }
-async function generateEmbeddingsForDb(db, config) {
-    if (!config.semanticSearch)
+async function generateEmbeddingsForDb(db, config, onProgress) {
+    if (!config.semanticSearch) {
+        onProgress({ phase: "embeddings", message: "Semantic search disabled; skipping embeddings." });
         return false;
+    }
     try {
         const { embedBatch } = await import("./embedder.js");
         const allEntries = getAllEntriesForEmbedding(db);
-        if (allEntries.length === 0)
+        if (allEntries.length === 0) {
+            onProgress({ phase: "embeddings", message: "Embeddings already up to date." });
             return true;
+        }
+        onProgress({
+            phase: "embeddings",
+            message: `Generating embeddings for ${allEntries.length} entr${allEntries.length === 1 ? "y" : "ies"}.`,
+        });
         const texts = allEntries.map((e) => e.searchText);
         const embeddings = await embedBatch(texts, config.embedding);
-        // HI-3: Wrap all embedding upserts in a single transaction so partial
+        // Wrap all embedding upserts in a single transaction so partial
         // state is rolled back on failure rather than leaving the table half-filled.
         db.transaction(() => {
             for (let i = 0; i < allEntries.length; i++) {
                 upsertEmbedding(db, allEntries[i].id, embeddings[i]);
             }
         })();
+        onProgress({
+            phase: "embeddings",
+            message: `Stored ${embeddings.length} embedding${embeddings.length === 1 ? "" : "s"}.`,
+        });
         return true;
     }
     catch (error) {
         warn("Embedding generation failed, continuing without:", error instanceof Error ? error.message : String(error));
+        onProgress({
+            phase: "embeddings",
+            message: `Embedding generation failed: ${error instanceof Error ? error.message : String(error)}`,
+        });
         return false;
     }
 }
@@ -289,7 +339,69 @@ function attachFileSize(entry, entryPath) {
         return entry;
     }
 }
-/** Set of all known type directory names */
+function buildIndexSummaryMessage(options) {
+    const stashSourceLabel = options.stashSources === 1 ? "stash source" : "stash sources";
+    const semanticDetail = getSemanticSearchLabel(options.semanticSearch, options.embeddingProvider, options.vecAvailable);
+    return `Starting ${options.mode} index (${options.stashSources} ${stashSourceLabel}, semantic search: ${semanticDetail}, LLM: ${options.llmEnabled ? "enabled" : "disabled"}).`;
+}
+function getEmbeddingProvider(embedding) {
+    return isHttpUrl(embedding?.endpoint) ? "remote" : "local";
+}
+function getSemanticSearchLabel(semanticSearch, embeddingProvider, vecAvailable) {
+    if (!semanticSearch)
+        return "disabled";
+    return `${embeddingProvider} embeddings, ${vecAvailable ? "sqlite-vec" : "JS fallback"}`;
+}
+function verifyIndexState(db, config, totalEntries) {
+    const embeddingCount = getEmbeddingCount(db);
+    const vecAvailable = isVecAvailable(db);
+    const embeddingProvider = getEmbeddingProvider(config.embedding);
+    if (totalEntries === 0) {
+        return {
+            ok: true,
+            message: "Index ready. No assets were found yet.",
+            semanticSearchEnabled: config.semanticSearch,
+            embeddingProvider,
+            entryCount: totalEntries,
+            embeddingCount,
+            vecAvailable,
+        };
+    }
+    if (!config.semanticSearch) {
+        return {
+            ok: true,
+            message: "Keyword index ready. Semantic search is disabled.",
+            semanticSearchEnabled: false,
+            embeddingProvider,
+            entryCount: totalEntries,
+            embeddingCount,
+            vecAvailable,
+        };
+    }
+    if (embeddingCount >= totalEntries) {
+        return {
+            ok: true,
+            message: `Semantic search ready (${embeddingCount}/${totalEntries} embeddings, ${vecAvailable ? "sqlite-vec active" : "JS fallback active"}).`,
+            semanticSearchEnabled: true,
+            embeddingProvider,
+            entryCount: totalEntries,
+            embeddingCount,
+            vecAvailable,
+        };
+    }
+    return {
+        ok: false,
+        message: `Semantic search verification failed (${embeddingCount}/${totalEntries} embeddings available).`,
+        guidance: embeddingProvider === "remote"
+            ? "Check your embedding endpoint and credentials, then retry `akm index --full --verbose`."
+            : "Retry `akm index --full --verbose`. If it still fails, confirm local model downloads are permitted and see docs/configuration.md for local embedding dependency setup.",
+        semanticSearchEnabled: true,
+        embeddingProvider,
+        entryCount: totalEntries,
+        embeddingCount,
+        vecAvailable,
+    };
+}
 function isDirStale(dirPath, currentFiles, previousEntries, builtAtMs) {
     // Check if file set changed (additions or deletions)
     const prevFileNames = new Set(previousEntries.map((ie) => ie.entry.filename).filter((e) => !!e));
@@ -321,7 +433,7 @@ function isDirStale(dirPath, currentFiles, previousEntries, builtAtMs) {
     }
     return false;
 }
-async function enhanceStashWithLlm(llmConfig, stash, _dirPath, files) {
+async function enhanceStashWithLlm(llmConfig, stash, files) {
     const { enhanceMetadata } = await import("./llm.js");
     const enhanced = [];
     for (const entry of stash.entries) {
@@ -391,30 +503,74 @@ export function matchEntryToFile(entryName, fileMap, files) {
     // Fallback to first file, or null if no files are available
     return files[0] || null;
 }
-export function buildSearchText(entry) {
-    const parts = [entry.name.replace(/[-_]/g, " ")];
-    if (entry.description)
-        parts.push(entry.description);
-    if (entry.tags)
-        parts.push(entry.tags.join(" "));
-    if (entry.examples)
-        parts.push(entry.examples.join(" "));
-    if (entry.aliases)
-        parts.push(entry.aliases.join(" "));
-    if (entry.searchHints)
-        parts.push(entry.searchHints.join(" "));
-    if (entry.usage)
-        parts.push(entry.usage.join(" "));
-    if (entry.intent) {
-        if (entry.intent.when)
-            parts.push(entry.intent.when);
-        if (entry.intent.input)
-            parts.push(entry.intent.input);
-        if (entry.intent.output)
-            parts.push(entry.intent.output);
+export { buildSearchFields, buildSearchText } from "./search-fields";
+// ── Utility score recomputation ──────────────────────────────────────────────
+/** Retention window for usage events: events older than this are purged. */
+const USAGE_EVENT_RETENTION_DAYS = 90;
+/**
+ * Recompute utility scores for all entries based on usage_events data.
+ *
+ * For each entry:
+ *   - Count search appearances (event_type = 'search')
+ *   - Count show events (event_type = 'show')
+ *   - Compute select_rate = showCount / searchCount, clamped to [0, 1]
+ *   - Update utility via EMA: utility = previousUtility * 0.7 + selectRate * 0.3
+ *
+ * Also purges usage_events older than 90 days and ensures the M-1
+ * usage_events table exists before querying.
+ *
+ * Called during `akm index` after FTS rebuild.
+ */
+export function recomputeUtilityScores(db) {
+    const EMA_DECAY = 0.7;
+    // Ensure usage_events table exists before querying
+    ensureUsageEventsSchema(db);
+    // Purge stale usage events (90-day retention)
+    purgeOldUsageEvents(db, USAGE_EVENT_RETENTION_DAYS);
+    // Time-proportional decay: apply one round of EMA per elapsed day so
+    // indexing frequency doesn't affect how fast scores decay.
+    const lastComputedAt = getMeta(db, "last_utility_computed_at");
+    let elapsedDays = 1; // default for first run
+    if (lastComputedAt) {
+        const ms = Date.now() - new Date(lastComputedAt).getTime();
+        elapsedDays = Math.max(1, ms / (1000 * 60 * 60 * 24));
+    }
+    const emaDecay = EMA_DECAY ** elapsedDays;
+    const emaNew = 1 - emaDecay; // complement so weights still sum to 1
+    // Single aggregate query instead of N+1 per-entry queries.
+    // Only processes entries that actually have usage events.
+    const usageRows = db
+        .prepare(`
+      SELECT entry_id,
+             SUM(CASE WHEN event_type = 'search' THEN 1 ELSE 0 END) AS search_count,
+             SUM(CASE WHEN event_type = 'show'   THEN 1 ELSE 0 END) AS show_count,
+             MAX(created_at) AS last_used_at
+      FROM usage_events
+      WHERE entry_id IS NOT NULL
+      GROUP BY entry_id
+    `)
+        .all();
+    if (usageRows.length === 0) {
+        setMeta(db, "last_utility_computed_at", new Date().toISOString());
+        return;
+    }
+    // Batch-load existing utility scores
+    const existingScores = new Map();
+    const scoreRows = db.prepare("SELECT entry_id, utility FROM utility_scores").all();
+    for (const row of scoreRows) {
+        existingScores.set(row.entry_id, row.utility);
     }
-    if (entry.toc) {
-        parts.push(entry.toc.map((h) => h.text).join(" "));
+    for (const row of usageRows) {
+        const selectRate = row.search_count > 0 ? Math.min(1, row.show_count / row.search_count) : 0;
+        const prevUtility = existingScores.get(row.entry_id) ?? 0;
+        const utility = prevUtility * emaDecay + selectRate * emaNew;
+        upsertUtilityScore(db, row.entry_id, {
+            utility,
+            showCount: row.show_count,
+            searchCount: row.search_count,
+            selectRate,
+            lastUsedAt: row.last_used_at ?? undefined,
+        });
     }
-    return parts.join(" ").toLowerCase();
+    setMeta(db, "last_utility_computed_at", new Date().toISOString());
 }

package/dist/info.js

@@ -0,0 +1,92 @@
+import fs from "node:fs";
+import { getAssetTypes } from "./asset-spec";
+import { loadConfig } from "./config";
+import { closeDatabase, getEntryCount, getMeta, isVecAvailable, openDatabase } from "./db";
+import { getDbPath } from "./paths";
+import { pkgVersion } from "./version";
+/**
+ * Assemble system info describing the current capabilities, configuration,
+ * and index state. Used by `akm info`.
+ *
+ * @param options.dbPath - Override the database path (useful for testing)
+ */
+export function assembleInfo(options) {
+    const config = loadConfig();
+    // Asset types
+    const assetTypes = getAssetTypes();
+    // Search modes
+    const searchModes = ["fts"];
+    if (config.semanticSearch) {
+        searchModes.push("semantic", "hybrid");
+    }
+    // Registries (strip sensitive fields like apiKey from options)
+    const registries = (config.registries ?? []).map((r) => ({
+        url: r.url,
+        ...(r.name ? { name: r.name } : {}),
+        ...(r.provider ? { provider: r.provider } : {}),
+        ...(r.enabled !== undefined ? { enabled: r.enabled } : {}),
+    }));
+    // Stash providers
+    const stashProviders = (config.stashes ?? []).map((s) => ({
+        type: s.type,
+        ...(s.name ? { name: s.name } : {}),
+        ...(s.path ? { path: s.path } : {}),
+        ...(s.url ? { url: s.url } : {}),
+        ...(s.enabled !== undefined ? { enabled: s.enabled } : {}),
+    }));
+    // Index stats
+    const indexStats = readIndexStats(options?.dbPath);
+    return {
+        schemaVersion: 1,
+        version: pkgVersion,
+        assetTypes,
+        searchModes,
+        registries,
+        stashProviders,
+        indexStats,
+    };
+}
+function readIndexStats(dbPath) {
+    const resolvedPath = dbPath ?? getDbPath();
+    // If no index file exists, return zeros
+    if (!fs.existsSync(resolvedPath)) {
+        return {
+            entryCount: 0,
+            lastBuiltAt: null,
+            hasEmbeddings: false,
+            vecAvailable: false,
+        };
+    }
+    let db;
+    try {
+        db = openDatabase(resolvedPath);
+        const entryCount = getEntryCount(db);
+        const lastBuiltAt = getMeta(db, "builtAt") ?? null;
+        const vecAvailable = isVecAvailable(db);
+        const hasEmbeddings = getMeta(db, "hasEmbeddings") === "1";
+        return {
+            entryCount,
+            lastBuiltAt,
+            hasEmbeddings,
+            vecAvailable,
+        };
+    }
+    catch {
+        return {
+            entryCount: 0,
+            lastBuiltAt: null,
+            hasEmbeddings: false,
+            vecAvailable: false,
+        };
+    }
+    finally {
+        if (db) {
+            try {
+                closeDatabase(db);
+            }
+            catch {
+                /* ignore */
+            }
+        }
+    }
+}