akm-cli 0.1.3 → 0.2.0

package/dist/indexer.js

@@ -1,9 +1,11 @@
 import fs from "node:fs";
 import path from "node:path";
 import { resolveStashDir } from "./common";
-import { closeDatabase, DB_VERSION, deleteEntriesByDir, deleteEntriesByStashDir, getEntriesByDir, getEntryCount, getMeta, isVecAvailable, openDatabase, rebuildFts, setMeta, upsertEmbedding, upsertEntry, warnIfVecMissing, } from "./db";
+import { closeDatabase, deleteEntriesByDir, deleteEntriesByStashDir, getEntriesByDir, getEntryCount, getMeta, isVecAvailable, openDatabase, rebuildFts, setMeta, upsertEmbedding, upsertEntry, upsertUtilityScore, warnIfVecMissing, } from "./db";
 import { generateMetadataFlat, loadStashFile } from "./metadata";
 import { getDbPath } from "./paths";
+import { buildSearchText } from "./search-fields";
+import { ensureUsageEventsSchema, purgeOldUsageEvents } from "./usage-events";
 import { walkStashFlat } from "./walker";
 import { warn } from "./warn";
 // ── Indexer ──────────────────────────────────────────────────────────────────
@@ -12,7 +14,10 @@ export async function akmIndex(options) {
     // Load config and resolve all stash sources
     const { loadConfig } = await import("./config.js");
     const config = loadConfig();
-    const { resolveAllStashDirs } = await import("./search-source.js");
+    // Ensure git stash caches are extracted before resolving stash dirs,
+    // so their content directories exist on disk for the walker to discover.
+    const { ensureGitCaches, resolveAllStashDirs } = await import("./search-source.js");
+    await ensureGitCaches(config);
     const allStashDirs = resolveAllStashDirs(stashDir);
     const t0 = Date.now();
     // Open database — pass embedding dimension from config if available
@@ -26,7 +31,7 @@ export async function akmIndex(options) {
         const isIncremental = !options?.full && prevStashDir === stashDir && !!prevBuiltAt;
         const builtAtMs = isIncremental && prevBuiltAt ? new Date(prevBuiltAt).getTime() : 0;
         if (options?.full || !isIncremental) {
-            // HI-5: the delete is now merged into the insert transaction inside
+            // The delete is now merged into the insert transaction inside
             // indexEntries() so that a reader never sees an empty database between
             // the wipe and the re-inserts.  The doFullDelete flag signals this path.
         }
@@ -59,9 +64,9 @@ export async function akmIndex(options) {
         const tWalkStart = Date.now();
         // Walk stash dirs and index entries.
         // doFullDelete=true merges the wipe into the same transaction as the
-        // inserts (HI-5) so readers never see an empty database mid-rebuild.
+        // inserts so readers never see an empty database mid-rebuild.
         const doFullDelete = options?.full || !isIncremental;
-        const { scannedDirs, skippedDirs, generatedCount, dirsNeedingLlm } = await indexEntries(db, allStashDirs, stashDir, isIncremental, builtAtMs, doFullDelete);
+        const { scannedDirs, skippedDirs, generatedCount, dirsNeedingLlm } = await indexEntries(db, allStashDirs, isIncremental, builtAtMs, doFullDelete);
         const tWalkEnd = Date.now();
         // Enhance entries with LLM if configured
         await enhanceDirsWithLlm(db, config, dirsNeedingLlm);
@@ -69,11 +74,12 @@ export async function akmIndex(options) {
         // Rebuild FTS after all inserts
         rebuildFts(db);
         const tFtsEnd = Date.now();
+        // Recompute utility scores from usage_events after FTS rebuild
+        recomputeUtilityScores(db);
         // Generate embeddings if semantic search is enabled
         const hasEmbeddings = await generateEmbeddingsForDb(db, config);
         const tEmbedEnd = Date.now();
         // Update metadata
-        setMeta(db, "version", String(DB_VERSION));
         setMeta(db, "builtAt", new Date().toISOString());
         setMeta(db, "stashDir", stashDir);
         setMeta(db, "stashDirs", JSON.stringify(allStashDirs));
@@ -104,7 +110,7 @@ export async function akmIndex(options) {
     }
 }
 // ── Extracted helpers for indexing ────────────────────────────────────────────
-async function indexEntries(db, allStashDirs, _stashDir, isIncremental, builtAtMs, doFullDelete = false) {
+async function indexEntries(db, allStashDirs, isIncremental, builtAtMs, doFullDelete = false) {
     let scannedDirs = 0;
     let skippedDirs = 0;
     let generatedCount = 0;
@@ -170,7 +176,7 @@ async function indexEntries(db, allStashDirs, _stashDir, isIncremental, builtAtM
     // occurrence wins.
     const indexedAssetIdentities = new Set();
     const insertTransaction = db.transaction(() => {
-        // HI-5: Perform the full-rebuild wipe as the FIRST step of the insert
+        // Perform the full-rebuild wipe as the FIRST step of the insert
         // transaction so delete and re-insert are atomic — a concurrent reader
         // never observes an empty database between the two operations.
         if (doFullDelete) {
@@ -189,6 +195,8 @@ async function indexEntries(db, allStashDirs, _stashDir, isIncremental, builtAtM
                 }
             }
             db.exec("DELETE FROM entries_fts");
+            db.exec("DELETE FROM utility_scores");
+            db.exec("DELETE FROM usage_events");
             db.exec("DELETE FROM entries");
         }
         for (const { dirPath, currentStashDir, files, stash, skip } of dirRecords) {
@@ -235,8 +243,8 @@ async function enhanceDirsWithLlm(db, config, dirsNeedingLlm) {
         if (generatedEntries.length === 0)
             continue;
         const generatedStash = { entries: generatedEntries };
-        const enhanced = await enhanceStashWithLlm(config.llm, generatedStash, dirPath, files);
-        // HI-2: Re-upsert the enhanced entries in a single transaction so a crash
+        const enhanced = await enhanceStashWithLlm(config.llm, generatedStash, files);
+        // Re-upsert the enhanced entries in a single transaction so a crash
         // cannot leave half the entries updated and the rest stale.
         db.transaction(() => {
             for (const entry of enhanced.entries) {
@@ -258,7 +266,7 @@ async function generateEmbeddingsForDb(db, config) {
             return true;
         const texts = allEntries.map((e) => e.searchText);
         const embeddings = await embedBatch(texts, config.embedding);
-        // HI-3: Wrap all embedding upserts in a single transaction so partial
+        // Wrap all embedding upserts in a single transaction so partial
         // state is rolled back on failure rather than leaving the table half-filled.
         db.transaction(() => {
             for (let i = 0; i < allEntries.length; i++) {
@@ -289,7 +297,6 @@ function attachFileSize(entry, entryPath) {
         return entry;
     }
 }
-/** Set of all known type directory names */
 function isDirStale(dirPath, currentFiles, previousEntries, builtAtMs) {
     // Check if file set changed (additions or deletions)
     const prevFileNames = new Set(previousEntries.map((ie) => ie.entry.filename).filter((e) => !!e));
@@ -321,7 +328,7 @@ function isDirStale(dirPath, currentFiles, previousEntries, builtAtMs) {
     }
     return false;
 }
-async function enhanceStashWithLlm(llmConfig, stash, _dirPath, files) {
+async function enhanceStashWithLlm(llmConfig, stash, files) {
     const { enhanceMetadata } = await import("./llm.js");
     const enhanced = [];
     for (const entry of stash.entries) {
@@ -391,30 +398,74 @@ export function matchEntryToFile(entryName, fileMap, files) {
     // Fallback to first file, or null if no files are available
     return files[0] || null;
 }
-export function buildSearchText(entry) {
-    const parts = [entry.name.replace(/[-_]/g, " ")];
-    if (entry.description)
-        parts.push(entry.description);
-    if (entry.tags)
-        parts.push(entry.tags.join(" "));
-    if (entry.examples)
-        parts.push(entry.examples.join(" "));
-    if (entry.aliases)
-        parts.push(entry.aliases.join(" "));
-    if (entry.searchHints)
-        parts.push(entry.searchHints.join(" "));
-    if (entry.usage)
-        parts.push(entry.usage.join(" "));
-    if (entry.intent) {
-        if (entry.intent.when)
-            parts.push(entry.intent.when);
-        if (entry.intent.input)
-            parts.push(entry.intent.input);
-        if (entry.intent.output)
-            parts.push(entry.intent.output);
+export { buildSearchFields, buildSearchText } from "./search-fields";
+// ── Utility score recomputation ──────────────────────────────────────────────
+/** Retention window for usage events: events older than this are purged. */
+const USAGE_EVENT_RETENTION_DAYS = 90;
+/**
+ * Recompute utility scores for all entries based on usage_events data.
+ *
+ * For each entry:
+ *   - Count search appearances (event_type = 'search')
+ *   - Count show events (event_type = 'show')
+ *   - Compute select_rate = showCount / searchCount, clamped to [0, 1]
+ *   - Update utility via EMA: utility = previousUtility * 0.7 + selectRate * 0.3
+ *
+ * Also purges usage_events older than 90 days and ensures the M-1
+ * usage_events table exists before querying.
+ *
+ * Called during `akm index` after FTS rebuild.
+ */
+export function recomputeUtilityScores(db) {
+    const EMA_DECAY = 0.7;
+    // Ensure usage_events table exists before querying
+    ensureUsageEventsSchema(db);
+    // Purge stale usage events (90-day retention)
+    purgeOldUsageEvents(db, USAGE_EVENT_RETENTION_DAYS);
+    // Time-proportional decay: apply one round of EMA per elapsed day so
+    // indexing frequency doesn't affect how fast scores decay.
+    const lastComputedAt = getMeta(db, "last_utility_computed_at");
+    let elapsedDays = 1; // default for first run
+    if (lastComputedAt) {
+        const ms = Date.now() - new Date(lastComputedAt).getTime();
+        elapsedDays = Math.max(1, ms / (1000 * 60 * 60 * 24));
+    }
+    const emaDecay = EMA_DECAY ** elapsedDays;
+    const emaNew = 1 - emaDecay; // complement so weights still sum to 1
+    // Single aggregate query instead of N+1 per-entry queries.
+    // Only processes entries that actually have usage events.
+    const usageRows = db
+        .prepare(`
+      SELECT entry_id,
+             SUM(CASE WHEN event_type = 'search' THEN 1 ELSE 0 END) AS search_count,
+             SUM(CASE WHEN event_type = 'show'   THEN 1 ELSE 0 END) AS show_count,
+             MAX(created_at) AS last_used_at
+      FROM usage_events
+      WHERE entry_id IS NOT NULL
+      GROUP BY entry_id
+    `)
+        .all();
+    if (usageRows.length === 0) {
+        setMeta(db, "last_utility_computed_at", new Date().toISOString());
+        return;
+    }
+    // Batch-load existing utility scores
+    const existingScores = new Map();
+    const scoreRows = db.prepare("SELECT entry_id, utility FROM utility_scores").all();
+    for (const row of scoreRows) {
+        existingScores.set(row.entry_id, row.utility);
     }
-    if (entry.toc) {
-        parts.push(entry.toc.map((h) => h.text).join(" "));
+    for (const row of usageRows) {
+        const selectRate = row.search_count > 0 ? Math.min(1, row.show_count / row.search_count) : 0;
+        const prevUtility = existingScores.get(row.entry_id) ?? 0;
+        const utility = prevUtility * emaDecay + selectRate * emaNew;
+        upsertUtilityScore(db, row.entry_id, {
+            utility,
+            showCount: row.show_count,
+            searchCount: row.search_count,
+            selectRate,
+            lastUsedAt: row.last_used_at ?? undefined,
+        });
     }
-    return parts.join(" ").toLowerCase();
+    setMeta(db, "last_utility_computed_at", new Date().toISOString());
 }

package/dist/info.js

@@ -0,0 +1,92 @@
+import fs from "node:fs";
+import { getAssetTypes } from "./asset-spec";
+import { loadConfig } from "./config";
+import { closeDatabase, getEntryCount, getMeta, isVecAvailable, openDatabase } from "./db";
+import { getDbPath } from "./paths";
+import { pkgVersion } from "./version";
+/**
+ * Assemble system info describing the current capabilities, configuration,
+ * and index state. Used by `akm info`.
+ *
+ * @param options.dbPath - Override the database path (useful for testing)
+ */
+export function assembleInfo(options) {
+    const config = loadConfig();
+    // Asset types
+    const assetTypes = getAssetTypes();
+    // Search modes
+    const searchModes = ["fts"];
+    if (config.semanticSearch) {
+        searchModes.push("semantic", "hybrid");
+    }
+    // Registries (strip sensitive fields like apiKey from options)
+    const registries = (config.registries ?? []).map((r) => ({
+        url: r.url,
+        ...(r.name ? { name: r.name } : {}),
+        ...(r.provider ? { provider: r.provider } : {}),
+        ...(r.enabled !== undefined ? { enabled: r.enabled } : {}),
+    }));
+    // Stash providers
+    const stashProviders = (config.stashes ?? []).map((s) => ({
+        type: s.type,
+        ...(s.name ? { name: s.name } : {}),
+        ...(s.path ? { path: s.path } : {}),
+        ...(s.url ? { url: s.url } : {}),
+        ...(s.enabled !== undefined ? { enabled: s.enabled } : {}),
+    }));
+    // Index stats
+    const indexStats = readIndexStats(options?.dbPath);
+    return {
+        schemaVersion: 1,
+        version: pkgVersion,
+        assetTypes,
+        searchModes,
+        registries,
+        stashProviders,
+        indexStats,
+    };
+}
+function readIndexStats(dbPath) {
+    const resolvedPath = dbPath ?? getDbPath();
+    // If no index file exists, return zeros
+    if (!fs.existsSync(resolvedPath)) {
+        return {
+            entryCount: 0,
+            lastBuiltAt: null,
+            hasEmbeddings: false,
+            vecAvailable: false,
+        };
+    }
+    let db;
+    try {
+        db = openDatabase(resolvedPath);
+        const entryCount = getEntryCount(db);
+        const lastBuiltAt = getMeta(db, "builtAt") ?? null;
+        const vecAvailable = isVecAvailable(db);
+        const hasEmbeddings = getMeta(db, "hasEmbeddings") === "1";
+        return {
+            entryCount,
+            lastBuiltAt,
+            hasEmbeddings,
+            vecAvailable,
+        };
+    }
+    catch {
+        return {
+            entryCount: 0,
+            lastBuiltAt: null,
+            hasEmbeddings: false,
+            vecAvailable: false,
+        };
+    }
+    finally {
+        if (db) {
+            try {
+                closeDatabase(db);
+            }
+            catch {
+                /* ignore */
+            }
+        }
+    }
+}