akm-cli 0.8.0-rc2 → 0.8.1

package/dist/indexer/indexer.js

@@ -1,12 +1,17 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
 import fs from "node:fs";
 import path from "node:path";
+import { SCRIPT_EXTENSIONS } from "../core/asset-spec";
 import { isHttpUrl, resolveStashDir, toErrorMessage } from "../core/common";
 import { concurrentMap } from "../core/concurrent";
 import { getDbPath } from "../core/paths";
 import { isVerbose, warn, warnVerbose } from "../core/warn";
 import { resolveIndexPassLLM } from "../llm/index-passes";
 import { takeWorkflowDocument } from "../workflows/document-cache";
-import { clearStaleCacheEntries, closeDatabase, deleteEntriesByDir, deleteEntriesByStashDir, deleteIndexDirStatesByStashDir, getAllEntriesForEmbedding, getEmbeddingCount, getEntriesByDir, getEntryCount, getIndexDirState, getMeta, isVecAvailable, openDatabase, openExistingDatabase, rebuildFts, relinkUsageEvents, setMeta, upsertEmbedding, upsertEntry, upsertIndexDirState, upsertUtilityScore, upsertWorkflowDocument, warnIfVecMissing, } from "./db";
+import { clearStaleCacheEntries, closeDatabase, deleteEntriesByDir, deleteEntriesByIds, deleteEntriesByStashDir, deleteIndexDirStatesByStashDir, getAllEntriesForEmbedding, getEmbeddingCount, getEntriesByDir, getEntryCount, getIndexDirState, getMeta, isVecAvailable, openDatabase, openExistingDatabase, rebuildFts, relinkUsageEvents, setMeta, upsertEmbedding, upsertEntry, upsertIndexDirState, upsertUtilityScore, upsertWorkflowDocument, warnIfVecMissing, } from "./db";
+import { deleteStoredGraph } from "./graph-db";
 import { applyCuratedFrontmatter, applyWikiFrontmatter, generateMetadataFlat, isEnrichmentComplete, isWorkflowSkipWarning, loadStashFile, shouldIndexStashFile, } from "./metadata";
 import { buildSearchText } from "./search-fields";
 import { classifySemanticFailure, clearSemanticStatus, deriveSemanticProviderFingerprint, writeSemanticStatus, } from "./semantic-status";
@@ -64,6 +69,7 @@ async function runSourceCachePhase(ctx) {
                     ctx.hadRemovedSources = true;
                     deleteEntriesByStashDir(db, dir);
                     deleteIndexDirStatesByStashDir(db, dir);
+                    deleteStoredGraph(db, dir);
                 }
             }
         }
@@ -193,6 +199,29 @@ async function runFinalizePhase(ctx) {
     // suppress unused warning — sources was previously used inline
     void sources;
 }
+// ── Clean pass ───────────────────────────────────────────────────────────────
+/**
+ * Post-index clean pass: scan the `entries` table for rows whose source file
+ * no longer exists on disk and remove them (unless `dryRun` is true).
+ *
+ * Only rows with a non-empty `file_path` are checked — remote/virtual entries
+ * that have no local path are always skipped.
+ */
+function runCleanPass(db, dryRun) {
+    const allEntries = db.prepare("SELECT id, entry_key AS ref, file_path AS path FROM entries").all();
+    // Only check entries that have a non-empty local path (skip remote/virtual).
+    const localEntries = allEntries.filter((e) => typeof e.path === "string" && e.path.trim() !== "");
+    const missing = localEntries.filter((e) => !fs.existsSync(e.path));
+    if (!dryRun && missing.length > 0) {
+        deleteEntriesByIds(db, missing.map((e) => e.id));
+    }
+    return {
+        checked: localEntries.length,
+        removed: dryRun ? 0 : missing.length,
+        removedRefs: missing.map((e) => e.ref),
+        dryRun,
+    };
+}
 // ── Indexer ──────────────────────────────────────────────────────────────────
 export async function akmIndex(options) {
     const stashDir = options?.stashDir || resolveStashDir();
@@ -200,6 +229,8 @@ export async function akmIndex(options) {
     const signal = options?.signal;
     const reEnrich = options?.reEnrich === true;
     const full = options?.full === true;
+    const clean = options?.clean === true;
+    const dryRun = options?.dryRun === true;
     // Load config and resolve all stash sources
     const { loadConfig } = await import("../core/config.js");
     const config = loadConfig();
@@ -269,6 +300,14 @@ export async function akmIndex(options) {
         // ────────────────────────────────────────────────────────────────────────
         const { _verification: verification, _totalEntries: totalEntries } = ctx;
         const { timing } = ctx;
+        // ── Clean pass ───────────────────────────────────────────────────────────
+        // After the normal index completes, remove entries whose source files no
+        // longer exist on disk. Remote entries (empty file_path) are skipped.
+        let cleanResult;
+        if (clean) {
+            cleanResult = runCleanPass(db, dryRun);
+        }
+        // ────────────────────────────────────────────────────────────────────────
         return {
             stashDir,
             totalEntries,
@@ -283,9 +322,10 @@ export async function akmIndex(options) {
                 totalMs: Date.now() - timing.t0,
                 walkMs: timing.tWalkEnd - timing.tWalkStart,
                 llmMs: timing.tLlmEnd - timing.tWalkEnd,
-                embedMs: timing.tEmbedEnd - timing.tFtsEnd,
-                ftsMs: timing.tFtsEnd - timing.tLlmEnd,
+                embedMs: timing.tEmbedEnd - timing.tLlmEnd,
+                ftsMs: timing.tFtsEnd - timing.tEmbedEnd,
             },
+            ...(cleanResult !== undefined ? { clean: cleanResult } : {}),
         };
     }
     finally {
@@ -567,7 +607,20 @@ async function indexEntries(db, allSourceEntries, isIncremental, builtAtMs, hadR
                 reason: persistedReason,
             });
             if (persistedRows === 0) {
-                warn(`[index] zero-row ${dirPath}: ${persistedReason}`);
+                // Warn only when the dir had files that *could* produce entries (.md or
+                // known script extensions). Dirs with only non-indexable types (.json,
+                // .yaml, .conf, .env, .gitkeep) or deduped-only rows are expected and
+                // not actionable at normal log level.
+                const hasIndexableExtension = files.some((f) => {
+                    const ext = path.extname(f).toLowerCase();
+                    return ext === ".md" || SCRIPT_EXTENSIONS.has(ext);
+                });
+                if (persistedReason !== "deduped-zero-row" && hasIndexableExtension) {
+                    warn(`[index] zero-row ${dirPath}: ${persistedReason}`);
+                }
+                else {
+                    warnVerbose(`[index] zero-row ${dirPath}: ${persistedReason}`);
+                }
             }
         }
     });
@@ -884,14 +937,24 @@ async function generateEmbeddingsForDb(db, config, onProgress, signal) {
         throwIfAborted(signal);
         // Wrap all embedding upserts in a single transaction so partial
         // state is rolled back on failure rather than leaving the table half-filled.
+        let storedCount = 0;
+        let skippedCount = 0;
         db.transaction(() => {
             for (let i = 0; i < allEntries.length; i++) {
-                upsertEmbedding(db, allEntries[i].id, embeddings[i]);
+                if (upsertEmbedding(db, allEntries[i].id, embeddings[i])) {
+                    storedCount++;
+                }
+                else {
+                    skippedCount++;
+                }
             }
         })();
+        if (skippedCount > 0) {
+            warn(`[embed] ${skippedCount} embedding${skippedCount === 1 ? "" : "s"} skipped (entry deleted between queue and write)`);
+        }
         onProgress({
             phase: "embeddings",
-            message: `Stored ${embeddings.length} embedding${embeddings.length === 1 ? "" : "s"}.`,
+            message: `Stored ${storedCount} embedding${storedCount === 1 ? "" : "s"}.`,
         });
         setMeta(db, "embeddingFingerprint", currentFingerprint);
         return { success: true };
@@ -1258,18 +1321,25 @@ export function recomputeUtilityScores(db) {
     const emaDecay = EMA_DECAY ** elapsedDays;
     const emaNew = 1 - emaDecay; // complement so weights still sum to 1
     // Single aggregate query instead of N+1 per-entry queries.
-    // Only processes entries that actually have usage events.
+    // Only processes entries that actually have usage events AND still exist
+    // in `entries`. The latter check is critical: usage_events has no FK to
+    // entries, so its entry_id can become stale (entry deleted, re-keyed,
+    // moved between sources). Without the JOIN, writing the derived row to
+    // utility_scores (which DOES have an FK) raises "FOREIGN KEY constraint
+    // failed" and rolls back the whole finalize transaction — failing every
+    // index run.
     const usageRows = db
         .prepare(`
-      SELECT entry_id,
-             SUM(CASE WHEN event_type = 'search' THEN 1 ELSE 0 END) AS search_count,
-             SUM(CASE WHEN event_type = 'show'   THEN 1 ELSE 0 END) AS show_count,
-             SUM(CASE WHEN event_type = 'feedback' AND signal = 'positive' THEN 1 ELSE 0 END) AS positive_feedback_count,
-             SUM(CASE WHEN event_type = 'feedback' AND signal = 'negative' THEN 1 ELSE 0 END) AS negative_feedback_count,
-             MAX(created_at) AS last_used_at
-      FROM usage_events
-      WHERE entry_id IS NOT NULL
-      GROUP BY entry_id
+      SELECT u.entry_id,
+             SUM(CASE WHEN u.event_type = 'search' THEN 1 ELSE 0 END) AS search_count,
+             SUM(CASE WHEN u.event_type = 'show'   THEN 1 ELSE 0 END) AS show_count,
+             SUM(CASE WHEN u.event_type = 'feedback' AND u.signal = 'positive' THEN 1 ELSE 0 END) AS positive_feedback_count,
+             SUM(CASE WHEN u.event_type = 'feedback' AND u.signal = 'negative' THEN 1 ELSE 0 END) AS negative_feedback_count,
+             MAX(u.created_at) AS last_used_at
+      FROM usage_events u
+      JOIN entries e ON e.id = u.entry_id
+      WHERE u.entry_id IS NOT NULL
+      GROUP BY u.entry_id
     `)
         .all();
     if (usageRows.length === 0) {

package/dist/indexer/llm-cache.js

@@ -1,33 +1,38 @@
-/**
- * Generic LLM-result cache wrapper shared across indexer passes.
- *
- * Each pass that calls an LLM and wants to skip re-processing unchanged
- * content can delegate the cache check/write to `withLlmCache` instead of
- * duplicating the hash-compute → lookup → write pattern inline.
- */
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
 import { computeBodyHash, getLlmCacheEntry, upsertLlmCacheEntry } from "./db";
 /**
  * Generic LLM cache wrapper. Returns cached result if body unchanged,
  * otherwise calls llmFn(), caches the result, and returns it.
  * Returns undefined if llmFn() returns undefined or throws.
  *
- * @param db        - SQLite database holding the LLM result cache.
- * @param cacheKey  - Stable identifier for this asset (typically its absolute path).
- * @param body      - The content being processed; its hash determines cache validity.
- * @param reEnrich  - When true the cache is bypassed and llmFn() is always called.
- * @param llmFn     - Async function that performs the actual LLM call.
- * @param validate  - Converts the raw parsed JSON back into the pass-specific type;
- *                    returns undefined when the cached data is unusable.
+ * @param db            - SQLite database holding the LLM result cache.
+ * @param cacheKey      - Stable identifier for this asset (typically its absolute path).
+ * @param body          - The content being processed; its hash determines cache validity.
+ * @param reEnrich      - When true the cache is bypassed and llmFn() is always called.
+ * @param llmFn         - Async function that performs the actual LLM call.
+ * @param validate      - Converts the raw parsed JSON back into the pass-specific type;
+ *                        returns undefined when the cached data is unusable.
+ * @param precomputedHash - Optional precomputed SHA-256 of `body`. When provided,
+ *                          the wrapper skips its internal hashing — callers that
+ *                          already hashed the body (e.g. to reuse it elsewhere)
+ *                          should pass this to avoid the redundant work.
+ * @param cacheVariant  - Namespace token for the cache row (e.g. staleness-detect
+ *                        sets one so its rows do not collide with memory-inference).
+ * @param hooks         - Optional event sink for telemetry (see {@link WithLlmCacheHooks}).
  */
-export async function withLlmCache(db, cacheKey, body, reEnrich, llmFn, validate) {
-    const bodyHash = computeBodyHash(body);
+export async function withLlmCache(db, cacheKey, body, reEnrich, llmFn, validate, precomputedHash, cacheVariant = "", hooks) {
+    const bodyHash = precomputedHash ?? computeBodyHash(body);
     if (!reEnrich) {
         try {
-            const cached = getLlmCacheEntry(db, cacheKey, bodyHash);
+            const cached = getLlmCacheEntry(db, cacheKey, bodyHash, cacheVariant);
             if (cached) {
                 const result = validate(JSON.parse(cached.resultJson));
-                if (result !== undefined)
+                if (result !== undefined) {
+                    hooks?.onCacheHit?.();
                     return result;
+                }
             }
         }
         catch {
@@ -37,7 +42,7 @@ export async function withLlmCache(db, cacheKey, body, reEnrich, llmFn, validate
     const result = await llmFn();
     if (result !== undefined) {
         try {
-            upsertLlmCacheEntry(db, cacheKey, bodyHash, JSON.stringify(result));
+            upsertLlmCacheEntry(db, cacheKey, bodyHash, JSON.stringify(result), cacheVariant);
         }
         catch {
             // Cache write failure is non-fatal

package/dist/indexer/manifest.js

@@ -1,3 +1,6 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
 /**
  * Manifest: compact asset listing for cheap capability discovery.
  *

package/dist/indexer/matchers.js

@@ -1,15 +1,185 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
 /**
  * Built-in asset matchers for the akm file classification system.
  *
- * Classification facts now live in `match-contributors.ts`. This module keeps
- * the existing matcher API and registration order intact by adapting those
- * facts back into `MatchResult` values.
+ * Each private `classifyBy*` function encapsulates the classification logic for
+ * one heuristic. The public `*Matcher` exports compose those facts into the
+ * `MatchResult` shape expected by the rest of the indexer.
  */
 import { defaultRendererRegistry } from "../core/asset-registry";
+import { SCRIPT_EXTENSIONS } from "../core/asset-spec";
+import { looksLikeWorkflow } from "../workflows/parser";
 import { registerMatcher } from "./file-context";
-import { directoryContributor, extensionContributor, parentDirHintContributor, smartMdContributor, wikiContributor, } from "./match-contributors";
-function toMatchResult(ctx, contributor) {
-    const fact = contributor.classify(ctx);
+// ---------------------------------------------------------------------------
+// Private data
+// ---------------------------------------------------------------------------
+const DIR_TYPE_MAP = [
+    {
+        dir: "scripts",
+        type: "script",
+        test: (ext) => SCRIPT_EXTENSIONS.has(ext),
+    },
+    {
+        dir: "commands",
+        type: "command",
+        test: (ext) => ext === ".md",
+    },
+    {
+        dir: "agents",
+        type: "agent",
+        test: (ext) => ext === ".md",
+    },
+    {
+        dir: "knowledge",
+        type: "knowledge",
+        test: (ext) => ext === ".md",
+    },
+    {
+        dir: "workflows",
+        type: "workflow",
+        test: (ext) => ext === ".md",
+    },
+    {
+        dir: "memories",
+        type: "memory",
+        test: (ext) => ext === ".md",
+    },
+    {
+        dir: "lessons",
+        type: "lesson",
+        test: (ext) => ext === ".md",
+    },
+    {
+        dir: "env",
+        type: "env",
+        test: (_, fileName) => fileName === ".env" || fileName.endsWith(".env"),
+    },
+    {
+        dir: "vaults",
+        type: "vault",
+        test: (_, fileName) => fileName === ".env" || fileName.endsWith(".env"),
+    },
+    {
+        dir: "secrets",
+        type: "secret",
+        // Any regular file under secrets/ is a secret value, except the lock and
+        // sensitive-marker sidecars. The whole file is the value (no extension or
+        // body parsing — see the secret-file renderer + indexer guards).
+        test: (_, fileName) => !fileName.endsWith(".lock") && !fileName.endsWith(".sensitive"),
+    },
+    {
+        dir: "tasks",
+        type: "task",
+        test: (ext) => ext === ".md",
+    },
+];
+const COMMAND_PLACEHOLDER_RE = /\$ARGUMENTS|\$[123]\b/;
+// Files that should never be treated as the typed asset for the surrounding
+// directory (e.g. `workflows/README.md` is documentation, not a workflow).
+// Lower-cased and matched case-insensitively against `ctx.fileName`. They are
+// still indexable — falling through to `classifyBySmartMd` typically routes
+// them to the generic `knowledge` type.
+const TYPED_DIR_DOC_FILES = new Set(["readme.md"]);
+function isTypedDirDocFile(fileName) {
+    return TYPED_DIR_DOC_FILES.has(fileName.toLowerCase());
+}
+// ---------------------------------------------------------------------------
+// Private helpers
+// ---------------------------------------------------------------------------
+function matchDirectoryHint(dirName, ctx, specificity) {
+    if (dirName === "skills" && ctx.fileName === "SKILL.md") {
+        return { type: "skill", specificity };
+    }
+    for (const rule of DIR_TYPE_MAP) {
+        if (rule.dir === dirName && rule.test(ctx.ext, ctx.fileName)) {
+            // Skip `README.md` (case-insensitive) so `workflows/README.md`,
+            // `agents/README.md`, etc. are not parsed as the typed asset and don't
+            // trip the workflow/agent metadata validators. They still get indexed
+            // as `knowledge` via the smart-md matcher.
+            if (isTypedDirDocFile(ctx.fileName))
+                return null;
+            return { type: rule.type, specificity };
+        }
+    }
+    return null;
+}
+function classifyByExtension(ctx) {
+    if (ctx.fileName === "SKILL.md" && !ctx.ancestorDirs.includes("wikis")) {
+        return { type: "skill", specificity: 25 };
+    }
+    if (SCRIPT_EXTENSIONS.has(ctx.ext)) {
+        return { type: "script", specificity: 3 };
+    }
+    return null;
+}
+function classifyByDirectory(ctx) {
+    for (const dir of ctx.ancestorDirs) {
+        const result = matchDirectoryHint(dir, ctx, 10);
+        if (result)
+            return result;
+    }
+    return null;
+}
+function classifyByParentDirHint(ctx) {
+    const { parentDir, ext, fileName } = ctx;
+    if (parentDir === "skills" && (fileName === "SKILL.md" || ext === ".md")) {
+        return { type: "skill", specificity: 15 };
+    }
+    return matchDirectoryHint(parentDir, ctx, 15);
+}
+function classifyBySmartMd(ctx) {
+    if (ctx.ext !== ".md")
+        return null;
+    // Never read the body of a file under secrets/ — the whole file is the
+    // secret value. The directory matcher classifies it as `secret` without
+    // touching content; bailing here keeps classifyBySmartMd from calling
+    // ctx.content()/frontmatter() on secret material.
+    if (ctx.ancestorDirs.includes("secrets"))
+        return null;
+    // README.md is documentation, never a workflow/agent/command even when the
+    // body shape would otherwise classify (e.g. step-list inside a project
+    // README under workflows/). Fall straight through to `knowledge`.
+    if (isTypedDirDocFile(ctx.fileName)) {
+        return { type: "knowledge", specificity: 5 };
+    }
+    const body = ctx.content();
+    if (looksLikeWorkflow(body)) {
+        return { type: "workflow", specificity: 19 };
+    }
+    const fm = ctx.frontmatter();
+    if (fm) {
+        if ("toolPolicy" in fm || "tools" in fm) {
+            return { type: "agent", specificity: 20 };
+        }
+        if ("agent" in fm) {
+            return { type: "command", specificity: 18 };
+        }
+    }
+    if (COMMAND_PLACEHOLDER_RE.test(body)) {
+        return { type: "command", specificity: 18 };
+    }
+    if (fm && "model" in fm) {
+        return { type: "agent", specificity: 8 };
+    }
+    return { type: "knowledge", specificity: 5 };
+}
+function classifyByWiki(ctx) {
+    if (ctx.ext !== ".md")
+        return null;
+    const idx = ctx.ancestorDirs.indexOf("wikis");
+    if (idx < 0)
+        return null;
+    if (idx + 1 >= ctx.ancestorDirs.length)
+        return null;
+    return { type: "wiki", specificity: 20 };
+}
+// ---------------------------------------------------------------------------
+// Adapter: MatchFact → MatchResult
+// ---------------------------------------------------------------------------
+function toMatchResult(ctx, classify) {
+    const fact = classify(ctx);
     if (!fact)
         return null;
     const renderer = defaultRendererRegistry.rendererNameFor(fact.type);
@@ -22,20 +192,23 @@ function toMatchResult(ctx, contributor) {
         ...(fact.meta ? { meta: fact.meta } : {}),
     };
 }
+// ---------------------------------------------------------------------------
+// Public matchers (API unchanged)
+// ---------------------------------------------------------------------------
 export function extensionMatcher(ctx) {
-    return toMatchResult(ctx, extensionContributor);
+    return toMatchResult(ctx, classifyByExtension);
 }
 export function directoryMatcher(ctx) {
-    return toMatchResult(ctx, directoryContributor);
+    return toMatchResult(ctx, classifyByDirectory);
 }
 export function parentDirHintMatcher(ctx) {
-    return toMatchResult(ctx, parentDirHintContributor);
+    return toMatchResult(ctx, classifyByParentDirHint);
 }
 export function smartMdMatcher(ctx) {
-    return toMatchResult(ctx, smartMdContributor);
+    return toMatchResult(ctx, classifyBySmartMd);
 }
 export function wikiMatcher(ctx) {
-    return toMatchResult(ctx, wikiContributor);
+    return toMatchResult(ctx, classifyByWiki);
 }
 const builtinMatchers = [
     extensionMatcher,