akm-cli 0.7.5 → 0.8.0-rc.3

package/dist/indexer/llm-cache.js

@@ -0,0 +1,47 @@
+/**
+ * Generic LLM-result cache wrapper shared across indexer passes.
+ *
+ * Each pass that calls an LLM and wants to skip re-processing unchanged
+ * content can delegate the cache check/write to `withLlmCache` instead of
+ * duplicating the hash-compute → lookup → write pattern inline.
+ */
+import { computeBodyHash, getLlmCacheEntry, upsertLlmCacheEntry } from "./db";
+/**
+ * Generic LLM cache wrapper. Returns cached result if body unchanged,
+ * otherwise calls llmFn(), caches the result, and returns it.
+ * Returns undefined if llmFn() returns undefined or throws.
+ *
+ * @param db        - SQLite database holding the LLM result cache.
+ * @param cacheKey  - Stable identifier for this asset (typically its absolute path).
+ * @param body      - The content being processed; its hash determines cache validity.
+ * @param reEnrich  - When true the cache is bypassed and llmFn() is always called.
+ * @param llmFn     - Async function that performs the actual LLM call.
+ * @param validate  - Converts the raw parsed JSON back into the pass-specific type;
+ *                    returns undefined when the cached data is unusable.
+ */
+export async function withLlmCache(db, cacheKey, body, reEnrich, llmFn, validate) {
+    const bodyHash = computeBodyHash(body);
+    if (!reEnrich) {
+        try {
+            const cached = getLlmCacheEntry(db, cacheKey, bodyHash);
+            if (cached) {
+                const result = validate(JSON.parse(cached.resultJson));
+                if (result !== undefined)
+                    return result;
+            }
+        }
+        catch {
+            // Cache corrupt — fall through
+        }
+    }
+    const result = await llmFn();
+    if (result !== undefined) {
+        try {
+            upsertLlmCacheEntry(db, cacheKey, bodyHash, JSON.stringify(result));
+        }
+        catch {
+            // Cache write failure is non-fatal
+        }
+    }
+    return result;
+}

package/dist/indexer/matchers.js

@@ -1,193 +1,128 @@
 /**
  * Built-in asset matchers for the akm file classification system.
  *
- * Five matchers are registered at module load time, each at a different
- * specificity level. Extension and content determine type; directories are
- * optional specificity boosts, not requirements.
- *
- * - `extensionMatcher` (3) -- classifies any file by extension alone.
- *   Ensures every known file type is discoverable regardless of directory.
- * - `directoryMatcher` (10) -- boosts specificity when an ancestor
- *   directory matches a known type name (e.g. `scripts/`, `agents/`).
- * - `parentDirHintMatcher` (15) -- boosts specificity based on the
- *   immediate parent directory name.
- * - `smartMdMatcher` (20 / 18 / 8 / 5) -- inspects markdown frontmatter
- *   and body content for agent/command signals; falls back to "knowledge"
- *   at specificity 5 when no signals are found. Command signals (`agent`
- *   frontmatter, `$ARGUMENTS`/`$1`-`$3` placeholders) return 18.
- * - `wikiMatcher` (20) -- classifies any `.md` under `wikis/<name>/…` as
- *   `wiki`. Registered last so the later-wins tiebreaker beats agent at 20.
+ * Each private `classifyBy*` function encapsulates the classification logic for
+ * one heuristic. The public `*Matcher` exports compose those facts into the
+ * `MatchResult` shape expected by the rest of the indexer.
  */
+import { defaultRendererRegistry } from "../core/asset-registry";
 import { SCRIPT_EXTENSIONS } from "../core/asset-spec";
 import { looksLikeWorkflow } from "../workflows/parser";
 import { registerMatcher } from "./file-context";
-// ── extensionMatcher (specificity: 3) ────────────────────────────────────────
-/**
- * Base-level matcher that classifies files purely by extension.
- *
- * This is the foundation of the classification system: every file with a
- * known extension gets a type, regardless of what directory it lives in.
- * Higher-specificity matchers (directory, content) can override this.
- *
- * .md files are NOT handled here -- smartMdMatcher provides richer
- * classification for markdown via frontmatter inspection.
- */
-export function extensionMatcher(ctx) {
-    // SKILL.md is a skill regardless of location — high specificity beats
-    // smartMdMatcher's knowledge fallback and all directory-based matchers.
-    // Exception: files under wikis/<name>/… are always wiki pages; the wiki
-    // directory is an authoritative signal that outranks the filename.
+// ---------------------------------------------------------------------------
+// Private data
+// ---------------------------------------------------------------------------
+const DIR_TYPE_MAP = [
+    {
+        dir: "scripts",
+        type: "script",
+        test: (ext) => SCRIPT_EXTENSIONS.has(ext),
+    },
+    {
+        dir: "commands",
+        type: "command",
+        test: (ext) => ext === ".md",
+    },
+    {
+        dir: "agents",
+        type: "agent",
+        test: (ext) => ext === ".md",
+    },
+    {
+        dir: "knowledge",
+        type: "knowledge",
+        test: (ext) => ext === ".md",
+    },
+    {
+        dir: "workflows",
+        type: "workflow",
+        test: (ext) => ext === ".md",
+    },
+    {
+        dir: "memories",
+        type: "memory",
+        test: (ext) => ext === ".md",
+    },
+    {
+        dir: "lessons",
+        type: "lesson",
+        test: (ext) => ext === ".md",
+    },
+    {
+        dir: "vaults",
+        type: "vault",
+        test: (_, fileName) => fileName === ".env" || fileName.endsWith(".env"),
+    },
+    {
+        dir: "tasks",
+        type: "task",
+        test: (ext) => ext === ".md",
+    },
+];
+const COMMAND_PLACEHOLDER_RE = /\$ARGUMENTS|\$[123]\b/;
+// ---------------------------------------------------------------------------
+// Private helpers
+// ---------------------------------------------------------------------------
+function matchDirectoryHint(dirName, ctx, specificity) {
+    if (dirName === "skills" && ctx.fileName === "SKILL.md") {
+        return { type: "skill", specificity };
+    }
+    for (const rule of DIR_TYPE_MAP) {
+        if (rule.dir === dirName && rule.test(ctx.ext, ctx.fileName)) {
+            return { type: rule.type, specificity };
+        }
+    }
+    return null;
+}
+function classifyByExtension(ctx) {
     if (ctx.fileName === "SKILL.md" && !ctx.ancestorDirs.includes("wikis")) {
-        return { type: "skill", specificity: 25, renderer: "skill-md" };
+        return { type: "skill", specificity: 25 };
     }
-    // Known script extensions (excluding .md, handled by smartMdMatcher)
     if (SCRIPT_EXTENSIONS.has(ctx.ext)) {
-        return { type: "script", specificity: 3, renderer: "script-source" };
+        return { type: "script", specificity: 3 };
     }
     return null;
 }
-// ── directoryMatcher (specificity: 10) ──────────────────────────────────────
-/**
- * Directory-based matcher that boosts specificity when an ancestor
- * directory segment from the stash root matches a known type name.
- *
- * The first matching type-like ancestor wins. This preserves intuitive
- * behavior for nested stash layouts such as `agent-stash/agents/blog/foo.md`
- * while still honoring earlier type roots like `commands/agents/foo.md`.
- */
-export function directoryMatcher(ctx) {
-    const ext = ctx.ext;
+function classifyByDirectory(ctx) {
     for (const dir of ctx.ancestorDirs) {
-        if (dir === "scripts" && SCRIPT_EXTENSIONS.has(ext)) {
-            return { type: "script", specificity: 10, renderer: "script-source" };
-        }
-        if (dir === "skills" && ctx.fileName === "SKILL.md") {
-            return { type: "skill", specificity: 10, renderer: "skill-md" };
-        }
-        if (dir === "commands" && ext === ".md") {
-            return { type: "command", specificity: 10, renderer: "command-md" };
-        }
-        if (dir === "agents" && ext === ".md") {
-            return { type: "agent", specificity: 10, renderer: "agent-md" };
-        }
-        if (dir === "knowledge" && ext === ".md") {
-            return { type: "knowledge", specificity: 10, renderer: "knowledge-md" };
-        }
-        if (dir === "workflows" && ext === ".md") {
-            return { type: "workflow", specificity: 10, renderer: "workflow-md" };
-        }
-        if (dir === "memories" && ext === ".md") {
-            return { type: "memory", specificity: 10, renderer: "memory-md" };
-        }
-        if (dir === "vaults" && (ctx.fileName === ".env" || ctx.fileName.endsWith(".env"))) {
-            return { type: "vault", specificity: 10, renderer: "vault-env" };
-        }
+        const result = matchDirectoryHint(dir, ctx, 10);
+        if (result)
+            return result;
     }
     return null;
 }
-// ── parentDirHintMatcher (specificity: 15) ──────────────────────────────────
-/**
- * Uses the immediate parent directory name as a hint. More specific than
- * the ancestor-based directory matcher because the file might be nested
- * several levels deep, yet its immediate parent can still carry strong
- * naming conventions (e.g. `my-project/agents/planning.md`).
- */
-export function parentDirHintMatcher(ctx) {
+function classifyByParentDirHint(ctx) {
     const { parentDir, ext, fileName } = ctx;
-    if (parentDir === "scripts" && SCRIPT_EXTENSIONS.has(ext)) {
-        return { type: "script", specificity: 15, renderer: "script-source" };
-    }
     if (parentDir === "skills" && (fileName === "SKILL.md" || ext === ".md")) {
-        return { type: "skill", specificity: 15, renderer: "skill-md" };
-    }
-    if (parentDir === "agents" && ext === ".md") {
-        return { type: "agent", specificity: 15, renderer: "agent-md" };
-    }
-    if (parentDir === "commands" && ext === ".md") {
-        return { type: "command", specificity: 15, renderer: "command-md" };
-    }
-    if (parentDir === "knowledge" && ext === ".md") {
-        return { type: "knowledge", specificity: 15, renderer: "knowledge-md" };
-    }
-    if (parentDir === "workflows" && ext === ".md") {
-        return { type: "workflow", specificity: 15, renderer: "workflow-md" };
+        return { type: "skill", specificity: 15 };
     }
-    if (parentDir === "memories" && ext === ".md") {
-        return { type: "memory", specificity: 15, renderer: "memory-md" };
-    }
-    if (parentDir === "vaults" && (fileName === ".env" || fileName.endsWith(".env"))) {
-        return { type: "vault", specificity: 15, renderer: "vault-env" };
-    }
-    return null;
+    return matchDirectoryHint(parentDir, ctx, 15);
 }
-// ── smartMdMatcher (specificity: 20 / 18 / 8 / 5) ──────────────────────────
-/** Pattern that matches OpenCode command placeholders in markdown body. */
-const COMMAND_PLACEHOLDER_RE = /\$ARGUMENTS|\$[123]\b/;
-/**
- * Content-based matcher for `.md` files. Inspects frontmatter keys and body
- * content to classify markdown as agent, command, or knowledge.
- *
- * Specificity levels:
- *   20 -- agent-exclusive signals (`tools`, `toolPolicy`)
- *   18 -- command content signals (`agent` frontmatter, `$ARGUMENTS`/`$1`-`$3`)
- *    8 -- weak agent signal (`model` alone)
- *    5 -- knowledge fallback (any unclassified `.md`)
- *
- * Command signals at 18 override directory hints (10/15) because the content
- * unambiguously identifies a command template. Agent-exclusive signals at 20
- * still win over command signals when both are present.
- */
-export function smartMdMatcher(ctx) {
+function classifyBySmartMd(ctx) {
     if (ctx.ext !== ".md")
         return null;
     const body = ctx.content();
     if (looksLikeWorkflow(body)) {
-        return { type: "workflow", specificity: 19, renderer: "workflow-md" };
+        return { type: "workflow", specificity: 19 };
     }
     const fm = ctx.frontmatter();
     if (fm) {
-        // Agent-exclusive indicators: toolPolicy or tools
-        // These return high specificity (20) to override everything else.
         if ("toolPolicy" in fm || "tools" in fm) {
-            return { type: "agent", specificity: 20, renderer: "agent-md" };
+            return { type: "agent", specificity: 20 };
         }
-        // Command signal: `agent` frontmatter key names a dispatch target.
-        // This is an OpenCode convention specific to commands.
         if ("agent" in fm) {
-            return { type: "command", specificity: 18, renderer: "command-md" };
+            return { type: "command", specificity: 18 };
         }
     }
-    // Command signal: body contains $ARGUMENTS or $1/$2/$3 placeholders.
-    // These are definitively command template patterns (OpenCode convention).
     if (COMMAND_PLACEHOLDER_RE.test(body)) {
-        return { type: "command", specificity: 18, renderer: "command-md" };
+        return { type: "command", specificity: 18 };
     }
-    if (fm) {
-        // model alone is a weaker agent signal (specificity 8) -- it can appear
-        // on commands too (OpenCode convention). Directory hints (10/15) win
-        // when the file lives in commands/, but model still classifies an .md
-        // as agent when no directory hint is present.
-        if ("model" in fm) {
-            return { type: "agent", specificity: 8, renderer: "agent-md" };
-        }
+    if (fm && "model" in fm) {
+        return { type: "agent", specificity: 8 };
     }
-    // Weak fallback: any .md file is assumed to be knowledge
-    return { type: "knowledge", specificity: 5, renderer: "knowledge-md" };
+    return { type: "knowledge", specificity: 5 };
 }
-// ── wikiMatcher (specificity: 20) ──────────────────────────────────────────
-/**
- * Classify any `.md` file that lives under `wikis/<name>/…` as `wiki`.
- *
- * Registered AFTER `smartMdMatcher` so the registered-later-wins tiebreaker
- * puts wiki ahead of agent at specificity 20. That means a wiki page with
- * agent-style frontmatter (e.g. `tools:`) still classifies as a wiki page,
- * not an agent. That's intentional — the directory is the authoritative
- * signal: files under `wikis/` are wiki content.
- *
- * Requires at least one path segment after `wikis/` (the wiki name) — a
- * stray `.md` at the bare `wikis/` root is not a wiki page.
- */
-export function wikiMatcher(ctx) {
+function classifyByWiki(ctx) {
     if (ctx.ext !== ".md")
         return null;
     const idx = ctx.ancestorDirs.indexOf("wikis");
@@ -195,10 +130,43 @@ export function wikiMatcher(ctx) {
         return null;
     if (idx + 1 >= ctx.ancestorDirs.length)
         return null;
-    return { type: "wiki", specificity: 20, renderer: "wiki-md" };
+    return { type: "wiki", specificity: 20 };
+}
+// ---------------------------------------------------------------------------
+// Adapter: MatchFact → MatchResult
+// ---------------------------------------------------------------------------
+function toMatchResult(ctx, classify) {
+    const fact = classify(ctx);
+    if (!fact)
+        return null;
+    const renderer = defaultRendererRegistry.rendererNameFor(fact.type);
+    if (!renderer)
+        return null;
+    return {
+        type: fact.type,
+        specificity: fact.specificity,
+        renderer,
+        ...(fact.meta ? { meta: fact.meta } : {}),
+    };
+}
+// ---------------------------------------------------------------------------
+// Public matchers (API unchanged)
+// ---------------------------------------------------------------------------
+export function extensionMatcher(ctx) {
+    return toMatchResult(ctx, classifyByExtension);
+}
+export function directoryMatcher(ctx) {
+    return toMatchResult(ctx, classifyByDirectory);
+}
+export function parentDirHintMatcher(ctx) {
+    return toMatchResult(ctx, classifyByParentDirHint);
+}
+export function smartMdMatcher(ctx) {
+    return toMatchResult(ctx, classifyBySmartMd);
+}
+export function wikiMatcher(ctx) {
+    return toMatchResult(ctx, classifyByWiki);
 }
-// ── Registration ────────────────────────────────────────────────────────────
-/** All built-in matchers in registration order (later wins ties). */
 const builtinMatchers = [
     extensionMatcher,
     directoryMatcher,
@@ -206,10 +174,6 @@ const builtinMatchers = [
     smartMdMatcher,
     wikiMatcher,
 ];
-/**
- * Register all built-in matchers with the file-context registry.
- * Called once from the CLI entry point (or ensureBuiltinsRegistered).
- */
 export function registerBuiltinMatchers() {
     for (const matcher of builtinMatchers) {
         registerMatcher(matcher);

package/dist/indexer/memory-inference.js

@@ -33,11 +33,14 @@ import fs from "node:fs";
 import path from "node:path";
 import { stringify as yamlStringify } from "yaml";
 import { parseAssetRef } from "../core/asset-ref";
+import { concurrentMap } from "../core/concurrent";
 import { parseFrontmatter, parseFrontmatterBlock } from "../core/frontmatter";
 import { warn } from "../core/warn";
 import { writeAssetToSource } from "../core/write-source";
 import { resolveIndexPassLLM } from "../llm/index-passes";
-import { compressMemoryToDerivedMemory } from "../llm/memory-infer";
+import * as memoryInfer from "../llm/memory-infer";
+import { withLlmCache } from "./llm-cache";
+import { walkMarkdownFiles } from "./walker";
 /**
  * Frontmatter keys this pass cares about. Constants so a future rename only
  * needs to touch one site.
@@ -60,7 +63,7 @@ const FM_SOURCE = "source";
  * Both must allow the call for the pass to run. Either set to `false`
  * short-circuits to a no-op result.
  */
-export async function runMemoryInferencePass(config, sources, signal) {
+export async function runMemoryInferencePass(config, sources, signal, db, reEnrich, onProgress, options = {}) {
     const result = {
         considered: 0,
         splitParents: 0,
@@ -82,26 +85,75 @@ export async function runMemoryInferencePass(config, sources, signal) {
     const primary = sources[0];
     if (!primary)
         return result;
-    const pending = collectPendingMemories(primary.path);
+    const pending = collectPendingMemories(primary.path).filter((record) => !options.candidateRefs || options.candidateRefs.has(record.ref));
     result.considered = pending.length;
     if (pending.length === 0)
         return result;
-    for (const record of pending) {
+    let processed = 0;
+    const total = pending.length;
+    onProgress?.({ processed, total, writtenFacts: 0, skippedNoFacts: 0 });
+    const perRecordResults = await concurrentMap(pending, async (record) => {
         if (signal?.aborted)
-            return result;
-        const derived = await compressMemoryToDerivedMemory(llmConfig, record.body, signal);
+            return undefined;
+        // Incremental cache: skip LLM call when body hash is unchanged and
+        // --re-enrich was not requested. The cache ref is the absolute file path.
+        const validate = (raw) => {
+            if (!raw || typeof raw !== "object")
+                return undefined;
+            const parsed = raw;
+            const title = typeof parsed.title === "string" ? parsed.title : "";
+            const description = typeof parsed.description === "string" ? parsed.description : "";
+            const content = typeof parsed.content === "string" ? parsed.content : "";
+            const tags = Array.isArray(parsed.tags) ? parsed.tags.filter((t) => typeof t === "string") : [];
+            const searchHints = Array.isArray(parsed.searchHints)
+                ? parsed.searchHints.filter((h) => typeof h === "string")
+                : [];
+            if (title && description && content && tags.length > 0 && searchHints.length > 0) {
+                return { title, description, tags, searchHints, content };
+            }
+            return undefined;
+        };
+        const derived = db
+            ? await withLlmCache(db, record.filePath, record.body, reEnrich ?? false, () => memoryInfer.compressMemoryToDerivedMemory(llmConfig, record.body, signal, config, (evt) => {
+                warn(`[akm] LLM fallback for ${evt.feature}: ${evt.reason}`);
+            }), validate)
+            : await memoryInfer.compressMemoryToDerivedMemory(llmConfig, record.body, signal, config, (evt) => {
+                warn(`[akm] LLM fallback for ${evt.feature}: ${evt.reason}`);
+            });
         if (!derived) {
-            result.skippedNoFacts += 1;
-            // Intentionally NOT marked processed — a transient LLM failure should
-            // be retried on the next index run.
-            continue;
+            return { skipped: true };
         }
         const written = await writeDerivedMemory(record, derived);
         if (written > 0) {
             markParentProcessed(record);
+            return { skipped: false, splitParent: true, written };
+        }
+        return { skipped: false, splitParent: false, written: 0 };
+    }, 
+    // Default concurrency of 4 for cloud APIs. Set `llm.concurrency: 1`
+    // in config.json for local model servers (LM Studio, Ollama).
+    llmConfig.concurrency ?? 1);
+    for (let i = 0; i < perRecordResults.length; i++) {
+        const res = perRecordResults[i];
+        if (!res)
+            continue;
+        if (res.skipped) {
+            result.skippedNoFacts += 1;
+            // Intentionally NOT marked processed — a transient LLM failure should
+            // be retried on the next index run.
+        }
+        else if (res.splitParent) {
             result.splitParents += 1;
-            result.writtenFacts += written;
+            result.writtenFacts += res.written;
         }
+        processed++;
+        onProgress?.({
+            processed,
+            total,
+            writtenFacts: result.writtenFacts,
+            skippedNoFacts: result.skippedNoFacts,
+            currentRef: pending[i]?.ref,
+        });
     }
     return result;
 }
@@ -155,24 +207,6 @@ export function isPendingMemory(frontmatter) {
         return false;
     return true;
 }
-function* walkMarkdownFiles(root) {
-    let entries;
-    try {
-        entries = fs.readdirSync(root, { withFileTypes: true });
-    }
-    catch {
-        return;
-    }
-    for (const entry of entries) {
-        const full = path.join(root, entry.name);
-        if (entry.isDirectory()) {
-            yield* walkMarkdownFiles(full);
-        }
-        else if (entry.isFile() && entry.name.toLowerCase().endsWith(".md")) {
-            yield full;
-        }
-    }
-}
 function toMemoryName(memoriesDir, filePath) {
     const rel = path.relative(memoriesDir, filePath);
     if (!rel || rel.startsWith(".."))

package/dist/indexer/metadata-contributors.js

@@ -0,0 +1,26 @@
+const contributors = [];
+let builtinsPromise;
+async function ensureBuiltinMetadataContributorsRegistered() {
+    if (!builtinsPromise) {
+        builtinsPromise = (async () => {
+            await import("../output/renderers.js");
+            await import("../workflows/renderer.js");
+        })();
+    }
+    return builtinsPromise;
+}
+export function registerMetadataContributor(contributor) {
+    contributors.push(contributor);
+}
+export async function getMetadataContributors() {
+    await ensureBuiltinMetadataContributorsRegistered();
+    return [...contributors];
+}
+export async function applyMetadataContributors(entry, ctx) {
+    const activeContributors = await getMetadataContributors();
+    for (const contributor of activeContributors) {
+        if (!contributor.appliesTo(ctx))
+            continue;
+        contributor.contribute(entry, ctx);
+    }
+}