akm-cli 0.7.4 → 0.8.0-rc.10

package/dist/indexer/memory-inference.js

@@ -1,43 +1,19 @@
-/**
- * Memory inference pass for `akm index` (#201).
- *
- * Detects memories pending inference, asks the configured LLM to compress each
- * into one higher-signal derived memory, and writes the result back as a new
- * memory file with frontmatter `inferred: true` + a `source:` backref to the
- * parent memory.
- *
- * Pending predicate (see {@link isPendingMemory}):
- *   - File lives under `<stashRoot>/memories/` and ends in `.md`.
- *   - Frontmatter does NOT have `inferenceProcessed: true` (parent already split).
- *   - Frontmatter does NOT have `inferred: true` (this is itself a child fact).
- *
- * Idempotency: after a successful split the parent's frontmatter is rewritten
- * with `inferenceProcessed: true`. A subsequent `akm index` therefore skips
- * the parent without re-running the LLM.
- *
- * Disabling — two orthogonal gates per v1 spec §14:
- *   1. `llm.features.memory_inference = false` blocks the pass at the
- *      locked feature-flag layer (no network call may ever issue).
- *   2. `index.memory.llm = false` (or no `akm.llm` block at all) opts the
- *      pass out at the per-pass layer (#208).
- *   A pass runs iff both layers allow it. Existing inferred children are
- *   NEVER deleted — the user keeps what was already produced.
- *
- * Locked v1 contract:
- *   - LLM access is exclusively via `resolveIndexPassLLM("memory", config)`.
- *   - All child memory writes go through `writeAssetToSource` in
- *     `src/core/write-source.ts`. The parent's frontmatter rewrite is an
- *     explicit narrow exception — see {@link markParentProcessed}.
- */
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
 import fs from "node:fs";
 import path from "node:path";
-import { stringify as yamlStringify } from "yaml";
 import { parseAssetRef } from "../core/asset-ref";
+import { assembleAsset } from "../core/asset-serialize";
+import { concurrentMap } from "../core/concurrent";
 import { parseFrontmatter, parseFrontmatterBlock } from "../core/frontmatter";
 import { warn } from "../core/warn";
 import { writeAssetToSource } from "../core/write-source";
+import { isProcessEnabled } from "../llm/feature-gate";
 import { resolveIndexPassLLM } from "../llm/index-passes";
-import { compressMemoryToDerivedMemory } from "../llm/memory-infer";
+import * as memoryInfer from "../llm/memory-infer";
+import { withLlmCache } from "./llm-cache";
+import { walkMarkdownFiles } from "./walker";
 /**
  * Frontmatter keys this pass cares about. Constants so a future rename only
  * needs to touch one site.
@@ -45,14 +21,15 @@ import { compressMemoryToDerivedMemory } from "../llm/memory-infer";
 const FM_INFERRED = "inferred";
 const FM_INFERENCE_PROCESSED = "inferenceProcessed";
 const FM_SOURCE = "source";
+const FM_CAPTURE_MODE = "captureMode";
 /**
  * Top-level entry point. Returns a no-op result when the pass is disabled.
  *
- * Two orthogonal gates per v1 spec §14:
+ * Two orthogonal gates:
  *
- *   1. **Feature gate** — `llm.features.memory_inference` (defaults to
- *      `true`). When `false`, no network call may issue regardless of
- *      per-pass settings. This is the locked spec-§14 gate.
+ *   1. **Feature gate** — `profiles.improve.default.processes.memoryInference.enabled`
+ *      (defaults to `true`). When `false`, no network call may issue regardless
+ *      of per-pass settings.
  *   2. **Per-pass gate** — `resolveIndexPassLLM("memory", config)` (which
  *      reads `index.memory.llm`). When `false`, the indexer simply skips
  *      this pass for the current run.
@@ -60,16 +37,21 @@ const FM_SOURCE = "source";
  * Both must allow the call for the pass to run. Either set to `false`
  * short-circuits to a no-op result.
  */
-export async function runMemoryInferencePass(config, sources, signal) {
+export async function runMemoryInferencePass(config, sources, signal, db, reEnrich, onProgress, options = {}) {
     const result = {
         considered: 0,
+        cacheHits: 0,
         splitParents: 0,
         writtenFacts: 0,
         skippedNoFacts: 0,
+        skippedChildExists: 0,
+        skippedAborted: 0,
+        unaccounted: 0,
     };
-    // Gate 1 — locked feature flag (§14). Defaults to enabled; only an
-    // explicit `false` disables the pass entirely.
-    if (config.llm?.features?.memory_inference === false)
+    // Gate 1 — feature gate via isProcessEnabled, which reads the 0.8.0 path
+    // (profiles.improve.default.processes.memoryInference.enabled). Defaults to
+    // enabled when the key is absent.
+    if (!isProcessEnabled("index", "memory_inference", config))
         return result;
     // Gate 2 — per-pass opt-out (#208). Returns the resolved llm config or
     // `undefined` when the pass should not run.
@@ -82,26 +64,124 @@ export async function runMemoryInferencePass(config, sources, signal) {
     const primary = sources[0];
     if (!primary)
         return result;
-    const pending = collectPendingMemories(primary.path);
+    const pending = collectPendingMemories(primary.path).filter((record) => !options.candidateRefs || options.candidateRefs.has(record.ref));
     result.considered = pending.length;
     if (pending.length === 0)
         return result;
-    for (const record of pending) {
+    let processed = 0;
+    const total = pending.length;
+    onProgress?.({ processed, total, writtenFacts: 0, skippedNoFacts: 0 });
+    const perRecordResults = await concurrentMap(pending, async (record) => {
+        // Aborted BEFORE a fresh LLM call. Returned as a typed outcome so the
+        // for-loop below increments `skippedAborted` instead of silently
+        // dropping the record (which historically inflated freshAttempts and
+        // dragged the health-reported yield rate down — see investigation
+        // 2026-05-26).
         if (signal?.aborted)
-            return result;
-        const derived = await compressMemoryToDerivedMemory(llmConfig, record.body, signal);
+            return { aborted: true };
+        // Incremental cache: skip LLM call when body hash is unchanged and
+        // --re-enrich was not requested. The cache ref is the absolute file path.
+        const validate = (raw) => {
+            if (!raw || typeof raw !== "object")
+                return undefined;
+            const parsed = raw;
+            const title = typeof parsed.title === "string" ? parsed.title : "";
+            const description = typeof parsed.description === "string" ? parsed.description : "";
+            const content = typeof parsed.content === "string" ? parsed.content : "";
+            const tags = Array.isArray(parsed.tags) ? parsed.tags.filter((t) => typeof t === "string") : [];
+            const searchHints = Array.isArray(parsed.searchHints)
+                ? parsed.searchHints.filter((h) => typeof h === "string")
+                : [];
+            if (title && description && content && tags.length > 0 && searchHints.length > 0) {
+                return { title, description, tags, searchHints, content };
+            }
+            return undefined;
+        };
+        // Track whether THIS candidate's result came from the body-hash
+        // cache vs. a fresh LLM call. The cache short-circuits when the
+        // parent body has not changed since a prior derived write — surfacing
+        // the hit count separately so the operational yield rate
+        // (writtenFacts / freshAttempts) is interpretable as the cache warms.
+        let fromCache = false;
+        const derived = db
+            ? await withLlmCache(db, record.filePath, record.body, reEnrich ?? false, () => memoryInfer.compressMemoryToDerivedMemory(llmConfig, record.body, signal, config, (evt) => {
+                warn(`[akm] LLM fallback for ${evt.feature}: ${evt.reason}`);
+            }), validate, undefined, "", {
+                onCacheHit: () => {
+                    fromCache = true;
+                },
+            })
+            : await memoryInfer.compressMemoryToDerivedMemory(llmConfig, record.body, signal, config, (evt) => {
+                warn(`[akm] LLM fallback for ${evt.feature}: ${evt.reason}`);
+            });
         if (!derived) {
-            result.skippedNoFacts += 1;
-            // Intentionally NOT marked processed — a transient LLM failure should
-            // be retried on the next index run.
-            continue;
+            return { skipped: true, fromCache };
         }
         const written = await writeDerivedMemory(record, derived);
         if (written > 0) {
             markParentProcessed(record);
+            return { skipped: false, splitParent: true, written, fromCache };
+        }
+        // LLM produced a valid derived draft but no file was written — either
+        // because `<parent>.derived.md` already exists on disk or
+        // `writeAssetToSource` threw. Categorise as `childExists` so the
+        // attempt is accounted for in health metrics rather than vanishing
+        // into the freshAttempts denominator.
+        return { skipped: false, splitParent: false, written: 0, fromCache, childExists: true };
+    }, 
+    // Default concurrency of 4 for cloud APIs. Set `llm.concurrency: 1`
+    // in config.json for local model servers (LM Studio, Ollama).
+    llmConfig.concurrency ?? 1);
+    for (let i = 0; i < perRecordResults.length; i++) {
+        const res = perRecordResults[i];
+        if (!res)
+            continue;
+        if ("aborted" in res && res.aborted) {
+            result.skippedAborted += 1;
+            processed++;
+            onProgress?.({
+                processed,
+                total,
+                writtenFacts: result.writtenFacts,
+                skippedNoFacts: result.skippedNoFacts,
+                currentRef: pending[i]?.ref,
+            });
+            continue;
+        }
+        if (res.fromCache) {
+            result.cacheHits += 1;
+        }
+        if (res.skipped) {
+            result.skippedNoFacts += 1;
+            // Intentionally NOT marked processed — a transient LLM failure should
+            // be retried on the next index run.
+        }
+        else if (res.splitParent) {
             result.splitParents += 1;
-            result.writtenFacts += written;
+            result.writtenFacts += res.written;
+        }
+        else if ("childExists" in res && res.childExists) {
+            // LLM call was consumed but the derived file already existed (or the
+            // write threw). Track separately so this category is observable in
+            // health output and stops bleeding into the freshAttempts denominator.
+            result.skippedChildExists += 1;
+            warn(`memory inference: derived child for ${pending[i]?.ref ?? "<unknown>"} already existed or write failed; counted as skippedChildExists`);
+        }
+        else {
+            // The per-record state machine should cover every outcome. A hit here
+            // means a new code path slipped past the categorisation — surface it
+            // loudly so health metrics stay honest and we get a signal to fix.
+            result.unaccounted += 1;
+            warn(`memory inference: unaccounted per-record outcome for ${pending[i]?.ref ?? "<unknown>"}`);
         }
+        processed++;
+        onProgress?.({
+            processed,
+            total,
+            writtenFacts: result.writtenFacts,
+            skippedNoFacts: result.skippedNoFacts,
+            currentRef: pending[i]?.ref,
+        });
     }
     return result;
 }
@@ -125,7 +205,7 @@ export function collectPendingMemories(stashRoot) {
             continue;
         }
         const parsed = parseFrontmatter(raw);
-        if (!isPendingMemory(parsed.data))
+        if (!isPendingMemory(parsed.data, filePath))
             continue;
         const relName = toMemoryName(memoriesDir, filePath);
         if (!relName)
@@ -145,34 +225,33 @@ export function collectPendingMemories(stashRoot) {
  * Predicate: true when the parsed frontmatter indicates the memory has not
  * yet been split AND is not itself an inferred child.
  *
+ * Also guards against `.derived` files whose `inferred:` frontmatter key has
+ * been dropped by a manual edit or schema-repair rewrite. The file name suffix
+ * is structural and immutable; frontmatter flags are mutable. A file whose
+ * path contains `.derived` is always treated as a derived child regardless of
+ * its frontmatter state — this prevents `<name>.derived.derived.md` chains.
+ *
+ * @param frontmatter - Parsed YAML frontmatter from the memory file.
+ * @param filePath    - Optional absolute path to the memory file. When
+ *                      supplied, the name-based guard is applied.
+ *
  * Exported for direct unit testing — keeping the predicate in one place
  * avoids drift between the walker, tests, and any future consumers.
  */
-export function isPendingMemory(frontmatter) {
+export function isPendingMemory(frontmatter, filePath) {
+    // Name-based guard: a `.derived` suffix in the path means this file is a
+    // derived child regardless of what its frontmatter currently says.
+    if (filePath !== undefined) {
+        const base = path.basename(filePath, ".md");
+        if (base.endsWith(".derived"))
+            return false;
+    }
     if (frontmatter[FM_INFERRED] === true)
         return false;
     if (frontmatter[FM_INFERENCE_PROCESSED] === true)
         return false;
     return true;
 }
-function* walkMarkdownFiles(root) {
-    let entries;
-    try {
-        entries = fs.readdirSync(root, { withFileTypes: true });
-    }
-    catch {
-        return;
-    }
-    for (const entry of entries) {
-        const full = path.join(root, entry.name);
-        if (entry.isDirectory()) {
-            yield* walkMarkdownFiles(full);
-        }
-        else if (entry.isFile() && entry.name.toLowerCase().endsWith(".md")) {
-            yield full;
-        }
-    }
-}
 function toMemoryName(memoriesDir, filePath) {
     const rel = path.relative(memoriesDir, filePath);
     if (!rel || rel.startsWith(".."))
@@ -214,6 +293,7 @@ async function writeDerivedMemory(parent, derived) {
 function renderDerivedMemory(parent, derived) {
     const fm = {
         [FM_INFERRED]: true,
+        [FM_CAPTURE_MODE]: "background",
         [FM_SOURCE]: parent.ref,
         description: derived.description,
         tags: derived.tags,
@@ -221,8 +301,7 @@ function renderDerivedMemory(parent, derived) {
         title: derived.title,
         derivedFrom: parent.name,
     };
-    const yaml = yamlStringify(fm).trimEnd();
-    return `---\n${yaml}\n---\n\n# ${derived.title.trim()}\n\n${derived.content.trim()}\n`;
+    return assembleAsset(fm, `# ${derived.title.trim()}\n\n${derived.content.trim()}\n`);
 }
 function markParentProcessed(parent) {
     // Frontmatter-only rewrite of an existing asset: not a new asset write,
@@ -239,10 +318,9 @@ function markParentProcessed(parent) {
         return;
     }
     const updatedFm = { ...parent.data, [FM_INFERENCE_PROCESSED]: true };
-    const yaml = yamlStringify(updatedFm).trimEnd();
     const block = parseFrontmatterBlock(raw);
     const body = block?.content ?? raw;
-    const next = `---\n${yaml}\n---\n${body.startsWith("\n") ? "" : "\n"}${body}`;
+    const next = assembleAsset(updatedFm, body);
     try {
         fs.writeFileSync(parent.filePath, next, "utf8");
     }

package/dist/indexer/metadata-contributors.js

@@ -0,0 +1,29 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+const contributors = [];
+let builtinsPromise;
+async function ensureBuiltinMetadataContributorsRegistered() {
+    if (!builtinsPromise) {
+        builtinsPromise = (async () => {
+            await import("../output/renderers.js");
+            await import("../workflows/renderer.js");
+        })();
+    }
+    return builtinsPromise;
+}
+export function registerMetadataContributor(contributor) {
+    contributors.push(contributor);
+}
+export async function getMetadataContributors() {
+    await ensureBuiltinMetadataContributorsRegistered();
+    return [...contributors];
+}
+export async function applyMetadataContributors(entry, ctx) {
+    const activeContributors = await getMetadataContributors();
+    for (const contributor of activeContributors) {
+        if (!contributor.appliesTo(ctx))
+            continue;
+        contributor.contribute(entry, ctx);
+    }
+}