akm-cli 0.9.0-beta.6 → 0.9.0-rc.0

package/dist/commands/improve/dedup.js

@@ -0,0 +1,482 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+// ── Deterministic near-duplicate MEMORY dedup (#617) ─────────────────────────
+//
+// A CHEAP, DETERMINISTIC fast path that collapses the obvious duplicates with
+// NO LLM call, run in front of the embedding-clustered LLM consolidation pass
+// (see consolidate.ts). The working stash accumulates near-duplicates faster
+// than the capped/rotating LLM consolidate pass can merge them; this pre-pass
+// clears the unambiguous twins so the (expensive) LLM only ever sees genuinely
+// distinct-but-related memories.
+//
+// Two collapse classes, both safe and reversible (archive + backup at the call
+// site in consolidate.ts; this module only computes the plan and applies the
+// file writes/deletes):
+//
+//   1. `.derived` ↔ origin pairs — a memory-inference `<parent>.derived` child
+//      whose normalized body is identical to (or, with embeddings, ≥ the strict
+//      cosine threshold of) its origin. Keep the canonical (non-derived) origin;
+//      drop the derived variant; preserve the variant's provenance on the
+//      canonical (`dedupedFrom`).
+//
+//   2. Content twins — two non-derived memories with identical normalized body
+//      hash, or ≥ the strict cosine threshold. Keep the deterministic canonical
+//      (lexicographically smallest name); merge the other into it, preserving
+//      provenance.
+//
+// Determinism: the plan is a pure function of the on-disk memory set + config.
+// Nothing here reads Date.now()/Math.random(); ordering is by memory name so the
+// canonical choice and op order are stable across runs.
+//
+// Gating: DEFAULT OFF. The pre-pass only runs when `dedup.enabled === true`. The
+// cosine path additionally requires an embedding config; absent embeddings the
+// pass still collapses exact normalized-hash twins. `cosineThreshold` defaults
+// to 0.97 — a strict floor chosen so distinct-but-related memories fall through
+// untouched to the LLM consolidation.
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import path from "node:path";
+import { parseAssetRef } from "../../core/asset/asset-ref.js";
+import { assembleAssetFromString, serializeFrontmatter } from "../../core/asset/asset-serialize.js";
+import { parseFrontmatter } from "../../core/asset/frontmatter.js";
+import { warn } from "../../core/warn.js";
+import { cosineSimilarity, embedBatch, resolveEmbeddingModelId } from "../../llm/embedder.js";
+import { getBodyEmbeddings, upsertBodyEmbeddings } from "../../storage/repositories/embeddings-repository.js";
+/** Default strict cosine floor — high enough to skip distinct-but-related memories. */
+export const DEFAULT_DEDUP_COSINE_THRESHOLD = 0.97;
+/**
+ * Strip frontmatter from raw memory content, returning the body text trimmed.
+ * Case and whitespace are preserved — this is the shared primitive used by
+ * both hash wrappers below. Falls back to `raw.trim()` on unparseable
+ * frontmatter (consistent with the pre-existing load-time hot guard).
+ */
+export function stripFrontmatterBody(raw) {
+    try {
+        return parseFrontmatter(raw).content.trim();
+    }
+    catch {
+        return raw.trim();
+    }
+}
+/**
+ * Normalize a memory body for content-twin equality. Strips frontmatter,
+ * lowercases, trims, and collapses all runs of whitespace to a single space so
+ * trivial reformatting (extra blank lines, trailing spaces, case) does not
+ * defeat the hash. Deterministic and pure.
+ *
+ * Use this for the DEDUP path only (exact-twin detection). For the change-
+ * detection / embedding-cache path use `cacheHash` instead.
+ */
+export function normalizeMemoryBody(raw) {
+    return stripFrontmatterBody(raw).toLowerCase().replace(/\s+/g, " ");
+}
+/**
+ * Hash used for content-twin detection: lowercase + whitespace-collapsed body.
+ * Two memories that differ only in case or whitespace produce the same hash
+ * and are considered identical twins. Use this key for the dedup buckets.
+ */
+export function dedupHash(raw) {
+    return createHash("sha256").update(normalizeMemoryBody(raw), "utf8").digest("hex");
+}
+/**
+ * Hash used for change-detection and the body-embedding cache: case-/whitespace-
+ * preserving stripped body. Two memories with the same wording but different
+ * casing produce DIFFERENT hashes here, which is intentional — we embed the
+ * exact text and cache by its precise content.
+ *
+ * This is the `content_hash` stored in `body_embeddings` and
+ * `consolidation_judged`. Do NOT reuse the `dedupHash` for those tables.
+ */
+export function cacheHash(raw) {
+    return createHash("sha256").update(stripFrontmatterBody(raw), "utf8").digest("hex");
+}
+/**
+ * Load every memory `.md` file (including `.derived` children) from the stash
+ * memories directory. Unlike the consolidate loader, this DOES include derived
+ * children — the whole point of class 1 is to collapse a derived child into its
+ * origin, so both must be visible here.
+ */
+export function loadDedupMemories(stashDir) {
+    const memoriesDir = path.join(stashDir, "memories");
+    if (!fs.existsSync(memoriesDir))
+        return [];
+    const out = [];
+    for (const fname of fs.readdirSync(memoriesDir).sort()) {
+        if (!fname.endsWith(".md"))
+            continue;
+        const name = fname.replace(/\.md$/, "");
+        const filePath = path.join(memoriesDir, fname);
+        let raw;
+        try {
+            raw = fs.readFileSync(filePath, "utf8");
+        }
+        catch {
+            continue;
+        }
+        let fm = {};
+        try {
+            fm = parseFrontmatter(raw).data ?? {};
+        }
+        catch {
+            fm = {};
+        }
+        const normalizedBody = normalizeMemoryBody(raw);
+        out.push({
+            name,
+            filePath,
+            derived: name.endsWith(".derived"),
+            derivedFrom: typeof fm.derivedFrom === "string" ? fm.derivedFrom : undefined,
+            raw,
+            normalizedBody,
+            bodyHash: dedupHash(raw),
+            hot: fm.captureMode === "hot",
+        });
+    }
+    return out;
+}
+/** Default cap on the O(n²) cosine-compare pool size. */
+export const DEFAULT_COSINE_CANDIDATE_LIMIT = 500;
+/**
+ * Build the deterministic collapse plan. Pure over (memories, similarities,
+ * threshold) — `embeddings` is optional; when absent only exact normalized-hash
+ * twins are matched.
+ *
+ * Pass invariants:
+ *   - A memory is consumed (collapsed) at most once.
+ *   - Hot (captureMode: hot) memories are never collapsed, as canonical OR
+ *     variant — user-explicit, only the user retires them.
+ *   - Class 1 (`.derived` ↔ origin) is matched first so a derived child is
+ *     always folded into its origin (never the reverse, never twin-matched).
+ */
+export function planDedup(memories, opts) {
+    const collapses = [];
+    const warnings = [];
+    const consumed = new Set();
+    const threshold = opts.cosineThreshold;
+    const byName = new Map();
+    for (const m of memories)
+        byName.set(m.name, m);
+    const sim = (a, b) => {
+        if (!opts.embeddings)
+            return undefined;
+        const ea = opts.embeddings.get(a.name);
+        const eb = opts.embeddings.get(b.name);
+        if (!ea || !eb)
+            return undefined;
+        return cosineSimilarity(ea, eb);
+    };
+    // ── Class 1: `.derived` children ↔ their origin ────────────────────────────
+    // Iterate derived children in sorted order (loadDedupMemories sorts), folding
+    // each into its origin when near-identical.
+    const derived = memories.filter((m) => m.derived).sort((a, b) => a.name.localeCompare(b.name));
+    for (const child of derived) {
+        if (consumed.has(child.name))
+            continue;
+        if (child.hot)
+            continue;
+        const originName = child.derivedFrom ?? child.name.replace(/\.derived$/, "");
+        const origin = byName.get(originName);
+        if (!origin || consumed.has(origin.name) || origin.derived || origin.hot)
+            continue;
+        let via;
+        let similarity;
+        if (child.bodyHash === origin.bodyHash) {
+            via = "derived-hash";
+        }
+        else {
+            const s = sim(child, origin);
+            if (s !== undefined && s >= threshold) {
+                via = "derived-cosine";
+                similarity = s;
+            }
+        }
+        if (!via)
+            continue;
+        collapses.push({ canonical: origin.name, variant: child.name, via, similarity });
+        consumed.add(child.name);
+    }
+    // ── Class 2: content twins among non-derived memories ──────────────────────
+    // Bucket by exact normalized-hash first (cheap, transitive, deterministic).
+    const remaining = memories
+        .filter((m) => !m.derived && !m.hot && !consumed.has(m.name))
+        .sort((a, b) => a.name.localeCompare(b.name));
+    const hashBuckets = new Map();
+    for (const m of remaining) {
+        const list = hashBuckets.get(m.bodyHash);
+        if (list)
+            list.push(m);
+        else
+            hashBuckets.set(m.bodyHash, [m]);
+    }
+    for (const bucket of hashBuckets.values()) {
+        if (bucket.length < 2)
+            continue;
+        // Canonical = lexicographically smallest name (already sorted).
+        const canonical = bucket[0];
+        for (let i = 1; i < bucket.length; i++) {
+            const variant = bucket[i];
+            if (consumed.has(variant.name))
+                continue;
+            collapses.push({ canonical: canonical.name, variant: variant.name, via: "twin-hash" });
+            consumed.add(variant.name);
+        }
+        consumed.add(canonical.name); // canonical kept but no longer a twin candidate
+    }
+    // Cosine twins (only when embeddings are available). O(n²) over the still-
+    // unconsumed non-derived pool; deterministic greedy: for each canonical in
+    // sorted order, claim every unconsumed later memory whose similarity ≥ floor.
+    // The pool is capped at `cosineCandidateLimit` (default 500) to bound the
+    // O(n²) cost (~0.1 s at 500; ~3 s at 2.6 k; ~85 s at 13 k). Exact-hash
+    // matches above always run over the full pool and are unaffected.
+    if (opts.embeddings) {
+        const limit = opts.cosineCandidateLimit ?? DEFAULT_COSINE_CANDIDATE_LIMIT;
+        const fullPool = remaining.filter((m) => !consumed.has(m.name));
+        const pool = fullPool.length > limit ? fullPool.slice(0, limit) : fullPool;
+        if (fullPool.length > limit) {
+            warnings.push(`dedup: cosine compare pool (${fullPool.length}) exceeds cosineCandidateLimit (${limit}); capping to first ${limit} memories (exact-hash matches unaffected).`);
+        }
+        for (let i = 0; i < pool.length; i++) {
+            const canonical = pool[i];
+            if (consumed.has(canonical.name))
+                continue;
+            for (let j = i + 1; j < pool.length; j++) {
+                const variant = pool[j];
+                if (consumed.has(variant.name))
+                    continue;
+                const s = sim(canonical, variant);
+                if (s !== undefined && s >= threshold) {
+                    collapses.push({ canonical: canonical.name, variant: variant.name, via: "twin-cosine", similarity: s });
+                    consumed.add(variant.name);
+                }
+            }
+            consumed.add(canonical.name);
+        }
+    }
+    return { collapses, warnings };
+}
+/**
+ * Fold a dropped variant's provenance into the canonical's frontmatter. Appends
+ * the variant ref to a `dedupedFrom` list (deduplicated, sorted) and carries
+ * any `source`/`sources` references the variant held that the canonical lacks.
+ * Pure string→string; never invents timestamps.
+ */
+export function applyProvenance(canonicalRaw, variant) {
+    let parsed;
+    try {
+        parsed = parseFrontmatter(canonicalRaw);
+    }
+    catch {
+        // Canonical frontmatter unparseable — leave content untouched (the file is
+        // never collapsed AS a canonical when hot/unparseable is the variant, but
+        // the canonical itself can still be odd; preserve bytes).
+        return canonicalRaw;
+    }
+    const fm = { ...parsed.data };
+    const variantRef = `memory:${variant.name}`;
+    const existing = Array.isArray(fm.dedupedFrom)
+        ? fm.dedupedFrom.filter((v) => typeof v === "string")
+        : typeof fm.dedupedFrom === "string"
+            ? [fm.dedupedFrom]
+            : [];
+    const next = Array.from(new Set([...existing, variantRef])).sort();
+    fm.dedupedFrom = next;
+    return assembleAssetFromString(serializeFrontmatter(fm), parsed.content);
+}
+/**
+ * Apply a collapse plan to disk: rewrite each canonical with merged provenance,
+ * delete each variant. Returns counts + consumed refs so the consolidate pass
+ * can prune them from the LLM pool. NO LLM call.
+ *
+ * `onArchive` (optional) is invoked with the variant file path before deletion
+ * so the caller can archive/back up exactly as it does for LLM merges.
+ */
+export function applyDedupPlan(plan, memories, onArchive) {
+    const byName = new Map();
+    for (const m of memories)
+        byName.set(m.name, m);
+    const warnings = [...plan.warnings];
+    const consumedRefs = [];
+    let collapsed = 0;
+    // Accumulate provenance per canonical so multiple variants folding into the
+    // same canonical produce a single rewrite.
+    const provByCanonical = new Map();
+    for (const c of plan.collapses) {
+        const list = provByCanonical.get(c.canonical);
+        if (list)
+            list.push(byName.get(c.variant));
+        else
+            provByCanonical.set(c.canonical, [byName.get(c.variant)]);
+    }
+    // 1. Rewrite canonicals with merged provenance.
+    for (const [canonicalName, variants] of provByCanonical) {
+        const canonical = byName.get(canonicalName);
+        if (!canonical)
+            continue;
+        let content = canonical.raw;
+        for (const v of variants.filter((x) => x).sort((a, b) => a.name.localeCompare(b.name))) {
+            content = applyProvenance(content, v);
+        }
+        try {
+            fs.writeFileSync(canonical.filePath, content.endsWith("\n") ? content : `${content}\n`, "utf8");
+        }
+        catch (e) {
+            warnings.push(`dedup: failed to rewrite canonical ${canonicalName}: ${String(e)}`);
+        }
+    }
+    // 2. Delete variants (archive first if requested).
+    for (const c of plan.collapses) {
+        const variant = byName.get(c.variant);
+        if (!variant)
+            continue;
+        if (!fs.existsSync(variant.filePath)) {
+            warnings.push(`dedup: variant ${c.variant} already absent — skipping delete.`);
+            continue;
+        }
+        if (onArchive) {
+            try {
+                onArchive(variant);
+            }
+            catch {
+                // best-effort archival; deletion proceeds
+            }
+        }
+        try {
+            fs.unlinkSync(variant.filePath);
+            collapsed++;
+            consumedRefs.push(`memory:${variant.name}`);
+        }
+        catch (e) {
+            warnings.push(`dedup: failed to delete variant ${c.variant}: ${String(e)}`);
+        }
+    }
+    return { collapsed, consumedRefs, warnings };
+}
+/**
+ * Top-level entry point for the consolidate pre-pass. Loads memories, optionally
+ * embeds them (only when a cosine path is reachable), plans, and applies the
+ * collapse. DEFAULT OFF — returns a no-op result when `config.enabled !== true`.
+ *
+ * `onArchive` lets the caller archive/back up each dropped variant before
+ * deletion (consolidate.ts wires this to its existing archive helper).
+ *
+ * `signal` (optional): an AbortSignal forwarded from the caller's budget
+ * controller. When aborted before the embedding call the function returns a
+ * no-op result immediately; the signal is also forwarded into `embedBatch`
+ * so a mid-embedding abort is handled cleanly.
+ */
+export async function runDeterministicDedup(stashDir, dedupConfig, akmConfig, onArchive, signal, 
+/** Optional open state.db handle for the body-embedding cache (WS-3a). */
+stateDb) {
+    if (!dedupConfig?.enabled) {
+        return { collapsed: 0, consumedRefs: [], warnings: [] };
+    }
+    if (signal?.aborted) {
+        return { collapsed: 0, consumedRefs: [], warnings: ["dedup: aborted before start"] };
+    }
+    const threshold = dedupConfig.cosineThreshold ?? DEFAULT_DEDUP_COSINE_THRESHOLD;
+    const candidateLimit = dedupConfig.cosineCandidateLimit ?? DEFAULT_COSINE_CANDIDATE_LIMIT;
+    const memories = loadDedupMemories(stashDir);
+    if (memories.length === 0) {
+        return { collapsed: 0, consumedRefs: [], warnings: [] };
+    }
+    // Embed only when embeddings are configured — exact-hash collapse still works
+    // without them. Fail-open: any embedding error degrades to hash-only matching.
+    // NOTE: embedBatch embeds the case-preserving stripped body (cacheHash domain),
+    // not the lowercase dedupHash body, so dedup cosine and the body_embeddings
+    // cache share the same canonical embedding input.
+    let embeddings;
+    if (akmConfig.embedding) {
+        try {
+            const eligible = memories.filter((m) => !m.hot);
+            // Use the case-preserving stripped body for embeddings (matching cacheHash
+            // canonical input) so the embedding cache can be shared with consolidate.
+            const modelId = resolveEmbeddingModelId(akmConfig.embedding);
+            // WS-3a: body-embedding cache — look up all content_hashes in one query,
+            // embed only the misses, then upsert the new vectors in one transaction.
+            const contentHashes = eligible.map((m) => cacheHash(m.raw));
+            const hashToName = new Map();
+            for (let i = 0; i < eligible.length; i++) {
+                hashToName.set(contentHashes[i], eligible[i].name);
+            }
+            let cachedVecs = new Map();
+            if (stateDb) {
+                try {
+                    cachedVecs = getBodyEmbeddings(stateDb, contentHashes, modelId);
+                }
+                catch {
+                    // Fail open: cache read errors degrade to full embed.
+                    cachedVecs = new Map();
+                }
+            }
+            const missIndices = [];
+            const missTexts = [];
+            for (let i = 0; i < eligible.length; i++) {
+                const hash = contentHashes[i];
+                if (!cachedVecs.has(hash)) {
+                    missIndices.push(i);
+                    missTexts.push(stripFrontmatterBody(eligible[i].raw) || eligible[i].name);
+                }
+            }
+            let missVecs = [];
+            if (missTexts.length > 0) {
+                missVecs = await embedBatch(missTexts, akmConfig.embedding, signal);
+                // Upsert new vectors into cache.
+                if (stateDb && missVecs.length === missTexts.length) {
+                    try {
+                        const toUpsert = missIndices.map((idx, pos) => ({
+                            contentHash: contentHashes[idx],
+                            embedding: missVecs[pos],
+                            modelId,
+                        }));
+                        upsertBodyEmbeddings(stateDb, toUpsert);
+                    }
+                    catch {
+                        // Fail open: cache write errors are non-fatal.
+                    }
+                }
+            }
+            // Assemble the full embeddings map (cache hits + freshly embedded misses).
+            if (missVecs.length === missTexts.length || cachedVecs.size > 0) {
+                embeddings = new Map();
+                // Add cache hits.
+                for (let i = 0; i < eligible.length; i++) {
+                    const hash = contentHashes[i];
+                    const cached = cachedVecs.get(hash);
+                    if (cached)
+                        embeddings.set(eligible[i].name, cached);
+                }
+                // Add freshly embedded misses.
+                for (let pos = 0; pos < missIndices.length; pos++) {
+                    const idx = missIndices[pos];
+                    const vec = missVecs[pos];
+                    if (vec)
+                        embeddings.set(eligible[idx].name, vec);
+                }
+            }
+        }
+        catch {
+            embeddings = undefined;
+        }
+    }
+    const plan = planDedup(memories, { cosineThreshold: threshold, embeddings, cosineCandidateLimit: candidateLimit });
+    if (plan.collapses.length === 0) {
+        return { collapsed: 0, consumedRefs: [], warnings: plan.warnings };
+    }
+    // Sanity: every variant ref must be a parseable memory ref before we touch
+    // disk (defends against a malformed name slipping through).
+    for (const c of plan.collapses) {
+        try {
+            parseAssetRef(`memory:${c.variant}`);
+            parseAssetRef(`memory:${c.canonical}`);
+        }
+        catch {
+            plan.warnings.push(`dedup: unparseable ref in collapse ${c.canonical} ← ${c.variant} — dropping op.`);
+        }
+    }
+    const result = applyDedupPlan(plan, memories, (v) => onArchive?.(v.filePath, v.name));
+    if (result.collapsed > 0) {
+        warn(`[consolidate] deterministic dedup collapsed ${result.collapsed} near-duplicate memor${result.collapsed === 1 ? "y" : "ies"} (no LLM).`);
+    }
+    return result;
+}

package/dist/commands/improve/distill/content-repair.js

@@ -0,0 +1,202 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * Pure content-repair + quality-validation stages for `akm distill`.
+ *
+ * Extracted verbatim from the inline body of `akmDistill` so each normalization
+ * pass is an independently testable unit. Every function is a pure transform of
+ * `(content, inputRef) → content | findings` with no I/O — logic is
+ * byte-identical to the pre-extraction inline code. The lesson-path guard
+ * (`effectiveProposalKind !== "knowledge"`) stays in the caller; these helpers
+ * assume the lesson path.
+ */
+import { assembleAssetFromString } from "../../../core/asset/asset-serialize.js";
+import { parseFrontmatter } from "../../../core/asset/frontmatter.js";
+import { repairTruncatedDescription } from "../../../core/text-truncation.js";
+import { detectDoubleFrontmatter, isValidDescription, isValidWhenToUse, } from "../../proposal/validators/proposal-quality-validators.js";
+/**
+ * Auto-repair missing frontmatter fields before hard-failing. Small models
+ * frequently produce a good lesson body but omit the YAML header entirely.
+ * Rather than discarding valid content, we extract description/when_to_use
+ * from the body and prepend the required frontmatter block.
+ *
+ * IMPORTANT: We do NOT synthesise placeholder strings here. If the body
+ * does not contain text that passes the post-LLM validators
+ * (`isValidDescription` / `isValidWhenToUse`), we leave the field missing
+ * and let the lesson lint reject the proposal as `validation_failed`.
+ * Emitting placeholders like `"Lesson distilled from <ref>"` or
+ * `"When working with <slug>"` is what produced the systematic broken
+ * proposals observed across 323 archived rejections.
+ */
+export function autoRepairLessonFrontmatter(content, inputRef) {
+    const parsed = parseFrontmatter(content);
+    const fm = (parsed.data ?? {});
+    const missingDesc = typeof fm.description !== "string" || !fm.description.trim();
+    const missingWtu = typeof fm.when_to_use !== "string" || !fm.when_to_use.trim();
+    if (!missingDesc && !missingWtu)
+        return content;
+    const body = parsed.content.trim();
+    // Strip markdown formatting tokens from a line so extracted text is clean.
+    const stripMd = (l) => l
+        .replace(/\*\*([^*]+)\*\*/g, "$1")
+        .replace(/\*([^*]+)\*/g, "$1")
+        .replace(/`([^`]+)`/g, "$1")
+        .replace(/^[#*\->_]+\s*/, "")
+        .replace(/:\s*$/, "")
+        .trim();
+    // Skip lines that look like YAML field assignments (key: value) or frontmatter delimiters.
+    // These appear when the LLM leaks frontmatter content into the body, causing
+    // auto-repair to produce description: "description: Key Takeaways".
+    const isYamlLike = (l) => /^---/.test(l) || /^[a-z_]+:\s/i.test(l);
+    const bodyLines = body.split("\n").map(stripMd);
+    // Extract description: first body line that BOTH looks like prose AND
+    // passes isValidDescription. If nothing qualifies, leave the field
+    // missing — the lint pass will reject the proposal cleanly.
+    let descLine;
+    for (const l of bodyLines) {
+        if (isYamlLike(l))
+            continue;
+        if (l.length <= 10 || l.length >= 400)
+            continue;
+        if (isValidDescription(l, inputRef).ok) {
+            descLine = l;
+            break;
+        }
+    }
+    // Extract when_to_use: a line starting with "When" / "Use when" / "Apply when"
+    // that ALSO passes isValidWhenToUse (rejects circular fallbacks).
+    let wtuLine;
+    for (const l of bodyLines) {
+        if (!/^(when |use when|apply when)/i.test(l))
+            continue;
+        if (l.length >= 400)
+            continue;
+        if (isValidWhenToUse(l, inputRef).ok) {
+            wtuLine = l;
+            break;
+        }
+    }
+    const repairedFm = {
+        ...fm,
+        ...(missingDesc && descLine ? { description: descLine } : {}),
+        ...(missingWtu && wtuLine ? { when_to_use: wtuLine } : {}),
+    };
+    const fmLines = Object.entries(repairedFm)
+        .map(([k, v]) => `${k}: ${JSON.stringify(v)}`)
+        .join("\n");
+    // Only rewrite content if we actually have at least one field to write.
+    // Otherwise leave the original content for the lint pass to reject.
+    if (Object.keys(repairedFm).length > 0) {
+        return assembleAssetFromString(fmLines, body);
+    }
+    return content;
+}
+/**
+ * Description ↔ when_to_use auto-swap normalization (recover ~93% of
+ * qwen-9b's `^when\b/i` rejections at zero LLM cost). When the LLM emits
+ * a conditional-framed description ("When X happens, do Y") and the
+ * when_to_use field looks like a declarative description (or is empty),
+ * the two fields are mis-fielded — exactly what `isValidDescription`'s
+ * error message says ("that pattern belongs in when_to_use"). We swap
+ * them and revalidate; the swap is committed only if BOTH fields pass
+ * their respective validators afterwards. If revalidation still fails,
+ * we fall through returning the original content (swapped: 0).
+ */
+export function autoSwapDescriptionWhenToUse(content, inputRef) {
+    const parsedSwap = parseFrontmatter(content);
+    const fmSwap = (parsedSwap.data ?? {});
+    const descRaw = typeof fmSwap.description === "string" ? fmSwap.description.trim() : "";
+    const wtuRaw = typeof fmSwap.when_to_use === "string" ? fmSwap.when_to_use.trim() : "";
+    const descStartsConditional = /^(when|if)\b/i.test(descRaw);
+    const wtuStartsConditional = /^(when|if)\b/i.test(wtuRaw);
+    if (descStartsConditional && !wtuStartsConditional && wtuRaw.length > 0) {
+        // Try the swap and revalidate. The when_to_use validator requires the
+        // value not match `/^when working with\b/i` (the circular fallback) —
+        // a real description rarely does, so this usually passes.
+        const swappedDescCheck = isValidDescription(wtuRaw, inputRef);
+        const swappedWtuCheck = isValidWhenToUse(descRaw, inputRef);
+        if (swappedDescCheck.ok && swappedWtuCheck.ok) {
+            const swappedFm = {
+                ...fmSwap,
+                description: wtuRaw,
+                when_to_use: descRaw,
+            };
+            const swappedFmLines = Object.entries(swappedFm)
+                .map(([k, v]) => `${k}: ${JSON.stringify(v)}`)
+                .join("\n");
+            return { content: assembleAssetFromString(swappedFmLines, parsedSwap.content), swapped: 1 };
+        }
+    }
+    return { content, swapped: 0 };
+}
+/**
+ * Post-generation truncation repair (#556): if the LLM sliced the
+ * description mid-sentence, deterministically complete it from its own text
+ * / the lesson body BEFORE the lint + quality validators run. No-op
+ * (byte-identical) for already-complete descriptions, so this never alters
+ * a valid proposal.
+ */
+export function repairLessonDescriptionTruncation(content) {
+    const parsedRepair = parseFrontmatter(content);
+    const fmRepair = (parsedRepair.data ?? {});
+    const descRepairRaw = typeof fmRepair.description === "string" ? fmRepair.description : "";
+    if (!descRepairRaw)
+        return content;
+    const repaired = repairTruncatedDescription(descRepairRaw, parsedRepair.content);
+    if (repaired === descRepairRaw)
+        return content;
+    const repairedFmLines = Object.entries({ ...fmRepair, description: repaired })
+        .map(([k, v]) => `${k}: ${JSON.stringify(v)}`)
+        .join("\n");
+    return assembleAssetFromString(repairedFmLines, parsedRepair.content);
+}
+/**
+ * Additional quality validators that run only on lessons whose lesson-lint
+ * pass was clean. lesson-lint checks "field is present and non-empty"; these
+ * reject the systematic failure modes observed across 323 archived rejected
+ * proposals:
+ *   - description is a body fragment, section heading, or placeholder
+ *   - when_to_use is the circular "When working with <ref>" fallback
+ *   - description == when_to_use (LLM duplicated a single sentence)
+ *   - body contains a second pseudo-frontmatter block
+ */
+export function collectLessonQualityFindings(content, inputRef) {
+    const findings = [];
+    const parsedQC = parseFrontmatter(content);
+    const fmQC = (parsedQC.data ?? {});
+    const descCheck = isValidDescription(fmQC.description, inputRef);
+    if (!descCheck.ok) {
+        findings.push({
+            kind: "invalid-description",
+            field: "description",
+            message: `Distilled lesson for ${inputRef} has an invalid description: ${descCheck.reason}.`,
+        });
+    }
+    const wtuCheck = isValidWhenToUse(fmQC.when_to_use, inputRef);
+    if (!wtuCheck.ok) {
+        findings.push({
+            kind: "invalid-when_to_use",
+            field: "when_to_use",
+            message: `Distilled lesson for ${inputRef} has an invalid when_to_use: ${wtuCheck.reason}.`,
+        });
+    }
+    // description and when_to_use must say different things.
+    if (descCheck.ok &&
+        wtuCheck.ok &&
+        typeof fmQC.description === "string" &&
+        typeof fmQC.when_to_use === "string" &&
+        fmQC.description.trim().toLowerCase() === fmQC.when_to_use.trim().toLowerCase()) {
+        findings.push({
+            kind: "description-equals-when_to_use",
+            field: "description",
+            message: `Distilled lesson for ${inputRef} has identical description and when_to_use.`,
+        });
+    }
+    // Double-frontmatter / pseudo-frontmatter pollution in the body.
+    const dfm = detectDoubleFrontmatter(content);
+    if (dfm) {
+        findings.push({ kind: dfm.kind, field: "body", message: `Distilled lesson for ${inputRef}: ${dfm.message}` });
+    }
+    return findings;
+}