akm-cli 0.9.0-beta.9 → 0.9.0-rc.0

package/dist/commands/improve/distill.js

@@ -51,23 +51,38 @@
  * be invoked from CI / automation without spinning up an agent harness.
  */
 import fs from "node:fs";
-import path from "node:path";
+import distillKnowledgeSystemPrompt from "../../assets/prompts/distill-knowledge-system.md" with { type: "text" };
+import distillLessonSystemPrompt from "../../assets/prompts/distill-lesson-system.md" with { type: "text" };
 import { parseAssetRef } from "../../core/asset/asset-ref.js";
 import { assembleAssetFromString } from "../../core/asset/asset-serialize.js";
-import { parseFrontmatter } from "../../core/asset/frontmatter.js";
+import { parseFrontmatter, writeSalienceToFrontmatter } from "../../core/asset/frontmatter.js";
 import { stripMarkdownFences } from "../../core/asset/markdown.js";
-import { resolveStashDir, timestampForFilename } from "../../core/common.js";
-import { getDefaultLlmConfig, loadConfig } from "../../core/config/config.js";
+import { authoringRulesForType } from "../../core/authoring-rules.js";
+import { resolveStashDir } from "../../core/common.js";
+import { getDefaultLlmConfig, getImproveProcessConfig, loadConfig } from "../../core/config/config.js";
 import { ConfigError, UsageError } from "../../core/errors.js";
 import { appendEvent, readEvents } from "../../core/events.js";
 import { lintLessonContent } from "../../core/lesson-lint.js";
+import { getDbPath } from "../../core/paths.js";
+import { resolveStashStandards } from "../../core/standards/resolve-stash-standards.js";
+import { withStateDb } from "../../core/state-db.js";
 import { warnVerbose } from "../../core/warn.js";
+import { closeDatabase, getAllEntries, openIndexDatabase } from "../../indexer/db/db.js";
 import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
 import { chatCompletion, parseEmbeddedJsonResponse } from "../../llm/client.js";
 import { isLlmFeatureEnabled, tryLlmFeature } from "../../llm/feature-gate.js";
-import { createProposal, isProposalSkipped, listProposals, } from "../proposal/validators/proposals.js";
-import { akmSearch } from "../read/search.js";
-import { assessMemoryKnowledgePromotionCandidate, deriveKnowledgeRef } from "./distill-promotion-policy.js";
+import { createProposal, isProposalSkipped, listProposals, } from "../proposal/repository.js";
+import { stripFrontmatterBody as stripBodyForFidelity } from "./dedup.js";
+import { autoRepairLessonFrontmatter, autoSwapDescriptionWhenToUse, collectLessonQualityFindings, repairLessonDescriptionTruncation, } from "./distill/content-repair.js";
+import { promoteMemoryToKnowledge } from "./distill/promote-memory.js";
+import { fetchTopSimilarLessons, persistOutputEncodingSalience, runLessonQualityJudge, writeQualityRejection, } from "./distill/quality-gate.js";
+import { buildClsContext, checkDistillFidelity } from "./distill-guards.js";
+import { deriveKnowledgeRef } from "./distill-promotion-policy.js";
+import { buildRefVocabulary, scoreEncodingSalience } from "./encoding-salience.js";
+import { computeSalience, upsertAssetSalience } from "./salience.js";
+// Re-exported for `reflect.ts`, which applies the same LLM-as-judge gate to
+// reflect proposals (R-5 / #374).
+export { runLessonQualityJudge };
 /**
  * Asset-ref types that `akm distill` structurally refuses as inputs.
  *
@@ -76,6 +91,11 @@ import { assessMemoryKnowledgePromotionCandidate, deriveKnowledgeRef } from "./d
  * `lesson:lesson-<name>-lesson-lesson` (double `-lesson` suffix) — the
  * recursive-ref defect observed across 323 archived rejected proposals.
  *
+ * 08-F2: `env` and `secret` are refused as a STRUCTURAL floor — distill reads
+ * the input asset's bytes via `readFileSync` and hands them to the LLM, so
+ * secret material must never be a distill input. This gate is code, not config:
+ * it holds even when `allowedTypes` config is mis-set in unattended cron.
+ *
  * The runtime gate inside {@link akmDistill} still refuses these inputs
  * defensively (returning an `outcome: "skipped"` envelope with `skipReason:
  * "recursive_lesson_input"`). This exported set is the planner-side companion:
@@ -87,7 +107,7 @@ import { assessMemoryKnowledgePromotionCandidate, deriveKnowledgeRef } from "./d
  * type means updating this constant — the planner picks the change up for
  * free.
  */
-export const DISTILL_REFUSED_INPUT_TYPES = new Set(["lesson"]);
+export const DISTILL_REFUSED_INPUT_TYPES = new Set(["lesson", "env", "secret"]);
 /**
  * Returns true when `type` is structurally refused as an input by
  * {@link akmDistill}. See {@link DISTILL_REFUSED_INPUT_TYPES}.
@@ -112,7 +132,6 @@ export function deriveLessonRef(inputRef) {
         .replace(/^-|-$/g, "");
     return `lesson:${safe}-lesson`;
 }
-import { repairTruncatedDescription } from "../../core/text-truncation.js";
 // ── Content quality validators ──────────────────────────────────────────────
 //
 // The actual implementations now live in `core/proposal-quality-validators.ts`
@@ -122,68 +141,8 @@ import { repairTruncatedDescription } from "../../core/text-truncation.js";
 import { detectDoubleFrontmatter, isValidDescription, isValidWhenToUse, } from "../proposal/validators/proposal-quality-validators.js";
 export { detectDoubleFrontmatter, isValidDescription, isValidWhenToUse };
 // ── Prompt assembly ─────────────────────────────────────────────────────────
-const LESSON_SYSTEM_PROMPT = [
-    "You are the akm `distill` distiller.",
-    "Given an asset and recent feedback events about it, produce a single",
-    "concise *lesson* an agent should remember next time it works on this",
-    "asset's domain.",
-    "",
-    "YOUR RESPONSE MUST START EXACTLY WITH `---` ON THE VERY FIRST LINE.",
-    "DO NOT output any prose, explanation, or code fences before or after.",
-    "",
-    "Required output format — copy this structure exactly:",
-    "---",
-    "description: <one complete sentence (ending with `.`) summarising what the lesson teaches>",
-    "when_to_use: <one complete sentence describing the concrete trigger condition>",
-    "---",
-    "",
-    "<lesson body — plain markdown, 1–3 short paragraphs of practical guidance>",
-    "",
-    "## description field (MANDATORY)",
-    "- A single complete sentence in present tense, 80-200 chars, NO markdown.",
-    "- Self-contained: a reviewer must understand the lesson from this field alone.",
-    '- DO NOT start with "When ", "If ", or a connector word — that belongs in when_to_use.',
-    '- DO NOT copy a section heading ("Key takeaways", "For example", "Key pitfalls").',
-    "- DO NOT begin with a numbered list marker, code fence, or markdown heading.",
-    "",
-    'GOOD: "Always validate ref existence before promoting a memory to knowledge; missing refs surface as silent 404s during accept."',
-    'BAD:  "Key pitfalls"',
-    'BAD:  "When working with the akm CLI"',
-    'BAD:  "For example, you might..."',
-    'BAD:  "1. Check the file"',
-    "",
-    "RULES:",
-    "- `when_to_use` MUST be a complete sentence describing a concrete trigger. Never write `When working with <asset-name>` — that is circular and useless.",
-    "- `description` and `when_to_use` MUST differ from each other.",
-    "- The lesson body MUST be non-empty markdown prose. Do NOT restate `description:` or `when_to_use:` inside the body (no `**description:** ...` or `**when_to_use:** ...` lines — the frontmatter is the only place those keys belong).",
-    "- Do NOT emit a second `---` fence after the opening frontmatter — there are exactly two `---` lines in the output, both belonging to the single frontmatter block at the top.",
-    "- Do NOT reproduce the source asset verbatim — distil what a caller needs to know.",
-    "- Output ONLY the lesson file. No preamble, no code fences, no trailing prose.",
-].join("\n");
-const KNOWLEDGE_SYSTEM_PROMPT = [
-    "You are the akm `distill` distiller.",
-    "Given an asset and recent feedback events about it, produce a concise",
-    "*knowledge* markdown document capturing the durable, reusable facts.",
-    "Prefer stable guidance over narrative recap.",
-    "",
-    "YOUR RESPONSE MUST START EXACTLY WITH `---` ON THE VERY FIRST LINE.",
-    "DO NOT output any prose, explanation, or code fences before or after.",
-    "",
-    "Required output format:",
-    "---",
-    "description: <one-line summary of the knowledge asset>",
-    "tags: [<tag1>, <tag2>]",
-    "---",
-    "",
-    "# <Title>",
-    "",
-    "<body — structured markdown, durable facts only>",
-    "",
-    "RULES:",
-    "- `description` MUST be a non-empty single-line string.",
-    "- Include a meaningful markdown body with a `# Title` heading.",
-    "- Output ONLY the knowledge file. No preamble, no code fences, no trailing prose.",
-].join("\n");
+const LESSON_SYSTEM_PROMPT = distillLessonSystemPrompt;
+const KNOWLEDGE_SYSTEM_PROMPT = distillKnowledgeSystemPrompt;
 // ── Structured-output schemas (responseSchema lift) ─────────────────────────
 //
 // PR 1 of the asset-writers decision (see knowledge:projects/akm/
@@ -358,6 +317,18 @@ export function buildDistillPrompt(input) {
     const lines = [];
     lines.push(`Asset ref: ${input.inputRef}`);
     lines.push("");
+    if (input.standardsContext?.trim()) {
+        lines.push("Standards to follow (the rulebook for this target):");
+        lines.push(input.standardsContext.trim());
+        lines.push("");
+    }
+    {
+        const authoringRules = authoringRulesForType(input.proposalKind ?? "lesson");
+        if (authoringRules) {
+            lines.push(authoringRules);
+            lines.push("");
+        }
+    }
     lines.push("Asset content:");
     if (input.assetContent) {
         const body = input.assetContent.trim().slice(0, 3000);
@@ -443,181 +414,6 @@ export function buildDistillPrompt(input) {
     }
     return lines.join("\n");
 }
-// ── D-4 / #390: Top-3 similar lessons retrieval ──────────────────────────────
-/**
- * Default implementation: use akmSearch to find top-N similar lesson assets.
- * Returns empty array when search fails or returns no results.
- * Requires embedding configured for semantic similarity; degrades gracefully.
- */
-async function fetchTopSimilarLessons(query, n, _stashDir) {
-    try {
-        const result = await akmSearch({
-            query,
-            type: "lesson",
-            limit: n,
-            skipLogging: true,
-            eventSource: "improve",
-        });
-        const hits = result?.hits ?? [];
-        return hits
-            .filter((h) => "path" in h && typeof h.path === "string")
-            .slice(0, n)
-            .map((h) => {
-            let content = "";
-            try {
-                if (h.path && fs.existsSync(h.path)) {
-                    content = fs.readFileSync(h.path, "utf8");
-                }
-            }
-            catch {
-                /* best-effort */
-            }
-            return { ref: h.ref, content };
-        });
-    }
-    catch {
-        return [];
-    }
-}
-// ── LLM-as-judge quality gate (P2-B) ────────────────────────────────────────
-/**
- * D-4 / #390: Build the LLM-as-judge prompt.
- *
- * When similarLessons are provided (top-3 by embedding similarity), they are
- * included in the context so the judge can lower the score for near-duplicates.
- * Voyager arXiv:2305.16291 — skill library admission requires similarity check
- * against the existing library. A-MEM arXiv:2502.12110 — new notes are checked
- * against existing notes before linking.
- */
-function buildJudgePrompt(lessonContent, sourceContent, similarLessons) {
-    const lines = [
-        "You are evaluating a proposed lesson asset for an akm knowledge base.",
-        "",
-        "Score this lesson on each criterion from 1 (poor) to 5 (excellent):",
-        "1. NOVELTY: Does the lesson add information not already present in the source asset?",
-        "2. ACTIONABILITY: Can an agent follow this lesson without additional context?",
-        "3. NON-REDUNDANCY: Is this lesson meaningfully different from what the source already says?",
-        "",
-        "Source asset content:",
-        "```",
-        sourceContent.slice(0, 2000),
-        "```",
-    ];
-    if (similarLessons && similarLessons.length > 0) {
-        lines.push("");
-        lines.push("Existing similar lessons (top-3 by similarity). Rate lower if the proposed lesson is substantially similar to any of these:");
-        for (const sl of similarLessons) {
-            lines.push(`\nExisting lesson ref: ${sl.ref}`);
-            lines.push("```");
-            lines.push(sl.content.slice(0, 500));
-            lines.push("```");
-        }
-    }
-    lines.push("");
-    lines.push("Proposed lesson content:");
-    lines.push("```");
-    lines.push(lessonContent.slice(0, 1000));
-    lines.push("```");
-    lines.push("");
-    lines.push('Return ONLY valid JSON, no prose: {"score": <average score 1-5 as float>, "reason": "<one sentence>"}');
-    return lines.join("\n");
-}
-/**
- * Run the LLM-as-judge quality gate on a proposal's content.
- *
- * Exported so reflect.ts can apply the same gate to reflect proposals (R-5 / #374).
- * Gated by the flag name `lesson_quality_gate` (or its alias
- * `proposal_quality_gate`) via {@link isLlmFeatureEnabled} — which reads
- * `profiles.improve.default.processes.distill.qualityGate.enabled` (and the
- * corresponding `.reflect.qualityGate.enabled` for proposals).
- *
- * Fail-open: returns `pass: true` on timeout, parse failure, or missing LLM.
- */
-export async function runLessonQualityJudge(config, lessonContent, sourceContent, chat, 
-/** D-4 / #390: top-3 similar existing lessons for dedup check. */
-similarLessons) {
-    const llmConfig = getDefaultLlmConfig(config);
-    if (!llmConfig) {
-        return { pass: true, score: -1, reason: "no LLM configured — passing through" };
-    }
-    const judgeLlmConfig = llmConfig.judgeModel ? { ...llmConfig, model: llmConfig.judgeModel } : llmConfig;
-    const JUDGE_TIMEOUT_MS = 8_000;
-    try {
-        const raw = await Promise.race([
-            chat(judgeLlmConfig, [
-                { role: "system", content: "Return only valid JSON. No prose." },
-                { role: "user", content: buildJudgePrompt(lessonContent, sourceContent, similarLessons) },
-            ]),
-            new Promise((_, reject) => setTimeout(() => reject(new Error("judge timeout")), JUDGE_TIMEOUT_MS)),
-        ]);
-        const parsed = parseEmbeddedJsonResponse(raw);
-        if (!parsed || typeof parsed.score !== "number") {
-            return { pass: true, score: -1, reason: "judge parse failed — passing through" };
-        }
-        // D-5 / #388: Three-band system (MT-Bench arXiv:2306.05685 — ~±0.5 judge variance).
-        //   >= 3.5: auto-queue as pending (pass: true)
-        //   2.5–3.5: review-needed band — uncertain, escalate to human (reviewNeeded: true)
-        //   < 2.5: auto-reject (pass: false)
-        const score = parsed.score;
-        const reason = parsed.reason ?? "";
-        if (score >= 3.5) {
-            return { pass: true, score, reason };
-        }
-        if (score >= 2.5) {
-            // Uncertainty band: treat as failed for auto-queuing but flag for review.
-            return { pass: false, score, reason, reviewNeeded: true };
-        }
-        return { pass: false, score, reason };
-    }
-    catch {
-        return { pass: true, score: -1, reason: "judge failed — passing through" };
-    }
-}
-// ── Quality-rejection helper ─────────────────────────────────────────────────
-/**
- * Write a rejected lesson to `.akm/distill-rejected/`, append a `distill_invoked`
- * quality-rejected event, and return the `quality_rejected` envelope.
- *
- * @param stash     - Root stash directory.
- * @param inputRef  - The original input ref (for the event).
- * @param lessonRef - The proposed lesson/knowledge ref.
- * @param content   - The raw content that failed the quality gate.
- * @param score     - Quality score from the judge.
- * @param reason    - Human-readable rejection reason.
- * @param extraMeta - Optional additional metadata for the event.
- */
-function writeQualityRejection(stash, inputRef, lessonRef, content, score, reason, extraMeta = {}, eligibilitySource) {
-    // D-5 / #388: reviewNeeded flag selects "review_needed" vs "quality_rejected" outcome.
-    const outcome = extraMeta.reviewNeeded ? "review_needed" : "quality_rejected";
-    const rejectDir = path.join(stash, ".akm", "distill-rejected");
-    fs.mkdirSync(rejectDir, { recursive: true });
-    const ts = timestampForFilename();
-    fs.writeFileSync(path.join(rejectDir, `${ts}-${lessonRef}.md`), `---\nscore: ${score}\nreason: ${reason}\noutcome: ${outcome}\n---\n\n${content}`, "utf8");
-    appendEvent({
-        eventType: "distill_invoked",
-        ref: inputRef,
-        metadata: {
-            outcome,
-            lessonRef,
-            score,
-            reason,
-            ...extraMeta,
-            // Attribution tagging: stamp the eligibility lane so distill_invoked can be
-            // sliced by lane downstream. See EligibilitySource.
-            ...(eligibilitySource ? { eligibilitySource } : {}),
-        },
-    });
-    return {
-        schemaVersion: 1,
-        ok: true,
-        outcome,
-        inputRef,
-        lessonRef,
-        score,
-        reason,
-        ...extraMeta,
-    };
-}
 // ── Main entry point ────────────────────────────────────────────────────────
 /**
  * Run a single bounded distillation pass for `ref`. Always emits exactly one
@@ -650,15 +446,21 @@ export async function akmDistill(options) {
     // the improve planner can skip these refs before queuing distill attempts;
     // this runtime check stays as a defensive backstop for direct callers.
     if (isDistillRefusedInputType(parsedInputRef.type)) {
-        const skippedRef = `lesson:${parsedInputRef.name}`;
+        // 08-F2: env/secret are a secret-material refusal (never read the bytes);
+        // lesson is the recursive-form refusal. Both skip BEFORE any readFileSync.
+        const isSecretInput = parsedInputRef.type === "env" || parsedInputRef.type === "secret";
+        const skippedRef = isSecretInput ? inputRef : `lesson:${parsedInputRef.name}`;
+        const message = isSecretInput
+            ? `Distill refuses ${parsedInputRef.type} inputs — secret material must never be sent to the LLM.`
+            : "Distill refuses lesson inputs — lessons are the distilled form, not a source.";
         appendEvent({
             eventType: "distill_invoked",
             ref: inputRef,
             metadata: {
                 outcome: "skipped",
                 lessonRef: skippedRef,
-                message: "distill refuses lesson inputs — lessons are the distilled form, not a source",
-                skipReason: "recursive_lesson_input",
+                message,
+                skipReason: isSecretInput ? "refused_secret_input" : "recursive_lesson_input",
                 ...eligMeta,
             },
         });
@@ -668,7 +470,7 @@ export async function akmDistill(options) {
             outcome: "skipped",
             inputRef,
             lessonRef: skippedRef,
-            message: "Distill refuses lesson inputs — lessons are the distilled form, not a source.",
+            message,
         };
     }
     const config = options.config ?? loadConfig();
@@ -676,20 +478,95 @@ export async function akmDistill(options) {
     const chat = options.chat ?? chatCompletion;
     const lookup = options.lookupFn ?? defaultLookup;
     const readEventsImpl = options.readEventsFn ?? readEvents;
+    // R1 opt-out must flow into every computeSalience call this command makes so
+    // distill-written rank_score rows use the same weights as preparation's.
+    const outcomeWeightEnabled = config.improve?.salience?.outcomeWeightEnabled !== false;
     // D-4 / #390: similar-lessons retrieval seam (test-injectable).
     const fetchSimilarLessonsFn = options.fetchSimilarLessonsFn ?? ((query, n) => fetchTopSimilarLessons(query, n, options.stashDir));
     // Best-effort load: when the asset is not yet indexed we still proceed —
     // the LLM is asked to distil from "available signal" (feedback alone).
     let assetContent = null;
+    let assetFilePath = null;
     try {
         const filePath = await lookup(inputRef);
         if (filePath && fs.existsSync(filePath)) {
+            assetFilePath = filePath;
             assetContent = fs.readFileSync(filePath, "utf8");
         }
     }
     catch {
         assetContent = null;
     }
+    // ── #608: Encoding-time salience scoring ────────────────────────────────
+    // Score the source asset with the three-signal model (novelty × 0.40 +
+    // magnitude × 0.35 + predictionError × 0.25) and persist the result to:
+    //   1. The asset's frontmatter (human-readable mirror; idempotent delta gate).
+    //   2. state.db :: asset_salience (canonical; feeds improve's high-salience gate).
+    // Both writes are best-effort — a DB error never blocks distillation.
+    //
+    // The bigram ref vocabulary is built ONCE per invocation — the novelty signal
+    // reuses it when scoring the distilled OUTPUT at proposal creation (G4).
+    let existingRefVocabulary = new Set();
+    try {
+        const embCfg = config?.embedding;
+        const indexDb = openIndexDatabase(getDbPath(), embCfg?.dimension ? { embeddingDim: embCfg.dimension } : undefined);
+        try {
+            const allRefs = getAllEntries(indexDb).map((e) => e.entryKey);
+            existingRefVocabulary = buildRefVocabulary(allRefs);
+        }
+        finally {
+            closeDatabase(indexDb);
+        }
+    }
+    catch {
+        // Index not available — novelty defaults to type-floor.
+    }
+    if (assetContent && assetFilePath) {
+        try {
+            const parsedRef = parseAssetRef(inputRef);
+            // G4: predictionError decays with revision count — the prior hardcoded
+            // `revisionCount: 0` made it a dead constant 1.0. Use the number of
+            // proposals ever raised against this ref as the revision proxy.
+            let revisionCount = 0;
+            try {
+                revisionCount = listProposals(stash, { ref: inputRef, includeArchive: true }).length;
+            }
+            catch {
+                // best-effort: unknown history scores as a first encounter
+            }
+            const salienceResult = scoreEncodingSalience({
+                body: assetContent,
+                type: parsedRef.type,
+                existingRefVocabulary,
+                revisionCount,
+            });
+            // 1. Write salience to the source asset frontmatter (idempotent).
+            const updatedContent = writeSalienceToFrontmatter(assetContent, salienceResult.score, salienceResult);
+            if (updatedContent !== assetContent) {
+                fs.writeFileSync(assetFilePath, updatedContent, "utf8");
+                assetContent = updatedContent;
+            }
+            // 2. Persist encoding_salience to state.db.
+            try {
+                withStateDb((stateDb) => {
+                    const vector = computeSalience({
+                        ref: inputRef,
+                        type: parsedRef.type,
+                        retrievalFreq: 0,
+                        encodingSalience: salienceResult.score,
+                        outcomeWeightEnabled,
+                    });
+                    upsertAssetSalience(stateDb, inputRef, vector);
+                });
+            }
+            catch {
+                // State DB unavailable — frontmatter mirror is the only persistence.
+            }
+        }
+        catch {
+            // Scoring errors never block distillation.
+        }
+    }
     const { events } = readEventsImpl({
         ref: inputRef,
         type: "feedback",
@@ -713,190 +590,34 @@ export async function akmDistill(options) {
         eventType: e.eventType,
         ...(e.metadata !== undefined ? { metadata: e.metadata } : {}),
     }));
-    const promotion = targetKind === "lesson"
-        ? null
-        : assessMemoryKnowledgePromotionCandidate({
-            inputRef,
-            assetContent,
-            feedbackEvents: filteredEvents.map((event) => ({
-                ...(event.metadata !== undefined ? { metadata: event.metadata } : {}),
-            })),
-        });
-    if (promotion?.promote && promotion.content && (targetKind === "knowledge" || targetKind === "auto")) {
-        // D-1 / #369: When the destination knowledge file already exists, route
-        // through the LLM for contradiction resolution instead of silently
-        // overwriting. Follows mem0 ADD/UPDATE/DELETE/NOOP pattern (arXiv:2504.19413 §3.2)
-        // and A-MEM dynamic linking (arXiv:2502.12110).
-        let resolvedPromotionContent = promotion.content;
-        const existingKnowledgePath = await lookup(promotion.knowledgeRef);
-        const existingKnowledgeContent = existingKnowledgePath && fs.existsSync(existingKnowledgePath)
-            ? (() => {
-                try {
-                    return fs.readFileSync(existingKnowledgePath, "utf8");
-                }
-                catch {
-                    return null;
-                }
-            })()
-            : null;
-        if (existingKnowledgeContent && config && getDefaultLlmConfig(config)) {
-            // Existing content found: call LLM for contradiction-resolution merge.
-            const mergePrompt = [
-                "You are merging two versions of a knowledge document.",
-                "Existing content is already committed; new content comes from a memory distillation run.",
-                "Choose one of: ADD (combine both), UPDATE (replace existing with new), NOOP (keep existing unchanged).",
-                'Return ONLY valid JSON: {"action": "ADD"|"UPDATE"|"NOOP", "content": "<merged markdown if ADD/UPDATE, empty string if NOOP>"}',
-                "",
-                "## Existing knowledge content",
-                "```",
-                existingKnowledgeContent.slice(0, 3000),
-                "```",
-                "",
-                "## New content from distillation",
-                "```",
-                promotion.content.slice(0, 3000),
-                "```",
-            ].join("\n");
-            try {
-                const mergeLlm = getDefaultLlmConfig(config);
-                if (!mergeLlm) {
-                    throw new ConfigError("LLM is not configured for distillation merge.", "LLM_NOT_CONFIGURED");
-                }
-                const mergeResponse = await chat(mergeLlm, [
-                    { role: "system", content: "Return only valid JSON. No prose." },
-                    { role: "user", content: mergePrompt },
-                ]);
-                const mergeResult = parseEmbeddedJsonResponse(mergeResponse);
-                if (mergeResult?.action === "NOOP") {
-                    // Existing content is authoritative — no update needed.
-                    appendEvent({
-                        eventType: "distill_invoked",
-                        ref: inputRef,
-                        metadata: {
-                            outcome: "skipped",
-                            lessonRef: promotion.knowledgeRef,
-                            message: "D-1: LLM resolved destination conflict as NOOP — existing content kept",
-                            ...eligMeta,
-                        },
-                    });
-                    return {
-                        schemaVersion: 1,
-                        ok: true,
-                        outcome: "skipped",
-                        inputRef,
-                        lessonRef: promotion.knowledgeRef,
-                        message: "Existing knowledge content unchanged (contradiction resolution: NOOP)",
-                    };
-                }
-                if (mergeResult?.action && (mergeResult.action === "ADD" || mergeResult.action === "UPDATE")) {
-                    if (mergeResult.content?.trim()) {
-                        resolvedPromotionContent = mergeResult.content;
-                    }
-                }
-            }
-            catch {
-                // LLM merge failed — fall through with the original promotion content.
-                // The reviewer will see both versions in the proposal diff.
-            }
-        }
-        else if (existingKnowledgeContent && config && !getDefaultLlmConfig(config)) {
-            // No LLM configured: include existing content as context in the proposal
-            // so the reviewer can do the contradiction resolution manually.
-            resolvedPromotionContent = [
-                promotion.content,
-                "",
-                "---",
-                "<!-- D-1 / #369: Existing knowledge content is shown below for reviewer reference. -->",
-                "<!-- Review: decide whether to ADD (merge), UPDATE (replace), or NOOP (keep existing). -->",
-                "",
-                "## Existing content (for reviewer reference)",
-                "",
-                existingKnowledgeContent,
-            ].join("\n");
-        }
-        // Apply quality gate to fast-path knowledge promotion (Risk 4 fix).
-        // D-5 / #388: Three-band system — review_needed band queues to proposal
-        // queue with review_needed outcome rather than auto-rejecting.
-        let knowledgeJudgeConfidence;
-        if (isLlmFeatureEnabled(config, "lesson_quality_gate")) {
-            // D-4 / #390: retrieve top-3 similar lessons for dedup check in judge.
-            const similarLessons = await fetchSimilarLessonsFn(resolvedPromotionContent.slice(0, 500), 3);
-            const judgeResult = await runLessonQualityJudge(config, resolvedPromotionContent, assetContent ?? "", chat, similarLessons.length > 0 ? similarLessons : undefined);
-            if (!judgeResult.pass) {
-                if (judgeResult.reviewNeeded) {
-                    // Uncertainty band (2.5–3.5): queue as review_needed instead of rejecting.
-                    return writeQualityRejection(stash, inputRef, promotion.knowledgeRef, resolvedPromotionContent, judgeResult.score, judgeResult.reason, { reviewNeeded: true }, options.eligibilitySource);
-                }
-                return writeQualityRejection(stash, inputRef, promotion.knowledgeRef, resolvedPromotionContent, judgeResult.score, judgeResult.reason, {}, options.eligibilitySource);
-            }
-            // Normalize 1-5 judge score to [0, 1]. Score of -1 means pass-through
-            // (no LLM / timeout / parse failure) — leave confidence undefined so
-            // the auto-accept gate treats the proposal as unscored and skips it.
-            if (judgeResult.score > 0)
-                knowledgeJudgeConfidence = judgeResult.score / 5;
-        }
-        const knowledgeParsed = parseFrontmatter(resolvedPromotionContent);
-        const proposalResult = createProposal(stash, {
-            ref: promotion.knowledgeRef,
-            source: "distill",
-            ...(options.sourceRun !== undefined ? { sourceRun: options.sourceRun } : {}),
-            payload: {
-                content: resolvedPromotionContent,
-                ...(Object.keys(knowledgeParsed.data).length > 0 ? { frontmatter: knowledgeParsed.data } : {}),
-            },
-            ...(knowledgeJudgeConfidence !== undefined ? { confidence: knowledgeJudgeConfidence } : {}),
-            // Attribution tagging: persist the eligibility lane on the proposal.
-            ...(options.eligibilitySource ? { eligibilitySource: options.eligibilitySource } : {}),
-        }, options.ctx);
-        if (isProposalSkipped(proposalResult)) {
-            appendEvent({
-                eventType: "distill_invoked",
-                ref: inputRef,
-                metadata: {
-                    outcome: "skipped",
-                    lessonRef: promotion.knowledgeRef,
-                    message: proposalResult.message,
-                    skipReason: proposalResult.reason,
-                    ...eligMeta,
-                },
-            });
-            return {
-                schemaVersion: 1,
-                ok: true,
-                outcome: "skipped",
-                inputRef,
-                lessonRef: promotion.knowledgeRef,
-                message: proposalResult.message,
-            };
-        }
-        const proposal = proposalResult;
-        appendEvent({
-            eventType: "distill_invoked",
-            ref: inputRef,
-            metadata: {
-                outcome: "queued",
-                lessonRef: promotion.knowledgeRef,
-                proposalRef: promotion.knowledgeRef,
-                proposalKind: "knowledge",
-                proposalId: proposal.id,
-                ...(options.sourceRun !== undefined ? { sourceRun: options.sourceRun } : {}),
-                ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
-                ...eligMeta,
-            },
-        });
-        return {
-            schemaVersion: 1,
-            ok: true,
-            outcome: "queued",
-            inputRef,
-            lessonRef: promotion.knowledgeRef,
-            proposalRef: promotion.knowledgeRef,
-            proposalKind: "knowledge",
-            proposalId: proposal.id,
-            proposal,
-            ...(exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
-        };
-    }
+    // Memory→knowledge promotion branch (D-1/#369). When the target ref is a
+    // reinforced memory, distill graduates it into a knowledge proposal instead
+    // of a lesson — the whole branch (LLM contradiction-merge, quality gate,
+    // proposal creation, event emit) lives in `promoteMemoryToKnowledge` and is
+    // terminal when it fires. A `null` return means "not a promotion candidate";
+    // fall through to the ordinary lesson/knowledge distillation path.
+    const promotionResult = await promoteMemoryToKnowledge({
+        targetKind,
+        inputRef,
+        assetContent,
+        filteredEvents,
+        config,
+        chat,
+        stash,
+        lookup,
+        fetchSimilarLessonsFn,
+        existingRefVocabulary,
+        outcomeWeightEnabled,
+        eligMeta,
+        eligibilitySource: options.eligibilitySource,
+        sourceRun: options.sourceRun,
+        proposalsCtx: options.ctx,
+        exclusionSetSize: exclusionSet.size,
+        filteredFeedbackCount,
+        feedbackFullyFiltered,
+    });
+    if (promotionResult)
+        return promotionResult;
     const effectiveProposalKind = targetKind === "knowledge" ? "knowledge" : "lesson";
     const effectiveLessonRef = effectiveProposalKind === "knowledge" ? deriveKnowledgeRef(inputRef) : deriveLessonRef(inputRef);
     // Inject last 1–3 rejected proposals for this ref as Reflexion-style
@@ -909,13 +630,36 @@ export async function akmDistill(options) {
         reason: p.review?.reason ?? "no reason given",
         contentPreview: p.payload.content.slice(0, 500),
     }));
-    const userPrompt = buildDistillPrompt({
+    // WS-3b CLS interleaving (step 9).
+    // When cls.enabled, inject embedding-retrieved adjacent lessons/knowledge
+    // into the distill prompt so the LLM avoids overwriting prior generalizations
+    // (catastrophic interference). DEFAULT OFF.
+    const clsConfig = getImproveProcessConfig(config, "distill", options.improveProfile)?.cls ?? {};
+    let clsContext = "";
+    if (clsConfig.enabled) {
+        try {
+            const adjacentCount = clsConfig.adjacentCount ?? 3;
+            // Use the asset content or input ref as the query for adjacent retrieval.
+            const clsQuery = assetContent ? assetContent.slice(0, 500) : inputRef;
+            const adjacentItems = await fetchSimilarLessonsFn(clsQuery, adjacentCount);
+            clsContext = buildClsContext(adjacentItems, clsConfig);
+        }
+        catch {
+            // Fail open — CLS is supplemental, never required.
+        }
+    }
+    // Distill output is a lesson/knowledge (non-wiki) → stash authoring
+    // standards. Resolved once for this single call.
+    const standardsContext = resolveStashStandards(stash);
+    const baseUserPrompt = buildDistillPrompt({
         inputRef,
         assetContent,
         feedback,
         proposalKind: effectiveProposalKind,
         ...(rejectedForRef.length > 0 ? { rejectedProposals: rejectedForRef } : {}),
+        ...(standardsContext.trim() ? { standardsContext } : {}),
     });
+    const userPrompt = clsContext ? `${baseUserPrompt}${clsContext}` : baseUserPrompt;
     const messages = [
         { role: "system", content: effectiveProposalKind === "knowledge" ? KNOWLEDGE_SYSTEM_PROMPT : LESSON_SYSTEM_PROMPT },
         { role: "user", content: userPrompt },
@@ -1027,136 +771,20 @@ export async function akmDistill(options) {
         // Strip any stray fence the LLM might have added around the markdown.
         content = stripMarkdownFences(raw);
     }
-    // Auto-repair missing frontmatter fields before hard-failing. Small models
-    // frequently produce a good lesson body but omit the YAML header entirely.
-    // Rather than discarding valid content, we extract description/when_to_use
-    // from the body and prepend the required frontmatter block.
-    //
-    // IMPORTANT: We do NOT synthesise placeholder strings here. If the body
-    // does not contain text that passes the post-LLM validators
-    // (`isValidDescription` / `isValidWhenToUse`), we leave the field missing
-    // and let the lesson lint reject the proposal as `validation_failed`.
-    // Emitting placeholders like `"Lesson distilled from <ref>"` or
-    // `"When working with <slug>"` is what produced the systematic broken
-    // proposals observed across 323 archived rejections.
+    // Lesson-path content normalization (see distill/content-repair): auto-repair
+    // missing frontmatter, description↔when_to_use auto-swap, and truncation
+    // repair. Knowledge output skips all three (no lesson frontmatter contract).
     if (effectiveProposalKind !== "knowledge") {
-        const parsed = parseFrontmatter(content);
-        const fm = (parsed.data ?? {});
-        const missingDesc = typeof fm.description !== "string" || !fm.description.trim();
-        const missingWtu = typeof fm.when_to_use !== "string" || !fm.when_to_use.trim();
-        if (missingDesc || missingWtu) {
-            const body = parsed.content.trim();
-            // Strip markdown formatting tokens from a line so extracted text is clean.
-            const stripMd = (l) => l
-                .replace(/\*\*([^*]+)\*\*/g, "$1")
-                .replace(/\*([^*]+)\*/g, "$1")
-                .replace(/`([^`]+)`/g, "$1")
-                .replace(/^[#*\->_]+\s*/, "")
-                .replace(/:\s*$/, "")
-                .trim();
-            // Skip lines that look like YAML field assignments (key: value) or frontmatter delimiters.
-            // These appear when the LLM leaks frontmatter content into the body, causing
-            // auto-repair to produce description: "description: Key Takeaways".
-            const isYamlLike = (l) => /^---/.test(l) || /^[a-z_]+:\s/i.test(l);
-            const bodyLines = body.split("\n").map(stripMd);
-            // Extract description: first body line that BOTH looks like prose AND
-            // passes isValidDescription. If nothing qualifies, leave the field
-            // missing — the lint pass will reject the proposal cleanly.
-            let descLine;
-            for (const l of bodyLines) {
-                if (isYamlLike(l))
-                    continue;
-                if (l.length <= 10 || l.length >= 400)
-                    continue;
-                if (isValidDescription(l, inputRef).ok) {
-                    descLine = l;
-                    break;
-                }
-            }
-            // Extract when_to_use: a line starting with "When" / "Use when" / "Apply when"
-            // that ALSO passes isValidWhenToUse (rejects circular fallbacks).
-            let wtuLine;
-            for (const l of bodyLines) {
-                if (!/^(when |use when|apply when)/i.test(l))
-                    continue;
-                if (l.length >= 400)
-                    continue;
-                if (isValidWhenToUse(l, inputRef).ok) {
-                    wtuLine = l;
-                    break;
-                }
-            }
-            const repairedFm = {
-                ...fm,
-                ...(missingDesc && descLine ? { description: descLine } : {}),
-                ...(missingWtu && wtuLine ? { when_to_use: wtuLine } : {}),
-            };
-            const fmLines = Object.entries(repairedFm)
-                .map(([k, v]) => `${k}: ${JSON.stringify(v)}`)
-                .join("\n");
-            // Only rewrite content if we actually have at least one field to write.
-            // Otherwise leave the original content for the lint pass to reject.
-            if (Object.keys(repairedFm).length > 0) {
-                content = assembleAssetFromString(fmLines, body);
-            }
-        }
+        content = autoRepairLessonFrontmatter(content, inputRef);
     }
-    // Description ↔ when_to_use auto-swap normalization (recover ~93% of
-    // qwen-9b's `^when\b/i` rejections at zero LLM cost). When the LLM emits
-    // a conditional-framed description ("When X happens, do Y") and the
-    // when_to_use field looks like a declarative description (or is empty),
-    // the two fields are mis-fielded — exactly what `isValidDescription`'s
-    // error message says ("that pattern belongs in when_to_use"). We swap
-    // them and revalidate; the swap is committed only if BOTH fields pass
-    // their respective validators afterwards. If revalidation still fails,
-    // we fall through to the existing reject path.
     let descriptionSwapped = 0;
     if (effectiveProposalKind !== "knowledge") {
-        const parsedSwap = parseFrontmatter(content);
-        const fmSwap = (parsedSwap.data ?? {});
-        const descRaw = typeof fmSwap.description === "string" ? fmSwap.description.trim() : "";
-        const wtuRaw = typeof fmSwap.when_to_use === "string" ? fmSwap.when_to_use.trim() : "";
-        const descStartsConditional = /^(when|if)\b/i.test(descRaw);
-        const wtuStartsConditional = /^(when|if)\b/i.test(wtuRaw);
-        if (descStartsConditional && !wtuStartsConditional && wtuRaw.length > 0) {
-            // Try the swap and revalidate. The when_to_use validator requires the
-            // value not match `/^when working with\b/i` (the circular fallback) —
-            // a real description rarely does, so this usually passes.
-            const swappedDescCheck = isValidDescription(wtuRaw, inputRef);
-            const swappedWtuCheck = isValidWhenToUse(descRaw, inputRef);
-            if (swappedDescCheck.ok && swappedWtuCheck.ok) {
-                const swappedFm = {
-                    ...fmSwap,
-                    description: wtuRaw,
-                    when_to_use: descRaw,
-                };
-                const swappedFmLines = Object.entries(swappedFm)
-                    .map(([k, v]) => `${k}: ${JSON.stringify(v)}`)
-                    .join("\n");
-                content = assembleAssetFromString(swappedFmLines, parsedSwap.content);
-                descriptionSwapped = 1;
-            }
-        }
+        const swapResult = autoSwapDescriptionWhenToUse(content, inputRef);
+        content = swapResult.content;
+        descriptionSwapped = swapResult.swapped;
     }
-    // Post-generation truncation repair (#556): if the LLM sliced the
-    // description mid-sentence, deterministically complete it from its own text
-    // / the lesson body BEFORE the lint + quality validators run. No-op
-    // (byte-identical) for already-complete descriptions, so this never alters
-    // a valid proposal. Runs on the lesson path only (knowledge has no
-    // description field gate here).
     if (effectiveProposalKind !== "knowledge") {
-        const parsedRepair = parseFrontmatter(content);
-        const fmRepair = (parsedRepair.data ?? {});
-        const descRepairRaw = typeof fmRepair.description === "string" ? fmRepair.description : "";
-        if (descRepairRaw) {
-            const repaired = repairTruncatedDescription(descRepairRaw, parsedRepair.content);
-            if (repaired !== descRepairRaw) {
-                const repairedFmLines = Object.entries({ ...fmRepair, description: repaired })
-                    .map(([k, v]) => `${k}: ${JSON.stringify(v)}`)
-                    .join("\n");
-                content = assembleAssetFromString(repairedFmLines, parsedRepair.content);
-            }
-        }
+        content = repairLessonDescriptionTruncation(content);
     }
     // Parse + lint the lesson before creating the proposal. The lint is the
     // canonical gate for required frontmatter (v1 spec §13). On failure we
@@ -1165,49 +793,10 @@ export async function akmDistill(options) {
     const findings = effectiveProposalKind === "knowledge"
         ? validateKnowledgeContent(content, inputRef)
         : lintLessonContent(content, `distill:${inputRef}`).findings;
-    // Additional quality validators run only on lessons. lesson-lint checks
-    // "field is present and non-empty"; these reject the systematic failure
-    // modes observed across 323 archived rejected proposals:
-    //   - description is a body fragment, section heading, or placeholder
-    //   - when_to_use is the circular "When working with <ref>" fallback
-    //   - description == when_to_use (LLM duplicated a single sentence)
-    //   - body contains a second pseudo-frontmatter block
+    // Additional lesson-only quality validators — reject the systematic failure
+    // modes seen across 323 archived rejected proposals (see distill/content-repair).
     if (effectiveProposalKind !== "knowledge" && findings.length === 0) {
-        const parsedQC = parseFrontmatter(content);
-        const fmQC = (parsedQC.data ?? {});
-        const descCheck = isValidDescription(fmQC.description, inputRef);
-        if (!descCheck.ok) {
-            findings.push({
-                kind: "invalid-description",
-                field: "description",
-                message: `Distilled lesson for ${inputRef} has an invalid description: ${descCheck.reason}.`,
-            });
-        }
-        const wtuCheck = isValidWhenToUse(fmQC.when_to_use, inputRef);
-        if (!wtuCheck.ok) {
-            findings.push({
-                kind: "invalid-when_to_use",
-                field: "when_to_use",
-                message: `Distilled lesson for ${inputRef} has an invalid when_to_use: ${wtuCheck.reason}.`,
-            });
-        }
-        // description and when_to_use must say different things.
-        if (descCheck.ok &&
-            wtuCheck.ok &&
-            typeof fmQC.description === "string" &&
-            typeof fmQC.when_to_use === "string" &&
-            fmQC.description.trim().toLowerCase() === fmQC.when_to_use.trim().toLowerCase()) {
-            findings.push({
-                kind: "description-equals-when_to_use",
-                field: "description",
-                message: `Distilled lesson for ${inputRef} has identical description and when_to_use.`,
-            });
-        }
-        // Double-frontmatter / pseudo-frontmatter pollution in the body.
-        const dfm = detectDoubleFrontmatter(content);
-        if (dfm) {
-            findings.push({ kind: dfm.kind, field: "body", message: `Distilled lesson for ${inputRef}: ${dfm.message}` });
-        }
+        findings.push(...collectLessonQualityFindings(content, inputRef));
     }
     if (findings.length > 0) {
         appendEvent({
@@ -1228,7 +817,8 @@ export async function akmDistill(options) {
             : "Lessons require non-empty `description` and `when_to_use` frontmatter fields. See v1 spec §13.");
     }
     // LLM-as-judge quality gate (P2-B). Only active when the feature flag is
-    // explicitly enabled. Fail-open: judge failures always pass through.
+    // explicitly enabled. Fail-CLOSED (07 P0-2): an unjudgeable proposal (no LLM
+    // / timeout / parse failure) is rejected, not passed through.
     // D-5 / #388: Three-band system — review_needed band queues a proposal
     // with review_needed outcome rather than auto-rejecting.
     let lessonJudgeConfidence;
@@ -1245,12 +835,38 @@ export async function akmDistill(options) {
             }
             return writeQualityRejection(stash, inputRef, effectiveLessonRef, content, judgeResult.score, judgeResult.reason, exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}, options.eligibilitySource);
         }
-        // Normalize 1-5 judge score to [0, 1]. Score of -1 means pass-through
-        // (no LLM / timeout / parse failure) — leave confidence undefined so
-        // the auto-accept gate treats the proposal as unscored and skips it.
+        // Normalize 1-5 judge score to [0, 1]. Only a real passing verdict
+        // reaches here (07 P0-2: the judge now fails CLOSED on no-LLM / timeout /
+        // parse failure, so those return pass:false and never fall through to
+        // this line). A defensive score>0 guard keeps confidence undefined for any
+        // non-positive score the auto-accept gate should treat as unscored.
         if (judgeResult.score > 0)
             lessonJudgeConfidence = judgeResult.score / 5;
     }
+    // WS-3b: Distill→source fidelity check (step 10).
+    // When fidelityCheck.enabled, check the distill proposal against its cited
+    // source memories. A contradiction flag routes to human review (not auto-accept).
+    // DEFAULT OFF. Fail-open: any error is treated as no-contradiction.
+    const fidelityConfig = getImproveProcessConfig(config, "distill", options.improveProfile)?.fidelityCheck ?? {};
+    if (fidelityConfig.enabled && assetContent) {
+        try {
+            const proposalBody = stripBodyForFidelity(content);
+            const sourceBodies = [stripBodyForFidelity(assetContent)];
+            const fidelityResult = checkDistillFidelity(proposalBody, sourceBodies, fidelityConfig);
+            if (fidelityResult.contradictionDetected) {
+                // Route to human review by writing a quality rejection with reviewNeeded=true.
+                return writeQualityRejection(stash, inputRef, effectiveLessonRef, content, 2.0, // below auto-accept threshold, signals review needed
+                fidelityResult.reason ?? "Proposal may contradict cited source memories.", {
+                    reviewNeeded: true,
+                    fidelityContradiction: true,
+                    ...(exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
+                }, options.eligibilitySource);
+            }
+        }
+        catch {
+            // Fail open — fidelity check is supplemental.
+        }
+    }
     // Round-trip the parsed frontmatter so the proposal carries it as a
     // structured payload alongside the raw content (matches the shape used by
     // other proposal sources).
@@ -1298,6 +914,10 @@ export async function akmDistill(options) {
         };
     }
     const proposal2 = proposalResult2;
+    // G4: content-score the distilled OUTPUT so it carries a real encoding
+    // salience (encoding_source='content') from creation — lessons never get
+    // another chance (they are refused as distill inputs).
+    persistOutputEncodingSalience(effectiveLessonRef, content, existingRefVocabulary, outcomeWeightEnabled);
     appendEvent({
         eventType: "distill_invoked",
         ref: inputRef,
@@ -1307,6 +927,9 @@ export async function akmDistill(options) {
             proposalRef: effectiveLessonRef,
             proposalKind: effectiveProposalKind,
             proposalId: proposal2.id,
+            // R3: judge verdicts are longitudinally queryable, not just a one-shot
+            // proposal.confidence write (normalized 1–5 score / 5).
+            ...(lessonJudgeConfidence !== undefined ? { judgeConfidence: lessonJudgeConfidence } : {}),
             ...(options.sourceRun !== undefined ? { sourceRun: options.sourceRun } : {}),
             ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
             ...(descriptionSwapped > 0 ? { descriptionSwapped } : {}),