akm-cli 0.9.0-beta.5 → 0.9.0-beta.51

package/dist/commands/improve/distill.js

@@ -52,22 +52,33 @@
  */
 import fs from "node:fs";
 import path from "node:path";
+import distillKnowledgeSystemPrompt from "../../assets/prompts/distill-knowledge-system.md" with { type: "text" };
+import distillLessonSystemPrompt from "../../assets/prompts/distill-lesson-system.md" with { type: "text" };
 import { parseAssetRef } from "../../core/asset/asset-ref.js";
 import { assembleAssetFromString } from "../../core/asset/asset-serialize.js";
-import { parseFrontmatter } from "../../core/asset/frontmatter.js";
+import { parseFrontmatter, writeSalienceToFrontmatter } from "../../core/asset/frontmatter.js";
 import { stripMarkdownFences } from "../../core/asset/markdown.js";
+import { authoringRulesForType } from "../../core/authoring-rules.js";
 import { resolveStashDir, timestampForFilename } from "../../core/common.js";
 import { getDefaultLlmConfig, loadConfig } from "../../core/config/config.js";
 import { ConfigError, UsageError } from "../../core/errors.js";
 import { appendEvent, readEvents } from "../../core/events.js";
 import { lintLessonContent } from "../../core/lesson-lint.js";
+import { getDbPath } from "../../core/paths.js";
+import { resolveStashStandards } from "../../core/standards/resolve-stash-standards.js";
+import { withStateDb } from "../../core/state-db.js";
 import { warnVerbose } from "../../core/warn.js";
+import { closeDatabase, getAllEntries, openIndexDatabase } from "../../indexer/db/db.js";
 import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
 import { chatCompletion, parseEmbeddedJsonResponse } from "../../llm/client.js";
 import { isLlmFeatureEnabled, tryLlmFeature } from "../../llm/feature-gate.js";
 import { createProposal, isProposalSkipped, listProposals, } from "../proposal/validators/proposals.js";
 import { akmSearch } from "../read/search.js";
+import { stripFrontmatterBody as stripBodyForFidelity } from "./dedup.js";
 import { assessMemoryKnowledgePromotionCandidate, deriveKnowledgeRef } from "./distill-promotion-policy.js";
+import { buildRefVocabulary, scoreEncodingSalience } from "./encoding-salience.js";
+import { buildClsContext, checkDistillFidelity } from "./homeostatic.js";
+import { computeSalience, upsertAssetSalience } from "./salience.js";
 /**
  * Asset-ref types that `akm distill` structurally refuses as inputs.
  *
@@ -122,68 +133,8 @@ import { repairTruncatedDescription } from "../../core/text-truncation.js";
 import { detectDoubleFrontmatter, isValidDescription, isValidWhenToUse, } from "../proposal/validators/proposal-quality-validators.js";
 export { detectDoubleFrontmatter, isValidDescription, isValidWhenToUse };
 // ── Prompt assembly ─────────────────────────────────────────────────────────
-const LESSON_SYSTEM_PROMPT = [
-    "You are the akm `distill` distiller.",
-    "Given an asset and recent feedback events about it, produce a single",
-    "concise *lesson* an agent should remember next time it works on this",
-    "asset's domain.",
-    "",
-    "YOUR RESPONSE MUST START EXACTLY WITH `---` ON THE VERY FIRST LINE.",
-    "DO NOT output any prose, explanation, or code fences before or after.",
-    "",
-    "Required output format — copy this structure exactly:",
-    "---",
-    "description: <one complete sentence (ending with `.`) summarising what the lesson teaches>",
-    "when_to_use: <one complete sentence describing the concrete trigger condition>",
-    "---",
-    "",
-    "<lesson body — plain markdown, 1–3 short paragraphs of practical guidance>",
-    "",
-    "## description field (MANDATORY)",
-    "- A single complete sentence in present tense, 80-200 chars, NO markdown.",
-    "- Self-contained: a reviewer must understand the lesson from this field alone.",
-    '- DO NOT start with "When ", "If ", or a connector word — that belongs in when_to_use.',
-    '- DO NOT copy a section heading ("Key takeaways", "For example", "Key pitfalls").',
-    "- DO NOT begin with a numbered list marker, code fence, or markdown heading.",
-    "",
-    'GOOD: "Always validate ref existence before promoting a memory to knowledge; missing refs surface as silent 404s during accept."',
-    'BAD:  "Key pitfalls"',
-    'BAD:  "When working with the akm CLI"',
-    'BAD:  "For example, you might..."',
-    'BAD:  "1. Check the file"',
-    "",
-    "RULES:",
-    "- `when_to_use` MUST be a complete sentence describing a concrete trigger. Never write `When working with <asset-name>` — that is circular and useless.",
-    "- `description` and `when_to_use` MUST differ from each other.",
-    "- The lesson body MUST be non-empty markdown prose. Do NOT restate `description:` or `when_to_use:` inside the body (no `**description:** ...` or `**when_to_use:** ...` lines — the frontmatter is the only place those keys belong).",
-    "- Do NOT emit a second `---` fence after the opening frontmatter — there are exactly two `---` lines in the output, both belonging to the single frontmatter block at the top.",
-    "- Do NOT reproduce the source asset verbatim — distil what a caller needs to know.",
-    "- Output ONLY the lesson file. No preamble, no code fences, no trailing prose.",
-].join("\n");
-const KNOWLEDGE_SYSTEM_PROMPT = [
-    "You are the akm `distill` distiller.",
-    "Given an asset and recent feedback events about it, produce a concise",
-    "*knowledge* markdown document capturing the durable, reusable facts.",
-    "Prefer stable guidance over narrative recap.",
-    "",
-    "YOUR RESPONSE MUST START EXACTLY WITH `---` ON THE VERY FIRST LINE.",
-    "DO NOT output any prose, explanation, or code fences before or after.",
-    "",
-    "Required output format:",
-    "---",
-    "description: <one-line summary of the knowledge asset>",
-    "tags: [<tag1>, <tag2>]",
-    "---",
-    "",
-    "# <Title>",
-    "",
-    "<body — structured markdown, durable facts only>",
-    "",
-    "RULES:",
-    "- `description` MUST be a non-empty single-line string.",
-    "- Include a meaningful markdown body with a `# Title` heading.",
-    "- Output ONLY the knowledge file. No preamble, no code fences, no trailing prose.",
-].join("\n");
+const LESSON_SYSTEM_PROMPT = distillLessonSystemPrompt;
+const KNOWLEDGE_SYSTEM_PROMPT = distillKnowledgeSystemPrompt;
 // ── Structured-output schemas (responseSchema lift) ─────────────────────────
 //
 // PR 1 of the asset-writers decision (see knowledge:projects/akm/
@@ -358,6 +309,18 @@ export function buildDistillPrompt(input) {
     const lines = [];
     lines.push(`Asset ref: ${input.inputRef}`);
     lines.push("");
+    if (input.standardsContext?.trim()) {
+        lines.push("Standards to follow (the rulebook for this target):");
+        lines.push(input.standardsContext.trim());
+        lines.push("");
+    }
+    {
+        const authoringRules = authoringRulesForType(input.proposalKind ?? "lesson");
+        if (authoringRules) {
+            lines.push(authoringRules);
+            lines.push("");
+        }
+    }
     lines.push("Asset content:");
     if (input.assetContent) {
         const body = input.assetContent.trim().slice(0, 3000);
@@ -586,7 +549,7 @@ similarLessons) {
  * @param reason    - Human-readable rejection reason.
  * @param extraMeta - Optional additional metadata for the event.
  */
-function writeQualityRejection(stash, inputRef, lessonRef, content, score, reason, extraMeta = {}) {
+function writeQualityRejection(stash, inputRef, lessonRef, content, score, reason, extraMeta = {}, eligibilitySource) {
     // D-5 / #388: reviewNeeded flag selects "review_needed" vs "quality_rejected" outcome.
     const outcome = extraMeta.reviewNeeded ? "review_needed" : "quality_rejected";
     const rejectDir = path.join(stash, ".akm", "distill-rejected");
@@ -602,6 +565,9 @@ function writeQualityRejection(stash, inputRef, lessonRef, content, score, reaso
             score,
             reason,
             ...extraMeta,
+            // Attribution tagging: stamp the eligibility lane so distill_invoked can be
+            // sliced by lane downstream. See EligibilitySource.
+            ...(eligibilitySource ? { eligibilitySource } : {}),
         },
     });
     return {
@@ -629,6 +595,12 @@ export async function akmDistill(options) {
     // Validate the ref shape up front so a typo never reaches the LLM.
     const parsedInputRef = parseAssetRef(inputRef);
     const targetKind = options.proposalKind ?? "lesson";
+    // Attribution tagging: spread into every distill_invoked event's metadata so
+    // the lane that selected this asset is recorded uniformly across all outcome
+    // branches. Empty object when no lane was supplied (direct `akm distill`).
+    const eligMeta = options.eligibilitySource
+        ? { eligibilitySource: options.eligibilitySource }
+        : {};
     // Recursive-distillation guard. Distill produces *lessons* from non-lesson
     // sources (memory, skill, knowledge, etc.). Calling distill on an existing
     // lesson would derive `lesson:lesson-<name>-lesson-lesson` (double `-lesson`
@@ -650,6 +622,7 @@ export async function akmDistill(options) {
                 lessonRef: skippedRef,
                 message: "distill refuses lesson inputs — lessons are the distilled form, not a source",
                 skipReason: "recursive_lesson_input",
+                ...eligMeta,
             },
         });
         return {
@@ -671,15 +644,74 @@ export async function akmDistill(options) {
     // Best-effort load: when the asset is not yet indexed we still proceed —
     // the LLM is asked to distil from "available signal" (feedback alone).
     let assetContent = null;
+    let assetFilePath = null;
     try {
         const filePath = await lookup(inputRef);
         if (filePath && fs.existsSync(filePath)) {
+            assetFilePath = filePath;
             assetContent = fs.readFileSync(filePath, "utf8");
         }
     }
     catch {
         assetContent = null;
     }
+    // ── #608: Encoding-time salience scoring ────────────────────────────────
+    // Score the source asset with the three-signal model (novelty × 0.40 +
+    // magnitude × 0.35 + predictionError × 0.25) and persist the result to:
+    //   1. The asset's frontmatter (human-readable mirror; idempotent delta gate).
+    //   2. state.db :: asset_salience (canonical; feeds improve's high-salience gate).
+    // Both writes are best-effort — a DB error never blocks distillation.
+    if (assetContent && assetFilePath) {
+        try {
+            const parsedRef = parseAssetRef(inputRef);
+            // Build bigram vocabulary from currently-indexed refs for novelty signal.
+            let existingRefVocabulary = new Set();
+            try {
+                const embCfg = config?.embedding;
+                const indexDb = openIndexDatabase(getDbPath(), embCfg?.dimension ? { embeddingDim: embCfg.dimension } : undefined);
+                try {
+                    const allRefs = getAllEntries(indexDb).map((e) => e.entryKey);
+                    existingRefVocabulary = buildRefVocabulary(allRefs);
+                }
+                finally {
+                    closeDatabase(indexDb);
+                }
+            }
+            catch {
+                // Index not available — novelty defaults to type-floor.
+            }
+            const salienceResult = scoreEncodingSalience({
+                body: assetContent,
+                type: parsedRef.type,
+                existingRefVocabulary,
+                revisionCount: 0,
+            });
+            // 1. Write salience to the source asset frontmatter (idempotent).
+            const updatedContent = writeSalienceToFrontmatter(assetContent, salienceResult.score, salienceResult);
+            if (updatedContent !== assetContent) {
+                fs.writeFileSync(assetFilePath, updatedContent, "utf8");
+                assetContent = updatedContent;
+            }
+            // 2. Persist encoding_salience to state.db.
+            try {
+                withStateDb((stateDb) => {
+                    const vector = computeSalience({
+                        ref: inputRef,
+                        type: parsedRef.type,
+                        retrievalFreq: 0,
+                        encodingSalience: salienceResult.score,
+                    });
+                    upsertAssetSalience(stateDb, inputRef, vector);
+                });
+            }
+            catch {
+                // State DB unavailable — frontmatter mirror is the only persistence.
+            }
+        }
+        catch {
+            // Scoring errors never block distillation.
+        }
+    }
     const { events } = readEventsImpl({
         ref: inputRef,
         type: "feedback",
@@ -766,6 +798,7 @@ export async function akmDistill(options) {
                             outcome: "skipped",
                             lessonRef: promotion.knowledgeRef,
                             message: "D-1: LLM resolved destination conflict as NOOP — existing content kept",
+                            ...eligMeta,
                         },
                     });
                     return {
@@ -814,9 +847,9 @@ export async function akmDistill(options) {
             if (!judgeResult.pass) {
                 if (judgeResult.reviewNeeded) {
                     // Uncertainty band (2.5–3.5): queue as review_needed instead of rejecting.
-                    return writeQualityRejection(stash, inputRef, promotion.knowledgeRef, resolvedPromotionContent, judgeResult.score, judgeResult.reason, { reviewNeeded: true });
+                    return writeQualityRejection(stash, inputRef, promotion.knowledgeRef, resolvedPromotionContent, judgeResult.score, judgeResult.reason, { reviewNeeded: true }, options.eligibilitySource);
                 }
-                return writeQualityRejection(stash, inputRef, promotion.knowledgeRef, resolvedPromotionContent, judgeResult.score, judgeResult.reason);
+                return writeQualityRejection(stash, inputRef, promotion.knowledgeRef, resolvedPromotionContent, judgeResult.score, judgeResult.reason, {}, options.eligibilitySource);
             }
             // Normalize 1-5 judge score to [0, 1]. Score of -1 means pass-through
             // (no LLM / timeout / parse failure) — leave confidence undefined so
@@ -834,6 +867,8 @@ export async function akmDistill(options) {
                 ...(Object.keys(knowledgeParsed.data).length > 0 ? { frontmatter: knowledgeParsed.data } : {}),
             },
             ...(knowledgeJudgeConfidence !== undefined ? { confidence: knowledgeJudgeConfidence } : {}),
+            // Attribution tagging: persist the eligibility lane on the proposal.
+            ...(options.eligibilitySource ? { eligibilitySource: options.eligibilitySource } : {}),
         }, options.ctx);
         if (isProposalSkipped(proposalResult)) {
             appendEvent({
@@ -844,6 +879,7 @@ export async function akmDistill(options) {
                     lessonRef: promotion.knowledgeRef,
                     message: proposalResult.message,
                     skipReason: proposalResult.reason,
+                    ...eligMeta,
                 },
             });
             return {
@@ -867,6 +903,7 @@ export async function akmDistill(options) {
                 proposalId: proposal.id,
                 ...(options.sourceRun !== undefined ? { sourceRun: options.sourceRun } : {}),
                 ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
+                ...eligMeta,
             },
         });
         return {
@@ -894,13 +931,36 @@ export async function akmDistill(options) {
         reason: p.review?.reason ?? "no reason given",
         contentPreview: p.payload.content.slice(0, 500),
     }));
-    const userPrompt = buildDistillPrompt({
+    // WS-3b CLS interleaving (step 9).
+    // When cls.enabled, inject embedding-retrieved adjacent lessons/knowledge
+    // into the distill prompt so the LLM avoids overwriting prior generalizations
+    // (catastrophic interference). DEFAULT OFF.
+    const clsConfig = config.profiles?.improve?.default?.processes?.distill?.cls ?? {};
+    let clsContext = "";
+    if (clsConfig.enabled) {
+        try {
+            const adjacentCount = clsConfig.adjacentCount ?? 3;
+            // Use the asset content or input ref as the query for adjacent retrieval.
+            const clsQuery = assetContent ? assetContent.slice(0, 500) : inputRef;
+            const adjacentItems = await fetchSimilarLessonsFn(clsQuery, adjacentCount);
+            clsContext = buildClsContext(adjacentItems, clsConfig);
+        }
+        catch {
+            // Fail open — CLS is supplemental, never required.
+        }
+    }
+    // Distill output is a lesson/knowledge (non-wiki) → stash authoring
+    // standards. Resolved once for this single call.
+    const standardsContext = resolveStashStandards(stash);
+    const baseUserPrompt = buildDistillPrompt({
         inputRef,
         assetContent,
         feedback,
         proposalKind: effectiveProposalKind,
         ...(rejectedForRef.length > 0 ? { rejectedProposals: rejectedForRef } : {}),
+        ...(standardsContext.trim() ? { standardsContext } : {}),
     });
+    const userPrompt = clsContext ? `${baseUserPrompt}${clsContext}` : baseUserPrompt;
     const messages = [
         { role: "system", content: effectiveProposalKind === "knowledge" ? KNOWLEDGE_SYSTEM_PROMPT : LESSON_SYSTEM_PROMPT },
         { role: "user", content: userPrompt },
@@ -979,6 +1039,7 @@ export async function akmDistill(options) {
                 lessonRef: effectiveLessonRef,
                 proposalKind: effectiveProposalKind,
                 ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
+                ...eligMeta,
             },
         });
         return {
@@ -1203,6 +1264,7 @@ export async function akmDistill(options) {
                 proposalKind: effectiveProposalKind,
                 findingKinds: findings.map((f) => f.kind),
                 ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
+                ...eligMeta,
             },
         });
         const message = findings.map((f) => f.message).join("\n");
@@ -1224,9 +1286,9 @@ export async function akmDistill(options) {
                 return writeQualityRejection(stash, inputRef, effectiveLessonRef, content, judgeResult.score, judgeResult.reason, {
                     reviewNeeded: true,
                     ...(exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
-                });
+                }, options.eligibilitySource);
             }
-            return writeQualityRejection(stash, inputRef, effectiveLessonRef, content, judgeResult.score, judgeResult.reason, exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {});
+            return writeQualityRejection(stash, inputRef, effectiveLessonRef, content, judgeResult.score, judgeResult.reason, exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}, options.eligibilitySource);
         }
         // Normalize 1-5 judge score to [0, 1]. Score of -1 means pass-through
         // (no LLM / timeout / parse failure) — leave confidence undefined so
@@ -1234,6 +1296,30 @@ export async function akmDistill(options) {
         if (judgeResult.score > 0)
             lessonJudgeConfidence = judgeResult.score / 5;
     }
+    // WS-3b: Distill→source fidelity check (step 10).
+    // When fidelityCheck.enabled, check the distill proposal against its cited
+    // source memories. A contradiction flag routes to human review (not auto-accept).
+    // DEFAULT OFF. Fail-open: any error is treated as no-contradiction.
+    const fidelityConfig = config.profiles?.improve?.default?.processes?.distill?.fidelityCheck ?? {};
+    if (fidelityConfig.enabled && assetContent) {
+        try {
+            const proposalBody = stripBodyForFidelity(content);
+            const sourceBodies = [stripBodyForFidelity(assetContent)];
+            const fidelityResult = checkDistillFidelity(proposalBody, sourceBodies, fidelityConfig);
+            if (fidelityResult.contradictionDetected) {
+                // Route to human review by writing a quality rejection with reviewNeeded=true.
+                return writeQualityRejection(stash, inputRef, effectiveLessonRef, content, 2.0, // below auto-accept threshold, signals review needed
+                fidelityResult.reason ?? "Proposal may contradict cited source memories.", {
+                    reviewNeeded: true,
+                    fidelityContradiction: true,
+                    ...(exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
+                }, options.eligibilitySource);
+            }
+        }
+        catch {
+            // Fail open — fidelity check is supplemental.
+        }
+    }
     // Round-trip the parsed frontmatter so the proposal carries it as a
     // structured payload alongside the raw content (matches the shape used by
     // other proposal sources).
@@ -1256,6 +1342,8 @@ export async function akmDistill(options) {
             frontmatter: frontmatterWithSources,
         },
         ...(lessonJudgeConfidence !== undefined ? { confidence: lessonJudgeConfidence } : {}),
+        // Attribution tagging: persist the eligibility lane on the proposal.
+        ...(options.eligibilitySource ? { eligibilitySource: options.eligibilitySource } : {}),
     }, options.ctx);
     if (isProposalSkipped(proposalResult2)) {
         appendEvent({
@@ -1266,6 +1354,7 @@ export async function akmDistill(options) {
                 lessonRef: effectiveLessonRef,
                 message: proposalResult2.message,
                 skipReason: proposalResult2.reason,
+                ...eligMeta,
             },
         });
         return {
@@ -1290,6 +1379,7 @@ export async function akmDistill(options) {
             ...(options.sourceRun !== undefined ? { sourceRun: options.sourceRun } : {}),
             ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
             ...(descriptionSwapped > 0 ? { descriptionSwapped } : {}),
+            ...eligMeta,
         },
     });
     return {