akm-cli 0.9.0-beta.4 → 0.9.0-beta.41

package/dist/commands/improve/distill.js

@@ -52,22 +52,33 @@
  */
 import fs from "node:fs";
 import path from "node:path";
+import distillKnowledgeSystemPrompt from "../../assets/prompts/distill-knowledge-system.md" with { type: "text" };
+import distillLessonSystemPrompt from "../../assets/prompts/distill-lesson-system.md" with { type: "text" };
 import { parseAssetRef } from "../../core/asset/asset-ref.js";
 import { assembleAssetFromString } from "../../core/asset/asset-serialize.js";
-import { parseFrontmatter } from "../../core/asset/frontmatter.js";
+import { parseFrontmatter, writeSalienceToFrontmatter } from "../../core/asset/frontmatter.js";
 import { stripMarkdownFences } from "../../core/asset/markdown.js";
+import { authoringRulesForType } from "../../core/authoring-rules.js";
 import { resolveStashDir, timestampForFilename } from "../../core/common.js";
 import { getDefaultLlmConfig, loadConfig } from "../../core/config/config.js";
 import { ConfigError, UsageError } from "../../core/errors.js";
 import { appendEvent, readEvents } from "../../core/events.js";
 import { lintLessonContent } from "../../core/lesson-lint.js";
+import { getDbPath } from "../../core/paths.js";
+import { resolveStashStandards } from "../../core/standards/resolve-stash-standards.js";
+import { openStateDatabase } from "../../core/state-db.js";
 import { warnVerbose } from "../../core/warn.js";
+import { closeDatabase, getAllEntries, openDatabase } from "../../indexer/db/db.js";
 import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
 import { chatCompletion, parseEmbeddedJsonResponse } from "../../llm/client.js";
 import { isLlmFeatureEnabled, tryLlmFeature } from "../../llm/feature-gate.js";
 import { createProposal, isProposalSkipped, listProposals, } from "../proposal/validators/proposals.js";
 import { akmSearch } from "../read/search.js";
+import { stripFrontmatterBody as stripBodyForFidelity } from "./dedup.js";
 import { assessMemoryKnowledgePromotionCandidate, deriveKnowledgeRef } from "./distill-promotion-policy.js";
+import { buildRefVocabulary, scoreEncodingSalience } from "./encoding-salience.js";
+import { buildClsContext, checkDistillFidelity } from "./homeostatic.js";
+import { computeSalience, upsertAssetSalience } from "./salience.js";
 /**
  * Asset-ref types that `akm distill` structurally refuses as inputs.
  *
@@ -122,68 +133,8 @@ import { repairTruncatedDescription } from "../../core/text-truncation.js";
 import { detectDoubleFrontmatter, isValidDescription, isValidWhenToUse, } from "../proposal/validators/proposal-quality-validators.js";
 export { detectDoubleFrontmatter, isValidDescription, isValidWhenToUse };
 // ── Prompt assembly ─────────────────────────────────────────────────────────
-const LESSON_SYSTEM_PROMPT = [
-    "You are the akm `distill` distiller.",
-    "Given an asset and recent feedback events about it, produce a single",
-    "concise *lesson* an agent should remember next time it works on this",
-    "asset's domain.",
-    "",
-    "YOUR RESPONSE MUST START EXACTLY WITH `---` ON THE VERY FIRST LINE.",
-    "DO NOT output any prose, explanation, or code fences before or after.",
-    "",
-    "Required output format — copy this structure exactly:",
-    "---",
-    "description: <one complete sentence (ending with `.`) summarising what the lesson teaches>",
-    "when_to_use: <one complete sentence describing the concrete trigger condition>",
-    "---",
-    "",
-    "<lesson body — plain markdown, 1–3 short paragraphs of practical guidance>",
-    "",
-    "## description field (MANDATORY)",
-    "- A single complete sentence in present tense, 80-200 chars, NO markdown.",
-    "- Self-contained: a reviewer must understand the lesson from this field alone.",
-    '- DO NOT start with "When ", "If ", or a connector word — that belongs in when_to_use.',
-    '- DO NOT copy a section heading ("Key takeaways", "For example", "Key pitfalls").',
-    "- DO NOT begin with a numbered list marker, code fence, or markdown heading.",
-    "",
-    'GOOD: "Always validate ref existence before promoting a memory to knowledge; missing refs surface as silent 404s during accept."',
-    'BAD:  "Key pitfalls"',
-    'BAD:  "When working with the akm CLI"',
-    'BAD:  "For example, you might..."',
-    'BAD:  "1. Check the file"',
-    "",
-    "RULES:",
-    "- `when_to_use` MUST be a complete sentence describing a concrete trigger. Never write `When working with <asset-name>` — that is circular and useless.",
-    "- `description` and `when_to_use` MUST differ from each other.",
-    "- The lesson body MUST be non-empty markdown prose. Do NOT restate `description:` or `when_to_use:` inside the body (no `**description:** ...` or `**when_to_use:** ...` lines — the frontmatter is the only place those keys belong).",
-    "- Do NOT emit a second `---` fence after the opening frontmatter — there are exactly two `---` lines in the output, both belonging to the single frontmatter block at the top.",
-    "- Do NOT reproduce the source asset verbatim — distil what a caller needs to know.",
-    "- Output ONLY the lesson file. No preamble, no code fences, no trailing prose.",
-].join("\n");
-const KNOWLEDGE_SYSTEM_PROMPT = [
-    "You are the akm `distill` distiller.",
-    "Given an asset and recent feedback events about it, produce a concise",
-    "*knowledge* markdown document capturing the durable, reusable facts.",
-    "Prefer stable guidance over narrative recap.",
-    "",
-    "YOUR RESPONSE MUST START EXACTLY WITH `---` ON THE VERY FIRST LINE.",
-    "DO NOT output any prose, explanation, or code fences before or after.",
-    "",
-    "Required output format:",
-    "---",
-    "description: <one-line summary of the knowledge asset>",
-    "tags: [<tag1>, <tag2>]",
-    "---",
-    "",
-    "# <Title>",
-    "",
-    "<body — structured markdown, durable facts only>",
-    "",
-    "RULES:",
-    "- `description` MUST be a non-empty single-line string.",
-    "- Include a meaningful markdown body with a `# Title` heading.",
-    "- Output ONLY the knowledge file. No preamble, no code fences, no trailing prose.",
-].join("\n");
+const LESSON_SYSTEM_PROMPT = distillLessonSystemPrompt;
+const KNOWLEDGE_SYSTEM_PROMPT = distillKnowledgeSystemPrompt;
 // ── Structured-output schemas (responseSchema lift) ─────────────────────────
 //
 // PR 1 of the asset-writers decision (see knowledge:projects/akm/
@@ -358,6 +309,18 @@ export function buildDistillPrompt(input) {
     const lines = [];
     lines.push(`Asset ref: ${input.inputRef}`);
     lines.push("");
+    if (input.standardsContext?.trim()) {
+        lines.push("Standards to follow (the rulebook for this target):");
+        lines.push(input.standardsContext.trim());
+        lines.push("");
+    }
+    {
+        const authoringRules = authoringRulesForType(input.proposalKind ?? "lesson");
+        if (authoringRules) {
+            lines.push(authoringRules);
+            lines.push("");
+        }
+    }
     lines.push("Asset content:");
     if (input.assetContent) {
         const body = input.assetContent.trim().slice(0, 3000);
@@ -586,7 +549,7 @@ similarLessons) {
  * @param reason    - Human-readable rejection reason.
  * @param extraMeta - Optional additional metadata for the event.
  */
-function writeQualityRejection(stash, inputRef, lessonRef, content, score, reason, extraMeta = {}) {
+function writeQualityRejection(stash, inputRef, lessonRef, content, score, reason, extraMeta = {}, eligibilitySource) {
     // D-5 / #388: reviewNeeded flag selects "review_needed" vs "quality_rejected" outcome.
     const outcome = extraMeta.reviewNeeded ? "review_needed" : "quality_rejected";
     const rejectDir = path.join(stash, ".akm", "distill-rejected");
@@ -602,6 +565,9 @@ function writeQualityRejection(stash, inputRef, lessonRef, content, score, reaso
             score,
             reason,
             ...extraMeta,
+            // Attribution tagging: stamp the eligibility lane so distill_invoked can be
+            // sliced by lane downstream. See EligibilitySource.
+            ...(eligibilitySource ? { eligibilitySource } : {}),
         },
     });
     return {
@@ -629,6 +595,12 @@ export async function akmDistill(options) {
     // Validate the ref shape up front so a typo never reaches the LLM.
     const parsedInputRef = parseAssetRef(inputRef);
     const targetKind = options.proposalKind ?? "lesson";
+    // Attribution tagging: spread into every distill_invoked event's metadata so
+    // the lane that selected this asset is recorded uniformly across all outcome
+    // branches. Empty object when no lane was supplied (direct `akm distill`).
+    const eligMeta = options.eligibilitySource
+        ? { eligibilitySource: options.eligibilitySource }
+        : {};
     // Recursive-distillation guard. Distill produces *lessons* from non-lesson
     // sources (memory, skill, knowledge, etc.). Calling distill on an existing
     // lesson would derive `lesson:lesson-<name>-lesson-lesson` (double `-lesson`
@@ -650,6 +622,7 @@ export async function akmDistill(options) {
                 lessonRef: skippedRef,
                 message: "distill refuses lesson inputs — lessons are the distilled form, not a source",
                 skipReason: "recursive_lesson_input",
+                ...eligMeta,
             },
         });
         return {
@@ -671,15 +644,78 @@ export async function akmDistill(options) {
     // Best-effort load: when the asset is not yet indexed we still proceed —
     // the LLM is asked to distil from "available signal" (feedback alone).
     let assetContent = null;
+    let assetFilePath = null;
     try {
         const filePath = await lookup(inputRef);
         if (filePath && fs.existsSync(filePath)) {
+            assetFilePath = filePath;
             assetContent = fs.readFileSync(filePath, "utf8");
         }
     }
     catch {
         assetContent = null;
     }
+    // ── #608: Encoding-time salience scoring ────────────────────────────────
+    // Score the source asset with the three-signal model (novelty × 0.40 +
+    // magnitude × 0.35 + predictionError × 0.25) and persist the result to:
+    //   1. The asset's frontmatter (human-readable mirror; idempotent delta gate).
+    //   2. state.db :: asset_salience (canonical; feeds improve's high-salience gate).
+    // Both writes are best-effort — a DB error never blocks distillation.
+    if (assetContent && assetFilePath) {
+        try {
+            const parsedRef = parseAssetRef(inputRef);
+            // Build bigram vocabulary from currently-indexed refs for novelty signal.
+            let existingRefVocabulary = new Set();
+            try {
+                const embCfg = config?.embedding;
+                const indexDb = openDatabase(getDbPath(), embCfg?.dimension ? { embeddingDim: embCfg.dimension } : undefined);
+                try {
+                    const allRefs = getAllEntries(indexDb).map((e) => e.entryKey);
+                    existingRefVocabulary = buildRefVocabulary(allRefs);
+                }
+                finally {
+                    closeDatabase(indexDb);
+                }
+            }
+            catch {
+                // Index not available — novelty defaults to type-floor.
+            }
+            const salienceResult = scoreEncodingSalience({
+                body: assetContent,
+                type: parsedRef.type,
+                existingRefVocabulary,
+                revisionCount: 0,
+            });
+            // 1. Write salience to the source asset frontmatter (idempotent).
+            const updatedContent = writeSalienceToFrontmatter(assetContent, salienceResult.score, salienceResult);
+            if (updatedContent !== assetContent) {
+                fs.writeFileSync(assetFilePath, updatedContent, "utf8");
+                assetContent = updatedContent;
+            }
+            // 2. Persist encoding_salience to state.db.
+            try {
+                const stateDb = openStateDatabase();
+                try {
+                    const vector = computeSalience({
+                        ref: inputRef,
+                        type: parsedRef.type,
+                        retrievalFreq: 0,
+                        encodingSalience: salienceResult.score,
+                    });
+                    upsertAssetSalience(stateDb, inputRef, vector);
+                }
+                finally {
+                    stateDb.close();
+                }
+            }
+            catch {
+                // State DB unavailable — frontmatter mirror is the only persistence.
+            }
+        }
+        catch {
+            // Scoring errors never block distillation.
+        }
+    }
     const { events } = readEventsImpl({
         ref: inputRef,
         type: "feedback",
@@ -766,6 +802,7 @@ export async function akmDistill(options) {
                             outcome: "skipped",
                             lessonRef: promotion.knowledgeRef,
                             message: "D-1: LLM resolved destination conflict as NOOP — existing content kept",
+                            ...eligMeta,
                         },
                     });
                     return {
@@ -814,9 +851,9 @@ export async function akmDistill(options) {
             if (!judgeResult.pass) {
                 if (judgeResult.reviewNeeded) {
                     // Uncertainty band (2.5–3.5): queue as review_needed instead of rejecting.
-                    return writeQualityRejection(stash, inputRef, promotion.knowledgeRef, resolvedPromotionContent, judgeResult.score, judgeResult.reason, { reviewNeeded: true });
+                    return writeQualityRejection(stash, inputRef, promotion.knowledgeRef, resolvedPromotionContent, judgeResult.score, judgeResult.reason, { reviewNeeded: true }, options.eligibilitySource);
                 }
-                return writeQualityRejection(stash, inputRef, promotion.knowledgeRef, resolvedPromotionContent, judgeResult.score, judgeResult.reason);
+                return writeQualityRejection(stash, inputRef, promotion.knowledgeRef, resolvedPromotionContent, judgeResult.score, judgeResult.reason, {}, options.eligibilitySource);
             }
             // Normalize 1-5 judge score to [0, 1]. Score of -1 means pass-through
             // (no LLM / timeout / parse failure) — leave confidence undefined so
@@ -834,6 +871,8 @@ export async function akmDistill(options) {
                 ...(Object.keys(knowledgeParsed.data).length > 0 ? { frontmatter: knowledgeParsed.data } : {}),
             },
             ...(knowledgeJudgeConfidence !== undefined ? { confidence: knowledgeJudgeConfidence } : {}),
+            // Attribution tagging: persist the eligibility lane on the proposal.
+            ...(options.eligibilitySource ? { eligibilitySource: options.eligibilitySource } : {}),
         }, options.ctx);
         if (isProposalSkipped(proposalResult)) {
             appendEvent({
@@ -844,6 +883,7 @@ export async function akmDistill(options) {
                     lessonRef: promotion.knowledgeRef,
                     message: proposalResult.message,
                     skipReason: proposalResult.reason,
+                    ...eligMeta,
                 },
             });
             return {
@@ -867,6 +907,7 @@ export async function akmDistill(options) {
                 proposalId: proposal.id,
                 ...(options.sourceRun !== undefined ? { sourceRun: options.sourceRun } : {}),
                 ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
+                ...eligMeta,
             },
         });
         return {
@@ -894,13 +935,36 @@ export async function akmDistill(options) {
         reason: p.review?.reason ?? "no reason given",
         contentPreview: p.payload.content.slice(0, 500),
     }));
-    const userPrompt = buildDistillPrompt({
+    // WS-3b CLS interleaving (step 9).
+    // When cls.enabled, inject embedding-retrieved adjacent lessons/knowledge
+    // into the distill prompt so the LLM avoids overwriting prior generalizations
+    // (catastrophic interference). DEFAULT OFF.
+    const clsConfig = config.profiles?.improve?.default?.processes?.distill?.cls ?? {};
+    let clsContext = "";
+    if (clsConfig.enabled) {
+        try {
+            const adjacentCount = clsConfig.adjacentCount ?? 3;
+            // Use the asset content or input ref as the query for adjacent retrieval.
+            const clsQuery = assetContent ? assetContent.slice(0, 500) : inputRef;
+            const adjacentItems = await fetchSimilarLessonsFn(clsQuery, adjacentCount);
+            clsContext = buildClsContext(adjacentItems, clsConfig);
+        }
+        catch {
+            // Fail open — CLS is supplemental, never required.
+        }
+    }
+    // Distill output is a lesson/knowledge (non-wiki) → stash authoring
+    // standards. Resolved once for this single call.
+    const standardsContext = resolveStashStandards(stash);
+    const baseUserPrompt = buildDistillPrompt({
         inputRef,
         assetContent,
         feedback,
         proposalKind: effectiveProposalKind,
         ...(rejectedForRef.length > 0 ? { rejectedProposals: rejectedForRef } : {}),
+        ...(standardsContext.trim() ? { standardsContext } : {}),
     });
+    const userPrompt = clsContext ? `${baseUserPrompt}${clsContext}` : baseUserPrompt;
     const messages = [
         { role: "system", content: effectiveProposalKind === "knowledge" ? KNOWLEDGE_SYSTEM_PROMPT : LESSON_SYSTEM_PROMPT },
         { role: "user", content: userPrompt },
@@ -979,6 +1043,7 @@ export async function akmDistill(options) {
                 lessonRef: effectiveLessonRef,
                 proposalKind: effectiveProposalKind,
                 ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
+                ...eligMeta,
             },
         });
         return {
@@ -1203,6 +1268,7 @@ export async function akmDistill(options) {
                 proposalKind: effectiveProposalKind,
                 findingKinds: findings.map((f) => f.kind),
                 ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
+                ...eligMeta,
             },
         });
         const message = findings.map((f) => f.message).join("\n");
@@ -1224,9 +1290,9 @@ export async function akmDistill(options) {
                 return writeQualityRejection(stash, inputRef, effectiveLessonRef, content, judgeResult.score, judgeResult.reason, {
                     reviewNeeded: true,
                     ...(exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
-                });
+                }, options.eligibilitySource);
             }
-            return writeQualityRejection(stash, inputRef, effectiveLessonRef, content, judgeResult.score, judgeResult.reason, exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {});
+            return writeQualityRejection(stash, inputRef, effectiveLessonRef, content, judgeResult.score, judgeResult.reason, exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}, options.eligibilitySource);
         }
         // Normalize 1-5 judge score to [0, 1]. Score of -1 means pass-through
         // (no LLM / timeout / parse failure) — leave confidence undefined so
@@ -1234,6 +1300,30 @@ export async function akmDistill(options) {
         if (judgeResult.score > 0)
             lessonJudgeConfidence = judgeResult.score / 5;
     }
+    // WS-3b: Distill→source fidelity check (step 10).
+    // When fidelityCheck.enabled, check the distill proposal against its cited
+    // source memories. A contradiction flag routes to human review (not auto-accept).
+    // DEFAULT OFF. Fail-open: any error is treated as no-contradiction.
+    const fidelityConfig = config.profiles?.improve?.default?.processes?.distill?.fidelityCheck ?? {};
+    if (fidelityConfig.enabled && assetContent) {
+        try {
+            const proposalBody = stripBodyForFidelity(content);
+            const sourceBodies = [stripBodyForFidelity(assetContent)];
+            const fidelityResult = checkDistillFidelity(proposalBody, sourceBodies, fidelityConfig);
+            if (fidelityResult.contradictionDetected) {
+                // Route to human review by writing a quality rejection with reviewNeeded=true.
+                return writeQualityRejection(stash, inputRef, effectiveLessonRef, content, 2.0, // below auto-accept threshold, signals review needed
+                fidelityResult.reason ?? "Proposal may contradict cited source memories.", {
+                    reviewNeeded: true,
+                    fidelityContradiction: true,
+                    ...(exclusionSet.size > 0 ? { filteredFeedbackCount, feedbackFullyFiltered } : {}),
+                }, options.eligibilitySource);
+            }
+        }
+        catch {
+            // Fail open — fidelity check is supplemental.
+        }
+    }
     // Round-trip the parsed frontmatter so the proposal carries it as a
     // structured payload alongside the raw content (matches the shape used by
     // other proposal sources).
@@ -1256,6 +1346,8 @@ export async function akmDistill(options) {
             frontmatter: frontmatterWithSources,
         },
         ...(lessonJudgeConfidence !== undefined ? { confidence: lessonJudgeConfidence } : {}),
+        // Attribution tagging: persist the eligibility lane on the proposal.
+        ...(options.eligibilitySource ? { eligibilitySource: options.eligibilitySource } : {}),
     }, options.ctx);
     if (isProposalSkipped(proposalResult2)) {
         appendEvent({
@@ -1266,6 +1358,7 @@ export async function akmDistill(options) {
                 lessonRef: effectiveLessonRef,
                 message: proposalResult2.message,
                 skipReason: proposalResult2.reason,
+                ...eligMeta,
             },
         });
         return {
@@ -1290,6 +1383,7 @@ export async function akmDistill(options) {
             ...(options.sourceRun !== undefined ? { sourceRun: options.sourceRun } : {}),
             ...(exclusionSet.size > 0 ? { filteredFeedbackCount } : {}),
             ...(descriptionSwapped > 0 ? { descriptionSwapped } : {}),
+            ...eligMeta,
         },
     });
     return {

package/dist/commands/improve/encoding-salience.js

@@ -0,0 +1,205 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * Encoding-time salience scoring for issue #608.
+ *
+ * Pure module — no I/O. All inputs are pre-fetched by the caller.
+ * Implements the three-signal model: novelty × 0.40 + magnitude × 0.35 +
+ * predictionError × 0.25, clamped to [0, 1].
+ */
+// ── Weights ────────────────────────────────────────────────────────────────────
+const W_NOVELTY = 0.4;
+const W_MAGNITUDE = 0.35;
+const W_PREDICTION_ERROR = 0.25;
+// Guard: weights must sum to 1.0 (consistent with the pattern in salience.ts).
+// This check fires at module load and catches any future re-tuning mistakes.
+if (Math.abs(W_NOVELTY + W_MAGNITUDE + W_PREDICTION_ERROR - 1.0) > 1e-9) {
+    throw new Error(`encoding-salience.ts: sub-signal weights must sum to 1.0 (got ${W_NOVELTY + W_MAGNITUDE + W_PREDICTION_ERROR})`);
+}
+// ── Novelty type floors ────────────────────────────────────────────────────────
+const TYPE_NOVELTY_FLOOR = Object.freeze({
+    skill: 0.8,
+    agent: 0.8,
+    memory: 0.4,
+});
+const DEFAULT_NOVELTY_FLOOR = 0.5;
+// ── Magnitude keyword sets ──────────────────────────────────────────────────────
+const SEVERITY_BUCKET = new Set(["error", "critical", "warning", "incident", "regression"]);
+// Note: "fails" and "urgent" are from the spec's initial keyword list and are included
+// here as strong constraint signals, even though the spec's final bucket table omits them.
+const CONSTRAINT_BUCKET = new Set(["must", "never", "always", "blocked", "breaking", "deprecated", "fails", "urgent"]);
+const ALL_MAGNITUDE_KEYWORDS = new Set([...SEVERITY_BUCKET, ...CONSTRAINT_BUCKET]);
+// Number of distinct keyword matches to reach full magnitude score.
+const MAGNITUDE_FULL_SCORE_THRESHOLD = 4;
+// Single-bucket magnitude cap: when only one semantic bucket matches.
+const MAGNITUDE_SINGLE_BUCKET_CAP = 0.5;
+// ── Stop-words (excluded from bigram tokenization) ────────────────────────────
+const STOP_WORDS = new Set([
+    "a",
+    "an",
+    "the",
+    "and",
+    "or",
+    "but",
+    "in",
+    "on",
+    "at",
+    "to",
+    "for",
+    "of",
+    "with",
+    "by",
+    "from",
+    "as",
+    "is",
+    "it",
+    "its",
+    "be",
+    "are",
+    "was",
+    "were",
+    "has",
+    "have",
+    "had",
+    "do",
+    "does",
+    "did",
+    "not",
+    "this",
+    "that",
+    "these",
+    "those",
+    "i",
+    "we",
+    "you",
+    "he",
+    "she",
+    "they",
+    "my",
+    "our",
+    "your",
+    "his",
+    "her",
+    "their",
+    "can",
+    "will",
+    "would",
+    "should",
+    "could",
+    "may",
+    "might",
+    "shall",
+    "so",
+    "if",
+    "then",
+    "than",
+    "when",
+    "while",
+    "what",
+    "which",
+    "who",
+    "how",
+    "no",
+    "any",
+    "all",
+    "more",
+    "also",
+]);
+// ── Tokenization helpers ──────────────────────────────────────────────────────
+/**
+ * Tokenize text into lowercase words, splitting on whitespace and punctuation,
+ * dropping stop-words. Used for body bigram extraction (novelty signal).
+ */
+function tokenizeBody(text) {
+    return text
+        .toLowerCase()
+        .split(/[\s\p{P}\p{S}\-_/]+/u)
+        .filter((t) => t.length > 1 && !STOP_WORDS.has(t));
+}
+/**
+ * Tokenize a ref name or tag into lowercase words without stop-word filtering.
+ * Ref names are identifiers, not prose — all tokens are significant.
+ */
+function tokenizeRef(ref) {
+    return ref
+        .toLowerCase()
+        .split(/[\s\p{P}\p{S}\-_/]+/u)
+        .filter((t) => t.length > 0);
+}
+/**
+ * Generate all consecutive bigrams from an array of tokens.
+ * Returns them as "token1 token2" strings.
+ */
+function bigrams(tokens) {
+    const result = [];
+    for (let i = 0; i < tokens.length - 1; i++) {
+        result.push(`${tokens[i]} ${tokens[i + 1]}`);
+    }
+    return result;
+}
+/**
+ * Build the existing-ref vocabulary: a Set of bigrams derived from tokenizing
+ * the provided ref names (and tags). Pass this to `scoreEncodingSalience` as
+ * `existingRefVocabulary` so the novelty signal can measure how much of the
+ * asset body is already represented in the stash vocabulary.
+ */
+export function buildRefVocabulary(refs) {
+    const vocab = new Set();
+    for (const ref of refs) {
+        const tokens = tokenizeRef(ref);
+        for (const bg of bigrams(tokens)) {
+            vocab.add(bg);
+        }
+    }
+    return vocab;
+}
+// ── Sub-signal computations ────────────────────────────────────────────────────
+function computeNovelty(body, type, vocab) {
+    const tokens = tokenizeBody(body);
+    const bgs = bigrams(tokens);
+    const floor = TYPE_NOVELTY_FLOOR[type] ?? DEFAULT_NOVELTY_FLOOR;
+    if (bgs.length === 0)
+        return floor;
+    const novelCount = bgs.filter((bg) => !vocab.has(bg)).length;
+    const bigramNoveltyFraction = novelCount / bgs.length;
+    return Math.max(floor, bigramNoveltyFraction);
+}
+function computeMagnitude(body) {
+    const lowerBody = body.toLowerCase();
+    // Split on word boundaries to avoid partial matches (e.g. "errors" matching "error").
+    const words = new Set(lowerBody.split(/\W+/).filter((w) => w.length > 0));
+    const matched = [...ALL_MAGNITUDE_KEYWORDS].filter((kw) => words.has(kw));
+    if (matched.length === 0)
+        return 0;
+    const distinctCount = matched.length;
+    const rawMagnitude = Math.min(1.0, distinctCount / MAGNITUDE_FULL_SCORE_THRESHOLD);
+    // Require at least one match from each bucket to lift the single-bucket cap.
+    const hasSeverity = matched.some((kw) => SEVERITY_BUCKET.has(kw));
+    const hasConstraint = matched.some((kw) => CONSTRAINT_BUCKET.has(kw));
+    if (hasSeverity && hasConstraint)
+        return rawMagnitude;
+    return Math.min(rawMagnitude, MAGNITUDE_SINGLE_BUCKET_CAP);
+}
+function computePredictionError(revisionCount) {
+    if (revisionCount === 0)
+        return 1.0;
+    return 1 / (1 + Math.log(1 + revisionCount));
+}
+// ── Main export ────────────────────────────────────────────────────────────────
+/**
+ * Compute the encoding-time salience score for an asset.
+ *
+ * Three sub-signals:
+ *   novelty (0.40) — how much of the body is absent from the stash vocabulary.
+ *   magnitude (0.35) — presence of severity/constraint keywords.
+ *   predictionError (0.25) — surprise: high for new assets, decays with revisions.
+ */
+export function scoreEncodingSalience(inputs) {
+    const novelty = computeNovelty(inputs.body, inputs.type, inputs.existingRefVocabulary);
+    const magnitude = computeMagnitude(inputs.body);
+    const predictionError = computePredictionError(inputs.revisionCount);
+    const raw = W_NOVELTY * novelty + W_MAGNITUDE * magnitude + W_PREDICTION_ERROR * predictionError;
+    const score = Math.min(1, Math.max(0, raw));
+    return { score, novelty, magnitude, predictionError };
+}