akm-cli 0.9.0-beta.4 → 0.9.0-beta.41

package/dist/core/asset/frontmatter.js

@@ -4,170 +4,43 @@
 /**
  * Shared frontmatter parsing utilities.
  *
- * Provides a single, canonical YAML-subset frontmatter parser used by both
- * the stash open logic and the metadata generator.
+ * Uses the `yaml` library for all YAML parsing so that the full YAML spec
+ * (block scalars, multi-line strings, nested objects, flow sequences, escape
+ * sequences) is handled correctly without a brittle hand-rolled state machine.
  */
+import { parse as yamlParse, stringify as yamlStringify } from "yaml";
 /**
- * Parse YAML-subset frontmatter from a Markdown (or similar) string.
+ * Parse YAML frontmatter from a Markdown (or similar) string.
  *
  * Returns the parsed key-value data and the remaining body content.
- *
- * **Limitations**: This is a hand-rolled YAML-subset parser with intentional
- * constraints for simplicity and safety:
- * - **Top-level values**: string, boolean, and number scalars are supported,
- *   as well as top-level list-valued keys using YAML block sequences
- *   (`- item`) or flow arrays (`[a, b, c]`).
- * - **List item types**: list items must be scalar values and may be strings,
- *   booleans, or numbers.
- * - **No nested objects beyond one level**: Only a single level of indented
- *   key-value pairs is supported.
- * - **Block scalars**: `|` (literal), `|-` (strip), and `|+` (keep) block
- *   scalars are supported for multi-line string values as emitted by the
- *   `yaml` library's `stringify`.
+ * Delegates all YAML parsing to the `yaml` library; the only responsibility
+ * of this function is extracting the `---…---` block and normalizing the
+ * parsed result (e.g. converting YAML timestamp values to ISO date strings).
  */
 export function parseFrontmatter(raw) {
     const parsedBlock = parseFrontmatterBlock(raw);
     if (!parsedBlock) {
         return { data: {}, content: raw, frontmatter: null, bodyStartLine: 1 };
     }
-    const data = {};
-    let currentKey = null;
-    /**
-     * "scalar" | "list" | "object" | "pending" | "block" —
-     * "pending" means empty value, mode determined by next line.
-     * "block" means we are accumulating lines for a `|`-block scalar.
-     */
-    let mode = "scalar";
-    let nested = null;
-    let currentList = null;
-    /** Lines collected while in "block" mode. */
-    let blockLines = null;
-    /** Block scalar chomping: "clip" (|), "strip" (|-), "keep" (|+). */
-    let blockChomping = "clip";
-    const flushPending = () => {
-        // Called when we start a new top-level key and the previous key was still "pending".
-        // An empty-value key followed by another top-level key means it was an empty scalar.
-        if (mode === "pending" && currentKey !== null) {
-            data[currentKey] = "";
-        }
-    };
-    const flushBlock = () => {
-        // Commit the accumulated block-scalar lines to `data[currentKey]`.
-        if (mode !== "block" || currentKey === null || blockLines === null)
-            return;
-        // De-indent: strip the common 2-space prefix `yaml.stringify` emits.
-        const deindented = blockLines.map((l) => (l.startsWith("  ") ? l.slice(2) : l));
-        // Chomping: apply trailing-newline policy.
-        // "clip" (|): single trailing newline.
-        // "strip" (|-): no trailing newline.
-        // "keep" (|+): keep all trailing newlines as-is.
-        if (blockChomping === "keep") {
-            data[currentKey] = deindented.join("\n");
-        }
-        else if (blockChomping === "strip") {
-            data[currentKey] = deindented.join("\n").replace(/\n+$/, "");
-        }
-        else {
-            // "clip": exactly one trailing newline
-            data[currentKey] = `${deindented.join("\n").replace(/\n+$/, "")}\n`;
-        }
-    };
-    for (const line of parsedBlock.frontmatter.split(/\r?\n/)) {
-        // If we are in block-scalar mode, collect indented lines or end the block.
-        if (mode === "block") {
-            if (line.startsWith("  ") || line === "") {
-                // Continuation of the block scalar (indented content or blank line).
-                blockLines.push(line);
-                continue;
+    let data = {};
+    if (parsedBlock.frontmatter.trim()) {
+        try {
+            const parsed = yamlParse(parsedBlock.frontmatter);
+            if (parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)) {
+                // Normalize Date objects: the yaml "core" schema parses YYYY-MM-DD
+                // literals as JS Date instances. Convert them back to ISO date strings
+                // to preserve the string type that callers (and yaml.stringify on write)
+                // expect.
+                data = normalizeYamlValues(parsed);
             }
-            // Non-indented line ends the block scalar — flush and fall through to
-            // parse the new line as a top-level key.
-            flushBlock();
-            mode = "scalar";
-            blockLines = null;
         }
-        // Block-sequence item: "- value" or "  - value" (optional 2-space indent)
-        // Only match when the current key is in list or pending mode.
-        const seqItem = line.match(/^(?: {2})?- (.*)$/);
-        if (seqItem && currentKey !== null && (mode === "list" || mode === "pending")) {
-            if (mode === "pending") {
-                // First block-sequence item after an empty-value key — switch to list mode
-                currentList = [];
-                data[currentKey] = currentList;
-                mode = "list";
-            }
-            currentList.push(parseYamlScalar(seqItem[1].trim()));
-            continue;
-        }
-        // Plain-style multi-line scalar continuation: a 2-space-indented line that
-        // is not a sequence item or nested key. YAML plain scalars fold newlines
-        // into a single space, so we append with a space. This handles LLM-emitted
-        // descriptions like:
-        //   description: Use 4-colon outer containers when mixing
-        //     nesting depths in markdown-it-container plugins.
-        // Without this, only the first line is captured and the truncation
-        // heuristic wrongly flags it as cut off mid-sentence.
-        if (mode === "scalar" && currentKey !== null && /^ {2}\S/.test(line)) {
-            data[currentKey] = `${String(data[currentKey])} ${line.trim()}`;
-            continue;
-        }
-        // Indented nested key-value (object under a key with empty value)
-        const indented = line.match(/^ {2}(\w[\w-]*):\s*(.+)$/);
-        if (indented && currentKey !== null && (mode === "object" || mode === "pending")) {
-            if (mode === "pending") {
-                // First indented k-v after an empty-value key — switch to object mode
-                nested = {};
-                data[currentKey] = nested;
-                mode = "object";
-            }
-            nested[indented[1]] = parseYamlScalar(indented[2].trim());
-            continue;
+        catch {
+            // Malformed YAML (e.g. unterminated quotes from LLM output corruption).
+            // Fall back to line-by-line best-effort extraction so callers still get
+            // whatever scalar values they can rather than a completely empty record.
+            data = parseFrontmatterLenient(parsedBlock.frontmatter);
         }
-        // Top-level key (possibly with inline value)
-        const top = line.match(/^(\w[\w-]*):\s*(.*)$/);
-        if (!top) {
-            continue;
-        }
-        // Starting a new top-level key — flush any pending empty-value key
-        flushPending();
-        currentKey = top[1];
-        const value = top[2].trim();
-        if (value === "|" || value === "|-" || value === "|+") {
-            // Block scalar header — collect subsequent indented lines.
-            mode = "block";
-            blockLines = [];
-            blockChomping = value === "|-" ? "strip" : value === "|+" ? "keep" : "clip";
-            nested = null;
-            currentList = null;
-        }
-        else if (value === "") {
-            // Defer mode decision until we see the next line
-            mode = "pending";
-            nested = null;
-            currentList = null;
-            // Don't store anything yet — flushPending will set "" if no continuation
-        }
-        else if (value.startsWith("[") && value.endsWith("]")) {
-            // Inline flow array: tags: [ops, networking]
-            mode = "list";
-            nested = null;
-            currentList = null;
-            currentList = parseFlowArray(value);
-            data[currentKey] = currentList;
-        }
-        else {
-            mode = "scalar";
-            nested = null;
-            currentList = null;
-            data[currentKey] = parseYamlScalar(value);
-        }
-    }
-    // Flush any in-progress block scalar at end of frontmatter.
-    if (mode === "block") {
-        flushBlock();
     }
-    // Flush the last key if it was still pending (empty value, no continuation)
-    flushPending();
     return {
         data,
         content: parsedBlock.content,
@@ -176,29 +49,89 @@ export function parseFrontmatter(raw) {
     };
 }
 /**
- * Parse a YAML flow array string like `[a, b, c]` into an array of scalars.
+ * Normalize YAML-parsed values to match expected AKM frontmatter types.
+ *
+ * Two conversions:
+ * 1. `Date` → YYYY-MM-DD string: the yaml "core" schema parses bare date
+ *    scalars like `2026-06-18` as JS Date instances. AKM frontmatter treats
+ *    `updated:` and similar fields as plain strings.
+ * 2. `null` → `""`: the yaml library parses empty-value keys (`key:` with no
+ *    value) as `null`, but AKM callers historically received `""` from the
+ *    hand-rolled parser. Convert to preserve backward compatibility.
  */
-function parseFlowArray(value) {
-    const inner = value.slice(1, -1).trim();
-    if (!inner)
-        return [];
-    return inner.split(",").map((item) => parseYamlScalar(item.trim()));
+function normalizeYamlValues(value) {
+    if (value instanceof Date) {
+        const y = value.getUTCFullYear();
+        const m = String(value.getUTCMonth() + 1).padStart(2, "0");
+        const d = String(value.getUTCDate()).padStart(2, "0");
+        return `${y}-${m}-${d}`;
+    }
+    if (value === null)
+        return "";
+    if (Array.isArray(value))
+        return value.map(normalizeYamlValues);
+    if (typeof value === "object") {
+        return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, normalizeYamlValues(v)]));
+    }
+    return value;
+}
+/**
+ * Best-effort line-by-line frontmatter extraction for malformed YAML.
+ *
+ * Used as a fallback when yaml.parse throws (e.g. unterminated quotes from LLM
+ * output corruption). Extracts simple `key: value` scalar pairs only — nested
+ * objects and sequences are skipped. Values that are individually parseable by
+ * yaml are normalized; otherwise stored as raw strings.
+ */
+function parseFrontmatterLenient(frontmatter) {
+    const data = {};
+    for (const line of frontmatter.split(/\r?\n/)) {
+        const m = line.match(/^([\w][\w-]*):\s*(.*)$/);
+        if (!m)
+            continue;
+        const key = m[1];
+        const rawValue = (m[2] ?? "").trim();
+        try {
+            const singleEntry = yamlParse(`k: ${rawValue}`);
+            if (singleEntry !== null && typeof singleEntry === "object" && !Array.isArray(singleEntry)) {
+                const v = singleEntry.k;
+                data[key] = v === null || v === undefined ? "" : v;
+            }
+            else {
+                data[key] = rawValue;
+            }
+        }
+        catch {
+            data[key] = rawValue;
+        }
+    }
+    return data;
 }
 export function parseFrontmatterBlock(raw) {
     // Handle both LF and CRLF line endings throughout.
     // The closing --- may be preceded by \r\n; capture and strip trailing \r
     // from the frontmatter block so key parsing sees clean LF-terminated lines.
     const match = raw.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r\n|\r|\n|$)([\s\S]*)$/);
-    if (!match)
-        return null;
-    // Strip any \r characters from the frontmatter block to normalise CRLF → LF
-    const frontmatter = match[1].replace(/\r/g, "");
-    const content = match[2];
-    return {
-        frontmatter,
-        content,
-        bodyStartLine: countLines(raw.slice(0, match[0].length - match[2].length)) + 1,
-    };
+    if (match) {
+        // Strip any \r characters from the frontmatter block to normalise CRLF → LF
+        const frontmatter = match[1].replace(/\r/g, "");
+        const content = match[2];
+        return {
+            frontmatter,
+            content,
+            bodyStartLine: countLines(raw.slice(0, match[0].length - match[2].length)) + 1,
+        };
+    }
+    // Empty frontmatter (---\n---): the content-bearing regex above requires at
+    // least one character between the fences. Handle the degenerate case so
+    // callers can reconstruct `---\nkey: val\n---\n\nbody` from a previously
+    // empty-frontmatter file without corrupting it by wrapping the entire raw
+    // string as body content.
+    const emptyMatch = raw.match(/^---\r?\n---(?:\r\n|\r|\n)([\s\S]*)$/);
+    if (emptyMatch) {
+        return { frontmatter: "", content: emptyMatch[1], bodyStartLine: 3 };
+    }
+    return null;
 }
 function countLines(text) {
     if (text.length === 0)
@@ -206,7 +139,13 @@ function countLines(text) {
     return text.split(/\r?\n/).length - 1;
 }
 /**
- * Parse a simple YAML scalar value (string, boolean, or number).
+ * Parse a YAML scalar value (string, boolean, or number).
+ *
+ * For quoted strings (single or double), delegates to the `yaml` library so
+ * escape sequences are handled correctly per spec. The previous hand-rolled
+ * `slice(1, -1)` only stripped one layer of quoting and left inner quotes and
+ * escape sequences as literal characters in the stored value, causing visible
+ * corruption when `yaml.stringify` re-quoted them on the next write.
  */
 export function parseYamlScalar(value) {
     if (value === "")
@@ -219,7 +158,67 @@ export function parseYamlScalar(value) {
     if (!Number.isNaN(asNumber))
         return asNumber;
     if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
+        try {
+            const parsed = yamlParse(value);
+            if (typeof parsed === "string")
+                return parsed;
+        }
+        catch {
+            // Fall through to raw slice on malformed YAML — better than throwing.
+        }
         return value.slice(1, -1);
     }
     return value;
 }
+// ── Minimum score delta to trigger a frontmatter salience rewrite ─────────────
+const SALIENCE_WRITE_DELTA_THRESHOLD = 0.05;
+/**
+ * Idempotently write `salience` and `salienceInputs` fields into the YAML
+ * frontmatter of a raw asset string.
+ *
+ * Skips the write when the existing `salience` field differs from `score` by
+ * less than {@link SALIENCE_WRITE_DELTA_THRESHOLD}, to avoid churn for minor
+ * floating-point drift. Returns the raw string unchanged when no write is needed
+ * or when no frontmatter block is present.
+ *
+ * The `salienceInputs` field is written for auditability only; no pipeline code
+ * reads it back. `state.db :: asset_salience` is the canonical store.
+ */
+export function writeSalienceToFrontmatter(raw, score, inputs) {
+    const parsed = parseFrontmatterBlock(raw);
+    if (!parsed)
+        return raw;
+    const existingData = parseFrontmatter(raw).data;
+    const existingSalience = typeof existingData.salience === "number" ? existingData.salience : undefined;
+    if (existingSalience !== undefined && Math.abs(existingSalience - score) < SALIENCE_WRITE_DELTA_THRESHOLD) {
+        return raw;
+    }
+    // Parse existing frontmatter into an object, then set/overwrite salience fields.
+    let fm = {};
+    if (parsed.frontmatter.trim()) {
+        try {
+            const p = yamlParse(parsed.frontmatter);
+            if (p !== null && typeof p === "object" && !Array.isArray(p)) {
+                fm = p;
+            }
+        }
+        catch {
+            // Malformed YAML — rebuild from best-effort parse
+            fm = parseFrontmatterLenient(parsed.frontmatter);
+        }
+    }
+    fm.salience = roundTo2dp(score);
+    fm.salienceInputs = {
+        novelty: roundTo2dp(inputs.novelty),
+        magnitude: roundTo2dp(inputs.magnitude),
+        predictionError: roundTo2dp(inputs.predictionError),
+    };
+    const newFrontmatter = yamlStringify(fm).trimEnd();
+    const body = parsed.content;
+    // Preserve original line ending style between frontmatter and body
+    const separator = body.startsWith("\n") ? "" : "\n";
+    return `---\n${newFrontmatter}\n---\n${separator}${body}`;
+}
+function roundTo2dp(n) {
+    return Math.round(n * 100) / 100;
+}

package/dist/core/asset/markdown.js

@@ -8,7 +8,15 @@ export function parseMarkdownToc(content) {
     const headings = [];
     const parsed = parseFrontmatter(content);
     const start = parsed.frontmatter ? parsed.bodyStartLine - 1 : 0;
+    let inFence = false;
     for (let i = start; i < lines.length; i++) {
+        // Track fenced code blocks (``` or ~~~) so headings inside them are skipped.
+        if (/^\s*(`{3,}|~{3,})/.test(lines[i])) {
+            inFence = !inFence;
+            continue;
+        }
+        if (inFence)
+            continue;
         const match = lines[i].match(/^(#{1,6})\s+(.+)$/);
         if (match) {
             headings.push({

package/dist/core/authoring-rules.js

@@ -0,0 +1,83 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at https://mozilla.org/MPL/2.0/.
+/**
+ * Canonical HARD authoring rules — the single source of truth shared by the
+ * proposal validators (which REJECT violations) and the improve/authoring
+ * prompts (which must TELL the agent the same rules, in the same words).
+ *
+ * Why this module exists: authoring rules were duplicated and drifted across
+ * prompt templates. `distill-lesson-system.md` told the model "80–200 chars"
+ * while the validator enforced 20–400; reflect's prompt omitted the
+ * no-pseudo-frontmatter / single-fence rules entirely, so reflect generated
+ * proposals that the gate then rejected and that got stuck in the queue.
+ *
+ * The fix: the numeric bounds live HERE and are imported by both the validators
+ * (`isValidDescription` / `isValidWhenToUse` in proposal-quality-validators.ts)
+ * and the prompt text (`authoringRulesForType`). The agent-facing rule prose
+ * sits next to the bounds it describes, so a developer changing a validator
+ * sees the prompt copy that must change with it. `tests/authoring-rules-*`
+ * asserts the two representations stay consistent.
+ *
+ * SCOPE: only HARD rules (a validator rejects the proposal if violated) belong
+ * here. Soft/style conventions (voice, paragraph count, "include a # Title")
+ * are user-editable and flow through the separate `standardsContext` seam
+ * (stash `category: convention` facts) — NOT this module.
+ */
+// ── Canonical numeric bounds (imported by the validators — do not duplicate) ──
+/** `description` length bounds (chars). Enforced by `isValidDescription`. */
+export const DESCRIPTION_MIN_CHARS = 20;
+export const DESCRIPTION_MAX_CHARS = 400;
+/** `when_to_use` length bounds (chars). Enforced by `isValidWhenToUse`. */
+export const WHEN_TO_USE_MIN_CHARS = 15;
+export const WHEN_TO_USE_MAX_CHARS = 400;
+// ── Agent-facing rule prose (mirrors the validator checks one-for-one) ────────
+/**
+ * Rules that apply to any markdown asset authored with YAML frontmatter + a
+ * body. Enforced by `detectDoubleFrontmatter` (currently fires for lesson
+ * proposals, but the rules are universally correct, so we state them for every
+ * type to prevent the same defect class elsewhere).
+ */
+const FRONTMATTER_BODY_RULES = [
+    "Emit EXACTLY TWO `---` fence lines — the opening and closing of the YAML frontmatter. Do NOT use `---` as a horizontal rule anywhere in the body.",
+    "Do NOT restate `description:` or `when_to_use:` inside the body (no `**description:** …` or `**when_to_use:** …` lines). Those keys belong in the frontmatter ONLY.",
+];
+/** Rules for the `description` frontmatter field. Enforced by `isValidDescription`. */
+const DESCRIPTION_RULES = [
+    `\`description\` must be ${DESCRIPTION_MIN_CHARS}–${DESCRIPTION_MAX_CHARS} characters of plain-prose sentence — no leading digit or markdown marker, balanced backticks, and it must NOT end with \`:\`, \`;\`, or \`,\` (those read as truncation).`,
+    '`description` must NOT be a section-heading fragment (e.g. "Overview", "Key points", "Summary"), a code fragment (must not start with `def`/`function`/`class`/`const`/…), or end on a hanging connector word ("a", "the", "and", "to", …).',
+    "`description` must NOT merely restate the asset's ref/name; write what the asset actually does.",
+    '`description` should NOT start with "When" — that phrasing belongs in `when_to_use`.',
+];
+/** Rules for the `when_to_use` frontmatter field. Enforced by `isValidWhenToUse`. */
+const WHEN_TO_USE_RULES = [
+    `\`when_to_use\` is REQUIRED and must be ${WHEN_TO_USE_MIN_CHARS}–${WHEN_TO_USE_MAX_CHARS} characters describing a concrete trigger. Never write the circular fallback "When working with <name>".`,
+    "`description` and `when_to_use` must be different from each other.",
+];
+/**
+ * Asset types that carry a `description` and a body where the
+ * frontmatter/body rules apply. (Types without those — if any are added later —
+ * simply fall through to the cross-cutting block.)
+ */
+const DESCRIPTION_TYPES = new Set(["lesson", "knowledge", "memory", "skill", "command", "agent", "workflow", "fact"]);
+/** Types where `when_to_use` is a HARD requirement (validator rejects if absent). */
+const WHEN_TO_USE_TYPES = new Set(["lesson"]);
+/**
+ * Build the hard-rules block for a given asset type, ready to inject as a prompt
+ * section. Returns `""` for an unknown type (no over-claiming). The block is
+ * deterministic so prompt snapshots stay stable.
+ *
+ * Inject this VERBATIM into every improve/authoring prompt that creates or edits
+ * an asset of `type`, so the agent is told exactly what the gate will reject.
+ */
+export function authoringRulesForType(type) {
+    const rules = [...FRONTMATTER_BODY_RULES];
+    if (DESCRIPTION_TYPES.has(type))
+        rules.push(...DESCRIPTION_RULES);
+    if (WHEN_TO_USE_TYPES.has(type))
+        rules.push(...WHEN_TO_USE_RULES);
+    if (rules.length === 0)
+        return "";
+    const heading = `Hard authoring rules for ${type} assets (the validator REJECTS proposals that violate these):`;
+    return [heading, ...rules.map((r) => `- ${r}`)].join("\n");
+}