akm-cli 0.9.16 → 0.9.17-alpha.10

package/dist/commands/improve/consolidate/sanitize.js

@@ -1,80 +1,34 @@
 // This Source Code Form is subject to the terms of the Mozilla Public
 // License, v. 2.0. If a copy of the MPL was not distributed with this
 // file, You can obtain one at https://mozilla.org/MPL/2.0/.
-// ── LLM-output sanitization ─────────────────────────────────────────────────
-//
-// Pure string/frontmatter transforms that defend against three classes of LLM
-// defect observed across hundreds of consolidate proposals:
-//
-//   1. Code-fence leakage: the entire merged asset is wrapped in
-//      ```markdown … ``` (or ```yaml … ```) despite the prompt forbidding
-//      fences. The post-processor used to pass this through verbatim, so the
-//      first character of the asset content became a backtick rather than
-//      `---`, defeating the frontmatter parser.
-//   2. YAML quote-escaping bugs: descriptions like `'"Specialty intro...:`
-//      with unbalanced quotes that break the YAML reader. The post-processor
-//      historically passed the LLM's raw scalar straight into a manually
-//      assembled `description: <raw>` line.
-//   3. Truncated descriptions hitting token cutoffs — the model's max_tokens
-//      runs out mid-sentence, leaving things like
-//      `description: "Tables in narrow column containers need max-width:100% +"`
-//      with no closing context.
-//
-// `sanitizeMergedContent` and `validateProposalFrontmatter` defend against
-// all three at the point where LLM output is consumed.
+/**
+ * Consolidate's defence against three recurring LLM output defects: the whole
+ * asset wrapped in a code fence, frontmatter with broken YAML quoting, and a
+ * missing closing `---`.
+ */
 import { parse as yamlParse } from "yaml";
 import { assembleAssetFromString, serializeFrontmatter } from "../../../core/asset/asset-serialize.js";
 import { parseFrontmatter } from "../../../core/asset/frontmatter.js";
+const FRONTMATTER_BLOCK = /^---\r?\n([\s\S]*?)\r?\n---(?:\r\n|\r|\n|$)([\s\S]*)$/;
 /**
- * Attempt to recover a frontmatter block that is missing its closing `---`.
- *
- * Scans lines after the opening `---` for the first blank line or the first
- * line that cannot be a YAML scalar (i.e. not a key-value, indented
- * continuation, comment, or list item). Injects `---` before that line so
- * the normal parser can proceed.
- *
- * Returns the patched string on success, or `null` if the structure is too
- * ambiguous to recover safely (e.g. no opening `---`, or no body content
- * found after the frontmatter key-value lines).
+ * Insert the missing closing `---` before the first blank line or the first
+ * line that cannot be YAML (key, indented continuation, comment, list item);
+ * `null` when there is no such line.
  */
 function recoverMalformedFrontmatter(raw) {
     if (!raw.startsWith("---"))
         return null;
     const lines = raw.split(/\r?\n/);
-    // Skip the opening `---` line (index 0).
-    let insertAt = -1;
-    for (let i = 1; i < lines.length; i++) {
-        const line = lines[i];
-        // A blank line marks the end of the frontmatter block in many YAML variants.
-        if (line.trim() === "") {
-            insertAt = i;
-            break;
-        }
-        // A line that is clearly body content: doesn't look like a YAML key, an
-        // indented continuation, a comment, or a sequence item.
-        const isYaml = /^\w[\w-]*\s*:/.test(line) || // key: value
-            /^\s+\S/.test(line) || // indented continuation / nested
-            /^\s*#/.test(line) || // YAML comment
-            /^\s*-\s/.test(line); // sequence item
-        if (!isYaml) {
-            insertAt = i;
-            break;
-        }
-    }
+    const insertAt = lines.findIndex((line, i) => i > 0 &&
+        (line.trim() === "" ||
+            !(/^\w[\w-]*\s*:/.test(line) || /^\s+\S/.test(line) || /^\s*#/.test(line) || /^\s*-\s/.test(line))));
     if (insertAt < 0)
         return null;
-    const result = [...lines.slice(0, insertAt), "---", ...lines.slice(insertAt)].join("\n");
-    return result;
+    return [...lines.slice(0, insertAt), "---", ...lines.slice(insertAt)].join("\n");
 }
 /**
- * Outer-fence stripper specific to consolidate. Unlike the shared
- * `stripMarkdownFences` helper (which only handles markdown fences), this
- * variant additionally recognises `yaml` and bare-language fences and refuses
- * to strip an unbalanced fence — i.e. a leading ``` with no trailing ``` is
- * treated as a malformed response, not partially sanitized.
- *
- * Returns `null` when only one half of a fence pair is present (caller
- * should reject the response entirely).
+ * Strip a markdown/yaml/bare code fence wrapping the whole response; `null`
+ * when only one half of the pair is present.
  */
 export function stripOuterCodeFence(raw) {
     const trimmed = raw.trim();
@@ -83,91 +37,63 @@ export function stripOuterCodeFence(raw) {
     if (!leading && !trailing)
         return { content: trimmed, stripped: false };
     if (!leading || !trailing)
-        return null; // unbalanced — refuse
-    const inner = trimmed.slice(leading[0].length, trimmed.length - trailing[0].length).trim();
-    return { content: inner, stripped: true };
+        return null;
+    return { content: trimmed.slice(leading[0].length, trimmed.length - trailing[0].length).trim(), stripped: true };
 }
+/**
+ * Model output as an asset: fence stripped, `<think>` removed, frontmatter
+ * re-serialized through the YAML library (the lenient parser recovers what the
+ * strict one rejects), or a reason it is unusable.
+ */
 export function sanitizeMergedContent(raw) {
-    // Step 1: Strip outer code fence.
-    // Recovery path: if only the leading fence is present, strip it and continue
-    // provided the inner content starts with `---`. Trailing-only fences are NOT
-    // recovered — a trailing ``` is more likely a body code block than a forgotten
-    // wrapper, so recovering would silently corrupt the body.
     let body;
-    {
-        const fenceResult = stripOuterCodeFence(raw);
-        if (fenceResult) {
-            body = fenceResult.content;
-        }
-        else {
-            const trimmed = raw.trim();
-            const leadingMatch = trimmed.match(/^```(?:markdown|md|yaml|yml)?\s*\r?\n([\s\S]*)$/i);
-            const inner = leadingMatch ? leadingMatch[1].trim() : null;
-            if (!inner?.startsWith("---")) {
-                return { ok: false, reason: "UNBALANCED_CODE_FENCE" };
-            }
-            body = inner;
-        }
+    const fenceResult = stripOuterCodeFence(raw);
+    if (fenceResult) {
+        body = fenceResult.content;
+    }
+    else {
+        // A leading fence alone is recovered when frontmatter follows; a trailing
+        // one alone is more likely a body code block, so it is not.
+        const inner = raw
+            .trim()
+            .match(/^```(?:markdown|md|yaml|yml)?\s*\r?\n([\s\S]*)$/i)?.[1]
+            ?.trim();
+        if (!inner?.startsWith("---"))
+            return { ok: false, reason: "UNBALANCED_CODE_FENCE" };
+        body = inner;
     }
-    // Strip <think> blocks (some local models still emit them despite system prompts).
     body = body.replace(/<think>[\s\S]*?<\/think>/gi, "").trim();
-    // Step 2: Verify frontmatter sentinel.
-    // Recovery path: LLM sometimes emits 1-2 lines of preamble (e.g. "Here is the
-    // merged content:") before the `---`. Accept if `---` appears within 300 chars.
-    // Beyond that it's more likely a body section divider, not a frontmatter start.
+    // A line or two of preamble before the frontmatter is tolerated; a `---`
+    // further in is more likely a section divider.
     if (!body.startsWith("---")) {
         const nlIdx = body.indexOf("\n---");
-        if (nlIdx >= 0 && nlIdx < 300) {
-            body = body.slice(nlIdx + 1);
-        }
-        else {
+        if (nlIdx < 0 || nlIdx >= 300)
             return { ok: false, reason: "MISSING_FRONTMATTER_SENTINEL" };
-        }
+        body = body.slice(nlIdx + 1);
     }
-    // Extract frontmatter block.
-    // Recovery path: LLM sometimes omits the closing `---` delimiter. Detect this
-    // by scanning lines after the opening `---` for the first blank line or the
-    // first line that isn't a YAML key-value pair, then inject `---` there.
-    let match = body.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r\n|\r|\n|$)([\s\S]*)$/);
+    let match = body.match(FRONTMATTER_BLOCK);
     if (!match) {
         const recovered = recoverMalformedFrontmatter(body);
-        if (recovered) {
-            match = recovered.match(/^---\r?\n([\s\S]*?)\r?\n---(?:\r\n|\r|\n|$)([\s\S]*)$/);
-        }
-        if (!match) {
+        match = recovered ? recovered.match(FRONTMATTER_BLOCK) : null;
+        if (!match)
             return { ok: false, reason: "MALFORMED_FRONTMATTER_BLOCK" };
-        }
     }
-    // Re-parse via the yaml library so any quote-escaping mistakes either get
-    // normalised or surface as a parse error we can reject.
-    // Recovery: if the strict yaml library fails, fall back to the lenient
-    // hand-rolled parseFrontmatter parser, which tolerates common LLM YAML
-    // quirks (unescaped special chars, bare scalars, etc.). If it recovers
-    // at least one key, proceed — serializeFrontmatter below will re-serialize
-    // cleanly. Only reject if both parsers fail to extract any data.
     let parsedFm;
     try {
         parsedFm = yamlParse(match[1]);
     }
     catch (e) {
         const fallback = parseFrontmatter(`---\n${match[1]}\n---\n${match[2]}`);
-        if (fallback.frontmatter !== null && Object.keys(fallback.data).length > 0) {
-            parsedFm = fallback.data;
-        }
-        else {
+        if (fallback.frontmatter === null || Object.keys(fallback.data).length === 0) {
             return { ok: false, reason: `INVALID_YAML: ${e instanceof Error ? e.message : String(e)}` };
         }
+        parsedFm = fallback.data;
     }
     if (parsedFm === null || typeof parsedFm !== "object" || Array.isArray(parsedFm)) {
         return { ok: false, reason: "FRONTMATTER_NOT_OBJECT" };
     }
     const fm = parsedFm;
-    // Normalise placeholder leaks like `updated: today`, `updated: {today: null}`,
-    // `updated: now`, etc. The consolidate prompt instructs the LLM not to emit
-    // these, but small models still do. Replace any such leak with today's ISO
-    // date OR drop the field if we can't safely normalise it.
     normalizeUpdatedField(fm);
-    // Re-serialise via yaml.stringify to fix any quoting quirks.
     let serialized;
     try {
         serialized = serializeFrontmatter(fm);
@@ -175,57 +101,36 @@ export function sanitizeMergedContent(raw) {
     catch (e) {
         return { ok: false, reason: `YAML_STRINGIFY_FAILED: ${e instanceof Error ? e.message : String(e)}` };
     }
-    const cleaned = assembleAssetFromString(serialized, match[2]);
-    return { ok: true, result: { content: cleaned, frontmatter: fm } };
+    return { ok: true, result: { content: assembleAssetFromString(serialized, match[2]), frontmatter: fm } };
 }
 /**
- * Mutate `fm.updated` in place to normalise placeholder leaks emitted by the
- * LLM. The consolidate prompt forbids these, but small models still produce
- * literal `today` / `{today: null}` / `now` values.
- *
- * Rules:
- *   - A real ISO-style date string (YYYY-MM-DD, optionally with time) stays as-is.
- *   - A Date object (some YAML parsers materialise dates) is converted to its
- *     ISO yyyy-mm-dd form.
- *   - A placeholder string ("today", "now", "{today}", "${today}", template
- *     variables) is replaced with today's ISO date.
- *   - A map/object (e.g. `{today: null}`) is replaced with today's ISO date.
- *   - `null`, empty string, missing → left alone (no field added; reviewers
- *     should not silently gain metadata they didn't write).
- *
- * Exported for unit testing.
+ * Normalize a leaked `updated` placeholder in place: a Date becomes
+ * `YYYY-MM-DD`; `today`/`now`/`{today}`-style templates and maps like
+ * `{today: null}` become today's date; real dates, unknown strings and empty
+ * values are left alone (visible in the diff, never invented).
  */
 export function normalizeUpdatedField(fm) {
-    if (!("updated" in fm))
-        return;
     const v = fm.updated;
     if (v === null || v === undefined || v === "")
         return;
     const todayIso = new Date().toISOString().slice(0, 10);
     if (v instanceof Date) {
         fm.updated = v.toISOString().slice(0, 10);
-        return;
     }
-    if (typeof v === "string") {
+    else if (typeof v === "string") {
         const trimmed = v.trim().toLowerCase();
         if (/^\d{4}-\d{2}-\d{2}/.test(v.trim()))
-            return; // already a real date
+            return;
         if (trimmed === "today" ||
             trimmed === "now" ||
-            trimmed === "{today}" ||
-            // biome-ignore lint/suspicious/noTemplateCurlyInString: matches the literal user-typed placeholder text "${today}" so we can normalize it to today's ISO date
+            // biome-ignore lint/suspicious/noTemplateCurlyInString: the literal "${today}" placeholder a model emits
             trimmed === "${today}" ||
             trimmed === "{{today}}" ||
             /^\{?\s*today\s*\}?$/.test(trimmed)) {
             fm.updated = todayIso;
-            return;
         }
-        // Unknown string format — leave alone so it's visible in the diff.
-        return;
     }
-    if (typeof v === "object") {
-        // Maps like `{today: null}`, `{now: null}` — clearly a template leak.
+    else if (typeof v === "object") {
         fm.updated = todayIso;
-        return;
     }
 }