@ai-matrx/content-ir 0.13.0 → 0.14.0

package/CHANGELOG.md

@@ -1,5 +1,76 @@
 # Changelog
 
+## 0.14.0 — 2026-09-25
+
+**`/source` — RC-B3 polish: the re-verification residuals, and the verifier's
+attacks made permanent** (common-docs
+`projects/rich-content-unification/evidence/verify-RC-B3.md` § Re-verification).
+Each residual had a failing test first (R1–R3 red on 0.13.0; the new
+cross-language vectors fail 10 cases on the old Python twin):
+
+- **R1** — an insertion carrying an exact copy of the next island at its
+  expected offset no longer causes a false `integrity` refusal: an island's
+  candidate positions are scored by how much of the original text still
+  surrounds them.
+- **R2** — `tokenizeSource(text, { streaming: true })`: the last,
+  still-unterminated line closes no fence (a bare ```` ``` ```` there may grow
+  into ```` ```bash ````). Stored text omits the option.
+- **R3** — `listIslands` loops instead of spreading, so a block with 150k
+  inline islands saves without a `RangeError`.
+- **R4** — the fence rule defines its whitespace: `FENCE_WHITESPACE`
+  (CommonMark ASCII whitespace) and `trimFenceLine`, which every caller uses
+  in place of a language's `trim()`. The vectors carry U+FEFF, U+0085,
+  U+00A0, U+001C–U+001F and U+2028 lines; aidream matrx-ai's Python twin
+  gained `FENCE_WHITESPACE` / `trim_fence_line` and passes them.
+- **R5** — tilde and odd-whitespace fences: matrx-frontend's
+  `source-covers-renderer` property test proves the tokenizer locks at least
+  every fenced block the renderer draws (both now share `trimFenceLine`).
+- **Permanent attack suite** — `__tests__/source-attack.test.ts`: seeded fuzz
+  of rejoin identity, offsets (independently counted), surrogate safety,
+  no-edit/identity saves, splice verdicts, island refusals, misaligned spans,
+  streaming prefixes (settled islands never move) and pathological timing
+  (budget + linear scaling), bounded in CI; `SOURCE_ATTACK=long` runs the
+  verifier-scale version.
+
+**Consumer action:** fence-line classification callers trim with
+`trimFenceLine(line)` (matrx-frontend's splitters do as of this release).
+
+**`/source` — islands for the extended markdown syntax** (RC-B8, common-docs
+`projects/rich-content-unification/REGISTER.md`). The renderer now draws
+callouts, directives, front matter, footnotes and wikilinks; an editor must
+never re-serialize their grammar, so each is now an island:
+
+- `directive` (block) — a generic-directive container `:::name[label]{attrs}`
+  … `:::` with nesting counted (tabs, columns, details, figures, callouts,
+  TOC), a leaf `::name` line (`meta.leaf`), or a MkDocs admonition
+  `!!! type "Title"` / `??? type` with its indented body (`meta.syntax: "mkdocs"`).
+  An unclosed container runs to the end with `complete: false` (streaming).
+- `callout` (block) — a blockquote opening `> [!TYPE]` (GitHub alert /
+  Obsidian callout; `meta.name`, `meta.fold`).
+- `footnote_def` (block) — `[^id]: …` with its continuation lines (`meta.id`).
+- `front_matter` now also recognizes TOML (`+++`); `meta.format` is `yaml` | `toml`.
+- `wikilink` (inline) — `[[Page]]`, `[[Page|alias]]`, `![[Page]]`
+  (`meta.target`, `meta.embed`); never across a line, never inside code.
+
+CSV/TSV fences were already `fence` islands (`meta.lang`). Guard:
+`__tests__/source-markdown-syntax-islands.test.ts` (7 of 10 red on 0.13.0).
+
+**Consumer action:** an exhaustive `switch` over `IslandType` /
+`BlockIslandType` gains the new members; nothing else changes (joinSource is
+still byte-identical, the corpus check stays green).
+
+**Single-dollar math: single-letter math shapes are math** (RC-B2 verification,
+common-docs `projects/rich-content-unification/evidence/verify-RC-B2.md`).
+`isSingleDollarMath` now also accepts a shape built only from single-letter
+identifiers (primes, standard function names), digits, parentheses, commas and
+`- * /` that applies, primes or juxtaposes them — `$f(g(x))$`, `$u v$`,
+`$f'(x)$`, `$(x, y)$`, `$ln(x)$`. Words keep it text (`$Tipping (fee)$`), and
+the pandoc spacing / digit rules still keep currency (`$5 and $10`, `$5-$10`)
+as text. Guard: `__tests__/single-dollar-math.test.ts` (5 of 12 red before).
+
+**Consumer action:** none — adopt the release; matrx-frontend's math
+normalizer and `/source` tokenizer pick the rule up through the import.
+
 ## 0.13.0 — 2026-09-25
 
 **`/source` — fixes from the RC-B3 independent verification**

package/dist/index.cjs

@@ -5184,11 +5184,22 @@ var NESTING_FENCE_LANGUAGES = /* @__PURE__ */ new Set([
 function fenceNestsInnerFences(language) {
   return !!language && NESTING_FENCE_LANGUAGES.has(language.toLowerCase());
 }
+var FENCE_WHITESPACE = " 	\n\r\f\v";
+function isFenceWhitespace(ch) {
+  return ch !== void 0 && ch !== "" && FENCE_WHITESPACE.includes(ch);
+}
+function trimFenceLine(line) {
+  let start = 0;
+  let end = line.length;
+  while (start < end && isFenceWhitespace(line[start])) start++;
+  while (end > start && isFenceWhitespace(line[end - 1])) end--;
+  return start === 0 && end === line.length ? line : line.slice(start, end);
+}
 function classifyInnerFenceLine(trimmed, openTicks, nests, nestedDepth) {
   let ticks = 0;
   while (ticks < trimmed.length && trimmed[ticks] === "`") ticks++;
   if (ticks < 3) return "content";
-  const info = trimmed.slice(ticks).trim();
+  const info = trimFenceLine(trimmed.slice(ticks));
   if (info === "") {
     if (ticks < openTicks) return "content";
     if (nests && nestedDepth > 0) return "close-nested";
@@ -5201,13 +5212,42 @@ function classifyInnerFenceLine(trimmed, openTicks, nests, nestedDepth) {
 // source/math.ts
 var TEX_SIGNAL = /\\[A-Za-z]+|[\\^_{}=<>+]/;
 var LONE_VARIABLE = /^[A-Za-z](?:'|[0-9])?$/;
+var MATH_SHAPE_CHARS = /^[A-Za-z0-9'\s(),.*\/-]+$/;
+var FUNCTION_NAMES = /* @__PURE__ */ new Set([
+  "sin",
+  "cos",
+  "tan",
+  "sec",
+  "csc",
+  "cot",
+  "log",
+  "ln",
+  "exp",
+  "lim",
+  "max",
+  "min",
+  "det",
+  "gcd",
+  "mod",
+  "arg",
+  "sinh",
+  "cosh",
+  "tanh"
+]);
+function isMathShape(content) {
+  if (!MATH_SHAPE_CHARS.test(content)) return false;
+  const words = content.match(/[A-Za-z]+/g);
+  if (!words) return false;
+  if (!words.every((w) => w.length === 1 || FUNCTION_NAMES.has(w))) return false;
+  return /[A-Za-z]'*\(|[A-Za-z]'|\b[A-Za-z]\s+[A-Za-z]\b|\([^)]*[A-Za-z][^)]*\)/.test(content);
+}
 function isSingleDollarMath(content, after) {
   if (!content || content.length > 400) return false;
   if (/^\s/.test(content) || /\s$/.test(content)) return false;
   if (after !== void 0 && /[0-9]/.test(after)) return false;
   if (LONE_VARIABLE.test(content)) return true;
   if (/^[0-9]/.test(content)) return /\\[A-Za-z]+|[\\^_{}]/.test(content);
-  return TEX_SIGNAL.test(content);
+  return TEX_SIGNAL.test(content) || isMathShape(content);
 }
 function singleDollarMathEnd(text, open, limit = text.length) {
   const bound = Math.min(limit, open + 403);
@@ -5443,6 +5483,10 @@ var XmlContainerTracker = class {
 
 // source/tokenize.ts
 var NOT_WHITESPACE = /\S/;
+var DIRECTIVE_OPEN = /^(:{2,})([A-Za-z][\w-]*)/;
+var MKDOCS_OPEN = /^(!!!|\?\?\?\+?)[ \t]+([A-Za-z][\w-]*)/;
+var CALLOUT_OPEN = /^>[ \t]*\[!([A-Za-z][\w-]*)\]([+-])?/;
+var FOOTNOTE_DEF = /^\[\^([^\]\s]+)\]:/;
 function countStructuralBraces(source) {
   let opens = 0;
   let closes = 0;
@@ -5570,8 +5614,9 @@ function isSplitterTreeLine(line) {
 }
 var MEDIA_REF_PREFIXES = ["[Image URL:", "[Video URL:", "[Audio URL:"];
 var Tokenizer = class {
-  constructor(text) {
+  constructor(text, streaming = false) {
     this.text = text;
+    this.streaming = streaming;
     this.n = text.length;
     this.mathPairs = pairDisplayMath(text);
     let start = 0;
@@ -5590,6 +5635,7 @@ var Tokenizer = class {
     }
   }
   text;
+  streaming;
   n;
   lineStarts = [];
   contentEnds = [];
@@ -5627,6 +5673,10 @@ var Tokenizer = class {
     }
     return lo < list.length ? list[lo] : -1;
   }
+  /** Streaming only: the final line, still waiting for its terminator. */
+  isOpenTail(k) {
+    return this.streaming && k === this.lineStarts.length - 1 && this.lineEnds[k] === this.contentEnds[k];
+  }
   get lineCount() {
     return this.lineStarts.length;
   }
@@ -5708,6 +5758,22 @@ var Tokenizer = class {
       const fm = this.frontMatter(li, t);
       if (fm) return fm;
     }
+    if (indent <= 3 && t.startsWith("::")) {
+      const directive = this.directive(p, li, t);
+      if (directive) return directive;
+    }
+    if (indent === 0 && (t.startsWith("!!!") || t.startsWith("???"))) {
+      const admonition = this.mkdocsAdmonition(p, li, t);
+      if (admonition) return admonition;
+    }
+    if (indent <= 3 && t.startsWith(">")) {
+      const callout = this.callout(p, li, t);
+      if (callout) return callout;
+    }
+    if (atParaStart && indent <= 3 && t.startsWith("[^")) {
+      const footnote = this.footnoteDefinition(p, li, t);
+      if (footnote) return footnote;
+    }
     if (t.startsWith("```")) return this.backtickFence(p, li, t);
     if (indent <= 3 && t.startsWith("~~~")) {
       const tilde = this.tildeFence(p, li, t);
@@ -5745,19 +5811,110 @@ var Tokenizer = class {
     return { type, start, end, complete, meta };
   }
   frontMatter(li, t) {
-    if (this.lineText(li) !== "---" || t !== "---") return null;
+    const fence = this.lineText(li);
+    if (fence !== "---" && fence !== "+++" || t !== fence) return null;
+    const toml = fence === "+++";
     let sawContent = false;
     for (let k = li + 1; k < this.lineCount; k++) {
       const line = this.lineText(k);
-      if (line === "---" || line === "...") {
-        return sawContent ? this.island("front_matter", 0, this.contentEnds[k], true) : null;
+      if (line === fence || !toml && line === "...") {
+        return sawContent ? this.island("front_matter", 0, this.contentEnds[k], true, { format: toml ? "toml" : "yaml" }) : null;
       }
       if (!NOT_WHITESPACE.test(line)) continue;
-      if (!/^(?:[A-Za-z0-9_"'-][^:]*:(?:\s|$)|\s+\S|-\s)/.test(line)) return null;
+      const shaped = toml ? /^\s*(?:[A-Za-z0-9_."-]+\s*=|\[|#)/.test(line) : /^(?:[A-Za-z0-9_"'-][^:]*:(?:\s|$)|\s+\S|-\s|#)/.test(line);
+      if (!shaped) return null;
       sawContent = true;
     }
     return null;
   }
+  /**
+   * `:::name[label]{attrs}` … closing `:::` (a colon run at least as long as
+   * the opener's, nesting counted), or a leaf `::name[…]{…}` line. The whole
+   * container is ONE island: its grammar (fence lengths, labels, attributes,
+   * nested tabs) is exactly what a visual editor would re-serialize wrongly.
+   */
+  directive(p, li, t) {
+    const open = DIRECTIVE_OPEN.exec(t);
+    if (!open) return null;
+    const colons = open[1].length;
+    const name = open[2];
+    if (colons === 2) {
+      return this.island("directive", p, this.contentEnds[li], true, { name, leaf: true });
+    }
+    const stack = [colons];
+    for (let k = li + 1; k < this.lineCount; k++) {
+      const line = this.lineText(k).trim();
+      const inner = DIRECTIVE_OPEN.exec(line);
+      if (inner && inner[1].length >= 3) {
+        stack.push(inner[1].length);
+        continue;
+      }
+      const close = /^(:{3,})\s*$/.exec(line);
+      if (close && close[1].length >= stack[stack.length - 1]) {
+        stack.pop();
+        if (stack.length === 0) return this.island("directive", p, this.contentEnds[k], true, { name });
+      }
+    }
+    return this.island("directive", p, this.n, false, { name });
+  }
+  /** MkDocs `!!! type "Title"` / `??? type` / `???+ type` and its indented (or blank-separated indented) body. */
+  mkdocsAdmonition(p, li, t) {
+    const open = MKDOCS_OPEN.exec(t);
+    if (!open) return null;
+    let last = li;
+    for (let k = li + 1; k < this.lineCount; k++) {
+      const line = this.lineText(k);
+      if (/^(?: {4}|\t)/.test(line) && NOT_WHITESPACE.test(line)) {
+        last = k;
+        continue;
+      }
+      if (!NOT_WHITESPACE.test(line)) continue;
+      break;
+    }
+    return this.island("directive", p, this.contentEnds[last], true, {
+      name: open[2].toLowerCase(),
+      syntax: "mkdocs"
+    });
+  }
+  /** A blockquote whose first line is `> [!TYPE]…` — every following `>` line belongs to it. */
+  callout(p, li, t) {
+    const open = CALLOUT_OPEN.exec(t);
+    if (!open) return null;
+    let last = li;
+    for (let k = li + 1; k < this.lineCount; k++) {
+      if (!/^ {0,3}>/.test(this.lineText(k))) break;
+      last = k;
+    }
+    return this.island("callout", p, this.contentEnds[last], true, {
+      name: open[1].toLowerCase(),
+      ...open[2] ? { fold: open[2] === "-" ? "closed" : "open" } : {}
+    });
+  }
+  /** `[^id]: text` plus its continuation lines (indented, or lazy text before a blank line). */
+  footnoteDefinition(p, li, t) {
+    const open = FOOTNOTE_DEF.exec(t);
+    if (!open) return null;
+    let last = li;
+    for (let k = li + 1; k < this.lineCount; k++) {
+      const line = this.lineText(k);
+      if (!NOT_WHITESPACE.test(line)) {
+        let j = k + 1;
+        while (j < this.lineCount && !NOT_WHITESPACE.test(this.lineText(j))) j++;
+        if (j < this.lineCount && /^(?: {4}|\t)/.test(this.lineText(j))) {
+          k = j - 1;
+          continue;
+        }
+        break;
+      }
+      if (/^(?: {4}|\t)/.test(line)) {
+        last = k;
+        continue;
+      }
+      if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
+      last = k;
+    }
+    return this.island("footnote_def", p, this.contentEnds[last], true, { id: open[1] });
+  }
   /**
    * Backtick fence — the renderer splitter's close rules: JSON-string aware for
    * ```json, and THE nested-fence rule (source/fence-nesting.ts) for markdown
@@ -5786,8 +5943,9 @@ var Tokenizer = class {
     let nestedDepth = 0;
     let sawNested = false;
     for (let k = li + 1; k < this.lineCount; k++) {
+      if (this.isOpenTail(k)) continue;
       const line = this.lineText(k);
-      const trimmedLine = line.trim();
+      const trimmedLine = trimFenceLine(line);
       if (trimmedLine.startsWith("```")) {
         const at2 = line.indexOf("```");
         if (isJson) {
@@ -5817,7 +5975,7 @@ var Tokenizer = class {
           }
         }
         const closeTicks = backtickRunLength(line, at);
-        const after = line.slice(at + closeTicks).trim();
+        const after = trimFenceLine(line.slice(at + closeTicks));
         if (closeTicks >= openTicks && after === "" && nestedDepth === 0) return k;
         if (isJson) state = jsonStringState(line, 0, line.length, state);
         continue;
@@ -5844,6 +6002,7 @@ var Tokenizer = class {
     if (normalized && SPECIAL_CODE_LANGUAGES.has(normalized)) meta.special = normalized;
     const close = new RegExp(`^ {0,3}~{${ticks},}[ \\t]*$`);
     for (let k = li + 1; k < this.lineCount; k++) {
+      if (this.isOpenTail(k)) continue;
       if (close.test(this.lineText(k))) {
         return this.island("fence", p, this.contentEnds[k], true, meta);
       }
@@ -6233,6 +6392,21 @@ var Tokenizer = class {
         i++;
         continue;
       }
+      if (ch === "[" && text[i + 1] === "[" || ch === "!" && text[i + 1] === "[" && text[i + 2] === "[") {
+        const open = ch === "!" ? i + 3 : i + 2;
+        const close = within("]]", open);
+        const newline = this.next("\n", open);
+        const inner = close === -1 ? "" : text.slice(open, close);
+        if (close !== -1 && (newline === -1 || newline > close) && inner.trim() && !/[[\]]/.test(inner)) {
+          const cut = inner.search(/\\?\|/);
+          const target = (cut < 0 ? inner : inner.slice(0, cut)).trim();
+          inlines.push(
+            this.island("wikilink", i, close + 2, true, ch === "!" ? { target, embed: true } : { target })
+          );
+          i = close + 2;
+          continue;
+        }
+      }
       if (ch === "[") {
         const media = MEDIA_REF_PREFIXES.find((prefix) => text.startsWith(prefix, i));
         if (media) {
@@ -6314,8 +6488,8 @@ var Tokenizer = class {
     return { inlines, breakIsland: null, orphan: false };
   }
 };
-function tokenizeSource(text) {
-  const raw = new Tokenizer(text).run();
+function tokenizeSource(text, options = {}) {
+  const raw = new Tokenizer(text, options.streaming ?? false).run();
   const cp = buildCodePointIndex(text);
   const inline = (island) => ({
     islandType: island.type,
@@ -6364,7 +6538,7 @@ function listIslands(blocks) {
         raw: block.raw
       });
     } else {
-      out.push(...block.inlines);
+      for (const inline of block.inlines) out.push(inline);
     }
   }
   return out;
@@ -6400,15 +6574,37 @@ function islandEdit(island, text) {
 function blockRangeEdit(first, last, text) {
   return { start: first.start, end: last.end, text };
 }
-function locateIsland(text, raw, from, expected, expectedFromEnd) {
-  if (expected >= from && text.startsWith(raw, expected)) return expected;
-  if (expectedFromEnd >= from && text.startsWith(raw, expectedFromEnd)) return expectedFromEnd;
+var CONTEXT_WINDOW = 64;
+function locateIsland(text, raw, from, expected, expectedFromEnd, oldText, oldStart, oldEnd) {
+  const candidates = /* @__PURE__ */ new Set();
+  if (expected >= from && text.startsWith(raw, expected)) candidates.add(expected);
+  if (expectedFromEnd >= from && text.startsWith(raw, expectedFromEnd)) candidates.add(expectedFromEnd);
   const after = text.indexOf(raw, Math.max(from, expected));
+  if (after !== -1) candidates.add(after);
   const before = expected > from ? text.lastIndexOf(raw, expected) : -1;
-  const beforeOk = before >= from ? before : -1;
-  if (after === -1) return beforeOk;
-  if (beforeOk === -1) return after;
-  return expected - beforeOk <= after - expected ? beforeOk : after;
+  if (before >= from) candidates.add(before);
+  if (candidates.size === 0) return text.indexOf(raw, from);
+  if (candidates.size === 1) return candidates.values().next().value;
+  let best = -1;
+  let bestScore = -1;
+  for (const at of candidates) {
+    let score = 0;
+    for (let k = 0; k < CONTEXT_WINDOW; k++) {
+      const n = text[at + raw.length + k];
+      if (n === void 0 || n !== oldText[oldEnd + k]) break;
+      score++;
+    }
+    for (let k = 1; k <= CONTEXT_WINDOW; k++) {
+      const n = text[at - k];
+      if (n === void 0 || n !== oldText[oldStart - k]) break;
+      score++;
+    }
+    if (score > bestScore || score === bestScore && at === expected) {
+      best = at;
+      bestScore = score;
+    }
+  }
+  return best;
 }
 function commonPrefix(a, b) {
   const max = Math.min(a.length, b.length);
@@ -6485,7 +6681,16 @@ function spliceSave(original, edits, options = {}) {
     let oldCursor2 = edit.start;
     let newCursor2 = 0;
     for (const island of inside) {
-      const at = locateIsland(edit.text, island.raw, newCursor2, newCursor2 + (island.start - oldCursor2), edit.text.length - (edit.end - island.start));
+      const at = locateIsland(
+        edit.text,
+        island.raw,
+        newCursor2,
+        newCursor2 + (island.start - oldCursor2),
+        edit.text.length - (edit.end - island.start),
+        original.slice(edit.start, edit.end),
+        island.start - edit.start,
+        island.end - edit.start
+      );
       if (at === -1) {
         const name = island.meta.tag ?? island.meta.name ?? island.meta.lang ?? "";
         throw new SourceSpliceError(
@@ -6628,6 +6833,7 @@ exports.CAPABILITY_BY_CLASS = CAPABILITY_BY_CLASS;
 exports.CLASSES = CLASSES;
 exports.DIRECTIVE_VERSION = DIRECTIVE_VERSION;
 exports.DirectiveDecodeError = DirectiveDecodeError;
+exports.FENCE_WHITESPACE = FENCE_WHITESPACE;
 exports.IN_CONTENT_CLASSES = IN_CONTENT_CLASSES;
 exports.IR_ENVELOPE_CACHE_VERSION = IR_ENVELOPE_CACHE_VERSION;
 exports.IR_ENVELOPE_KEY = IR_ENVELOPE_KEY;
@@ -6759,6 +6965,7 @@ exports.titleCaseToken = titleCaseToken;
 exports.toCodePointOffset = toCodePointOffset;
 exports.toUtf16Offset = toUtf16Offset;
 exports.tokenizeSource = tokenizeSource;
+exports.trimFenceLine = trimFenceLine;
 exports.tryDecodeDirective = tryDecodeDirective;
 exports.tryDecodeDirectiveContent = tryDecodeDirectiveContent;
 exports.validateBlockSchemaSavePlan = validateBlockSchemaSavePlan;