@ai-matrx/content-ir 0.21.0 → 0.21.2

package/CHANGELOG.md

@@ -1,5 +1,35 @@
 # Changelog
 
+## 0.21.2
+
+### `/source`: a ```json fence's string state never crosses a line end (never-raw R11 H2)
+
+- `FenceReader` (behind `closeFence`, `fenceLineKinds`, `findCodeRanges`, the XML / directive balance walkers) read
+  JSON-string state across lines inside a ```json fence. A JSON string never holds a raw newline, so a body line
+  that ends inside one (`{\"__kind\":\"note\"}` — escaped quotes, a cut value) swallowed the closing ``` and every
+  following list item / quoted line. The state is now read per line; a backtick run inside a string on the same line
+  is still text.
+
+### Consumer action
+
+- None required. matrx-frontend may drop its splitter workaround (`jsonLineForFenceReader` in
+  content-splitter-core.ts) once on 0.21.2.
+
+## 0.21.1
+
+### `/source`: the tokenizer locks what the renderer draws (fuzz `source-covers-renderer`)
+
+- A `~~~` island now ends at its closer. The 0.14.2 "renderer overflow" assumed the renderer did not know `~~~`
+  (it has since matrx-frontend A4, 2026-09-30), and swallowed the next fence's opener into the tilde island,
+  leaving a following ````` ````md ````` block editable.
+- A JSON-family fence (unlabelled / json / jsonc / json5) opened on a quoted line (`> ```) is an island up to its
+  own closer, which may carry no `>`: the renderer lifts such a region out of its quote (V1 quoted-kind-lift). A
+  quoted fence of another language stays prose (source).
+
+### Consumer action
+
+- None required.
+
 ## 0.21.0
 
 ### `/source`: the markdown text helpers move in from matrx-frontend (chat-package-move P14)

package/dist/index.cjs

@@ -5487,9 +5487,9 @@ function fenceOpenerOf(content) {
 function opensStrictFence(line) {
   return fenceOpenerOf(line) !== null;
 }
+var LINE_START = { inString: false, escaped: false };
 var FenceReader = class {
   opener;
-  state = { inString: false, escaped: false };
   depth = 0;
   /** A nested fence was opened at some point. */
   sawNested = false;
@@ -5510,17 +5510,8 @@ var FenceReader = class {
     if (this.tildeCloser) return this.tildeCloser.test(line);
     const trimmedLine = trimFenceLine(line);
     const at = line.indexOf("```");
-    if (at === -1) {
-      if (this.isJson) this.state = jsonStringState(line, 0, line.length, this.state);
-      return false;
-    }
-    if (this.isJson) {
-      const probe = jsonStringState(line, 0, at, this.state);
-      if (probe.inString) {
-        this.state = jsonStringState(line, at, line.length, probe);
-        return false;
-      }
-    }
+    if (at === -1) return false;
+    if (this.isJson && jsonStringState(line, 0, at, LINE_START).inString) return false;
     if (trimmedLine.startsWith("```")) {
       const kind = classifyInnerFenceLine(trimmedLine, this.opener.ticks, this.nesting, this.depth);
       if (kind === "close-outer") return true;
@@ -5534,7 +5525,6 @@ var FenceReader = class {
       const after = trimFenceLine(line.slice(at + closeTicks));
       if (closeTicks >= this.opener.ticks && after === "" && this.depth === 0) return true;
     }
-    if (this.isJson) this.state = jsonStringState(line, 0, line.length, this.state);
     return false;
   }
 };
@@ -6418,6 +6408,8 @@ function findTableEnd(lines, start) {
 }
 
 // source/tokenize.ts
+var QUOTE_PREFIX = /^ {0,3}>(?: ?>)* ?/;
+var QUOTED_JSON_FENCE_LANGS = /* @__PURE__ */ new Set(["", "json", "jsonc", "json5"]);
 var NOT_WHITESPACE = /\S/;
 var DIRECTIVE_OPEN = /^(:{2,})([A-Za-z][\w-]*)/;
 var MKDOCS_OPEN = /^(!!!|\?\?\?\+?)[ \t]+([A-Za-z][\w-]*)/;
@@ -6907,6 +6899,8 @@ var Tokenizer = class {
     if (indent <= 3 && !listNested && t.startsWith(">")) {
       const callout = this.callout(p, li, t);
       if (callout) return callout;
+      const lifted = this.quotedJsonFence(p, li, seg);
+      if (lifted) return lifted;
     }
     if (atParaStart && indent <= 3 && t.startsWith("[^")) {
       const footnote = this.footnoteDefinition(p, li, t);
@@ -7102,6 +7096,42 @@ var Tokenizer = class {
       streaming: this.streaming
     };
   }
+  /**
+   * A JSON-family fence (unlabelled, json, jsonc, json5) opening on a quoted line. The renderer LIFTS it out
+   * of the quote (matrx-frontend V1, never-raw law): the quote prefix is removed from every line of the
+   * region, so the region is an ordinary top-level fence — drawn as code up to its own closer, which may sit on
+   * a line that carries no `>` at all. The editor must lock at least what the renderer draws, so the island
+   * runs from the quoted opener line to that closer (found on the lines with their quote prefix removed).
+   */
+  quotedJsonFence(p, li, seg) {
+    const m = QUOTE_PREFIX.exec(seg);
+    if (!m) return null;
+    const opener = fenceOpenerOf(seg.slice(m[0].length));
+    if (!opener || opener.char !== "`" || !QUOTED_JSON_FENCE_LANGS.has(opener.lang.toLowerCase())) return null;
+    const base2 = this.fenceLines;
+    const lines = {
+      ...base2,
+      line: (k) => {
+        const text = base2.line(k);
+        if (text === null) return null;
+        const q = QUOTE_PREFIX.exec(text);
+        return q ? text.slice(q[0].length) : text;
+      }
+    };
+    let first = li;
+    while (first > 0 && li - first < 400 && QUOTE_PREFIX.test(this.lineText(first - 1))) first--;
+    for (let k = first; k < li; k++) {
+      const o = fenceOpenerOf(lines.line(k) ?? "");
+      if (!o || o.char !== "`" || QUOTED_JSON_FENCE_LANGS.has(o.lang.toLowerCase())) continue;
+      const c = closeFence(lines, k, o);
+      if (!c.closed || c.line >= li) return null;
+      k = c.line;
+    }
+    const end = closeFence(lines, li, opener);
+    const meta = { fence: "`", ticks: opener.ticks, lang: opener.lang };
+    if (end.closed) return this.island("fence", p, this.contentEnds[end.line], true, meta);
+    return this.island("fence", p, this.n, false, meta);
+  }
   /**
    * Backtick fence — THE one closer (`closeFence`, source/code-ranges.ts): the
    * nested-fence rule for markdown fences with the strict-CommonMark retry, the
@@ -7134,44 +7164,14 @@ var Tokenizer = class {
     if (normalized && SPECIAL_CODE_LANGUAGES.has(normalized)) meta.special = normalized;
     const close = closeFence(this.fenceLines, li, opener);
     if (close.closed) {
-      const tildeEnd = this.contentEnds[close.line];
-      const end = this.rendererOverflow(li + 1, close.line, tildeEnd);
-      return this.island("fence", p, end, end === this.n ? this.overflowComplete : true, meta);
+      return this.island("fence", p, this.contentEnds[close.line], true, meta);
     }
     return this.island("fence", p, this.n, false, meta);
   }
-  /** Completion of the last `rendererOverflow` that ran to the end of the text. */
-  overflowComplete = true;
-  /**
-   * The renderer's splitter does not know ~~~ fences: it reads the lines inside
-   * one as ordinary lines, so a ```` ``` ```` fence (or an XML section, a JSON
-   * block…) that STARTS inside the tilde fence can run past its closer. The
-   * editor must lock at least what the renderer draws (verify-RC-B3 residual
-   * R5), so a tilde island extends over every block the renderer would detect
-   * starting inside it. Returns the island's end.
-   */
-  rendererOverflow(firstLine, closeLine, tildeEnd) {
-    let end = tildeEnd;
-    this.overflowComplete = true;
-    for (let j = firstLine; j <= closeLine && j < this.lineCount; j++) {
-      const start = this.lineStarts[j];
-      if (this.lineIsBlank(j)) continue;
-      if (fenceOpenerOf(this.lineText(j))?.char === "~") continue;
-      const inner = this.detectBlockUncached(start, j, false);
-      if (!inner) continue;
-      if (this.streaming && inner.end >= tildeEnd && this.paragraphMayGrow(inner.end)) {
-        this.overflowComplete = false;
-        return this.n;
-      }
-      if (inner.end > end) {
-        end = inner.end;
-        this.overflowComplete = inner.complete;
-      }
-      const lastLine = this.lineOf(Math.max(start, inner.end - 1));
-      if (lastLine > j) j = lastLine;
-    }
-    return end;
-  }
+  // A tilde island ends at its closer. The renderer's splitter treats `~~~` as a real fence since
+  // matrx-frontend A4 (2026-09-30): the lines inside are code, so nothing that starts inside one runs
+  // past the closer (the 0.14.2 "overflow" assumed the renderer did not know ~~~ and, kept, swallowed the
+  // next fence's opener into the tilde island: a ````md block after a ~~~ block was left unlocked).
   /**
    * A line-leading `$$` that OPENS a math pair (source/math-pairs.ts — the
    * core's rule). A closer, an unpaired `$$`, or a prose pair is never a block: