@ai-matrx/content-ir 0.16.2 → 0.18.0

package/dist/index.js

@@ -5221,65 +5221,313 @@ function classifyInnerFenceLine(trimmed, openTicks, nests, nestedDepth) {
   return "content";
 }
 
-// source/math.ts
-var TEX_SIGNAL = /\\[A-Za-z]+|[\\^_{}=<>+]/;
-var LONE_VARIABLE = /^[A-Za-z](?:'|[0-9])?$/;
-var MATH_SHAPE_CHARS = /^[A-Za-z0-9'\s(),.*\/-]+$/;
-var FUNCTION_NAMES = /* @__PURE__ */ new Set([
-  "sin",
-  "cos",
-  "tan",
-  "sec",
-  "csc",
-  "cot",
-  "log",
-  "ln",
-  "exp",
-  "lim",
-  "max",
-  "min",
-  "det",
-  "gcd",
-  "mod",
-  "arg",
-  "sinh",
-  "cosh",
-  "tanh"
-]);
-function isMathShape(content) {
-  if (!MATH_SHAPE_CHARS.test(content)) return false;
-  const words = content.match(/[A-Za-z]+/g);
-  if (!words) return false;
-  if (!words.every((w) => w.length === 1 || FUNCTION_NAMES.has(w))) return false;
-  if (/[A-Za-z]'+[A-Za-z]/.test(content) || /(?:^|[\s(,])[A-Za-z]\.(?:[A-Za-z]\.|\s|$)/.test(content)) return false;
-  return /[A-Za-z]'*\(|[A-Za-z]'|\b[A-Za-z]\s+[A-Za-z]\b|\([^)]*[A-Za-z][^)]*\)/.test(content);
+// source/code-ranges.ts
+function fenceOpenerOf(content) {
+  const t = content.trimStart();
+  if (t.startsWith("```")) {
+    let ticks = 0;
+    while (ticks < t.length && t[ticks] === "`") ticks++;
+    return { char: "`", ticks, lang: parseFenceOpener(t)?.lang ?? "" };
+  }
+  if (content.length - t.length <= 3 && t.startsWith("~~~")) {
+    const open = /^(~{3,})(.*)$/.exec(t);
+    if (!open) return null;
+    return { char: "~", ticks: open[1].length, lang: open[2].trim().split(/\s/)[0] ?? "" };
+  }
+  return null;
 }
-function isSingleDollarMath(content, after) {
-  if (!content || content.length > 400) return false;
-  if (/^\s/.test(content) || /\s$/.test(content)) return false;
-  if (after !== void 0 && /[0-9]/.test(after)) return false;
-  if (LONE_VARIABLE.test(content)) return true;
-  if (/^[0-9]/.test(content)) return /\\[A-Za-z]+|[\\^_{}]/.test(content);
-  return TEX_SIGNAL.test(content) || isMathShape(content);
+var FenceReader = class {
+  constructor(opener, options = {}) {
+    this.opener = opener;
+    this.isJson = opener.char === "`" && opener.lang === "json";
+    this.nesting = opener.char === "`" && (options.nesting ?? fenceNestsInnerFences(opener.lang));
+    this.tildeCloser = opener.char === "~" ? new RegExp(`^ {0,3}~{${opener.ticks},}[ \\t]*$`) : null;
+  }
+  opener;
+  state = { inString: false, escaped: false };
+  depth = 0;
+  /** A nested fence was opened at some point. */
+  sawNested = false;
+  isJson;
+  nesting;
+  tildeCloser;
+  /** How many nested fences are open right now. */
+  get nestedDepth() {
+    return this.depth;
+  }
+  feed(line) {
+    if (this.tildeCloser) return this.tildeCloser.test(line);
+    const trimmedLine = trimFenceLine(line);
+    const at = line.indexOf("```");
+    if (at === -1) {
+      if (this.isJson) this.state = jsonStringState(line, 0, line.length, this.state);
+      return false;
+    }
+    if (this.isJson) {
+      const probe = jsonStringState(line, 0, at, this.state);
+      if (probe.inString) {
+        this.state = jsonStringState(line, at, line.length, probe);
+        return false;
+      }
+    }
+    if (trimmedLine.startsWith("```")) {
+      const kind = classifyInnerFenceLine(trimmedLine, this.opener.ticks, this.nesting, this.depth);
+      if (kind === "close-outer") return true;
+      if (kind === "open-nested") {
+        this.depth++;
+        this.sawNested = true;
+      }
+      if (kind === "close-nested") this.depth--;
+    } else {
+      const closeTicks = backtickRunLength(line, at);
+      const after = trimFenceLine(line.slice(at + closeTicks));
+      if (closeTicks >= this.opener.ticks && after === "" && this.depth === 0) return true;
+    }
+    if (this.isJson) this.state = jsonStringState(line, 0, line.length, this.state);
+    return false;
+  }
+};
+function closeFence(src, open, opener) {
+  const first = readFence(src, open, new FenceReader(opener));
+  if (first.closed || !first.reader.sawNested || first.reader.nestedDepth <= 0) return { line: first.line, closed: first.closed };
+  const strict = readFence(src, open, new FenceReader(opener, { nesting: false }));
+  return { line: strict.line, closed: strict.closed };
 }
-function singleDollarMathEnd(text, open, limit = text.length) {
-  const bound = Math.min(limit, open + 403);
-  let close = -1;
-  for (let k = open + 1; k < bound; k += 1) {
-    const c = text[k];
-    if (c === "\n") break;
-    if (c === "\\") {
-      k += 1;
+function readFence(src, open, reader) {
+  for (let k = open + 1; k < src.count; k++) {
+    if (src.openTail?.(k)) continue;
+    const line = src.line(k);
+    if (line === null) return { line: k - 1, closed: false, reader };
+    if (reader.feed(line)) return { line: k, closed: true, reader };
+  }
+  return { line: src.count - 1, closed: false, reader };
+}
+function jsonStringState(line, from, upTo, state) {
+  let { inString, escaped } = state;
+  for (let idx = from; idx < upTo; idx++) {
+    const ch = line[idx];
+    if (escaped) {
+      escaped = false;
       continue;
     }
-    if (c === "$") {
-      close = k;
-      break;
+    if (ch === "\\" && inString) {
+      escaped = true;
+      continue;
     }
+    if (ch === '"') inString = !inString;
   }
-  if (close === -1 || text[close + 1] === "$") return -1;
-  const content = text.slice(open + 1, close);
-  return isSingleDollarMath(content, text[close + 1]) ? close + 1 : -1;
+  return { inString, escaped };
+}
+function backtickRunLength(str2, pos) {
+  let n = 0;
+  while (pos + n < str2.length && str2[pos + n] === "`") n++;
+  return n;
+}
+function quotePrefix(line) {
+  let depth = 0;
+  let offset = 0;
+  for (; ; ) {
+    const m = /^ {0,3}> ?/.exec(line.slice(offset));
+    if (!m) return { depth, offset };
+    depth++;
+    offset += m[0].length;
+  }
+}
+function stripQuotes(line, depth) {
+  let offset = 0;
+  for (let d = 0; d < depth; d++) {
+    const m = /^ {0,3}> ?/.exec(line.slice(offset));
+    if (!m) return null;
+    offset += m[0].length;
+  }
+  return line.slice(offset);
+}
+var LIST_ITEM = /^( *)([-*+]|\d{1,9}[.)])( +|\t|$)/;
+function leadingSpaces(s) {
+  let i = 0;
+  while (i < s.length && s[i] === " ") i++;
+  return i;
+}
+function scan(text) {
+  const starts = [];
+  const ends = [];
+  for (let pos = 0; ; ) {
+    const nl = text.indexOf("\n", pos);
+    const lineEnd = nl === -1 ? text.length : nl;
+    starts.push(pos);
+    ends.push(lineEnd > pos && text.charCodeAt(lineEnd - 1) === 13 ? lineEnd - 1 : lineEnd);
+    if (nl === -1) break;
+    pos = nl + 1;
+  }
+  const count = starts.length;
+  const raw = (k) => text.slice(starts[k], ends[k]);
+  const out = { ranges: [], paragraphs: [], endsInFence: false, pendingPara: -1 };
+  const rawLines = { count, line: raw };
+  let segment = 0;
+  for (let k = 0; k < count; k++) {
+    const opener = fenceOpenerOf(raw(k));
+    if (!opener || opener.char !== "`") continue;
+    prose(text, starts, ends, raw, segment, k, out);
+    const end = closeFence(rawLines, k, opener);
+    out.ranges.push({ start: starts[k], end: ends[end.line], kind: "fence" });
+    out.pendingPara = -1;
+    if (!end.closed && end.line === count - 1) out.endsInFence = true;
+    k = end.line;
+    segment = k + 1;
+  }
+  prose(text, starts, ends, raw, segment, count, out);
+  return out;
+}
+function prose(text, starts, ends, raw, from, to, out) {
+  let para = -1;
+  let quoteDepth = 0;
+  let lists = [];
+  const closePara = (at) => {
+    if (para !== -1) out.paragraphs.push([para, at]);
+    para = -1;
+  };
+  for (let k = from; k < to; k++) {
+    const line = raw(k);
+    const { depth, offset } = quotePrefix(line);
+    if (depth !== quoteDepth) {
+      quoteDepth = depth;
+      lists = [];
+    }
+    let content = line.slice(offset);
+    if (content.trim() === "") {
+      closePara(starts[k]);
+      if (!(k === starts.length - 1 && line === "")) out.pendingPara = -1;
+      continue;
+    }
+    const indent = leadingSpaces(content);
+    while (lists.length && indent < lists[lists.length - 1]) lists.pop();
+    const item = LIST_ITEM.exec(content);
+    let column = lists.length ? lists[lists.length - 1] : 0;
+    if (item) {
+      const gap = item[3].length;
+      column = item[1].length + item[2].length + (gap >= 1 && gap <= 4 ? gap : 1);
+      lists.push(column);
+      content = content.slice(Math.min(column, content.length));
+    } else {
+      content = content.slice(Math.min(indent, column));
+    }
+    const opener = fenceOpenerOf(content);
+    if (!opener) {
+      if (para === -1) para = starts[k];
+      if (out.pendingPara === -1) out.pendingPara = starts[k];
+      continue;
+    }
+    closePara(starts[k]);
+    out.pendingPara = -1;
+    const col = column;
+    const src = {
+      count: to,
+      line: (j) => {
+        const inner = stripQuotes(raw(j), depth);
+        if (inner === null) return null;
+        if (opener.char === "~" && col > 0) {
+          const ind = leadingSpaces(inner);
+          if (inner.trim() !== "" && ind < col) return null;
+          return inner.slice(Math.min(ind, col));
+        }
+        return inner;
+      }
+    };
+    const end = closeFence(src, k, opener);
+    out.ranges.push({ start: starts[k], end: ends[end.line], kind: "fence" });
+    if (!end.closed && end.line === starts.length - 1) out.endsInFence = true;
+    k = end.line;
+  }
+  closePara(to < starts.length ? starts[to] : text.length);
+}
+var BacktickRuns = class {
+  /** Every run, in order: [start, length]. */
+  all = [];
+  byLength = /* @__PURE__ */ new Map();
+  indexOf = /* @__PURE__ */ new Map();
+  constructor(text, from, to) {
+    for (let k = from; k < to; ) {
+      if (text.charCodeAt(k) !== 96) {
+        k++;
+        continue;
+      }
+      let len = 1;
+      while (k + len < to && text.charCodeAt(k + len) === 96) len++;
+      this.indexOf.set(k, this.all.length);
+      this.all.push([k, len]);
+      const list = this.byLength.get(len);
+      if (list) list.push(k);
+      else this.byLength.set(len, [k]);
+      k += len;
+    }
+  }
+  /** Start of the first run of exactly `n` backticks at or after `after`, or -1. */
+  next(n, after) {
+    const list = this.byLength.get(n);
+    if (!list) return -1;
+    let lo = 0;
+    let hi = list.length;
+    while (lo < hi) {
+      const mid = lo + hi >>> 1;
+      if (list[mid] < after) lo = mid + 1;
+      else hi = mid;
+    }
+    return list[lo] ?? -1;
+  }
+  /** Index of the run starting at `start` in `all`. */
+  index(start) {
+    return this.indexOf.get(start) ?? -1;
+  }
+};
+function codeSpans(text, from, to, out) {
+  const runs = new BacktickRuns(text, from, to);
+  for (let r = 0; r < runs.all.length; ) {
+    const [start, n] = runs.all[r];
+    const close = runs.next(n, start + n);
+    if (close === -1) {
+      r++;
+      continue;
+    }
+    out.push({ start, end: close + n, kind: "span" });
+    r = runs.index(close) + 1;
+  }
+}
+function findCodeRanges(text) {
+  const s = scan(text);
+  const ranges = [...s.ranges];
+  for (const [from, to] of s.paragraphs) codeSpans(text, from, to, ranges);
+  return ranges.sort((a, b) => a.start - b.start);
+}
+function pendingCodeSpanStart(text) {
+  const n = text.length;
+  if (n === 0 || /\n[ \t]*\n[ \t]*$/.test(text)) return -1;
+  const s = scan(text);
+  const para = s.pendingPara;
+  if (s.endsInFence || para === -1) return -1;
+  const runs = [];
+  for (let k = para; k < n; ) {
+    if (text.charCodeAt(k) !== 96) {
+      k++;
+      continue;
+    }
+    let len = 1;
+    while (k + len < n && text.charCodeAt(k + len) === 96) len++;
+    runs.push([k, len]);
+    k += len;
+  }
+  const growing = (r) => {
+    const [start, len] = runs[r];
+    return start + len === n;
+  };
+  for (let r = 0; r < runs.length; ) {
+    if (growing(r)) return runs[r][0];
+    const len = runs[r][1];
+    let j = r + 1;
+    while (j < runs.length && (runs[j][1] !== len || growing(j))) j++;
+    if (j >= runs.length) return runs[r][0];
+    r = j + 1;
+  }
+  return -1;
 }
 
 // source/math-pairs.ts
@@ -5308,86 +5556,7 @@ function looksLikeDisplayMath(inner, flow = false) {
   return (s.match(PROSE_WORD) ?? []).length < PROSE_WORD_LIMIT;
 }
 function codeRanges(text) {
-  const ranges = [];
-  const n = text.length;
-  let fence = null;
-  let para = -1;
-  const spans = (from, to) => {
-    const runs = [];
-    for (let k = from; k < to; ) {
-      if (text.charCodeAt(k) !== 96) {
-        k++;
-        continue;
-      }
-      let len = 1;
-      while (k + len < to && text.charCodeAt(k + len) === 96) len++;
-      runs.push([k, len]);
-      k += len;
-    }
-    const nextSame = new Array(runs.length).fill(-1);
-    const last = /* @__PURE__ */ new Map();
-    for (let r = runs.length - 1; r >= 0; r--) {
-      const len = runs[r][1];
-      nextSame[r] = last.get(len) ?? -1;
-      last.set(len, r);
-    }
-    for (let r = 0; r < runs.length; ) {
-      const j = nextSame[r];
-      if (j === -1) {
-        r++;
-        continue;
-      }
-      const [s0] = runs[r];
-      const [s1, l1] = runs[j];
-      ranges.push([s0, s1 + l1]);
-      r = j + 1;
-    }
-  };
-  let pos = 0;
-  while (pos <= n) {
-    const nl = text.indexOf("\n", pos);
-    const lineEnd = nl === -1 ? n : nl;
-    const line = text.slice(pos, lineEnd);
-    let indent = 0;
-    while (indent < line.length && line[indent] === " ") indent++;
-    const body = line.slice(indent);
-    if (fence) {
-      const trimmed = trimFenceLine(line);
-      let closes = false;
-      if (fence.char === "`") {
-        const kind = classifyInnerFenceLine(trimmed, fence.len, fence.nests, fence.depth);
-        if (kind === "open-nested") fence.depth++;
-        else if (kind === "close-nested") fence.depth--;
-        else if (kind === "close-outer") closes = indent <= 3;
-      } else {
-        let t = 0;
-        while (t < trimmed.length && trimmed[t] === "~") t++;
-        closes = indent <= 3 && t >= fence.len && trimFenceLine(trimmed.slice(t)) === "";
-      }
-      if (closes) {
-        ranges.push([fence.start, lineEnd]);
-        fence = null;
-      }
-    } else {
-      const opener = indent <= 3 ? /^(`{3,}|~{3,})(.*)$/.exec(body) : null;
-      const info = opener ? opener[2] : "";
-      if (opener && !(opener[1][0] === "`" && info.includes("`"))) {
-        if (para !== -1) spans(para, pos);
-        para = -1;
-        const char = opener[1][0];
-        const parsed = char === "`" ? parseFenceOpener(line) : null;
-        fence = { char, len: opener[1].length, start: pos, nests: fenceNestsInnerFences(parsed?.lang), depth: 0 };
-      } else if (body.trim() === "") {
-        if (para !== -1) spans(para, pos);
-        para = -1;
-      } else if (para === -1) para = pos;
-    }
-    if (nl === -1) break;
-    pos = nl + 1;
-  }
-  if (fence) ranges.push([fence.start, n]);
-  else if (para !== -1) spans(para, n);
-  return ranges.sort((a, b) => a[0] - b[0]);
+  return findCodeRanges(text).map((r) => [r.start, r.end]);
 }
 function pendingDisplayMathOpener(text) {
   if (!text.includes("$$")) return -1;
@@ -5471,6 +5640,83 @@ function pairDisplayMath(text) {
   return pairs;
 }
 
+// source/math.ts
+var TEX_SIGNAL = /\\[A-Za-z]+|[\\^_{}=<>+]/;
+var LONE_VARIABLE = /^[A-Za-z](?:'|[0-9])?$/;
+var MATH_SHAPE_CHARS = /^[A-Za-z0-9'\s(),.*\/-]+$/;
+function isSingleDollarMath(content, after) {
+  if (!content || content.length > 400) return false;
+  if (/^\s/.test(content) || /\s$/.test(content)) return false;
+  if (after !== void 0 && /[0-9]/.test(after)) return false;
+  if (/(?:^|[^\\])%/.test(content)) return false;
+  if (/^[)\]}"/]/.test(content) || /[(\[{":;,/]$/.test(content)) return false;
+  if (/^[0-9]/.test(content) && after !== void 0 && /[A-Za-z]/.test(after)) return false;
+  if (isMathPlaceholder(content)) return false;
+  return TEX_SIGNAL.test(content) || !readsAsProse(content);
+}
+var FUNCTION_NAMES = /* @__PURE__ */ new Set([
+  "sin",
+  "cos",
+  "tan",
+  "sec",
+  "csc",
+  "cot",
+  "log",
+  "ln",
+  "exp",
+  "lim",
+  "max",
+  "min",
+  "det",
+  "gcd",
+  "mod",
+  "arg",
+  "sinh",
+  "cosh",
+  "tanh"
+]);
+function readsAsProse(content) {
+  if (/[A-Za-z]'[a-z]/.test(content)) return true;
+  if (/(?:^|[\s(,])[A-Za-z]\.(?:[A-Za-z]\.|\s|$)/.test(content)) return true;
+  if (!/\s/.test(content)) return false;
+  const words = (content.match(/[A-Za-z]{3,}/g) ?? []).filter((w) => !FUNCTION_NAMES.has(w.toLowerCase()));
+  return words.length >= 2;
+}
+function looksLikeOpenInlineMath(partial) {
+  if (!partial || partial.length > 400) return false;
+  if (/^\s/.test(partial) || /\s$/.test(partial)) return false;
+  if (LONE_VARIABLE.test(partial)) return true;
+  if (/^[0-9]/.test(partial)) return /\\[A-Za-z]+|[\\^_{}]/.test(partial);
+  return TEX_SIGNAL.test(partial) || isMathShape(partial);
+}
+function isMathShape(content) {
+  if (!MATH_SHAPE_CHARS.test(content)) return false;
+  const words = content.match(/[A-Za-z]+/g);
+  if (!words) return false;
+  if (!words.every((w) => w.length === 1 || FUNCTION_NAMES.has(w))) return false;
+  if (/[A-Za-z]'+[A-Za-z]/.test(content) || /(?:^|[\s(,])[A-Za-z]\.(?:[A-Za-z]\.|\s|$)/.test(content)) return false;
+  return /[A-Za-z]'*\(|[A-Za-z]'|\b[A-Za-z]\s+[A-Za-z]\b|\([^)]*[A-Za-z][^)]*\)/.test(content);
+}
+function singleDollarMathEnd(text, open, limit = text.length) {
+  const bound = Math.min(limit, open + 403);
+  let close = -1;
+  for (let k = open + 1; k < bound; k += 1) {
+    const c = text[k];
+    if (c === "\n") break;
+    if (c === "\\") {
+      k += 1;
+      continue;
+    }
+    if (c === "$") {
+      close = k;
+      break;
+    }
+  }
+  if (close === -1 || text[close + 1] === "$") return -1;
+  const content = text.slice(open + 1, close);
+  return isSingleDollarMath(content, text[close + 1]) ? close + 1 : -1;
+}
+
 // source/xml-tag.ts
 var XML_NAME_START = /[A-Za-z_]/;
 var XML_NAME_CHARACTER = /[\w.:-]/;
@@ -5763,27 +6009,6 @@ function isUnclosedGenericXmlOpening(source) {
   }
   return true;
 }
-function jsonStringState(line, from, upTo, state) {
-  let { inString, escaped } = state;
-  for (let idx = from; idx < upTo; idx++) {
-    const ch = line[idx];
-    if (escaped) {
-      escaped = false;
-      continue;
-    }
-    if (ch === "\\" && inString) {
-      escaped = true;
-      continue;
-    }
-    if (ch === '"') inString = !inString;
-  }
-  return { inString, escaped };
-}
-function backtickRunLength(str2, pos) {
-  let n = 0;
-  while (pos + n < str2.length && str2[pos + n] === "`") n++;
-  return n;
-}
 function isSplitterTreeLine(line) {
   return TREE_GLYPHS.test(line) || /^[\s│|]*[├└+|][\s─\-]+/.test(line);
 }
@@ -5799,6 +6024,14 @@ var Tokenizer = class {
     this.provisionalFrom = streaming ? pendingDisplayMathOpener(text) : -1;
     if (this.provisionalFrom === -1) this.provisionalFrom = Number.POSITIVE_INFINITY;
     if (streaming && text.includes("$$")) {
+      const span = pendingCodeSpanStart(text);
+      if (span !== -1) {
+        const firstToken = text.indexOf("$$", span);
+        if (firstToken !== -1) this.provisionalFrom = Math.min(this.provisionalFrom, firstToken);
+        for (const [open, close] of this.mathPairs) {
+          if (close >= span) this.provisionalFrom = Math.min(this.provisionalFrom, open);
+        }
+      }
       for (const pad of ["`", "``", "```", "~~~"]) {
         const alt = pairDisplayMath(text + pad);
         for (const [open, close] of this.mathPairs) {
@@ -6007,11 +6240,9 @@ var Tokenizer = class {
       const footnote = this.footnoteDefinition(p, li, t);
       if (footnote) return footnote;
     }
-    if (t.startsWith("```")) return this.backtickFence(p, li, t);
-    if (indent <= 3 && t.startsWith("~~~")) {
-      const tilde = this.tildeFence(p, li, t);
-      if (tilde) return tilde;
-    }
+    const fence = fenceOpenerOf(seg);
+    if (fence?.char === "`") return this.backtickFence(p, li, fence);
+    if (fence?.char === "~") return this.tildeFence(p, li, fence);
     if (t.startsWith("$$")) {
       const math = this.mathBlock(p, tp);
       if (math) return math;
@@ -6154,8 +6385,8 @@ var Tokenizer = class {
         last = k;
         continue;
       }
-      if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
       if (this.isOpenTail(k)) return this.heldFootnote(p, open[1]);
+      if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
       last = k;
     }
     const end = this.contentEnds[last];
@@ -6166,74 +6397,31 @@ var Tokenizer = class {
   heldFootnote(p, id) {
     return this.island("footnote_def", p, this.n, false, { id });
   }
+  /** The text's lines as the one fence closer reads them (streaming: the open tail closes nothing). */
+  get fenceLines() {
+    return {
+      count: this.lineCount,
+      line: (k) => this.lineText(k),
+      openTail: (k) => this.isOpenTail(k)
+    };
+  }
   /**
-   * Backtick fence — the renderer splitter's close rules: JSON-string aware for
-   * ```json, and THE nested-fence rule (source/fence-nesting.ts) for markdown
-   * fences, with the splitter's strict-CommonMark retry when a nested fence is
-   * still open at the end of the text.
+   * Backtick fence — THE one closer (`closeFence`, source/code-ranges.ts): the
+   * nested-fence rule for markdown fences with the strict-CommonMark retry, the
+   * renderer's line-end closer, JSON-string awareness for ```json.
    */
-  backtickFence(p, li, t) {
-    const openTicks = backtickRunLength(t, 0);
-    const lang = parseFenceOpener(t)?.lang ?? "";
+  backtickFence(p, li, opener) {
+    const lang = opener.lang;
     const isJson = lang === "json";
     const normalized = normalizeFenceLanguage(lang);
-    const meta = { fence: "`", ticks: openTicks, lang };
+    const meta = { fence: "`", ticks: opener.ticks, lang };
     if (normalized && SPECIAL_CODE_LANGUAGES.has(normalized)) meta.special = normalized;
-    const nesting = fenceNestsInnerFences(lang);
-    let close = this.backtickFenceClose(li, openTicks, isJson, nesting);
-    if (close === "retry-strict") close = this.backtickFenceClose(li, openTicks, isJson, false);
-    if (typeof close === "number") {
-      return this.fenceIsland(p, this.contentEnds[close], true, meta, li, close, isJson);
+    const end = closeFence(this.fenceLines, li, opener);
+    if (end.closed) {
+      return this.fenceIsland(p, this.contentEnds[end.line], true, meta, li, end.line, isJson);
     }
     return this.fenceIsland(p, this.n, false, meta, li, this.lineCount, isJson);
   }
-  /** The closing line of a backtick fence opened on line `li`, or null when it never closes. */
-  backtickFenceClose(li, openTicks, isJson, nesting) {
-    let state = { inString: false, escaped: false };
-    let nestedDepth = 0;
-    let sawNested = false;
-    for (let k = li + 1; k < this.lineCount; k++) {
-      if (this.isOpenTail(k)) continue;
-      const line = this.lineText(k);
-      const trimmedLine = trimFenceLine(line);
-      if (trimmedLine.startsWith("```")) {
-        const at2 = line.indexOf("```");
-        if (isJson) {
-          const probe = jsonStringState(line, 0, at2, state);
-          if (probe.inString) {
-            state = jsonStringState(line, at2, line.length, probe);
-            continue;
-          }
-        }
-        const kind = classifyInnerFenceLine(trimmedLine, openTicks, nesting, nestedDepth);
-        if (kind === "close-outer") return k;
-        if (kind === "open-nested") {
-          nestedDepth++;
-          sawNested = true;
-        }
-        if (kind === "close-nested") nestedDepth--;
-        if (isJson) state = jsonStringState(line, 0, line.length, state);
-        continue;
-      }
-      const at = line.indexOf("```");
-      if (at !== -1) {
-        if (isJson) {
-          const probe = jsonStringState(line, 0, at, state);
-          if (probe.inString) {
-            state = jsonStringState(line, at, line.length, probe);
-            continue;
-          }
-        }
-        const closeTicks = backtickRunLength(line, at);
-        const after = trimFenceLine(line.slice(at + closeTicks));
-        if (closeTicks >= openTicks && after === "" && nestedDepth === 0) return k;
-        if (isJson) state = jsonStringState(line, 0, line.length, state);
-        continue;
-      }
-      if (isJson) state = jsonStringState(line, 0, line.length, state);
-    }
-    return sawNested && nestedDepth > 0 ? "retry-strict" : null;
-  }
   fenceIsland(p, end, complete, meta, openLine, closeLine, isJson) {
     if (isJson && openLine + 1 < this.lineCount) {
       const bodyEnd = closeLine < this.lineCount ? this.lineStarts[closeLine] : this.n;
@@ -6242,22 +6430,16 @@ var Tokenizer = class {
     }
     return this.island("fence", p, end, complete, meta);
   }
-  tildeFence(p, li, t) {
-    const open = /^(~{3,})(.*)$/.exec(t);
-    if (!open) return null;
-    const ticks = open[1].length;
-    const lang = open[2].trim().split(/\s/)[0] ?? "";
-    const meta = { fence: "~", ticks, lang };
+  tildeFence(p, li, opener) {
+    const lang = opener.lang;
+    const meta = { fence: "~", ticks: opener.ticks, lang };
     const normalized = normalizeFenceLanguage(lang);
     if (normalized && SPECIAL_CODE_LANGUAGES.has(normalized)) meta.special = normalized;
-    const close = new RegExp(`^ {0,3}~{${ticks},}[ \\t]*$`);
-    for (let k = li + 1; k < this.lineCount; k++) {
-      if (this.isOpenTail(k)) continue;
-      if (close.test(this.lineText(k))) {
-        const tildeEnd = this.contentEnds[k];
-        const end = this.rendererOverflow(li + 1, k, tildeEnd);
-        return this.island("fence", p, end, end === this.n ? this.overflowComplete : true, meta);
-      }
+    const close = closeFence(this.fenceLines, li, opener);
+    if (close.closed) {
+      const tildeEnd = this.contentEnds[close.line];
+      const end = this.rendererOverflow(li + 1, close.line, tildeEnd);
+      return this.island("fence", p, end, end === this.n ? this.overflowComplete : true, meta);
     }
     return this.island("fence", p, this.n, false, meta);
   }
@@ -6277,7 +6459,7 @@ var Tokenizer = class {
     for (let j = firstLine; j <= closeLine && j < this.lineCount; j++) {
       const start = this.lineStarts[j];
       if (this.lineIsBlank(j)) continue;
-      if (this.lineText(j).trimStart().startsWith("~~~")) continue;
+      if (fenceOpenerOf(this.lineText(j))?.char === "~") continue;
       const inner = this.detectBlockUncached(start, j, false);
       if (!inner) continue;
       if (inner.end > end) {
@@ -6525,9 +6707,9 @@ var Tokenizer = class {
           const tree = { ...breakIsland, start: treeStart };
           if (treeStart > pos) {
             const proseEnd = this.contentEnds[root - 1];
-            const scan2 = this.scanInline(pos, proseEnd);
-            if (!scan2.breakIsland) {
-              this.push({ kind: "prose", start: pos, end: proseEnd, island: null, inlines: scan2.inlines });
+            const scan3 = this.scanInline(pos, proseEnd);
+            if (!scan3.breakIsland) {
+              this.push({ kind: "prose", start: pos, end: proseEnd, island: null, inlines: scan3.inlines });
               this.push({ kind: "gap", start: proseEnd, end: treeStart, island: null, inlines: [] });
               this.push({ kind: "island", start: treeStart, end: tree.end, island: tree, inlines: [] });
               return tree.end;
@@ -6539,13 +6721,13 @@ var Tokenizer = class {
         }
       }
     }
-    const scan = this.scanInline(pos, end);
-    if (!scan.breakIsland) {
-      this.push({ kind: "prose", start: pos, end, island: null, inlines: scan.inlines });
+    const scan2 = this.scanInline(pos, end);
+    if (!scan2.breakIsland) {
+      this.push({ kind: "prose", start: pos, end, island: null, inlines: scan2.inlines });
       return end;
     }
-    const br = scan.breakIsland;
-    if (scan.orphan) {
+    const br = scan2.breakIsland;
+    if (scan2.orphan) {
       let start = pos;
       while (this.blocks.length) {
         const last = this.blocks[this.blocks.length - 1];
@@ -6562,7 +6744,7 @@ var Tokenizer = class {
       start: pos,
       end: br.start,
       island: null,
-      inlines: scan.inlines.filter((inline) => inline.end <= br.start)
+      inlines: scan2.inlines.filter((inline) => inline.end <= br.start)
     });
     this.push({ kind: "island", start: br.start, end: br.end, island: br, inlines: [] });
     return br.end;
@@ -6582,30 +6764,10 @@ var Tokenizer = class {
     const codeSpanClose = (n, from) => {
       if (!runs) {
         this.tick(pe - ps);
-        runs = /* @__PURE__ */ new Map();
-        for (let k = ps; k < pe; ) {
-          if (text.charCodeAt(k) !== 96) {
-            k++;
-            continue;
-          }
-          const len = backtickRunLength(text, k);
-          const list2 = runs.get(len);
-          if (list2) list2.push(k);
-          else runs.set(len, [k]);
-          k += len;
-        }
+        runs = new BacktickRuns(text, ps, pe);
       }
-      const list = runs.get(n);
-      if (!list) return -1;
-      let lo = 0;
-      let hi = list.length;
-      while (lo < hi) {
-        const mid = lo + hi >>> 1;
-        if (list[mid] < from) lo = mid + 1;
-        else hi = mid;
-      }
-      const at = list[lo];
-      return at !== void 0 && at + n <= pe ? at : -1;
+      const at = runs.next(n, from);
+      return at !== -1 && at + n <= pe ? at : -1;
     };
     let i = ps;
     const scanEnd = Math.min(pe, this.provisionalFrom);
@@ -7142,6 +7304,6 @@ function mapRange(changes, start, end, unit = "utf16") {
   return { start: newStart, end: newEnd, touched, collapsed: newEnd === newStart && end > start };
 }
 
-export { ACTION_BY_CLASS, CALLOUT_NAMES, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DirectiveDecodeError, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, IN_CONTENT_CLASSES, IR_ENVELOPE_CACHE_VERSION, IR_ENVELOPE_KEY, IR_PARTIAL_KEY, IR_VERSION, IrTree, JSON_DISCRIMINATOR, JsonStreamTokenizer, KIND_KEY, KindStorageError, KindStreamParser, NESTING_FENCE_LANGUAGES, NODE_OUTCOME_KIND, ParseSession, RESERVED_PREFIX, ROOT_STORAGE_NAME, RUN_RESULT_KIND, SIDE_EFFECT_CLASSES, SLUG_PREFIX, SourceSpliceError, XmlContainerTracker, advancePartialKind, asKindInstance, blockAt, blockEdit, blockRangeEdit, buildAgentSchemaWithRenderBlockSupport, buildCodePointIndex, buildCompliantKindSnapshot, buildDirectiveSlug, buildKindDirective, capabilityOf, classifyInboundEnvelopeMetadata, classifyInnerFenceLine, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, createFingerprinter, createKindStreamParser, decodeDirective, describeDualGateFailure, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, disposeParseSession, emitPayloadFence, emitPayloadJson, emptyValueForFieldSchema, envelopeCacheFromEnvelopes, envelopeFromCompleteValue, executesAtOutputRoot, fenceDiscriminator, fenceNestsInnerFences, fieldsToDbPayload, fingerprintText, formatBlockLabel, getParseSession, injectKindIntoObjectSchema, irPathIsUnderOrEqual, irPathKey, irPathLabel, irPathsEqual, isCanonicalBlockIR, isDirectiveClass, isDuplicateBlockSlug, isEmptyResidue, isIrEnvelopeCache, isJsonAnyField, isKindDirective, isLegacyShell, isMathPlaceholder, isProvisionalKind, isReservedDirectiveSlug, isScalarArrayType, isSingleDollarMath, isTerminalKindEvent, islandEdit, itemFacts, itemSubtitle, itemTitle, joinSource, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, kindSchemaToStorage, kindVerdictOf, legacyShellUses, listIslands, looksLikeDirectiveHead, looksLikeDisplayMath, makePartialKindStalenessGate, mapPosition, mapRange, mergeResidueIntoValue, normalizeAiSchemaInput, normalizeJsonRegion, nounFamily, nounLabel, nounTitleColumn, openParseSession, pairDisplayMath, parseDirectiveSlug, parseFenceOpener, pendingDisplayMathOpener, readEnvelope, readNodeOutcomeValue, readObjectKind, readOutputRef, readPartialKindEvent, readRunResultValue, readXmlTag, reconstructRegionValue, rehydrateNodeOutcome, rehydrateRunResult, resetLegacyShellUses, resolvesInContent, reuseEnvelopeIfCurrent, runKindDualGate, runSchemaConversion, sanitizeInboundEnvelopeMetadata, sanitizeInboundPartialKindMetadata, scalarArrayItemType, schemaLayoutMode, schemaStructureDepth, setJsonRootKeyLookup, singleDollarMathEnd, spliceSave, splitsSurrogatePair, storageToKindSchema, stripKindDeep, titleCaseToken, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine, tryDecodeDirective, tryDecodeDirectiveContent, validateBlockSchemaSavePlan, validateStructuralLeg, withRootKind, xmlDiscriminator };
+export { ACTION_BY_CLASS, CALLOUT_NAMES, CAPABILITY_BY_CLASS, CLASSES, DIRECTIVE_VERSION, DirectiveDecodeError, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, IN_CONTENT_CLASSES, IR_ENVELOPE_CACHE_VERSION, IR_ENVELOPE_KEY, IR_PARTIAL_KEY, IR_VERSION, IrTree, JSON_DISCRIMINATOR, JsonStreamTokenizer, KIND_KEY, KindStorageError, KindStreamParser, NESTING_FENCE_LANGUAGES, NODE_OUTCOME_KIND, ParseSession, RESERVED_PREFIX, ROOT_STORAGE_NAME, RUN_RESULT_KIND, SIDE_EFFECT_CLASSES, SLUG_PREFIX, SourceSpliceError, XmlContainerTracker, advancePartialKind, asKindInstance, blockAt, blockEdit, blockRangeEdit, buildAgentSchemaWithRenderBlockSupport, buildCodePointIndex, buildCompliantKindSnapshot, buildDirectiveSlug, buildKindDirective, capabilityOf, classifyInboundEnvelopeMetadata, classifyInnerFenceLine, collectReferencedKinds, collectSchemaReferencedKinds, compareWithExistingKindSchema, convertAiSchemaToBlockFields, createFingerprinter, createKindStreamParser, decodeDirective, describeDualGateFailure, directiveDisplay, directiveItemKindFromEdges, directiveSlugOf, disposeParseSession, emitPayloadFence, emitPayloadJson, emptyValueForFieldSchema, envelopeCacheFromEnvelopes, envelopeFromCompleteValue, executesAtOutputRoot, fenceDiscriminator, fenceNestsInnerFences, fieldsToDbPayload, findCodeRanges, fingerprintText, formatBlockLabel, getParseSession, injectKindIntoObjectSchema, irPathIsUnderOrEqual, irPathKey, irPathLabel, irPathsEqual, isCanonicalBlockIR, isDirectiveClass, isDuplicateBlockSlug, isEmptyResidue, isIrEnvelopeCache, isJsonAnyField, isKindDirective, isLegacyShell, isMathPlaceholder, isProvisionalKind, isReservedDirectiveSlug, isScalarArrayType, isSingleDollarMath, isTerminalKindEvent, islandEdit, itemFacts, itemSubtitle, itemTitle, joinSource, kindSchemaFromJsonSchema, kindSchemaToJsonSchema, kindSchemaToStorage, kindVerdictOf, legacyShellUses, listIslands, looksLikeDirectiveHead, looksLikeDisplayMath, looksLikeOpenInlineMath, makePartialKindStalenessGate, mapPosition, mapRange, mergeResidueIntoValue, normalizeAiSchemaInput, normalizeJsonRegion, nounFamily, nounLabel, nounTitleColumn, openParseSession, pairDisplayMath, parseDirectiveSlug, parseFenceOpener, pendingDisplayMathOpener, readEnvelope, readNodeOutcomeValue, readObjectKind, readOutputRef, readPartialKindEvent, readRunResultValue, readXmlTag, reconstructRegionValue, rehydrateNodeOutcome, rehydrateRunResult, resetLegacyShellUses, resolvesInContent, reuseEnvelopeIfCurrent, runKindDualGate, runSchemaConversion, sanitizeInboundEnvelopeMetadata, sanitizeInboundPartialKindMetadata, scalarArrayItemType, schemaLayoutMode, schemaStructureDepth, setJsonRootKeyLookup, singleDollarMathEnd, spliceSave, splitsSurrogatePair, storageToKindSchema, stripKindDeep, titleCaseToken, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine, tryDecodeDirective, tryDecodeDirectiveContent, validateBlockSchemaSavePlan, validateStructuralLeg, withRootKind, xmlDiscriminator };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map