@ai-matrx/content-ir 0.12.0 → 0.14.0

package/dist/source.js

@@ -200,20 +200,84 @@ function normalizeFenceLanguage(language) {
   return CODE_LANGUAGE_ALIASES[lower] ?? lower;
 }
 
+// source/fence-nesting.ts
+var NESTING_FENCE_LANGUAGES = /* @__PURE__ */ new Set([
+  "markdown",
+  "md",
+  "mdx"
+]);
+function fenceNestsInnerFences(language) {
+  return !!language && NESTING_FENCE_LANGUAGES.has(language.toLowerCase());
+}
+var FENCE_WHITESPACE = " 	\n\r\f\v";
+function isFenceWhitespace(ch) {
+  return ch !== void 0 && ch !== "" && FENCE_WHITESPACE.includes(ch);
+}
+function trimFenceLine(line) {
+  let start = 0;
+  let end = line.length;
+  while (start < end && isFenceWhitespace(line[start])) start++;
+  while (end > start && isFenceWhitespace(line[end - 1])) end--;
+  return start === 0 && end === line.length ? line : line.slice(start, end);
+}
+function classifyInnerFenceLine(trimmed, openTicks, nests, nestedDepth) {
+  let ticks = 0;
+  while (ticks < trimmed.length && trimmed[ticks] === "`") ticks++;
+  if (ticks < 3) return "content";
+  const info = trimFenceLine(trimmed.slice(ticks));
+  if (info === "") {
+    if (ticks < openTicks) return "content";
+    if (nests && nestedDepth > 0) return "close-nested";
+    return "close-outer";
+  }
+  if (nests && ticks >= openTicks && !info.includes("`")) return "open-nested";
+  return "content";
+}
+
 // source/math.ts
 var TEX_SIGNAL = /\\[A-Za-z]+|[\\^_{}=<>+]/;
 var LONE_VARIABLE = /^[A-Za-z](?:'|[0-9])?$/;
+var MATH_SHAPE_CHARS = /^[A-Za-z0-9'\s(),.*\/-]+$/;
+var FUNCTION_NAMES = /* @__PURE__ */ new Set([
+  "sin",
+  "cos",
+  "tan",
+  "sec",
+  "csc",
+  "cot",
+  "log",
+  "ln",
+  "exp",
+  "lim",
+  "max",
+  "min",
+  "det",
+  "gcd",
+  "mod",
+  "arg",
+  "sinh",
+  "cosh",
+  "tanh"
+]);
+function isMathShape(content) {
+  if (!MATH_SHAPE_CHARS.test(content)) return false;
+  const words = content.match(/[A-Za-z]+/g);
+  if (!words) return false;
+  if (!words.every((w) => w.length === 1 || FUNCTION_NAMES.has(w))) return false;
+  return /[A-Za-z]'*\(|[A-Za-z]'|\b[A-Za-z]\s+[A-Za-z]\b|\([^)]*[A-Za-z][^)]*\)/.test(content);
+}
 function isSingleDollarMath(content, after) {
   if (!content || content.length > 400) return false;
   if (/^\s/.test(content) || /\s$/.test(content)) return false;
   if (after !== void 0 && /[0-9]/.test(after)) return false;
   if (LONE_VARIABLE.test(content)) return true;
   if (/^[0-9]/.test(content)) return /\\[A-Za-z]+|[\\^_{}]/.test(content);
-  return TEX_SIGNAL.test(content);
+  return TEX_SIGNAL.test(content) || isMathShape(content);
 }
 function singleDollarMathEnd(text, open, limit = text.length) {
+  const bound = Math.min(limit, open + 403);
   let close = -1;
-  for (let k = open + 1; k < limit; k += 1) {
+  for (let k = open + 1; k < bound; k += 1) {
     const c = text[k];
     if (c === "\n") break;
     if (c === "\\") {
@@ -230,11 +294,69 @@ function singleDollarMathEnd(text, open, limit = text.length) {
   return isSingleDollarMath(content, text[close + 1]) ? close + 1 : -1;
 }
 
+// source/math-pairs.ts
+var MAX_MATH_SPAN = 600;
+var STRUCTURAL_MARKDOWN = /\]\(|https?:\/\/|\*\*|(?:^|\n)[ \t]{0,3}#{1,6}[ \t]|(?:^|\n)[ \t]*[-*+][ \t]+|(?:^|\n)[ \t]*\d+[.)][ \t]/;
+var LATEX_COMMAND = /\\[a-zA-Z]/;
+var PROSE_WORD = /[A-Za-z]{3,}/g;
+var PROSE_WORD_LIMIT = 6;
+function looksLikeDisplayMath(inner) {
+  const s = inner.trim();
+  if (!s) return false;
+  if (STRUCTURAL_MARKDOWN.test(s)) return false;
+  if (s.length > MAX_MATH_SPAN) return false;
+  if (LATEX_COMMAND.test(s)) {
+    return (s.match(PROSE_WORD) ?? []).length < PROSE_WORD_LIMIT * 3;
+  }
+  if (/\n[ \t]*\n/.test(s)) return false;
+  return (s.match(PROSE_WORD) ?? []).length < PROSE_WORD_LIMIT;
+}
+function codeRanges(text) {
+  const ranges = [];
+  for (const re of [/```[\s\S]*?(?:```|$)/g, /~~~[\s\S]*?(?:~~~|$)/g, /`[^`\n]*`/g]) {
+    let m;
+    while ((m = re.exec(text)) !== null) {
+      ranges.push([m.index, m.index + m[0].length]);
+      if (m[0].length === 0) re.lastIndex++;
+    }
+  }
+  return ranges.sort((a, b) => a[0] - b[0]);
+}
+function pairDisplayMath(text) {
+  const pairs = /* @__PURE__ */ new Map();
+  if (!text.includes("$$")) return pairs;
+  const ranges = codeRanges(text);
+  const tokens = [];
+  let r = 0;
+  let maxEnd = -1;
+  for (let i = text.indexOf("$$"); i !== -1 && i < text.length - 1; i = text.indexOf("$$", i)) {
+    while (r < ranges.length && ranges[r][0] <= i) {
+      maxEnd = Math.max(maxEnd, ranges[r][1]);
+      r++;
+    }
+    if (maxEnd <= i) tokens.push(i);
+    i += 2;
+  }
+  let j = 0;
+  while (j < tokens.length) {
+    const open = tokens[j];
+    const close = tokens[j + 1];
+    if (close === void 0) break;
+    if (looksLikeDisplayMath(text.slice(open + 2, close))) {
+      pairs.set(open, close);
+      j += 2;
+      continue;
+    }
+    j += 1;
+  }
+  return pairs;
+}
+
 // source/xml-tag.ts
 var XML_NAME_START = /[A-Za-z_]/;
 var XML_NAME_CHARACTER = /[\w.:-]/;
 var WHITESPACE = /\s/;
-function readXmlTag(content, start) {
+function readXmlTag(content, start, find = (needle, from) => content.indexOf(needle, from)) {
   let cursor = start + 1;
   const isClosing = content[cursor] === "/";
   if (isClosing) cursor++;
@@ -287,8 +409,9 @@ function readXmlTag(content, start) {
     const quote = content[cursor];
     if (quote !== '"' && quote !== "'") return null;
     const valueStart = ++cursor;
-    while (cursor < content.length && content[cursor] !== quote) cursor++;
-    if (cursor === content.length) return null;
+    const valueEnd = find(quote, valueStart);
+    if (valueEnd === -1) return null;
+    cursor = valueEnd;
     attributes.push({ name, value: content.slice(valueStart, cursor) });
     cursor++;
   }
@@ -385,6 +508,10 @@ var XmlContainerTracker = class {
 
 // source/tokenize.ts
 var NOT_WHITESPACE = /\S/;
+var DIRECTIVE_OPEN = /^(:{2,})([A-Za-z][\w-]*)/;
+var MKDOCS_OPEN = /^(!!!|\?\?\?\+?)[ \t]+([A-Za-z][\w-]*)/;
+var CALLOUT_OPEN = /^>[ \t]*\[!([A-Za-z][\w-]*)\]([+-])?/;
+var FOOTNOTE_DEF = /^\[\^([^\]\s]+)\]:/;
 function countStructuralBraces(source) {
   let opens = 0;
   let closes = 0;
@@ -507,11 +634,16 @@ function backtickRunLength(str, pos) {
   while (pos + n < str.length && str[pos + n] === "`") n++;
   return n;
 }
+function isSplitterTreeLine(line) {
+  return TREE_GLYPHS.test(line) || /^[\s│|]*[├└+|][\s─\-]+/.test(line);
+}
 var MEDIA_REF_PREFIXES = ["[Image URL:", "[Video URL:", "[Audio URL:"];
 var Tokenizer = class {
-  constructor(text) {
+  constructor(text, streaming = false) {
     this.text = text;
+    this.streaming = streaming;
     this.n = text.length;
+    this.mathPairs = pairDisplayMath(text);
     let start = 0;
     for (let i = 0; i < text.length; i++) {
       if (text.charCodeAt(i) === 10) {
@@ -528,6 +660,7 @@ var Tokenizer = class {
     }
   }
   text;
+  streaming;
   n;
   lineStarts = [];
   contentEnds = [];
@@ -537,6 +670,38 @@ var Tokenizer = class {
   /** Lazily built: for line i, first line j >= i where brace net from i drops to <= 0. */
   braceStop = null;
   braceNet = null;
+  /** Every occurrence offset of each needle searched so far (overlapping, like indexOf). */
+  occurrences = /* @__PURE__ */ new Map();
+  /** `$$` opener → closer for the pairs the core renders as math. */
+  mathPairs;
+  finder = (needle, from) => this.next(needle, from);
+  /**
+   * `text.indexOf(needle, from)` in O(log k): the first query for a needle
+   * indexes all of its occurrences once, so repeated searches for a closer
+   * that never comes stay linear over the whole text.
+   */
+  next(needle, from) {
+    let list = this.occurrences.get(needle);
+    if (!list) {
+      list = [];
+      for (let at = this.text.indexOf(needle); at !== -1; at = this.text.indexOf(needle, at + 1)) {
+        list.push(at);
+      }
+      this.occurrences.set(needle, list);
+    }
+    let lo = 0;
+    let hi = list.length;
+    while (lo < hi) {
+      const mid = lo + hi >>> 1;
+      if (list[mid] < from) lo = mid + 1;
+      else hi = mid;
+    }
+    return lo < list.length ? list[lo] : -1;
+  }
+  /** Streaming only: the final line, still waiting for its terminator. */
+  isOpenTail(k) {
+    return this.streaming && k === this.lineStarts.length - 1 && this.lineEnds[k] === this.contentEnds[k];
+  }
   get lineCount() {
     return this.lineStarts.length;
   }
@@ -618,6 +783,22 @@ var Tokenizer = class {
       const fm = this.frontMatter(li, t);
       if (fm) return fm;
     }
+    if (indent <= 3 && t.startsWith("::")) {
+      const directive = this.directive(p, li, t);
+      if (directive) return directive;
+    }
+    if (indent === 0 && (t.startsWith("!!!") || t.startsWith("???"))) {
+      const admonition = this.mkdocsAdmonition(p, li, t);
+      if (admonition) return admonition;
+    }
+    if (indent <= 3 && t.startsWith(">")) {
+      const callout = this.callout(p, li, t);
+      if (callout) return callout;
+    }
+    if (atParaStart && indent <= 3 && t.startsWith("[^")) {
+      const footnote = this.footnoteDefinition(p, li, t);
+      if (footnote) return footnote;
+    }
     if (t.startsWith("```")) return this.backtickFence(p, li, t);
     if (indent <= 3 && t.startsWith("~~~")) {
       const tilde = this.tildeFence(p, li, t);
@@ -655,20 +836,116 @@ var Tokenizer = class {
     return { type, start, end, complete, meta };
   }
   frontMatter(li, t) {
-    if (this.lineText(li) !== "---" || t !== "---") return null;
+    const fence = this.lineText(li);
+    if (fence !== "---" && fence !== "+++" || t !== fence) return null;
+    const toml = fence === "+++";
     let sawContent = false;
     for (let k = li + 1; k < this.lineCount; k++) {
       const line = this.lineText(k);
-      if (line === "---" || line === "...") {
-        return sawContent ? this.island("front_matter", 0, this.contentEnds[k], true) : null;
+      if (line === fence || !toml && line === "...") {
+        return sawContent ? this.island("front_matter", 0, this.contentEnds[k], true, { format: toml ? "toml" : "yaml" }) : null;
       }
       if (!NOT_WHITESPACE.test(line)) continue;
-      if (!/^(?:[A-Za-z0-9_"'-][^:]*:(?:\s|$)|\s+\S|-\s)/.test(line)) return null;
+      const shaped = toml ? /^\s*(?:[A-Za-z0-9_."-]+\s*=|\[|#)/.test(line) : /^(?:[A-Za-z0-9_"'-][^:]*:(?:\s|$)|\s+\S|-\s|#)/.test(line);
+      if (!shaped) return null;
       sawContent = true;
     }
     return null;
   }
-  /** Backtick fence — the splitter's close rules, JSON-string aware for ```json. */
+  /**
+   * `:::name[label]{attrs}` … closing `:::` (a colon run at least as long as
+   * the opener's, nesting counted), or a leaf `::name[…]{…}` line. The whole
+   * container is ONE island: its grammar (fence lengths, labels, attributes,
+   * nested tabs) is exactly what a visual editor would re-serialize wrongly.
+   */
+  directive(p, li, t) {
+    const open = DIRECTIVE_OPEN.exec(t);
+    if (!open) return null;
+    const colons = open[1].length;
+    const name = open[2];
+    if (colons === 2) {
+      return this.island("directive", p, this.contentEnds[li], true, { name, leaf: true });
+    }
+    const stack = [colons];
+    for (let k = li + 1; k < this.lineCount; k++) {
+      const line = this.lineText(k).trim();
+      const inner = DIRECTIVE_OPEN.exec(line);
+      if (inner && inner[1].length >= 3) {
+        stack.push(inner[1].length);
+        continue;
+      }
+      const close = /^(:{3,})\s*$/.exec(line);
+      if (close && close[1].length >= stack[stack.length - 1]) {
+        stack.pop();
+        if (stack.length === 0) return this.island("directive", p, this.contentEnds[k], true, { name });
+      }
+    }
+    return this.island("directive", p, this.n, false, { name });
+  }
+  /** MkDocs `!!! type "Title"` / `??? type` / `???+ type` and its indented (or blank-separated indented) body. */
+  mkdocsAdmonition(p, li, t) {
+    const open = MKDOCS_OPEN.exec(t);
+    if (!open) return null;
+    let last = li;
+    for (let k = li + 1; k < this.lineCount; k++) {
+      const line = this.lineText(k);
+      if (/^(?: {4}|\t)/.test(line) && NOT_WHITESPACE.test(line)) {
+        last = k;
+        continue;
+      }
+      if (!NOT_WHITESPACE.test(line)) continue;
+      break;
+    }
+    return this.island("directive", p, this.contentEnds[last], true, {
+      name: open[2].toLowerCase(),
+      syntax: "mkdocs"
+    });
+  }
+  /** A blockquote whose first line is `> [!TYPE]…` — every following `>` line belongs to it. */
+  callout(p, li, t) {
+    const open = CALLOUT_OPEN.exec(t);
+    if (!open) return null;
+    let last = li;
+    for (let k = li + 1; k < this.lineCount; k++) {
+      if (!/^ {0,3}>/.test(this.lineText(k))) break;
+      last = k;
+    }
+    return this.island("callout", p, this.contentEnds[last], true, {
+      name: open[1].toLowerCase(),
+      ...open[2] ? { fold: open[2] === "-" ? "closed" : "open" } : {}
+    });
+  }
+  /** `[^id]: text` plus its continuation lines (indented, or lazy text before a blank line). */
+  footnoteDefinition(p, li, t) {
+    const open = FOOTNOTE_DEF.exec(t);
+    if (!open) return null;
+    let last = li;
+    for (let k = li + 1; k < this.lineCount; k++) {
+      const line = this.lineText(k);
+      if (!NOT_WHITESPACE.test(line)) {
+        let j = k + 1;
+        while (j < this.lineCount && !NOT_WHITESPACE.test(this.lineText(j))) j++;
+        if (j < this.lineCount && /^(?: {4}|\t)/.test(this.lineText(j))) {
+          k = j - 1;
+          continue;
+        }
+        break;
+      }
+      if (/^(?: {4}|\t)/.test(line)) {
+        last = k;
+        continue;
+      }
+      if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
+      last = k;
+    }
+    return this.island("footnote_def", p, this.contentEnds[last], true, { id: open[1] });
+  }
+  /**
+   * Backtick fence — the renderer splitter's close rules: JSON-string aware for
+   * ```json, and THE nested-fence rule (source/fence-nesting.ts) for markdown
+   * fences, with the splitter's strict-CommonMark retry when a nested fence is
+   * still open at the end of the text.
+   */
   backtickFence(p, li, t) {
     const openTicks = backtickRunLength(t, 0);
     const info = t.slice(openTicks).trim();
@@ -677,10 +954,23 @@ var Tokenizer = class {
     const normalized = normalizeFenceLanguage(lang);
     const meta = { fence: "`", ticks: openTicks, lang };
     if (normalized && SPECIAL_CODE_LANGUAGES.has(normalized)) meta.special = normalized;
+    const nesting = fenceNestsInnerFences(lang);
+    let close = this.backtickFenceClose(li, openTicks, isJson, nesting);
+    if (close === "retry-strict") close = this.backtickFenceClose(li, openTicks, isJson, false);
+    if (typeof close === "number") {
+      return this.fenceIsland(p, this.contentEnds[close], true, meta, li, close, isJson);
+    }
+    return this.fenceIsland(p, this.n, false, meta, li, this.lineCount, isJson);
+  }
+  /** The closing line of a backtick fence opened on line `li`, or null when it never closes. */
+  backtickFenceClose(li, openTicks, isJson, nesting) {
     let state = { inString: false, escaped: false };
+    let nestedDepth = 0;
+    let sawNested = false;
     for (let k = li + 1; k < this.lineCount; k++) {
+      if (this.isOpenTail(k)) continue;
       const line = this.lineText(k);
-      const trimmedLine = line.trim();
+      const trimmedLine = trimFenceLine(line);
       if (trimmedLine.startsWith("```")) {
         const at2 = line.indexOf("```");
         if (isJson) {
@@ -690,11 +980,13 @@ var Tokenizer = class {
             continue;
           }
         }
-        const closeTicks = backtickRunLength(trimmedLine, 0);
-        const isBare = trimmedLine.slice(closeTicks).trim() === "";
-        if (isBare && closeTicks >= openTicks) {
-          return this.fenceIsland(p, this.contentEnds[k], true, meta, li, k, isJson);
+        const kind = classifyInnerFenceLine(trimmedLine, openTicks, nesting, nestedDepth);
+        if (kind === "close-outer") return k;
+        if (kind === "open-nested") {
+          nestedDepth++;
+          sawNested = true;
         }
+        if (kind === "close-nested") nestedDepth--;
         if (isJson) state = jsonStringState(line, 0, line.length, state);
         continue;
       }
@@ -708,16 +1000,14 @@ var Tokenizer = class {
           }
         }
         const closeTicks = backtickRunLength(line, at);
-        const after = line.slice(at + closeTicks).trim();
-        if (closeTicks >= openTicks && after === "") {
-          return this.fenceIsland(p, this.contentEnds[k], true, meta, li, k, isJson);
-        }
+        const after = trimFenceLine(line.slice(at + closeTicks));
+        if (closeTicks >= openTicks && after === "" && nestedDepth === 0) return k;
         if (isJson) state = jsonStringState(line, 0, line.length, state);
         continue;
       }
       if (isJson) state = jsonStringState(line, 0, line.length, state);
     }
-    return this.fenceIsland(p, this.n, false, meta, li, this.lineCount, isJson);
+    return sawNested && nestedDepth > 0 ? "retry-strict" : null;
   }
   fenceIsland(p, end, complete, meta, openLine, closeLine, isJson) {
     if (isJson && openLine + 1 < this.lineCount) {
@@ -737,24 +1027,29 @@ var Tokenizer = class {
     if (normalized && SPECIAL_CODE_LANGUAGES.has(normalized)) meta.special = normalized;
     const close = new RegExp(`^ {0,3}~{${ticks},}[ \\t]*$`);
     for (let k = li + 1; k < this.lineCount; k++) {
+      if (this.isOpenTail(k)) continue;
       if (close.test(this.lineText(k))) {
         return this.island("fence", p, this.contentEnds[k], true, meta);
       }
     }
     return this.island("fence", p, this.n, false, meta);
   }
+  /**
+   * A line-leading `$$` that OPENS a math pair (source/math-pairs.ts — the
+   * core's rule). A closer, an unpaired `$$`, or a prose pair is never a block:
+   * a lone `$$` renders as literal text and locks nothing.
+   */
   mathBlock(p, tp) {
+    const close = this.mathPairs.get(tp);
+    if (close === void 0) return null;
+    const end = close + 2;
     const li = this.lineOf(tp);
     const ce = this.contentEnds[li];
-    const sameLine = this.text.indexOf("$$", tp + 2);
-    if (sameLine !== -1 && sameLine + 2 <= ce) {
-      if (!this.isBlank(sameLine + 2, ce)) return null;
-      if (sameLine === tp + 2) return null;
-      return this.island("math_block", p, sameLine + 2, true, { delimiter: "$$" });
-    }
-    const close = this.text.indexOf("$$", tp + 2);
-    if (close === -1) return this.island("math_block", p, this.n, false, { delimiter: "$$" });
-    return this.island("math_block", p, close + 2, true, { delimiter: "$$" });
+    if (end <= ce && !this.isBlank(end, ce)) return null;
+    const endLine = this.lineOf(close);
+    const endCe = this.contentEnds[endLine];
+    if (endLine !== li && !this.isBlank(end, endCe)) return null;
+    return this.island("math_block", p, end, true, { delimiter: "$$" });
   }
   /** `<artifact …>`, `<decision …>`, … at the start of a line. */
   attributeXml(p, tp, t) {
@@ -771,7 +1066,7 @@ var Tokenizer = class {
   attributeElement(name, p, tp, type) {
     const li = this.lineOf(tp);
     const ce = this.contentEnds[li];
-    const tag = readXmlTag(this.text, tp);
+    const tag = readXmlTag(this.text, tp, this.finder);
     let openEnd;
     if (tag && !tag.isClosing) {
       if (tag.isSelfClosing) {
@@ -779,13 +1074,13 @@ var Tokenizer = class {
       }
       openEnd = tp + tag.raw.length;
     } else {
-      const gt = this.text.indexOf(">", tp);
+      const gt = this.next(">", tp);
       if (gt === -1) return this.island(type, p, this.n, false, { tag: name });
       if (gt >= ce) return null;
       openEnd = gt + 1;
     }
     const closer = `</${name}>`;
-    const close = this.text.indexOf(closer, openEnd);
+    const close = this.next(closer, openEnd);
     if (close === -1) return this.island(type, p, this.n, false, { tag: name });
     return this.island(type, p, close + closer.length, true, { tag: name });
   }
@@ -795,7 +1090,7 @@ var Tokenizer = class {
       const opener = `<${name}>`;
       if (!t.startsWith(opener)) continue;
       const closer = `</${name}>`;
-      const close = this.text.indexOf(closer, tp + opener.length);
+      const close = this.next(closer, tp + opener.length);
       if (close === -1) return this.island("xml_region", p, this.n, false, { tag: name });
       return this.island("xml_region", p, close + closer.length, true, { tag: name });
     }
@@ -831,7 +1126,7 @@ var Tokenizer = class {
     return null;
   }
   blockComment(p, tp, li) {
-    const close = this.text.indexOf("-->", tp + 4);
+    const close = this.next("-->", tp + 4);
     if (close === -1) return this.island("html_comment", p, this.n, false);
     const end = close + 3;
     const raw = this.text.slice(tp, end);
@@ -909,9 +1204,8 @@ var Tokenizer = class {
       }
     }
     if (lastLine === null) {
-      const partial = this.text.slice(tp);
-      if (isRecognizedJsonHead(partial)) {
-        return this.island("json", p, this.n, false, jsonMeta(partial));
+      if (isRecognizedJsonHead(this.text.slice(tp, tp + 512))) {
+        return this.island("json", p, this.n, false, jsonMeta(this.text.slice(tp)));
       }
       return null;
     }
@@ -946,11 +1240,43 @@ var Tokenizer = class {
   // ── prose ──────────────────────────────────────────────────────────────
   paragraph(pos, li) {
     let end = this.contentEnds[li];
+    let breakLine = -1;
     for (let k = li + 1; k < this.lineCount; k++) {
       if (this.lineIsBlank(k)) break;
-      if (this.detectBlock(this.lineStarts[k], k, false)) break;
+      if (this.detectBlock(this.lineStarts[k], k, false)) {
+        breakLine = k;
+        break;
+      }
       end = this.contentEnds[k];
     }
+    if (breakLine !== -1) {
+      const breakIsland = this.detectBlock(this.lineStarts[breakLine], breakLine, false);
+      if (breakIsland?.type === "tree") {
+        let root = breakLine;
+        for (let j = breakLine - 1; j >= li; j--) {
+          const trimmed = this.lineText(j).trim();
+          if (!trimmed || isSplitterTreeLine(trimmed) || /^#{1,6}\s/.test(trimmed)) break;
+          root = j;
+        }
+        if (root < breakLine) {
+          const treeStart = root === li ? pos : this.lineStarts[root];
+          const tree = { ...breakIsland, start: treeStart };
+          if (treeStart > pos) {
+            const proseEnd = this.contentEnds[root - 1];
+            const scan2 = this.scanInline(pos, proseEnd);
+            if (!scan2.breakIsland) {
+              this.push({ kind: "prose", start: pos, end: proseEnd, island: null, inlines: scan2.inlines });
+              this.push({ kind: "gap", start: proseEnd, end: treeStart, island: null, inlines: [] });
+              this.push({ kind: "island", start: treeStart, end: tree.end, island: tree, inlines: [] });
+              return tree.end;
+            }
+          } else {
+            this.push({ kind: "island", start: treeStart, end: tree.end, island: tree, inlines: [] });
+            return tree.end;
+          }
+        }
+      }
+    }
     const scan = this.scanInline(pos, end);
     if (!scan.breakIsland) {
       this.push({ kind: "prose", start: pos, end, island: null, inlines: scan.inlines });
@@ -983,17 +1309,49 @@ var Tokenizer = class {
     const text = this.text;
     const inlines = [];
     const hasKind = text.slice(ps, pe).includes("__kind");
+    let kindBudget = 4 * (pe - ps) + 65536;
     const stop = (island, orphan = false) => ({ inlines, breakIsland: island, orphan });
+    const within = (needle, from) => {
+      const at = this.next(needle, from);
+      return at !== -1 && at + needle.length <= pe ? at : -1;
+    };
+    let runs = null;
+    const codeSpanClose = (n, from) => {
+      if (!runs) {
+        runs = /* @__PURE__ */ new Map();
+        for (let k = ps; k < pe; ) {
+          if (text.charCodeAt(k) !== 96) {
+            k++;
+            continue;
+          }
+          const len = backtickRunLength(text, k);
+          const list2 = runs.get(len);
+          if (list2) list2.push(k);
+          else runs.set(len, [k]);
+          k += len;
+        }
+      }
+      const list = runs.get(n);
+      if (!list) return -1;
+      let lo = 0;
+      let hi = list.length;
+      while (lo < hi) {
+        const mid = lo + hi >>> 1;
+        if (list[mid] < from) lo = mid + 1;
+        else hi = mid;
+      }
+      const at = list[lo];
+      return at !== void 0 && at + n <= pe ? at : -1;
+    };
     let i = ps;
     while (i < pe) {
       const ch = text[i];
       if (ch === "\\") {
-        const next = text[i + 1];
-        if (next === "(" || next === "[") {
-          const closer = next === "(" ? "\\)" : "\\]";
-          const close = text.indexOf(closer, i + 2);
-          if (close !== -1 && close + 2 <= pe) {
-            inlines.push(this.island("math_inline", i, close + 2, true, { delimiter: next === "(" ? "\\(" : "\\[" }));
+        const nextCh = text[i + 1];
+        if (nextCh === "(" || nextCh === "[") {
+          const close = within(nextCh === "(" ? "\\)" : "\\]", i + 2);
+          if (close !== -1) {
+            inlines.push(this.island("math_inline", i, close + 2, true, { delimiter: nextCh === "(" ? "\\(" : "\\[" }));
             i = close + 2;
             continue;
           }
@@ -1003,33 +1361,20 @@ var Tokenizer = class {
       }
       if (ch === "`") {
         const n = backtickRunLength(text, i);
-        const run = "`".repeat(n);
-        let k = i + n;
-        let close = -1;
-        while (k < pe) {
-          const idx = text.indexOf(run, k);
-          if (idx === -1 || idx >= pe) break;
-          if (text[idx + n] === "`") {
-            let m = idx;
-            while (text[m] === "`") m++;
-            k = m;
-            continue;
-          }
-          close = idx;
-          break;
-        }
+        const close = codeSpanClose(n, i + n);
         i = close === -1 ? i + n : close + n;
         continue;
       }
       if (ch === "$") {
         if (text[i + 1] === "$") {
-          const close = text.indexOf("$$", i + 2);
-          if (close !== -1 && close + 2 <= pe && close > i + 2) {
-            inlines.push(this.island("math_inline", i, close + 2, true, { delimiter: "$$" }));
-            i = close + 2;
-          } else {
+          const close = this.mathPairs.get(i);
+          if (close === void 0) {
             i += 2;
+            continue;
           }
+          if (close + 2 > pe) return stop(this.island("math_block", i, close + 2, true, { delimiter: "$$" }));
+          inlines.push(this.island("math_inline", i, close + 2, true, { delimiter: "$$" }));
+          i = close + 2;
           continue;
         }
         const end = singleDollarMathEnd(text, i, pe);
@@ -1043,7 +1388,7 @@ var Tokenizer = class {
       }
       if (ch === "{") {
         if (text[i + 1] === "{") {
-          const close = text.indexOf("}}", i + 2);
+          const close = this.next("}}", i + 2);
           if (close !== -1 && close + 2 <= pe && close - i <= 200 && !text.slice(i, close).includes("\n")) {
             const raw = text.slice(i, close + 2);
             const strict = STRICT_VARIABLE_RE.exec(raw);
@@ -1056,8 +1401,10 @@ var Tokenizer = class {
           i += 2;
           continue;
         }
-        if (hasKind) {
-          const end = matchingJsonObjectEnd(text, i, pe);
+        if (hasKind && kindBudget > 0 && /^\{\s*"/.test(text.slice(i, i + 64))) {
+          const limit = Math.min(pe, i + 65536);
+          const end = matchingJsonObjectEnd(text, i, limit);
+          kindBudget -= (end ?? limit) - i;
           if (end !== null) {
             const kind = declaredKind(text.slice(i, end));
             if (kind) {
@@ -1070,11 +1417,27 @@ var Tokenizer = class {
         i++;
         continue;
       }
+      if (ch === "[" && text[i + 1] === "[" || ch === "!" && text[i + 1] === "[" && text[i + 2] === "[") {
+        const open = ch === "!" ? i + 3 : i + 2;
+        const close = within("]]", open);
+        const newline = this.next("\n", open);
+        const inner = close === -1 ? "" : text.slice(open, close);
+        if (close !== -1 && (newline === -1 || newline > close) && inner.trim() && !/[[\]]/.test(inner)) {
+          const cut = inner.search(/\\?\|/);
+          const target = (cut < 0 ? inner : inner.slice(0, cut)).trim();
+          inlines.push(
+            this.island("wikilink", i, close + 2, true, ch === "!" ? { target, embed: true } : { target })
+          );
+          i = close + 2;
+          continue;
+        }
+      }
       if (ch === "[") {
         const media = MEDIA_REF_PREFIXES.find((prefix) => text.startsWith(prefix, i));
         if (media) {
-          const close = text.indexOf("]", i + media.length);
-          if (close !== -1 && close < pe && !text.slice(i, close).includes("\n")) {
+          const close = within("]", i + media.length);
+          const newline = this.next("\n", i);
+          if (close !== -1 && (newline === -1 || newline > close)) {
             inlines.push(this.island("media_ref", i, close + 1, true, { media: media.slice(1, 6).toLowerCase() }));
             i = close + 1;
             continue;
@@ -1085,10 +1448,10 @@ var Tokenizer = class {
       }
       if (ch === "<") {
         if (text.startsWith("<!--", i)) {
-          const close = text.indexOf("-->", i + 4);
+          const close = this.next("-->", i + 4);
           if (close === -1) return stop(this.island("html_comment", i, this.n, false));
           const end = close + 3;
-          const anchor = PINNED_ANCHOR_RE.exec(text.slice(i, end));
+          const anchor = end - i <= 64 ? PINNED_ANCHOR_RE.exec(text.slice(i, end)) : null;
           const type = anchor ? "anchor" : "html_comment";
           const meta = anchor ? { id: anchor[1] } : {};
           if (end > pe) return stop(this.island(type, i, end, true, meta));
@@ -1102,7 +1465,7 @@ var Tokenizer = class {
           i += cite[0].length;
           continue;
         }
-        const tag = readXmlTag(text, i);
+        const tag = readXmlTag(text, i, this.finder);
         if (tag && i + tag.raw.length <= pe) {
           const name = tag.tagName;
           const tagEnd = i + tag.raw.length;
@@ -1115,7 +1478,7 @@ var Tokenizer = class {
               continue;
             }
             const closer = `</${name}>`;
-            const close = text.indexOf(closer, tagEnd);
+            const close = this.next(closer, tagEnd);
             const blockType = isAttr ? "xml_attr" : "xml_region";
             if (close === -1) return stop(this.island(blockType, i, this.n, false, { tag: name }));
             const end = close + closer.length;
@@ -1139,7 +1502,7 @@ var Tokenizer = class {
         const pending = ATTRIBUTE_XML_NAMES.find(
           (name) => text.startsWith(`<${name}`, i) && /\s/.test(text[i + name.length + 1] ?? "")
         );
-        if (pending && text.indexOf(">", i) === -1) {
+        if (pending && this.next(">", i) === -1) {
           return stop(this.island("xml_attr", i, this.n, false, { tag: pending }));
         }
         i++;
@@ -1150,8 +1513,8 @@ var Tokenizer = class {
     return { inlines, breakIsland: null, orphan: false };
   }
 };
-function tokenizeSource(text) {
-  const raw = new Tokenizer(text).run();
+function tokenizeSource(text, options = {}) {
+  const raw = new Tokenizer(text, options.streaming ?? false).run();
   const cp = buildCodePointIndex(text);
   const inline = (island) => ({
     islandType: island.type,
@@ -1200,7 +1563,7 @@ function listIslands(blocks) {
         raw: block.raw
       });
     } else {
-      out.push(...block.inlines);
+      for (const inline of block.inlines) out.push(inline);
     }
   }
   return out;
@@ -1230,9 +1593,44 @@ var SourceSpliceError = class extends Error {
 function blockEdit(block, text) {
   return { start: block.start, end: block.end, text };
 }
+function islandEdit(island, text) {
+  return { start: island.start, end: island.end, text, island: true };
+}
 function blockRangeEdit(first, last, text) {
   return { start: first.start, end: last.end, text };
 }
+var CONTEXT_WINDOW = 64;
+function locateIsland(text, raw, from, expected, expectedFromEnd, oldText, oldStart, oldEnd) {
+  const candidates = /* @__PURE__ */ new Set();
+  if (expected >= from && text.startsWith(raw, expected)) candidates.add(expected);
+  if (expectedFromEnd >= from && text.startsWith(raw, expectedFromEnd)) candidates.add(expectedFromEnd);
+  const after = text.indexOf(raw, Math.max(from, expected));
+  if (after !== -1) candidates.add(after);
+  const before = expected > from ? text.lastIndexOf(raw, expected) : -1;
+  if (before >= from) candidates.add(before);
+  if (candidates.size === 0) return text.indexOf(raw, from);
+  if (candidates.size === 1) return candidates.values().next().value;
+  let best = -1;
+  let bestScore = -1;
+  for (const at of candidates) {
+    let score = 0;
+    for (let k = 0; k < CONTEXT_WINDOW; k++) {
+      const n = text[at + raw.length + k];
+      if (n === void 0 || n !== oldText[oldEnd + k]) break;
+      score++;
+    }
+    for (let k = 1; k <= CONTEXT_WINDOW; k++) {
+      const n = text[at - k];
+      if (n === void 0 || n !== oldText[oldStart - k]) break;
+      score++;
+    }
+    if (score > bestScore || score === bestScore && at === expected) {
+      best = at;
+      bestScore = score;
+    }
+  }
+  return best;
+}
 function commonPrefix(a, b) {
   const max = Math.min(a.length, b.length);
   let i = 0;
@@ -1257,13 +1655,23 @@ function spliceSave(original, edits, options = {}) {
     boundaries.add(block.start);
     boundaries.add(block.end);
   }
+  const islands = listIslands(blocks);
+  const islandAt = /* @__PURE__ */ new Map();
+  for (const island of islands) islandAt.set(`${island.start}:${island.end}`, island);
   const sorted = [...edits].sort((a, b) => a.start - b.start || a.end - b.end);
   let previousEnd = -1;
   for (const edit of sorted) {
     if (edit.start < 0 || edit.end > original.length || edit.start > edit.end) {
       throw new SourceSpliceError("out_of_range", `edit [${edit.start}, ${edit.end}) is outside the text`);
     }
-    if (!boundaries.has(edit.start) || !boundaries.has(edit.end)) {
+    if (edit.island) {
+      if (!islandAt.has(`${edit.start}:${edit.end}`)) {
+        throw new SourceSpliceError(
+          "misaligned",
+          `islandEdit [${edit.start}, ${edit.end}) does not name an island's exact span`
+        );
+      }
+    } else if (!boundaries.has(edit.start) || !boundaries.has(edit.end)) {
       throw new SourceSpliceError(
         "misaligned",
         `edit [${edit.start}, ${edit.end}) does not start and end on block boundaries`
@@ -1275,18 +1683,51 @@ function spliceSave(original, edits, options = {}) {
     previousEnd = edit.end;
   }
   const effective = [];
-  for (const edit of sorted) {
-    const old = original.slice(edit.start, edit.end);
-    if (old === edit.text) continue;
-    const prefix = commonPrefix(old, edit.text);
-    const suffix = commonSuffix(old, edit.text, prefix);
+  const replacedIslands = /* @__PURE__ */ new Set();
+  const pushSegment = (oldStart, oldEnd, replacement) => {
+    const old = original.slice(oldStart, oldEnd);
+    if (old === replacement) return;
+    const prefix = commonPrefix(old, replacement);
+    const suffix = commonSuffix(old, replacement, prefix);
     effective.push({
-      blockStart: edit.start,
-      blockEnd: edit.end,
-      oldStart: edit.start + prefix,
-      oldEnd: edit.end - suffix,
-      replacement: edit.text.slice(prefix, edit.text.length - suffix)
+      oldStart: oldStart + prefix,
+      oldEnd: oldEnd - suffix,
+      replacement: replacement.slice(prefix, replacement.length - suffix)
     });
+  };
+  for (const edit of sorted) {
+    if (edit.island) {
+      replacedIslands.add(`${edit.start}:${edit.end}`);
+      pushSegment(edit.start, edit.end, edit.text);
+      continue;
+    }
+    if (edit.text === original.slice(edit.start, edit.end)) continue;
+    const inside = islands.filter((island) => island.start >= edit.start && island.end <= edit.end);
+    let oldCursor2 = edit.start;
+    let newCursor2 = 0;
+    for (const island of inside) {
+      const at = locateIsland(
+        edit.text,
+        island.raw,
+        newCursor2,
+        newCursor2 + (island.start - oldCursor2),
+        edit.text.length - (edit.end - island.start),
+        original.slice(edit.start, edit.end),
+        island.start - edit.start,
+        island.end - edit.start
+      );
+      if (at === -1) {
+        const name = island.meta.tag ?? island.meta.name ?? island.meta.lang ?? "";
+        throw new SourceSpliceError(
+          "island_edit",
+          `edit [${edit.start}, ${edit.end}) changes or removes the protected ${island.islandType}${name ? ` (${String(name)})` : ""} island at [${island.start}, ${island.end}); islands change only through islandEdit()`
+        );
+      }
+      pushSegment(oldCursor2, island.start, edit.text.slice(newCursor2, at));
+      oldCursor2 = island.end;
+      newCursor2 = at + island.raw.length;
+    }
+    pushSegment(oldCursor2, edit.end, edit.text.slice(newCursor2));
   }
   if (effective.length === 0) {
     return {
@@ -1306,9 +1747,10 @@ function spliceSave(original, edits, options = {}) {
     const newStart = edit.oldStart + delta;
     text += edit.replacement;
     const newEnd = newStart + edit.replacement.length;
+    const owner = sorted.find((e) => e.start <= edit.oldStart && e.end >= edit.oldEnd);
     pending.push({
-      blockStart: edit.blockStart,
-      blockEnd: edit.blockEnd,
+      blockStart: owner?.start ?? edit.oldStart,
+      blockEnd: owner?.end ?? edit.oldEnd,
       oldStart: edit.oldStart,
       oldEnd: edit.oldEnd,
       newStart,
@@ -1346,11 +1788,8 @@ function spliceSave(original, edits, options = {}) {
     else newIslandsByStart.set(island.start, [island.raw]);
   }
   const disturbed = [];
-  for (const island of listIslands(blocks)) {
-    const touched = effective.some(
-      (edit) => island.start < edit.blockEnd && island.end > edit.blockStart
-    );
-    if (touched) continue;
+  for (const island of islands) {
+    if (replacedIslands.has(`${island.start}:${island.end}`)) continue;
     const mappedStart = mapPosition(changes, island.start, { assoc: 1 }).pos;
     const candidates = newIslandsByStart.get(mappedStart);
     if (!candidates || !candidates.includes(island.raw)) {
@@ -1368,10 +1807,11 @@ function spliceSave(original, edits, options = {}) {
     bytesOutsideEditsIdentical,
     disturbed
   };
-  if (options.requireIntegrity && !integrity.ok) {
+  if ((options.requireIntegrity ?? true) && !integrity.ok) {
+    const named = disturbed.slice(0, 5).map((island) => `${island.islandType}@${island.start}`).join(", ");
     throw new SourceSpliceError(
       "integrity",
-      `the edit disturbed ${disturbed.length} protected island(s) outside the edited blocks`
+      `integrity: the save disturbed ${disturbed.length} protected island(s) it did not replace${named ? ` (${named}${disturbed.length > 5 ? ", \u2026" : ""})` : ""}${bytesOutsideEditsIdentical ? "" : "; bytes outside the edits changed"}`
     );
   }
   return { text, changed: true, changes, integrity, blocks: newBlocks };
@@ -1413,6 +1853,6 @@ function mapRange(changes, start, end, unit = "utf16") {
   return { start: newStart, end: newEnd, touched, collapsed: newEnd === newStart && end > start };
 }
 
-export { SourceSpliceError, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, isSingleDollarMath, joinSource, listIslands, mapPosition, mapRange, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource };
+export { FENCE_WHITESPACE, NESTING_FENCE_LANGUAGES, SourceSpliceError, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
 //# sourceMappingURL=source.js.map
 //# sourceMappingURL=source.js.map