@ai-matrx/content-ir 0.13.0 → 0.14.0

package/dist/source.d.cts

@@ -51,12 +51,25 @@ type BlockIslandType =
  | "anchor"
 /** `$$ … $$` display math on its own lines. */
  | "math_block"
-/** YAML front matter at the start of the text. */
+/** YAML (`---`) or TOML (`+++`) front matter at the start of the text (meta.format). */
  | "front_matter"
 /** A box-drawing tree diagram. */
- | "tree";
+ | "tree"
+/**
+ * A generic-directive container `:::name[label]{attrs}` … `:::` (tabs,
+ * columns, details, figures, callouts, TOC — meta.name), a leaf directive
+ * `::name` on its own line (meta.leaf), or a MkDocs admonition
+ * `!!! type "Title"` / `??? type` with its indented body (meta.syntax "mkdocs").
+ */
+ | "directive"
+/** A GitHub alert / Obsidian callout: a blockquote opening `> [!TYPE]` (meta.name). */
+ | "callout"
+/** A footnote definition `[^id]: …` with its indented continuation lines (meta.id). */
+ | "footnote_def";
 /** Island types that live inside a prose block. */
-type InlineIslandType = "variable" | "cite" | "math_inline" | "html_tag" | "xml_tag" | "xml_inline" | "html_comment" | "anchor" | "kind_json" | "media_ref";
+type InlineIslandType = "variable" | "cite" | "math_inline" | "html_tag" | "xml_tag" | "xml_inline" | "html_comment" | "anchor" | "kind_json" | "media_ref"
+/** `[[Page]]`, `[[Page|alias]]` (meta.target) — or `![[Page]]` (meta.embed). */
+ | "wikilink";
 type IslandType = BlockIslandType | InlineIslandType;
 type IslandMeta = Readonly<Record<string, string | number | boolean>>;
 interface SourceSpan {
@@ -92,7 +105,16 @@ interface SourceBlock extends SourceSpan {
  * Cut `text` into blocks whose raw concatenation is exactly `text`.
  * Never throws; unclosed constructs are islands with `complete: false`.
  */
-declare function tokenizeSource(text: string): SourceBlock[];
+interface TokenizeOptions {
+    /**
+     * The text is a prefix of a message still arriving. The final line (no
+     * terminator yet) can still grow — a bare ``` there may become ```bash — so
+     * it closes no fence until its newline arrives (verify-RC-B3 residual R2).
+     * Omit for stored text: a complete document's last line is final.
+     */
+    readonly streaming?: boolean;
+}
+declare function tokenizeSource(text: string, options?: TokenizeOptions): SourceBlock[];
 /** The exact source the blocks were cut from. */
 declare function joinSource(blocks: readonly SourceBlock[]): string;
 /** Every island — block-level and inline — in document order. */
@@ -272,8 +294,9 @@ declare function toUtf16Offset(text: string, codePoint: number): number;
  *
  * Pandoc's rule — the opening `$` is not followed by whitespace, the closing
  * `$` is not preceded by whitespace and not followed by a digit — plus a math
- * signal: a TeX command or operator (`\alpha`, `^`, `_`, `{`, `=`, …), or a
- * lone variable (`$x$`, `$n$`, `$x'$`). Content that opens with a digit needs
+ * signal: a TeX command or operator (`\alpha`, `^`, `_`, `{`, `=`, …), a
+ * lone variable (`$x$`, `$n$`, `$x'$`), or a single-letter math shape
+ * (`$f(g(x))$`, `$u v$`, `$f'(x)$` — see `isMathShape`). Content that opens with a digit needs
  * a real TeX signal, so a price range can never read as math.
  */
 declare function isSingleDollarMath(content: string, after: string | undefined): boolean;
@@ -342,13 +365,26 @@ type InnerFenceLine =
 /** Bare ``` that closes the OUTER fence. */
  | "close-outer";
 /**
- * Classify one TRIMMED line inside an open backtick fence.
+ * Classify one line inside an open backtick fence, already trimmed with
+ * {@link trimFenceLine} (never a language's own `trim()`).
  *
  * @param trimmed     the line with surrounding whitespace removed
  * @param openTicks   backtick count of the outer fence's opener
  * @param nests       whether the outer fence follows the nesting rule
  * @param nestedDepth how many nested fences are open right now
  */
+/**
+ * The whitespace a fence line may carry around its backticks and info string:
+ * CommonMark's ASCII whitespace — space, tab, LF, CR, form feed, vertical tab.
+ * Defined HERE, not by a language's `trim()`: JavaScript's `trim()` also strips
+ * U+FEFF, U+00A0 and the Unicode spaces, while Python's `strip()` also strips
+ * U+0085 and U+001C–U+001F — which made the two twins disagree on lines such
+ * as ```` ```\uFEFF ```` (verify-RC-B3 residual R4). Every caller trims a line
+ * with `trimFenceLine` before classifying it; the Python twin mirrors both.
+ */
+declare const FENCE_WHITESPACE = " \t\n\r\f\v";
+/** Strip {@link FENCE_WHITESPACE} (and nothing else) from both ends of a line. */
+declare function trimFenceLine(line: string): string;
 declare function classifyInnerFenceLine(trimmed: string, openTicks: number, nests: boolean, nestedDepth: number): InnerFenceLine;
 
 /**
@@ -371,4 +407,4 @@ declare function looksLikeDisplayMath(inner: string): boolean;
 /** Map of `$$` opener offset → closer offset for every pair the renderer shows as math. */
 declare function pairDisplayMath(text: string): Map<number, number>;
 
-export { type BlockIslandType, type CodePointIndex, type DisturbedIsland, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource };
+export { type BlockIslandType, type CodePointIndex, type DisturbedIsland, FENCE_WHITESPACE, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };

package/dist/source.d.ts

@@ -51,12 +51,25 @@ type BlockIslandType =
  | "anchor"
 /** `$$ … $$` display math on its own lines. */
  | "math_block"
-/** YAML front matter at the start of the text. */
+/** YAML (`---`) or TOML (`+++`) front matter at the start of the text (meta.format). */
  | "front_matter"
 /** A box-drawing tree diagram. */
- | "tree";
+ | "tree"
+/**
+ * A generic-directive container `:::name[label]{attrs}` … `:::` (tabs,
+ * columns, details, figures, callouts, TOC — meta.name), a leaf directive
+ * `::name` on its own line (meta.leaf), or a MkDocs admonition
+ * `!!! type "Title"` / `??? type` with its indented body (meta.syntax "mkdocs").
+ */
+ | "directive"
+/** A GitHub alert / Obsidian callout: a blockquote opening `> [!TYPE]` (meta.name). */
+ | "callout"
+/** A footnote definition `[^id]: …` with its indented continuation lines (meta.id). */
+ | "footnote_def";
 /** Island types that live inside a prose block. */
-type InlineIslandType = "variable" | "cite" | "math_inline" | "html_tag" | "xml_tag" | "xml_inline" | "html_comment" | "anchor" | "kind_json" | "media_ref";
+type InlineIslandType = "variable" | "cite" | "math_inline" | "html_tag" | "xml_tag" | "xml_inline" | "html_comment" | "anchor" | "kind_json" | "media_ref"
+/** `[[Page]]`, `[[Page|alias]]` (meta.target) — or `![[Page]]` (meta.embed). */
+ | "wikilink";
 type IslandType = BlockIslandType | InlineIslandType;
 type IslandMeta = Readonly<Record<string, string | number | boolean>>;
 interface SourceSpan {
@@ -92,7 +105,16 @@ interface SourceBlock extends SourceSpan {
  * Cut `text` into blocks whose raw concatenation is exactly `text`.
  * Never throws; unclosed constructs are islands with `complete: false`.
  */
-declare function tokenizeSource(text: string): SourceBlock[];
+interface TokenizeOptions {
+    /**
+     * The text is a prefix of a message still arriving. The final line (no
+     * terminator yet) can still grow — a bare ``` there may become ```bash — so
+     * it closes no fence until its newline arrives (verify-RC-B3 residual R2).
+     * Omit for stored text: a complete document's last line is final.
+     */
+    readonly streaming?: boolean;
+}
+declare function tokenizeSource(text: string, options?: TokenizeOptions): SourceBlock[];
 /** The exact source the blocks were cut from. */
 declare function joinSource(blocks: readonly SourceBlock[]): string;
 /** Every island — block-level and inline — in document order. */
@@ -272,8 +294,9 @@ declare function toUtf16Offset(text: string, codePoint: number): number;
  *
  * Pandoc's rule — the opening `$` is not followed by whitespace, the closing
  * `$` is not preceded by whitespace and not followed by a digit — plus a math
- * signal: a TeX command or operator (`\alpha`, `^`, `_`, `{`, `=`, …), or a
- * lone variable (`$x$`, `$n$`, `$x'$`). Content that opens with a digit needs
+ * signal: a TeX command or operator (`\alpha`, `^`, `_`, `{`, `=`, …), a
+ * lone variable (`$x$`, `$n$`, `$x'$`), or a single-letter math shape
+ * (`$f(g(x))$`, `$u v$`, `$f'(x)$` — see `isMathShape`). Content that opens with a digit needs
  * a real TeX signal, so a price range can never read as math.
  */
 declare function isSingleDollarMath(content: string, after: string | undefined): boolean;
@@ -342,13 +365,26 @@ type InnerFenceLine =
 /** Bare ``` that closes the OUTER fence. */
  | "close-outer";
 /**
- * Classify one TRIMMED line inside an open backtick fence.
+ * Classify one line inside an open backtick fence, already trimmed with
+ * {@link trimFenceLine} (never a language's own `trim()`).
  *
  * @param trimmed     the line with surrounding whitespace removed
  * @param openTicks   backtick count of the outer fence's opener
  * @param nests       whether the outer fence follows the nesting rule
  * @param nestedDepth how many nested fences are open right now
  */
+/**
+ * The whitespace a fence line may carry around its backticks and info string:
+ * CommonMark's ASCII whitespace — space, tab, LF, CR, form feed, vertical tab.
+ * Defined HERE, not by a language's `trim()`: JavaScript's `trim()` also strips
+ * U+FEFF, U+00A0 and the Unicode spaces, while Python's `strip()` also strips
+ * U+0085 and U+001C–U+001F — which made the two twins disagree on lines such
+ * as ```` ```\uFEFF ```` (verify-RC-B3 residual R4). Every caller trims a line
+ * with `trimFenceLine` before classifying it; the Python twin mirrors both.
+ */
+declare const FENCE_WHITESPACE = " \t\n\r\f\v";
+/** Strip {@link FENCE_WHITESPACE} (and nothing else) from both ends of a line. */
+declare function trimFenceLine(line: string): string;
 declare function classifyInnerFenceLine(trimmed: string, openTicks: number, nests: boolean, nestedDepth: number): InnerFenceLine;
 
 /**
@@ -371,4 +407,4 @@ declare function looksLikeDisplayMath(inner: string): boolean;
 /** Map of `$$` opener offset → closer offset for every pair the renderer shows as math. */
 declare function pairDisplayMath(text: string): Map<number, number>;
 
-export { type BlockIslandType, type CodePointIndex, type DisturbedIsland, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource };
+export { type BlockIslandType, type CodePointIndex, type DisturbedIsland, FENCE_WHITESPACE, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };

package/dist/source.js

@@ -209,11 +209,22 @@ var NESTING_FENCE_LANGUAGES = /* @__PURE__ */ new Set([
 function fenceNestsInnerFences(language) {
   return !!language && NESTING_FENCE_LANGUAGES.has(language.toLowerCase());
 }
+var FENCE_WHITESPACE = " 	\n\r\f\v";
+function isFenceWhitespace(ch) {
+  return ch !== void 0 && ch !== "" && FENCE_WHITESPACE.includes(ch);
+}
+function trimFenceLine(line) {
+  let start = 0;
+  let end = line.length;
+  while (start < end && isFenceWhitespace(line[start])) start++;
+  while (end > start && isFenceWhitespace(line[end - 1])) end--;
+  return start === 0 && end === line.length ? line : line.slice(start, end);
+}
 function classifyInnerFenceLine(trimmed, openTicks, nests, nestedDepth) {
   let ticks = 0;
   while (ticks < trimmed.length && trimmed[ticks] === "`") ticks++;
   if (ticks < 3) return "content";
-  const info = trimmed.slice(ticks).trim();
+  const info = trimFenceLine(trimmed.slice(ticks));
   if (info === "") {
     if (ticks < openTicks) return "content";
     if (nests && nestedDepth > 0) return "close-nested";
@@ -226,13 +237,42 @@ function classifyInnerFenceLine(trimmed, openTicks, nests, nestedDepth) {
 // source/math.ts
 var TEX_SIGNAL = /\\[A-Za-z]+|[\\^_{}=<>+]/;
 var LONE_VARIABLE = /^[A-Za-z](?:'|[0-9])?$/;
+var MATH_SHAPE_CHARS = /^[A-Za-z0-9'\s(),.*\/-]+$/;
+var FUNCTION_NAMES = /* @__PURE__ */ new Set([
+  "sin",
+  "cos",
+  "tan",
+  "sec",
+  "csc",
+  "cot",
+  "log",
+  "ln",
+  "exp",
+  "lim",
+  "max",
+  "min",
+  "det",
+  "gcd",
+  "mod",
+  "arg",
+  "sinh",
+  "cosh",
+  "tanh"
+]);
+function isMathShape(content) {
+  if (!MATH_SHAPE_CHARS.test(content)) return false;
+  const words = content.match(/[A-Za-z]+/g);
+  if (!words) return false;
+  if (!words.every((w) => w.length === 1 || FUNCTION_NAMES.has(w))) return false;
+  return /[A-Za-z]'*\(|[A-Za-z]'|\b[A-Za-z]\s+[A-Za-z]\b|\([^)]*[A-Za-z][^)]*\)/.test(content);
+}
 function isSingleDollarMath(content, after) {
   if (!content || content.length > 400) return false;
   if (/^\s/.test(content) || /\s$/.test(content)) return false;
   if (after !== void 0 && /[0-9]/.test(after)) return false;
   if (LONE_VARIABLE.test(content)) return true;
   if (/^[0-9]/.test(content)) return /\\[A-Za-z]+|[\\^_{}]/.test(content);
-  return TEX_SIGNAL.test(content);
+  return TEX_SIGNAL.test(content) || isMathShape(content);
 }
 function singleDollarMathEnd(text, open, limit = text.length) {
   const bound = Math.min(limit, open + 403);
@@ -468,6 +508,10 @@ var XmlContainerTracker = class {
 
 // source/tokenize.ts
 var NOT_WHITESPACE = /\S/;
+var DIRECTIVE_OPEN = /^(:{2,})([A-Za-z][\w-]*)/;
+var MKDOCS_OPEN = /^(!!!|\?\?\?\+?)[ \t]+([A-Za-z][\w-]*)/;
+var CALLOUT_OPEN = /^>[ \t]*\[!([A-Za-z][\w-]*)\]([+-])?/;
+var FOOTNOTE_DEF = /^\[\^([^\]\s]+)\]:/;
 function countStructuralBraces(source) {
   let opens = 0;
   let closes = 0;
@@ -595,8 +639,9 @@ function isSplitterTreeLine(line) {
 }
 var MEDIA_REF_PREFIXES = ["[Image URL:", "[Video URL:", "[Audio URL:"];
 var Tokenizer = class {
-  constructor(text) {
+  constructor(text, streaming = false) {
     this.text = text;
+    this.streaming = streaming;
     this.n = text.length;
     this.mathPairs = pairDisplayMath(text);
     let start = 0;
@@ -615,6 +660,7 @@ var Tokenizer = class {
     }
   }
   text;
+  streaming;
   n;
   lineStarts = [];
   contentEnds = [];
@@ -652,6 +698,10 @@ var Tokenizer = class {
     }
     return lo < list.length ? list[lo] : -1;
   }
+  /** Streaming only: the final line, still waiting for its terminator. */
+  isOpenTail(k) {
+    return this.streaming && k === this.lineStarts.length - 1 && this.lineEnds[k] === this.contentEnds[k];
+  }
   get lineCount() {
     return this.lineStarts.length;
   }
@@ -733,6 +783,22 @@ var Tokenizer = class {
       const fm = this.frontMatter(li, t);
       if (fm) return fm;
     }
+    if (indent <= 3 && t.startsWith("::")) {
+      const directive = this.directive(p, li, t);
+      if (directive) return directive;
+    }
+    if (indent === 0 && (t.startsWith("!!!") || t.startsWith("???"))) {
+      const admonition = this.mkdocsAdmonition(p, li, t);
+      if (admonition) return admonition;
+    }
+    if (indent <= 3 && t.startsWith(">")) {
+      const callout = this.callout(p, li, t);
+      if (callout) return callout;
+    }
+    if (atParaStart && indent <= 3 && t.startsWith("[^")) {
+      const footnote = this.footnoteDefinition(p, li, t);
+      if (footnote) return footnote;
+    }
     if (t.startsWith("```")) return this.backtickFence(p, li, t);
     if (indent <= 3 && t.startsWith("~~~")) {
       const tilde = this.tildeFence(p, li, t);
@@ -770,19 +836,110 @@ var Tokenizer = class {
     return { type, start, end, complete, meta };
   }
   frontMatter(li, t) {
-    if (this.lineText(li) !== "---" || t !== "---") return null;
+    const fence = this.lineText(li);
+    if (fence !== "---" && fence !== "+++" || t !== fence) return null;
+    const toml = fence === "+++";
     let sawContent = false;
     for (let k = li + 1; k < this.lineCount; k++) {
       const line = this.lineText(k);
-      if (line === "---" || line === "...") {
-        return sawContent ? this.island("front_matter", 0, this.contentEnds[k], true) : null;
+      if (line === fence || !toml && line === "...") {
+        return sawContent ? this.island("front_matter", 0, this.contentEnds[k], true, { format: toml ? "toml" : "yaml" }) : null;
       }
       if (!NOT_WHITESPACE.test(line)) continue;
-      if (!/^(?:[A-Za-z0-9_"'-][^:]*:(?:\s|$)|\s+\S|-\s)/.test(line)) return null;
+      const shaped = toml ? /^\s*(?:[A-Za-z0-9_."-]+\s*=|\[|#)/.test(line) : /^(?:[A-Za-z0-9_"'-][^:]*:(?:\s|$)|\s+\S|-\s|#)/.test(line);
+      if (!shaped) return null;
       sawContent = true;
     }
     return null;
   }
+  /**
+   * `:::name[label]{attrs}` … closing `:::` (a colon run at least as long as
+   * the opener's, nesting counted), or a leaf `::name[…]{…}` line. The whole
+   * container is ONE island: its grammar (fence lengths, labels, attributes,
+   * nested tabs) is exactly what a visual editor would re-serialize wrongly.
+   */
+  directive(p, li, t) {
+    const open = DIRECTIVE_OPEN.exec(t);
+    if (!open) return null;
+    const colons = open[1].length;
+    const name = open[2];
+    if (colons === 2) {
+      return this.island("directive", p, this.contentEnds[li], true, { name, leaf: true });
+    }
+    const stack = [colons];
+    for (let k = li + 1; k < this.lineCount; k++) {
+      const line = this.lineText(k).trim();
+      const inner = DIRECTIVE_OPEN.exec(line);
+      if (inner && inner[1].length >= 3) {
+        stack.push(inner[1].length);
+        continue;
+      }
+      const close = /^(:{3,})\s*$/.exec(line);
+      if (close && close[1].length >= stack[stack.length - 1]) {
+        stack.pop();
+        if (stack.length === 0) return this.island("directive", p, this.contentEnds[k], true, { name });
+      }
+    }
+    return this.island("directive", p, this.n, false, { name });
+  }
+  /** MkDocs `!!! type "Title"` / `??? type` / `???+ type` and its indented (or blank-separated indented) body. */
+  mkdocsAdmonition(p, li, t) {
+    const open = MKDOCS_OPEN.exec(t);
+    if (!open) return null;
+    let last = li;
+    for (let k = li + 1; k < this.lineCount; k++) {
+      const line = this.lineText(k);
+      if (/^(?: {4}|\t)/.test(line) && NOT_WHITESPACE.test(line)) {
+        last = k;
+        continue;
+      }
+      if (!NOT_WHITESPACE.test(line)) continue;
+      break;
+    }
+    return this.island("directive", p, this.contentEnds[last], true, {
+      name: open[2].toLowerCase(),
+      syntax: "mkdocs"
+    });
+  }
+  /** A blockquote whose first line is `> [!TYPE]…` — every following `>` line belongs to it. */
+  callout(p, li, t) {
+    const open = CALLOUT_OPEN.exec(t);
+    if (!open) return null;
+    let last = li;
+    for (let k = li + 1; k < this.lineCount; k++) {
+      if (!/^ {0,3}>/.test(this.lineText(k))) break;
+      last = k;
+    }
+    return this.island("callout", p, this.contentEnds[last], true, {
+      name: open[1].toLowerCase(),
+      ...open[2] ? { fold: open[2] === "-" ? "closed" : "open" } : {}
+    });
+  }
+  /** `[^id]: text` plus its continuation lines (indented, or lazy text before a blank line). */
+  footnoteDefinition(p, li, t) {
+    const open = FOOTNOTE_DEF.exec(t);
+    if (!open) return null;
+    let last = li;
+    for (let k = li + 1; k < this.lineCount; k++) {
+      const line = this.lineText(k);
+      if (!NOT_WHITESPACE.test(line)) {
+        let j = k + 1;
+        while (j < this.lineCount && !NOT_WHITESPACE.test(this.lineText(j))) j++;
+        if (j < this.lineCount && /^(?: {4}|\t)/.test(this.lineText(j))) {
+          k = j - 1;
+          continue;
+        }
+        break;
+      }
+      if (/^(?: {4}|\t)/.test(line)) {
+        last = k;
+        continue;
+      }
+      if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
+      last = k;
+    }
+    return this.island("footnote_def", p, this.contentEnds[last], true, { id: open[1] });
+  }
   /**
    * Backtick fence — the renderer splitter's close rules: JSON-string aware for
    * ```json, and THE nested-fence rule (source/fence-nesting.ts) for markdown
@@ -811,8 +968,9 @@ var Tokenizer = class {
     let nestedDepth = 0;
     let sawNested = false;
     for (let k = li + 1; k < this.lineCount; k++) {
+      if (this.isOpenTail(k)) continue;
       const line = this.lineText(k);
-      const trimmedLine = line.trim();
+      const trimmedLine = trimFenceLine(line);
       if (trimmedLine.startsWith("```")) {
         const at2 = line.indexOf("```");
         if (isJson) {
@@ -842,7 +1000,7 @@ var Tokenizer = class {
           }
         }
         const closeTicks = backtickRunLength(line, at);
-        const after = line.slice(at + closeTicks).trim();
+        const after = trimFenceLine(line.slice(at + closeTicks));
         if (closeTicks >= openTicks && after === "" && nestedDepth === 0) return k;
         if (isJson) state = jsonStringState(line, 0, line.length, state);
         continue;
@@ -869,6 +1027,7 @@ var Tokenizer = class {
     if (normalized && SPECIAL_CODE_LANGUAGES.has(normalized)) meta.special = normalized;
     const close = new RegExp(`^ {0,3}~{${ticks},}[ \\t]*$`);
     for (let k = li + 1; k < this.lineCount; k++) {
+      if (this.isOpenTail(k)) continue;
       if (close.test(this.lineText(k))) {
         return this.island("fence", p, this.contentEnds[k], true, meta);
       }
@@ -1258,6 +1417,21 @@ var Tokenizer = class {
         i++;
         continue;
       }
+      if (ch === "[" && text[i + 1] === "[" || ch === "!" && text[i + 1] === "[" && text[i + 2] === "[") {
+        const open = ch === "!" ? i + 3 : i + 2;
+        const close = within("]]", open);
+        const newline = this.next("\n", open);
+        const inner = close === -1 ? "" : text.slice(open, close);
+        if (close !== -1 && (newline === -1 || newline > close) && inner.trim() && !/[[\]]/.test(inner)) {
+          const cut = inner.search(/\\?\|/);
+          const target = (cut < 0 ? inner : inner.slice(0, cut)).trim();
+          inlines.push(
+            this.island("wikilink", i, close + 2, true, ch === "!" ? { target, embed: true } : { target })
+          );
+          i = close + 2;
+          continue;
+        }
+      }
       if (ch === "[") {
         const media = MEDIA_REF_PREFIXES.find((prefix) => text.startsWith(prefix, i));
         if (media) {
@@ -1339,8 +1513,8 @@ var Tokenizer = class {
     return { inlines, breakIsland: null, orphan: false };
   }
 };
-function tokenizeSource(text) {
-  const raw = new Tokenizer(text).run();
+function tokenizeSource(text, options = {}) {
+  const raw = new Tokenizer(text, options.streaming ?? false).run();
   const cp = buildCodePointIndex(text);
   const inline = (island) => ({
     islandType: island.type,
@@ -1389,7 +1563,7 @@ function listIslands(blocks) {
         raw: block.raw
       });
     } else {
-      out.push(...block.inlines);
+      for (const inline of block.inlines) out.push(inline);
     }
   }
   return out;
@@ -1425,15 +1599,37 @@ function islandEdit(island, text) {
 function blockRangeEdit(first, last, text) {
   return { start: first.start, end: last.end, text };
 }
-function locateIsland(text, raw, from, expected, expectedFromEnd) {
-  if (expected >= from && text.startsWith(raw, expected)) return expected;
-  if (expectedFromEnd >= from && text.startsWith(raw, expectedFromEnd)) return expectedFromEnd;
+var CONTEXT_WINDOW = 64;
+function locateIsland(text, raw, from, expected, expectedFromEnd, oldText, oldStart, oldEnd) {
+  const candidates = /* @__PURE__ */ new Set();
+  if (expected >= from && text.startsWith(raw, expected)) candidates.add(expected);
+  if (expectedFromEnd >= from && text.startsWith(raw, expectedFromEnd)) candidates.add(expectedFromEnd);
   const after = text.indexOf(raw, Math.max(from, expected));
+  if (after !== -1) candidates.add(after);
   const before = expected > from ? text.lastIndexOf(raw, expected) : -1;
-  const beforeOk = before >= from ? before : -1;
-  if (after === -1) return beforeOk;
-  if (beforeOk === -1) return after;
-  return expected - beforeOk <= after - expected ? beforeOk : after;
+  if (before >= from) candidates.add(before);
+  if (candidates.size === 0) return text.indexOf(raw, from);
+  if (candidates.size === 1) return candidates.values().next().value;
+  let best = -1;
+  let bestScore = -1;
+  for (const at of candidates) {
+    let score = 0;
+    for (let k = 0; k < CONTEXT_WINDOW; k++) {
+      const n = text[at + raw.length + k];
+      if (n === void 0 || n !== oldText[oldEnd + k]) break;
+      score++;
+    }
+    for (let k = 1; k <= CONTEXT_WINDOW; k++) {
+      const n = text[at - k];
+      if (n === void 0 || n !== oldText[oldStart - k]) break;
+      score++;
+    }
+    if (score > bestScore || score === bestScore && at === expected) {
+      best = at;
+      bestScore = score;
+    }
+  }
+  return best;
 }
 function commonPrefix(a, b) {
   const max = Math.min(a.length, b.length);
@@ -1510,7 +1706,16 @@ function spliceSave(original, edits, options = {}) {
     let oldCursor2 = edit.start;
     let newCursor2 = 0;
     for (const island of inside) {
-      const at = locateIsland(edit.text, island.raw, newCursor2, newCursor2 + (island.start - oldCursor2), edit.text.length - (edit.end - island.start));
+      const at = locateIsland(
+        edit.text,
+        island.raw,
+        newCursor2,
+        newCursor2 + (island.start - oldCursor2),
+        edit.text.length - (edit.end - island.start),
+        original.slice(edit.start, edit.end),
+        island.start - edit.start,
+        island.end - edit.start
+      );
       if (at === -1) {
         const name = island.meta.tag ?? island.meta.name ?? island.meta.lang ?? "";
         throw new SourceSpliceError(
@@ -1648,6 +1853,6 @@ function mapRange(changes, start, end, unit = "utf16") {
   return { start: newStart, end: newEnd, touched, collapsed: newEnd === newStart && end > start };
 }
 
-export { NESTING_FENCE_LANGUAGES, SourceSpliceError, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource };
+export { FENCE_WHITESPACE, NESTING_FENCE_LANGUAGES, SourceSpliceError, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
 //# sourceMappingURL=source.js.map
 //# sourceMappingURL=source.js.map