@ai-matrx/content-ir 0.13.0 → 0.14.1

package/dist/source.cjs

@@ -211,11 +211,22 @@ var NESTING_FENCE_LANGUAGES = /* @__PURE__ */ new Set([
 function fenceNestsInnerFences(language) {
   return !!language && NESTING_FENCE_LANGUAGES.has(language.toLowerCase());
 }
+var FENCE_WHITESPACE = " 	\n\r\f\v";
+function isFenceWhitespace(ch) {
+  return ch !== void 0 && ch !== "" && FENCE_WHITESPACE.includes(ch);
+}
+function trimFenceLine(line) {
+  let start = 0;
+  let end = line.length;
+  while (start < end && isFenceWhitespace(line[start])) start++;
+  while (end > start && isFenceWhitespace(line[end - 1])) end--;
+  return start === 0 && end === line.length ? line : line.slice(start, end);
+}
 function classifyInnerFenceLine(trimmed, openTicks, nests, nestedDepth) {
   let ticks = 0;
   while (ticks < trimmed.length && trimmed[ticks] === "`") ticks++;
   if (ticks < 3) return "content";
-  const info = trimmed.slice(ticks).trim();
+  const info = trimFenceLine(trimmed.slice(ticks));
   if (info === "") {
     if (ticks < openTicks) return "content";
     if (nests && nestedDepth > 0) return "close-nested";
@@ -228,13 +239,43 @@ function classifyInnerFenceLine(trimmed, openTicks, nests, nestedDepth) {
 // source/math.ts
 var TEX_SIGNAL = /\\[A-Za-z]+|[\\^_{}=<>+]/;
 var LONE_VARIABLE = /^[A-Za-z](?:'|[0-9])?$/;
+var MATH_SHAPE_CHARS = /^[A-Za-z0-9'\s(),.*\/-]+$/;
+var FUNCTION_NAMES = /* @__PURE__ */ new Set([
+  "sin",
+  "cos",
+  "tan",
+  "sec",
+  "csc",
+  "cot",
+  "log",
+  "ln",
+  "exp",
+  "lim",
+  "max",
+  "min",
+  "det",
+  "gcd",
+  "mod",
+  "arg",
+  "sinh",
+  "cosh",
+  "tanh"
+]);
+function isMathShape(content) {
+  if (!MATH_SHAPE_CHARS.test(content)) return false;
+  const words = content.match(/[A-Za-z]+/g);
+  if (!words) return false;
+  if (!words.every((w) => w.length === 1 || FUNCTION_NAMES.has(w))) return false;
+  if (/[A-Za-z]'+[A-Za-z]/.test(content) || /(?:^|[\s(,])[A-Za-z]\.(?:[A-Za-z]\.|\s|$)/.test(content)) return false;
+  return /[A-Za-z]'*\(|[A-Za-z]'|\b[A-Za-z]\s+[A-Za-z]\b|\([^)]*[A-Za-z][^)]*\)/.test(content);
+}
 function isSingleDollarMath(content, after) {
   if (!content || content.length > 400) return false;
   if (/^\s/.test(content) || /\s$/.test(content)) return false;
   if (after !== void 0 && /[0-9]/.test(after)) return false;
   if (LONE_VARIABLE.test(content)) return true;
   if (/^[0-9]/.test(content)) return /\\[A-Za-z]+|[\\^_{}]/.test(content);
-  return TEX_SIGNAL.test(content);
+  return TEX_SIGNAL.test(content) || isMathShape(content);
 }
 function singleDollarMathEnd(text, open, limit = text.length) {
   const bound = Math.min(limit, open + 403);
@@ -470,6 +511,10 @@ var XmlContainerTracker = class {
 
 // source/tokenize.ts
 var NOT_WHITESPACE = /\S/;
+var DIRECTIVE_OPEN = /^(:{2,})([A-Za-z][\w-]*)/;
+var MKDOCS_OPEN = /^(!!!|\?\?\?\+?)[ \t]+([A-Za-z][\w-]*)/;
+var CALLOUT_OPEN = /^>[ \t]*\[!([A-Za-z][\w-]*)\]([+-])?/;
+var FOOTNOTE_DEF = /^\[\^([^\]\s]+)\]:/;
 function countStructuralBraces(source) {
   let opens = 0;
   let closes = 0;
@@ -597,8 +642,9 @@ function isSplitterTreeLine(line) {
 }
 var MEDIA_REF_PREFIXES = ["[Image URL:", "[Video URL:", "[Audio URL:"];
 var Tokenizer = class {
-  constructor(text) {
+  constructor(text, streaming = false) {
     this.text = text;
+    this.streaming = streaming;
     this.n = text.length;
     this.mathPairs = pairDisplayMath(text);
     let start = 0;
@@ -617,6 +663,7 @@ var Tokenizer = class {
     }
   }
   text;
+  streaming;
   n;
   lineStarts = [];
   contentEnds = [];
@@ -654,6 +701,10 @@ var Tokenizer = class {
     }
     return lo < list.length ? list[lo] : -1;
   }
+  /** Streaming only: the final line, still waiting for its terminator. */
+  isOpenTail(k) {
+    return this.streaming && k === this.lineStarts.length - 1 && this.lineEnds[k] === this.contentEnds[k];
+  }
   get lineCount() {
     return this.lineStarts.length;
   }
@@ -735,6 +786,22 @@ var Tokenizer = class {
       const fm = this.frontMatter(li, t);
       if (fm) return fm;
     }
+    if (indent <= 3 && t.startsWith("::")) {
+      const directive = this.directive(p, li, t);
+      if (directive) return directive;
+    }
+    if (indent === 0 && (t.startsWith("!!!") || t.startsWith("???"))) {
+      const admonition = this.mkdocsAdmonition(p, li, t);
+      if (admonition) return admonition;
+    }
+    if (indent <= 3 && t.startsWith(">")) {
+      const callout = this.callout(p, li, t);
+      if (callout) return callout;
+    }
+    if (atParaStart && indent <= 3 && t.startsWith("[^")) {
+      const footnote = this.footnoteDefinition(p, li, t);
+      if (footnote) return footnote;
+    }
     if (t.startsWith("```")) return this.backtickFence(p, li, t);
     if (indent <= 3 && t.startsWith("~~~")) {
       const tilde = this.tildeFence(p, li, t);
@@ -772,19 +839,110 @@ var Tokenizer = class {
     return { type, start, end, complete, meta };
   }
   frontMatter(li, t) {
-    if (this.lineText(li) !== "---" || t !== "---") return null;
+    const fence = this.lineText(li);
+    if (fence !== "---" && fence !== "+++" || t !== fence) return null;
+    const toml = fence === "+++";
     let sawContent = false;
     for (let k = li + 1; k < this.lineCount; k++) {
       const line = this.lineText(k);
-      if (line === "---" || line === "...") {
-        return sawContent ? this.island("front_matter", 0, this.contentEnds[k], true) : null;
+      if (line === fence || !toml && line === "...") {
+        return sawContent ? this.island("front_matter", 0, this.contentEnds[k], true, { format: toml ? "toml" : "yaml" }) : null;
       }
       if (!NOT_WHITESPACE.test(line)) continue;
-      if (!/^(?:[A-Za-z0-9_"'-][^:]*:(?:\s|$)|\s+\S|-\s)/.test(line)) return null;
+      const shaped = toml ? /^\s*(?:[A-Za-z0-9_."-]+\s*=|\[|#)/.test(line) : /^(?:[A-Za-z0-9_"'-][^:]*:(?:\s|$)|\s+\S|-\s|#)/.test(line);
+      if (!shaped) return null;
       sawContent = true;
     }
     return null;
   }
+  /**
+   * `:::name[label]{attrs}` … closing `:::` (a colon run at least as long as
+   * the opener's, nesting counted), or a leaf `::name[…]{…}` line. The whole
+   * container is ONE island: its grammar (fence lengths, labels, attributes,
+   * nested tabs) is exactly what a visual editor would re-serialize wrongly.
+   */
+  directive(p, li, t) {
+    const open = DIRECTIVE_OPEN.exec(t);
+    if (!open) return null;
+    const colons = open[1].length;
+    const name = open[2];
+    if (colons === 2) {
+      return this.island("directive", p, this.contentEnds[li], true, { name, leaf: true });
+    }
+    const stack = [colons];
+    for (let k = li + 1; k < this.lineCount; k++) {
+      const line = this.lineText(k).trim();
+      const inner = DIRECTIVE_OPEN.exec(line);
+      if (inner && inner[1].length >= 3) {
+        stack.push(inner[1].length);
+        continue;
+      }
+      const close = /^(:{3,})\s*$/.exec(line);
+      if (close && close[1].length >= stack[stack.length - 1]) {
+        stack.pop();
+        if (stack.length === 0) return this.island("directive", p, this.contentEnds[k], true, { name });
+      }
+    }
+    return this.island("directive", p, this.n, false, { name });
+  }
+  /** MkDocs `!!! type "Title"` / `??? type` / `???+ type` and its indented (or blank-separated indented) body. */
+  mkdocsAdmonition(p, li, t) {
+    const open = MKDOCS_OPEN.exec(t);
+    if (!open) return null;
+    let last = li;
+    for (let k = li + 1; k < this.lineCount; k++) {
+      const line = this.lineText(k);
+      if (/^(?: {4}|\t)/.test(line) && NOT_WHITESPACE.test(line)) {
+        last = k;
+        continue;
+      }
+      if (!NOT_WHITESPACE.test(line)) continue;
+      break;
+    }
+    return this.island("directive", p, this.contentEnds[last], true, {
+      name: open[2].toLowerCase(),
+      syntax: "mkdocs"
+    });
+  }
+  /** A blockquote whose first line is `> [!TYPE]…` — every following `>` line belongs to it. */
+  callout(p, li, t) {
+    const open = CALLOUT_OPEN.exec(t);
+    if (!open) return null;
+    let last = li;
+    for (let k = li + 1; k < this.lineCount; k++) {
+      if (!/^ {0,3}>/.test(this.lineText(k))) break;
+      last = k;
+    }
+    return this.island("callout", p, this.contentEnds[last], true, {
+      name: open[1].toLowerCase(),
+      ...open[2] ? { fold: open[2] === "-" ? "closed" : "open" } : {}
+    });
+  }
+  /** `[^id]: text` plus its continuation lines (indented, or lazy text before a blank line). */
+  footnoteDefinition(p, li, t) {
+    const open = FOOTNOTE_DEF.exec(t);
+    if (!open) return null;
+    let last = li;
+    for (let k = li + 1; k < this.lineCount; k++) {
+      const line = this.lineText(k);
+      if (!NOT_WHITESPACE.test(line)) {
+        let j = k + 1;
+        while (j < this.lineCount && !NOT_WHITESPACE.test(this.lineText(j))) j++;
+        if (j < this.lineCount && /^(?: {4}|\t)/.test(this.lineText(j))) {
+          k = j - 1;
+          continue;
+        }
+        break;
+      }
+      if (/^(?: {4}|\t)/.test(line)) {
+        last = k;
+        continue;
+      }
+      if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
+      last = k;
+    }
+    return this.island("footnote_def", p, this.contentEnds[last], true, { id: open[1] });
+  }
   /**
    * Backtick fence — the renderer splitter's close rules: JSON-string aware for
    * ```json, and THE nested-fence rule (source/fence-nesting.ts) for markdown
@@ -813,8 +971,9 @@ var Tokenizer = class {
     let nestedDepth = 0;
     let sawNested = false;
     for (let k = li + 1; k < this.lineCount; k++) {
+      if (this.isOpenTail(k)) continue;
       const line = this.lineText(k);
-      const trimmedLine = line.trim();
+      const trimmedLine = trimFenceLine(line);
       if (trimmedLine.startsWith("```")) {
         const at2 = line.indexOf("```");
         if (isJson) {
@@ -844,7 +1003,7 @@ var Tokenizer = class {
           }
         }
         const closeTicks = backtickRunLength(line, at);
-        const after = line.slice(at + closeTicks).trim();
+        const after = trimFenceLine(line.slice(at + closeTicks));
         if (closeTicks >= openTicks && after === "" && nestedDepth === 0) return k;
         if (isJson) state = jsonStringState(line, 0, line.length, state);
         continue;
@@ -871,6 +1030,7 @@ var Tokenizer = class {
     if (normalized && SPECIAL_CODE_LANGUAGES.has(normalized)) meta.special = normalized;
     const close = new RegExp(`^ {0,3}~{${ticks},}[ \\t]*$`);
     for (let k = li + 1; k < this.lineCount; k++) {
+      if (this.isOpenTail(k)) continue;
       if (close.test(this.lineText(k))) {
         return this.island("fence", p, this.contentEnds[k], true, meta);
       }
@@ -1260,6 +1420,21 @@ var Tokenizer = class {
         i++;
         continue;
       }
+      if (ch === "[" && text[i + 1] === "[" || ch === "!" && text[i + 1] === "[" && text[i + 2] === "[") {
+        const open = ch === "!" ? i + 3 : i + 2;
+        const close = within("]]", open);
+        const newline = this.next("\n", open);
+        const inner = close === -1 ? "" : text.slice(open, close);
+        if (close !== -1 && (newline === -1 || newline > close) && inner.trim() && !/[[\]]/.test(inner)) {
+          const cut = inner.search(/\\?\|/);
+          const target = (cut < 0 ? inner : inner.slice(0, cut)).trim();
+          inlines.push(
+            this.island("wikilink", i, close + 2, true, ch === "!" ? { target, embed: true } : { target })
+          );
+          i = close + 2;
+          continue;
+        }
+      }
       if (ch === "[") {
         const media = MEDIA_REF_PREFIXES.find((prefix) => text.startsWith(prefix, i));
         if (media) {
@@ -1341,8 +1516,8 @@ var Tokenizer = class {
     return { inlines, breakIsland: null, orphan: false };
   }
 };
-function tokenizeSource(text) {
-  const raw = new Tokenizer(text).run();
+function tokenizeSource(text, options = {}) {
+  const raw = new Tokenizer(text, options.streaming ?? false).run();
   const cp = buildCodePointIndex(text);
   const inline = (island) => ({
     islandType: island.type,
@@ -1391,7 +1566,7 @@ function listIslands(blocks) {
         raw: block.raw
       });
     } else {
-      out.push(...block.inlines);
+      for (const inline of block.inlines) out.push(inline);
     }
   }
   return out;
@@ -1427,15 +1602,37 @@ function islandEdit(island, text) {
 function blockRangeEdit(first, last, text) {
   return { start: first.start, end: last.end, text };
 }
-function locateIsland(text, raw, from, expected, expectedFromEnd) {
-  if (expected >= from && text.startsWith(raw, expected)) return expected;
-  if (expectedFromEnd >= from && text.startsWith(raw, expectedFromEnd)) return expectedFromEnd;
+var CONTEXT_WINDOW = 64;
+function locateIsland(text, raw, from, expected, expectedFromEnd, oldText, oldStart, oldEnd) {
+  const candidates = /* @__PURE__ */ new Set();
+  if (expected >= from && text.startsWith(raw, expected)) candidates.add(expected);
+  if (expectedFromEnd >= from && text.startsWith(raw, expectedFromEnd)) candidates.add(expectedFromEnd);
   const after = text.indexOf(raw, Math.max(from, expected));
+  if (after !== -1) candidates.add(after);
   const before = expected > from ? text.lastIndexOf(raw, expected) : -1;
-  const beforeOk = before >= from ? before : -1;
-  if (after === -1) return beforeOk;
-  if (beforeOk === -1) return after;
-  return expected - beforeOk <= after - expected ? beforeOk : after;
+  if (before >= from) candidates.add(before);
+  if (candidates.size === 0) return text.indexOf(raw, from);
+  if (candidates.size === 1) return candidates.values().next().value;
+  let best = -1;
+  let bestScore = -1;
+  for (const at of candidates) {
+    let score = 0;
+    for (let k = 0; k < CONTEXT_WINDOW; k++) {
+      const n = text[at + raw.length + k];
+      if (n === void 0 || n !== oldText[oldEnd + k]) break;
+      score++;
+    }
+    for (let k = 1; k <= CONTEXT_WINDOW; k++) {
+      const n = text[at - k];
+      if (n === void 0 || n !== oldText[oldStart - k]) break;
+      score++;
+    }
+    if (score > bestScore || score === bestScore && at === expected) {
+      best = at;
+      bestScore = score;
+    }
+  }
+  return best;
 }
 function commonPrefix(a, b) {
   const max = Math.min(a.length, b.length);
@@ -1512,7 +1709,16 @@ function spliceSave(original, edits, options = {}) {
     let oldCursor2 = edit.start;
     let newCursor2 = 0;
     for (const island of inside) {
-      const at = locateIsland(edit.text, island.raw, newCursor2, newCursor2 + (island.start - oldCursor2), edit.text.length - (edit.end - island.start));
+      const at = locateIsland(
+        edit.text,
+        island.raw,
+        newCursor2,
+        newCursor2 + (island.start - oldCursor2),
+        edit.text.length - (edit.end - island.start),
+        original.slice(edit.start, edit.end),
+        island.start - edit.start,
+        island.end - edit.start
+      );
       if (at === -1) {
         const name = island.meta.tag ?? island.meta.name ?? island.meta.lang ?? "";
         throw new SourceSpliceError(
@@ -1650,6 +1856,7 @@ function mapRange(changes, start, end, unit = "utf16") {
   return { start: newStart, end: newEnd, touched, collapsed: newEnd === newStart && end > start };
 }
 
+exports.FENCE_WHITESPACE = FENCE_WHITESPACE;
 exports.NESTING_FENCE_LANGUAGES = NESTING_FENCE_LANGUAGES;
 exports.SourceSpliceError = SourceSpliceError;
 exports.XmlContainerTracker = XmlContainerTracker;
@@ -1674,5 +1881,6 @@ exports.splitsSurrogatePair = splitsSurrogatePair;
 exports.toCodePointOffset = toCodePointOffset;
 exports.toUtf16Offset = toUtf16Offset;
 exports.tokenizeSource = tokenizeSource;
+exports.trimFenceLine = trimFenceLine;
 //# sourceMappingURL=source.cjs.map
 //# sourceMappingURL=source.cjs.map