@ai-matrx/content-ir 0.16.1 → 0.16.2

package/dist/source.cjs

@@ -321,6 +321,9 @@ var STRUCTURAL_MARKDOWN = /\]\(|https?:\/\/|\*\*|(?:^|\n)[ \t]{0,3}#{1,6}[ \t]|(
 var LATEX_COMMAND = /\\[a-zA-Z]/;
 var PROSE_WORD = /[A-Za-z]{3,}/g;
 var PROSE_WORD_LIMIT = 6;
+function isMathPlaceholder(inner) {
+  return /^[\s.…·⋯_-]*$/.test(inner) && /[.…·⋯]/.test(inner);
+}
 function opensFlowBlock(text, open) {
   for (let k = open - 1; k >= 0 && text[k] !== "\n"; k--) if (text[k] !== " " && text[k] !== "	") return false;
   for (let k = open + 2; k < text.length && text[k] !== "\n"; k++) if (text[k] !== " " && text[k] !== "	" && text[k] !== "\r") return false;
@@ -339,13 +342,84 @@ function looksLikeDisplayMath(inner, flow = false) {
 }
 function codeRanges(text) {
   const ranges = [];
-  for (const re of [/```[\s\S]*?(?:```|$)/g, /~~~[\s\S]*?(?:~~~|$)/g, /`[^`\n]*`/g]) {
-    let m;
-    while ((m = re.exec(text)) !== null) {
-      ranges.push([m.index, m.index + m[0].length]);
-      if (m[0].length === 0) re.lastIndex++;
+  const n = text.length;
+  let fence = null;
+  let para = -1;
+  const spans = (from, to) => {
+    const runs = [];
+    for (let k = from; k < to; ) {
+      if (text.charCodeAt(k) !== 96) {
+        k++;
+        continue;
+      }
+      let len = 1;
+      while (k + len < to && text.charCodeAt(k + len) === 96) len++;
+      runs.push([k, len]);
+      k += len;
+    }
+    const nextSame = new Array(runs.length).fill(-1);
+    const last = /* @__PURE__ */ new Map();
+    for (let r = runs.length - 1; r >= 0; r--) {
+      const len = runs[r][1];
+      nextSame[r] = last.get(len) ?? -1;
+      last.set(len, r);
+    }
+    for (let r = 0; r < runs.length; ) {
+      const j = nextSame[r];
+      if (j === -1) {
+        r++;
+        continue;
+      }
+      const [s0] = runs[r];
+      const [s1, l1] = runs[j];
+      ranges.push([s0, s1 + l1]);
+      r = j + 1;
     }
-  }
+  };
+  let pos = 0;
+  while (pos <= n) {
+    const nl = text.indexOf("\n", pos);
+    const lineEnd = nl === -1 ? n : nl;
+    const line = text.slice(pos, lineEnd);
+    let indent = 0;
+    while (indent < line.length && line[indent] === " ") indent++;
+    const body = line.slice(indent);
+    if (fence) {
+      const trimmed = trimFenceLine(line);
+      let closes = false;
+      if (fence.char === "`") {
+        const kind = classifyInnerFenceLine(trimmed, fence.len, fence.nests, fence.depth);
+        if (kind === "open-nested") fence.depth++;
+        else if (kind === "close-nested") fence.depth--;
+        else if (kind === "close-outer") closes = indent <= 3;
+      } else {
+        let t = 0;
+        while (t < trimmed.length && trimmed[t] === "~") t++;
+        closes = indent <= 3 && t >= fence.len && trimFenceLine(trimmed.slice(t)) === "";
+      }
+      if (closes) {
+        ranges.push([fence.start, lineEnd]);
+        fence = null;
+      }
+    } else {
+      const opener = indent <= 3 ? /^(`{3,}|~{3,})(.*)$/.exec(body) : null;
+      const info = opener ? opener[2] : "";
+      if (opener && !(opener[1][0] === "`" && info.includes("`"))) {
+        if (para !== -1) spans(para, pos);
+        para = -1;
+        const char = opener[1][0];
+        const parsed = char === "`" ? parseFenceOpener(line) : null;
+        fence = { char, len: opener[1].length, start: pos, nests: fenceNestsInnerFences(parsed?.lang), depth: 0 };
+      } else if (body.trim() === "") {
+        if (para !== -1) spans(para, pos);
+        para = -1;
+      } else if (para === -1) para = pos;
+    }
+    if (nl === -1) break;
+    pos = nl + 1;
+  }
+  if (fence) ranges.push([fence.start, n]);
+  else if (para !== -1) spans(para, n);
   return ranges.sort((a, b) => a[0] - b[0]);
 }
 function pendingDisplayMathOpener(text) {
@@ -368,6 +442,10 @@ function pendingDisplayMathOpener(text) {
     const open = tokens[j];
     const close = tokens[j + 1];
     if (close === void 0) break;
+    if (isMathPlaceholder(text.slice(open + 2, close))) {
+      j += 2;
+      continue;
+    }
     if (looksLikeDisplayMath(text.slice(open + 2, close), opensFlowBlock(text, open))) {
       paired.add(open);
       paired.add(close);
@@ -412,6 +490,10 @@ function pairDisplayMath(text) {
     const open = tokens[j];
     const close = tokens[j + 1];
     if (close === void 0) break;
+    if (isMathPlaceholder(text.slice(open + 2, close))) {
+      j += 2;
+      continue;
+    }
     if (looksLikeDisplayMath(text.slice(open + 2, close), opensFlowBlock(text, open))) {
       pairs.set(open, close);
       j += 2;
@@ -749,6 +831,19 @@ var Tokenizer = class {
     this.mathPairs = pairDisplayMath(text);
     this.provisionalFrom = streaming ? pendingDisplayMathOpener(text) : -1;
     if (this.provisionalFrom === -1) this.provisionalFrom = Number.POSITIVE_INFINITY;
+    if (streaming && text.includes("$$")) {
+      for (const pad of ["`", "``", "```", "~~~"]) {
+        const alt = pairDisplayMath(text + pad);
+        for (const [open, close] of this.mathPairs) {
+          if (alt.get(open) !== close) this.provisionalFrom = Math.min(this.provisionalFrom, open);
+        }
+        for (const [open, close] of alt) {
+          if (close < text.length && this.mathPairs.get(open) !== close) {
+            this.provisionalFrom = Math.min(this.provisionalFrom, open);
+          }
+        }
+      }
+    }
     let start = 0;
     for (let i = 0; i < text.length; i++) {
       if (text.charCodeAt(i) === 10) {
@@ -789,13 +884,17 @@ var Tokenizer = class {
   /**
    * Streaming: an open last line that begins with `{` is undecided — each new
    * byte can balance or unbalance its braces and flip the whole line between
-   * prose and a JSON island — so it is provisional until it ends (G5).
+   * prose and a JSON island — and so is a line that is exactly one allowed
+   * HTML tag so far; both are provisional until they end (G5).
    */
   markOpenJsonTail() {
     const last = this.lineStarts.length - 1;
     if (!this.isOpenTail(last)) return;
     const start = this.lineStarts[last];
-    if (this.lineText(last).trimStart().startsWith("{")) {
+    const t = this.lineText(last).trim();
+    const oneTag = t.startsWith("<") ? readXmlTag(t, 0) : null;
+    const loneTag = !!oneTag && oneTag.raw === t && ALLOWED_RAW_HTML_TAGS.has(oneTag.tagName.toLowerCase());
+    if (t.startsWith("{") || loneTag) {
       this.provisionalFrom = Math.min(this.provisionalFrom, start);
     }
   }
@@ -1089,10 +1188,16 @@ var Tokenizer = class {
         continue;
       }
       if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
-      if (this.isOpenTail(k)) break;
+      if (this.isOpenTail(k)) return this.heldFootnote(p, open[1]);
       last = k;
     }
-    return this.island("footnote_def", p, this.contentEnds[last], true, { id: open[1] });
+    const end = this.contentEnds[last];
+    if (this.paragraphMayGrow(end)) return this.heldFootnote(p, open[1]);
+    return this.island("footnote_def", p, end, true, { id: open[1] });
+  }
+  /** A footnote whose continuation is still arriving: open to the end of the text. */
+  heldFootnote(p, id) {
+    return this.island("footnote_def", p, this.n, false, { id });
   }
   /**
    * Backtick fence — the renderer splitter's close rules: JSON-string aware for
@@ -1336,7 +1441,12 @@ var Tokenizer = class {
       return this.island("html_block", p, close.index + close[0].length, true, { tag: name });
     }
     const typeSix = HTML_BLOCK_LEVEL_TAGS.has(name);
-    const typeSeven = atParaStart && t.trimEnd() === tag.raw && !this.isOpenTail(li);
+    const loneTag = atParaStart && t.trimEnd() === tag.raw;
+    if (loneTag && this.isOpenTail(li)) {
+      this.provisionalFrom = Math.min(this.provisionalFrom, p);
+      return null;
+    }
+    const typeSeven = loneTag;
     if (!typeSix && !typeSeven) return null;
     let end = this.contentEnds[li];
     for (let k = li + 1; k < this.lineCount && !this.lineIsBlank(k); k++) {
@@ -1428,10 +1538,6 @@ var Tokenizer = class {
     let breakLine = -1;
     for (let k = li + 1; k < this.lineCount; k++) {
       if (this.lineIsBlank(k)) break;
-      if (this.lineStarts[k] >= this.provisionalFrom) {
-        end = this.n;
-        break;
-      }
       if (this.detectBlock(this.lineStarts[k], k, false)) {
         breakLine = k;
         break;
@@ -1574,7 +1680,11 @@ var Tokenizer = class {
         }
         const end = singleDollarMathEnd(text, i, pe);
         this.tick(end > 0 ? end - i : Math.min(403, pe - i));
-        if (end > 0 && !(this.streaming && end === this.n)) {
+        if (end > 0 && this.streaming && end === this.n) {
+          this.provisionalFrom = Math.min(this.provisionalFrom, i);
+          return { inlines, breakIsland: null, orphan: false };
+        }
+        if (end > 0) {
           inlines.push(this.island("math_inline", i, end, true, { delimiter: "$" }));
           i = end;
           continue;
@@ -2077,6 +2187,7 @@ exports.blockRangeEdit = blockRangeEdit;
 exports.buildCodePointIndex = buildCodePointIndex;
 exports.classifyInnerFenceLine = classifyInnerFenceLine;
 exports.fenceNestsInnerFences = fenceNestsInnerFences;
+exports.isMathPlaceholder = isMathPlaceholder;
 exports.isSingleDollarMath = isSingleDollarMath;
 exports.islandEdit = islandEdit;
 exports.joinSource = joinSource;