@ai-matrx/content-ir 0.16.1 → 0.17.0

package/dist/source.cjs

@@ -315,12 +315,167 @@ function singleDollarMathEnd(text, open, limit = text.length) {
   return isSingleDollarMath(content, text[close + 1]) ? close + 1 : -1;
 }
 
+// source/code-ranges.ts
+function findCodeRanges(text) {
+  const ranges = [];
+  const n = text.length;
+  let fence = null;
+  let para = -1;
+  const spans = (from, to) => {
+    const runs = [];
+    for (let k = from; k < to; ) {
+      if (text.charCodeAt(k) !== 96) {
+        k++;
+        continue;
+      }
+      let len = 1;
+      while (k + len < to && text.charCodeAt(k + len) === 96) len++;
+      runs.push([k, len]);
+      k += len;
+    }
+    const nextSame = new Array(runs.length).fill(-1);
+    const last = /* @__PURE__ */ new Map();
+    for (let r = runs.length - 1; r >= 0; r--) {
+      const len = runs[r][1];
+      nextSame[r] = last.get(len) ?? -1;
+      last.set(len, r);
+    }
+    for (let r = 0; r < runs.length; ) {
+      const j = nextSame[r];
+      if (j === -1) {
+        r++;
+        continue;
+      }
+      const [s0] = runs[r];
+      const [s1, l1] = runs[j];
+      ranges.push({ start: s0, end: s1 + l1, kind: "span" });
+      r = j + 1;
+    }
+  };
+  let pos = 0;
+  while (pos <= n) {
+    const nl = text.indexOf("\n", pos);
+    const lineEnd = nl === -1 ? n : nl;
+    const line = text.slice(pos, lineEnd);
+    let indent = 0;
+    while (indent < line.length && line[indent] === " ") indent++;
+    const body = line.slice(indent);
+    if (fence) {
+      const trimmed = trimFenceLine(line);
+      let closes = false;
+      if (fence.char === "`") {
+        const kind = classifyInnerFenceLine(trimmed, fence.len, fence.nests, fence.depth);
+        if (kind === "open-nested") fence.depth++;
+        else if (kind === "close-nested") fence.depth--;
+        else if (kind === "close-outer") closes = indent <= 3;
+      } else {
+        let t = 0;
+        while (t < trimmed.length && trimmed[t] === "~") t++;
+        closes = indent <= 3 && t >= fence.len && trimFenceLine(trimmed.slice(t)) === "";
+      }
+      if (closes) {
+        ranges.push({ start: fence.start, end: lineEnd, kind: "fence" });
+        fence = null;
+      }
+    } else {
+      const opener = indent <= 3 ? /^(`{3,}|~{3,})(.*)$/.exec(body) : null;
+      const info = opener ? opener[2] : "";
+      if (opener && !(opener[1][0] === "`" && info.includes("`"))) {
+        if (para !== -1) spans(para, pos);
+        para = -1;
+        const char = opener[1][0];
+        const parsed = char === "`" ? parseFenceOpener(line) : null;
+        fence = { char, len: opener[1].length, start: pos, nests: fenceNestsInnerFences(parsed?.lang), depth: 0 };
+      } else if (body.trim() === "") {
+        if (para !== -1) spans(para, pos);
+        para = -1;
+      } else if (para === -1) para = pos;
+    }
+    if (nl === -1) break;
+    pos = nl + 1;
+  }
+  if (fence) ranges.push({ start: fence.start, end: n, kind: "fence" });
+  else if (para !== -1) spans(para, n);
+  return ranges.sort((a, b) => a.start - b.start);
+}
+function pendingCodeSpanStart(text) {
+  const n = text.length;
+  if (n === 0 || /\n[ \t]*\n[ \t]*$/.test(text)) return -1;
+  let fence = null;
+  let para = -1;
+  let pos = 0;
+  while (pos <= n) {
+    const nl = text.indexOf("\n", pos);
+    const lineEnd = nl === -1 ? n : nl;
+    const line = text.slice(pos, lineEnd);
+    let indent = 0;
+    while (indent < line.length && line[indent] === " ") indent++;
+    const body = line.slice(indent);
+    if (fence) {
+      const trimmed = trimFenceLine(line);
+      let closes = false;
+      if (fence.char === "`") {
+        const kind = classifyInnerFenceLine(trimmed, fence.len, fence.nests, fence.depth);
+        if (kind === "open-nested") fence.depth++;
+        else if (kind === "close-nested") fence.depth--;
+        else if (kind === "close-outer") closes = indent <= 3;
+      } else {
+        let t = 0;
+        while (t < trimmed.length && trimmed[t] === "~") t++;
+        closes = indent <= 3 && t >= fence.len && trimFenceLine(trimmed.slice(t)) === "";
+      }
+      if (closes) fence = null;
+    } else {
+      const opener = indent <= 3 ? /^(`{3,}|~{3,})(.*)$/.exec(body) : null;
+      const info = opener ? opener[2] : "";
+      if (opener && !(opener[1][0] === "`" && info.includes("`"))) {
+        para = -1;
+        const char = opener[1][0];
+        const parsed = char === "`" ? parseFenceOpener(line) : null;
+        fence = { char, len: opener[1].length, nests: fenceNestsInnerFences(parsed?.lang), depth: 0 };
+      } else if (body.trim() === "") {
+        if (!(nl === -1 && line === "")) para = -1;
+      } else if (para === -1) para = pos;
+    }
+    if (nl === -1) break;
+    pos = nl + 1;
+  }
+  if (fence || para === -1) return -1;
+  const runs = [];
+  for (let k = para; k < n; ) {
+    if (text.charCodeAt(k) !== 96) {
+      k++;
+      continue;
+    }
+    let len = 1;
+    while (k + len < n && text.charCodeAt(k + len) === 96) len++;
+    runs.push([k, len]);
+    k += len;
+  }
+  const growing = (r) => {
+    const [start, len] = runs[r];
+    return start + len === n;
+  };
+  for (let r = 0; r < runs.length; ) {
+    if (growing(r)) return runs[r][0];
+    const len = runs[r][1];
+    let j = r + 1;
+    while (j < runs.length && (runs[j][1] !== len || growing(j))) j++;
+    if (j >= runs.length) return runs[r][0];
+    r = j + 1;
+  }
+  return -1;
+}
+
 // source/math-pairs.ts
 var MAX_MATH_SPAN = 600;
 var STRUCTURAL_MARKDOWN = /\]\(|https?:\/\/|\*\*|(?:^|\n)[ \t]{0,3}#{1,6}[ \t]|(?:^|\n)[ \t]*[-*+][ \t]+|(?:^|\n)[ \t]*\d+[.)][ \t]/;
 var LATEX_COMMAND = /\\[a-zA-Z]/;
 var PROSE_WORD = /[A-Za-z]{3,}/g;
 var PROSE_WORD_LIMIT = 6;
+function isMathPlaceholder(inner) {
+  return /^[\s.…·⋯_-]*$/.test(inner) && /[.…·⋯]/.test(inner);
+}
 function opensFlowBlock(text, open) {
   for (let k = open - 1; k >= 0 && text[k] !== "\n"; k--) if (text[k] !== " " && text[k] !== "	") return false;
   for (let k = open + 2; k < text.length && text[k] !== "\n"; k++) if (text[k] !== " " && text[k] !== "	" && text[k] !== "\r") return false;
@@ -338,15 +493,7 @@ function looksLikeDisplayMath(inner, flow = false) {
   return (s.match(PROSE_WORD) ?? []).length < PROSE_WORD_LIMIT;
 }
 function codeRanges(text) {
-  const ranges = [];
-  for (const re of [/```[\s\S]*?(?:```|$)/g, /~~~[\s\S]*?(?:~~~|$)/g, /`[^`\n]*`/g]) {
-    let m;
-    while ((m = re.exec(text)) !== null) {
-      ranges.push([m.index, m.index + m[0].length]);
-      if (m[0].length === 0) re.lastIndex++;
-    }
-  }
-  return ranges.sort((a, b) => a[0] - b[0]);
+  return findCodeRanges(text).map((r) => [r.start, r.end]);
 }
 function pendingDisplayMathOpener(text) {
   if (!text.includes("$$")) return -1;
@@ -368,6 +515,10 @@ function pendingDisplayMathOpener(text) {
     const open = tokens[j];
     const close = tokens[j + 1];
     if (close === void 0) break;
+    if (isMathPlaceholder(text.slice(open + 2, close))) {
+      j += 2;
+      continue;
+    }
     if (looksLikeDisplayMath(text.slice(open + 2, close), opensFlowBlock(text, open))) {
       paired.add(open);
       paired.add(close);
@@ -412,6 +563,10 @@ function pairDisplayMath(text) {
     const open = tokens[j];
     const close = tokens[j + 1];
     if (close === void 0) break;
+    if (isMathPlaceholder(text.slice(open + 2, close))) {
+      j += 2;
+      continue;
+    }
     if (looksLikeDisplayMath(text.slice(open + 2, close), opensFlowBlock(text, open))) {
       pairs.set(open, close);
       j += 2;
@@ -749,6 +904,27 @@ var Tokenizer = class {
     this.mathPairs = pairDisplayMath(text);
     this.provisionalFrom = streaming ? pendingDisplayMathOpener(text) : -1;
     if (this.provisionalFrom === -1) this.provisionalFrom = Number.POSITIVE_INFINITY;
+    if (streaming && text.includes("$$")) {
+      const span = pendingCodeSpanStart(text);
+      if (span !== -1) {
+        const firstToken = text.indexOf("$$", span);
+        if (firstToken !== -1) this.provisionalFrom = Math.min(this.provisionalFrom, firstToken);
+        for (const [open, close] of this.mathPairs) {
+          if (close >= span) this.provisionalFrom = Math.min(this.provisionalFrom, open);
+        }
+      }
+      for (const pad of ["`", "``", "```", "~~~"]) {
+        const alt = pairDisplayMath(text + pad);
+        for (const [open, close] of this.mathPairs) {
+          if (alt.get(open) !== close) this.provisionalFrom = Math.min(this.provisionalFrom, open);
+        }
+        for (const [open, close] of alt) {
+          if (close < text.length && this.mathPairs.get(open) !== close) {
+            this.provisionalFrom = Math.min(this.provisionalFrom, open);
+          }
+        }
+      }
+    }
     let start = 0;
     for (let i = 0; i < text.length; i++) {
       if (text.charCodeAt(i) === 10) {
@@ -789,13 +965,17 @@ var Tokenizer = class {
   /**
    * Streaming: an open last line that begins with `{` is undecided — each new
    * byte can balance or unbalance its braces and flip the whole line between
-   * prose and a JSON island — so it is provisional until it ends (G5).
+   * prose and a JSON island — and so is a line that is exactly one allowed
+   * HTML tag so far; both are provisional until they end (G5).
    */
   markOpenJsonTail() {
     const last = this.lineStarts.length - 1;
     if (!this.isOpenTail(last)) return;
     const start = this.lineStarts[last];
-    if (this.lineText(last).trimStart().startsWith("{")) {
+    const t = this.lineText(last).trim();
+    const oneTag = t.startsWith("<") ? readXmlTag(t, 0) : null;
+    const loneTag = !!oneTag && oneTag.raw === t && ALLOWED_RAW_HTML_TAGS.has(oneTag.tagName.toLowerCase());
+    if (t.startsWith("{") || loneTag) {
       this.provisionalFrom = Math.min(this.provisionalFrom, start);
     }
   }
@@ -1088,11 +1268,17 @@ var Tokenizer = class {
         last = k;
         continue;
       }
+      if (this.isOpenTail(k)) return this.heldFootnote(p, open[1]);
       if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
-      if (this.isOpenTail(k)) break;
       last = k;
     }
-    return this.island("footnote_def", p, this.contentEnds[last], true, { id: open[1] });
+    const end = this.contentEnds[last];
+    if (this.paragraphMayGrow(end)) return this.heldFootnote(p, open[1]);
+    return this.island("footnote_def", p, end, true, { id: open[1] });
+  }
+  /** A footnote whose continuation is still arriving: open to the end of the text. */
+  heldFootnote(p, id) {
+    return this.island("footnote_def", p, this.n, false, { id });
   }
   /**
    * Backtick fence — the renderer splitter's close rules: JSON-string aware for
@@ -1336,7 +1522,12 @@ var Tokenizer = class {
       return this.island("html_block", p, close.index + close[0].length, true, { tag: name });
     }
     const typeSix = HTML_BLOCK_LEVEL_TAGS.has(name);
-    const typeSeven = atParaStart && t.trimEnd() === tag.raw && !this.isOpenTail(li);
+    const loneTag = atParaStart && t.trimEnd() === tag.raw;
+    if (loneTag && this.isOpenTail(li)) {
+      this.provisionalFrom = Math.min(this.provisionalFrom, p);
+      return null;
+    }
+    const typeSeven = loneTag;
     if (!typeSix && !typeSeven) return null;
     let end = this.contentEnds[li];
     for (let k = li + 1; k < this.lineCount && !this.lineIsBlank(k); k++) {
@@ -1428,10 +1619,6 @@ var Tokenizer = class {
     let breakLine = -1;
     for (let k = li + 1; k < this.lineCount; k++) {
       if (this.lineIsBlank(k)) break;
-      if (this.lineStarts[k] >= this.provisionalFrom) {
-        end = this.n;
-        break;
-      }
       if (this.detectBlock(this.lineStarts[k], k, false)) {
         breakLine = k;
         break;
@@ -1574,7 +1761,11 @@ var Tokenizer = class {
         }
         const end = singleDollarMathEnd(text, i, pe);
         this.tick(end > 0 ? end - i : Math.min(403, pe - i));
-        if (end > 0 && !(this.streaming && end === this.n)) {
+        if (end > 0 && this.streaming && end === this.n) {
+          this.provisionalFrom = Math.min(this.provisionalFrom, i);
+          return { inlines, breakIsland: null, orphan: false };
+        }
+        if (end > 0) {
           inlines.push(this.island("math_inline", i, end, true, { delimiter: "$" }));
           i = end;
           continue;
@@ -2077,6 +2268,8 @@ exports.blockRangeEdit = blockRangeEdit;
 exports.buildCodePointIndex = buildCodePointIndex;
 exports.classifyInnerFenceLine = classifyInnerFenceLine;
 exports.fenceNestsInnerFences = fenceNestsInnerFences;
+exports.findCodeRanges = findCodeRanges;
+exports.isMathPlaceholder = isMathPlaceholder;
 exports.isSingleDollarMath = isSingleDollarMath;
 exports.islandEdit = islandEdit;
 exports.joinSource = joinSource;