@ai-matrx/content-ir 0.15.1 → 0.16.0

package/dist/source.cjs

@@ -348,6 +348,50 @@ function codeRanges(text) {
   }
   return ranges.sort((a, b) => a[0] - b[0]);
 }
+function pendingDisplayMathOpener(text) {
+  if (!text.includes("$$")) return -1;
+  const ranges = codeRanges(text);
+  const tokens = [];
+  let r = 0;
+  let maxEnd = -1;
+  for (let i = text.indexOf("$$"); i !== -1 && i < text.length - 1; i = text.indexOf("$$", i)) {
+    while (r < ranges.length && ranges[r][0] <= i) {
+      maxEnd = Math.max(maxEnd, ranges[r][1]);
+      r++;
+    }
+    if (maxEnd <= i) tokens.push(i);
+    i += 2;
+  }
+  const paired = /* @__PURE__ */ new Set();
+  let j = 0;
+  while (j < tokens.length) {
+    const open = tokens[j];
+    const close = tokens[j + 1];
+    if (close === void 0) break;
+    if (looksLikeDisplayMath(text.slice(open + 2, close), opensFlowBlock(text, open))) {
+      paired.add(open);
+      paired.add(close);
+      j += 2;
+      continue;
+    }
+    j += 1;
+  }
+  let trail = 0;
+  while (trail < text.length && /\s/.test(text[text.length - 1 - trail])) trail++;
+  for (const token of tokens) {
+    if (paired.has(token)) continue;
+    let lead = 0;
+    while (lead <= MAX_MATH_SPAN && /\s/.test(text[token + 2 + lead] ?? "x")) lead++;
+    if (text.length - (token + 2) - lead - trail > MAX_MATH_SPAN) continue;
+    const inner = text.slice(token + 2);
+    const s = inner.trim();
+    if (STRUCTURAL_MARKDOWN.test(s) || s.length > MAX_MATH_SPAN) continue;
+    if (!opensFlowBlock(text, token) && /\n[ \t]*\n/.test(inner)) continue;
+    if ((s.match(PROSE_WORD) ?? []).length >= PROSE_WORD_LIMIT * 3) continue;
+    return token;
+  }
+  return -1;
+}
 function pairDisplayMath(text) {
   const pairs = /* @__PURE__ */ new Map();
   if (!text.includes("$$")) return pairs;
@@ -696,11 +740,15 @@ function isSplitterTreeLine(line) {
 }
 var MEDIA_REF_PREFIXES = ["[Image URL:", "[Video URL:", "[Audio URL:"];
 var Tokenizer = class {
-  constructor(text, streaming = false) {
+  constructor(text, streaming = false, meter = void 0) {
     this.text = text;
     this.streaming = streaming;
+    this.meter = meter;
     this.n = text.length;
+    this.tick(text.length);
     this.mathPairs = pairDisplayMath(text);
+    this.provisionalFrom = streaming ? pendingDisplayMathOpener(text) : -1;
+    if (this.provisionalFrom === -1) this.provisionalFrom = Number.POSITIVE_INFINITY;
     let start = 0;
     for (let i = 0; i < text.length; i++) {
       if (text.charCodeAt(i) === 10) {
@@ -711,13 +759,15 @@ var Tokenizer = class {
       }
     }
     if (start < text.length || text.length === 0) {
+      this.pendingCR = streaming && text.length > start && text.charCodeAt(text.length - 1) === 13;
       this.lineStarts.push(start);
-      this.contentEnds.push(text.length);
+      this.contentEnds.push(this.pendingCR ? text.length - 1 : text.length);
       this.lineEnds.push(text.length);
     }
   }
   text;
   streaming;
+  meter;
   n;
   lineStarts = [];
   contentEnds = [];
@@ -732,6 +782,23 @@ var Tokenizer = class {
   /** `$$` opener → closer for the pairs the core renders as math. */
   mathPairs;
   finder = (needle, from) => this.next(needle, from);
+  /** Streaming: where provisional text starts (a pending `$$`), or +∞. */
+  provisionalFrom = Number.POSITIVE_INFINITY;
+  /** Streaming: the text ends in a lone CR whose LF has not arrived yet. */
+  pendingCR = false;
+  /**
+   * Streaming: an open last line that begins with `{` is undecided — each new
+   * byte can balance or unbalance its braces and flip the whole line between
+   * prose and a JSON island — so it is provisional until it ends (G5).
+   */
+  markOpenJsonTail() {
+    const last = this.lineStarts.length - 1;
+    if (!this.isOpenTail(last)) return;
+    const start = this.lineStarts[last];
+    if (this.lineText(last).trimStart().startsWith("{")) {
+      this.provisionalFrom = Math.min(this.provisionalFrom, start);
+    }
+  }
   /**
    * `text.indexOf(needle, from)` in O(log k): the first query for a needle
    * indexes all of its occurrences once, so repeated searches for a closer
@@ -755,9 +822,21 @@ var Tokenizer = class {
     }
     return lo < list.length ? list[lo] : -1;
   }
+  /** Charge `n` visited characters to the diagnostic meter (complexity tests). */
+  tick(n) {
+    if (this.meter) this.meter.work += n;
+  }
+  /** Streaming only: a paragraph ending at `pe` may still be continued by a line not yet arrived. */
+  paragraphMayGrow(pe) {
+    if (!this.streaming) return false;
+    const rest = this.n - pe;
+    if (rest === 0) return true;
+    const tail = this.text.slice(pe);
+    return tail === "\n" || tail === "\r\n" || tail === "\r";
+  }
   /** Streaming only: the final line, still waiting for its terminator. */
   isOpenTail(k) {
-    return this.streaming && k === this.lineStarts.length - 1 && this.lineEnds[k] === this.contentEnds[k];
+    return this.streaming && k === this.lineStarts.length - 1 && (this.lineEnds[k] === this.contentEnds[k] || this.pendingCR);
   }
   get lineCount() {
     return this.lineStarts.length;
@@ -782,6 +861,7 @@ var Tokenizer = class {
     return this.text.slice(this.lineStarts[li], this.contentEnds[li]);
   }
   run() {
+    if (this.streaming) this.markOpenJsonTail();
     let pos = 0;
     while (pos < this.n) {
       const li = this.lineOf(pos);
@@ -790,6 +870,10 @@ var Tokenizer = class {
         pos = this.gap(pos, li);
         continue;
       }
+      if (pos >= this.provisionalFrom) {
+        this.push({ kind: "prose", start: pos, end: this.n, island: null, inlines: [] });
+        break;
+      }
       const island = this.detectBlock(pos, li, true);
       if (island) {
         this.push({ kind: "island", start: island.start, end: island.end, island, inlines: [] });
@@ -993,6 +1077,7 @@ var Tokenizer = class {
         continue;
       }
       if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
+      if (this.isOpenTail(k)) break;
       last = k;
     }
     return this.island("footnote_def", p, this.contentEnds[last], true, { id: open[1] });
@@ -1154,6 +1239,7 @@ var Tokenizer = class {
     const li = this.lineOf(tp);
     const ce = this.contentEnds[li];
     const tag = readXmlTag(this.text, tp, this.finder);
+    this.tick(tag ? tag.raw.length : 1);
     let openEnd;
     if (tag && !tag.isClosing) {
       if (tag.isSelfClosing) {
@@ -1238,7 +1324,7 @@ var Tokenizer = class {
       return this.island("html_block", p, close.index + close[0].length, true, { tag: name });
     }
     const typeSix = HTML_BLOCK_LEVEL_TAGS.has(name);
-    const typeSeven = atParaStart && t.trimEnd() === tag.raw;
+    const typeSeven = atParaStart && t.trimEnd() === tag.raw && !this.isOpenTail(li);
     if (!typeSix && !typeSeven) return null;
     let end = this.contentEnds[li];
     for (let k = li + 1; k < this.lineCount && !this.lineIsBlank(k); k++) {
@@ -1330,6 +1416,10 @@ var Tokenizer = class {
     let breakLine = -1;
     for (let k = li + 1; k < this.lineCount; k++) {
       if (this.lineIsBlank(k)) break;
+      if (this.lineStarts[k] >= this.provisionalFrom) {
+        end = this.n;
+        break;
+      }
       if (this.detectBlock(this.lineStarts[k], k, false)) {
         breakLine = k;
         break;
@@ -1395,6 +1485,7 @@ var Tokenizer = class {
   scanInline(ps, pe) {
     const text = this.text;
     const inlines = [];
+    this.tick(pe - ps);
     const hasKind = text.slice(ps, pe).includes("__kind");
     let kindBudget = 4 * (pe - ps) + 65536;
     const stop = (island, orphan = false) => ({ inlines, breakIsland: island, orphan });
@@ -1405,6 +1496,7 @@ var Tokenizer = class {
     let runs = null;
     const codeSpanClose = (n, from) => {
       if (!runs) {
+        this.tick(pe - ps);
         runs = /* @__PURE__ */ new Map();
         for (let k = ps; k < pe; ) {
           if (text.charCodeAt(k) !== 96) {
@@ -1431,7 +1523,8 @@ var Tokenizer = class {
       return at !== void 0 && at + n <= pe ? at : -1;
     };
     let i = ps;
-    while (i < pe) {
+    const scanEnd = Math.min(pe, this.provisionalFrom);
+    while (i < scanEnd) {
       const ch = text[i];
       if (ch === "\\") {
         const nextCh = text[i + 1];
@@ -1449,6 +1542,9 @@ var Tokenizer = class {
       if (ch === "`") {
         const n = backtickRunLength(text, i);
         const close = codeSpanClose(n, i + n);
+        if (close === -1 && this.streaming && this.paragraphMayGrow(pe)) {
+          return { inlines, breakIsland: null, orphan: false };
+        }
         i = close === -1 ? i + n : close + n;
         continue;
       }
@@ -1465,6 +1561,7 @@ var Tokenizer = class {
           continue;
         }
         const end = singleDollarMathEnd(text, i, pe);
+        this.tick(end > 0 ? end - i : Math.min(403, pe - i));
         if (end > 0 && !(this.streaming && end === this.n)) {
           inlines.push(this.island("math_inline", i, end, true, { delimiter: "$" }));
           i = end;
@@ -1492,6 +1589,7 @@ var Tokenizer = class {
           const limit = Math.min(pe, i + 65536);
           const end = matchingJsonObjectEnd(text, i, limit);
           kindBudget -= (end ?? limit) - i;
+          this.tick((end ?? limit) - i);
           if (end !== null) {
             const kind = declaredKind(text.slice(i, end));
             if (kind) {
@@ -1553,6 +1651,7 @@ var Tokenizer = class {
           continue;
         }
         const tag = readXmlTag(text, i, this.finder);
+        this.tick(tag ? tag.raw.length : 1);
         if (tag && i + tag.raw.length <= pe) {
           const name = tag.tagName;
           const tagEnd = i + tag.raw.length;
@@ -1601,7 +1700,7 @@ var Tokenizer = class {
   }
 };
 function tokenizeSource(text, options = {}) {
-  const raw = new Tokenizer(text, options.streaming ?? false).run();
+  const raw = new Tokenizer(text, options.streaming ?? false, options.meter).run();
   const cp = buildCodePointIndex(text);
   const inline = (island) => ({
     islandType: island.type,
@@ -1975,6 +2074,7 @@ exports.mapPosition = mapPosition;
 exports.mapRange = mapRange;
 exports.pairDisplayMath = pairDisplayMath;
 exports.parseFenceOpener = parseFenceOpener;
+exports.pendingDisplayMathOpener = pendingDisplayMathOpener;
 exports.readXmlTag = readXmlTag;
 exports.singleDollarMathEnd = singleDollarMathEnd;
 exports.spliceSave = spliceSave;