@ai-matrx/content-ir 0.18.4 → 0.18.6

package/dist/source.cjs

@@ -323,6 +323,7 @@ var FenceReader = class {
 function closeFence(src, open, opener) {
   const first = readFence(src, open, new FenceReader(opener));
   if (first.closed || !first.reader.sawNested || first.reader.nestedDepth <= 0) return { line: first.line, closed: first.closed };
+  if (src.streaming) return { line: first.line, closed: false };
   const strict = readFence(src, open, new FenceReader(opener, { nesting: false }));
   return { line: strict.line, closed: strict.closed };
 }
@@ -586,6 +587,8 @@ function fenceLineKinds(text) {
 function pendingCodeSpanStart(text) {
   const n = text.length;
   if (n === 0 || /\n[ \t]*\n[ \t]*$/.test(text)) return -1;
+  const lastLine = text.slice(text.lastIndexOf("\n") + 1);
+  if (lastLine && /^(?: {0,3}>[ \t]?)*[ \t]*$/.test(lastLine)) return pendingCodeSpanStart(`${text}x`);
   const s = scan(text);
   const para = s.pendingPara;
   if (s.endsInFence || para === -1) return -1;
@@ -732,13 +735,26 @@ var MATH_SHAPE_CHARS = /^[A-Za-z0-9'\s(),.*\/-]+$/;
 function isSingleDollarMath(content, after) {
   if (!content || content.length > 400) return false;
   if (/^\s/.test(content) || /\s$/.test(content)) return false;
-  if (after !== void 0 && /[0-9]/.test(after)) return false;
+  if (after !== void 0 && /^[0-9]/.test(after)) return false;
+  if (isVariableChain(content, after)) return false;
   if (/(?:^|[^\\])%/.test(content)) return false;
   if (/^[)\]}"/]/.test(content) || /[(\[{":;,/]$/.test(content)) return false;
-  if (/^[0-9]/.test(content) && after !== void 0 && /[A-Za-z]/.test(after)) return false;
+  if (/^[0-9]/.test(content) && after !== void 0 && /^[A-Za-z]/.test(after)) return false;
   if (isMathPlaceholder(content)) return false;
   return TEX_SIGNAL.test(content) || !readsAsProse(content);
 }
+var IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]*$/;
+var MATH_SUFFIX = /^(?:s|es|th|st|nd|rd)$/i;
+function isVariableChain(content, after) {
+  if (/^\{[A-Za-z_][\w.]*\}$/.test(content)) return true;
+  if (!after) return false;
+  if (after[0] === "{") return IDENTIFIER.test(content);
+  const run = /^[A-Za-z0-9_]+/.exec(after)?.[0];
+  if (!run) return false;
+  if (after.length === 1) return false;
+  if (/^[A-Za-z_][A-Za-z0-9_]*[.@-]$/.test(content)) return true;
+  return IDENTIFIER.test(content) && !MATH_SUFFIX.test(run);
+}
 var FUNCTION_NAMES = /* @__PURE__ */ new Set([
   "sin",
   "cos",
@@ -782,6 +798,12 @@ function isMathShape(content) {
   if (/[A-Za-z]'+[A-Za-z]/.test(content) || /(?:^|[\s(,])[A-Za-z]\.(?:[A-Za-z]\.|\s|$)/.test(content)) return false;
   return /[A-Za-z]'*\(|[A-Za-z]'|\b[A-Za-z]\s+[A-Za-z]\b|\([^)]*[A-Za-z][^)]*\)/.test(content);
 }
+var AFTER_WINDOW = 32;
+function singleDollarAfter(text, close) {
+  const rest = text.slice(close + 1, close + 1 + AFTER_WINDOW);
+  return rest.length < AFTER_WINDOW ? `${rest}
+` : rest;
+}
 function singleDollarMathEnd(text, open, limit = text.length) {
   const bound = Math.min(limit, open + 403);
   let close = -1;
@@ -799,7 +821,7 @@ function singleDollarMathEnd(text, open, limit = text.length) {
   }
   if (close === -1 || text[close + 1] === "$") return -1;
   const content = text.slice(open + 1, close);
-  return isSingleDollarMath(content, text[close + 1]) ? close + 1 : -1;
+  return isSingleDollarMath(content, singleDollarAfter(text, close)) ? close + 1 : -1;
 }
 var BRACKET_TEX_SIGNAL = /\\[A-Za-z]+|[\^{}=<>+]|_\{|(?:^|[^A-Za-z0-9])[A-Za-z]_[A-Za-z0-9](?![A-Za-z0-9])/;
 function isEscapedBracketMath(content, before) {
@@ -1170,18 +1192,14 @@ var Tokenizer = class {
   /**
    * Streaming: an open last line that begins with `{` is undecided — each new
    * byte can balance or unbalance its braces and flip the whole line between
-   * prose and a JSON island — and so is a line that is exactly one allowed
-   * HTML tag so far; both are provisional until they end (G5).
+   * prose and a JSON island — so it is provisional until it ends (G5).
+   * (A lone HTML tag on that line is held by {@link holdUndecidedTail}.)
    */
   markOpenJsonTail() {
     const last = this.lineStarts.length - 1;
     if (!this.isOpenTail(last)) return;
-    const start = this.lineStarts[last];
-    const t = this.lineText(last).trim();
-    const oneTag = t.startsWith("<") ? readXmlTag(t, 0) : null;
-    const loneTag = !!oneTag && oneTag.raw === t && ALLOWED_RAW_HTML_TAGS.has(oneTag.tagName.toLowerCase());
-    if (t.startsWith("{") || loneTag) {
-      this.provisionalFrom = Math.min(this.provisionalFrom, start);
+    if (this.lineText(last).trim().startsWith("{")) {
+      this.provisionalFrom = Math.min(this.provisionalFrom, this.lineStarts[last]);
     }
   }
   /**
@@ -1214,11 +1232,25 @@ var Tokenizer = class {
   /** Streaming only: a paragraph ending at `pe` may still be continued by a line not yet arrived. */
   paragraphMayGrow(pe) {
     if (!this.streaming) return false;
+    if (this.undecidedBreaks.has(pe)) return true;
     const rest = this.n - pe;
     if (rest === 0) return true;
     const tail = this.text.slice(pe);
-    return tail === "\n" || tail === "\r\n" || tail === "\r";
+    return /^(?:\r\n|\r|\n)[ \t]*$/.test(tail);
   }
+  /**
+   * Streaming: an open block that may still turn back into prose — a `$$`
+   * pair whose closer is on the open last line (prose after it makes it
+   * inline), a JSON head whose braces have not balanced (it may never parse),
+   * or any opener that is itself still arriving on the open last line.
+   */
+  mayDissolve(block) {
+    if (!this.streaming || block.complete) return false;
+    if (block.type === "math_block" || block.type === "json") return true;
+    return this.isOpenTail(this.lineOf(block.start));
+  }
+  /** Streaming: paragraph ends where an undecided, dissolvable block cut them short. */
+  undecidedBreaks = /* @__PURE__ */ new Set();
   /** Streaming only: the final line, still waiting for its terminator. */
   isOpenTail(k) {
     return this.streaming && k === this.lineStarts.length - 1 && (this.lineEnds[k] === this.contentEnds[k] || this.pendingCR);
@@ -1236,6 +1268,12 @@ var Tokenizer = class {
     }
     return lo;
   }
+  /** How many whitespace characters open the text at `from` (up to `to`). */
+  leadingSpace(from, to) {
+    let k = from;
+    while (k < to && /\s/.test(this.text[k])) k++;
+    return k - from;
+  }
   isBlank(from, to) {
     return !NOT_WHITESPACE.test(this.text.slice(from, to));
   }
@@ -1260,6 +1298,10 @@ var Tokenizer = class {
         break;
       }
       const island = this.detectBlock(pos, li, true);
+      if (island && pos + this.leadingSpace(pos, this.contentEnds[li]) >= this.provisionalFrom) {
+        this.push({ kind: "prose", start: pos, end: this.n, island: null, inlines: [] });
+        break;
+      }
       if (island) {
         this.push({ kind: "island", start: island.start, end: island.end, island, inlines: [] });
         pos = island.end;
@@ -1267,8 +1309,122 @@ var Tokenizer = class {
       }
       pos = this.paragraph(pos, li);
     }
+    if (this.streaming) this.holdUndecidedTail();
     return this.blocks;
   }
+  /**
+   * Streaming: settle nothing the next bytes can still change (verify-RC-B3 F3).
+   *
+   * 1. A block island that ends on the open last line is undecided — the next
+   *    bytes on that line can re-read it or dissolve it (`<!--@a:k1-->` +
+   *    ` more` is an inline anchor, `<br>` + `x` is prose). It runs open to
+   *    the end of the text.
+   * 2. A box-drawing tree still forming at the end of the text claims, as its
+   *    root label, the lines directly above its first connector (the
+   *    renderer's `findTreeBlockStart`), and the open last line itself may
+   *    become one of its connector lines — so inlines on those lines are held
+   *    back as prose until a blank line, a heading or a tree line decides.
+   *    This also holds everything a closer still able to arrive on the open
+   *    last line could change (`$x$` + `5` is currency, G4; `{{v}}` + `}`).
+   */
+  holdUndecidedTail() {
+    const last = this.lineStarts.length - 1;
+    if (last < 0) return;
+    if (this.isOpenTail(last)) {
+      const lineStart = this.lineStarts[last];
+      for (const block2 of this.blocks) {
+        if (block2.kind === "prose" && block2.end > lineStart) {
+          block2.inlines = block2.inlines.filter((inline) => inline.end <= lineStart);
+        }
+      }
+      let b = this.blocks.length - 1;
+      while (b >= 0 && this.blocks[b].kind === "gap") b--;
+      const block = this.blocks[b];
+      if (block?.island?.complete && block.island.end > this.lineStarts[last]) {
+        block.island.complete = false;
+        block.island.end = this.n;
+        block.end = this.n;
+        this.blocks.length = b + 1;
+      }
+    }
+    const from = this.treeRootHoldFrom();
+    for (let b = this.blocks.length - 1; b >= 0; b--) {
+      const block = this.blocks[b];
+      if (block.end <= from) break;
+      if (block.kind === "prose") block.inlines = block.inlines.filter((inline) => inline.end <= from);
+      if (block.island) this.blocks[b] = { kind: "prose", start: block.start, end: block.end, island: null, inlines: [] };
+    }
+  }
+  /**
+   * Streaming: where a tree that may still form at the end of the text would
+   * start claiming lines (`n` when none can). Only paragraph lines can be
+   * claimed — a line that starts inside an island (a fence body, an XML
+   * region…) is never a tree line nor a root label, so it stops the walk.
+   */
+  treeRootHoldFrom() {
+    const claimFrom = (k2) => {
+      const lineStart = this.lineStarts[k2];
+      const lineEnd = this.contentEnds[k2];
+      let pos = lineStart;
+      for (; ; ) {
+        const block = this.blockAt(pos);
+        if (!block || block.kind !== "island" || !block.island) return pos;
+        if (block.island.fromParagraph === true || this.mayDissolve(block.island)) return pos;
+        if (block.start < lineStart) {
+          const opened = this.lineOf(block.start);
+          if (this.lineStarts[opened] !== block.start && claimFrom(opened) !== -1) return pos;
+          if (block.end >= lineEnd) return -1;
+          pos = block.end;
+          continue;
+        }
+        if (block.end >= lineEnd) return -1;
+        pos = block.end;
+      }
+    };
+    let k = this.lineCount - 1;
+    let start = this.lineCount;
+    let startPos = this.n;
+    if (k >= 0 && this.isOpenTail(k)) {
+      const at = claimFrom(k);
+      if (at === -1) return this.n;
+      start = k;
+      startPos = at;
+      k--;
+    }
+    for (; k >= 0; k--) {
+      this.tick(1);
+      const at = claimFrom(k);
+      if (at === -1) break;
+      if (this.lineIsBlank(k)) continue;
+      if (!TREE_GLYPHS.test(this.lineText(k))) break;
+      start = k;
+      startPos = at;
+    }
+    let rootPos = startPos;
+    for (let j = start - 1; j >= 0; j--) {
+      const at = claimFrom(j);
+      if (at === -1) break;
+      const trimmed = this.lineText(j).trim();
+      this.tick(trimmed.length + 1);
+      if (!trimmed || isSplitterTreeLine(trimmed) || /^#{1,6}\s/.test(trimmed)) break;
+      rootPos = at;
+      if (at > this.lineStarts[j]) break;
+    }
+    return rootPos;
+  }
+  /** The block (so far) containing offset `pos`, or null. */
+  blockAt(pos) {
+    let lo = 0;
+    let hi = this.blocks.length - 1;
+    while (lo <= hi) {
+      const mid = lo + hi >>> 1;
+      const block = this.blocks[mid];
+      if (pos < block.start) hi = mid - 1;
+      else if (pos >= block.end) lo = mid + 1;
+      else return block;
+    }
+    return null;
+  }
   push(block) {
     if (block.end <= block.start) return;
     if (block.kind === "prose" && this.isBlank(block.start, block.end)) {
@@ -1472,7 +1628,10 @@ var Tokenizer = class {
         continue;
       }
       if (this.isOpenTail(k)) return this.heldFootnote(p, open[1]);
-      if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
+      if (FOOTNOTE_DEF.test(line.trimStart())) break;
+      const block = this.detectBlock(this.lineStarts[k], k, false);
+      if (block && this.mayDissolve(block)) return this.heldFootnote(p, open[1]);
+      if (block) break;
       last = k;
     }
     const end = this.contentEnds[last];
@@ -1488,7 +1647,8 @@ var Tokenizer = class {
     return {
       count: this.lineCount,
       line: (k) => this.lineText(k),
-      openTail: (k) => this.isOpenTail(k)
+      openTail: (k) => this.isOpenTail(k),
+      streaming: this.streaming
     };
   }
   /**
@@ -1548,6 +1708,10 @@ var Tokenizer = class {
       if (fenceOpenerOf(this.lineText(j))?.char === "~") continue;
       const inner = this.detectBlockUncached(start, j, false);
       if (!inner) continue;
+      if (this.streaming && inner.end >= tildeEnd && this.paragraphMayGrow(inner.end)) {
+        this.overflowComplete = false;
+        return this.n;
+      }
       if (inner.end > end) {
         end = inner.end;
         this.overflowComplete = inner.complete;
@@ -1676,12 +1840,7 @@ var Tokenizer = class {
       return this.island("html_block", p, close.index + close[0].length, true, { tag: name });
     }
     const typeSix = HTML_BLOCK_LEVEL_TAGS.has(name);
-    const loneTag = atParaStart && t.trimEnd() === tag.raw;
-    if (loneTag && this.isOpenTail(li)) {
-      this.provisionalFrom = Math.min(this.provisionalFrom, p);
-      return null;
-    }
-    const typeSeven = loneTag;
+    const typeSeven = atParaStart && t.trimEnd() === tag.raw;
     if (!typeSix && !typeSeven) return null;
     let end = this.contentEnds[li];
     for (let k = li + 1; k < this.lineCount && !this.lineIsBlank(k); k++) {
@@ -1739,6 +1898,13 @@ var Tokenizer = class {
       }
       return null;
     }
+    if (this.isOpenTail(lastLine)) {
+      if (isRecognizedJsonHead(this.text.slice(tp, tp + 512))) {
+        return this.island("json", p, this.n, false, jsonMeta(this.text.slice(tp)));
+      }
+      this.provisionalFrom = Math.min(this.provisionalFrom, p);
+      return null;
+    }
     if (net !== 0) return null;
     const end = this.contentEnds[lastLine];
     const content = this.text.slice(tp, end).trim();
@@ -1773,8 +1939,10 @@ var Tokenizer = class {
     let breakLine = -1;
     for (let k = li + 1; k < this.lineCount; k++) {
       if (this.lineIsBlank(k)) break;
-      if (this.detectBlock(this.lineStarts[k], k, false)) {
+      const block = this.detectBlock(this.lineStarts[k], k, false);
+      if (block) {
         breakLine = k;
+        if (this.mayDissolve(block)) this.undecidedBreaks.add(end);
         break;
       }
       end = this.contentEnds[k];
@@ -1812,7 +1980,7 @@ var Tokenizer = class {
       this.push({ kind: "prose", start: pos, end, island: null, inlines: scan2.inlines });
       return end;
     }
-    const br = scan2.breakIsland;
+    const br = { ...scan2.breakIsland, fromParagraph: true };
     if (scan2.orphan) {
       let start = pos;
       while (this.blocks.length) {
@@ -1835,6 +2003,15 @@ var Tokenizer = class {
     this.push({ kind: "island", start: br.start, end: br.end, island: br, inlines: [] });
     return br.end;
   }
+  /**
+   * An island that opens inside a paragraph and closes past its end becomes a
+   * block. Streaming: while the paragraph may still grow over its closer, the
+   * island may yet be inline — undecided, so it runs open to the end (F3).
+   */
+  spill(type, start, end, pe, meta) {
+    if (this.paragraphMayGrow(pe)) return this.island(type, start, this.n, false, meta);
+    return this.island(type, start, end, true, meta);
+  }
   scanInline(ps, pe) {
     const text = this.text;
     const inlines = [];
@@ -1869,6 +2046,9 @@ var Tokenizer = class {
             i = close + 2;
             continue;
           }
+          if (close === -1 && this.streaming && this.paragraphMayGrow(pe)) {
+            return { inlines, breakIsland: null, orphan: false };
+          }
         }
         i += 2;
         continue;
@@ -1889,17 +2069,13 @@ var Tokenizer = class {
             i += 2;
             continue;
           }
-          if (close + 2 > pe) return stop(this.island("math_block", i, close + 2, true, { delimiter: "$$" }));
+          if (close + 2 > pe) return stop(this.spill("math_block", i, close + 2, pe, { delimiter: "$$" }));
           inlines.push(this.island("math_inline", i, close + 2, true, { delimiter: "$$" }));
           i = close + 2;
           continue;
         }
         const end = singleDollarMathEnd(text, i, pe);
         this.tick(end > 0 ? end - i : Math.min(403, pe - i));
-        if (end > 0 && this.streaming && end === this.n) {
-          this.provisionalFrom = Math.min(this.provisionalFrom, i);
-          return { inlines, breakIsland: null, orphan: false };
-        }
         if (end > 0) {
           inlines.push(this.island("math_inline", i, end, true, { delimiter: "$" }));
           i = end;
@@ -1977,7 +2153,7 @@ var Tokenizer = class {
           const anchor = end - i <= 64 ? PINNED_ANCHOR_RE.exec(text.slice(i, end)) : null;
           const type = anchor ? "anchor" : "html_comment";
           const meta = anchor ? { id: anchor[1] } : {};
-          if (end > pe) return stop(this.island(type, i, end, true, meta));
+          if (end > pe) return stop(this.spill(type, i, end, pe, meta));
           inlines.push(this.island(type, i, end, true, meta));
           i = end;
           continue;
@@ -2004,9 +2180,13 @@ var Tokenizer = class {
             const closer = `</${name}>`;
             const close = this.next(closer, tagEnd);
             const blockType = isAttr ? "xml_attr" : "xml_region";
+            if (close === -1 && isThinking && !this.streaming) {
+              i = tagEnd;
+              continue;
+            }
             if (close === -1) return stop(this.island(blockType, i, this.n, false, { tag: name }));
             const end = close + closer.length;
-            if (end > pe) return stop(this.island(blockType, i, end, true, { tag: name }));
+            if (end > pe) return stop(this.spill(blockType, i, end, pe, { tag: name }));
             inlines.push(this.island("xml_inline", i, end, true, { tag: name }));
             i = end;
             continue;
@@ -2391,6 +2571,7 @@ function mapRange(changes, start, end, unit = "utf16") {
   return { start: newStart, end: newEnd, touched, collapsed: newEnd === newStart && end > start };
 }
 
+exports.AFTER_WINDOW = AFTER_WINDOW;
 exports.BacktickRuns = BacktickRuns;
 exports.CALLOUT_NAMES = CALLOUT_NAMES;
 exports.FENCE_OPENER_WHITESPACE = FENCE_OPENER_WHITESPACE;
@@ -2426,6 +2607,7 @@ exports.pairDisplayMath = pairDisplayMath;
 exports.parseFenceOpener = parseFenceOpener;
 exports.pendingDisplayMathOpener = pendingDisplayMathOpener;
 exports.readXmlTag = readXmlTag;
+exports.singleDollarAfter = singleDollarAfter;
 exports.singleDollarMathEnd = singleDollarMathEnd;
 exports.spliceSave = spliceSave;
 exports.splitsSurrogatePair = splitsSurrogatePair;