@ai-matrx/content-ir 0.18.5 → 0.19.0

package/dist/source.js

@@ -268,12 +268,6 @@ function fenceOpenerOf(content) {
   return null;
 }
 var FenceReader = class {
-  constructor(opener, options = {}) {
-    this.opener = opener;
-    this.isJson = opener.char === "`" && opener.lang === "json";
-    this.nesting = opener.char === "`" && (options.nesting ?? fenceNestsInnerFences(opener.lang));
-    this.tildeCloser = opener.char === "~" ? new RegExp(`^ {0,3}~{${opener.ticks},}[ \\t]*$`) : null;
-  }
   opener;
   state = { inString: false, escaped: false };
   depth = 0;
@@ -282,6 +276,12 @@ var FenceReader = class {
   isJson;
   nesting;
   tildeCloser;
+  constructor(opener, options = {}) {
+    this.opener = opener;
+    this.isJson = opener.char === "`" && opener.lang === "json";
+    this.nesting = opener.char === "`" && (options.nesting ?? fenceNestsInnerFences(opener.lang));
+    this.tildeCloser = opener.char === "~" ? new RegExp(`^ {0,3}~{${opener.ticks},}[ \\t]*$`) : null;
+  }
   /** How many nested fences are open right now. */
   get nestedDepth() {
     return this.depth;
@@ -899,9 +899,6 @@ function isEscapedBacktick(line, offset) {
   return slashes % 2 === 1;
 }
 var XmlContainerTracker = class {
-  constructor(rootTag) {
-    this.rootTag = rootTag;
-  }
   rootTag;
   depth = 0;
   inComment = false;
@@ -909,6 +906,9 @@ var XmlContainerTracker = class {
   inlineTicks = 0;
   /** The open fence, read by THE one closer (source/code-ranges.ts). */
   fence = null;
+  constructor(rootTag) {
+    this.rootTag = rootTag;
+  }
   consumeLine(line, startOffset = 0) {
     if (this.fence) {
       if (this.fence.feed(line)) this.fence = null;
@@ -1120,6 +1120,23 @@ function isSplitterTreeLine(line) {
 }
 var MEDIA_REF_PREFIXES = ["[Image URL:", "[Video URL:", "[Audio URL:"];
 var Tokenizer = class {
+  text;
+  streaming;
+  meter;
+  n;
+  lineStarts = [];
+  contentEnds = [];
+  lineEnds = [];
+  blocks = [];
+  blockCache = /* @__PURE__ */ new Map();
+  /** Lazily built: for line i, first line j >= i where brace net from i drops to <= 0. */
+  braceStop = null;
+  braceNet = null;
+  /** Every occurrence offset of each needle searched so far (overlapping, like indexOf). */
+  occurrences = /* @__PURE__ */ new Map();
+  /** `$$` opener → closer for the pairs the core renders as math. */
+  mathPairs;
+  finder = (needle, from) => this.next(needle, from);
   constructor(text, streaming = false, meter = void 0) {
     this.text = text;
     this.streaming = streaming;
@@ -1166,23 +1183,6 @@ var Tokenizer = class {
       this.lineEnds.push(text.length);
     }
   }
-  text;
-  streaming;
-  meter;
-  n;
-  lineStarts = [];
-  contentEnds = [];
-  lineEnds = [];
-  blocks = [];
-  blockCache = /* @__PURE__ */ new Map();
-  /** Lazily built: for line i, first line j >= i where brace net from i drops to <= 0. */
-  braceStop = null;
-  braceNet = null;
-  /** Every occurrence offset of each needle searched so far (overlapping, like indexOf). */
-  occurrences = /* @__PURE__ */ new Map();
-  /** `$$` opener → closer for the pairs the core renders as math. */
-  mathPairs;
-  finder = (needle, from) => this.next(needle, from);
   /** Streaming: where provisional text starts (a pending `$$`), or +∞. */
   provisionalFrom = Number.POSITIVE_INFINITY;
   /** Streaming: the text ends in a lone CR whose LF has not arrived yet. */
@@ -1239,10 +1239,13 @@ var Tokenizer = class {
   /**
    * Streaming: an open block that may still turn back into prose — a `$$`
    * pair whose closer is on the open last line (prose after it makes it
-   * inline), or a JSON head whose braces have not balanced (it may never parse).
+   * inline), a JSON head whose braces have not balanced (it may never parse),
+   * or any opener that is itself still arriving on the open last line.
    */
   mayDissolve(block) {
-    return this.streaming && !block.complete && (block.type === "math_block" || block.type === "json");
+    if (!this.streaming || block.complete) return false;
+    if (block.type === "math_block" || block.type === "json") return true;
+    return this.isOpenTail(this.lineOf(block.start));
   }
   /** Streaming: paragraph ends where an undecided, dissolvable block cut them short. */
   undecidedBreaks = /* @__PURE__ */ new Set();
@@ -1263,6 +1266,12 @@ var Tokenizer = class {
     }
     return lo;
   }
+  /** How many whitespace characters open the text at `from` (up to `to`). */
+  leadingSpace(from, to) {
+    let k = from;
+    while (k < to && /\s/.test(this.text[k])) k++;
+    return k - from;
+  }
   isBlank(from, to) {
     return !NOT_WHITESPACE.test(this.text.slice(from, to));
   }
@@ -1287,6 +1296,10 @@ var Tokenizer = class {
         break;
       }
       const island = this.detectBlock(pos, li, true);
+      if (island && pos + this.leadingSpace(pos, this.contentEnds[li]) >= this.provisionalFrom) {
+        this.push({ kind: "prose", start: pos, end: this.n, island: null, inlines: [] });
+        break;
+      }
       if (island) {
         this.push({ kind: "island", start: island.start, end: island.end, island, inlines: [] });
         pos = island.end;
@@ -1316,12 +1329,6 @@ var Tokenizer = class {
     const last = this.lineStarts.length - 1;
     if (last < 0) return;
     if (this.isOpenTail(last)) {
-      const lineStart = this.lineStarts[last];
-      for (const block2 of this.blocks) {
-        if (block2.kind === "prose" && block2.end > lineStart) {
-          block2.inlines = block2.inlines.filter((inline) => inline.end <= lineStart);
-        }
-      }
       let b = this.blocks.length - 1;
       while (b >= 0 && this.blocks[b].kind === "gap") b--;
       const block = this.blocks[b];
@@ -1347,36 +1354,55 @@ var Tokenizer = class {
    * region…) is never a tree line nor a root label, so it stops the walk.
    */
   treeRootHoldFrom() {
-    const claimable = (k2) => {
+    const claimFrom = (k2) => {
+      const lineStart = this.lineStarts[k2];
+      const lineEnd = this.contentEnds[k2];
+      let pos = lineStart;
       for (; ; ) {
-        const block = this.blockAt(this.lineStarts[k2]);
-        if (!block || block.kind !== "island" || block.island?.fromParagraph === true) return true;
-        const opened = this.lineOf(block.start);
-        if (this.lineStarts[opened] === block.start || opened === k2) return false;
-        k2 = opened;
+        const block = this.blockAt(pos);
+        if (!block || block.kind !== "island" || !block.island) return pos;
+        if (block.island.fromParagraph === true || this.mayDissolve(block.island)) return pos;
+        if (block.start < lineStart) {
+          const opened = this.lineOf(block.start);
+          if (this.lineStarts[opened] !== block.start && claimFrom(opened) !== -1) return pos;
+          if (block.end >= lineEnd) return -1;
+          pos = block.end;
+          continue;
+        }
+        if (block.end >= lineEnd) return -1;
+        pos = block.end;
       }
     };
     let k = this.lineCount - 1;
     let start = this.lineCount;
+    let startPos = this.n;
     if (k >= 0 && this.isOpenTail(k)) {
-      if (!claimable(k)) return this.n;
+      const at = claimFrom(k);
+      if (at === -1) return this.n;
       start = k;
+      startPos = at;
       k--;
     }
-    for (; k >= 0 && claimable(k); k--) {
+    for (; k >= 0; k--) {
       this.tick(1);
+      const at = claimFrom(k);
+      if (at === -1) break;
       if (this.lineIsBlank(k)) continue;
       if (!TREE_GLYPHS.test(this.lineText(k))) break;
       start = k;
+      startPos = at;
     }
-    let root = start;
-    for (let j = start - 1; j >= 0 && claimable(j); j--) {
+    let rootPos = startPos;
+    for (let j = start - 1; j >= 0; j--) {
+      const at = claimFrom(j);
+      if (at === -1) break;
       const trimmed = this.lineText(j).trim();
       this.tick(trimmed.length + 1);
       if (!trimmed || isSplitterTreeLine(trimmed) || /^#{1,6}\s/.test(trimmed)) break;
-      root = j;
+      rootPos = at;
+      if (at > this.lineStarts[j]) break;
     }
-    return root < this.lineCount ? this.lineStarts[root] : this.n;
+    return rootPos;
   }
   /** The block (so far) containing offset `pos`, or null. */
   blockAt(pos) {
@@ -1908,7 +1934,9 @@ var Tokenizer = class {
       const block = this.detectBlock(this.lineStarts[k], k, false);
       if (block) {
         breakLine = k;
-        if (this.mayDissolve(block)) this.undecidedBreaks.add(end);
+        if (this.mayDissolve(block) || this.streaming && block.start >= this.provisionalFrom) {
+          this.undecidedBreaks.add(end);
+        }
         break;
       }
       end = this.contentEnds[k];
@@ -2146,6 +2174,10 @@ var Tokenizer = class {
             const closer = `</${name}>`;
             const close = this.next(closer, tagEnd);
             const blockType = isAttr ? "xml_attr" : "xml_region";
+            if (close === -1 && isThinking && !this.streaming) {
+              i = tagEnd;
+              continue;
+            }
             if (close === -1) return stop(this.island(blockType, i, this.n, false, { tag: name }));
             const end = close + closer.length;
             if (end > pe) return stop(this.spill(blockType, i, end, pe, { tag: name }));
@@ -2249,12 +2281,12 @@ function blockAt(blocks, offset) {
 
 // source/splice.ts
 var SourceSpliceError = class extends Error {
+  code;
   constructor(code, message) {
     super(message);
     this.code = code;
     this.name = "SourceSpliceError";
   }
-  code;
 };
 function blockEdit(block, text) {
   return { start: block.start, end: block.end, text };