@ai-matrx/content-ir 0.16.1 → 0.16.2

package/dist/source.d.cts

@@ -421,20 +421,11 @@ declare function parseFenceOpener(line: string): {
 declare function classifyInnerFenceLine(trimmed: string, openTicks: number, nests: boolean, nestedDepth: number): InnerFenceLine;
 
 /**
- * `$$` pairing — the core's rule for which `$$` delimiters are real math.
- *
- * Ported from `@ai-matrx/kit/delimiter-guard` (`guardMathDelimiters`), which
- * every math-capable renderer runs before remark-math. Its decisions:
- *   - `$$` tokens outside code (``` / ~~~ fences, `inline code`) pair in
- *     document order, open → close;
- *   - a pair whose inside reads as prose or carries markdown structure is NOT
- *     math: the renderer neutralizes its opener and the closer is retried as
- *     the next opener;
- *   - an unpaired `$$` is inert — it renders as literal text.
- * So the tokenizer protects exactly the pairs the renderer shows as math, and
- * a lone `$$` (a stray closer, or an opener that never closed) never locks
- * anything. Linear time: one regex pass per code shape, one sweep.
+ * `$$...$$`, `$$…$$`, `$$ · $$`: a PLACEHOLDER — prose describing the syntax,
+ * never a formula. The pair is consumed whole as literal text (neither `$$`
+ * may open the next span).
  */
+declare function isMathPlaceholder(inner: string): boolean;
 /**
  * The guard's `looksLikeMath`. `flow` = the opener is alone on its line: a
  * flow block may span blank lines (like a fence); a `$$` with text beside it
@@ -453,4 +444,4 @@ declare function looksLikeDisplayMath(inner: string, flow?: boolean): boolean;
 declare function pendingDisplayMathOpener(text: string): number;
 declare function pairDisplayMath(text: string): Map<number, number>;
 
-export { type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
+export { type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isMathPlaceholder, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };

package/dist/source.d.ts

@@ -421,20 +421,11 @@ declare function parseFenceOpener(line: string): {
 declare function classifyInnerFenceLine(trimmed: string, openTicks: number, nests: boolean, nestedDepth: number): InnerFenceLine;
 
 /**
- * `$$` pairing — the core's rule for which `$$` delimiters are real math.
- *
- * Ported from `@ai-matrx/kit/delimiter-guard` (`guardMathDelimiters`), which
- * every math-capable renderer runs before remark-math. Its decisions:
- *   - `$$` tokens outside code (``` / ~~~ fences, `inline code`) pair in
- *     document order, open → close;
- *   - a pair whose inside reads as prose or carries markdown structure is NOT
- *     math: the renderer neutralizes its opener and the closer is retried as
- *     the next opener;
- *   - an unpaired `$$` is inert — it renders as literal text.
- * So the tokenizer protects exactly the pairs the renderer shows as math, and
- * a lone `$$` (a stray closer, or an opener that never closed) never locks
- * anything. Linear time: one regex pass per code shape, one sweep.
+ * `$$...$$`, `$$…$$`, `$$ · $$`: a PLACEHOLDER — prose describing the syntax,
+ * never a formula. The pair is consumed whole as literal text (neither `$$`
+ * may open the next span).
  */
+declare function isMathPlaceholder(inner: string): boolean;
 /**
  * The guard's `looksLikeMath`. `flow` = the opener is alone on its line: a
  * flow block may span blank lines (like a fence); a `$$` with text beside it
@@ -453,4 +444,4 @@ declare function looksLikeDisplayMath(inner: string, flow?: boolean): boolean;
 declare function pendingDisplayMathOpener(text: string): number;
 declare function pairDisplayMath(text: string): Map<number, number>;
 
-export { type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
+export { type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isMathPlaceholder, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };

package/dist/source.js

@@ -319,6 +319,9 @@ var STRUCTURAL_MARKDOWN = /\]\(|https?:\/\/|\*\*|(?:^|\n)[ \t]{0,3}#{1,6}[ \t]|(
 var LATEX_COMMAND = /\\[a-zA-Z]/;
 var PROSE_WORD = /[A-Za-z]{3,}/g;
 var PROSE_WORD_LIMIT = 6;
+function isMathPlaceholder(inner) {
+  return /^[\s.…·⋯_-]*$/.test(inner) && /[.…·⋯]/.test(inner);
+}
 function opensFlowBlock(text, open) {
   for (let k = open - 1; k >= 0 && text[k] !== "\n"; k--) if (text[k] !== " " && text[k] !== "	") return false;
   for (let k = open + 2; k < text.length && text[k] !== "\n"; k++) if (text[k] !== " " && text[k] !== "	" && text[k] !== "\r") return false;
@@ -337,13 +340,84 @@ function looksLikeDisplayMath(inner, flow = false) {
 }
 function codeRanges(text) {
   const ranges = [];
-  for (const re of [/```[\s\S]*?(?:```|$)/g, /~~~[\s\S]*?(?:~~~|$)/g, /`[^`\n]*`/g]) {
-    let m;
-    while ((m = re.exec(text)) !== null) {
-      ranges.push([m.index, m.index + m[0].length]);
-      if (m[0].length === 0) re.lastIndex++;
+  const n = text.length;
+  let fence = null;
+  let para = -1;
+  const spans = (from, to) => {
+    const runs = [];
+    for (let k = from; k < to; ) {
+      if (text.charCodeAt(k) !== 96) {
+        k++;
+        continue;
+      }
+      let len = 1;
+      while (k + len < to && text.charCodeAt(k + len) === 96) len++;
+      runs.push([k, len]);
+      k += len;
+    }
+    const nextSame = new Array(runs.length).fill(-1);
+    const last = /* @__PURE__ */ new Map();
+    for (let r = runs.length - 1; r >= 0; r--) {
+      const len = runs[r][1];
+      nextSame[r] = last.get(len) ?? -1;
+      last.set(len, r);
+    }
+    for (let r = 0; r < runs.length; ) {
+      const j = nextSame[r];
+      if (j === -1) {
+        r++;
+        continue;
+      }
+      const [s0] = runs[r];
+      const [s1, l1] = runs[j];
+      ranges.push([s0, s1 + l1]);
+      r = j + 1;
     }
-  }
+  };
+  let pos = 0;
+  while (pos <= n) {
+    const nl = text.indexOf("\n", pos);
+    const lineEnd = nl === -1 ? n : nl;
+    const line = text.slice(pos, lineEnd);
+    let indent = 0;
+    while (indent < line.length && line[indent] === " ") indent++;
+    const body = line.slice(indent);
+    if (fence) {
+      const trimmed = trimFenceLine(line);
+      let closes = false;
+      if (fence.char === "`") {
+        const kind = classifyInnerFenceLine(trimmed, fence.len, fence.nests, fence.depth);
+        if (kind === "open-nested") fence.depth++;
+        else if (kind === "close-nested") fence.depth--;
+        else if (kind === "close-outer") closes = indent <= 3;
+      } else {
+        let t = 0;
+        while (t < trimmed.length && trimmed[t] === "~") t++;
+        closes = indent <= 3 && t >= fence.len && trimFenceLine(trimmed.slice(t)) === "";
+      }
+      if (closes) {
+        ranges.push([fence.start, lineEnd]);
+        fence = null;
+      }
+    } else {
+      const opener = indent <= 3 ? /^(`{3,}|~{3,})(.*)$/.exec(body) : null;
+      const info = opener ? opener[2] : "";
+      if (opener && !(opener[1][0] === "`" && info.includes("`"))) {
+        if (para !== -1) spans(para, pos);
+        para = -1;
+        const char = opener[1][0];
+        const parsed = char === "`" ? parseFenceOpener(line) : null;
+        fence = { char, len: opener[1].length, start: pos, nests: fenceNestsInnerFences(parsed?.lang), depth: 0 };
+      } else if (body.trim() === "") {
+        if (para !== -1) spans(para, pos);
+        para = -1;
+      } else if (para === -1) para = pos;
+    }
+    if (nl === -1) break;
+    pos = nl + 1;
+  }
+  if (fence) ranges.push([fence.start, n]);
+  else if (para !== -1) spans(para, n);
   return ranges.sort((a, b) => a[0] - b[0]);
 }
 function pendingDisplayMathOpener(text) {
@@ -366,6 +440,10 @@ function pendingDisplayMathOpener(text) {
     const open = tokens[j];
     const close = tokens[j + 1];
     if (close === void 0) break;
+    if (isMathPlaceholder(text.slice(open + 2, close))) {
+      j += 2;
+      continue;
+    }
     if (looksLikeDisplayMath(text.slice(open + 2, close), opensFlowBlock(text, open))) {
       paired.add(open);
       paired.add(close);
@@ -410,6 +488,10 @@ function pairDisplayMath(text) {
     const open = tokens[j];
     const close = tokens[j + 1];
     if (close === void 0) break;
+    if (isMathPlaceholder(text.slice(open + 2, close))) {
+      j += 2;
+      continue;
+    }
     if (looksLikeDisplayMath(text.slice(open + 2, close), opensFlowBlock(text, open))) {
       pairs.set(open, close);
       j += 2;
@@ -747,6 +829,19 @@ var Tokenizer = class {
     this.mathPairs = pairDisplayMath(text);
     this.provisionalFrom = streaming ? pendingDisplayMathOpener(text) : -1;
     if (this.provisionalFrom === -1) this.provisionalFrom = Number.POSITIVE_INFINITY;
+    if (streaming && text.includes("$$")) {
+      for (const pad of ["`", "``", "```", "~~~"]) {
+        const alt = pairDisplayMath(text + pad);
+        for (const [open, close] of this.mathPairs) {
+          if (alt.get(open) !== close) this.provisionalFrom = Math.min(this.provisionalFrom, open);
+        }
+        for (const [open, close] of alt) {
+          if (close < text.length && this.mathPairs.get(open) !== close) {
+            this.provisionalFrom = Math.min(this.provisionalFrom, open);
+          }
+        }
+      }
+    }
     let start = 0;
     for (let i = 0; i < text.length; i++) {
       if (text.charCodeAt(i) === 10) {
@@ -787,13 +882,17 @@ var Tokenizer = class {
   /**
    * Streaming: an open last line that begins with `{` is undecided — each new
    * byte can balance or unbalance its braces and flip the whole line between
-   * prose and a JSON island — so it is provisional until it ends (G5).
+   * prose and a JSON island — and so is a line that is exactly one allowed
+   * HTML tag so far; both are provisional until they end (G5).
    */
   markOpenJsonTail() {
     const last = this.lineStarts.length - 1;
     if (!this.isOpenTail(last)) return;
     const start = this.lineStarts[last];
-    if (this.lineText(last).trimStart().startsWith("{")) {
+    const t = this.lineText(last).trim();
+    const oneTag = t.startsWith("<") ? readXmlTag(t, 0) : null;
+    const loneTag = !!oneTag && oneTag.raw === t && ALLOWED_RAW_HTML_TAGS.has(oneTag.tagName.toLowerCase());
+    if (t.startsWith("{") || loneTag) {
       this.provisionalFrom = Math.min(this.provisionalFrom, start);
     }
   }
@@ -1087,10 +1186,16 @@ var Tokenizer = class {
         continue;
       }
       if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
-      if (this.isOpenTail(k)) break;
+      if (this.isOpenTail(k)) return this.heldFootnote(p, open[1]);
       last = k;
     }
-    return this.island("footnote_def", p, this.contentEnds[last], true, { id: open[1] });
+    const end = this.contentEnds[last];
+    if (this.paragraphMayGrow(end)) return this.heldFootnote(p, open[1]);
+    return this.island("footnote_def", p, end, true, { id: open[1] });
+  }
+  /** A footnote whose continuation is still arriving: open to the end of the text. */
+  heldFootnote(p, id) {
+    return this.island("footnote_def", p, this.n, false, { id });
   }
   /**
    * Backtick fence — the renderer splitter's close rules: JSON-string aware for
@@ -1334,7 +1439,12 @@ var Tokenizer = class {
       return this.island("html_block", p, close.index + close[0].length, true, { tag: name });
     }
     const typeSix = HTML_BLOCK_LEVEL_TAGS.has(name);
-    const typeSeven = atParaStart && t.trimEnd() === tag.raw && !this.isOpenTail(li);
+    const loneTag = atParaStart && t.trimEnd() === tag.raw;
+    if (loneTag && this.isOpenTail(li)) {
+      this.provisionalFrom = Math.min(this.provisionalFrom, p);
+      return null;
+    }
+    const typeSeven = loneTag;
     if (!typeSix && !typeSeven) return null;
     let end = this.contentEnds[li];
     for (let k = li + 1; k < this.lineCount && !this.lineIsBlank(k); k++) {
@@ -1426,10 +1536,6 @@ var Tokenizer = class {
     let breakLine = -1;
     for (let k = li + 1; k < this.lineCount; k++) {
       if (this.lineIsBlank(k)) break;
-      if (this.lineStarts[k] >= this.provisionalFrom) {
-        end = this.n;
-        break;
-      }
       if (this.detectBlock(this.lineStarts[k], k, false)) {
         breakLine = k;
         break;
@@ -1572,7 +1678,11 @@ var Tokenizer = class {
         }
         const end = singleDollarMathEnd(text, i, pe);
         this.tick(end > 0 ? end - i : Math.min(403, pe - i));
-        if (end > 0 && !(this.streaming && end === this.n)) {
+        if (end > 0 && this.streaming && end === this.n) {
+          this.provisionalFrom = Math.min(this.provisionalFrom, i);
+          return { inlines, breakIsland: null, orphan: false };
+        }
+        if (end > 0) {
           inlines.push(this.island("math_inline", i, end, true, { delimiter: "$" }));
           i = end;
           continue;
@@ -2063,6 +2173,6 @@ function mapRange(changes, start, end, unit = "utf16") {
   return { start: newStart, end: newEnd, touched, collapsed: newEnd === newStart && end > start };
 }
 
-export { CALLOUT_NAMES, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, NESTING_FENCE_LANGUAGES, SourceSpliceError, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
+export { CALLOUT_NAMES, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, NESTING_FENCE_LANGUAGES, SourceSpliceError, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isMathPlaceholder, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
 //# sourceMappingURL=source.js.map
 //# sourceMappingURL=source.js.map