@ai-matrx/content-ir 0.15.1 → 0.16.0

package/dist/source.d.cts

@@ -118,6 +118,16 @@ interface TokenizeOptions {
      * Omit for stored text: a complete document's last line is final.
      */
     readonly streaming?: boolean;
+    /**
+     * DIAGNOSTIC — complexity tests only. When given, every loop that walks the
+     * text itself adds the characters it visits to `meter.work`, so a test can
+     * assert the tokenizer's work grows linearly with its input (deterministic,
+     * unlike wall-clock timing). Loops over a slice are not charged here: the
+     * slice's own cost covers them. Omit in production.
+     */
+    readonly meter?: {
+        work: number;
+    };
 }
 declare function tokenizeSource(text: string, options?: TokenizeOptions): SourceBlock[];
 /** The exact source the blocks were cut from. */
@@ -433,6 +443,14 @@ declare function classifyInnerFenceLine(trimmed: string, openTicks: number, nest
  */
 declare function looksLikeDisplayMath(inner: string, flow?: boolean): boolean;
 /** Map of `$$` opener offset → closer offset for every pair the renderer shows as math. */
+/**
+ * STREAMING: the last `$$` token that is still unpaired AND could still pair
+ * with a `$$` not yet arrived — its span so far is short enough, carries no
+ * markdown structure, not too many prose words, and (unless it opens a flow
+ * block) no blank line. Everything after it may yet become math, so a
+ * streaming tokenizer treats it as provisional. -1 when there is none.
+ */
+declare function pendingDisplayMathOpener(text: string): number;
 declare function pairDisplayMath(text: string): Map<number, number>;
 
-export { type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
+export { type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };

package/dist/source.d.ts

@@ -118,6 +118,16 @@ interface TokenizeOptions {
      * Omit for stored text: a complete document's last line is final.
      */
     readonly streaming?: boolean;
+    /**
+     * DIAGNOSTIC — complexity tests only. When given, every loop that walks the
+     * text itself adds the characters it visits to `meter.work`, so a test can
+     * assert the tokenizer's work grows linearly with its input (deterministic,
+     * unlike wall-clock timing). Loops over a slice are not charged here: the
+     * slice's own cost covers them. Omit in production.
+     */
+    readonly meter?: {
+        work: number;
+    };
 }
 declare function tokenizeSource(text: string, options?: TokenizeOptions): SourceBlock[];
 /** The exact source the blocks were cut from. */
@@ -433,6 +443,14 @@ declare function classifyInnerFenceLine(trimmed: string, openTicks: number, nest
  */
 declare function looksLikeDisplayMath(inner: string, flow?: boolean): boolean;
 /** Map of `$$` opener offset → closer offset for every pair the renderer shows as math. */
+/**
+ * STREAMING: the last `$$` token that is still unpaired AND could still pair
+ * with a `$$` not yet arrived — its span so far is short enough, carries no
+ * markdown structure, not too many prose words, and (unless it opens a flow
+ * block) no blank line. Everything after it may yet become math, so a
+ * streaming tokenizer treats it as provisional. -1 when there is none.
+ */
+declare function pendingDisplayMathOpener(text: string): number;
 declare function pairDisplayMath(text: string): Map<number, number>;
 
-export { type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
+export { type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };

package/dist/source.js

@@ -346,6 +346,50 @@ function codeRanges(text) {
   }
   return ranges.sort((a, b) => a[0] - b[0]);
 }
+function pendingDisplayMathOpener(text) {
+  if (!text.includes("$$")) return -1;
+  const ranges = codeRanges(text);
+  const tokens = [];
+  let r = 0;
+  let maxEnd = -1;
+  for (let i = text.indexOf("$$"); i !== -1 && i < text.length - 1; i = text.indexOf("$$", i)) {
+    while (r < ranges.length && ranges[r][0] <= i) {
+      maxEnd = Math.max(maxEnd, ranges[r][1]);
+      r++;
+    }
+    if (maxEnd <= i) tokens.push(i);
+    i += 2;
+  }
+  const paired = /* @__PURE__ */ new Set();
+  let j = 0;
+  while (j < tokens.length) {
+    const open = tokens[j];
+    const close = tokens[j + 1];
+    if (close === void 0) break;
+    if (looksLikeDisplayMath(text.slice(open + 2, close), opensFlowBlock(text, open))) {
+      paired.add(open);
+      paired.add(close);
+      j += 2;
+      continue;
+    }
+    j += 1;
+  }
+  let trail = 0;
+  while (trail < text.length && /\s/.test(text[text.length - 1 - trail])) trail++;
+  for (const token of tokens) {
+    if (paired.has(token)) continue;
+    let lead = 0;
+    while (lead <= MAX_MATH_SPAN && /\s/.test(text[token + 2 + lead] ?? "x")) lead++;
+    if (text.length - (token + 2) - lead - trail > MAX_MATH_SPAN) continue;
+    const inner = text.slice(token + 2);
+    const s = inner.trim();
+    if (STRUCTURAL_MARKDOWN.test(s) || s.length > MAX_MATH_SPAN) continue;
+    if (!opensFlowBlock(text, token) && /\n[ \t]*\n/.test(inner)) continue;
+    if ((s.match(PROSE_WORD) ?? []).length >= PROSE_WORD_LIMIT * 3) continue;
+    return token;
+  }
+  return -1;
+}
 function pairDisplayMath(text) {
   const pairs = /* @__PURE__ */ new Map();
   if (!text.includes("$$")) return pairs;
@@ -694,11 +738,15 @@ function isSplitterTreeLine(line) {
 }
 var MEDIA_REF_PREFIXES = ["[Image URL:", "[Video URL:", "[Audio URL:"];
 var Tokenizer = class {
-  constructor(text, streaming = false) {
+  constructor(text, streaming = false, meter = void 0) {
     this.text = text;
     this.streaming = streaming;
+    this.meter = meter;
     this.n = text.length;
+    this.tick(text.length);
     this.mathPairs = pairDisplayMath(text);
+    this.provisionalFrom = streaming ? pendingDisplayMathOpener(text) : -1;
+    if (this.provisionalFrom === -1) this.provisionalFrom = Number.POSITIVE_INFINITY;
     let start = 0;
     for (let i = 0; i < text.length; i++) {
       if (text.charCodeAt(i) === 10) {
@@ -709,13 +757,15 @@ var Tokenizer = class {
       }
     }
     if (start < text.length || text.length === 0) {
+      this.pendingCR = streaming && text.length > start && text.charCodeAt(text.length - 1) === 13;
       this.lineStarts.push(start);
-      this.contentEnds.push(text.length);
+      this.contentEnds.push(this.pendingCR ? text.length - 1 : text.length);
       this.lineEnds.push(text.length);
     }
   }
   text;
   streaming;
+  meter;
   n;
   lineStarts = [];
   contentEnds = [];
@@ -730,6 +780,23 @@ var Tokenizer = class {
   /** `$$` opener → closer for the pairs the core renders as math. */
   mathPairs;
   finder = (needle, from) => this.next(needle, from);
+  /** Streaming: where provisional text starts (a pending `$$`), or +∞. */
+  provisionalFrom = Number.POSITIVE_INFINITY;
+  /** Streaming: the text ends in a lone CR whose LF has not arrived yet. */
+  pendingCR = false;
+  /**
+   * Streaming: an open last line that begins with `{` is undecided — each new
+   * byte can balance or unbalance its braces and flip the whole line between
+   * prose and a JSON island — so it is provisional until it ends (G5).
+   */
+  markOpenJsonTail() {
+    const last = this.lineStarts.length - 1;
+    if (!this.isOpenTail(last)) return;
+    const start = this.lineStarts[last];
+    if (this.lineText(last).trimStart().startsWith("{")) {
+      this.provisionalFrom = Math.min(this.provisionalFrom, start);
+    }
+  }
   /**
    * `text.indexOf(needle, from)` in O(log k): the first query for a needle
    * indexes all of its occurrences once, so repeated searches for a closer
@@ -753,9 +820,21 @@ var Tokenizer = class {
     }
     return lo < list.length ? list[lo] : -1;
   }
+  /** Charge `n` visited characters to the diagnostic meter (complexity tests). */
+  tick(n) {
+    if (this.meter) this.meter.work += n;
+  }
+  /** Streaming only: a paragraph ending at `pe` may still be continued by a line not yet arrived. */
+  paragraphMayGrow(pe) {
+    if (!this.streaming) return false;
+    const rest = this.n - pe;
+    if (rest === 0) return true;
+    const tail = this.text.slice(pe);
+    return tail === "\n" || tail === "\r\n" || tail === "\r";
+  }
   /** Streaming only: the final line, still waiting for its terminator. */
   isOpenTail(k) {
-    return this.streaming && k === this.lineStarts.length - 1 && this.lineEnds[k] === this.contentEnds[k];
+    return this.streaming && k === this.lineStarts.length - 1 && (this.lineEnds[k] === this.contentEnds[k] || this.pendingCR);
   }
   get lineCount() {
     return this.lineStarts.length;
@@ -780,6 +859,7 @@ var Tokenizer = class {
     return this.text.slice(this.lineStarts[li], this.contentEnds[li]);
   }
   run() {
+    if (this.streaming) this.markOpenJsonTail();
     let pos = 0;
     while (pos < this.n) {
       const li = this.lineOf(pos);
@@ -788,6 +868,10 @@ var Tokenizer = class {
         pos = this.gap(pos, li);
         continue;
       }
+      if (pos >= this.provisionalFrom) {
+        this.push({ kind: "prose", start: pos, end: this.n, island: null, inlines: [] });
+        break;
+      }
       const island = this.detectBlock(pos, li, true);
       if (island) {
         this.push({ kind: "island", start: island.start, end: island.end, island, inlines: [] });
@@ -991,6 +1075,7 @@ var Tokenizer = class {
         continue;
       }
       if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
+      if (this.isOpenTail(k)) break;
       last = k;
     }
     return this.island("footnote_def", p, this.contentEnds[last], true, { id: open[1] });
@@ -1152,6 +1237,7 @@ var Tokenizer = class {
     const li = this.lineOf(tp);
     const ce = this.contentEnds[li];
     const tag = readXmlTag(this.text, tp, this.finder);
+    this.tick(tag ? tag.raw.length : 1);
     let openEnd;
     if (tag && !tag.isClosing) {
       if (tag.isSelfClosing) {
@@ -1236,7 +1322,7 @@ var Tokenizer = class {
       return this.island("html_block", p, close.index + close[0].length, true, { tag: name });
     }
     const typeSix = HTML_BLOCK_LEVEL_TAGS.has(name);
-    const typeSeven = atParaStart && t.trimEnd() === tag.raw;
+    const typeSeven = atParaStart && t.trimEnd() === tag.raw && !this.isOpenTail(li);
     if (!typeSix && !typeSeven) return null;
     let end = this.contentEnds[li];
     for (let k = li + 1; k < this.lineCount && !this.lineIsBlank(k); k++) {
@@ -1328,6 +1414,10 @@ var Tokenizer = class {
     let breakLine = -1;
     for (let k = li + 1; k < this.lineCount; k++) {
       if (this.lineIsBlank(k)) break;
+      if (this.lineStarts[k] >= this.provisionalFrom) {
+        end = this.n;
+        break;
+      }
       if (this.detectBlock(this.lineStarts[k], k, false)) {
         breakLine = k;
         break;
@@ -1393,6 +1483,7 @@ var Tokenizer = class {
   scanInline(ps, pe) {
     const text = this.text;
     const inlines = [];
+    this.tick(pe - ps);
     const hasKind = text.slice(ps, pe).includes("__kind");
     let kindBudget = 4 * (pe - ps) + 65536;
     const stop = (island, orphan = false) => ({ inlines, breakIsland: island, orphan });
@@ -1403,6 +1494,7 @@ var Tokenizer = class {
     let runs = null;
     const codeSpanClose = (n, from) => {
       if (!runs) {
+        this.tick(pe - ps);
         runs = /* @__PURE__ */ new Map();
         for (let k = ps; k < pe; ) {
           if (text.charCodeAt(k) !== 96) {
@@ -1429,7 +1521,8 @@ var Tokenizer = class {
       return at !== void 0 && at + n <= pe ? at : -1;
     };
     let i = ps;
-    while (i < pe) {
+    const scanEnd = Math.min(pe, this.provisionalFrom);
+    while (i < scanEnd) {
       const ch = text[i];
       if (ch === "\\") {
         const nextCh = text[i + 1];
@@ -1447,6 +1540,9 @@ var Tokenizer = class {
       if (ch === "`") {
         const n = backtickRunLength(text, i);
         const close = codeSpanClose(n, i + n);
+        if (close === -1 && this.streaming && this.paragraphMayGrow(pe)) {
+          return { inlines, breakIsland: null, orphan: false };
+        }
         i = close === -1 ? i + n : close + n;
         continue;
       }
@@ -1463,6 +1559,7 @@ var Tokenizer = class {
           continue;
         }
         const end = singleDollarMathEnd(text, i, pe);
+        this.tick(end > 0 ? end - i : Math.min(403, pe - i));
         if (end > 0 && !(this.streaming && end === this.n)) {
           inlines.push(this.island("math_inline", i, end, true, { delimiter: "$" }));
           i = end;
@@ -1490,6 +1587,7 @@ var Tokenizer = class {
           const limit = Math.min(pe, i + 65536);
           const end = matchingJsonObjectEnd(text, i, limit);
           kindBudget -= (end ?? limit) - i;
+          this.tick((end ?? limit) - i);
           if (end !== null) {
             const kind = declaredKind(text.slice(i, end));
             if (kind) {
@@ -1551,6 +1649,7 @@ var Tokenizer = class {
           continue;
         }
         const tag = readXmlTag(text, i, this.finder);
+        this.tick(tag ? tag.raw.length : 1);
         if (tag && i + tag.raw.length <= pe) {
           const name = tag.tagName;
           const tagEnd = i + tag.raw.length;
@@ -1599,7 +1698,7 @@ var Tokenizer = class {
   }
 };
 function tokenizeSource(text, options = {}) {
-  const raw = new Tokenizer(text, options.streaming ?? false).run();
+  const raw = new Tokenizer(text, options.streaming ?? false, options.meter).run();
   const cp = buildCodePointIndex(text);
   const inline = (island) => ({
     islandType: island.type,
@@ -1952,6 +2051,6 @@ function mapRange(changes, start, end, unit = "utf16") {
   return { start: newStart, end: newEnd, touched, collapsed: newEnd === newStart && end > start };
 }
 
-export { CALLOUT_NAMES, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, NESTING_FENCE_LANGUAGES, SourceSpliceError, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
+export { CALLOUT_NAMES, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, NESTING_FENCE_LANGUAGES, SourceSpliceError, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
 //# sourceMappingURL=source.js.map
 //# sourceMappingURL=source.js.map