@ai-matrx/content-ir 0.15.1 → 0.16.0

package/CHANGELOG.md

@@ -1,5 +1,40 @@
 # Changelog
 
+## 0.16.0 — 2026-09-25
+
+**`/source` — a deterministic attack suite, and streaming that only ever
+grows** (verify-RC-B3 § Re-verification (0.15.0): G3, G4, G5). The verifier's
+planted mutations M1–M6 plus M7 (the trailing-`$` streaming guard removed)
+and M8 (the footnote open-tail rule removed) each turn
+`__tests__/source-attack.test.ts` red ON ITS OWN; unmutated code is green.
+
+- **G3 — work, not wall-clock.** `tokenizeSource(text, { meter })` (diagnostic)
+  counts every character the tokenizer's own loops visit; the attack suite adds
+  native string / regex work (indexOf / includes / lastIndexOf spans, trim,
+  split, replace, regex exec / test to where they stop) under a V8 cost model,
+  and asserts work at 4× input stays under 6× — deterministic on any host. One
+  generous wall-clock smoke bound remains (1 MB mixed in < 20 s). The meter
+  immediately caught two real quadratics, both fixed: the new pending-`$$`
+  scan (206 s at 1 MB of `$$\n`, now linear) — and it replaced a timing
+  check that flaked 3 of 12 runs under load.
+- **G4** — a streamed `$x$` that the next byte turns into currency (`$x$5`)
+  never settles as math first; guarded by a message in the suite.
+- **G5 — trailing islands only grow.** The suite asserts that a line-grown
+  island (tree, callout, footnote, HTML block, directive) keeps its start and
+  type and never shrinks, at every next prefix, over random documents. Fixed
+  classes it found: a footnote absorbing an unterminated line that becomes
+  the next note (`[^n]: short\n[`); a one-tag HTML line still arriving; a
+  trailing lone CR (the first half of CRLF) counted as content; text after an
+  unclosed backtick run while its paragraph can still grow; an open last line
+  starting with `{`; and anything after a `$$` that could still find its
+  partner (`pendingDisplayMathOpener` — the earliest pairable unpaired token).
+  In streaming mode all of these are provisional prose until decided.
+- Distinct-document counting: the structure attack counts the distinct
+  documents it tokenized (mulberry32) and requires ≥ 95% of its rounds.
+
+**Consumer action:** none for stored text. Streaming callers see fewer
+islands near the tail until the text decides them.
+
 ## 0.15.1
 
 - **Fixed (`/source` `pairDisplayMath`):** a stray `$$` in prose paired with

package/dist/index.cjs

@@ -5321,6 +5321,50 @@ function codeRanges(text) {
   }
   return ranges.sort((a, b) => a[0] - b[0]);
 }
+function pendingDisplayMathOpener(text) {
+  if (!text.includes("$$")) return -1;
+  const ranges = codeRanges(text);
+  const tokens = [];
+  let r = 0;
+  let maxEnd = -1;
+  for (let i = text.indexOf("$$"); i !== -1 && i < text.length - 1; i = text.indexOf("$$", i)) {
+    while (r < ranges.length && ranges[r][0] <= i) {
+      maxEnd = Math.max(maxEnd, ranges[r][1]);
+      r++;
+    }
+    if (maxEnd <= i) tokens.push(i);
+    i += 2;
+  }
+  const paired = /* @__PURE__ */ new Set();
+  let j = 0;
+  while (j < tokens.length) {
+    const open = tokens[j];
+    const close = tokens[j + 1];
+    if (close === void 0) break;
+    if (looksLikeDisplayMath(text.slice(open + 2, close), opensFlowBlock(text, open))) {
+      paired.add(open);
+      paired.add(close);
+      j += 2;
+      continue;
+    }
+    j += 1;
+  }
+  let trail = 0;
+  while (trail < text.length && /\s/.test(text[text.length - 1 - trail])) trail++;
+  for (const token of tokens) {
+    if (paired.has(token)) continue;
+    let lead = 0;
+    while (lead <= MAX_MATH_SPAN && /\s/.test(text[token + 2 + lead] ?? "x")) lead++;
+    if (text.length - (token + 2) - lead - trail > MAX_MATH_SPAN) continue;
+    const inner = text.slice(token + 2);
+    const s = inner.trim();
+    if (STRUCTURAL_MARKDOWN.test(s) || s.length > MAX_MATH_SPAN) continue;
+    if (!opensFlowBlock(text, token) && /\n[ \t]*\n/.test(inner)) continue;
+    if ((s.match(PROSE_WORD) ?? []).length >= PROSE_WORD_LIMIT * 3) continue;
+    return token;
+  }
+  return -1;
+}
 function pairDisplayMath(text) {
   const pairs = /* @__PURE__ */ new Map();
   if (!text.includes("$$")) return pairs;
@@ -5669,11 +5713,15 @@ function isSplitterTreeLine(line) {
 }
 var MEDIA_REF_PREFIXES = ["[Image URL:", "[Video URL:", "[Audio URL:"];
 var Tokenizer = class {
-  constructor(text, streaming = false) {
+  constructor(text, streaming = false, meter = void 0) {
     this.text = text;
     this.streaming = streaming;
+    this.meter = meter;
     this.n = text.length;
+    this.tick(text.length);
     this.mathPairs = pairDisplayMath(text);
+    this.provisionalFrom = streaming ? pendingDisplayMathOpener(text) : -1;
+    if (this.provisionalFrom === -1) this.provisionalFrom = Number.POSITIVE_INFINITY;
     let start = 0;
     for (let i = 0; i < text.length; i++) {
       if (text.charCodeAt(i) === 10) {
@@ -5684,13 +5732,15 @@ var Tokenizer = class {
       }
     }
     if (start < text.length || text.length === 0) {
+      this.pendingCR = streaming && text.length > start && text.charCodeAt(text.length - 1) === 13;
       this.lineStarts.push(start);
-      this.contentEnds.push(text.length);
+      this.contentEnds.push(this.pendingCR ? text.length - 1 : text.length);
       this.lineEnds.push(text.length);
     }
   }
   text;
   streaming;
+  meter;
   n;
   lineStarts = [];
   contentEnds = [];
@@ -5705,6 +5755,23 @@ var Tokenizer = class {
   /** `$$` opener → closer for the pairs the core renders as math. */
   mathPairs;
   finder = (needle, from) => this.next(needle, from);
+  /** Streaming: where provisional text starts (a pending `$$`), or +∞. */
+  provisionalFrom = Number.POSITIVE_INFINITY;
+  /** Streaming: the text ends in a lone CR whose LF has not arrived yet. */
+  pendingCR = false;
+  /**
+   * Streaming: an open last line that begins with `{` is undecided — each new
+   * byte can balance or unbalance its braces and flip the whole line between
+   * prose and a JSON island — so it is provisional until it ends (G5).
+   */
+  markOpenJsonTail() {
+    const last = this.lineStarts.length - 1;
+    if (!this.isOpenTail(last)) return;
+    const start = this.lineStarts[last];
+    if (this.lineText(last).trimStart().startsWith("{")) {
+      this.provisionalFrom = Math.min(this.provisionalFrom, start);
+    }
+  }
   /**
    * `text.indexOf(needle, from)` in O(log k): the first query for a needle
    * indexes all of its occurrences once, so repeated searches for a closer
@@ -5728,9 +5795,21 @@ var Tokenizer = class {
     }
     return lo < list.length ? list[lo] : -1;
   }
+  /** Charge `n` visited characters to the diagnostic meter (complexity tests). */
+  tick(n) {
+    if (this.meter) this.meter.work += n;
+  }
+  /** Streaming only: a paragraph ending at `pe` may still be continued by a line not yet arrived. */
+  paragraphMayGrow(pe) {
+    if (!this.streaming) return false;
+    const rest = this.n - pe;
+    if (rest === 0) return true;
+    const tail = this.text.slice(pe);
+    return tail === "\n" || tail === "\r\n" || tail === "\r";
+  }
   /** Streaming only: the final line, still waiting for its terminator. */
   isOpenTail(k) {
-    return this.streaming && k === this.lineStarts.length - 1 && this.lineEnds[k] === this.contentEnds[k];
+    return this.streaming && k === this.lineStarts.length - 1 && (this.lineEnds[k] === this.contentEnds[k] || this.pendingCR);
   }
   get lineCount() {
     return this.lineStarts.length;
@@ -5755,6 +5834,7 @@ var Tokenizer = class {
     return this.text.slice(this.lineStarts[li], this.contentEnds[li]);
   }
   run() {
+    if (this.streaming) this.markOpenJsonTail();
     let pos = 0;
     while (pos < this.n) {
       const li = this.lineOf(pos);
@@ -5763,6 +5843,10 @@ var Tokenizer = class {
         pos = this.gap(pos, li);
         continue;
       }
+      if (pos >= this.provisionalFrom) {
+        this.push({ kind: "prose", start: pos, end: this.n, island: null, inlines: [] });
+        break;
+      }
       const island = this.detectBlock(pos, li, true);
       if (island) {
         this.push({ kind: "island", start: island.start, end: island.end, island, inlines: [] });
@@ -5966,6 +6050,7 @@ var Tokenizer = class {
         continue;
       }
       if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
+      if (this.isOpenTail(k)) break;
       last = k;
     }
     return this.island("footnote_def", p, this.contentEnds[last], true, { id: open[1] });
@@ -6127,6 +6212,7 @@ var Tokenizer = class {
     const li = this.lineOf(tp);
     const ce = this.contentEnds[li];
     const tag = readXmlTag(this.text, tp, this.finder);
+    this.tick(tag ? tag.raw.length : 1);
     let openEnd;
     if (tag && !tag.isClosing) {
       if (tag.isSelfClosing) {
@@ -6211,7 +6297,7 @@ var Tokenizer = class {
       return this.island("html_block", p, close.index + close[0].length, true, { tag: name });
     }
     const typeSix = HTML_BLOCK_LEVEL_TAGS.has(name);
-    const typeSeven = atParaStart && t.trimEnd() === tag.raw;
+    const typeSeven = atParaStart && t.trimEnd() === tag.raw && !this.isOpenTail(li);
     if (!typeSix && !typeSeven) return null;
     let end = this.contentEnds[li];
     for (let k = li + 1; k < this.lineCount && !this.lineIsBlank(k); k++) {
@@ -6303,6 +6389,10 @@ var Tokenizer = class {
     let breakLine = -1;
     for (let k = li + 1; k < this.lineCount; k++) {
       if (this.lineIsBlank(k)) break;
+      if (this.lineStarts[k] >= this.provisionalFrom) {
+        end = this.n;
+        break;
+      }
       if (this.detectBlock(this.lineStarts[k], k, false)) {
         breakLine = k;
         break;
@@ -6368,6 +6458,7 @@ var Tokenizer = class {
   scanInline(ps, pe) {
     const text = this.text;
     const inlines = [];
+    this.tick(pe - ps);
     const hasKind = text.slice(ps, pe).includes("__kind");
     let kindBudget = 4 * (pe - ps) + 65536;
     const stop = (island, orphan = false) => ({ inlines, breakIsland: island, orphan });
@@ -6378,6 +6469,7 @@ var Tokenizer = class {
     let runs = null;
     const codeSpanClose = (n, from) => {
       if (!runs) {
+        this.tick(pe - ps);
         runs = /* @__PURE__ */ new Map();
         for (let k = ps; k < pe; ) {
           if (text.charCodeAt(k) !== 96) {
@@ -6404,7 +6496,8 @@ var Tokenizer = class {
       return at !== void 0 && at + n <= pe ? at : -1;
     };
     let i = ps;
-    while (i < pe) {
+    const scanEnd = Math.min(pe, this.provisionalFrom);
+    while (i < scanEnd) {
       const ch = text[i];
       if (ch === "\\") {
         const nextCh = text[i + 1];
@@ -6422,6 +6515,9 @@ var Tokenizer = class {
       if (ch === "`") {
         const n = backtickRunLength(text, i);
         const close = codeSpanClose(n, i + n);
+        if (close === -1 && this.streaming && this.paragraphMayGrow(pe)) {
+          return { inlines, breakIsland: null, orphan: false };
+        }
         i = close === -1 ? i + n : close + n;
         continue;
       }
@@ -6438,6 +6534,7 @@ var Tokenizer = class {
           continue;
         }
         const end = singleDollarMathEnd(text, i, pe);
+        this.tick(end > 0 ? end - i : Math.min(403, pe - i));
         if (end > 0 && !(this.streaming && end === this.n)) {
           inlines.push(this.island("math_inline", i, end, true, { delimiter: "$" }));
           i = end;
@@ -6465,6 +6562,7 @@ var Tokenizer = class {
           const limit = Math.min(pe, i + 65536);
           const end = matchingJsonObjectEnd(text, i, limit);
           kindBudget -= (end ?? limit) - i;
+          this.tick((end ?? limit) - i);
           if (end !== null) {
             const kind = declaredKind(text.slice(i, end));
             if (kind) {
@@ -6526,6 +6624,7 @@ var Tokenizer = class {
           continue;
         }
         const tag = readXmlTag(text, i, this.finder);
+        this.tick(tag ? tag.raw.length : 1);
         if (tag && i + tag.raw.length <= pe) {
           const name = tag.tagName;
           const tagEnd = i + tag.raw.length;
@@ -6574,7 +6673,7 @@ var Tokenizer = class {
   }
 };
 function tokenizeSource(text, options = {}) {
-  const raw = new Tokenizer(text, options.streaming ?? false).run();
+  const raw = new Tokenizer(text, options.streaming ?? false, options.meter).run();
   const cp = buildCodePointIndex(text);
   const inline = (island) => ({
     islandType: island.type,
@@ -7037,6 +7136,7 @@ exports.openParseSession = openParseSession;
 exports.pairDisplayMath = pairDisplayMath;
 exports.parseDirectiveSlug = parseDirectiveSlug;
 exports.parseFenceOpener = parseFenceOpener;
+exports.pendingDisplayMathOpener = pendingDisplayMathOpener;
 exports.readEnvelope = readEnvelope;
 exports.readNodeOutcomeValue = readNodeOutcomeValue;
 exports.readObjectKind = readObjectKind;