@ai-matrx/content-ir 0.16.2 → 0.17.0

package/dist/source.d.cts

@@ -420,6 +420,30 @@ declare function parseFenceOpener(line: string): {
 } | null;
 declare function classifyInnerFenceLine(trimmed: string, openTicks: number, nests: boolean, nestedDepth: number): InnerFenceLine;
 
+interface CodeRange {
+    /** UTF-16 offset of the first character (the fence line start, or the opening backtick run). */
+    start: number;
+    /** Offset just past the closing fence line's last character / closing backtick run (end of text when unclosed). */
+    end: number;
+    kind: "fence" | "span";
+}
+/**
+ * Where code is, by CommonMark — THE ONE code-range rule for every package.
+ * Math pairing (`pairDisplayMath`), the tokenizer's math islands, print's math
+ * lifting and kit's delimiter guard all ask this function; a second copy of
+ * code detection anywhere in `apps/shared` fails
+ * `__tests__/one-code-range-rule.test.ts`.
+ *   - A fence opens only at the START of a line (≤ 3 spaces of indent): a
+ *     mid-sentence "```python" is text, not a fence that swallows everything
+ *     after it (verify-RC-B9: one inline ``` turned math off for the rest of
+ *     the answer). Closing follows THE ONE nested-fence rule
+ *     (`classifyInnerFenceLine`); an unclosed fence runs to the end.
+ *   - An inline code span closes only at a backtick run of the SAME length in
+ *     the same paragraph; an unmatched run is literal text.
+ * Linear: one pass over lines, one over each paragraph's backtick runs.
+ */
+declare function findCodeRanges(text: string): CodeRange[];
+
 /**
  * `$$...$$`, `$$…$$`, `$$ · $$`: a PLACEHOLDER — prose describing the syntax,
  * never a formula. The pair is consumed whole as literal text (neither `$$`
@@ -433,7 +457,7 @@ declare function isMathPlaceholder(inner: string): boolean;
  * pairing with the next paragraph's formula.
  */
 declare function looksLikeDisplayMath(inner: string, flow?: boolean): boolean;
-/** Map of `$$` opener offset → closer offset for every pair the renderer shows as math. */
+
 /**
  * STREAMING: the last `$$` token that is still unpaired AND could still pair
  * with a `$$` not yet arrived — its span so far is short enough, carries no
@@ -442,6 +466,7 @@ declare function looksLikeDisplayMath(inner: string, flow?: boolean): boolean;
  * streaming tokenizer treats it as provisional. -1 when there is none.
  */
 declare function pendingDisplayMathOpener(text: string): number;
+/** Map of `$$` opener offset → closer offset for every pair the renderer shows as math. */
 declare function pairDisplayMath(text: string): Map<number, number>;
 
-export { type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isMathPlaceholder, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
+export { type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type CodeRange, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, findCodeRanges, isMathPlaceholder, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };

package/dist/source.d.ts

@@ -420,6 +420,30 @@ declare function parseFenceOpener(line: string): {
 } | null;
 declare function classifyInnerFenceLine(trimmed: string, openTicks: number, nests: boolean, nestedDepth: number): InnerFenceLine;
 
+interface CodeRange {
+    /** UTF-16 offset of the first character (the fence line start, or the opening backtick run). */
+    start: number;
+    /** Offset just past the closing fence line's last character / closing backtick run (end of text when unclosed). */
+    end: number;
+    kind: "fence" | "span";
+}
+/**
+ * Where code is, by CommonMark — THE ONE code-range rule for every package.
+ * Math pairing (`pairDisplayMath`), the tokenizer's math islands, print's math
+ * lifting and kit's delimiter guard all ask this function; a second copy of
+ * code detection anywhere in `apps/shared` fails
+ * `__tests__/one-code-range-rule.test.ts`.
+ *   - A fence opens only at the START of a line (≤ 3 spaces of indent): a
+ *     mid-sentence "```python" is text, not a fence that swallows everything
+ *     after it (verify-RC-B9: one inline ``` turned math off for the rest of
+ *     the answer). Closing follows THE ONE nested-fence rule
+ *     (`classifyInnerFenceLine`); an unclosed fence runs to the end.
+ *   - An inline code span closes only at a backtick run of the SAME length in
+ *     the same paragraph; an unmatched run is literal text.
+ * Linear: one pass over lines, one over each paragraph's backtick runs.
+ */
+declare function findCodeRanges(text: string): CodeRange[];
+
 /**
  * `$$...$$`, `$$…$$`, `$$ · $$`: a PLACEHOLDER — prose describing the syntax,
  * never a formula. The pair is consumed whole as literal text (neither `$$`
@@ -433,7 +457,7 @@ declare function isMathPlaceholder(inner: string): boolean;
  * pairing with the next paragraph's formula.
  */
 declare function looksLikeDisplayMath(inner: string, flow?: boolean): boolean;
-/** Map of `$$` opener offset → closer offset for every pair the renderer shows as math. */
+
 /**
  * STREAMING: the last `$$` token that is still unpaired AND could still pair
  * with a `$$` not yet arrived — its span so far is short enough, carries no
@@ -442,6 +466,7 @@ declare function looksLikeDisplayMath(inner: string, flow?: boolean): boolean;
  * streaming tokenizer treats it as provisional. -1 when there is none.
  */
 declare function pendingDisplayMathOpener(text: string): number;
+/** Map of `$$` opener offset → closer offset for every pair the renderer shows as math. */
 declare function pairDisplayMath(text: string): Map<number, number>;
 
-export { type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isMathPlaceholder, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
+export { type BlockIslandType, CALLOUT_NAMES, type CodePointIndex, type CodeRange, type DisturbedIsland, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, type InlineIslandType, type InnerFenceLine, type IslandMeta, type IslandType, type MapOptions, type MappedPosition, type MappedRange, NESTING_FENCE_LANGUAGES, type ReadXmlTag, type SourceBlock, type SourceBlockKind, type SourceChange, type SourceEdit, type SourceIsland, type SourceSpan, SourceSpliceError, type SourceSpliceErrorCode, type SpliceIntegrity, type SpliceOptions, type SpliceResult, type TokenizeOptions, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, findCodeRanges, isMathPlaceholder, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };

package/dist/source.js

@@ -313,32 +313,8 @@ function singleDollarMathEnd(text, open, limit = text.length) {
   return isSingleDollarMath(content, text[close + 1]) ? close + 1 : -1;
 }
 
-// source/math-pairs.ts
-var MAX_MATH_SPAN = 600;
-var STRUCTURAL_MARKDOWN = /\]\(|https?:\/\/|\*\*|(?:^|\n)[ \t]{0,3}#{1,6}[ \t]|(?:^|\n)[ \t]*[-*+][ \t]+|(?:^|\n)[ \t]*\d+[.)][ \t]/;
-var LATEX_COMMAND = /\\[a-zA-Z]/;
-var PROSE_WORD = /[A-Za-z]{3,}/g;
-var PROSE_WORD_LIMIT = 6;
-function isMathPlaceholder(inner) {
-  return /^[\s.…·⋯_-]*$/.test(inner) && /[.…·⋯]/.test(inner);
-}
-function opensFlowBlock(text, open) {
-  for (let k = open - 1; k >= 0 && text[k] !== "\n"; k--) if (text[k] !== " " && text[k] !== "	") return false;
-  for (let k = open + 2; k < text.length && text[k] !== "\n"; k++) if (text[k] !== " " && text[k] !== "	" && text[k] !== "\r") return false;
-  return true;
-}
-function looksLikeDisplayMath(inner, flow = false) {
-  const s = inner.trim();
-  if (!s) return false;
-  if (STRUCTURAL_MARKDOWN.test(s)) return false;
-  if (s.length > MAX_MATH_SPAN) return false;
-  if (!flow && /\n[ \t]*\n/.test(inner)) return false;
-  if (LATEX_COMMAND.test(s)) {
-    return (s.match(PROSE_WORD) ?? []).length < PROSE_WORD_LIMIT * 3;
-  }
-  return (s.match(PROSE_WORD) ?? []).length < PROSE_WORD_LIMIT;
-}
-function codeRanges(text) {
+// source/code-ranges.ts
+function findCodeRanges(text) {
   const ranges = [];
   const n = text.length;
   let fence = null;
@@ -370,7 +346,7 @@ function codeRanges(text) {
       }
       const [s0] = runs[r];
       const [s1, l1] = runs[j];
-      ranges.push([s0, s1 + l1]);
+      ranges.push({ start: s0, end: s1 + l1, kind: "span" });
       r = j + 1;
     }
   };
@@ -396,7 +372,7 @@ function codeRanges(text) {
         closes = indent <= 3 && t >= fence.len && trimFenceLine(trimmed.slice(t)) === "";
       }
       if (closes) {
-        ranges.push([fence.start, lineEnd]);
+        ranges.push({ start: fence.start, end: lineEnd, kind: "fence" });
         fence = null;
       }
     } else {
@@ -416,9 +392,106 @@ function codeRanges(text) {
     if (nl === -1) break;
     pos = nl + 1;
   }
-  if (fence) ranges.push([fence.start, n]);
+  if (fence) ranges.push({ start: fence.start, end: n, kind: "fence" });
   else if (para !== -1) spans(para, n);
-  return ranges.sort((a, b) => a[0] - b[0]);
+  return ranges.sort((a, b) => a.start - b.start);
+}
+function pendingCodeSpanStart(text) {
+  const n = text.length;
+  if (n === 0 || /\n[ \t]*\n[ \t]*$/.test(text)) return -1;
+  let fence = null;
+  let para = -1;
+  let pos = 0;
+  while (pos <= n) {
+    const nl = text.indexOf("\n", pos);
+    const lineEnd = nl === -1 ? n : nl;
+    const line = text.slice(pos, lineEnd);
+    let indent = 0;
+    while (indent < line.length && line[indent] === " ") indent++;
+    const body = line.slice(indent);
+    if (fence) {
+      const trimmed = trimFenceLine(line);
+      let closes = false;
+      if (fence.char === "`") {
+        const kind = classifyInnerFenceLine(trimmed, fence.len, fence.nests, fence.depth);
+        if (kind === "open-nested") fence.depth++;
+        else if (kind === "close-nested") fence.depth--;
+        else if (kind === "close-outer") closes = indent <= 3;
+      } else {
+        let t = 0;
+        while (t < trimmed.length && trimmed[t] === "~") t++;
+        closes = indent <= 3 && t >= fence.len && trimFenceLine(trimmed.slice(t)) === "";
+      }
+      if (closes) fence = null;
+    } else {
+      const opener = indent <= 3 ? /^(`{3,}|~{3,})(.*)$/.exec(body) : null;
+      const info = opener ? opener[2] : "";
+      if (opener && !(opener[1][0] === "`" && info.includes("`"))) {
+        para = -1;
+        const char = opener[1][0];
+        const parsed = char === "`" ? parseFenceOpener(line) : null;
+        fence = { char, len: opener[1].length, nests: fenceNestsInnerFences(parsed?.lang), depth: 0 };
+      } else if (body.trim() === "") {
+        if (!(nl === -1 && line === "")) para = -1;
+      } else if (para === -1) para = pos;
+    }
+    if (nl === -1) break;
+    pos = nl + 1;
+  }
+  if (fence || para === -1) return -1;
+  const runs = [];
+  for (let k = para; k < n; ) {
+    if (text.charCodeAt(k) !== 96) {
+      k++;
+      continue;
+    }
+    let len = 1;
+    while (k + len < n && text.charCodeAt(k + len) === 96) len++;
+    runs.push([k, len]);
+    k += len;
+  }
+  const growing = (r) => {
+    const [start, len] = runs[r];
+    return start + len === n;
+  };
+  for (let r = 0; r < runs.length; ) {
+    if (growing(r)) return runs[r][0];
+    const len = runs[r][1];
+    let j = r + 1;
+    while (j < runs.length && (runs[j][1] !== len || growing(j))) j++;
+    if (j >= runs.length) return runs[r][0];
+    r = j + 1;
+  }
+  return -1;
+}
+
+// source/math-pairs.ts
+var MAX_MATH_SPAN = 600;
+var STRUCTURAL_MARKDOWN = /\]\(|https?:\/\/|\*\*|(?:^|\n)[ \t]{0,3}#{1,6}[ \t]|(?:^|\n)[ \t]*[-*+][ \t]+|(?:^|\n)[ \t]*\d+[.)][ \t]/;
+var LATEX_COMMAND = /\\[a-zA-Z]/;
+var PROSE_WORD = /[A-Za-z]{3,}/g;
+var PROSE_WORD_LIMIT = 6;
+function isMathPlaceholder(inner) {
+  return /^[\s.…·⋯_-]*$/.test(inner) && /[.…·⋯]/.test(inner);
+}
+function opensFlowBlock(text, open) {
+  for (let k = open - 1; k >= 0 && text[k] !== "\n"; k--) if (text[k] !== " " && text[k] !== "	") return false;
+  for (let k = open + 2; k < text.length && text[k] !== "\n"; k++) if (text[k] !== " " && text[k] !== "	" && text[k] !== "\r") return false;
+  return true;
+}
+function looksLikeDisplayMath(inner, flow = false) {
+  const s = inner.trim();
+  if (!s) return false;
+  if (STRUCTURAL_MARKDOWN.test(s)) return false;
+  if (s.length > MAX_MATH_SPAN) return false;
+  if (!flow && /\n[ \t]*\n/.test(inner)) return false;
+  if (LATEX_COMMAND.test(s)) {
+    return (s.match(PROSE_WORD) ?? []).length < PROSE_WORD_LIMIT * 3;
+  }
+  return (s.match(PROSE_WORD) ?? []).length < PROSE_WORD_LIMIT;
+}
+function codeRanges(text) {
+  return findCodeRanges(text).map((r) => [r.start, r.end]);
 }
 function pendingDisplayMathOpener(text) {
   if (!text.includes("$$")) return -1;
@@ -830,6 +903,14 @@ var Tokenizer = class {
     this.provisionalFrom = streaming ? pendingDisplayMathOpener(text) : -1;
     if (this.provisionalFrom === -1) this.provisionalFrom = Number.POSITIVE_INFINITY;
     if (streaming && text.includes("$$")) {
+      const span = pendingCodeSpanStart(text);
+      if (span !== -1) {
+        const firstToken = text.indexOf("$$", span);
+        if (firstToken !== -1) this.provisionalFrom = Math.min(this.provisionalFrom, firstToken);
+        for (const [open, close] of this.mathPairs) {
+          if (close >= span) this.provisionalFrom = Math.min(this.provisionalFrom, open);
+        }
+      }
       for (const pad of ["`", "``", "```", "~~~"]) {
         const alt = pairDisplayMath(text + pad);
         for (const [open, close] of this.mathPairs) {
@@ -1185,8 +1266,8 @@ var Tokenizer = class {
         last = k;
         continue;
       }
-      if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
       if (this.isOpenTail(k)) return this.heldFootnote(p, open[1]);
+      if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
       last = k;
     }
     const end = this.contentEnds[last];
@@ -2173,6 +2254,6 @@ function mapRange(changes, start, end, unit = "utf16") {
   return { start: newStart, end: newEnd, touched, collapsed: newEnd === newStart && end > start };
 }
 
-export { CALLOUT_NAMES, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, NESTING_FENCE_LANGUAGES, SourceSpliceError, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, isMathPlaceholder, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
+export { CALLOUT_NAMES, FENCE_OPENER_WHITESPACE, FENCE_WHITESPACE, NESTING_FENCE_LANGUAGES, SourceSpliceError, XmlContainerTracker, blockAt, blockEdit, blockRangeEdit, buildCodePointIndex, classifyInnerFenceLine, fenceNestsInnerFences, findCodeRanges, isMathPlaceholder, isSingleDollarMath, islandEdit, joinSource, listIslands, looksLikeDisplayMath, mapPosition, mapRange, pairDisplayMath, parseFenceOpener, pendingDisplayMathOpener, readXmlTag, singleDollarMathEnd, spliceSave, splitsSurrogatePair, toCodePointOffset, toUtf16Offset, tokenizeSource, trimFenceLine };
 //# sourceMappingURL=source.js.map
 //# sourceMappingURL=source.js.map