@ai-matrx/content-ir 0.16.2 → 0.17.0

package/dist/source.cjs

@@ -315,32 +315,8 @@ function singleDollarMathEnd(text, open, limit = text.length) {
   return isSingleDollarMath(content, text[close + 1]) ? close + 1 : -1;
 }
 
-// source/math-pairs.ts
-var MAX_MATH_SPAN = 600;
-var STRUCTURAL_MARKDOWN = /\]\(|https?:\/\/|\*\*|(?:^|\n)[ \t]{0,3}#{1,6}[ \t]|(?:^|\n)[ \t]*[-*+][ \t]+|(?:^|\n)[ \t]*\d+[.)][ \t]/;
-var LATEX_COMMAND = /\\[a-zA-Z]/;
-var PROSE_WORD = /[A-Za-z]{3,}/g;
-var PROSE_WORD_LIMIT = 6;
-function isMathPlaceholder(inner) {
-  return /^[\s.…·⋯_-]*$/.test(inner) && /[.…·⋯]/.test(inner);
-}
-function opensFlowBlock(text, open) {
-  for (let k = open - 1; k >= 0 && text[k] !== "\n"; k--) if (text[k] !== " " && text[k] !== "	") return false;
-  for (let k = open + 2; k < text.length && text[k] !== "\n"; k++) if (text[k] !== " " && text[k] !== "	" && text[k] !== "\r") return false;
-  return true;
-}
-function looksLikeDisplayMath(inner, flow = false) {
-  const s = inner.trim();
-  if (!s) return false;
-  if (STRUCTURAL_MARKDOWN.test(s)) return false;
-  if (s.length > MAX_MATH_SPAN) return false;
-  if (!flow && /\n[ \t]*\n/.test(inner)) return false;
-  if (LATEX_COMMAND.test(s)) {
-    return (s.match(PROSE_WORD) ?? []).length < PROSE_WORD_LIMIT * 3;
-  }
-  return (s.match(PROSE_WORD) ?? []).length < PROSE_WORD_LIMIT;
-}
-function codeRanges(text) {
+// source/code-ranges.ts
+function findCodeRanges(text) {
   const ranges = [];
   const n = text.length;
   let fence = null;
@@ -372,7 +348,7 @@ function codeRanges(text) {
       }
       const [s0] = runs[r];
       const [s1, l1] = runs[j];
-      ranges.push([s0, s1 + l1]);
+      ranges.push({ start: s0, end: s1 + l1, kind: "span" });
       r = j + 1;
     }
   };
@@ -398,7 +374,7 @@ function codeRanges(text) {
         closes = indent <= 3 && t >= fence.len && trimFenceLine(trimmed.slice(t)) === "";
       }
       if (closes) {
-        ranges.push([fence.start, lineEnd]);
+        ranges.push({ start: fence.start, end: lineEnd, kind: "fence" });
         fence = null;
       }
     } else {
@@ -418,9 +394,106 @@ function codeRanges(text) {
     if (nl === -1) break;
     pos = nl + 1;
   }
-  if (fence) ranges.push([fence.start, n]);
+  if (fence) ranges.push({ start: fence.start, end: n, kind: "fence" });
   else if (para !== -1) spans(para, n);
-  return ranges.sort((a, b) => a[0] - b[0]);
+  return ranges.sort((a, b) => a.start - b.start);
+}
+function pendingCodeSpanStart(text) {
+  const n = text.length;
+  if (n === 0 || /\n[ \t]*\n[ \t]*$/.test(text)) return -1;
+  let fence = null;
+  let para = -1;
+  let pos = 0;
+  while (pos <= n) {
+    const nl = text.indexOf("\n", pos);
+    const lineEnd = nl === -1 ? n : nl;
+    const line = text.slice(pos, lineEnd);
+    let indent = 0;
+    while (indent < line.length && line[indent] === " ") indent++;
+    const body = line.slice(indent);
+    if (fence) {
+      const trimmed = trimFenceLine(line);
+      let closes = false;
+      if (fence.char === "`") {
+        const kind = classifyInnerFenceLine(trimmed, fence.len, fence.nests, fence.depth);
+        if (kind === "open-nested") fence.depth++;
+        else if (kind === "close-nested") fence.depth--;
+        else if (kind === "close-outer") closes = indent <= 3;
+      } else {
+        let t = 0;
+        while (t < trimmed.length && trimmed[t] === "~") t++;
+        closes = indent <= 3 && t >= fence.len && trimFenceLine(trimmed.slice(t)) === "";
+      }
+      if (closes) fence = null;
+    } else {
+      const opener = indent <= 3 ? /^(`{3,}|~{3,})(.*)$/.exec(body) : null;
+      const info = opener ? opener[2] : "";
+      if (opener && !(opener[1][0] === "`" && info.includes("`"))) {
+        para = -1;
+        const char = opener[1][0];
+        const parsed = char === "`" ? parseFenceOpener(line) : null;
+        fence = { char, len: opener[1].length, nests: fenceNestsInnerFences(parsed?.lang), depth: 0 };
+      } else if (body.trim() === "") {
+        if (!(nl === -1 && line === "")) para = -1;
+      } else if (para === -1) para = pos;
+    }
+    if (nl === -1) break;
+    pos = nl + 1;
+  }
+  if (fence || para === -1) return -1;
+  const runs = [];
+  for (let k = para; k < n; ) {
+    if (text.charCodeAt(k) !== 96) {
+      k++;
+      continue;
+    }
+    let len = 1;
+    while (k + len < n && text.charCodeAt(k + len) === 96) len++;
+    runs.push([k, len]);
+    k += len;
+  }
+  const growing = (r) => {
+    const [start, len] = runs[r];
+    return start + len === n;
+  };
+  for (let r = 0; r < runs.length; ) {
+    if (growing(r)) return runs[r][0];
+    const len = runs[r][1];
+    let j = r + 1;
+    while (j < runs.length && (runs[j][1] !== len || growing(j))) j++;
+    if (j >= runs.length) return runs[r][0];
+    r = j + 1;
+  }
+  return -1;
+}
+
+// source/math-pairs.ts
+var MAX_MATH_SPAN = 600;
+var STRUCTURAL_MARKDOWN = /\]\(|https?:\/\/|\*\*|(?:^|\n)[ \t]{0,3}#{1,6}[ \t]|(?:^|\n)[ \t]*[-*+][ \t]+|(?:^|\n)[ \t]*\d+[.)][ \t]/;
+var LATEX_COMMAND = /\\[a-zA-Z]/;
+var PROSE_WORD = /[A-Za-z]{3,}/g;
+var PROSE_WORD_LIMIT = 6;
+function isMathPlaceholder(inner) {
+  return /^[\s.…·⋯_-]*$/.test(inner) && /[.…·⋯]/.test(inner);
+}
+function opensFlowBlock(text, open) {
+  for (let k = open - 1; k >= 0 && text[k] !== "\n"; k--) if (text[k] !== " " && text[k] !== "	") return false;
+  for (let k = open + 2; k < text.length && text[k] !== "\n"; k++) if (text[k] !== " " && text[k] !== "	" && text[k] !== "\r") return false;
+  return true;
+}
+function looksLikeDisplayMath(inner, flow = false) {
+  const s = inner.trim();
+  if (!s) return false;
+  if (STRUCTURAL_MARKDOWN.test(s)) return false;
+  if (s.length > MAX_MATH_SPAN) return false;
+  if (!flow && /\n[ \t]*\n/.test(inner)) return false;
+  if (LATEX_COMMAND.test(s)) {
+    return (s.match(PROSE_WORD) ?? []).length < PROSE_WORD_LIMIT * 3;
+  }
+  return (s.match(PROSE_WORD) ?? []).length < PROSE_WORD_LIMIT;
+}
+function codeRanges(text) {
+  return findCodeRanges(text).map((r) => [r.start, r.end]);
 }
 function pendingDisplayMathOpener(text) {
   if (!text.includes("$$")) return -1;
@@ -832,6 +905,14 @@ var Tokenizer = class {
     this.provisionalFrom = streaming ? pendingDisplayMathOpener(text) : -1;
     if (this.provisionalFrom === -1) this.provisionalFrom = Number.POSITIVE_INFINITY;
     if (streaming && text.includes("$$")) {
+      const span = pendingCodeSpanStart(text);
+      if (span !== -1) {
+        const firstToken = text.indexOf("$$", span);
+        if (firstToken !== -1) this.provisionalFrom = Math.min(this.provisionalFrom, firstToken);
+        for (const [open, close] of this.mathPairs) {
+          if (close >= span) this.provisionalFrom = Math.min(this.provisionalFrom, open);
+        }
+      }
       for (const pad of ["`", "``", "```", "~~~"]) {
         const alt = pairDisplayMath(text + pad);
         for (const [open, close] of this.mathPairs) {
@@ -1187,8 +1268,8 @@ var Tokenizer = class {
         last = k;
         continue;
       }
-      if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
       if (this.isOpenTail(k)) return this.heldFootnote(p, open[1]);
+      if (FOOTNOTE_DEF.test(line.trimStart()) || this.detectBlock(this.lineStarts[k], k, false)) break;
       last = k;
     }
     const end = this.contentEnds[last];
@@ -2187,6 +2268,7 @@ exports.blockRangeEdit = blockRangeEdit;
 exports.buildCodePointIndex = buildCodePointIndex;
 exports.classifyInnerFenceLine = classifyInnerFenceLine;
 exports.fenceNestsInnerFences = fenceNestsInnerFences;
+exports.findCodeRanges = findCodeRanges;
 exports.isMathPlaceholder = isMathPlaceholder;
 exports.isSingleDollarMath = isSingleDollarMath;
 exports.islandEdit = islandEdit;