@ai-react-markdown/engine 2.5.0 → 2.5.2

package/dist/index.cjs

@@ -65,6 +65,8 @@ __export(src_exports, {
   hasLoneSurrogate: () => hasLoneSurrogate,
   highlight: () => highlight,
   isFootnoteSection: () => isFootnoteSection,
+  isWhitespaceText: () => isWhitespaceText,
+  lastMeaningfulIdx: () => lastMeaningfulIdx,
   measureStage: () => measureStage,
   mergeClassNameAllowlist: () => mergeClassNameAllowlist,
   normalizeForMatch: () => normalizeForMatch,
@@ -751,6 +753,7 @@ function processConfirmedLine(cp, ln, text) {
       cp.openTotal += 1;
     }
   };
+  const strayTablePart = (tag) => TABLE_PART_NAMES.has(tag) && (cp.tagBalance.get("table") ?? 0) === 0;
   const commentOpenAtLineStart = cp.commentOpen;
   const rawOpenAtLineStart = cp.commentOpen || cp.piOpen || cp.declOpen || cp.cdataOpen || cp.bogusOpen;
   if (cp.inFence) {
@@ -1079,7 +1082,7 @@ ${cont(scanText)}` };
       if (cp.commentOpen) continue;
       const closing = m[1] === "/";
       const tag = m[2].toLowerCase();
-      if (TABLE_PART_NAMES.has(tag)) cp.phasePoisonedAt = Math.min(cp.phasePoisonedAt, ln.start + m.index);
+      if (strayTablePart(tag)) cp.phasePoisonedAt = Math.min(cp.phasePoisonedAt, ln.start + m.index);
       let attrs = m[3] ?? "";
       if (cp.htmlFlowReal && (cp.rawTextOpen === null || closing && tag === cp.rawTextOpen)) {
         const attrStart = m.index + 1 + (closing ? 1 : 0) + m[2].length;
@@ -1123,7 +1126,7 @@ ${cont(scanText)}` };
         if (startMasked || wholeVisible || inRaw(mr.index) || cp.commentOpen) continue;
         const closing = mr[1] === "/";
         const tag = mr[2].toLowerCase();
-        if (TABLE_PART_NAMES.has(tag)) cp.phasePoisonedAt = Math.min(cp.phasePoisonedAt, ln.start + mr.index);
+        if (strayTablePart(tag)) cp.phasePoisonedAt = Math.min(cp.phasePoisonedAt, ln.start + mr.index);
         if (closing && mr[3] !== void 0 && !/^\s*$/.test(mr[3])) continue;
         const selfClosing = mr[3] !== void 0 && /\/\s*$/.test(mr[3]);
         if (VOID_TAGS.has(tag) || selfClosing && honoursSelfClosing(tag)) continue;
@@ -1131,7 +1134,7 @@ ${cont(scanText)}` };
       }
     }
     if (cp.pendingTruncatedTags.length > 0 && ln.text.includes(">")) {
-      if (cp.pendingTruncatedTags.some((t) => TABLE_PART_NAMES.has(t))) {
+      if (cp.pendingTruncatedTags.some((t) => strayTablePart(t))) {
         cp.phasePoisonedAt = Math.min(cp.phasePoisonedAt, ln.start);
       }
       cp.pendingTruncatedTags = [];
@@ -1148,7 +1151,7 @@ ${cont(scanText)}` };
         if (m2) {
           const closing = m2[1] === "/";
           const tag = m2[2].toLowerCase();
-          if (TABLE_PART_NAMES.has(tag) && cp.htmlFlowReal) {
+          if (strayTablePart(tag) && cp.htmlFlowReal) {
             cp.phasePoisonedAt = Math.min(cp.phasePoisonedAt, ln.start + lastLt);
           }
           if (cp.htmlFlowReal) {
@@ -1198,6 +1201,15 @@ function isFootnoteSection(node) {
   const props = node.properties;
   return props?.dataFootnotes !== void 0;
 }
+function isWhitespaceText(c) {
+  return c.type === "text" && /^\s*$/.test(c.value);
+}
+function lastMeaningfulIdx(children) {
+  for (let i = children.length - 1; i >= 0; i--) {
+    if (!isWhitespaceText(children[i])) return i;
+  }
+  return -1;
+}
 
 // src/components/incrementalParse/attributeHastChildren.ts
 function attributeHastChildren(mdast, hast, stopAt = Infinity) {
@@ -1373,6 +1385,24 @@ function rebaseDualWalk(node, segments, maxEnd, offsetDelta, lineDelta) {
   }
 }
 var TABLE_PART_TAG_RE = /<(?:td|th|tr|tbody|thead|tfoot|caption|col|colgroup)\b/i;
+var TABLE_TOKEN_RE = /<(\/?)(table|td|th|tr|tbody|thead|tfoot|caption|col|colgroup)\b/gi;
+function hasStrayTablePart(values) {
+  let depth = 0;
+  for (const value of values) {
+    TABLE_TOKEN_RE.lastIndex = 0;
+    let m;
+    while ((m = TABLE_TOKEN_RE.exec(value)) !== null) {
+      const closing = m[1] === "/";
+      const tag = m[2].toLowerCase();
+      if (tag === "table") {
+        depth = closing ? Math.max(0, depth - 1) : depth + 1;
+        continue;
+      }
+      if (depth === 0) return true;
+    }
+  }
+  return false;
+}
 var STRAY_SYNTHESIZED_END_TAG_RE = /<\/(?:br|p)\b/i;
 function spliceTrees(input) {
   const { prevMdast, prevHast, tailMdast, tailHast, content, boundary, injectionPrefix, injectedSegments } = input;
@@ -1430,7 +1460,7 @@ function spliceTrees(input) {
     return !(start !== void 0 && start < injectedLen);
   });
   const tailWrapVisible = tailMdastChildren.some((child) => !isWrapInvisible(child));
-  if (prefixMdast.some((c) => c.type === "html" && TABLE_PART_TAG_RE.test(c.value))) return null;
+  if (hasStrayTablePart(prefixMdast.flatMap((c) => c.type === "html" ? [c.value] : []))) return null;
   for (const child of tailMdastChildren) {
     if (isWrapInvisible(child)) continue;
     if (child.type !== "html") break;
@@ -1962,16 +1992,6 @@ function isBackrefAnchor(c) {
   if (el.tagName !== "a") return false;
   return Boolean(el.properties && "dataFootnoteBackref" in el.properties);
 }
-function isWhitespaceText(c) {
-  if (c.type !== "text") return false;
-  return /^\s*$/.test(c.value);
-}
-function lastMeaningfulIdx(children) {
-  for (let i = children.length - 1; i >= 0; i--) {
-    if (!isWhitespaceText(children[i])) return i;
-  }
-  return -1;
-}
 function dropTrailingBackrefs(children) {
   let trailingWsStart = children.length;
   while (trailingWsStart > 0 && isWhitespaceText(children[trailingWsStart - 1])) {
@@ -4088,6 +4108,14 @@ var graphemeEnds = (text, base) => {
   }
   return ends;
 };
+var RESUME_LOOKBACK = 64;
+var isMidSurrogatePair = (text, at) => at > 0 && at < text.length && (text.charCodeAt(at - 1) & 64512) === 55296 && (text.charCodeAt(at) & 64512) === 56320;
+var RI_FIRST = 127462;
+var RI_LAST = 127487;
+var isRegionalIndicatorAt = (text, at) => {
+  const cp = text.codePointAt(at);
+  return cp !== void 0 && cp >= RI_FIRST && cp <= RI_LAST;
+};
 var createSmoothStreamController = (options = {}) => {
   const now = options.now ?? defaultNow;
   const schedule = options.schedule ?? defaultSchedule;
@@ -4096,6 +4124,7 @@ var createSmoothStreamController = (options = {}) => {
   let pending = [];
   let tentativeEnd = 0;
   let finished = false;
+  let seam;
   let drainDeadlineAt;
   let initialized = false;
   let visibleCache = "";
@@ -4179,8 +4208,19 @@ var createSmoothStreamController = (options = {}) => {
     if (!disposed && !cancelFrame && pending.length > 0) cancelFrame = schedule(tick);
   };
   const resegmentTail = () => {
+    if (seam !== void 0 && seam < source.length && pending[pending.length - 1] === seam) {
+      pending.pop();
+    }
     const from = pending.length > 0 ? pending[pending.length - 1] : visibleEnd;
-    const ends = graphemeEnds(source.slice(from), from);
+    let anchor = from;
+    if (seam !== void 0 && from <= seam) {
+      anchor = Math.max(0, from - RESUME_LOOKBACK);
+      if (isMidSurrogatePair(source, anchor)) anchor -= 1;
+      while (anchor >= 2 && isRegionalIndicatorAt(source, anchor - 2)) anchor -= 2;
+    } else {
+      seam = void 0;
+    }
+    const ends = graphemeEnds(source.slice(anchor), anchor);
     if (ends.length === 0) {
       tentativeEnd = from;
       return;
@@ -4192,7 +4232,7 @@ var createSmoothStreamController = (options = {}) => {
       if (last >= 55296 && last <= 56319) hold = 2;
     }
     const confirmedEnds = finished ? ends : ends.slice(0, -hold);
-    for (const end of confirmedEnds) pending.push(end);
+    for (const end of confirmedEnds) if (end > from) pending.push(end);
   };
   const snap = (next) => {
     disposed = false;
@@ -4206,6 +4246,7 @@ var createSmoothStreamController = (options = {}) => {
     visibleEnd = next.length;
     tentativeEnd = next.length;
     pending = [];
+    seam = next.length;
     credit = 0;
     cancelScheduled();
     if (visibleCache !== next) notify();
@@ -4240,6 +4281,7 @@ var createSmoothStreamController = (options = {}) => {
       if (tentativeEnd > (pending.length > 0 ? pending[pending.length - 1] : visibleEnd)) {
         pending.push(tentativeEnd);
       }
+      seam = source.length;
       ensureScheduled();
     },
     snap,
@@ -4616,21 +4658,7 @@ function convertSingleToDoubleDollar(text) {
 }
 function preprocessLaTeX(str) {
   if (!hasLatexTrigger(str)) return str;
-  const segments = splitByProtectedRegions(str);
-  const result = segments.map((segment) => {
-    if (segment.isCode) return segment.text;
-    let text = segment.text;
-    text = escapeMhchemCommands(text);
-    text = escapeCurrencyDollarSigns(text);
-    text = convertLatexDelimiters(text);
-    text = escapeLatexPipes(text);
-    text = escapeLatexPipesInUnclosed(text);
-    text = escapeTextUnderscores(text);
-    text = convertSingleToDoubleDollar(text);
-    text = truncateUnclosedLatexBlock(text);
-    return text;
-  });
-  return result.join("");
+  return processSlice(str, false).out;
 }
 function hasLatexTrigger(str) {
   return str.includes("$") || str.includes("\\[") || str.includes("\\(");
@@ -4659,7 +4687,7 @@ function hasUnclosedTextCommand(text) {
 }
 var RESIDUAL_OPEN_BRACKET_RE = /(?<!!)\\\[/;
 var LEADING_DOUBLE_DOLLAR_RE = /^\s*\$\$/;
-function processSliceInstrumented(slice, probe = true) {
+function processSlice(slice, probe = true) {
   const segments = splitByProtectedRegions(slice);
   const parts = [];
   let quiescent = true;
@@ -4783,13 +4811,13 @@ function createIncrementalLatexPreprocessor(options) {
       };
       const cut = findRawSafeCut(active);
       if (cut > 0) {
-        const candidate = processSliceInstrumented(active.slice(0, cut));
+        const candidate = processSlice(active.slice(0, cut));
         if (candidate.quiescent) freeze(cut, candidate);
       }
       nextAttemptLen = advanced || !backoff ? 0 : active.length * 2;
       onAttempt?.({ activeLength, frozenBytes });
     }
-    const tail = processSliceInstrumented(active, false);
+    const tail = processSlice(active, false);
     const head = tail.truncatedAtSeamStart ? frozenOut.replace(/\s+$/, "") : frozenOut;
     const out = head + tail.out;
     prevSource = source;
@@ -4854,6 +4882,8 @@ function createRemendPreprocessor(options) {
   hasLoneSurrogate,
   highlight,
   isFootnoteSection,
+  isWhitespaceText,
+  lastMeaningfulIdx,
   measureStage,
   mergeClassNameAllowlist,
   normalizeForMatch,