@ai-react-markdown/engine 2.4.3 → 2.4.5

package/dist/index.js

@@ -176,6 +176,11 @@ var defaultUrlTransform = (value) => {
 import { htmlBlockNames } from "micromark-util-html-tag-name";
 import { normalizeIdentifier } from "micromark-util-normalize-identifier";
 var TYPE6_NAMES = new Set(htmlBlockNames);
+var TABLE_PART_NAMES = /* @__PURE__ */ new Set(["td", "th", "tr", "tbody", "thead", "tfoot", "caption", "col", "colgroup"]);
+var TYPE1_NAMES = /* @__PURE__ */ new Set(["script", "pre", "style", "textarea"]);
+var TYPE6_START_RE = /^<\/?([A-Za-z][A-Za-z0-9-]*)(?:[ \t\r]|\/?>|$)/;
+var TYPE1_START_RE = /^<(script|pre|style|textarea)(?:[ \t\r]|>|$)/i;
+var TYPE7_LINE_RE = /^<(\/?)([A-Za-z][A-Za-z0-9-]*)(?:[ \t\r][^>]*|\/)?>[ \t\r]*$/;
 var VOID_TAGS = /* @__PURE__ */ new Set([
   "area",
   "base",
@@ -314,7 +319,11 @@ function freshCheckpoint(defListEnabled, mathFlow, referenceTaint) {
     htmlFlowSinceBlank: false,
     htmlSeamPending: false,
     phasePoisonedAt: Infinity,
-    pendingTruncatedTags: []
+    pendingTruncatedTags: [],
+    pendingTruncatedCloses: [],
+    tagAcrossLines: false,
+    tagAcrossLinesIndent: 0,
+    htmlFlowReal: false
   };
 }
 function isPlausibleLinkDefRest(rest) {
@@ -611,6 +620,8 @@ function processConfirmedLine(cp, ln, text) {
       for (const tag of cp.pendingTruncatedTags) applyTag(tag, true);
       cp.pendingTruncatedTags = [];
     }
+    cp.pendingTruncatedCloses = [];
+    cp.tagAcrossLines = false;
     cp.blankRun += 1;
     cp.lastBlankStart = ln.start;
     cp.candidates.push({
@@ -624,6 +635,7 @@ function processConfirmedLine(cp, ln, text) {
     cp.paragraphHasUnpairedRun = false;
     cp.openBracket = null;
     cp.htmlFlowSinceBlank = false;
+    cp.htmlFlowReal = false;
     cp.prevLineBlank = true;
     cp.prevLineWasText = false;
     cp.prevLineWasValidDef = false;
@@ -639,6 +651,16 @@ function processConfirmedLine(cp, ln, text) {
   if (tagStart) {
     cp.htmlFlowSinceBlank = true;
     if (!TYPE6_NAMES.has(tagStart[1].toLowerCase())) cp.hazardVerdict = true;
+    if (!cp.htmlFlowReal) {
+      const t = mdTrimStart(ln.text);
+      const t6 = TYPE6_START_RE.exec(t);
+      const t7 = TYPE7_LINE_RE.exec(t);
+      if (t6 !== null && TYPE6_NAMES.has(t6[1].toLowerCase()) || TYPE1_START_RE.test(t) || // Type 7 cannot interrupt a paragraph, and excludes the raw-text
+      // names (those are type 1 as start tags, paragraph as end tags).
+      t7 !== null && !cp.prevLineWasText && !TYPE1_NAMES.has(t7[2].toLowerCase())) {
+        cp.htmlFlowReal = true;
+      }
+    }
   }
   const inRawText = cp.htmlFlowSinceBlank || rawOpenAtLineStart;
   const rawFlowStart = ln.indent <= 3 && /^<(?:!--|\?|![A-Za-z]|!\[CDATA\[)/.test(mdTrimStart(ln.text));
@@ -779,7 +801,21 @@ ${cont(scanText)}` };
   for (const [from, to] of rawSpans) {
     tagText = tagText.slice(0, from) + " ".repeat(to - from) + tagText.slice(to);
   }
-  {
+  let skipTagScan = false;
+  if (cp.tagAcrossLines) {
+    if (ln.indent < cp.tagAcrossLinesIndent) poisonRawDivergence();
+    const gt = ln.text.indexOf(">");
+    if (gt === -1) {
+      skipTagScan = true;
+    } else {
+      if (/["']/.test(ln.text.slice(0, gt))) poisonRawDivergence();
+      for (const tag of cp.pendingTruncatedCloses) applyTag(tag, true);
+      cp.pendingTruncatedCloses = [];
+      cp.tagAcrossLines = false;
+      tagText = " ".repeat(gt + 1) + tagText.slice(gt + 1);
+    }
+  }
+  if (!skipTagScan) {
     TAG_OR_COMMENT_RE.lastIndex = 0;
     let m;
     let lastCommentOpenerIdx = -1;
@@ -809,6 +845,7 @@ ${cont(scanText)}` };
       if (cp.commentOpen) continue;
       const closing = m[1] === "/";
       const tag = m[2].toLowerCase();
+      if (TABLE_PART_NAMES.has(tag)) cp.phasePoisonedAt = Math.min(cp.phasePoisonedAt, ln.start + m.index);
       const selfClosing = m[3] !== void 0 && /\/\s*$/.test(m[3]);
       if (VOID_TAGS.has(tag) || selfClosing) continue;
       applyTag(tag, closing);
@@ -829,6 +866,7 @@ ${cont(scanText)}` };
         if (startMasked || wholeVisible || inRaw(mr.index) || cp.commentOpen) continue;
         const closing = mr[1] === "/";
         const tag = mr[2].toLowerCase();
+        if (TABLE_PART_NAMES.has(tag)) cp.phasePoisonedAt = Math.min(cp.phasePoisonedAt, ln.start + mr.index);
         const selfClosing = mr[3] !== void 0 && /\/\s*$/.test(mr[3]);
         if (VOID_TAGS.has(tag) || selfClosing) continue;
         applyTag(tag, closing);
@@ -844,7 +882,14 @@ ${cont(scanText)}` };
         if (m2) {
           const closing = m2[1] === "/";
           const tag = m2[2].toLowerCase();
-          if (!VOID_TAGS.has(tag)) {
+          if (TABLE_PART_NAMES.has(tag)) cp.phasePoisonedAt = Math.min(cp.phasePoisonedAt, ln.start + lastLt);
+          if (cp.htmlFlowReal) {
+            cp.tagAcrossLines = true;
+            cp.tagAcrossLinesIndent = ln.indent;
+          }
+          if (closing) {
+            if (!VOID_TAGS.has(tag) && cp.htmlFlowReal) cp.pendingTruncatedCloses.push(tag);
+          } else if (!VOID_TAGS.has(tag)) {
             applyTag(tag, closing);
             const rawLastLt = ln.text.lastIndexOf("<");
             const rawTruncated = rawLastLt !== -1 && !ln.text.includes(">", rawLastLt);
@@ -1387,6 +1432,9 @@ function countTrailingNewlines(value) {
 function countNewlines(text, end = text.length) {
   let count = 0;
   for (let i = text.indexOf("\n"); i !== -1 && i < end; i = text.indexOf("\n", i + 1)) count += 1;
+  for (let i = text.indexOf("\r"); i !== -1 && i < end; i = text.indexOf("\r", i + 1)) {
+    if (text.charCodeAt(i + 1) !== 10) count += 1;
+  }
   return count;
 }
 
@@ -1506,7 +1554,7 @@ function normalizeId(s) {
   return normalizeIdentifier3(s);
 }
 function normalizeForMatch(s) {
-  return normalizeIdentifier3(s.replace(/\\(.)/g, "$1"));
+  return normalizeIdentifier3(s.replace(/\\([!-/:-@[-`{-~])/g, "$1"));
 }
 
 // src/components/collectDefLabels.ts
@@ -3932,10 +3980,16 @@ function lineIndentBefore(content, pos) {
 function findClosingBacktickRun(content, start, n) {
   let i = start;
   while (i < content.length) {
-    if (content[i] === "`") {
+    const ch = content[i];
+    if (ch === "`") {
       const runLen = getRepeatedMarkerLength(content, i, "`");
       if (runLen === n) return i;
       i += runLen;
+    } else if (ch === "\n") {
+      let j = i + 1;
+      while (j < content.length && (content[j] === " " || content[j] === "	" || content[j] === "\r")) j += 1;
+      if (j >= content.length || content[j] === "\n") return -1;
+      i += 1;
     } else {
       i += 1;
     }
@@ -4057,16 +4111,20 @@ function escapeCurrencyDollarSigns(text) {
       currentLineProcessed += segment;
     }
     let needEscape = true;
-    let restBeforeNextMatchOrEnd = "";
-    if (i < currencyMatches.length - 1) {
-      const nextMatch = currencyMatches[i + 1];
-      if (nextMatch.index - match.index > 1) {
-        restBeforeNextMatchOrEnd = text.substring(match.index + 1, nextMatch.index);
+    const restStart = match.index + 1;
+    const restEnd = i < currencyMatches.length - 1 ? currencyMatches[i + 1].index : text.length;
+    let firstLineBeforeNextMatch = "";
+    if (restEnd - restStart > 0) {
+      let eol = restEnd;
+      for (let k = restStart; k < restEnd; k++) {
+        const c = text.charCodeAt(k);
+        if (c === 10 || c === 13) {
+          eol = k;
+          break;
+        }
       }
-    } else {
-      restBeforeNextMatchOrEnd = text.substring(match.index + 1);
+      firstLineBeforeNextMatch = text.substring(restStart, eol);
     }
-    const firstLineBeforeNextMatch = restBeforeNextMatchOrEnd.split(/\r\n|\r|\n/g)[0];
     if (Array.from(firstLineBeforeNextMatch.matchAll(NO_ESCAPED_DOLLAR_REGEX)).length % 2 !== 0) {
       const wholeLineBeforeNextMatchWithoutCurrentDollar = currentLineProcessed + firstLineBeforeNextMatch;
       if (Array.from(wholeLineBeforeNextMatchWithoutCurrentDollar.matchAll(NO_ESCAPED_DOLLAR_REGEX)).length % 2 !== 0) {
@@ -4232,39 +4290,49 @@ function hasUnclosedTextCommand(text) {
   return false;
 }
 var RESIDUAL_OPEN_BRACKET_RE = /(?<!!)\\\[/;
-function processSliceInstrumented(slice) {
+var LEADING_DOUBLE_DOLLAR_RE = /^\s*\$\$/;
+function processSliceInstrumented(slice, probe = true) {
   const segments = splitByProtectedRegions(slice);
-  let out = "";
+  const parts = [];
   let quiescent = true;
   let truncatedAtSeamStart = false;
   for (let index = 0; index < segments.length; index++) {
     const segment = segments[index];
     if (segment.isCode) {
-      out += segment.text;
+      parts.push(segment.text);
       continue;
     }
     let text = segment.text;
     text = escapeMhchemCommands(text);
     text = escapeCurrencyDollarSigns(text);
     text = convertLatexDelimiters(text);
-    if (RESIDUAL_OPEN_BRACKET_RE.test(text)) quiescent = false;
+    if (probe && RESIDUAL_OPEN_BRACKET_RE.test(text)) quiescent = false;
     text = escapeLatexPipes(text);
-    if (findUnclosedDelimiterStart(text, "both") !== -1) quiescent = false;
+    if (probe && findUnclosedDelimiterStart(text, "both") !== -1) quiescent = false;
     text = escapeLatexPipesInUnclosed(text);
-    if (hasUnclosedTextCommand(text)) quiescent = false;
+    if (probe && hasUnclosedTextCommand(text)) quiescent = false;
     text = escapeTextUnderscores(text);
     text = convertSingleToDoubleDollar(text);
-    const unclosedDouble = findUnclosedDelimiterStart(text, "double-only");
-    if (unclosedDouble !== -1) {
-      quiescent = false;
-      if (index === 0 && text.slice(0, unclosedDouble).trim() === "") {
-        truncatedAtSeamStart = true;
+    if (probe || index === 0 && LEADING_DOUBLE_DOLLAR_RE.test(text)) {
+      const unclosedDouble = findUnclosedDelimiterStart(text, "double-only");
+      if (unclosedDouble !== -1) {
+        quiescent = false;
+        if (index === 0 && text.slice(0, unclosedDouble).trim() === "") {
+          truncatedAtSeamStart = true;
+        }
       }
     }
     text = truncateUnclosedLatexBlock(text);
-    out += text;
+    parts.push(text);
   }
-  return { out, quiescent, truncatedAtSeamStart };
+  return { out: parts.join(""), quiescent, truncatedAtSeamStart };
+}
+function isBlankRawLine(text, from, to) {
+  for (let i = from; i < to; i++) {
+    const c = text.charCodeAt(i);
+    if (c !== 32 && c !== 9 && c !== 13) return false;
+  }
+  return true;
 }
 function findRawSafeCut(active) {
   const segments = splitByProtectedRegions(active);
@@ -4279,10 +4347,12 @@ function findRawSafeCut(active) {
       continue;
     }
     const text = segment.text;
+    let atLineStart = offset === 0 || active.charCodeAt(offset - 1) === 10;
     let lineStart = 0;
     while (lineStart <= text.length) {
       const nl = text.indexOf("\n", lineStart);
       const lineEnd = nl === -1 ? text.length : nl;
+      if (atLineStart && nl !== -1 && isBlankRawLine(text, lineStart, lineEnd)) backtickHazard = false;
       for (let i = lineStart; i < lineEnd; i++) {
         const ch = text[i];
         if (ch === "`") backtickHazard = true;
@@ -4295,6 +4365,7 @@ function findRawSafeCut(active) {
       if (nl === -1) break;
       if (!backtickHazard && !latentLt) lastCut = offset + nl + 1;
       lineStart = nl + 1;
+      atLineStart = true;
     }
     offset += text.length;
   }
@@ -4303,11 +4374,14 @@ function findRawSafeCut(active) {
 var DEFAULT_FREEZE_ATTEMPT_THRESHOLD = 512;
 function createIncrementalLatexPreprocessor(options) {
   const freezeThreshold = options?.freezeThreshold ?? DEFAULT_FREEZE_ATTEMPT_THRESHOLD;
+  const onAttempt = options?.onAttempt;
+  const backoff = options?.backoff ?? true;
   let prevSource = "";
   let prevOutput = "";
   let frozenSrcEnd = 0;
   let frozenOut = "";
   let triggered = false;
+  let nextAttemptLen = 0;
   return function incrementalPreprocessLaTeX(source) {
     if (source === prevSource) return prevOutput;
     const isAppend = source.length > prevSource.length && source.startsWith(prevSource);
@@ -4315,6 +4389,7 @@ function createIncrementalLatexPreprocessor(options) {
       frozenSrcEnd = 0;
       frozenOut = "";
       triggered = false;
+      nextAttemptLen = 0;
     }
     if (!triggered) {
       const checkFrom = isAppend ? Math.max(0, prevSource.length - 1) : 0;
@@ -4326,18 +4401,26 @@ function createIncrementalLatexPreprocessor(options) {
       triggered = true;
     }
     let active = source.slice(frozenSrcEnd);
-    if (active.length > freezeThreshold) {
+    if (active.length > freezeThreshold && active.length >= nextAttemptLen) {
+      const activeLength = active.length;
+      let advanced = false;
+      let frozenBytes = 0;
+      const freeze = (cut2, slice) => {
+        frozenOut += slice.out;
+        frozenSrcEnd += cut2;
+        active = source.slice(frozenSrcEnd);
+        advanced = true;
+        frozenBytes = cut2;
+      };
       const cut = findRawSafeCut(active);
       if (cut > 0) {
         const candidate = processSliceInstrumented(active.slice(0, cut));
-        if (candidate.quiescent) {
-          frozenOut += candidate.out;
-          frozenSrcEnd += cut;
-          active = source.slice(frozenSrcEnd);
-        }
+        if (candidate.quiescent) freeze(cut, candidate);
       }
+      nextAttemptLen = advanced || !backoff ? 0 : active.length * 2;
+      onAttempt?.({ activeLength, frozenBytes });
     }
-    const tail = processSliceInstrumented(active);
+    const tail = processSliceInstrumented(active, false);
     const head = tail.truncatedAtSeamStart ? frozenOut.replace(/\s+$/, "") : frozenOut;
     const out = head + tail.out;
     prevSource = source;