@ai-react-markdown/engine 2.4.2 → 2.4.5

package/dist/index.js

@@ -176,6 +176,11 @@ var defaultUrlTransform = (value) => {
 import { htmlBlockNames } from "micromark-util-html-tag-name";
 import { normalizeIdentifier } from "micromark-util-normalize-identifier";
 var TYPE6_NAMES = new Set(htmlBlockNames);
+var TABLE_PART_NAMES = /* @__PURE__ */ new Set(["td", "th", "tr", "tbody", "thead", "tfoot", "caption", "col", "colgroup"]);
+var TYPE1_NAMES = /* @__PURE__ */ new Set(["script", "pre", "style", "textarea"]);
+var TYPE6_START_RE = /^<\/?([A-Za-z][A-Za-z0-9-]*)(?:[ \t\r]|\/?>|$)/;
+var TYPE1_START_RE = /^<(script|pre|style|textarea)(?:[ \t\r]|>|$)/i;
+var TYPE7_LINE_RE = /^<(\/?)([A-Za-z][A-Za-z0-9-]*)(?:[ \t\r][^>]*|\/)?>[ \t\r]*$/;
 var VOID_TAGS = /* @__PURE__ */ new Set([
   "area",
   "base",
@@ -314,7 +319,11 @@ function freshCheckpoint(defListEnabled, mathFlow, referenceTaint) {
     htmlFlowSinceBlank: false,
     htmlSeamPending: false,
     phasePoisonedAt: Infinity,
-    pendingTruncatedTags: []
+    pendingTruncatedTags: [],
+    pendingTruncatedCloses: [],
+    tagAcrossLines: false,
+    tagAcrossLinesIndent: 0,
+    htmlFlowReal: false
   };
 }
 function isPlausibleLinkDefRest(rest) {
@@ -611,6 +620,8 @@ function processConfirmedLine(cp, ln, text) {
       for (const tag of cp.pendingTruncatedTags) applyTag(tag, true);
       cp.pendingTruncatedTags = [];
     }
+    cp.pendingTruncatedCloses = [];
+    cp.tagAcrossLines = false;
     cp.blankRun += 1;
     cp.lastBlankStart = ln.start;
     cp.candidates.push({
@@ -624,6 +635,7 @@ function processConfirmedLine(cp, ln, text) {
     cp.paragraphHasUnpairedRun = false;
     cp.openBracket = null;
     cp.htmlFlowSinceBlank = false;
+    cp.htmlFlowReal = false;
     cp.prevLineBlank = true;
     cp.prevLineWasText = false;
     cp.prevLineWasValidDef = false;
@@ -639,6 +651,16 @@ function processConfirmedLine(cp, ln, text) {
   if (tagStart) {
     cp.htmlFlowSinceBlank = true;
     if (!TYPE6_NAMES.has(tagStart[1].toLowerCase())) cp.hazardVerdict = true;
+    if (!cp.htmlFlowReal) {
+      const t = mdTrimStart(ln.text);
+      const t6 = TYPE6_START_RE.exec(t);
+      const t7 = TYPE7_LINE_RE.exec(t);
+      if (t6 !== null && TYPE6_NAMES.has(t6[1].toLowerCase()) || TYPE1_START_RE.test(t) || // Type 7 cannot interrupt a paragraph, and excludes the raw-text
+      // names (those are type 1 as start tags, paragraph as end tags).
+      t7 !== null && !cp.prevLineWasText && !TYPE1_NAMES.has(t7[2].toLowerCase())) {
+        cp.htmlFlowReal = true;
+      }
+    }
   }
   const inRawText = cp.htmlFlowSinceBlank || rawOpenAtLineStart;
   const rawFlowStart = ln.indent <= 3 && /^<(?:!--|\?|![A-Za-z]|!\[CDATA\[)/.test(mdTrimStart(ln.text));
@@ -779,7 +801,21 @@ ${cont(scanText)}` };
   for (const [from, to] of rawSpans) {
     tagText = tagText.slice(0, from) + " ".repeat(to - from) + tagText.slice(to);
   }
-  {
+  let skipTagScan = false;
+  if (cp.tagAcrossLines) {
+    if (ln.indent < cp.tagAcrossLinesIndent) poisonRawDivergence();
+    const gt = ln.text.indexOf(">");
+    if (gt === -1) {
+      skipTagScan = true;
+    } else {
+      if (/["']/.test(ln.text.slice(0, gt))) poisonRawDivergence();
+      for (const tag of cp.pendingTruncatedCloses) applyTag(tag, true);
+      cp.pendingTruncatedCloses = [];
+      cp.tagAcrossLines = false;
+      tagText = " ".repeat(gt + 1) + tagText.slice(gt + 1);
+    }
+  }
+  if (!skipTagScan) {
     TAG_OR_COMMENT_RE.lastIndex = 0;
     let m;
     let lastCommentOpenerIdx = -1;
@@ -809,6 +845,7 @@ ${cont(scanText)}` };
       if (cp.commentOpen) continue;
       const closing = m[1] === "/";
       const tag = m[2].toLowerCase();
+      if (TABLE_PART_NAMES.has(tag)) cp.phasePoisonedAt = Math.min(cp.phasePoisonedAt, ln.start + m.index);
       const selfClosing = m[3] !== void 0 && /\/\s*$/.test(m[3]);
       if (VOID_TAGS.has(tag) || selfClosing) continue;
       applyTag(tag, closing);
@@ -829,6 +866,7 @@ ${cont(scanText)}` };
         if (startMasked || wholeVisible || inRaw(mr.index) || cp.commentOpen) continue;
         const closing = mr[1] === "/";
         const tag = mr[2].toLowerCase();
+        if (TABLE_PART_NAMES.has(tag)) cp.phasePoisonedAt = Math.min(cp.phasePoisonedAt, ln.start + mr.index);
         const selfClosing = mr[3] !== void 0 && /\/\s*$/.test(mr[3]);
         if (VOID_TAGS.has(tag) || selfClosing) continue;
         applyTag(tag, closing);
@@ -844,7 +882,14 @@ ${cont(scanText)}` };
         if (m2) {
           const closing = m2[1] === "/";
           const tag = m2[2].toLowerCase();
-          if (!VOID_TAGS.has(tag)) {
+          if (TABLE_PART_NAMES.has(tag)) cp.phasePoisonedAt = Math.min(cp.phasePoisonedAt, ln.start + lastLt);
+          if (cp.htmlFlowReal) {
+            cp.tagAcrossLines = true;
+            cp.tagAcrossLinesIndent = ln.indent;
+          }
+          if (closing) {
+            if (!VOID_TAGS.has(tag) && cp.htmlFlowReal) cp.pendingTruncatedCloses.push(tag);
+          } else if (!VOID_TAGS.has(tag)) {
             applyTag(tag, closing);
             const rawLastLt = ln.text.lastIndexOf("<");
             const rawTruncated = rawLastLt !== -1 && !ln.text.includes(">", rawLastLt);
@@ -1056,6 +1101,8 @@ function rebaseDualWalk(node, segments, maxEnd, offsetDelta, lineDelta) {
     for (const child of children) rebaseDualWalk(child, segments, maxEnd, offsetDelta, lineDelta);
   }
 }
+var TABLE_PART_TAG_RE = /<(?:td|th|tr|tbody|thead|tfoot|caption|col|colgroup)\b/i;
+var STRAY_SYNTHESIZED_END_TAG_RE = /<\/(?:br|p)\b/i;
 function spliceTrees(input) {
   const { prevMdast, prevHast, tailMdast, tailHast, content, boundary, injectionPrefix, injectedSegments } = input;
   const injectedLen = injectionPrefix.length;
@@ -1112,6 +1159,12 @@ function spliceTrees(input) {
     return !(start !== void 0 && start < injectedLen);
   });
   const tailWrapVisible = tailMdastChildren.some((child) => !isWrapInvisible(child));
+  if (prefixMdast.some((c) => c.type === "html" && TABLE_PART_TAG_RE.test(c.value))) return null;
+  for (const child of tailMdastChildren) {
+    if (isWrapInvisible(child)) continue;
+    if (child.type !== "html") break;
+    if (STRAY_SYNTHESIZED_END_TAG_RE.test(child.value) || TABLE_PART_TAG_RE.test(child.value)) return null;
+  }
   const aligned = alignPrefixCut(prefixMdast, cutRegion, tailWrapVisible);
   if (aligned === null) return null;
   const hastChildren = aligned.children;
@@ -1237,6 +1290,9 @@ function alignPrefixCut(prefixMdast, cutRegion, tailWrapVisible) {
     return null;
   }
   const lastIsLiteral = last !== void 0 && last.type === "text" && last.value.trim() !== "";
+  if (lastIsLiteral && pairIdx >= 0 && visibles[pairIdx].type !== "html") {
+    return null;
+  }
   const litOwnerEnd = pairIdx >= 0 ? visibles[pairIdx].position?.end?.offset : void 0;
   const litEnd = lastIsLiteral ? last.position?.end?.offset : void 0;
   if (lastIsLiteral && last.position !== void 0 && (litEnd === void 0 || litOwnerEnd === void 0)) {
@@ -1325,8 +1381,9 @@ function stripInjectedHast(tailMdast, tailHast, injectedLen, tailWrapVisible) {
 }
 function tailLeadingTextIsHoist(tailMdastChildren, tailHastChildren) {
   const firstText = tailHastChildren[0];
-  if (!firstText) return false;
   const firstVisible = tailMdastChildren.find((c) => !isWrapInvisible(c));
+  if (firstVisible?.type === "html" && STRAY_SYNTHESIZED_END_TAG_RE.test(firstVisible.value)) return null;
+  if (!firstText) return false;
   if (!isSeparatorText(firstText)) {
     if (firstText.type === "text" && firstText.position === void 0 && firstVisible?.type === "html") {
       if (/^\s*<\/[A-Za-z][A-Za-z0-9-]*\s*>/.test(firstVisible.value)) return true;
@@ -1375,6 +1432,9 @@ function countTrailingNewlines(value) {
 function countNewlines(text, end = text.length) {
   let count = 0;
   for (let i = text.indexOf("\n"); i !== -1 && i < end; i = text.indexOf("\n", i + 1)) count += 1;
+  for (let i = text.indexOf("\r"); i !== -1 && i < end; i = text.indexOf("\r", i + 1)) {
+    if (text.charCodeAt(i + 1) !== 10) count += 1;
+  }
   return count;
 }
 
@@ -1494,7 +1554,7 @@ function normalizeId(s) {
   return normalizeIdentifier3(s);
 }
 function normalizeForMatch(s) {
-  return normalizeIdentifier3(s.replace(/\\(.)/g, "$1"));
+  return normalizeIdentifier3(s.replace(/\\([!-/:-@[-`{-~])/g, "$1"));
 }
 
 // src/components/collectDefLabels.ts
@@ -1767,17 +1827,8 @@ function phantomSuffixCloser(content) {
 }
 
 // src/components/extractContributions.ts
-function fakeAnchorElement(url) {
-  return { type: "element", tagName: "a", properties: { href: url }, children: [] };
-}
-function sanitizeDefUrl(url, urlTransform) {
-  if (!urlTransform) return url;
-  const result = urlTransform(url, "href", fakeAnchorElement(url));
-  return result == null ? "" : String(result);
-}
 function* extractContributions(mdast, options = {}) {
   const phantomFn = options.phantomFootnoteLabels;
-  const urlTransform = options.urlTransform;
   const out = [];
   visit3(mdast, (n) => {
     if (n.type === "footnoteReference") {
@@ -1805,7 +1856,7 @@ function* extractContributions(mdast, options = {}) {
       out.push({
         kind: "linkDef",
         label: normalizeId(d.identifier),
-        url: sanitizeDefUrl(d.url, urlTransform),
+        url: d.url,
         title: d.title
       });
     }
@@ -3519,8 +3570,7 @@ function isProtocolAllowed(url, allowed) {
 }
 function sanitizeCrossChunkUrl(rawUrl, key, tagName, urlTransform, schema) {
   rawUrl = normalizeUri2(rawUrl);
-  const callerProtocols = schema.protocols;
-  const allowed = callerProtocols === void 0 || callerProtocols === null ? sanitizeSchema.protocols?.[key] : callerProtocols[key];
+  const allowed = Object.hasOwn(schema, "protocols") ? schema.protocols?.[key] : sanitizeSchema.protocols?.[key];
   if (allowed && allowed.length > 0 && !isProtocolAllowed(rawUrl, allowed)) return null;
   const transformed = urlTransform(rawUrl, key, fakeElement(tagName, key, rawUrl));
   if (transformed == null) return null;
@@ -3918,23 +3968,28 @@ function lineHasBacktick(content, pos) {
   }
   return false;
 }
-function isAtLineStart(content, pos) {
+function lineIndentBefore(content, pos) {
   let i = pos - 1;
-  let spaces = 0;
-  while (i >= 0 && content[i] === " ") {
-    spaces++;
-    if (spaces > 3) return false;
+  let indent = 0;
+  while (i >= 0 && (content[i] === " " || content[i] === "	")) {
+    indent += content[i] === "	" ? 4 : 1;
     i--;
   }
-  return i < 0 || content[i] === "\n" || content[i] === "\r";
+  return i < 0 || content[i] === "\n" || content[i] === "\r" ? indent : -1;
 }
 function findClosingBacktickRun(content, start, n) {
   let i = start;
   while (i < content.length) {
-    if (content[i] === "`") {
+    const ch = content[i];
+    if (ch === "`") {
       const runLen = getRepeatedMarkerLength(content, i, "`");
       if (runLen === n) return i;
       i += runLen;
+    } else if (ch === "\n") {
+      let j = i + 1;
+      while (j < content.length && (content[j] === " " || content[j] === "	" || content[j] === "\r")) j += 1;
+      if (j >= content.length || content[j] === "\n") return -1;
+      i += 1;
     } else {
       i += 1;
     }
@@ -3947,6 +4002,7 @@ function splitByProtectedRegions(content) {
   let multilineStart = -1;
   let multilineFenceMarker = null;
   let multilineFenceLength = 0;
+  let multilineFenceIndent = 0;
   function pushProtected(start, end) {
     if (start > lastIndex) {
       segments.push({ text: content.substring(lastIndex, start), isCode: false });
@@ -3960,7 +4016,8 @@ function splitByProtectedRegions(content) {
     if (multilineStart !== -1) {
       if (char === multilineFenceMarker) {
         const runLen = getRepeatedMarkerLength(content, i, multilineFenceMarker);
-        if (runLen >= multilineFenceLength && isAtLineStart(content, i) && restOfLineIsBlank(content, i + runLen)) {
+        const closerIndent = lineIndentBefore(content, i);
+        if (runLen >= multilineFenceLength && closerIndent !== -1 && closerIndent <= multilineFenceIndent + 3 && restOfLineIsBlank(content, i + runLen)) {
           pushProtected(multilineStart, i + runLen);
           multilineStart = -1;
           multilineFenceMarker = null;
@@ -3976,10 +4033,12 @@ function splitByProtectedRegions(content) {
     }
     if (char === "`" || char === "~") {
       const runLen = getRepeatedMarkerLength(content, i, char);
-      if (runLen >= 3 && isAtLineStart(content, i) && !(char === "`" && lineHasBacktick(content, i + runLen))) {
+      const openerIndent = lineIndentBefore(content, i);
+      if (runLen >= 3 && openerIndent !== -1 && !(char === "`" && lineHasBacktick(content, i + runLen))) {
         multilineStart = i;
         multilineFenceMarker = char;
         multilineFenceLength = runLen;
+        multilineFenceIndent = openerIndent;
         i += runLen;
         continue;
       }
@@ -4052,16 +4111,20 @@ function escapeCurrencyDollarSigns(text) {
       currentLineProcessed += segment;
     }
     let needEscape = true;
-    let restBeforeNextMatchOrEnd = "";
-    if (i < currencyMatches.length - 1) {
-      const nextMatch = currencyMatches[i + 1];
-      if (nextMatch.index - match.index > 1) {
-        restBeforeNextMatchOrEnd = text.substring(match.index + 1, nextMatch.index);
+    const restStart = match.index + 1;
+    const restEnd = i < currencyMatches.length - 1 ? currencyMatches[i + 1].index : text.length;
+    let firstLineBeforeNextMatch = "";
+    if (restEnd - restStart > 0) {
+      let eol = restEnd;
+      for (let k = restStart; k < restEnd; k++) {
+        const c = text.charCodeAt(k);
+        if (c === 10 || c === 13) {
+          eol = k;
+          break;
+        }
       }
-    } else {
-      restBeforeNextMatchOrEnd = text.substring(match.index + 1);
+      firstLineBeforeNextMatch = text.substring(restStart, eol);
     }
-    const firstLineBeforeNextMatch = restBeforeNextMatchOrEnd.split(/\r\n|\r|\n/g)[0];
     if (Array.from(firstLineBeforeNextMatch.matchAll(NO_ESCAPED_DOLLAR_REGEX)).length % 2 !== 0) {
       const wholeLineBeforeNextMatchWithoutCurrentDollar = currentLineProcessed + firstLineBeforeNextMatch;
       if (Array.from(wholeLineBeforeNextMatchWithoutCurrentDollar.matchAll(NO_ESCAPED_DOLLAR_REGEX)).length % 2 !== 0) {
@@ -4227,39 +4290,49 @@ function hasUnclosedTextCommand(text) {
   return false;
 }
 var RESIDUAL_OPEN_BRACKET_RE = /(?<!!)\\\[/;
-function processSliceInstrumented(slice) {
+var LEADING_DOUBLE_DOLLAR_RE = /^\s*\$\$/;
+function processSliceInstrumented(slice, probe = true) {
   const segments = splitByProtectedRegions(slice);
-  let out = "";
+  const parts = [];
   let quiescent = true;
   let truncatedAtSeamStart = false;
   for (let index = 0; index < segments.length; index++) {
     const segment = segments[index];
     if (segment.isCode) {
-      out += segment.text;
+      parts.push(segment.text);
       continue;
     }
     let text = segment.text;
     text = escapeMhchemCommands(text);
     text = escapeCurrencyDollarSigns(text);
     text = convertLatexDelimiters(text);
-    if (RESIDUAL_OPEN_BRACKET_RE.test(text)) quiescent = false;
+    if (probe && RESIDUAL_OPEN_BRACKET_RE.test(text)) quiescent = false;
     text = escapeLatexPipes(text);
-    if (findUnclosedDelimiterStart(text, "both") !== -1) quiescent = false;
+    if (probe && findUnclosedDelimiterStart(text, "both") !== -1) quiescent = false;
     text = escapeLatexPipesInUnclosed(text);
-    if (hasUnclosedTextCommand(text)) quiescent = false;
+    if (probe && hasUnclosedTextCommand(text)) quiescent = false;
     text = escapeTextUnderscores(text);
     text = convertSingleToDoubleDollar(text);
-    const unclosedDouble = findUnclosedDelimiterStart(text, "double-only");
-    if (unclosedDouble !== -1) {
-      quiescent = false;
-      if (index === 0 && text.slice(0, unclosedDouble).trim() === "") {
-        truncatedAtSeamStart = true;
+    if (probe || index === 0 && LEADING_DOUBLE_DOLLAR_RE.test(text)) {
+      const unclosedDouble = findUnclosedDelimiterStart(text, "double-only");
+      if (unclosedDouble !== -1) {
+        quiescent = false;
+        if (index === 0 && text.slice(0, unclosedDouble).trim() === "") {
+          truncatedAtSeamStart = true;
+        }
       }
     }
     text = truncateUnclosedLatexBlock(text);
-    out += text;
+    parts.push(text);
   }
-  return { out, quiescent, truncatedAtSeamStart };
+  return { out: parts.join(""), quiescent, truncatedAtSeamStart };
+}
+function isBlankRawLine(text, from, to) {
+  for (let i = from; i < to; i++) {
+    const c = text.charCodeAt(i);
+    if (c !== 32 && c !== 9 && c !== 13) return false;
+  }
+  return true;
 }
 function findRawSafeCut(active) {
   const segments = splitByProtectedRegions(active);
@@ -4274,10 +4347,12 @@ function findRawSafeCut(active) {
       continue;
     }
     const text = segment.text;
+    let atLineStart = offset === 0 || active.charCodeAt(offset - 1) === 10;
     let lineStart = 0;
     while (lineStart <= text.length) {
       const nl = text.indexOf("\n", lineStart);
       const lineEnd = nl === -1 ? text.length : nl;
+      if (atLineStart && nl !== -1 && isBlankRawLine(text, lineStart, lineEnd)) backtickHazard = false;
       for (let i = lineStart; i < lineEnd; i++) {
         const ch = text[i];
         if (ch === "`") backtickHazard = true;
@@ -4290,6 +4365,7 @@ function findRawSafeCut(active) {
       if (nl === -1) break;
       if (!backtickHazard && !latentLt) lastCut = offset + nl + 1;
       lineStart = nl + 1;
+      atLineStart = true;
     }
     offset += text.length;
   }
@@ -4298,11 +4374,14 @@ function findRawSafeCut(active) {
 var DEFAULT_FREEZE_ATTEMPT_THRESHOLD = 512;
 function createIncrementalLatexPreprocessor(options) {
   const freezeThreshold = options?.freezeThreshold ?? DEFAULT_FREEZE_ATTEMPT_THRESHOLD;
+  const onAttempt = options?.onAttempt;
+  const backoff = options?.backoff ?? true;
   let prevSource = "";
   let prevOutput = "";
   let frozenSrcEnd = 0;
   let frozenOut = "";
   let triggered = false;
+  let nextAttemptLen = 0;
   return function incrementalPreprocessLaTeX(source) {
     if (source === prevSource) return prevOutput;
     const isAppend = source.length > prevSource.length && source.startsWith(prevSource);
@@ -4310,6 +4389,7 @@ function createIncrementalLatexPreprocessor(options) {
       frozenSrcEnd = 0;
       frozenOut = "";
       triggered = false;
+      nextAttemptLen = 0;
     }
     if (!triggered) {
       const checkFrom = isAppend ? Math.max(0, prevSource.length - 1) : 0;
@@ -4321,18 +4401,26 @@ function createIncrementalLatexPreprocessor(options) {
       triggered = true;
     }
     let active = source.slice(frozenSrcEnd);
-    if (active.length > freezeThreshold) {
+    if (active.length > freezeThreshold && active.length >= nextAttemptLen) {
+      const activeLength = active.length;
+      let advanced = false;
+      let frozenBytes = 0;
+      const freeze = (cut2, slice) => {
+        frozenOut += slice.out;
+        frozenSrcEnd += cut2;
+        active = source.slice(frozenSrcEnd);
+        advanced = true;
+        frozenBytes = cut2;
+      };
       const cut = findRawSafeCut(active);
       if (cut > 0) {
         const candidate = processSliceInstrumented(active.slice(0, cut));
-        if (candidate.quiescent) {
-          frozenOut += candidate.out;
-          frozenSrcEnd += cut;
-          active = source.slice(frozenSrcEnd);
-        }
+        if (candidate.quiescent) freeze(cut, candidate);
       }
+      nextAttemptLen = advanced || !backoff ? 0 : active.length * 2;
+      onAttempt?.({ activeLength, frozenBytes });
     }
-    const tail = processSliceInstrumented(active);
+    const tail = processSliceInstrumented(active, false);
     const head = tail.truncatedAtSeamStart ? frozenOut.replace(/\s+$/, "") : frozenOut;
     const out = head + tail.out;
     prevSource = source;