@ai-react-markdown/engine 2.3.3 → 2.4.0

package/dist/index.cjs

@@ -72,6 +72,7 @@ __export(src_exports, {
   normalizeId: () => normalizeId,
   pangu: () => pangu,
   parseStage: () => parseStage,
+  phantomSuffixCloser: () => phantomSuffixCloser,
   preprocessAIMDContent: () => preprocessAIMDContent,
   preprocessLaTeX: () => preprocessLaTeX,
   rehypeFooterAdorn: () => rehypeFooterAdorn,
@@ -289,7 +290,7 @@ var DEF_LIST_DD_RE = /^ {0,3}:[ \t]/;
 var DEF_RE = /^ {0,3}\[((?:[^[\]\\]|\\.)+)\]:/;
 var FENCE_RE = /^ {0,3}(`{3,}|~{3,})/;
 var MATH_RUN_RE = /^ {0,3}(\$\$+)/;
-var TAG_OR_COMMENT_RE = /<(\/?)([A-Za-z][A-Za-z0-9-]*)(?=[\s/>])([^>]*)>|<!--|-->/g;
+var TAG_OR_COMMENT_RE = /<(\/?)([A-Za-z][A-Za-z0-9-]*)(?=[\s/>])([^>]*)>|<!--|-->|--!>/g;
 var TRUNCATED_TAG_RE = /<(\/?)([A-Za-z][A-Za-z0-9-]*)([^<>]*)$/;
 var REF_RE = /!?\[((?:[^[\]\\]|\\.)*)\]/g;
 var BACKTICK_RUN_RE = /`+/g;
@@ -371,6 +372,7 @@ function freshCheckpoint(defListEnabled, mathFlow, referenceTaint) {
     fenceLen: 0,
     inMath: false,
     mathFenceLen: 0,
+    openIndent: 0,
     blankRun: 0,
     lastBlankStart: -1,
     hazardVerdict: false,
@@ -381,21 +383,15 @@ function freshCheckpoint(defListEnabled, mathFlow, referenceTaint) {
     paragraphHasUnpairedRun: false,
     htmlFlowSinceBlank: false,
     htmlSeamPending: false,
-    phasePoisonedAt: Infinity
+    phasePoisonedAt: Infinity,
+    pendingTruncatedTags: []
   };
 }
 function isPlausibleLinkDefRest(rest) {
   const t = rest.trim();
   if (t === "") return false;
-  let destEnd;
-  if (t.startsWith("<")) {
-    const close = t.indexOf(">");
-    if (close === -1) return false;
-    destEnd = close + 1;
-  } else {
-    const ws = t.search(/[ \t]/);
-    destEnd = ws === -1 ? t.length : ws;
-  }
+  const destEnd = linkDestinationEnd(t);
+  if (destEnd === -1) return false;
   const after = t.slice(destEnd).trim();
   if (after === "") return true;
   const opener = after[0];
@@ -410,6 +406,39 @@ function isPlausibleLinkDefRest(rest) {
   }
   return false;
 }
+function linkDestinationEnd(t) {
+  if (t.startsWith("<")) {
+    for (let i2 = 1; i2 < t.length; i2++) {
+      const ch = t[i2];
+      if (ch === "\\" && (t[i2 + 1] === "<" || t[i2 + 1] === ">" || t[i2 + 1] === "\\")) {
+        i2 += 1;
+        continue;
+      }
+      if (ch === ">") return i2 + 1;
+      if (ch === "<") return -1;
+    }
+    return -1;
+  }
+  let balance = 0;
+  let i = 0;
+  for (; i < t.length; i++) {
+    const code = t.charCodeAt(i);
+    if (code === 32 || code === 9) break;
+    if (code < 32 || code === 127) return -1;
+    const ch = t[i];
+    if (ch === "\\" && (t[i + 1] === "(" || t[i + 1] === ")" || t[i + 1] === "\\")) {
+      i += 1;
+      continue;
+    }
+    if (ch === "(") balance += 1;
+    else if (ch === ")") {
+      if (balance === 0) break;
+      balance -= 1;
+    }
+  }
+  if (balance !== 0 || i === 0) return -1;
+  return i;
+}
 function classifyBlockStart(text, indent, defListEnabled) {
   if (indent >= 4) return true;
   if (LIST_MARKER_RE.test(text) || FOOTNOTE_DEF_RE.test(text)) return true;
@@ -471,6 +500,48 @@ function computeFreezeBoundary(text, options, resume) {
   }
   return { boundary, checkpoint: cp };
 }
+function floatingResidue(text, commentOpenAtStart) {
+  let out = "";
+  let open = commentOpenAtStart;
+  let last = 0;
+  TAG_OR_COMMENT_RE.lastIndex = 0;
+  let m;
+  while ((m = TAG_OR_COMMENT_RE.exec(text)) !== null) {
+    if (m[0] === "<!--") {
+      const next = text.slice(m.index + 4, m.index + 6);
+      const overlapLen = next.startsWith(">") ? 5 : next === "->" ? 6 : 0;
+      if (open) {
+        if (overlapLen) {
+          open = false;
+          last = m.index + overlapLen;
+          TAG_OR_COMMENT_RE.lastIndex = last;
+        }
+        continue;
+      }
+      out += text.slice(last, m.index);
+      if (overlapLen) {
+        last = m.index + overlapLen;
+        TAG_OR_COMMENT_RE.lastIndex = last;
+        continue;
+      }
+      open = true;
+      continue;
+    }
+    if (m[0] === "-->") {
+      if (open) {
+        open = false;
+        last = m.index + 3;
+      }
+      continue;
+    }
+    if (m[0] === "--!>") continue;
+    if (open) continue;
+    out += text.slice(last, m.index);
+    last = m.index + m[0].length;
+  }
+  if (!open) out += text.slice(last);
+  return out;
+}
 function processConfirmedLine(cp, ln, text) {
   const newest = cp.candidates[cp.candidates.length - 1];
   if (newest && newest.defListSettled === null) {
@@ -525,6 +596,7 @@ function processConfirmedLine(cp, ln, text) {
       cp.inFence = true;
       cp.fenceChar = open[1][0];
       cp.fenceLen = open[1].length;
+      cp.openIndent = ln.indent;
       cp.blankRun = 0;
       cp.paragraphHasUnpairedRun = false;
       cp.prevLineBlank = false;
@@ -559,6 +631,7 @@ function processConfirmedLine(cp, ln, text) {
         }
         cp.inMath = true;
         cp.mathFenceLen = mathRun[1].length;
+        cp.openIndent = ln.indent;
         cp.blankRun = 0;
         cp.paragraphHasUnpairedRun = false;
         cp.prevLineBlank = false;
@@ -569,6 +642,10 @@ function processConfirmedLine(cp, ln, text) {
     }
   }
   if (ln.blank) {
+    if (cp.pendingTruncatedTags.length > 0) {
+      for (const tag of cp.pendingTruncatedTags) applyTag(tag, true);
+      cp.pendingTruncatedTags = [];
+    }
     cp.blankRun += 1;
     cp.lastBlankStart = ln.start;
     cp.candidates.push({
@@ -598,6 +675,7 @@ function processConfirmedLine(cp, ln, text) {
     if (!TYPE6_NAMES.has(tagStart[1].toLowerCase())) cp.hazardVerdict = true;
   }
   const inRawText = cp.htmlFlowSinceBlank || rawOpenAtLineStart;
+  const rawFlowStart = ln.indent <= 3 && /^<(?:!--|\?|![A-Za-z]|!\[CDATA\[)/.test(ln.text.trimStart());
   const { masked, unpaired } = inRawText ? { masked: null, unpaired: false } : maskIntraLineCodeSpans(ln.text, cp.paragraphHasUnpairedRun);
   if (unpaired) cp.paragraphHasUnpairedRun = true;
   const scanText = masked ?? ln.text;
@@ -642,9 +720,14 @@ function processConfirmedLine(cp, ln, text) {
   const rawOpenAtStart = cp.piOpen || cp.declOpen || cp.cdataOpen;
   const rawSpans = [];
   let pos = 0;
+  const poisonRawDivergence = () => {
+    cp.phasePoisonedAt = Math.min(cp.phasePoisonedAt, ln.start);
+  };
   while (pos < scanText.length) {
     if (cp.piOpen) {
       const c = scanText.indexOf("?>", pos);
+      const gt = scanText.indexOf(">", pos);
+      if (gt !== -1 && (c === -1 || gt !== c + 1)) poisonRawDivergence();
       if (c === -1) {
         rawSpans.push([pos, scanText.length]);
         break;
@@ -656,6 +739,8 @@ function processConfirmedLine(cp, ln, text) {
     }
     if (cp.cdataOpen) {
       const c = scanText.indexOf("]]>", pos);
+      const gt = scanText.indexOf(">", pos);
+      if (gt !== -1 && (c === -1 || gt !== c + 2)) poisonRawDivergence();
       if (c === -1) {
         rawSpans.push([pos, scanText.length]);
         break;
@@ -688,6 +773,12 @@ function processConfirmedLine(cp, ln, text) {
       cp.cdataOpen = true;
       pos = cd + 9;
     } else if (first === pi) {
+      if (scanText[pi + 2] === ">") {
+        rawSpans.push([pi, pi + 3]);
+        pos = pi + 3;
+        if (scanText.slice(0, pi).trim() !== "" || ln.indent > 3) poisonRawDivergence();
+        continue;
+      }
       rawSpans.push([pi, pi + 2]);
       cp.piOpen = true;
       pos = pi + 2;
@@ -704,6 +795,15 @@ function processConfirmedLine(cp, ln, text) {
     let lastCommentOpenerIdx = -1;
     while ((m = TAG_OR_COMMENT_RE.exec(scanText)) !== null) {
       if (m[0] === "<!--") {
+        const next = scanText.slice(m.index + 4, m.index + 6);
+        if (cp.commentOpen) {
+          if (next.startsWith(">") || next === "->") cp.commentOpen = false;
+          else if (next === "!>" || next === "-!") poisonRawDivergence();
+          continue;
+        }
+        if (next.startsWith(">") || next === "->") {
+          continue;
+        }
         cp.commentOpen = true;
         lastCommentOpenerIdx = m.index;
         continue;
@@ -712,6 +812,10 @@ function processConfirmedLine(cp, ln, text) {
         cp.commentOpen = false;
         continue;
       }
+      if (m[0] === "--!>") {
+        if (cp.commentOpen) poisonRawDivergence();
+        continue;
+      }
       if (cp.commentOpen) continue;
       const closing = m[1] === "/";
       const tag = m[2].toLowerCase();
@@ -724,6 +828,9 @@ function processConfirmedLine(cp, ln, text) {
         cp.phasePoisonedAt = Math.min(cp.phasePoisonedAt, ln.start + lastCommentOpenerIdx);
       }
     }
+    if (cp.pendingTruncatedTags.length > 0 && scanText.includes(">")) {
+      cp.pendingTruncatedTags = [];
+    }
     if (!cp.commentOpen) {
       const lastLt = scanText.lastIndexOf("<");
       if (lastLt !== -1 && !scanText.includes(">", lastLt)) {
@@ -731,22 +838,20 @@ function processConfirmedLine(cp, ln, text) {
         if (m2) {
           const closing = m2[1] === "/";
           const tag = m2[2].toLowerCase();
-          if (!VOID_TAGS.has(tag)) applyTag(tag, closing);
+          if (!VOID_TAGS.has(tag)) {
+            applyTag(tag, closing);
+            if (!closing && !inRawText) cp.pendingTruncatedTags.push(tag);
+          }
         }
       }
     }
   }
-  if (inRawText && cp.openTotal === 0) {
-    let residue = scanText;
+  if ((inRawText || rawFlowStart) && cp.openTotal === 0) {
+    let masked2 = scanText;
     for (const [from, to] of rawSpans) {
-      residue = residue.slice(0, from) + " ".repeat(to - from) + residue.slice(to);
-    }
-    residue = residue.replace(/<!--[\s\S]*?-->/g, " ");
-    if (commentOpenAtLineStart) {
-      residue = residue.includes("-->") ? residue.replace(/[\s\S]*?-->/, " ") : "";
+      masked2 = masked2.slice(0, from) + " ".repeat(to - from) + masked2.slice(to);
     }
-    residue = residue.replace(TAG_OR_COMMENT_RE, "");
-    if (residue.trim() !== "") {
+    if (floatingResidue(masked2, commentOpenAtLineStart).trim() !== "") {
       cp.htmlSeamPending = true;
     }
   }
@@ -756,6 +861,9 @@ function processConfirmedLine(cp, ln, text) {
   cp.prevLineWasValidDef = validDef && !def[1].startsWith("^");
 }
 
+// src/components/incrementalParse/spliceParse.ts
+var import_micromark_util_normalize_identifier2 = require("micromark-util-normalize-identifier");
+
 // src/components/hastPredicates.ts
 function isFootnoteSection(node) {
   if (node.tagName !== "section") return false;
@@ -850,16 +958,20 @@ function collectPrefixInjection(mdast, content, boundary, resume) {
       for (const child of children) visit6(child, true);
     }
   };
+  let lastStart = -1;
   for (const child of mdast.children) {
     const start = child.position?.start?.offset;
+    if (start !== void 0) {
+      if (start < lastStart) return { events: [], uninjectable: true, cacheable: false };
+      lastStart = start;
+    }
     if (start === void 0) {
       if (resumeAt > 0) return collectPrefixInjection(mdast, content, boundary, null);
       cacheable = false;
       visit6(child, false);
       continue;
     }
-    if (start >= boundary) break;
-    if (start < resumeAt) continue;
+    if (start >= boundary || start < resumeAt) continue;
     visit6(child, false);
   }
   return { events, uninjectable, cacheable };
@@ -873,7 +985,7 @@ function cloneEventsForAppend(events) {
 var TERMINATOR_LABEL = "__aimd_injection_terminator__";
 var INJECTION_TERMINATOR = `[${TERMINATOR_LABEL}]: __aimd_sentinel_link__`;
 function tailMentionsTerminator(tailSource) {
-  return tailSource.includes(`[${TERMINATOR_LABEL}`);
+  return (0, import_micromark_util_normalize_identifier2.normalizeIdentifier)(tailSource).includes(`[${(0, import_micromark_util_normalize_identifier2.normalizeIdentifier)(TERMINATOR_LABEL)}`);
 }
 function buildInjectionPrefix(events) {
   if (events.length === 0) return { text: "", segments: [] };
@@ -1321,11 +1433,12 @@ var import_remark_gfm = __toESM(require("remark-gfm"), 1);
 var import_unist_util_visit = require("unist-util-visit");
 
 // src/components/normalizeId.ts
+var import_micromark_util_normalize_identifier3 = require("micromark-util-normalize-identifier");
 function normalizeId(s) {
-  return s.replace(/\s+/g, " ").toUpperCase();
+  return (0, import_micromark_util_normalize_identifier3.normalizeIdentifier)(s);
 }
 function normalizeForMatch(s) {
-  return s.replace(/\\(.)/g, "$1").replace(/\s+/g, " ").toUpperCase();
+  return (0, import_micromark_util_normalize_identifier3.normalizeIdentifier)(s.replace(/\\(.)/g, "$1"));
 }
 
 // src/components/collectDefLabels.ts
@@ -1580,6 +1693,18 @@ function buildPhantomSuffix(phantoms) {
   }
   return suffix;
 }
+function phantomSuffixCloser(content) {
+  if (content === "") return "";
+  const endsWithNewline = content.endsWith("\n");
+  const confirmed = endsWithNewline ? content : content + "\n";
+  const { checkpoint } = computeFreezeBoundary(confirmed, { defListEnabled: false, referenceTaint: false });
+  if (checkpoint.phasePoisonedAt !== Infinity) return "";
+  const nl = endsWithNewline ? "" : "\n";
+  const indent = " ".repeat(checkpoint.openIndent);
+  if (checkpoint.inFence) return `${nl}${indent}${checkpoint.fenceChar.repeat(checkpoint.fenceLen)}`;
+  if (checkpoint.inMath) return `${nl}${indent}${"$".repeat(checkpoint.mathFenceLen)}`;
+  return "";
+}
 
 // src/components/extractContributions.ts
 function fakeAnchorElement(url) {
@@ -1658,6 +1783,14 @@ function createRegistry(onEmpty) {
     releaseSymbol(reactId) {
       const entry = this._reactIdMap.get(reactId);
       if (!entry) return;
+      if (entry.refcount <= 0) {
+        if (false) {
+          console.warn(
+            `[ai-react-markdown] Registry.releaseSymbol("${reactId}") called with no matching allocateSymbol \u2014 ignoring (unbalanced release).`
+          );
+        }
+        return;
+      }
       entry.refcount--;
       if (entry.refcount === 0) {
         queueMicrotask(() => {
@@ -1954,6 +2087,11 @@ function buildCoreRemarkRehypeOptions(enableDefinitionList) {
 }
 
 // src/components/customMdastHandlers.ts
+function localDefProps(s, id) {
+  const def = s.definitionById.get(id);
+  if (!def || typeof def.url !== "string" || def.url === SENTINEL_LINK_URL) return {};
+  return def.title ? { localUrl: def.url, localTitle: def.title } : { localUrl: def.url };
+}
 function buildCrossChunkHandlers() {
   return {
     footnoteDefinition: (state, node) => {
@@ -1986,7 +2124,8 @@ function buildCrossChunkHandlers() {
           // internally, so cross-chunk case-insensitive matching still works.
           label: node.label ?? node.identifier,
           referenceType: node.referenceType,
-          documentId: s.options.documentId
+          documentId: s.options.documentId,
+          ...localDefProps(s, id)
         },
         children: s.all(node)
       };
@@ -2003,7 +2142,8 @@ function buildCrossChunkHandlers() {
           label: node.label ?? node.identifier,
           referenceType: node.referenceType,
           alt: node.alt ?? "",
-          documentId: s.options.documentId
+          documentId: s.options.documentId,
+          ...localDefProps(s, id)
         },
         children: []
       };
@@ -2032,6 +2172,12 @@ function buildCrossChunkHandlers() {
         properties: {
           label: node.identifier,
           localOccurrence,
+          // The number mdast-util-to-hast would give this reference in a
+          // standalone render (footnoteOrder position) — the placeholder's
+          // fallback while the registry has no global number (server render /
+          // first client frame), where the local synthetic footer is what
+          // renders, so marks and footer agree (core-render-02).
+          localNumber: s.footnoteOrder.indexOf(id) + 1,
           documentId: s.options.documentId
         },
         children: []
@@ -3240,9 +3386,9 @@ var sanitizeSchema = cloneDeep_default({
   attributes: {
     ...import_rehype_sanitize2.defaultSchema.attributes,
     code: mergeClassNameAllowlist(import_rehype_sanitize2.defaultSchema.attributes?.code, ["math-inline", "math-display"]),
-    "cross-chunk-link": ["label", "referenceType", "documentId"],
-    "cross-chunk-image": ["label", "referenceType", "documentId", "alt"],
-    "footnote-sup": ["label", "localOccurrence", "documentId"]
+    "cross-chunk-link": ["label", "referenceType", "documentId", "localUrl", "localTitle"],
+    "cross-chunk-image": ["label", "referenceType", "documentId", "alt", "localUrl", "localTitle"],
+    "footnote-sup": ["label", "localOccurrence", "localNumber", "documentId"]
   }
 });
 
@@ -3611,9 +3757,9 @@ var createSmoothStreamController = (options = {}) => {
     snap,
     flush() {
       disposed = false;
-      if (visibleEnd === source.length) return;
-      visibleEnd = source.length;
-      tentativeEnd = source.length;
+      const target = finished ? source.length : pending.length > 0 ? pending[pending.length - 1] : visibleEnd;
+      if (target <= visibleEnd) return;
+      visibleEnd = target;
       pending = [];
       credit = 0;
       cancelScheduled();
@@ -4140,6 +4286,7 @@ function createRemendPreprocessor(options) {
   normalizeId,
   pangu,
   parseStage,
+  phantomSuffixCloser,
   preprocessAIMDContent,
   preprocessLaTeX,
   rehypeFooterAdorn,