@ai-react-markdown/engine 2.10.1 → 2.12.0

package/dist/index.js

@@ -2131,6 +2131,38 @@ function normalizeForMatch(s) {
   return normalizeIdentifier3(s);
 }
 
+// src/components/blankLineScanner.ts
+function createBlankLineScanner() {
+  let end = 0;
+  let newline = false;
+  let cr = false;
+  return (source, from = 0) => {
+    if (from === 0) {
+      end = 0;
+      newline = false;
+      cr = false;
+    }
+    for (let i = from; i < source.length; i++) {
+      const c = source[i];
+      if (c === "\n") {
+        if (newline) {
+          end = i + 1;
+          newline = false;
+        } else newline = true;
+        cr = false;
+      } else if (newline) {
+        if ((c === " " || c === "	") && !cr) continue;
+        if (c === "\r" && !cr) cr = true;
+        else {
+          newline = false;
+          cr = false;
+        }
+      }
+    }
+    return end;
+  };
+}
+
 // src/components/collectDefLabels.ts
 var SCANNER_BOUNDARY_PROFILE = { defListEnabled: false, mathFlow: false, referenceTaint: false };
 function buildProcessor() {
@@ -2162,19 +2194,12 @@ var setsEqual = (a, b) => {
   for (const v of a) if (!b.has(v)) return false;
   return true;
 };
-var BLANK_LINE_RE = /\r?\n[ \t]*\r?\n/g;
-function lastRegionStart(source) {
-  BLANK_LINE_RE.lastIndex = 0;
-  let start = 0;
-  for (let m = BLANK_LINE_RE.exec(source); m !== null; m = BLANK_LINE_RE.exec(source)) {
-    start = m.index + m[0].length;
-  }
-  return start;
-}
 var DEF_LINE_START_RE = /^[ \t>*+\d.)-]*\[(?:[^\]\\]|\\[\s\S])*\]:/m;
 function createDefLabelScanner(parse = collectDefLabels) {
   let prevSource = null;
   let prevLabels = null;
+  const scanBlankLines = createBlankLineScanner();
+  let regionStart = 0;
   let frozenEnd = 0;
   let frozenFootnotes = /* @__PURE__ */ new Set();
   let frozenLinks = /* @__PURE__ */ new Set();
@@ -2187,13 +2212,16 @@ function createDefLabelScanner(parse = collectDefLabels) {
   };
   return {
     scan(source) {
+      if (source === prevSource && prevLabels !== null) return prevLabels;
+      const previousRegionStart = regionStart;
+      const appended = prevSource !== null && source.startsWith(prevSource);
+      regionStart = scanBlankLines(source, appended ? prevSource.length : 0);
       let isAppend = false;
       if (prevSource !== null && prevLabels !== null) {
         if (source === prevSource) return prevLabels;
         if (source.startsWith(prevSource)) {
           isAppend = true;
-          const regionStart = lastRegionStart(prevSource);
-          const region = prevSource.slice(regionStart) + source.slice(prevSource.length);
+          const region = source.slice(previousRegionStart);
           if (!DEF_LINE_START_RE.test(region)) {
             prevSource = source;
             return prevLabels;
@@ -2426,8 +2454,47 @@ function* extractContributions(mdast, options = {}) {
   for (const c of out) yield c;
 }
 
+// src/components/registryIndex.ts
+function buildRegistryIndex(registry) {
+  const index = {
+    footnotes: /* @__PURE__ */ new Map(),
+    links: /* @__PURE__ */ new Map(),
+    numbers: /* @__PURE__ */ new Map(),
+    counts: /* @__PURE__ */ new Map(),
+    occurrences: /* @__PURE__ */ new Map()
+  };
+  for (const sym of registry.chunkOrder) {
+    const data = registry.chunkData.get(sym);
+    if (!data) continue;
+    for (const label of data.defs.keys()) if (!index.footnotes.has(label)) index.footnotes.set(label, sym);
+    for (const label of data.linkDefs.keys()) if (!index.links.has(label)) index.links.set(label, sym);
+    const local = /* @__PURE__ */ new Map();
+    index.occurrences.set(sym, local);
+    for (const ref of data.refs) {
+      if (ref.kind !== "footnote") continue;
+      const label = ref.label;
+      if (!index.numbers.has(label)) index.numbers.set(label, index.numbers.size + 1);
+      const total = (index.counts.get(label) ?? 0) + 1;
+      index.counts.set(label, total);
+      const prior = local.get(label);
+      if (prior) prior.count++;
+      else local.set(label, { start: total, count: 1 });
+    }
+  }
+  return index;
+}
+
 // src/components/documentRegistry.ts
 function createRegistry(onEmpty) {
+  let index;
+  let indexedVersion = -1;
+  const getIndex = () => {
+    if (!index || indexedVersion !== reg.version) {
+      index = buildRegistryIndex(reg);
+      indexedVersion = reg.version;
+    }
+    return index;
+  };
   const reg = {
     chunkOrder: [],
     chunkData: /* @__PURE__ */ new Map(),
@@ -2579,72 +2646,25 @@ function createRegistry(onEmpty) {
       };
     },
     canonicalFootnoteFor(label) {
-      const id = normalizeId(label);
-      for (const sym of this.chunkOrder) {
-        const data = this.chunkData.get(sym);
-        if (data?.defs.has(id)) return sym;
-      }
-      return null;
+      return getIndex().footnotes.get(normalizeId(label)) ?? null;
     },
     canonicalLinkFor(label) {
-      const id = normalizeId(label);
-      for (const sym of this.chunkOrder) {
-        const data = this.chunkData.get(sym);
-        if (data?.linkDefs.has(id)) return sym;
-      }
-      return null;
+      return getIndex().links.get(normalizeId(label)) ?? null;
     },
     globalNumber(label) {
-      const id = normalizeId(label);
-      let n = 0;
-      const seen = /* @__PURE__ */ new Set();
-      for (const sym of this.chunkOrder) {
-        const data = this.chunkData.get(sym);
-        if (!data) continue;
-        for (const ref of data.refs) {
-          if (ref.kind !== "footnote") continue;
-          if (!seen.has(ref.label)) {
-            seen.add(ref.label);
-            n++;
-            if (ref.label === id) return n;
-          }
-        }
-      }
-      return null;
+      return getIndex().numbers.get(normalizeId(label)) ?? null;
     },
     resolveLinkDef(label) {
-      const sym = this.canonicalLinkFor(label);
-      if (!sym) return null;
-      return this.chunkData.get(sym)?.linkDefs.get(normalizeId(label)) ?? null;
+      const id = normalizeId(label);
+      const sym = getIndex().links.get(id);
+      return sym ? this.chunkData.get(sym)?.linkDefs.get(id) ?? null : null;
     },
     getRefsForLabel(label) {
-      const id = normalizeId(label);
-      let n = 0;
-      for (const sym of this.chunkOrder) {
-        const data = this.chunkData.get(sym);
-        if (!data) continue;
-        for (const ref of data.refs) {
-          if (ref.kind === "footnote" && ref.label === id) n++;
-        }
-      }
-      return n;
+      return getIndex().counts.get(normalizeId(label)) ?? 0;
     },
     globalOccurrenceForRef(chunkSym, label, localOccurrence) {
-      const id = normalizeId(label);
-      let global2 = 0;
-      for (const sym of this.chunkOrder) {
-        const data = this.chunkData.get(sym);
-        if (!data) continue;
-        let localCount = 0;
-        for (const ref of data.refs) {
-          if (ref.kind !== "footnote") continue;
-          if (ref.label !== id) continue;
-          localCount++;
-          global2++;
-          if (sym === chunkSym && localCount === localOccurrence) return global2;
-        }
-      }
-      return null;
+      const range = getIndex().occurrences.get(chunkSym)?.get(normalizeId(label));
+      return range && Number.isInteger(localOccurrence) && localOccurrence > 0 && localOccurrence <= range.count ? range.start + localOccurrence - 1 : null;
     },
     _notify() {
       this.version++;
@@ -2701,6 +2721,55 @@ function rehypeUnwrapCrossChunkImages() {
 
 // src/components/pluginChain.ts
 import rehypeSanitize from "rehype-sanitize";
+
+// src/components/rehypeVerifyEngineTags.ts
+var ENGINE_PLACEHOLDER_TAGS = /* @__PURE__ */ new Set([
+  "footnote-sup",
+  "cross-chunk-link",
+  "cross-chunk-image"
+]);
+var ENGINE_PROVENANCE_PROPERTY = "engineProvenance";
+function walk(parent, provenance, ancestors) {
+  const children = parent.children;
+  let i = 0;
+  while (i < children.length) {
+    const node = children[i];
+    if (node.type === "element" && ENGINE_PLACEHOLDER_TAGS.has(node.tagName)) {
+      const props = node.properties ?? {};
+      const stamped = props[ENGINE_PROVENANCE_PROPERTY];
+      const genuine = provenance !== "" && typeof stamped === "string" && stamped === provenance;
+      if (genuine) {
+        delete props[ENGINE_PROVENANCE_PROPERTY];
+        if (ancestors && (node.tagName === "cross-chunk-link" || node.tagName === "cross-chunk-image")) {
+          (node.data ??= {}).referenceAncestors = ancestors.slice();
+        }
+        ancestors?.push(
+          node.tagName === "cross-chunk-link" ? "a" : node.tagName === "cross-chunk-image" ? "img" : node.tagName
+        );
+        walk(node, provenance, ancestors);
+        ancestors?.pop();
+        i += 1;
+      } else {
+        children.splice(i, 1, ...node.children);
+      }
+      continue;
+    }
+    if (node.type === "element") {
+      ancestors?.push(node.tagName);
+      walk(node, provenance, ancestors);
+      ancestors?.pop();
+    }
+    i += 1;
+  }
+}
+function rehypeVerifyEngineTags(options) {
+  const provenance = options?.provenance ?? "";
+  return function transformer(tree) {
+    walk(tree, provenance, options?.referenceAncestors ? [] : void 0);
+  };
+}
+
+// src/components/pluginChain.ts
 import remarkBreaks from "remark-breaks";
 import remarkCjkFriendly from "remark-cjk-friendly";
 import remarkCjkFriendlyGfmStrikethrough from "remark-cjk-friendly-gfm-strikethrough";
@@ -2717,16 +2786,18 @@ import remarkRemoveComments from "remark-remove-comments";
 // src/components/rehypeRebaseHashLinks.ts
 import { visit as visit5 } from "unist-util-visit";
 var DEFAULT_PREFIX = "user-content-";
+function rebaseHashHref(href, prefix) {
+  const hashPrefix = "#" + prefix;
+  return href.startsWith("#") && !href.startsWith(hashPrefix) ? hashPrefix + href.slice(1) : href;
+}
 var rehypeRebaseHashLinks = (options) => {
   const prefix = options?.prefix ?? DEFAULT_PREFIX;
-  const hashPrefix = "#" + prefix;
   return (tree) => {
     visit5(tree, "element", (node) => {
       if (node.tagName !== "a") return;
       const href = node.properties?.href;
       if (typeof href !== "string" || !href.startsWith("#")) return;
-      if (href.startsWith(hashPrefix)) return;
-      node.properties.href = hashPrefix + href.slice(1);
+      node.properties.href = rebaseHashHref(href, prefix);
     });
   };
 };
@@ -2806,10 +2877,21 @@ function buildCoreRemarkPlugins(enginePlugins) {
     ...DISPLAY_OPTIMIZE_CHAIN.filter(([name]) => selected.has(name)).map(([, plugin]) => plugin)
   ];
 }
-function buildCoreRehypePlugins(sanitizeSchema2, clobberPrefix) {
+function buildCoreRehypePlugins(sanitizeSchema2, clobberPrefix, options) {
   return [
     // Allow raw HTML through so rehype-sanitize can handle it.
     [rehypeRaw, { passThrough: [] }],
+    // Unwrap forged engine placeholders BEFORE sanitize admits their tag
+    // names. Only when the caller holds a credential (see the option's doc).
+    ...options ? [
+      [
+        rehypeVerifyEngineTags,
+        {
+          provenance: options.provenance,
+          ...sanitizeSchema2.ancestors?.a || sanitizeSchema2.ancestors?.img ? { referenceAncestors: true } : {}
+        }
+      ]
+    ] : [],
     // Sanitize HTML while allowing <mark> (highlight), KaTeX class names,
     // and any extra protocols the caller permitted via the `sanitizeSchema`
     // prop. Override `clobberPrefix` with the instance-scoped value — the
@@ -2849,6 +2931,10 @@ function buildCoreRemarkRehypeOptions(enableDefinitionList) {
 }
 
 // src/components/customMdastHandlers.ts
+function provenanceProps(s) {
+  const provenance = s.options.provenance;
+  return typeof provenance === "string" ? { engineProvenance: provenance } : {};
+}
 function localDefProps(s, id) {
   const def = s.definitionById.get(id);
   if (!def || typeof def.url !== "string" || def.url === SENTINEL_LINK_URL) return {};
@@ -2879,15 +2965,14 @@ function buildCrossChunkHandlers() {
         type: "element",
         tagName: "cross-chunk-link",
         properties: {
-          // `label` is the ORIGINAL source text (mdast's `label` field), NOT
-          // the normalized `identifier`. The placeholder uses it to construct
-          // hrefs that line up with mdast-util-to-hast's default `<li id>`
-          // which also preserves source case. Registry lookups normalize
-          // internally, so cross-chunk case-insensitive matching still works.
+          // Display labels decode escapes; registry identifiers must retain
+          // those bytes. Never use the display label as the lookup key.
+          identifier: node.identifier,
           label: node.label ?? node.identifier,
           referenceType: node.referenceType,
           documentId: s.options.documentId,
-          ...localDefProps(s, id)
+          ...localDefProps(s, id),
+          ...provenanceProps(s)
         },
         children: s.all(node)
       };
@@ -2901,11 +2986,13 @@ function buildCrossChunkHandlers() {
         type: "element",
         tagName: "cross-chunk-image",
         properties: {
+          identifier: node.identifier,
           label: node.label ?? node.identifier,
           referenceType: node.referenceType,
           alt: node.alt ?? "",
           documentId: s.options.documentId,
-          ...localDefProps(s, id)
+          ...localDefProps(s, id),
+          ...provenanceProps(s)
         },
         children: []
       };
@@ -2922,7 +3009,8 @@ function buildCrossChunkHandlers() {
           properties: {
             label: node.identifier,
             localOccurrence,
-            documentId: s.options.documentId
+            documentId: s.options.documentId,
+            ...provenanceProps(s)
           },
           children: []
         };
@@ -2940,7 +3028,8 @@ function buildCrossChunkHandlers() {
           // first client frame), where the local synthetic footer is what
           // renders, so marks and footer agree (core-render-02).
           localNumber: s.footnoteOrder.indexOf(id) + 1,
-          documentId: s.options.documentId
+          documentId: s.options.documentId,
+          ...provenanceProps(s)
         },
         children: []
       };
@@ -4160,8 +4249,8 @@ var sanitizeSchema = deepFreeze(
     attributes: {
       ...defaultSchema.attributes,
       code: mergeClassNameAllowlist(defaultSchema.attributes?.code, ["math-inline", "math-display"]),
-      "cross-chunk-link": ["label", "referenceType", "documentId", "localUrl", "localTitle"],
-      "cross-chunk-image": ["label", "referenceType", "documentId", "alt", "localUrl", "localTitle"],
+      "cross-chunk-link": ["identifier", "label", "referenceType", "documentId", "localUrl", "localTitle"],
+      "cross-chunk-image": ["identifier", "label", "referenceType", "documentId", "alt", "localUrl", "localTitle"],
       "footnote-sup": ["label", "localOccurrence", "localNumber", "documentId"]
     },
     strip: [.../* @__PURE__ */ new Set([...defaultSchema.strip || [], ...STRIPPED_TAGS])]
@@ -4197,6 +4286,49 @@ function sanitizeCrossChunkUrl(rawUrl, key, tagName, urlTransform, schema) {
   return String(transformed);
 }
 
+// src/components/resolveCrossChunkReference.ts
+import rehypeSanitize2 from "rehype-sanitize";
+import { normalizeUri as normalizeUri3 } from "micromark-util-sanitize-uri";
+function resolveCrossChunkReference(input, schema, urlTransform, clobberPrefix) {
+  const key = input.tagName === "a" ? "href" : "src";
+  const element = {
+    type: "element",
+    tagName: input.tagName,
+    properties: {
+      [key]: normalizeUri3(input.url),
+      ...input.tagName === "img" ? { alt: input.alt ?? "" } : {},
+      ...input.title !== void 0 ? { title: input.title } : {}
+    },
+    children: input.tagName === "a" ? [{ type: "text", value: "__reference_children__" }] : []
+  };
+  const requiredAncestors = schema.ancestors?.[input.tagName];
+  const recordedAncestors = input.node?.data?.referenceAncestors;
+  let finalSchema = schema;
+  if (requiredAncestors && Array.isArray(recordedAncestors) && requiredAncestors.some((tag) => recordedAncestors.includes(tag))) {
+    const ancestors = { ...schema.ancestors };
+    delete ancestors[input.tagName];
+    finalSchema = { ...schema, ancestors };
+  }
+  const root2 = rehypeSanitize2({ ...finalSchema, clobberPrefix })({ type: "root", children: [element] });
+  const node = root2.children[0];
+  if (node?.type !== "element") return { element: null, keepChildren: node?.type === "text" };
+  if (node.tagName === "a" && typeof node.properties.href === "string") {
+    node.properties.href = rebaseHashHref(node.properties.href, clobberPrefix);
+  }
+  node.children = input.node?.children ?? [];
+  if (input.node?.position) node.position = input.node.position;
+  buildTransform({
+    allowedElements: void 0,
+    disallowedElements: void 0,
+    allowElement: void 0,
+    skipHtml: void 0,
+    unwrapDisallowed: void 0,
+    urlTransform
+  })(node, 0, root2);
+  node.children = [];
+  return { element: node, keepChildren: false };
+}
+
 // src/plugins/defs.ts
 function getEnginePluginInternals(plugin) {
   const candidate = plugin;
@@ -4401,6 +4533,8 @@ var createSmoothStreamController = (options = {}) => {
   let source = "";
   let visibleEnd = 0;
   let pending = [];
+  let pendingHead = 0;
+  const pendingCount = () => pending.length - pendingHead;
   let tentativeEnd = 0;
   let finished = false;
   let seam;
@@ -4445,7 +4579,7 @@ var createSmoothStreamController = (options = {}) => {
     cancelFrame = void 0;
   };
   const ensureScheduled = () => {
-    if (disposed || cancelFrame || pending.length === 0) return;
+    if (disposed || cancelFrame || pendingCount() === 0) return;
     lastTickAt = now();
     credit = 0;
     cancelFrame = schedule(tick);
@@ -4459,31 +4593,38 @@ var createSmoothStreamController = (options = {}) => {
     const params = resolveParams();
     let rate;
     if (finished && drainDeadlineAt !== void 0) {
-      rate = Math.max(params.minCharsPerSecond, pending.length * 1e3 / Math.max(1, drainDeadlineAt - t));
+      rate = Math.max(params.minCharsPerSecond, pendingCount() * 1e3 / Math.max(1, drainDeadlineAt - t));
     } else if (gapWindow.length > 0 && lastArrivalAt !== void 0) {
       const gaps = gapWindow.map((s) => s.gap).sort((a, b) => a - b);
       const intervalQ = gaps[Math.min(gaps.length - 1, Math.floor(gaps.length * INTERVAL_QUANTILE))];
       const horizon = Math.max(16, Math.min(params.bufferFactor * intervalQ + HORIZON_PAD_MS, params.maxLagMs));
       const deadline = Math.max(lastArrivalAt + horizon, t + DEADLINE_FLOOR_MS);
-      rate = Math.max(params.minCharsPerSecond, pending.length * 1e3 / Math.max(1, deadline - t));
+      rate = Math.max(params.minCharsPerSecond, pendingCount() * 1e3 / Math.max(1, deadline - t));
     } else {
-      rate = Math.max(params.minCharsPerSecond, pending.length * 1e3 / Math.max(1, params.correctionTauMs));
+      rate = Math.max(params.minCharsPerSecond, pendingCount() * 1e3 / Math.max(1, params.correctionTauMs));
     }
     credit += rate * dt / 1e3;
-    const reveal = Math.min(Math.floor(credit), pending.length);
+    const reveal = Math.min(Math.floor(credit), pendingCount());
     if (reveal > 0) {
-      visibleEnd = pending[reveal - 1];
-      pending = pending.slice(reveal);
-      credit = pending.length > 0 ? credit - reveal : 0;
+      visibleEnd = pending[pendingHead + reveal - 1];
+      pendingHead += reveal;
+      if (pendingHead === pending.length) {
+        pending = [];
+        pendingHead = 0;
+      } else if (pendingHead >= 1024 && pendingHead * 2 >= pending.length) {
+        pending = pending.slice(pendingHead);
+        pendingHead = 0;
+      }
+      credit = pendingCount() > 0 ? credit - reveal : 0;
       notify();
     }
-    if (!disposed && !cancelFrame && pending.length > 0) cancelFrame = schedule(tick);
+    if (!disposed && !cancelFrame && pendingCount() > 0) cancelFrame = schedule(tick);
   };
   const resegmentTail = () => {
-    if (seam !== void 0 && seam < source.length && pending[pending.length - 1] === seam) {
+    if (pendingCount() > 0 && seam !== void 0 && seam < source.length && pending[pending.length - 1] === seam) {
       pending.pop();
     }
-    const from = pending.length > 0 ? pending[pending.length - 1] : visibleEnd;
+    const from = pendingCount() > 0 ? pending[pending.length - 1] : visibleEnd;
     let anchor = from;
     if (seam !== void 0 && from <= seam) {
       anchor = Math.max(0, from - RESUME_LOOKBACK);
@@ -4517,6 +4658,7 @@ var createSmoothStreamController = (options = {}) => {
     visibleEnd = next.length;
     tentativeEnd = next.length;
     pending = [];
+    pendingHead = 0;
     seam = next.length;
     credit = 0;
     cancelScheduled();
@@ -4548,7 +4690,7 @@ var createSmoothStreamController = (options = {}) => {
       if (finished) return;
       finished = true;
       lastArrivalAt = void 0;
-      if (tentativeEnd > (pending.length > 0 ? pending[pending.length - 1] : visibleEnd)) {
+      if (tentativeEnd > (pendingCount() > 0 ? pending[pending.length - 1] : visibleEnd)) {
         pending.push(tentativeEnd);
       }
       const params = resolveParams();
@@ -4574,10 +4716,11 @@ var createSmoothStreamController = (options = {}) => {
     snap,
     flush() {
       disposed = false;
-      const target = finished ? source.length : pending.length > 0 ? pending[pending.length - 1] : visibleEnd;
+      const target = finished ? source.length : pendingCount() > 0 ? pending[pending.length - 1] : visibleEnd;
       if (target <= visibleEnd) return;
       visibleEnd = target;
       pending = [];
+      pendingHead = 0;
       credit = 0;
       cancelScheduled();
       notify();
@@ -4600,6 +4743,12 @@ var createSmoothStreamController = (options = {}) => {
 };
 
 // src/preprocessors/latex.ts
+function isHardBoundary(kind) {
+  return kind === "code" || kind === "literal" || kind === "multilineTag";
+}
+function hasLineEnding(text) {
+  return text.includes("\n") || text.includes("\r");
+}
 function getRepeatedMarkerLength(content, start, marker) {
   let end = start;
   while (end < content.length && content[end] === marker) {
@@ -4667,11 +4816,11 @@ function splitByProtectedRegions(content) {
   let multilineFenceMarker = null;
   let multilineFenceLength = 0;
   let multilineFenceIndent = 0;
-  function pushProtected(start, end) {
+  function pushProtected(start, end, kind) {
     if (start > lastIndex) {
-      segments.push({ text: content.substring(lastIndex, start), isCode: false });
+      segments.push({ kind: "text", text: content.substring(lastIndex, start) });
     }
-    segments.push({ text: content.substring(start, end), isCode: true });
+    segments.push({ kind, text: content.substring(start, end) });
     lastIndex = end;
   }
   let i = 0;
@@ -4682,7 +4831,7 @@ function splitByProtectedRegions(content) {
         const runLen = getRepeatedMarkerLength(content, i, multilineFenceMarker);
         const closerIndent = lineIndentBefore(content, i);
         if (runLen >= multilineFenceLength && closerIndent !== -1 && closerIndent <= multilineFenceIndent + 3 && restOfLineIsBlank(content, i + runLen)) {
-          pushProtected(multilineStart, i + runLen);
+          pushProtected(multilineStart, i + runLen, "code");
           multilineStart = -1;
           multilineFenceMarker = null;
           multilineFenceLength = 0;
@@ -4709,7 +4858,7 @@ function splitByProtectedRegions(content) {
       if (char === "`") {
         const closeIdx = findClosingBacktickRun(content, i + runLen, runLen);
         if (closeIdx !== -1) {
-          pushProtected(i, closeIdx + runLen);
+          pushProtected(i, closeIdx + runLen, "code");
           i = closeIdx + runLen;
           continue;
         }
@@ -4734,7 +4883,11 @@ function splitByProtectedRegions(content) {
             endIndex = content.length;
           }
         }
-        pushProtected(i, endIndex);
+        pushProtected(
+          i,
+          endIndex,
+          isOpeningPairedTag ? "literal" : hasLineEnding(content.substring(i, endIndex)) ? "multilineTag" : "tag"
+        );
         i = endIndex;
         continue;
       }
@@ -4742,10 +4895,10 @@ function splitByProtectedRegions(content) {
     i += 1;
   }
   if (multilineStart !== -1) {
-    pushProtected(multilineStart, content.length);
+    pushProtected(multilineStart, content.length, "code");
   }
   if (lastIndex < content.length) {
-    segments.push({ text: content.substring(lastIndex), isCode: false });
+    segments.push({ kind: "text", text: content.substring(lastIndex) });
   }
   return segments;
 }
@@ -4899,20 +5052,22 @@ function escapeLatexPipesInUnclosed(text) {
   const tail = text.substring(unclosedStart + delimLen);
   return before + delim + replaceUnescapedPipes(tail);
 }
-function opensMathFlow(text, pos) {
+function opensMathFlow(text, pos, runStartsAtLineStart) {
   let i = pos;
   let spaces = 0;
-  while (i > 0 && text[i - 1] !== "\n") {
+  while (i > 0) {
+    const prev = text[i - 1];
+    if (prev === "\n" || prev === "\r") return true;
     i -= 1;
     if (text[i] !== " ") return false;
     spaces += 1;
     if (spaces > 3) return false;
   }
-  return true;
+  return runStartsAtLineStart;
 }
-function truncateUnclosedLatexBlock(text, unclosedStart = findUnclosedDelimiterStart(text, "double-only")) {
+function truncateUnclosedLatexBlock(text, runStartsAtLineStart, unclosedStart = findUnclosedDelimiterStart(text, "double-only")) {
   if (unclosedStart === -1) return text;
-  if (!opensMathFlow(text, unclosedStart)) return text;
+  if (!opensMathFlow(text, unclosedStart, runStartsAtLineStart)) return text;
   return text.substring(0, unclosedStart).trimEnd();
 }
 function escapeTextUnderscores(text) {
@@ -4957,7 +5112,8 @@ function convertSingleToDoubleDollar(text) {
 }
 function preprocessLaTeX(str) {
   if (!hasLatexTrigger(str)) return str;
-  return processSlice(str, false).out;
+  const mask = selectMask(str);
+  return (mask === null ? processSlice(str, { legacy: true, probe: false }) : processSlice(str, { probe: false, mask })).out;
 }
 function hasLatexTrigger(str) {
   return str.includes("$") || str.includes("\\[") || str.includes("\\(");
@@ -4986,42 +5142,240 @@ function hasUnclosedTextCommand(text) {
 }
 var RESIDUAL_OPEN_BRACKET_RE = /(?<!!)\\\[/;
 var LEADING_DOUBLE_DOLLAR_RE = /^\s*\$\$/;
-function processSlice(slice, probe = true) {
+var PUA_START = 57344;
+var PUA_END = 63743;
+var PUA_SIZE = PUA_END - PUA_START + 1;
+function selectMask(source) {
+  const first = String.fromCharCode(PUA_START);
+  if (source.indexOf(first) === -1) return first;
+  const seen = new Uint8Array(PUA_SIZE);
+  for (let i = 0; i < source.length; i++) {
+    const c = source.charCodeAt(i);
+    if (c >= PUA_START && c <= PUA_END) seen[c - PUA_START] = 1;
+  }
+  for (let k = 0; k < PUA_SIZE; k++) if (seen[k] === 0) return String.fromCharCode(PUA_START + k);
+  return null;
+}
+var PuaPresence = class {
+  bits = new Uint8Array(PUA_SIZE);
+  distinct = 0;
+  reset() {
+    this.bits.fill(0);
+    this.distinct = 0;
+  }
+  add(text, from) {
+    for (let i = from; i < text.length; i++) {
+      const c = text.charCodeAt(i);
+      if (c >= PUA_START && c <= PUA_END) {
+        const k = c - PUA_START;
+        if (this.bits[k] === 0) {
+          this.bits[k] = 1;
+          this.distinct += 1;
+        }
+      }
+    }
+  }
+  select() {
+    if (this.distinct === 0) return String.fromCharCode(PUA_START);
+    if (this.distinct >= PUA_SIZE) return null;
+    for (let k = 0; k < PUA_SIZE; k++) if (this.bits[k] === 0) return String.fromCharCode(PUA_START + k);
+    return null;
+  }
+};
+function transformRun(input, probe, runStartsAtLineStart, seamEligible) {
+  let text = input;
+  let tailSensitive = false;
+  let truncatedAtSeamStart = false;
+  text = escapeMhchemCommands(text);
+  text = escapeCurrencyDollarSigns(text);
+  text = convertLatexDelimiters(text);
+  if (probe && RESIDUAL_OPEN_BRACKET_RE.test(text)) tailSensitive = true;
+  text = escapeLatexPipes(text);
+  if (probe && findUnclosedDelimiterStart(text, "both") !== -1) tailSensitive = true;
+  text = escapeLatexPipesInUnclosed(text);
+  if (probe && hasUnclosedTextCommand(text)) tailSensitive = true;
+  text = escapeTextUnderscores(text);
+  text = convertSingleToDoubleDollar(text);
+  let unclosedDouble;
+  if (probe || seamEligible && LEADING_DOUBLE_DOLLAR_RE.test(text)) {
+    unclosedDouble = findUnclosedDelimiterStart(text, "double-only");
+    if (unclosedDouble !== -1) {
+      tailSensitive = true;
+      if (seamEligible && opensMathFlow(text, unclosedDouble, runStartsAtLineStart) && text.slice(0, unclosedDouble).trim() === "") {
+        truncatedAtSeamStart = true;
+      }
+    }
+  }
+  text = truncateUnclosedLatexBlock(text, runStartsAtLineStart, unclosedDouble);
+  return { out: text, tailSensitive, truncatedAtSeamStart };
+}
+function processSliceLegacy(slice, probe) {
   const segments = splitByProtectedRegions(slice);
   const parts = [];
   let quiescent = true;
   let truncatedAtSeamStart = false;
   for (let index = 0; index < segments.length; index++) {
     const segment = segments[index];
-    if (segment.isCode) {
+    if (segment.kind !== "text") {
       parts.push(segment.text);
       continue;
     }
-    let text = segment.text;
-    text = escapeMhchemCommands(text);
-    text = escapeCurrencyDollarSigns(text);
-    text = convertLatexDelimiters(text);
-    if (probe && RESIDUAL_OPEN_BRACKET_RE.test(text)) quiescent = false;
-    text = escapeLatexPipes(text);
-    if (probe && findUnclosedDelimiterStart(text, "both") !== -1) quiescent = false;
-    text = escapeLatexPipesInUnclosed(text);
-    if (probe && hasUnclosedTextCommand(text)) quiescent = false;
-    text = escapeTextUnderscores(text);
-    text = convertSingleToDoubleDollar(text);
-    let unclosedDouble;
-    if (probe || index === 0 && LEADING_DOUBLE_DOLLAR_RE.test(text)) {
-      unclosedDouble = findUnclosedDelimiterStart(text, "double-only");
-      if (unclosedDouble !== -1) {
-        quiescent = false;
-        if (index === 0 && opensMathFlow(text, unclosedDouble) && text.slice(0, unclosedDouble).trim() === "") {
-          truncatedAtSeamStart = true;
+    const r = transformRun(segment.text, probe, true, index === 0);
+    if (r.tailSensitive) quiescent = false;
+    if (r.truncatedAtSeamStart) truncatedAtSeamStart = true;
+    parts.push(r.out);
+  }
+  return { out: parts.join(""), quiescent, truncatedAtSeamStart, degradedReason: null };
+}
+var SCOPE_DEPTH_CAP = 8;
+var VOID_TAGS2 = /* @__PURE__ */ new Set(["br", "hr", "img", "wbr", "input", "source"]);
+var TAG_NAME_RE = /^<(\/?)([A-Za-z][A-Za-z0-9]*)/;
+function tagInfo(tag) {
+  const m = TAG_NAME_RE.exec(tag);
+  const closing = m?.[1] === "/";
+  const name = (m?.[2] ?? "").toLowerCase();
+  const opensScope = !closing && !tag.endsWith("/>") && !VOID_TAGS2.has(name);
+  return { name, closing, opensScope };
+}
+function buildRunTree(segments) {
+  const root2 = [];
+  const stack = [];
+  const current = () => stack.length > 0 ? stack[stack.length - 1].children : root2;
+  const unwind = () => {
+    while (stack.length > 0) {
+      const frame = stack.pop();
+      current().push({ type: "atom", text: frame.open }, ...frame.children);
+    }
+  };
+  for (const segment of segments) {
+    if (segment.kind === "tag") {
+      const info = tagInfo(segment.text);
+      if (info.closing) {
+        const top = stack[stack.length - 1];
+        if (top !== void 0 && top.name === info.name) {
+          stack.pop();
+          const parent = current();
+          if (top.suppressed) {
+            parent.push({ type: "atom", text: top.open }, ...top.children, { type: "atom", text: segment.text });
+          } else {
+            parent.push({ type: "scope", open: top.open, close: segment.text, children: top.children });
+          }
+        } else {
+          current().push({ type: "atom", text: segment.text });
         }
+      } else if (info.opensScope) {
+        stack.push({ name: info.name, open: segment.text, suppressed: stack.length >= SCOPE_DEPTH_CAP, children: [] });
+      } else {
+        current().push({ type: "atom", text: segment.text });
       }
+      continue;
+    }
+    const t = segment.text;
+    let start = 0;
+    for (let i = 0; i < t.length; i++) {
+      const c = t.charCodeAt(i);
+      if (c !== 10 && c !== 13) continue;
+      if (i > start) current().push({ type: "text", text: t.slice(start, i) });
+      const end = c === 13 && t.charCodeAt(i + 1) === 10 ? i + 2 : i + 1;
+      unwind();
+      root2.push({ type: "text", text: t.slice(i, end) });
+      start = end;
+      i = end - 1;
+    }
+    if (start < t.length) current().push({ type: "text", text: t.slice(start) });
+  }
+  unwind();
+  return root2;
+}
+var restoreFailureInjector = null;
+function restore(out, atoms, mask) {
+  if (restoreFailureInjector !== null && restoreFailureInjector(atoms)) return null;
+  let result = "";
+  let k = 0;
+  let last = 0;
+  for (; ; ) {
+    const idx = out.indexOf(mask, last);
+    if (idx === -1) break;
+    if (k >= atoms.length) return null;
+    result += out.slice(last, idx) + atoms[k];
+    k += 1;
+    last = idx + 1;
+  }
+  return result + out.slice(last);
+}
+function emitRun(nodes, mask) {
+  let text = "";
+  const atoms = [];
+  for (const node of nodes) {
+    if (node.type === "text") {
+      text += node.text;
+    } else if (node.type === "atom") {
+      atoms.push(node.text);
+      text += mask;
+    } else {
+      const inner = emitRun(node.children, mask);
+      if (inner === null) return null;
+      const transformed = transformRun(inner.text, false, false, false);
+      const restored = restore(transformed.out, inner.atoms, mask);
+      if (restored === null) return null;
+      atoms.push(node.open + restored + node.close);
+      text += mask;
     }
-    text = truncateUnclosedLatexBlock(text, unclosedDouble);
-    parts.push(text);
   }
-  return { out: parts.join(""), quiescent, truncatedAtSeamStart };
+  return { text, atoms };
+}
+function reportRestoreViolation() {
+  if (false) {
+    console.error(
+      "[ai-react-markdown] LaTeX preprocessor: atom restoration violated its invariant (more masks in the output than atoms); the slice was re-processed on the legacy path. This is an engine defect \u2014 please report it."
+    );
+  }
+}
+function processSliceDefault(slice, probe, mask) {
+  const segments = splitByProtectedRegions(slice);
+  const parts = [];
+  let quiescent = true;
+  let truncatedAtSeamStart = false;
+  let offset = 0;
+  let i = 0;
+  while (i < segments.length) {
+    const segment = segments[i];
+    if (isHardBoundary(segment.kind)) {
+      parts.push(segment.text);
+      offset += segment.text.length;
+      i += 1;
+      continue;
+    }
+    const runStart = offset;
+    const runSegments = [];
+    while (i < segments.length && !isHardBoundary(segments[i].kind)) {
+      runSegments.push(segments[i]);
+      offset += segments[i].text.length;
+      i += 1;
+    }
+    const before = runStart === 0 ? -1 : slice.charCodeAt(runStart - 1);
+    const runStartsAtLineStart = before === -1 || before === 10 || before === 13;
+    const seamEligible = runStart === 0;
+    const emitted = emitRun(buildRunTree(runSegments), mask);
+    if (emitted === null) {
+      reportRestoreViolation();
+      return { ...processSliceLegacy(slice, probe), degradedReason: "restore-invariant" };
+    }
+    const r = transformRun(emitted.text, probe, runStartsAtLineStart, seamEligible);
+    const restored = restore(r.out, emitted.atoms, mask);
+    if (restored === null) {
+      reportRestoreViolation();
+      return { ...processSliceLegacy(slice, probe), degradedReason: "restore-invariant" };
+    }
+    if (r.tailSensitive) quiescent = false;
+    if (r.truncatedAtSeamStart) truncatedAtSeamStart = true;
+    parts.push(restored);
+  }
+  return { out: parts.join(""), quiescent, truncatedAtSeamStart, degradedReason: null };
+}
+function processSlice(slice, options) {
+  if (options.legacy === true) return processSliceLegacy(slice, options.probe);
+  return processSliceDefault(slice, options.probe, options.mask);
 }
 function isBlankRawLine(text, from, to) {
   for (let i = from; i < to; i++) {
@@ -5037,7 +5391,7 @@ function findRawSafeCut(active) {
   let backtickHazard = false;
   let latentLt = false;
   for (const segment of segments) {
-    if (segment.isCode) {
+    if (segment.kind !== "text") {
       if (segment.text.includes(">")) latentLt = false;
       offset += segment.text.length;
       continue;
@@ -5072,12 +5426,28 @@ function createIncrementalLatexPreprocessor(options) {
   const freezeThreshold = options?.freezeThreshold ?? DEFAULT_FREEZE_ATTEMPT_THRESHOLD;
   const onAttempt = options?.onAttempt;
   const backoff = options?.backoff ?? true;
+  const onDegrade = options?.onDegrade;
   let prevSource = "";
   let prevOutput = "";
   let frozenSrcEnd = 0;
   let frozenOut = "";
   let triggered = false;
   let nextAttemptLen = 0;
+  let lineageDegraded = false;
+  const presence = new PuaPresence();
+  const commit = (source, out) => {
+    prevSource = source;
+    prevOutput = out;
+    return out;
+  };
+  const legacyWhole = (source) => processSlice(source, { legacy: true, probe: false }).out;
+  const degrade = (source, reason) => {
+    lineageDegraded = true;
+    frozenSrcEnd = 0;
+    frozenOut = "";
+    onDegrade?.(reason);
+    return commit(source, legacyWhole(source));
+  };
   return function incrementalPreprocessLaTeX(source) {
     if (source === prevSource) return prevOutput;
     const isAppend = source.length > prevSource.length && source.startsWith(prevSource);
@@ -5086,16 +5456,20 @@ function createIncrementalLatexPreprocessor(options) {
       frozenOut = "";
       triggered = false;
       nextAttemptLen = 0;
+      lineageDegraded = false;
+      presence.reset();
+      presence.add(source, 0);
+    } else {
+      presence.add(source, prevSource.length);
     }
     if (!triggered) {
       const checkFrom = isAppend ? Math.max(0, prevSource.length - 1) : 0;
-      if (!hasLatexTrigger(source.slice(checkFrom))) {
-        prevSource = source;
-        prevOutput = source;
-        return source;
-      }
+      if (!hasLatexTrigger(source.slice(checkFrom))) return commit(source, source);
       triggered = true;
     }
+    if (lineageDegraded) return commit(source, legacyWhole(source));
+    const mask = presence.select();
+    if (mask === null) return degrade(source, "mask-exhausted");
     let active = source.slice(frozenSrcEnd);
     if (active.length > freezeThreshold && active.length >= nextAttemptLen) {
       const activeLength = active.length;
@@ -5110,18 +5484,20 @@ function createIncrementalLatexPreprocessor(options) {
       };
       const cut = findRawSafeCut(active);
       if (cut > 0) {
-        const candidate = processSlice(active.slice(0, cut));
+        const candidate = processSlice(active.slice(0, cut), { probe: true, mask });
+        if (candidate.degradedReason !== null) {
+          onAttempt?.({ activeLength, frozenBytes: 0 });
+          return degrade(source, candidate.degradedReason);
+        }
         if (candidate.quiescent) freeze(cut, candidate);
       }
       nextAttemptLen = advanced || !backoff ? 0 : active.length * 2;
       onAttempt?.({ activeLength, frozenBytes });
     }
-    const tail = processSlice(active, false);
+    const tail = processSlice(active, { probe: false, mask });
+    if (tail.degradedReason !== null) return degrade(source, tail.degradedReason);
     const head = tail.truncatedAtSeamStart ? frozenOut.replace(/\s+$/, "") : frozenOut;
-    const out = head + tail.out;
-    prevSource = source;
-    prevOutput = out;
-    return out;
+    return commit(source, head + tail.out);
   };
 }
 
@@ -5146,6 +5522,8 @@ function createRemendPreprocessor(options) {
 }
 export {
   DEFAULT_PAYLOAD,
+  ENGINE_PLACEHOLDER_TAGS,
+  ENGINE_PROVENANCE_PROPERTY,
   PIPELINE_STAGES,
   SENTINEL_FN_CONTENT,
   SENTINEL_LINK_URL,
@@ -5190,7 +5568,9 @@ export {
   preprocessLaTeX,
   rehypeFooterAdorn,
   rehypeRebaseHashLinks_default as rehypeRebaseHashLinks,
+  rehypeVerifyEngineTags,
   removeComments,
+  resolveCrossChunkReference,
   sanitizeCrossChunkUrl,
   sanitizeSchema,
   shortenDocumentId,