@ai-react-markdown/engine 2.10.1 → 2.11.0

package/dist/index.js

@@ -2701,6 +2701,44 @@ function rehypeUnwrapCrossChunkImages() {
 
 // src/components/pluginChain.ts
 import rehypeSanitize from "rehype-sanitize";
+
+// src/components/rehypeVerifyEngineTags.ts
+var ENGINE_PLACEHOLDER_TAGS = /* @__PURE__ */ new Set([
+  "footnote-sup",
+  "cross-chunk-link",
+  "cross-chunk-image"
+]);
+var ENGINE_PROVENANCE_PROPERTY = "engineProvenance";
+function walk(parent, provenance) {
+  const children = parent.children;
+  let i = 0;
+  while (i < children.length) {
+    const node = children[i];
+    if (node.type === "element" && ENGINE_PLACEHOLDER_TAGS.has(node.tagName)) {
+      const props = node.properties ?? {};
+      const stamped = props[ENGINE_PROVENANCE_PROPERTY];
+      const genuine = provenance !== "" && typeof stamped === "string" && stamped === provenance;
+      if (genuine) {
+        delete props[ENGINE_PROVENANCE_PROPERTY];
+        walk(node, provenance);
+        i += 1;
+      } else {
+        children.splice(i, 1, ...node.children);
+      }
+      continue;
+    }
+    if (node.type === "element") walk(node, provenance);
+    i += 1;
+  }
+}
+function rehypeVerifyEngineTags(options) {
+  const provenance = options?.provenance ?? "";
+  return function transformer(tree) {
+    walk(tree, provenance);
+  };
+}
+
+// src/components/pluginChain.ts
 import remarkBreaks from "remark-breaks";
 import remarkCjkFriendly from "remark-cjk-friendly";
 import remarkCjkFriendlyGfmStrikethrough from "remark-cjk-friendly-gfm-strikethrough";
@@ -2806,10 +2844,13 @@ function buildCoreRemarkPlugins(enginePlugins) {
     ...DISPLAY_OPTIMIZE_CHAIN.filter(([name]) => selected.has(name)).map(([, plugin]) => plugin)
   ];
 }
-function buildCoreRehypePlugins(sanitizeSchema2, clobberPrefix) {
+function buildCoreRehypePlugins(sanitizeSchema2, clobberPrefix, options) {
   return [
     // Allow raw HTML through so rehype-sanitize can handle it.
     [rehypeRaw, { passThrough: [] }],
+    // Unwrap forged engine placeholders BEFORE sanitize admits their tag
+    // names. Only when the caller holds a credential (see the option's doc).
+    ...options ? [[rehypeVerifyEngineTags, { provenance: options.provenance }]] : [],
     // Sanitize HTML while allowing <mark> (highlight), KaTeX class names,
     // and any extra protocols the caller permitted via the `sanitizeSchema`
     // prop. Override `clobberPrefix` with the instance-scoped value — the
@@ -2849,6 +2890,10 @@ function buildCoreRemarkRehypeOptions(enableDefinitionList) {
 }
 
 // src/components/customMdastHandlers.ts
+function provenanceProps(s) {
+  const provenance = s.options.provenance;
+  return typeof provenance === "string" ? { engineProvenance: provenance } : {};
+}
 function localDefProps(s, id) {
   const def = s.definitionById.get(id);
   if (!def || typeof def.url !== "string" || def.url === SENTINEL_LINK_URL) return {};
@@ -2887,7 +2932,8 @@ function buildCrossChunkHandlers() {
           label: node.label ?? node.identifier,
           referenceType: node.referenceType,
           documentId: s.options.documentId,
-          ...localDefProps(s, id)
+          ...localDefProps(s, id),
+          ...provenanceProps(s)
         },
         children: s.all(node)
       };
@@ -2905,7 +2951,8 @@ function buildCrossChunkHandlers() {
           referenceType: node.referenceType,
           alt: node.alt ?? "",
           documentId: s.options.documentId,
-          ...localDefProps(s, id)
+          ...localDefProps(s, id),
+          ...provenanceProps(s)
         },
         children: []
       };
@@ -2922,7 +2969,8 @@ function buildCrossChunkHandlers() {
           properties: {
             label: node.identifier,
             localOccurrence,
-            documentId: s.options.documentId
+            documentId: s.options.documentId,
+            ...provenanceProps(s)
           },
           children: []
         };
@@ -2940,7 +2988,8 @@ function buildCrossChunkHandlers() {
           // first client frame), where the local synthetic footer is what
           // renders, so marks and footer agree (core-render-02).
           localNumber: s.footnoteOrder.indexOf(id) + 1,
-          documentId: s.options.documentId
+          documentId: s.options.documentId,
+          ...provenanceProps(s)
         },
         children: []
       };
@@ -4600,6 +4649,12 @@ var createSmoothStreamController = (options = {}) => {
 };
 
 // src/preprocessors/latex.ts
+function isHardBoundary(kind) {
+  return kind === "code" || kind === "literal" || kind === "multilineTag";
+}
+function hasLineEnding(text) {
+  return text.includes("\n") || text.includes("\r");
+}
 function getRepeatedMarkerLength(content, start, marker) {
   let end = start;
   while (end < content.length && content[end] === marker) {
@@ -4667,11 +4722,11 @@ function splitByProtectedRegions(content) {
   let multilineFenceMarker = null;
   let multilineFenceLength = 0;
   let multilineFenceIndent = 0;
-  function pushProtected(start, end) {
+  function pushProtected(start, end, kind) {
     if (start > lastIndex) {
-      segments.push({ text: content.substring(lastIndex, start), isCode: false });
+      segments.push({ kind: "text", text: content.substring(lastIndex, start) });
     }
-    segments.push({ text: content.substring(start, end), isCode: true });
+    segments.push({ kind, text: content.substring(start, end) });
     lastIndex = end;
   }
   let i = 0;
@@ -4682,7 +4737,7 @@ function splitByProtectedRegions(content) {
         const runLen = getRepeatedMarkerLength(content, i, multilineFenceMarker);
         const closerIndent = lineIndentBefore(content, i);
         if (runLen >= multilineFenceLength && closerIndent !== -1 && closerIndent <= multilineFenceIndent + 3 && restOfLineIsBlank(content, i + runLen)) {
-          pushProtected(multilineStart, i + runLen);
+          pushProtected(multilineStart, i + runLen, "code");
           multilineStart = -1;
           multilineFenceMarker = null;
           multilineFenceLength = 0;
@@ -4709,7 +4764,7 @@ function splitByProtectedRegions(content) {
       if (char === "`") {
         const closeIdx = findClosingBacktickRun(content, i + runLen, runLen);
         if (closeIdx !== -1) {
-          pushProtected(i, closeIdx + runLen);
+          pushProtected(i, closeIdx + runLen, "code");
           i = closeIdx + runLen;
           continue;
         }
@@ -4734,7 +4789,11 @@ function splitByProtectedRegions(content) {
             endIndex = content.length;
           }
         }
-        pushProtected(i, endIndex);
+        pushProtected(
+          i,
+          endIndex,
+          isOpeningPairedTag ? "literal" : hasLineEnding(content.substring(i, endIndex)) ? "multilineTag" : "tag"
+        );
         i = endIndex;
         continue;
       }
@@ -4742,10 +4801,10 @@ function splitByProtectedRegions(content) {
     i += 1;
   }
   if (multilineStart !== -1) {
-    pushProtected(multilineStart, content.length);
+    pushProtected(multilineStart, content.length, "code");
   }
   if (lastIndex < content.length) {
-    segments.push({ text: content.substring(lastIndex), isCode: false });
+    segments.push({ kind: "text", text: content.substring(lastIndex) });
   }
   return segments;
 }
@@ -4899,20 +4958,22 @@ function escapeLatexPipesInUnclosed(text) {
   const tail = text.substring(unclosedStart + delimLen);
   return before + delim + replaceUnescapedPipes(tail);
 }
-function opensMathFlow(text, pos) {
+function opensMathFlow(text, pos, runStartsAtLineStart) {
   let i = pos;
   let spaces = 0;
-  while (i > 0 && text[i - 1] !== "\n") {
+  while (i > 0) {
+    const prev = text[i - 1];
+    if (prev === "\n" || prev === "\r") return true;
     i -= 1;
     if (text[i] !== " ") return false;
     spaces += 1;
     if (spaces > 3) return false;
   }
-  return true;
+  return runStartsAtLineStart;
 }
-function truncateUnclosedLatexBlock(text, unclosedStart = findUnclosedDelimiterStart(text, "double-only")) {
+function truncateUnclosedLatexBlock(text, runStartsAtLineStart, unclosedStart = findUnclosedDelimiterStart(text, "double-only")) {
   if (unclosedStart === -1) return text;
-  if (!opensMathFlow(text, unclosedStart)) return text;
+  if (!opensMathFlow(text, unclosedStart, runStartsAtLineStart)) return text;
   return text.substring(0, unclosedStart).trimEnd();
 }
 function escapeTextUnderscores(text) {
@@ -4957,7 +5018,8 @@ function convertSingleToDoubleDollar(text) {
 }
 function preprocessLaTeX(str) {
   if (!hasLatexTrigger(str)) return str;
-  return processSlice(str, false).out;
+  const mask = selectMask(str);
+  return (mask === null ? processSlice(str, { legacy: true, probe: false }) : processSlice(str, { probe: false, mask })).out;
 }
 function hasLatexTrigger(str) {
   return str.includes("$") || str.includes("\\[") || str.includes("\\(");
@@ -4986,42 +5048,240 @@ function hasUnclosedTextCommand(text) {
 }
 var RESIDUAL_OPEN_BRACKET_RE = /(?<!!)\\\[/;
 var LEADING_DOUBLE_DOLLAR_RE = /^\s*\$\$/;
-function processSlice(slice, probe = true) {
+var PUA_START = 57344;
+var PUA_END = 63743;
+var PUA_SIZE = PUA_END - PUA_START + 1;
+function selectMask(source) {
+  const first = String.fromCharCode(PUA_START);
+  if (source.indexOf(first) === -1) return first;
+  const seen = new Uint8Array(PUA_SIZE);
+  for (let i = 0; i < source.length; i++) {
+    const c = source.charCodeAt(i);
+    if (c >= PUA_START && c <= PUA_END) seen[c - PUA_START] = 1;
+  }
+  for (let k = 0; k < PUA_SIZE; k++) if (seen[k] === 0) return String.fromCharCode(PUA_START + k);
+  return null;
+}
+var PuaPresence = class {
+  bits = new Uint8Array(PUA_SIZE);
+  distinct = 0;
+  reset() {
+    this.bits.fill(0);
+    this.distinct = 0;
+  }
+  add(text, from) {
+    for (let i = from; i < text.length; i++) {
+      const c = text.charCodeAt(i);
+      if (c >= PUA_START && c <= PUA_END) {
+        const k = c - PUA_START;
+        if (this.bits[k] === 0) {
+          this.bits[k] = 1;
+          this.distinct += 1;
+        }
+      }
+    }
+  }
+  select() {
+    if (this.distinct === 0) return String.fromCharCode(PUA_START);
+    if (this.distinct >= PUA_SIZE) return null;
+    for (let k = 0; k < PUA_SIZE; k++) if (this.bits[k] === 0) return String.fromCharCode(PUA_START + k);
+    return null;
+  }
+};
+function transformRun(input, probe, runStartsAtLineStart, seamEligible) {
+  let text = input;
+  let tailSensitive = false;
+  let truncatedAtSeamStart = false;
+  text = escapeMhchemCommands(text);
+  text = escapeCurrencyDollarSigns(text);
+  text = convertLatexDelimiters(text);
+  if (probe && RESIDUAL_OPEN_BRACKET_RE.test(text)) tailSensitive = true;
+  text = escapeLatexPipes(text);
+  if (probe && findUnclosedDelimiterStart(text, "both") !== -1) tailSensitive = true;
+  text = escapeLatexPipesInUnclosed(text);
+  if (probe && hasUnclosedTextCommand(text)) tailSensitive = true;
+  text = escapeTextUnderscores(text);
+  text = convertSingleToDoubleDollar(text);
+  let unclosedDouble;
+  if (probe || seamEligible && LEADING_DOUBLE_DOLLAR_RE.test(text)) {
+    unclosedDouble = findUnclosedDelimiterStart(text, "double-only");
+    if (unclosedDouble !== -1) {
+      tailSensitive = true;
+      if (seamEligible && opensMathFlow(text, unclosedDouble, runStartsAtLineStart) && text.slice(0, unclosedDouble).trim() === "") {
+        truncatedAtSeamStart = true;
+      }
+    }
+  }
+  text = truncateUnclosedLatexBlock(text, runStartsAtLineStart, unclosedDouble);
+  return { out: text, tailSensitive, truncatedAtSeamStart };
+}
+function processSliceLegacy(slice, probe) {
   const segments = splitByProtectedRegions(slice);
   const parts = [];
   let quiescent = true;
   let truncatedAtSeamStart = false;
   for (let index = 0; index < segments.length; index++) {
     const segment = segments[index];
-    if (segment.isCode) {
+    if (segment.kind !== "text") {
       parts.push(segment.text);
       continue;
     }
-    let text = segment.text;
-    text = escapeMhchemCommands(text);
-    text = escapeCurrencyDollarSigns(text);
-    text = convertLatexDelimiters(text);
-    if (probe && RESIDUAL_OPEN_BRACKET_RE.test(text)) quiescent = false;
-    text = escapeLatexPipes(text);
-    if (probe && findUnclosedDelimiterStart(text, "both") !== -1) quiescent = false;
-    text = escapeLatexPipesInUnclosed(text);
-    if (probe && hasUnclosedTextCommand(text)) quiescent = false;
-    text = escapeTextUnderscores(text);
-    text = convertSingleToDoubleDollar(text);
-    let unclosedDouble;
-    if (probe || index === 0 && LEADING_DOUBLE_DOLLAR_RE.test(text)) {
-      unclosedDouble = findUnclosedDelimiterStart(text, "double-only");
-      if (unclosedDouble !== -1) {
-        quiescent = false;
-        if (index === 0 && opensMathFlow(text, unclosedDouble) && text.slice(0, unclosedDouble).trim() === "") {
-          truncatedAtSeamStart = true;
+    const r = transformRun(segment.text, probe, true, index === 0);
+    if (r.tailSensitive) quiescent = false;
+    if (r.truncatedAtSeamStart) truncatedAtSeamStart = true;
+    parts.push(r.out);
+  }
+  return { out: parts.join(""), quiescent, truncatedAtSeamStart, degradedReason: null };
+}
+var SCOPE_DEPTH_CAP = 8;
+var VOID_TAGS2 = /* @__PURE__ */ new Set(["br", "hr", "img", "wbr", "input", "source"]);
+var TAG_NAME_RE = /^<(\/?)([A-Za-z][A-Za-z0-9]*)/;
+function tagInfo(tag) {
+  const m = TAG_NAME_RE.exec(tag);
+  const closing = m?.[1] === "/";
+  const name = (m?.[2] ?? "").toLowerCase();
+  const opensScope = !closing && !tag.endsWith("/>") && !VOID_TAGS2.has(name);
+  return { name, closing, opensScope };
+}
+function buildRunTree(segments) {
+  const root2 = [];
+  const stack = [];
+  const current = () => stack.length > 0 ? stack[stack.length - 1].children : root2;
+  const unwind = () => {
+    while (stack.length > 0) {
+      const frame = stack.pop();
+      current().push({ type: "atom", text: frame.open }, ...frame.children);
+    }
+  };
+  for (const segment of segments) {
+    if (segment.kind === "tag") {
+      const info = tagInfo(segment.text);
+      if (info.closing) {
+        const top = stack[stack.length - 1];
+        if (top !== void 0 && top.name === info.name) {
+          stack.pop();
+          const parent = current();
+          if (top.suppressed) {
+            parent.push({ type: "atom", text: top.open }, ...top.children, { type: "atom", text: segment.text });
+          } else {
+            parent.push({ type: "scope", open: top.open, close: segment.text, children: top.children });
+          }
+        } else {
+          current().push({ type: "atom", text: segment.text });
         }
+      } else if (info.opensScope) {
+        stack.push({ name: info.name, open: segment.text, suppressed: stack.length >= SCOPE_DEPTH_CAP, children: [] });
+      } else {
+        current().push({ type: "atom", text: segment.text });
       }
+      continue;
     }
-    text = truncateUnclosedLatexBlock(text, unclosedDouble);
-    parts.push(text);
+    const t = segment.text;
+    let start = 0;
+    for (let i = 0; i < t.length; i++) {
+      const c = t.charCodeAt(i);
+      if (c !== 10 && c !== 13) continue;
+      if (i > start) current().push({ type: "text", text: t.slice(start, i) });
+      const end = c === 13 && t.charCodeAt(i + 1) === 10 ? i + 2 : i + 1;
+      unwind();
+      root2.push({ type: "text", text: t.slice(i, end) });
+      start = end;
+      i = end - 1;
+    }
+    if (start < t.length) current().push({ type: "text", text: t.slice(start) });
+  }
+  unwind();
+  return root2;
+}
+var restoreFailureInjector = null;
+function restore(out, atoms, mask) {
+  if (restoreFailureInjector !== null && restoreFailureInjector(atoms)) return null;
+  let result = "";
+  let k = 0;
+  let last = 0;
+  for (; ; ) {
+    const idx = out.indexOf(mask, last);
+    if (idx === -1) break;
+    if (k >= atoms.length) return null;
+    result += out.slice(last, idx) + atoms[k];
+    k += 1;
+    last = idx + 1;
+  }
+  return result + out.slice(last);
+}
+function emitRun(nodes, mask) {
+  let text = "";
+  const atoms = [];
+  for (const node of nodes) {
+    if (node.type === "text") {
+      text += node.text;
+    } else if (node.type === "atom") {
+      atoms.push(node.text);
+      text += mask;
+    } else {
+      const inner = emitRun(node.children, mask);
+      if (inner === null) return null;
+      const transformed = transformRun(inner.text, false, false, false);
+      const restored = restore(transformed.out, inner.atoms, mask);
+      if (restored === null) return null;
+      atoms.push(node.open + restored + node.close);
+      text += mask;
+    }
+  }
+  return { text, atoms };
+}
+function reportRestoreViolation() {
+  if (false) {
+    console.error(
+      "[ai-react-markdown] LaTeX preprocessor: atom restoration violated its invariant (more masks in the output than atoms); the slice was re-processed on the legacy path. This is an engine defect \u2014 please report it."
+    );
   }
-  return { out: parts.join(""), quiescent, truncatedAtSeamStart };
+}
+function processSliceDefault(slice, probe, mask) {
+  const segments = splitByProtectedRegions(slice);
+  const parts = [];
+  let quiescent = true;
+  let truncatedAtSeamStart = false;
+  let offset = 0;
+  let i = 0;
+  while (i < segments.length) {
+    const segment = segments[i];
+    if (isHardBoundary(segment.kind)) {
+      parts.push(segment.text);
+      offset += segment.text.length;
+      i += 1;
+      continue;
+    }
+    const runStart = offset;
+    const runSegments = [];
+    while (i < segments.length && !isHardBoundary(segments[i].kind)) {
+      runSegments.push(segments[i]);
+      offset += segments[i].text.length;
+      i += 1;
+    }
+    const before = runStart === 0 ? -1 : slice.charCodeAt(runStart - 1);
+    const runStartsAtLineStart = before === -1 || before === 10 || before === 13;
+    const seamEligible = runStart === 0;
+    const emitted = emitRun(buildRunTree(runSegments), mask);
+    if (emitted === null) {
+      reportRestoreViolation();
+      return { ...processSliceLegacy(slice, probe), degradedReason: "restore-invariant" };
+    }
+    const r = transformRun(emitted.text, probe, runStartsAtLineStart, seamEligible);
+    const restored = restore(r.out, emitted.atoms, mask);
+    if (restored === null) {
+      reportRestoreViolation();
+      return { ...processSliceLegacy(slice, probe), degradedReason: "restore-invariant" };
+    }
+    if (r.tailSensitive) quiescent = false;
+    if (r.truncatedAtSeamStart) truncatedAtSeamStart = true;
+    parts.push(restored);
+  }
+  return { out: parts.join(""), quiescent, truncatedAtSeamStart, degradedReason: null };
+}
+function processSlice(slice, options) {
+  if (options.legacy === true) return processSliceLegacy(slice, options.probe);
+  return processSliceDefault(slice, options.probe, options.mask);
 }
 function isBlankRawLine(text, from, to) {
   for (let i = from; i < to; i++) {
@@ -5037,7 +5297,7 @@ function findRawSafeCut(active) {
   let backtickHazard = false;
   let latentLt = false;
   for (const segment of segments) {
-    if (segment.isCode) {
+    if (segment.kind !== "text") {
       if (segment.text.includes(">")) latentLt = false;
       offset += segment.text.length;
       continue;
@@ -5072,12 +5332,28 @@ function createIncrementalLatexPreprocessor(options) {
   const freezeThreshold = options?.freezeThreshold ?? DEFAULT_FREEZE_ATTEMPT_THRESHOLD;
   const onAttempt = options?.onAttempt;
   const backoff = options?.backoff ?? true;
+  const onDegrade = options?.onDegrade;
   let prevSource = "";
   let prevOutput = "";
   let frozenSrcEnd = 0;
   let frozenOut = "";
   let triggered = false;
   let nextAttemptLen = 0;
+  let lineageDegraded = false;
+  const presence = new PuaPresence();
+  const commit = (source, out) => {
+    prevSource = source;
+    prevOutput = out;
+    return out;
+  };
+  const legacyWhole = (source) => processSlice(source, { legacy: true, probe: false }).out;
+  const degrade = (source, reason) => {
+    lineageDegraded = true;
+    frozenSrcEnd = 0;
+    frozenOut = "";
+    onDegrade?.(reason);
+    return commit(source, legacyWhole(source));
+  };
   return function incrementalPreprocessLaTeX(source) {
     if (source === prevSource) return prevOutput;
     const isAppend = source.length > prevSource.length && source.startsWith(prevSource);
@@ -5086,16 +5362,20 @@ function createIncrementalLatexPreprocessor(options) {
       frozenOut = "";
       triggered = false;
       nextAttemptLen = 0;
+      lineageDegraded = false;
+      presence.reset();
+      presence.add(source, 0);
+    } else {
+      presence.add(source, prevSource.length);
     }
     if (!triggered) {
       const checkFrom = isAppend ? Math.max(0, prevSource.length - 1) : 0;
-      if (!hasLatexTrigger(source.slice(checkFrom))) {
-        prevSource = source;
-        prevOutput = source;
-        return source;
-      }
+      if (!hasLatexTrigger(source.slice(checkFrom))) return commit(source, source);
       triggered = true;
     }
+    if (lineageDegraded) return commit(source, legacyWhole(source));
+    const mask = presence.select();
+    if (mask === null) return degrade(source, "mask-exhausted");
     let active = source.slice(frozenSrcEnd);
     if (active.length > freezeThreshold && active.length >= nextAttemptLen) {
       const activeLength = active.length;
@@ -5110,18 +5390,20 @@ function createIncrementalLatexPreprocessor(options) {
       };
       const cut = findRawSafeCut(active);
       if (cut > 0) {
-        const candidate = processSlice(active.slice(0, cut));
+        const candidate = processSlice(active.slice(0, cut), { probe: true, mask });
+        if (candidate.degradedReason !== null) {
+          onAttempt?.({ activeLength, frozenBytes: 0 });
+          return degrade(source, candidate.degradedReason);
+        }
         if (candidate.quiescent) freeze(cut, candidate);
       }
       nextAttemptLen = advanced || !backoff ? 0 : active.length * 2;
       onAttempt?.({ activeLength, frozenBytes });
     }
-    const tail = processSlice(active, false);
+    const tail = processSlice(active, { probe: false, mask });
+    if (tail.degradedReason !== null) return degrade(source, tail.degradedReason);
     const head = tail.truncatedAtSeamStart ? frozenOut.replace(/\s+$/, "") : frozenOut;
-    const out = head + tail.out;
-    prevSource = source;
-    prevOutput = out;
-    return out;
+    return commit(source, head + tail.out);
   };
 }
 
@@ -5146,6 +5428,8 @@ function createRemendPreprocessor(options) {
 }
 export {
   DEFAULT_PAYLOAD,
+  ENGINE_PLACEHOLDER_TAGS,
+  ENGINE_PROVENANCE_PROPERTY,
   PIPELINE_STAGES,
   SENTINEL_FN_CONTENT,
   SENTINEL_LINK_URL,
@@ -5190,6 +5474,7 @@ export {
   preprocessLaTeX,
   rehypeFooterAdorn,
   rehypeRebaseHashLinks_default as rehypeRebaseHashLinks,
+  rehypeVerifyEngineTags,
   removeComments,
   sanitizeCrossChunkUrl,
   sanitizeSchema,