@stream-mdx/core 0.1.0 → 0.2.0

package/dist/index.cjs

@@ -67,12 +67,14 @@ __export(index_exports, {
   findClosingHtmlTag: () => findClosingHtmlTag,
   generateBlockId: () => generateBlockId,
   getBlockKey: () => getBlockKey,
+  getDefaultCodeWrapperAttributes: () => getDefaultCodeWrapperAttributes,
   initializeSecurity: () => initializeSecurity,
   initializeTrustedTypesPolicy: () => initializeTrustedTypesPolicy,
   inlineNodesToPlainText: () => inlineNodesToPlainText,
   isLikelyMdxComponent: () => isLikelyMdxComponent,
   normalizeBlockquoteText: () => normalizeBlockquoteText,
   normalizeFormatAnticipation: () => normalizeFormatAnticipation,
+  normalizeHighlightedLines: () => normalizeHighlightedLines,
   normalizeLang: () => normalizeLang,
   parseCodeFenceInfo: () => parseCodeFenceInfo,
   prepareInlineStreamingContent: () => prepareInlineStreamingContent,
@@ -230,6 +232,22 @@ function manualExtractHighlightedLines(html, fallbackLength) {
   }
   return normalizeHighlightedLines(lines, fallbackLength);
 }
+function getDefaultCodeWrapperAttributes(lang, themes = { dark: "github-dark", light: "github-light" }) {
+  const language = lang && typeof lang === "string" && lang.length > 0 ? lang : "text";
+  const themeLabel = `${themes.dark} ${themes.light}`;
+  return {
+    preAttrs: {
+      class: `shiki shiki-themes ${themeLabel}`,
+      "data-language": language,
+      style: "--shiki-dark-bg: transparent; --shiki-light-bg: transparent"
+    },
+    codeAttrs: {
+      "data-language": language,
+      "data-theme": themeLabel,
+      style: "display: grid;"
+    }
+  };
+}
 function dedentIndentedCode(raw) {
   if (!raw) return "";
   const normalized = normalizeNewlines(raw);
@@ -636,6 +654,130 @@ function splitTextByRegexWithPrecedence(text, regex, toNode) {
   return result;
 }
 
+// src/streaming/inline-streaming.ts
+var DEFAULT_FORMAT_ANTICIPATION = {
+  inline: false,
+  mathInline: false,
+  mathBlock: false,
+  html: false,
+  mdx: false,
+  regex: false
+};
+function normalizeFormatAnticipation(input) {
+  if (input === true) {
+    return { ...DEFAULT_FORMAT_ANTICIPATION, inline: true };
+  }
+  if (!input) {
+    return { ...DEFAULT_FORMAT_ANTICIPATION };
+  }
+  return {
+    inline: input.inline ?? false,
+    mathInline: input.mathInline ?? false,
+    mathBlock: input.mathBlock ?? false,
+    html: input.html ?? false,
+    mdx: input.mdx ?? false,
+    regex: input.regex ?? false
+  };
+}
+function prepareInlineStreamingContent(content, options) {
+  const enableMath = options?.math !== false;
+  const anticipation = normalizeFormatAnticipation(options?.formatAnticipation);
+  const enableInlineAnticipation = anticipation.inline;
+  const enableMathInlineAnticipation = anticipation.mathInline;
+  const enableMathBlockAnticipation = anticipation.mathBlock;
+  const stack = [];
+  const toggleToken = (token) => {
+    const last = stack[stack.length - 1];
+    if (last === token) {
+      stack.pop();
+    } else {
+      stack.push(token);
+    }
+  };
+  let mathDisplayOpen = false;
+  let mathDisplayCrossedNewline = false;
+  for (let i = 0; i < content.length; i++) {
+    const code = content.charCodeAt(i);
+    if (code === 10 || code === 13) {
+      if (mathDisplayOpen) {
+        mathDisplayCrossedNewline = true;
+      }
+      continue;
+    }
+    if (code === 96) {
+      toggleToken("code");
+      continue;
+    }
+    if (code === 126 && i + 1 < content.length && content.charCodeAt(i + 1) === 126) {
+      toggleToken("strike");
+      i += 1;
+      continue;
+    }
+    if (code === 42) {
+      if (i + 1 < content.length && content.charCodeAt(i + 1) === 42) {
+        toggleToken("strong");
+        i += 1;
+      } else {
+        toggleToken("em");
+      }
+      continue;
+    }
+    if (enableMath && code === 36) {
+      if (i + 1 < content.length && content.charCodeAt(i + 1) === 36) {
+        toggleToken("math-display");
+        if (mathDisplayOpen) {
+          mathDisplayOpen = false;
+          mathDisplayCrossedNewline = false;
+        } else {
+          mathDisplayOpen = true;
+          mathDisplayCrossedNewline = false;
+        }
+        i += 1;
+      } else {
+        toggleToken("math-inline");
+      }
+    }
+  }
+  const hasIncompleteFormatting = stack.some((token) => token === "code" || token === "strike" || token === "strong" || token === "em");
+  const hasIncompleteMathInline = stack.includes("math-inline");
+  const hasIncompleteMathDisplay = stack.includes("math-display");
+  const hasIncompleteMath = hasIncompleteMathInline || hasIncompleteMathDisplay;
+  if (enableMath && hasIncompleteMath) {
+    if (hasIncompleteMathInline && !enableMathInlineAnticipation) {
+      return { kind: "raw", status: "raw", reason: "incomplete-math" };
+    }
+    if (hasIncompleteMathDisplay && (!enableMathBlockAnticipation || mathDisplayCrossedNewline)) {
+      return { kind: "raw", status: "raw", reason: "incomplete-math" };
+    }
+  }
+  if (hasIncompleteFormatting && !enableInlineAnticipation) {
+    return { kind: "raw", status: "raw", reason: "incomplete-formatting" };
+  }
+  if (!hasIncompleteFormatting && !hasIncompleteMath) {
+    return { kind: "parse", status: "complete", content, appended: "" };
+  }
+  const appendForToken = (token) => {
+    switch (token) {
+      case "code":
+        return "`";
+      case "strike":
+        return "~~";
+      case "strong":
+        return "**";
+      case "em":
+        return "*";
+      case "math-inline":
+        return "$";
+      case "math-display":
+        return "$$";
+      default:
+        return "";
+    }
+  };
+  const appended = stack.slice().reverse().map((token) => appendForToken(token)).join("");
+  return { kind: "parse", status: "anticipated", content: content + appended, appended };
+}
+
 // src/worker-html-sanitizer.ts
 var rehypeParse = __toESM(require("rehype-parse"), 1);
 var rehypeSanitize = __toESM(require("rehype-sanitize"), 1);
@@ -817,6 +959,8 @@ function splitByTagSegments(source, baseOffset, parseInline, options) {
   const mdxAutoClose = options?.mdx?.autoClose === true;
   const mdxMaxNewlines = normalizeNewlineLimit(options?.mdx?.maxNewlines);
   const mdxAllowlist = normalizeComponentAllowlist(options?.mdx?.componentAllowlist);
+  const protectedRanges = options?.protectedRanges ?? [];
+  const protectedKinds = protectedRanges.length ? new Set(options?.protectedRangeKinds ?? ["math-inline", "math-display", "code-inline", "code-block", "autolink"]) : null;
   while (match !== null) {
     const start = match.index;
     const tagName = match[1];
@@ -831,6 +975,18 @@ function splitByTagSegments(source, baseOffset, parseInline, options) {
       continue;
     }
     let end = tagPattern.lastIndex;
+    if (protectedKinds && protectedRanges.length > 0) {
+      const absoluteStart = baseIsFinite ? baseOffset + start : start;
+      const absoluteEnd = baseIsFinite ? baseOffset + end : end;
+      const covered = protectedRanges.some(
+        (range) => protectedKinds.has(range.kind) && typeof range.from === "number" && typeof range.to === "number" && range.from <= absoluteStart && range.to >= absoluteEnd
+      );
+      if (covered) {
+        tagPattern.lastIndex = start + 1;
+        match = tagPattern.exec(source);
+        continue;
+      }
+    }
     if (!isSelfClosing && !mdxAllowed) {
       const closingIndex = findClosingHtmlTag(lowerSource, tagName.toLowerCase(), end);
       if (closingIndex === -1) {
@@ -1128,23 +1284,33 @@ function getBlockKey(block) {
 }
 function detectMDX(content, options) {
   const inlineCodeRanges = collectInlineCodeRanges(content);
+  const protectedRanges = options?.protectedRanges ?? [];
+  const baseOffset = typeof options?.baseOffset === "number" ? options.baseOffset : 0;
+  const protectedKinds = protectedRanges.length ? /* @__PURE__ */ new Set(["math-inline", "math-display", "code-inline", "code-block", "html-inline", "html-block", "autolink"]) : null;
   const componentPattern = /<([A-Z][\w-]*)(\s|\/?>)/g;
   let componentMatch = componentPattern.exec(content);
   while (componentMatch !== null) {
     const start = componentMatch.index;
     const end = start + componentMatch[0].length;
-    if (!isWithinRanges(start, end, inlineCodeRanges)) {
-      return true;
+    if (isWithinRanges(start, end, inlineCodeRanges)) {
+      componentMatch = componentPattern.exec(content);
+      continue;
+    }
+    if (protectedKinds) {
+      const absoluteStart = baseOffset + start;
+      const absoluteEnd = baseOffset + end;
+      const covered = protectedRanges.some((range) => protectedKinds.has(range.kind) && range.from <= absoluteStart && range.to >= absoluteEnd);
+      if (covered) {
+        componentMatch = componentPattern.exec(content);
+        continue;
+      }
     }
-    componentMatch = componentPattern.exec(content);
+    return true;
   }
   if (/(^|\n)\s*(import|export)\s/.test(content)) {
     return true;
   }
   const expressionPattern = /\{[^{}]+\}/g;
-  const protectedRanges = options?.protectedRanges ?? [];
-  const baseOffset = typeof options?.baseOffset === "number" ? options.baseOffset : 0;
-  const protectedKinds = protectedRanges.length ? /* @__PURE__ */ new Set(["math-inline", "math-display", "code-inline", "code-block"]) : null;
   for (let match = expressionPattern.exec(content); match !== null; match = expressionPattern.exec(content)) {
     const index = match.index;
     const prev = index > 0 ? content[index - 1] : "";
@@ -1392,7 +1558,12 @@ function buildListItemSnapshot(block, listItemNode, ordered, index, id, baseOffs
       }
       if (name === "Paragraph") {
         const paragraphRaw = raw.slice(cursor.from, cursor.to);
-        const paragraphData = processListItemParagraph(paragraphRaw);
+        const meta = block.payload.meta ?? {};
+        const paragraphData = processListItemParagraph(paragraphRaw, {
+          formatAnticipation: meta.formatAnticipation,
+          math: meta.mathEnabled,
+          streaming: !block.isFinalized
+        });
         const parsedInline = paragraphData.inline;
         if (!paragraphHandled) {
           inlineNodes = parsedInline;
@@ -1424,7 +1595,10 @@ function buildListItemSnapshot(block, listItemNode, ordered, index, id, baseOffs
       } else if (name === "BulletList" || name === "OrderedList") {
         const nestedId = `${id}::list:${subListIndex++}`;
         const nestedOrdered = name === "OrderedList";
-        childSnapshots.push(buildListNodeSnapshot(block, cursor.node, nestedOrdered, nestedId, baseOffset, raw));
+        const nestedSnapshot = buildListNodeSnapshot(block, cursor.node, nestedOrdered, nestedId, baseOffset, raw);
+        if (Array.isArray(nestedSnapshot.children) && nestedSnapshot.children.length > 0) {
+          childSnapshots.push(nestedSnapshot);
+        }
       } else if (name === "Blockquote") {
         const quoteId = `${id}::blockquote:${blockquoteIndex++}`;
         childSnapshots.push(buildBlockquoteSnapshot(block, cursor.node, quoteId, baseOffset, raw));
@@ -1460,11 +1634,31 @@ function buildListItemSnapshot(block, listItemNode, ordered, index, id, baseOffs
   };
   return itemSnapshot;
 }
-function processListItemParagraph(raw) {
+function parseListInline(raw, options) {
+  if (!options?.streaming || !options.formatAnticipation) {
+    return listInlineParser.parse(raw);
+  }
+  const prepared = prepareInlineStreamingContent(raw, { formatAnticipation: options.formatAnticipation, math: options.math });
+  if (prepared.kind === "raw") {
+    return [{ kind: "text", text: raw }];
+  }
+  let preparedContent = prepared.content;
+  let appended = prepared.appended;
+  const normalized = normalizeFormatAnticipation(options.formatAnticipation);
+  if (normalized.regex) {
+    const regexAppend = listInlineParser.getRegexAnticipationAppend(raw);
+    if (regexAppend) {
+      preparedContent += regexAppend;
+      appended += regexAppend;
+    }
+  }
+  return listInlineParser.parse(preparedContent, { cache: false });
+}
+function processListItemParagraph(raw, options) {
   const normalized = normalizeParagraphText(raw);
   const { content, task } = stripTaskMarker(normalized);
-  const inline = listInlineParser.parse(content);
-  const segments = extractMixedContentSegments(content, void 0, (value) => listInlineParser.parse(value));
+  const inline = parseListInline(content, options);
+  const segments = extractMixedContentSegments(content, void 0, (value) => parseListInline(value, options));
   return {
     inline,
     segments,
@@ -1749,28 +1943,82 @@ function enrichTableSnapshot(block, snapshot) {
 function enrichCodeSnapshot(block, snapshot) {
   const source = typeof block.payload.meta?.code === "string" ? block.payload.meta?.code : block.payload.raw ?? "";
   const lines = extractCodeLines(source);
+  const meta = block.payload.meta;
   const highlightedHtml = block.payload.highlightedHtml ?? "";
-  const highlightedLines = extractHighlightedLines(highlightedHtml, lines.length);
-  const { preAttrs, codeAttrs } = extractCodeWrapperAttributes(highlightedHtml);
-  const lang = typeof block.payload.meta?.lang === "string" ? String(block.payload.meta?.lang) : void 0;
+  const hasBlockHighlight = typeof block.payload.highlightedHtml === "string" && block.payload.highlightedHtml.length > 0;
+  const metaLines = Array.isArray(meta?.highlightedLines) ? meta?.highlightedLines : null;
+  const includeLineHtml = metaLines ? true : !hasBlockHighlight || lines.length >= 200;
+  const highlightedLines = metaLines ? normalizeHighlightedLines(metaLines, lines.length) : extractHighlightedLines(highlightedHtml, lines.length);
+  const metaTokenLines = Array.isArray(meta?.tokenLines) ? meta?.tokenLines : null;
+  const tokenLines = metaTokenLines ? normalizeTokenLines(metaTokenLines, lines.length) : null;
+  const metaDiffKind = Array.isArray(meta?.diffKind) ? meta?.diffKind : null;
+  const diffKindLines = metaDiffKind ? normalizeOptionalArray(metaDiffKind, lines.length) : null;
+  const metaOldNo = Array.isArray(meta?.oldNo) ? meta?.oldNo : null;
+  const oldNoLines = metaOldNo ? normalizeOptionalArray(metaOldNo, lines.length) : null;
+  const metaNewNo = Array.isArray(meta?.newNo) ? meta?.newNo : null;
+  const newNoLines = metaNewNo ? normalizeOptionalArray(metaNewNo, lines.length) : null;
+  const lang = typeof meta?.lang === "string" ? String(meta.lang) : void 0;
+  let { preAttrs, codeAttrs } = extractCodeWrapperAttributes(highlightedHtml);
+  if (!preAttrs || !codeAttrs) {
+    const defaults = getDefaultCodeWrapperAttributes(lang);
+    preAttrs = preAttrs ?? defaults.preAttrs;
+    codeAttrs = codeAttrs ?? defaults.codeAttrs;
+  }
   snapshot.props = {
     ...snapshot.props ?? {},
     lang,
     preAttrs,
     codeAttrs
   };
-  snapshot.children = lines.map((line, index) => ({
-    id: `${block.id}::line:${index}`,
-    type: "code-line",
-    props: {
+  snapshot.children = lines.map((line, index) => {
+    const props = {
       index,
       text: line,
-      html: highlightedLines[index] ?? null
-    },
-    children: []
-  }));
+      html: includeLineHtml ? highlightedLines[index] ?? null : null
+    };
+    if (tokenLines) {
+      props.tokens = tokenLines[index] ?? null;
+    }
+    if (diffKindLines) {
+      props.diffKind = diffKindLines[index] ?? null;
+    }
+    if (oldNoLines) {
+      props.oldNo = oldNoLines[index] ?? null;
+    }
+    if (newNoLines) {
+      props.newNo = newNoLines[index] ?? null;
+    }
+    return {
+      id: `${block.id}::line:${index}`,
+      type: "code-line",
+      props,
+      children: []
+    };
+  });
   return snapshot;
 }
+function normalizeTokenLines(lines, fallbackLength) {
+  if (!lines || lines.length === 0) {
+    return new Array(Math.max(0, fallbackLength)).fill(null);
+  }
+  const length = Math.max(fallbackLength, lines.length);
+  const result = new Array(length).fill(null);
+  for (let i = 0; i < lines.length; i++) {
+    result[i] = lines[i] ?? null;
+  }
+  return result;
+}
+function normalizeOptionalArray(lines, fallbackLength) {
+  if (!lines || lines.length === 0) {
+    return new Array(Math.max(0, fallbackLength)).fill(null);
+  }
+  const length = Math.max(fallbackLength, lines.length);
+  const result = new Array(length).fill(null);
+  for (let i = 0; i < lines.length; i++) {
+    result[i] = lines[i] ?? null;
+  }
+  return result;
+}
 function cloneInlineNodes(nodes) {
   return nodes.map((node) => {
     if ("children" in node && Array.isArray(node.children)) {
@@ -1939,7 +2187,17 @@ function applyPatchBatch(snapshot, patches) {
       case "appendLines": {
         const parent = resolveTargetNode(snapshot, patch.at);
         if (!parent) break;
-        appendLinesToCodeNode(snapshot, parent, patch.startIndex, patch.lines ?? [], patch.highlight ?? []);
+        appendLinesToCodeNode(
+          snapshot,
+          parent,
+          patch.startIndex,
+          patch.lines ?? [],
+          patch.highlight ?? [],
+          patch.tokens,
+          patch.diffKind,
+          patch.oldNo,
+          patch.newNo
+        );
         break;
       }
       case "setHTML": {
@@ -2061,7 +2319,7 @@ function applyPropsToNode(node, props) {
   }
   node.props = next;
 }
-function appendLinesToCodeNode(snapshot, parent, startIndex, lines, highlights) {
+function appendLinesToCodeNode(snapshot, parent, startIndex, lines, highlights, tokens, diffKind, oldNo, newNo) {
   if (parent.type !== "code") return;
   const insertionIndex = Math.max(0, Math.min(startIndex, parent.children.length));
   let currentIndex = insertionIndex;
@@ -2079,6 +2337,18 @@ function appendLinesToCodeNode(snapshot, parent, startIndex, lines, highlights)
         html: highlights[i] ?? null
       }
     };
+    if (tokens && Object.prototype.hasOwnProperty.call(tokens, i)) {
+      child.props.tokens = tokens[i] ?? null;
+    }
+    if (diffKind && Object.prototype.hasOwnProperty.call(diffKind, i)) {
+      child.props.diffKind = diffKind[i] ?? null;
+    }
+    if (oldNo && Object.prototype.hasOwnProperty.call(oldNo, i)) {
+      child.props.oldNo = oldNo[i] ?? null;
+    }
+    if (newNo && Object.prototype.hasOwnProperty.call(newNo, i)) {
+      child.props.newNo = newNo[i] ?? null;
+    }
     snapshot.nodes.set(child.id, child);
     parent.children.splice(currentIndex, 0, child.id);
     currentIndex++;
@@ -2634,6 +2904,14 @@ function mergeAppendLines(window2, startIndex) {
   }
   const lines = [...base.lines ?? []];
   const highlight = Array.isArray(base.highlight) ? [...base.highlight] : [];
+  const tokens = Array.isArray(base.tokens) ? [...base.tokens] : [];
+  const diffKind = Array.isArray(base.diffKind) ? [...base.diffKind] : [];
+  const oldNo = Array.isArray(base.oldNo) ? [...base.oldNo] : [];
+  const newNo = Array.isArray(base.newNo) ? [...base.newNo] : [];
+  let includeTokens = Array.isArray(base.tokens);
+  let includeDiffKind = Array.isArray(base.diffKind);
+  let includeOldNo = Array.isArray(base.oldNo);
+  let includeNewNo = Array.isArray(base.newNo);
   const baseStart = base.startIndex;
   let expectedStart = baseStart + lines.length;
   let j = startIndex + 1;
@@ -2641,6 +2919,7 @@ function mergeAppendLines(window2, startIndex) {
   while (j < window2.length && mergedCount < APPEND_MERGE_LIMIT) {
     const next = window2[j];
     if (next.op === "appendLines" && next.at.blockId === base.at.blockId && next.at.nodeId === base.at.nodeId && typeof next.startIndex === "number" && next.startIndex === expectedStart) {
+      const priorLineCount = lines.length;
       lines.push(...next.lines ?? []);
       const nextHighlights = Array.isArray(next.highlight) ? next.highlight : [];
       const appendedCount = next.lines?.length ?? 0;
@@ -2654,6 +2933,50 @@ function mergeAppendLines(window2, startIndex) {
           highlight.push(null);
         }
       }
+      if (Array.isArray(next.tokens)) {
+        if (!includeTokens) {
+          tokens.push(...new Array(priorLineCount).fill(null));
+          includeTokens = true;
+        }
+        for (let idx = 0; idx < appendedCount; idx++) {
+          tokens.push(idx < next.tokens.length ? next.tokens[idx] ?? null : null);
+        }
+      } else if (includeTokens) {
+        tokens.push(...new Array(appendedCount).fill(null));
+      }
+      if (Array.isArray(next.diffKind)) {
+        if (!includeDiffKind) {
+          diffKind.push(...new Array(priorLineCount).fill(null));
+          includeDiffKind = true;
+        }
+        for (let idx = 0; idx < appendedCount; idx++) {
+          diffKind.push(idx < next.diffKind.length ? next.diffKind[idx] ?? null : null);
+        }
+      } else if (includeDiffKind) {
+        diffKind.push(...new Array(appendedCount).fill(null));
+      }
+      if (Array.isArray(next.oldNo)) {
+        if (!includeOldNo) {
+          oldNo.push(...new Array(priorLineCount).fill(null));
+          includeOldNo = true;
+        }
+        for (let idx = 0; idx < appendedCount; idx++) {
+          oldNo.push(idx < next.oldNo.length ? next.oldNo[idx] ?? null : null);
+        }
+      } else if (includeOldNo) {
+        oldNo.push(...new Array(appendedCount).fill(null));
+      }
+      if (Array.isArray(next.newNo)) {
+        if (!includeNewNo) {
+          newNo.push(...new Array(priorLineCount).fill(null));
+          includeNewNo = true;
+        }
+        for (let idx = 0; idx < appendedCount; idx++) {
+          newNo.push(idx < next.newNo.length ? next.newNo[idx] ?? null : null);
+        }
+      } else if (includeNewNo) {
+        newNo.push(...new Array(appendedCount).fill(null));
+      }
       expectedStart = baseStart + lines.length;
       mergedCount++;
       j++;
@@ -2664,7 +2987,11 @@ function mergeAppendLines(window2, startIndex) {
   const combined = {
     ...base,
     lines,
-    highlight: highlight.length > 0 ? highlight : void 0
+    highlight: highlight.length > 0 ? highlight : void 0,
+    tokens: includeTokens && tokens.length > 0 ? tokens : void 0,
+    diffKind: includeDiffKind && diffKind.length > 0 ? diffKind : void 0,
+    oldNo: includeOldNo && oldNo.length > 0 ? oldNo : void 0,
+    newNo: includeNewNo && newNo.length > 0 ? newNo : void 0
   };
   return { patch: combined, nextIndex: j };
 }
@@ -3004,130 +3331,6 @@ var CustomStreamingMatcher = class {
     return false;
   }
 };
-
-// src/streaming/inline-streaming.ts
-var DEFAULT_FORMAT_ANTICIPATION = {
-  inline: false,
-  mathInline: false,
-  mathBlock: false,
-  html: false,
-  mdx: false,
-  regex: false
-};
-function normalizeFormatAnticipation(input) {
-  if (input === true) {
-    return { ...DEFAULT_FORMAT_ANTICIPATION, inline: true };
-  }
-  if (!input) {
-    return { ...DEFAULT_FORMAT_ANTICIPATION };
-  }
-  return {
-    inline: input.inline ?? false,
-    mathInline: input.mathInline ?? false,
-    mathBlock: input.mathBlock ?? false,
-    html: input.html ?? false,
-    mdx: input.mdx ?? false,
-    regex: input.regex ?? false
-  };
-}
-function prepareInlineStreamingContent(content, options) {
-  const enableMath = options?.math !== false;
-  const anticipation = normalizeFormatAnticipation(options?.formatAnticipation);
-  const enableInlineAnticipation = anticipation.inline;
-  const enableMathInlineAnticipation = anticipation.mathInline;
-  const enableMathBlockAnticipation = anticipation.mathBlock;
-  const stack = [];
-  const toggleToken = (token) => {
-    const last = stack[stack.length - 1];
-    if (last === token) {
-      stack.pop();
-    } else {
-      stack.push(token);
-    }
-  };
-  let mathDisplayOpen = false;
-  let mathDisplayCrossedNewline = false;
-  for (let i = 0; i < content.length; i++) {
-    const code = content.charCodeAt(i);
-    if (code === 10 || code === 13) {
-      if (mathDisplayOpen) {
-        mathDisplayCrossedNewline = true;
-      }
-      continue;
-    }
-    if (code === 96) {
-      toggleToken("code");
-      continue;
-    }
-    if (code === 126 && i + 1 < content.length && content.charCodeAt(i + 1) === 126) {
-      toggleToken("strike");
-      i += 1;
-      continue;
-    }
-    if (code === 42) {
-      if (i + 1 < content.length && content.charCodeAt(i + 1) === 42) {
-        toggleToken("strong");
-        i += 1;
-      } else {
-        toggleToken("em");
-      }
-      continue;
-    }
-    if (enableMath && code === 36) {
-      if (i + 1 < content.length && content.charCodeAt(i + 1) === 36) {
-        toggleToken("math-display");
-        if (mathDisplayOpen) {
-          mathDisplayOpen = false;
-          mathDisplayCrossedNewline = false;
-        } else {
-          mathDisplayOpen = true;
-          mathDisplayCrossedNewline = false;
-        }
-        i += 1;
-      } else {
-        toggleToken("math-inline");
-      }
-    }
-  }
-  const hasIncompleteFormatting = stack.some((token) => token === "code" || token === "strike" || token === "strong" || token === "em");
-  const hasIncompleteMathInline = stack.includes("math-inline");
-  const hasIncompleteMathDisplay = stack.includes("math-display");
-  const hasIncompleteMath = hasIncompleteMathInline || hasIncompleteMathDisplay;
-  if (enableMath && hasIncompleteMath) {
-    if (hasIncompleteMathInline && !enableMathInlineAnticipation) {
-      return { kind: "raw", status: "raw", reason: "incomplete-math" };
-    }
-    if (hasIncompleteMathDisplay && (!enableMathBlockAnticipation || mathDisplayCrossedNewline)) {
-      return { kind: "raw", status: "raw", reason: "incomplete-math" };
-    }
-  }
-  if (hasIncompleteFormatting && !enableInlineAnticipation) {
-    return { kind: "raw", status: "raw", reason: "incomplete-formatting" };
-  }
-  if (!hasIncompleteFormatting && !hasIncompleteMath) {
-    return { kind: "parse", status: "complete", content, appended: "" };
-  }
-  const appendForToken = (token) => {
-    switch (token) {
-      case "code":
-        return "`";
-      case "strike":
-        return "~~";
-      case "strong":
-        return "**";
-      case "em":
-        return "*";
-      case "math-inline":
-        return "$";
-      case "math-display":
-        return "$$";
-      default:
-        return "";
-    }
-  };
-  const appended = stack.slice().reverse().map((token) => appendForToken(token)).join("");
-  return { kind: "parse", status: "anticipated", content: content + appended, appended };
-}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   CSP_HEADERS,
@@ -3167,12 +3370,14 @@ function prepareInlineStreamingContent(content, options) {
   findClosingHtmlTag,
   generateBlockId,
   getBlockKey,
+  getDefaultCodeWrapperAttributes,
   initializeSecurity,
   initializeTrustedTypesPolicy,
   inlineNodesToPlainText,
   isLikelyMdxComponent,
   normalizeBlockquoteText,
   normalizeFormatAnticipation,
+  normalizeHighlightedLines,
   normalizeLang,
   parseCodeFenceInfo,
   prepareInlineStreamingContent,

package/dist/index.d.cts

@@ -1,6 +1,6 @@
 import { Block, NodeSnapshot, InlineNode, Patch } from './types.cjs';
-export { ASTInlinePlugin, CoalescingMetrics, CompiledMdxModule, FormatAnticipationConfig, InlineHtmlDescriptor, InlinePlugin, LANGUAGE_ALIASES, MixedContentSegment, NodePath, PATCH_ROOT_ID, PatchMetrics, PerformanceMetrics, ProtectedRange, ProtectedRangeKind, RegexAnticipationPattern, RegexInlinePlugin, SetPropsBatchEntry, WorkerErrorPayload, WorkerIn, WorkerOut, WorkerPhase } from './types.cjs';
-export { HighlightedLine, dedentIndentedCode, extractCodeLines, extractCodeWrapperAttributes, extractHighlightedLines, stripCodeFence } from './code-highlighting.cjs';
+export { ASTInlinePlugin, CoalescingMetrics, CodeHighlightOutputMode, CodeHighlightingMode, CompiledMdxModule, DiffKind, FormatAnticipationConfig, InlineHtmlDescriptor, InlinePlugin, LANGUAGE_ALIASES, LazyTokenizationPriority, MixedContentSegment, NodePath, PATCH_ROOT_ID, PatchMetrics, PerformanceMetrics, ProtectedRange, ProtectedRangeKind, RegexAnticipationPattern, RegexInlinePlugin, SetPropsBatchEntry, TokenLineV1, TokenSpan, TokenStyle, WorkerErrorPayload, WorkerIn, WorkerOut, WorkerPhase } from './types.cjs';
+export { HighlightedLine, dedentIndentedCode, extractCodeLines, extractCodeWrapperAttributes, extractHighlightedLines, getDefaultCodeWrapperAttributes, normalizeHighlightedLines, stripCodeFence } from './code-highlighting.cjs';
 export { PerformanceTimer, StringBuffer, applyUpdate, debounce, detectMDX, generateBlockId, getBlockKey, normalizeBlockquoteText, normalizeLang, parseCodeFenceInfo, removeHeadingMarkers } from './utils.cjs';
 export { MixedContentAutoCloseHtmlOptions, MixedContentAutoCloseMdxOptions, MixedContentOptions, extractMixedContentSegments, findClosingHtmlTag, isLikelyMdxComponent } from './mixed-content.cjs';
 export { CSP_HEADERS, SanitizationPolicy, createSanitizationConfig, createTrustedHTML, initializeSecurity, initializeTrustedTypesPolicy, sanitizeCodeHTML, sanitizeHTML, sanitizeMathHTML, sanitizeURL } from './security.cjs';

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 import { Block, NodeSnapshot, InlineNode, Patch } from './types.js';
-export { ASTInlinePlugin, CoalescingMetrics, CompiledMdxModule, FormatAnticipationConfig, InlineHtmlDescriptor, InlinePlugin, LANGUAGE_ALIASES, MixedContentSegment, NodePath, PATCH_ROOT_ID, PatchMetrics, PerformanceMetrics, ProtectedRange, ProtectedRangeKind, RegexAnticipationPattern, RegexInlinePlugin, SetPropsBatchEntry, WorkerErrorPayload, WorkerIn, WorkerOut, WorkerPhase } from './types.js';
-export { HighlightedLine, dedentIndentedCode, extractCodeLines, extractCodeWrapperAttributes, extractHighlightedLines, stripCodeFence } from './code-highlighting.js';
+export { ASTInlinePlugin, CoalescingMetrics, CodeHighlightOutputMode, CodeHighlightingMode, CompiledMdxModule, DiffKind, FormatAnticipationConfig, InlineHtmlDescriptor, InlinePlugin, LANGUAGE_ALIASES, LazyTokenizationPriority, MixedContentSegment, NodePath, PATCH_ROOT_ID, PatchMetrics, PerformanceMetrics, ProtectedRange, ProtectedRangeKind, RegexAnticipationPattern, RegexInlinePlugin, SetPropsBatchEntry, TokenLineV1, TokenSpan, TokenStyle, WorkerErrorPayload, WorkerIn, WorkerOut, WorkerPhase } from './types.js';
+export { HighlightedLine, dedentIndentedCode, extractCodeLines, extractCodeWrapperAttributes, extractHighlightedLines, getDefaultCodeWrapperAttributes, normalizeHighlightedLines, stripCodeFence } from './code-highlighting.js';
 export { PerformanceTimer, StringBuffer, applyUpdate, debounce, detectMDX, generateBlockId, getBlockKey, normalizeBlockquoteText, normalizeLang, parseCodeFenceInfo, removeHeadingMarkers } from './utils.js';
 export { MixedContentAutoCloseHtmlOptions, MixedContentAutoCloseMdxOptions, MixedContentOptions, extractMixedContentSegments, findClosingHtmlTag, isLikelyMdxComponent } from './mixed-content.js';
 export { CSP_HEADERS, SanitizationPolicy, createSanitizationConfig, createTrustedHTML, initializeSecurity, initializeTrustedTypesPolicy, sanitizeCodeHTML, sanitizeHTML, sanitizeMathHTML, sanitizeURL } from './security.js';