@stream-mdx/worker 0.1.1 → 0.2.0

package/dist/hosted/markdown-worker.js

@@ -17946,7 +17946,7 @@ function getGroupContents(expression, contentsStartPos) {
   return expression.slice(contentsStartPos, contentsEndPos);
 }
 
-// ../../node_modules/regex/src/subclass.js
+// node_modules/regex/src/subclass.js
 var emulationGroupMarker = "$E$";
 var RegExpSubclass = class _RegExpSubclass extends RegExp {
   /**
@@ -18059,13 +18059,13 @@ function unmarkEmulationGroups(expression) {
   };
 }
 
-// ../../node_modules/regex/src/utils-internals.js
+// node_modules/regex/src/utils-internals.js
 var noncapturingDelim = String.raw`\(\?(?:[:=!>A-Za-z\-]|<[=!]|\(DEFINE\))`;
 function spliceStr(str, pos, oldValue, newValue) {
   return str.slice(0, pos) + newValue + str.slice(pos + oldValue.length);
 }
 
-// ../../node_modules/regex/src/atomic.js
+// node_modules/regex/src/atomic.js
 var atomicPluginToken = new RegExp(String.raw`(?<noncapturingStart>${noncapturingDelim})|(?<capturingStart>\((?:\?<[^>]+>)?)|\\?.`, "gsu");
 function atomic(expression, data2) {
   if (!/\(\?>/.test(expression)) {
@@ -18206,7 +18206,7 @@ function possessive(expression) {
   return expression;
 }
 
-// ../../node_modules/regex-recursion/src/index.js
+// node_modules/regex-recursion/src/index.js
 var r2 = String.raw;
 var gRToken = r2`\\g<(?<gRNameOrNum>[^>&]+)&R=(?<gRDepth>[^>]+)>`;
 var recursiveToken = r2`\(\?R=(?<rDepth>[^\)]+)\)|${gRToken}`;
@@ -18356,7 +18356,7 @@ function emulationGroupMarkerLength(expression, index2) {
   return match ? match[0].length : 0;
 }
 
-// ../../node_modules/oniguruma-to-es/dist/esm/index.js
+// node_modules/oniguruma-to-es/dist/esm/index.js
 var cp = String.fromCodePoint;
 var r3 = String.raw;
 var envSupportsFlagGroups = (() => {
@@ -21379,7 +21379,7 @@ function toRegExp(pattern, options) {
   return new RegExp(result.pattern, result.flags);
 }
 
-// ../../node_modules/@shikijs/engine-javascript/dist/shared/engine-javascript.hzpS1_41.mjs
+// node_modules/@shikijs/engine-javascript/dist/shared/engine-javascript.hzpS1_41.mjs
 var MAX = 4294967295;
 var JavaScriptScanner = class {
   constructor(patterns, options = {}) {
@@ -21472,7 +21472,7 @@ var JavaScriptScanner = class {
   }
 };
 
-// ../../node_modules/@shikijs/engine-javascript/dist/engine-compile.mjs
+// node_modules/@shikijs/engine-javascript/dist/engine-compile.mjs
 function defaultJavaScriptRegexConstructor(pattern, options) {
   return toRegExp(
     pattern,
@@ -43324,6 +43324,22 @@ function manualExtractHighlightedLines(html9, fallbackLength) {
   }
   return normalizeHighlightedLines(lines, fallbackLength);
 }
+function getDefaultCodeWrapperAttributes(lang236, themes = { dark: "github-dark", light: "github-light" }) {
+  const language = lang236 && typeof lang236 === "string" && lang236.length > 0 ? lang236 : "text";
+  const themeLabel = `${themes.dark} ${themes.light}`;
+  return {
+    preAttrs: {
+      class: `shiki shiki-themes ${themeLabel}`,
+      "data-language": language,
+      style: "--shiki-dark-bg: transparent; --shiki-light-bg: transparent"
+    },
+    codeAttrs: {
+      "data-language": language,
+      "data-theme": themeLabel,
+      style: "display: grid;"
+    }
+  };
+}
 function dedentIndentedCode(raw2) {
   if (!raw2) return "";
   const normalized = normalizeNewlines(raw2);
@@ -43697,6 +43713,128 @@ function splitTextByRegexWithPrecedence(text12, regex3, toNode3) {
   }
   return result;
 }
+var DEFAULT_FORMAT_ANTICIPATION = {
+  inline: false,
+  mathInline: false,
+  mathBlock: false,
+  html: false,
+  mdx: false,
+  regex: false
+};
+function normalizeFormatAnticipation(input) {
+  if (input === true) {
+    return { ...DEFAULT_FORMAT_ANTICIPATION, inline: true };
+  }
+  if (!input) {
+    return { ...DEFAULT_FORMAT_ANTICIPATION };
+  }
+  return {
+    inline: input.inline ?? false,
+    mathInline: input.mathInline ?? false,
+    mathBlock: input.mathBlock ?? false,
+    html: input.html ?? false,
+    mdx: input.mdx ?? false,
+    regex: input.regex ?? false
+  };
+}
+function prepareInlineStreamingContent(content4, options) {
+  const enableMath2 = options?.math !== false;
+  const anticipation = normalizeFormatAnticipation(options?.formatAnticipation);
+  const enableInlineAnticipation = anticipation.inline;
+  const enableMathInlineAnticipation = anticipation.mathInline;
+  const enableMathBlockAnticipation = anticipation.mathBlock;
+  const stack = [];
+  const toggleToken = (token2) => {
+    const last = stack[stack.length - 1];
+    if (last === token2) {
+      stack.pop();
+    } else {
+      stack.push(token2);
+    }
+  };
+  let mathDisplayOpen = false;
+  let mathDisplayCrossedNewline = false;
+  for (let i = 0; i < content4.length; i++) {
+    const code4 = content4.charCodeAt(i);
+    if (code4 === 10 || code4 === 13) {
+      if (mathDisplayOpen) {
+        mathDisplayCrossedNewline = true;
+      }
+      continue;
+    }
+    if (code4 === 96) {
+      toggleToken("code");
+      continue;
+    }
+    if (code4 === 126 && i + 1 < content4.length && content4.charCodeAt(i + 1) === 126) {
+      toggleToken("strike");
+      i += 1;
+      continue;
+    }
+    if (code4 === 42) {
+      if (i + 1 < content4.length && content4.charCodeAt(i + 1) === 42) {
+        toggleToken("strong");
+        i += 1;
+      } else {
+        toggleToken("em");
+      }
+      continue;
+    }
+    if (enableMath2 && code4 === 36) {
+      if (i + 1 < content4.length && content4.charCodeAt(i + 1) === 36) {
+        toggleToken("math-display");
+        if (mathDisplayOpen) {
+          mathDisplayOpen = false;
+          mathDisplayCrossedNewline = false;
+        } else {
+          mathDisplayOpen = true;
+          mathDisplayCrossedNewline = false;
+        }
+        i += 1;
+      } else {
+        toggleToken("math-inline");
+      }
+    }
+  }
+  const hasIncompleteFormatting = stack.some((token2) => token2 === "code" || token2 === "strike" || token2 === "strong" || token2 === "em");
+  const hasIncompleteMathInline = stack.includes("math-inline");
+  const hasIncompleteMathDisplay = stack.includes("math-display");
+  const hasIncompleteMath = hasIncompleteMathInline || hasIncompleteMathDisplay;
+  if (enableMath2 && hasIncompleteMath) {
+    if (hasIncompleteMathInline && !enableMathInlineAnticipation) {
+      return { kind: "raw", status: "raw", reason: "incomplete-math" };
+    }
+    if (hasIncompleteMathDisplay && (!enableMathBlockAnticipation || mathDisplayCrossedNewline)) {
+      return { kind: "raw", status: "raw", reason: "incomplete-math" };
+    }
+  }
+  if (hasIncompleteFormatting && !enableInlineAnticipation) {
+    return { kind: "raw", status: "raw", reason: "incomplete-formatting" };
+  }
+  if (!hasIncompleteFormatting && !hasIncompleteMath) {
+    return { kind: "parse", status: "complete", content: content4, appended: "" };
+  }
+  const appendForToken = (token2) => {
+    switch (token2) {
+      case "code":
+        return "`";
+      case "strike":
+        return "~~";
+      case "strong":
+        return "**";
+      case "em":
+        return "*";
+      case "math-inline":
+        return "$";
+      case "math-display":
+        return "$$";
+      default:
+        return "";
+    }
+  };
+  const appended = stack.slice().reverse().map((token2) => appendForToken(token2)).join("");
+  return { kind: "parse", status: "anticipated", content: content4 + appended, appended };
+}
 var rehypeSanitizeModule = rehype_sanitize_exports;
 var defaultSchema2 = rehypeSanitizeModule.defaultSchema;
 var resolvePlugin = (mod) => {
@@ -43871,6 +44009,8 @@ function splitByTagSegments(source, baseOffset, parseInline, options) {
   const mdxAutoClose = options?.mdx?.autoClose === true;
   const mdxMaxNewlines = normalizeNewlineLimit(options?.mdx?.maxNewlines);
   const mdxAllowlist = normalizeComponentAllowlist(options?.mdx?.componentAllowlist);
+  const protectedRanges = options?.protectedRanges ?? [];
+  const protectedKinds = protectedRanges.length ? new Set(options?.protectedRangeKinds ?? ["math-inline", "math-display", "code-inline", "code-block", "autolink"]) : null;
   while (match !== null) {
     const start2 = match.index;
     const tagName = match[1];
@@ -43885,6 +44025,18 @@ function splitByTagSegments(source, baseOffset, parseInline, options) {
       continue;
     }
     let end = tagPattern.lastIndex;
+    if (protectedKinds && protectedRanges.length > 0) {
+      const absoluteStart = baseIsFinite ? baseOffset + start2 : start2;
+      const absoluteEnd = baseIsFinite ? baseOffset + end : end;
+      const covered = protectedRanges.some(
+        (range2) => protectedKinds.has(range2.kind) && typeof range2.from === "number" && typeof range2.to === "number" && range2.from <= absoluteStart && range2.to >= absoluteEnd
+      );
+      if (covered) {
+        tagPattern.lastIndex = start2 + 1;
+        match = tagPattern.exec(source);
+        continue;
+      }
+    }
     if (!isSelfClosing && !mdxAllowed) {
       const closingIndex = findClosingHtmlTag(lowerSource, tagName.toLowerCase(), end);
       if (closingIndex === -1) {
@@ -44164,23 +44316,33 @@ function normalizeLang(raw2) {
 }
 function detectMDX(content4, options) {
   const inlineCodeRanges = collectInlineCodeRanges(content4);
+  const protectedRanges = options?.protectedRanges ?? [];
+  const baseOffset = typeof options?.baseOffset === "number" ? options.baseOffset : 0;
+  const protectedKinds = protectedRanges.length ? /* @__PURE__ */ new Set(["math-inline", "math-display", "code-inline", "code-block", "html-inline", "html-block", "autolink"]) : null;
   const componentPattern = /<([A-Z][\w-]*)(\s|\/?>)/g;
   let componentMatch = componentPattern.exec(content4);
   while (componentMatch !== null) {
     const start2 = componentMatch.index;
     const end = start2 + componentMatch[0].length;
-    if (!isWithinRanges(start2, end, inlineCodeRanges)) {
-      return true;
+    if (isWithinRanges(start2, end, inlineCodeRanges)) {
+      componentMatch = componentPattern.exec(content4);
+      continue;
     }
-    componentMatch = componentPattern.exec(content4);
+    if (protectedKinds) {
+      const absoluteStart = baseOffset + start2;
+      const absoluteEnd = baseOffset + end;
+      const covered = protectedRanges.some((range2) => protectedKinds.has(range2.kind) && range2.from <= absoluteStart && range2.to >= absoluteEnd);
+      if (covered) {
+        componentMatch = componentPattern.exec(content4);
+        continue;
+      }
+    }
+    return true;
   }
   if (/(^|\n)\s*(import|export)\s/.test(content4)) {
     return true;
   }
   const expressionPattern = /\{[^{}]+\}/g;
-  const protectedRanges = options?.protectedRanges ?? [];
-  const baseOffset = typeof options?.baseOffset === "number" ? options.baseOffset : 0;
-  const protectedKinds = protectedRanges.length ? /* @__PURE__ */ new Set(["math-inline", "math-display", "code-inline", "code-block"]) : null;
   for (let match = expressionPattern.exec(content4); match !== null; match = expressionPattern.exec(content4)) {
     const index2 = match.index;
     const prev = index2 > 0 ? content4[index2 - 1] : "";
@@ -44397,7 +44559,12 @@ function buildListItemSnapshot(block, listItemNode, ordered, index2, id, baseOff
       }
       if (name3 === "Paragraph") {
         const paragraphRaw = raw2.slice(cursor.from, cursor.to);
-        const paragraphData = processListItemParagraph(paragraphRaw);
+        const meta2 = block.payload.meta ?? {};
+        const paragraphData = processListItemParagraph(paragraphRaw, {
+          formatAnticipation: meta2.formatAnticipation,
+          math: meta2.mathEnabled,
+          streaming: !block.isFinalized
+        });
         const parsedInline = paragraphData.inline;
         if (!paragraphHandled) {
           inlineNodes = parsedInline;
@@ -44429,7 +44596,10 @@ function buildListItemSnapshot(block, listItemNode, ordered, index2, id, baseOff
       } else if (name3 === "BulletList" || name3 === "OrderedList") {
         const nestedId = `${id}::list:${subListIndex++}`;
         const nestedOrdered = name3 === "OrderedList";
-        childSnapshots.push(buildListNodeSnapshot(block, cursor.node, nestedOrdered, nestedId, baseOffset, raw2));
+        const nestedSnapshot = buildListNodeSnapshot(block, cursor.node, nestedOrdered, nestedId, baseOffset, raw2);
+        if (Array.isArray(nestedSnapshot.children) && nestedSnapshot.children.length > 0) {
+          childSnapshots.push(nestedSnapshot);
+        }
       } else if (name3 === "Blockquote") {
         const quoteId = `${id}::blockquote:${blockquoteIndex++}`;
         childSnapshots.push(buildBlockquoteSnapshot(block, cursor.node, quoteId, baseOffset, raw2));
@@ -44465,11 +44635,31 @@ function buildListItemSnapshot(block, listItemNode, ordered, index2, id, baseOff
   };
   return itemSnapshot;
 }
-function processListItemParagraph(raw2) {
+function parseListInline(raw2, options) {
+  if (!options?.streaming || !options.formatAnticipation) {
+    return listInlineParser.parse(raw2);
+  }
+  const prepared = prepareInlineStreamingContent(raw2, { formatAnticipation: options.formatAnticipation, math: options.math });
+  if (prepared.kind === "raw") {
+    return [{ kind: "text", text: raw2 }];
+  }
+  let preparedContent = prepared.content;
+  let appended = prepared.appended;
+  const normalized = normalizeFormatAnticipation(options.formatAnticipation);
+  if (normalized.regex) {
+    const regexAppend = listInlineParser.getRegexAnticipationAppend(raw2);
+    if (regexAppend) {
+      preparedContent += regexAppend;
+      appended += regexAppend;
+    }
+  }
+  return listInlineParser.parse(preparedContent, { cache: false });
+}
+function processListItemParagraph(raw2, options) {
   const normalized = normalizeParagraphText(raw2);
   const { content: content4, task } = stripTaskMarker(normalized);
-  const inline = listInlineParser.parse(content4);
-  const segments = extractMixedContentSegments(content4, void 0, (value) => listInlineParser.parse(value));
+  const inline = parseListInline(content4, options);
+  const segments = extractMixedContentSegments(content4, void 0, (value) => parseListInline(value, options));
   return {
     inline,
     segments,
@@ -44754,28 +44944,82 @@ function enrichTableSnapshot(block, snapshot) {
 function enrichCodeSnapshot(block, snapshot) {
   const source = typeof block.payload.meta?.code === "string" ? block.payload.meta?.code : block.payload.raw ?? "";
   const lines = extractCodeLines(source);
+  const meta2 = block.payload.meta;
   const highlightedHtml = block.payload.highlightedHtml ?? "";
-  const highlightedLines = extractHighlightedLines(highlightedHtml, lines.length);
-  const { preAttrs, codeAttrs } = extractCodeWrapperAttributes(highlightedHtml);
-  const lang236 = typeof block.payload.meta?.lang === "string" ? String(block.payload.meta?.lang) : void 0;
+  const hasBlockHighlight = typeof block.payload.highlightedHtml === "string" && block.payload.highlightedHtml.length > 0;
+  const metaLines = Array.isArray(meta2?.highlightedLines) ? meta2?.highlightedLines : null;
+  const includeLineHtml = metaLines ? true : !hasBlockHighlight || lines.length >= 200;
+  const highlightedLines = metaLines ? normalizeHighlightedLines(metaLines, lines.length) : extractHighlightedLines(highlightedHtml, lines.length);
+  const metaTokenLines = Array.isArray(meta2?.tokenLines) ? meta2?.tokenLines : null;
+  const tokenLines = metaTokenLines ? normalizeTokenLines(metaTokenLines, lines.length) : null;
+  const metaDiffKind = Array.isArray(meta2?.diffKind) ? meta2?.diffKind : null;
+  const diffKindLines = metaDiffKind ? normalizeOptionalArray(metaDiffKind, lines.length) : null;
+  const metaOldNo = Array.isArray(meta2?.oldNo) ? meta2?.oldNo : null;
+  const oldNoLines = metaOldNo ? normalizeOptionalArray(metaOldNo, lines.length) : null;
+  const metaNewNo = Array.isArray(meta2?.newNo) ? meta2?.newNo : null;
+  const newNoLines = metaNewNo ? normalizeOptionalArray(metaNewNo, lines.length) : null;
+  const lang236 = typeof meta2?.lang === "string" ? String(meta2.lang) : void 0;
+  let { preAttrs, codeAttrs } = extractCodeWrapperAttributes(highlightedHtml);
+  if (!preAttrs || !codeAttrs) {
+    const defaults = getDefaultCodeWrapperAttributes(lang236);
+    preAttrs = preAttrs ?? defaults.preAttrs;
+    codeAttrs = codeAttrs ?? defaults.codeAttrs;
+  }
   snapshot.props = {
     ...snapshot.props ?? {},
     lang: lang236,
     preAttrs,
     codeAttrs
   };
-  snapshot.children = lines.map((line, index2) => ({
-    id: `${block.id}::line:${index2}`,
-    type: "code-line",
-    props: {
+  snapshot.children = lines.map((line, index2) => {
+    const props = {
       index: index2,
       text: line,
-      html: highlightedLines[index2] ?? null
-    },
-    children: []
-  }));
+      html: includeLineHtml ? highlightedLines[index2] ?? null : null
+    };
+    if (tokenLines) {
+      props.tokens = tokenLines[index2] ?? null;
+    }
+    if (diffKindLines) {
+      props.diffKind = diffKindLines[index2] ?? null;
+    }
+    if (oldNoLines) {
+      props.oldNo = oldNoLines[index2] ?? null;
+    }
+    if (newNoLines) {
+      props.newNo = newNoLines[index2] ?? null;
+    }
+    return {
+      id: `${block.id}::line:${index2}`,
+      type: "code-line",
+      props,
+      children: []
+    };
+  });
   return snapshot;
 }
+function normalizeTokenLines(lines, fallbackLength) {
+  if (!lines || lines.length === 0) {
+    return new Array(Math.max(0, fallbackLength)).fill(null);
+  }
+  const length = Math.max(fallbackLength, lines.length);
+  const result = new Array(length).fill(null);
+  for (let i = 0; i < lines.length; i++) {
+    result[i] = lines[i] ?? null;
+  }
+  return result;
+}
+function normalizeOptionalArray(lines, fallbackLength) {
+  if (!lines || lines.length === 0) {
+    return new Array(Math.max(0, fallbackLength)).fill(null);
+  }
+  const length = Math.max(fallbackLength, lines.length);
+  const result = new Array(length).fill(null);
+  for (let i = 0; i < lines.length; i++) {
+    result[i] = lines[i] ?? null;
+  }
+  return result;
+}
 function cloneInlineNodes(nodes) {
   return nodes.map((node2) => {
     if ("children" in node2 && Array.isArray(node2.children)) {
@@ -44892,128 +45136,6 @@ function computeHeavyPatchBudget(credit, config = DEFAULT_BACKPRESSURE_CONFIG) {
   return Math.min(config.maxHeavyPatchBudget, budget);
 }
 var USE_LINEAR_COALESCING = typeof process === "undefined" ? true : process.env.V2_USE_LINEAR_COALESCING !== "false" && true;
-var DEFAULT_FORMAT_ANTICIPATION = {
-  inline: false,
-  mathInline: false,
-  mathBlock: false,
-  html: false,
-  mdx: false,
-  regex: false
-};
-function normalizeFormatAnticipation(input) {
-  if (input === true) {
-    return { ...DEFAULT_FORMAT_ANTICIPATION, inline: true };
-  }
-  if (!input) {
-    return { ...DEFAULT_FORMAT_ANTICIPATION };
-  }
-  return {
-    inline: input.inline ?? false,
-    mathInline: input.mathInline ?? false,
-    mathBlock: input.mathBlock ?? false,
-    html: input.html ?? false,
-    mdx: input.mdx ?? false,
-    regex: input.regex ?? false
-  };
-}
-function prepareInlineStreamingContent(content4, options) {
-  const enableMath2 = options?.math !== false;
-  const anticipation = normalizeFormatAnticipation(options?.formatAnticipation);
-  const enableInlineAnticipation = anticipation.inline;
-  const enableMathInlineAnticipation = anticipation.mathInline;
-  const enableMathBlockAnticipation = anticipation.mathBlock;
-  const stack = [];
-  const toggleToken = (token2) => {
-    const last = stack[stack.length - 1];
-    if (last === token2) {
-      stack.pop();
-    } else {
-      stack.push(token2);
-    }
-  };
-  let mathDisplayOpen = false;
-  let mathDisplayCrossedNewline = false;
-  for (let i = 0; i < content4.length; i++) {
-    const code4 = content4.charCodeAt(i);
-    if (code4 === 10 || code4 === 13) {
-      if (mathDisplayOpen) {
-        mathDisplayCrossedNewline = true;
-      }
-      continue;
-    }
-    if (code4 === 96) {
-      toggleToken("code");
-      continue;
-    }
-    if (code4 === 126 && i + 1 < content4.length && content4.charCodeAt(i + 1) === 126) {
-      toggleToken("strike");
-      i += 1;
-      continue;
-    }
-    if (code4 === 42) {
-      if (i + 1 < content4.length && content4.charCodeAt(i + 1) === 42) {
-        toggleToken("strong");
-        i += 1;
-      } else {
-        toggleToken("em");
-      }
-      continue;
-    }
-    if (enableMath2 && code4 === 36) {
-      if (i + 1 < content4.length && content4.charCodeAt(i + 1) === 36) {
-        toggleToken("math-display");
-        if (mathDisplayOpen) {
-          mathDisplayOpen = false;
-          mathDisplayCrossedNewline = false;
-        } else {
-          mathDisplayOpen = true;
-          mathDisplayCrossedNewline = false;
-        }
-        i += 1;
-      } else {
-        toggleToken("math-inline");
-      }
-    }
-  }
-  const hasIncompleteFormatting = stack.some((token2) => token2 === "code" || token2 === "strike" || token2 === "strong" || token2 === "em");
-  const hasIncompleteMathInline = stack.includes("math-inline");
-  const hasIncompleteMathDisplay = stack.includes("math-display");
-  const hasIncompleteMath = hasIncompleteMathInline || hasIncompleteMathDisplay;
-  if (enableMath2 && hasIncompleteMath) {
-    if (hasIncompleteMathInline && !enableMathInlineAnticipation) {
-      return { kind: "raw", status: "raw", reason: "incomplete-math" };
-    }
-    if (hasIncompleteMathDisplay && (!enableMathBlockAnticipation || mathDisplayCrossedNewline)) {
-      return { kind: "raw", status: "raw", reason: "incomplete-math" };
-    }
-  }
-  if (hasIncompleteFormatting && !enableInlineAnticipation) {
-    return { kind: "raw", status: "raw", reason: "incomplete-formatting" };
-  }
-  if (!hasIncompleteFormatting && !hasIncompleteMath) {
-    return { kind: "parse", status: "complete", content: content4, appended: "" };
-  }
-  const appendForToken = (token2) => {
-    switch (token2) {
-      case "code":
-        return "`";
-      case "strike":
-        return "~~";
-      case "strong":
-        return "**";
-      case "em":
-        return "*";
-      case "math-inline":
-        return "$";
-      case "math-display":
-        return "$$";
-      default:
-        return "";
-    }
-  };
-  const appended = stack.slice().reverse().map((token2) => appendForToken(token2)).join("");
-  return { kind: "parse", status: "anticipated", content: content4 + appended, appended };
-}
 
 // ../markdown-v2-core/dist/perf/patch-batching.mjs
 var LIGHT_APPEND_LINE_THRESHOLD = 4;
@@ -45679,9 +45801,26 @@ var MDXDetectionPlugin = {
       if (block.type !== "paragraph" && block.type !== "html") continue;
       const raw2 = block.payload.raw;
       const blockRange = block.payload.range;
-      const baseOffset = typeof blockRange?.from === "number" ? blockRange.from : 0;
-      const relevantProtected = filterProtectedRanges(ctx.protectedRanges, baseOffset, baseOffset + raw2.length);
-      if (detectMDX(raw2, { protectedRanges: relevantProtected, baseOffset })) {
+      const baseOffset = typeof blockRange?.from === "number" ? blockRange.from : null;
+      let protectedBase = baseOffset ?? 0;
+      let relevantProtected = filterProtectedRanges(ctx.protectedRanges, protectedBase, protectedBase + raw2.length);
+      if (relevantProtected.length === 0) {
+        const metaProtected = Array.isArray(block.payload.meta?.protectedRanges) ? block.payload.meta.protectedRanges ?? [] : [];
+        if (metaProtected.length > 0) {
+          if (baseOffset === null) {
+            protectedBase = 0;
+            relevantProtected = metaProtected;
+          } else {
+            protectedBase = baseOffset;
+            relevantProtected = metaProtected.map((range2) => ({
+              ...range2,
+              from: baseOffset + range2.from,
+              to: baseOffset + range2.to
+            }));
+          }
+        }
+      }
+      if (detectMDX(raw2, { protectedRanges: relevantProtected, baseOffset: protectedBase })) {
         block.payload.meta = { ...block.payload.meta || {}, originalType: block.type };
         block.type = "mdx";
         if ("sanitizedHtml" in block.payload) {
@@ -45774,6 +45913,7 @@ var BLOCK_NODE_TYPES = /* @__PURE__ */ new Set([
   "OrderedList",
   "HTMLBlock",
   "ThematicBreak",
+  "HorizontalRule",
   "ATXHeading",
   "SetextHeading"
 ]);
@@ -45813,6 +45953,7 @@ function mapLezerNodeToBlockType(nodeType) {
     case "HTMLBlock":
       return "html";
     case "ThematicBreak":
+    case "HorizontalRule":
       return "hr";
     default:
       return "paragraph";
@@ -45861,6 +46002,36 @@ function computeParagraphPatchLimit(patches, {
   return Math.min(scaledLimit, patches.length);
 }
 
+// src/lazy-tokenization.ts
+var PRIORITY_ORDER = {
+  visible: 2,
+  prefetch: 1
+};
+function clampLazyRange(startLine, endLine, totalLines) {
+  const clampedStart = Math.max(0, Math.min(Math.floor(startLine), totalLines));
+  const clampedEnd = Math.max(clampedStart, Math.min(Math.floor(endLine), totalLines));
+  return { startLine: clampedStart, endLine: clampedEnd };
+}
+function compareLazyPriority(a, b) {
+  return (PRIORITY_ORDER[a] ?? 0) - (PRIORITY_ORDER[b] ?? 0);
+}
+function mergeLazyRequests(existing, next2) {
+  const priority = compareLazyPriority(existing.priority, next2.priority) >= 0 ? existing.priority : next2.priority;
+  const startLine = Math.min(existing.startLine, next2.startLine);
+  const endLine = Math.max(existing.endLine, next2.endLine);
+  const useNextTimestamp = compareLazyPriority(next2.priority, existing.priority) >= 0;
+  return {
+    blockId: existing.blockId,
+    startLine,
+    endLine,
+    priority,
+    requestedAt: useNextTimestamp ? next2.requestedAt : existing.requestedAt
+  };
+}
+function lazyRequestRangeSize(request) {
+  return Math.max(0, request.endLine - request.startLine);
+}
+
 // ../../node_modules/markdown-extensions/index.js
 var markdownExtension = [
   "md",
@@ -81979,9 +82150,21 @@ var deferredPatchQueue = [];
 var MAX_DEFERRED_FLUSH_PATCHES = 120;
 var CODE_HIGHLIGHT_CACHE = /* @__PURE__ */ new Map();
 var MAX_CODE_HIGHLIGHT_CACHE_ENTRIES = 200;
+var incrementalHighlightStates = /* @__PURE__ */ new Map();
+var lazyTokenizationStates = /* @__PURE__ */ new Map();
+var lazyTokenizationQueue = /* @__PURE__ */ new Map();
+var lazyTokenizationScheduled = false;
+var lazyTokenizationProcessing = false;
+var DEFAULT_LAZY_TOKENIZATION_THRESHOLD = 200;
+var MIN_LAZY_TOKENIZATION_THRESHOLD = 50;
+var MAX_LAZY_TOKENIZATION_THRESHOLD = 1e4;
+var lazyTokenizationEnabled = true;
+var lazyTokenizationThresholdLines = DEFAULT_LAZY_TOKENIZATION_THRESHOLD;
+var CODE_HIGHLIGHT_THEMES = { dark: "github-dark", light: "github-light" };
 var mdxCompileMode = "server";
 var formatAnticipationConfig = normalizeFormatAnticipation(false);
-var enableLiveCodeHighlighting = false;
+var codeHighlightingMode = "final";
+var highlightOutputMode = "html";
 var enableMath = true;
 var mdxComponentAllowlist = null;
 var WORKER_MDX_CACHE = /* @__PURE__ */ new Map();
@@ -82083,6 +82266,12 @@ var WorkerMetricsCollector = class {
     this.appendLineBatchCount = 0;
     this.appendLineTotalLines = 0;
     this.appendLineMaxLines = 0;
+    this.lazyTokenizationRequests = 0;
+    this.lazyTokenizationRangeTotal = 0;
+    this.lazyTokenizationRangeMax = 0;
+    this.lazyTokenizationLatencyTotal = 0;
+    this.lazyTokenizationLatencyMax = 0;
+    this.lazyTokenizationMaxQueue = 0;
     this.grammarEngine = grammarEngine;
     this.startedAt = now();
   }
@@ -82137,6 +82326,29 @@ var WorkerMetricsCollector = class {
       this.appendLineMaxLines = normalized;
     }
   }
+  recordLazyTokenization(rangeLines, latencyMs, queueDepth) {
+    if (Number.isFinite(rangeLines ?? Number.NaN)) {
+      const normalized = Math.max(0, Math.floor(Number(rangeLines)));
+      this.lazyTokenizationRequests += 1;
+      this.lazyTokenizationRangeTotal += normalized;
+      if (normalized > this.lazyTokenizationRangeMax) {
+        this.lazyTokenizationRangeMax = normalized;
+      }
+    }
+    if (Number.isFinite(latencyMs ?? Number.NaN)) {
+      const normalizedLatency = Math.max(0, Number(latencyMs));
+      this.lazyTokenizationLatencyTotal += normalizedLatency;
+      if (normalizedLatency > this.lazyTokenizationLatencyMax) {
+        this.lazyTokenizationLatencyMax = normalizedLatency;
+      }
+    }
+    if (Number.isFinite(queueDepth ?? Number.NaN)) {
+      const normalizedQueue = Math.max(0, Math.floor(Number(queueDepth)));
+      if (normalizedQueue > this.lazyTokenizationMaxQueue) {
+        this.lazyTokenizationMaxQueue = normalizedQueue;
+      }
+    }
+  }
   markDiffStart() {
     this.diffStart = now();
   }
@@ -82199,7 +82411,15 @@ var WorkerMetricsCollector = class {
       highlightByLanguage: mapToHighlightRecord(this.highlightByLanguage),
       appendLineBatches: this.appendLineBatchCount || void 0,
       appendLineTotalLines: this.appendLineTotalLines || void 0,
-      appendLineMaxLines: this.appendLineMaxLines || void 0
+      appendLineMaxLines: this.appendLineMaxLines || void 0,
+      lazyTokenization: this.lazyTokenizationRequests > 0 ? {
+        requests: this.lazyTokenizationRequests,
+        avgRangeLines: roundMetric(this.lazyTokenizationRangeTotal / this.lazyTokenizationRequests),
+        maxRangeLines: this.lazyTokenizationRangeMax,
+        avgLatencyMs: roundMetric(this.lazyTokenizationLatencyTotal / this.lazyTokenizationRequests),
+        maxLatencyMs: roundMetric(this.lazyTokenizationLatencyMax),
+        maxQueue: this.lazyTokenizationMaxQueue
+      } : void 0
     };
   }
 };
@@ -82362,6 +82582,11 @@ async function initialize(initialContent = "", prewarmLangs = [], docPlugins, md
   lastTree = null;
   currentContent = "";
   deferredPatchQueue = [];
+  resetIncrementalHighlightState();
+  lazyTokenizationStates.clear();
+  lazyTokenizationQueue.clear();
+  lazyTokenizationScheduled = false;
+  lazyTokenizationProcessing = false;
   mdxCompileMode = mdxOptions?.compileMode ?? "server";
   try {
     if (DEBUG_MDX) {
@@ -82378,11 +82603,29 @@ async function initialize(initialContent = "", prewarmLangs = [], docPlugins, md
     callouts: docPlugins?.callouts ?? false,
     math: docPlugins?.math ?? true,
     formatAnticipation: docPlugins?.formatAnticipation ?? false,
+    codeHighlighting: docPlugins?.codeHighlighting,
     liveCodeHighlighting: docPlugins?.liveCodeHighlighting ?? false
   };
   enableMath = enable.math;
   formatAnticipationConfig = normalizeFormatAnticipation(enable.formatAnticipation);
-  enableLiveCodeHighlighting = enable.liveCodeHighlighting;
+  if (enable.codeHighlighting === "incremental" || enable.codeHighlighting === "live" || enable.codeHighlighting === "final") {
+    codeHighlightingMode = enable.codeHighlighting;
+  } else {
+    codeHighlightingMode = enable.liveCodeHighlighting ? "live" : "final";
+  }
+  if (docPlugins?.outputMode === "html" || docPlugins?.outputMode === "tokens" || docPlugins?.outputMode === "both") {
+    highlightOutputMode = docPlugins.outputMode;
+  } else {
+    highlightOutputMode = "html";
+  }
+  if (docPlugins?.lazyTokenization) {
+    lazyTokenizationEnabled = docPlugins.lazyTokenization.enabled ?? true;
+    const desiredThreshold = docPlugins.lazyTokenization.thresholdLines ?? DEFAULT_LAZY_TOKENIZATION_THRESHOLD;
+    lazyTokenizationThresholdLines = clampInt(desiredThreshold, MIN_LAZY_TOKENIZATION_THRESHOLD, MAX_LAZY_TOKENIZATION_THRESHOLD);
+  } else {
+    lazyTokenizationEnabled = true;
+    lazyTokenizationThresholdLines = DEFAULT_LAZY_TOKENIZATION_THRESHOLD;
+  }
   if (Array.isArray(docPlugins?.mdxComponentNames) && docPlugins?.mdxComponentNames.length > 0) {
     mdxComponentAllowlist = new Set(docPlugins.mdxComponentNames);
   } else {
@@ -82396,7 +82639,7 @@ async function initialize(initialContent = "", prewarmLangs = [], docPlugins, md
   if (enable.html) globalDocumentPluginRegistry.register(HTMLBlockPlugin);
   if (enable.mdx) globalDocumentPluginRegistry.register(MDXDetectionPlugin);
   performanceTimer.mark("highlighter-init");
-  const coreLangs = ["javascript", "typescript", "json", "text", "markdown"];
+  const coreLangs = ["javascript", "typescript", "json", "text", "markdown", "diff"];
   const initialLangs = [...coreLangs, ...prewarmLangs];
   highlighter = await createHighlighter({
     engine: createJavaScriptRegexEngine(),
@@ -82493,6 +82736,7 @@ async function appendAndReparse(appendedText, metrics) {
   blocks = changedBlocks;
   lastTree = newTree;
   currentContent = newContent;
+  pruneLazyTokenizationStates(blocks);
   performanceTimer.measure("incremental-parse");
   metrics?.markParseEnd();
   metrics?.setBlocksProduced(blocks.length);
@@ -82671,22 +82915,35 @@ async function enrichBlock(block) {
         nextMeta.inlineStatus = void 0;
         metaChanged = true;
       }
-      const mathRanges = enableMath ? collectMathProtectedRanges(rawParagraph) : [];
-      if (mathRanges.length > 0) {
-        nextMeta.protectedRanges = mathRanges;
-        metaChanged = true;
-      } else if (Object.prototype.hasOwnProperty.call(nextMeta, "protectedRanges")) {
-        nextMeta.protectedRanges = void 0;
-        metaChanged = true;
+      const baseOffset = typeof block.payload.range?.from === "number" ? block.payload.range.from : void 0;
+      const protectedRanges = [];
+      if (enableMath) {
+        protectedRanges.push(...collectMathProtectedRanges(rawParagraph));
+      }
+      const autolinkRanges = collectAutolinkProtectedRanges(rawParagraph);
+      if (autolinkRanges.length > 0) {
+        protectedRanges.push(...autolinkRanges);
       }
       const shouldExtractSegments = typeof rawParagraph === "string" && (rawParagraph.includes("<") || rawParagraph.includes("{"));
       if (shouldExtractSegments) {
-        const baseOffset = typeof block.payload.range?.from === "number" ? block.payload.range.from : void 0;
         const mixedOptions = !block.isFinalized && shouldAllowMixedStreaming() ? {
           html: formatAnticipationConfig.html ? { autoClose: true, maxNewlines: MIXED_CONTENT_AUTOCLOSE_NEWLINES } : void 0,
           mdx: formatAnticipationConfig.mdx ? { autoClose: true, maxNewlines: MIXED_CONTENT_AUTOCLOSE_NEWLINES, componentAllowlist: mdxComponentAllowlist ?? void 0 } : void 0
         } : void 0;
-        const segments = extractMixedContentSegments(rawParagraph, baseOffset, (value) => inlineParse(value), mixedOptions);
+        const protectedRangesAbsolute = typeof baseOffset === "number" && protectedRanges.length > 0 ? protectedRanges.map((range2) => ({
+          ...range2,
+          from: baseOffset + range2.from,
+          to: baseOffset + range2.to
+        })) : void 0;
+        const mixedSegmentOptions = mixedOptions || protectedRangesAbsolute ? {
+          ...mixedOptions,
+          protectedRanges: protectedRangesAbsolute
+        } : void 0;
+        const segments = extractMixedContentSegments(rawParagraph, baseOffset, (value) => inlineParse(value), mixedSegmentOptions);
+        const htmlRanges = collectHtmlProtectedRangesFromSegments(segments, baseOffset);
+        if (htmlRanges.length > 0) {
+          protectedRanges.push(...htmlRanges);
+        }
         if (segments.length > 0) {
           nextMeta.mixedSegments = segments;
           metaChanged = true;
@@ -82713,6 +82970,13 @@ async function enrichBlock(block) {
           metaChanged = true;
         }
       }
+      if (protectedRanges.length > 0) {
+        nextMeta.protectedRanges = protectedRanges;
+        metaChanged = true;
+      } else if (Object.prototype.hasOwnProperty.call(nextMeta, "protectedRanges")) {
+        nextMeta.protectedRanges = void 0;
+        metaChanged = true;
+      }
       if (metaChanged) {
         if (Object.keys(nextMeta).length > 0) {
           block.payload.meta = nextMeta;
@@ -82728,9 +82992,14 @@ async function enrichBlock(block) {
       await enrichCodeBlock(block);
       break;
     case "html": {
-      const sanitized = sanitizeHtmlInWorker(block.payload.raw);
+      const rawHtml = typeof block.payload.raw === "string" ? block.payload.raw : "";
+      const sanitized = sanitizeHtmlInWorker(rawHtml);
       block.payload.sanitizedHtml = sanitized;
-      block.payload.meta = { ...block.payload.meta || {}, sanitized: true };
+      const nextMeta = { ...block.payload.meta || {}, sanitized: true };
+      if (rawHtml) {
+        nextMeta.protectedRanges = [{ from: 0, to: rawHtml.length, kind: "html-block" }];
+      }
+      block.payload.meta = nextMeta;
       break;
     }
     case "list":
@@ -82750,8 +83019,12 @@ async function enrichBlock(block) {
     const mdxOptions = normalizedRanges && normalizedRanges.length > 0 ? { protectedRanges: normalizedRanges, baseOffset } : baseOffset ? { baseOffset } : void 0;
     const mdxDetectStart = now();
     let shouldConvertToMDX = detectMDX(block.payload.raw, mdxOptions);
+    if (!shouldConvertToMDX && block.type === "html") {
+      const fallbackOptions = baseOffset ? { baseOffset } : void 0;
+      shouldConvertToMDX = detectMDX(block.payload.raw, fallbackOptions);
+    }
     metrics?.recordMdxDetect(now() - mdxDetectStart);
-    if (shouldConvertToMDX && protectedRanges && protectedRanges.length > 0) {
+    if (shouldConvertToMDX && protectedRanges && protectedRanges.length > 0 && block.type !== "html") {
       const exprPattern = /\{[^{}]+\}/g;
       let match;
       while (true) {
@@ -82838,65 +83111,898 @@ function collectMathProtectedRanges(content4) {
   }
   return ranges;
 }
+function collectAutolinkProtectedRanges(content4) {
+  if (!content4) return [];
+  const ranges = [];
+  const autolinkPattern = /<((?:https?:\/\/|mailto:)[^>\s]+)>/gi;
+  let match = autolinkPattern.exec(content4);
+  while (match !== null) {
+    ranges.push({ from: match.index, to: match.index + match[0].length, kind: "autolink" });
+    match = autolinkPattern.exec(content4);
+  }
+  return ranges;
+}
+function collectHtmlProtectedRangesFromSegments(segments, baseOffset) {
+  if (!segments || segments.length === 0) return [];
+  if (typeof baseOffset !== "number" || !Number.isFinite(baseOffset)) {
+    return [];
+  }
+  const ranges = [];
+  for (const segment of segments) {
+    if (segment.kind !== "html") continue;
+    const range2 = segment.range;
+    if (!range2) continue;
+    const from = range2.from - baseOffset;
+    const to = range2.to - baseOffset;
+    if (!Number.isFinite(from) || !Number.isFinite(to) || to <= from) continue;
+    ranges.push({ from, to, kind: "html-inline" });
+  }
+  return ranges;
+}
+var FONT_STYLE_ITALIC = 1;
+var FONT_STYLE_BOLD = 2;
+var FONT_STYLE_UNDERLINE = 4;
+var FONT_STYLE_STRIKETHROUGH = 8;
+function escapeHtmlText(text12) {
+  return text12.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;").replace(/'/g, "&#39;");
+}
+function renderShikiToken(token2) {
+  const dark = token2.variants.dark;
+  const light = token2.variants.light;
+  const styles2 = [];
+  if (dark?.color) {
+    styles2.push(`--shiki-dark:${dark.color}`);
+  }
+  if (light?.color) {
+    styles2.push(`--shiki-light:${light.color}`);
+  }
+  const fontStyle = light?.fontStyle ?? dark?.fontStyle;
+  if (typeof fontStyle === "number" && fontStyle > 0) {
+    if (fontStyle & FONT_STYLE_ITALIC) {
+      styles2.push("font-style: italic");
+    }
+    if (fontStyle & FONT_STYLE_BOLD) {
+      styles2.push("font-weight: bold");
+    }
+    const decorations2 = [];
+    if (fontStyle & FONT_STYLE_UNDERLINE) {
+      decorations2.push("underline");
+    }
+    if (fontStyle & FONT_STYLE_STRIKETHROUGH) {
+      decorations2.push("line-through");
+    }
+    if (decorations2.length > 0) {
+      styles2.push(`text-decoration: ${decorations2.join(" ")}`);
+    }
+  }
+  const styleAttr = styles2.length > 0 ? ` style="${styles2.join(";")}"` : "";
+  return `<span${styleAttr}>${escapeHtmlText(token2.content)}</span>`;
+}
+function getTokenFontStyle(token2) {
+  const dark = token2.variants?.dark?.fontStyle ?? 0;
+  const light = token2.variants?.light?.fontStyle ?? 0;
+  return (typeof dark === "number" ? dark : 0) | (typeof light === "number" ? light : 0);
+}
+function mergeWhitespaceTokens2(tokens) {
+  return tokens.map((line) => {
+    const merged = [];
+    let carryContent = "";
+    let carryOffset = 0;
+    let carryVariants = null;
+    line.forEach((token2, idx) => {
+      const fontStyle = getTokenFontStyle(token2);
+      const couldMerge = (fontStyle & FONT_STYLE_UNDERLINE) === 0;
+      if (couldMerge && /^\s+$/.test(token2.content) && line[idx + 1]) {
+        if (!carryContent) {
+          carryOffset = token2.offset ?? 0;
+          carryVariants = token2.variants;
+        }
+        carryContent += token2.content;
+        return;
+      }
+      if (carryContent) {
+        if (couldMerge) {
+          merged.push({ ...token2, offset: carryOffset, content: carryContent + token2.content });
+        } else {
+          merged.push({ content: carryContent, offset: carryOffset, variants: carryVariants ?? token2.variants });
+          merged.push(token2);
+        }
+        carryContent = "";
+        carryOffset = 0;
+        carryVariants = null;
+        return;
+      }
+      merged.push(token2);
+    });
+    return merged;
+  });
+}
+function renderShikiLines(tokens) {
+  const merged = mergeWhitespaceTokens2(tokens);
+  return merged.map((lineTokens) => lineTokens.map(renderShikiToken).join(""));
+}
+function detectDiffLanguage(lang236, meta2) {
+  const normalized = (lang236 || "").trim().toLowerCase();
+  if (!normalized) {
+    return { isDiff: false, diffLang: "diff", baseLang: null };
+  }
+  let baseLang = null;
+  if (normalized === "diff" || normalized === "udiff" || normalized === "diff-unified") {
+    baseLang = null;
+  } else if (normalized.startsWith("diff-")) {
+    baseLang = normalized.slice("diff-".length) || null;
+  } else if (normalized.endsWith("-diff")) {
+    baseLang = normalized.slice(0, Math.max(0, normalized.length - "-diff".length)) || null;
+  } else {
+    return { isDiff: false, diffLang: "diff", baseLang: null };
+  }
+  if (!baseLang) {
+    const metaLang = typeof meta2.lang === "string" ? meta2.lang : typeof meta2.language === "string" ? meta2.language : typeof meta2.base === "string" ? meta2.base : null;
+    if (metaLang) {
+      baseLang = metaLang;
+    } else {
+      const metaKeys = Object.keys(meta2);
+      if (metaKeys.length > 0) {
+        baseLang = metaKeys[0] ?? null;
+      }
+    }
+  }
+  return {
+    isDiff: true,
+    diffLang: "diff",
+    baseLang: baseLang ? normalizeLang(String(baseLang)) : null
+  };
+}
+function parseDiffHunkHeader(line) {
+  const match = line.match(/^@@\s+-(\d+)(?:,(\d+))?\s+\+(\d+)(?:,(\d+))?\s+@@/);
+  if (!match) return null;
+  return {
+    oldStart: Number.parseInt(match[1] ?? "0", 10),
+    newStart: Number.parseInt(match[3] ?? "0", 10)
+  };
+}
+function isDiffMetaLine(line) {
+  if (!line) return false;
+  return line.startsWith("diff ") || line.startsWith("index ") || line.startsWith("new file") || line.startsWith("deleted file") || line.startsWith("similarity index") || line.startsWith("rename from") || line.startsWith("rename to") || line.startsWith("mode ") || line.startsWith("\\\\ No newline");
+}
+function parseUnifiedDiffLine(line, cursor) {
+  if (line.startsWith("@@")) {
+    const header = parseDiffHunkHeader(line);
+    if (header) {
+      cursor.oldLine = header.oldStart;
+      cursor.newLine = header.newStart;
+    }
+    return { text: line, kind: "hunk", prefix: "", content: line, oldNo: null, newNo: null };
+  }
+  if (line.startsWith("--- ") || line.startsWith("+++ ") || isDiffMetaLine(line)) {
+    return { text: line, kind: "meta", prefix: "", content: line, oldNo: null, newNo: null };
+  }
+  if (line.startsWith("+")) {
+    if (line.startsWith("+++")) {
+      return { text: line, kind: "meta", prefix: "", content: line, oldNo: null, newNo: null };
+    }
+    const newNo = cursor.newLine;
+    if (cursor.newLine !== null) cursor.newLine += 1;
+    return { text: line, kind: "add", prefix: "+", content: line.slice(1), oldNo: null, newNo };
+  }
+  if (line.startsWith("-")) {
+    if (line.startsWith("---")) {
+      return { text: line, kind: "meta", prefix: "", content: line, oldNo: null, newNo: null };
+    }
+    const oldNo = cursor.oldLine;
+    if (cursor.oldLine !== null) cursor.oldLine += 1;
+    return { text: line, kind: "remove", prefix: "-", content: line.slice(1), oldNo, newNo: null };
+  }
+  if (line.startsWith(" ")) {
+    const oldNo = cursor.oldLine;
+    const newNo = cursor.newLine;
+    if (cursor.oldLine !== null) cursor.oldLine += 1;
+    if (cursor.newLine !== null) cursor.newLine += 1;
+    return { text: line, kind: "context", prefix: " ", content: line.slice(1), oldNo, newNo };
+  }
+  return { text: line, kind: "meta", prefix: "", content: line, oldNo: null, newNo: null };
+}
+function tokenizeDiffRun(lines, language, grammarState) {
+  if (!highlighter) {
+    return { tokens: [], nextGrammarState: grammarState };
+  }
+  const content4 = lines.map((line) => line.content).join("\\n");
+  if (!content4 && lines.length > 0) {
+    return { tokens: lines.map(() => []), nextGrammarState: grammarState };
+  }
+  const tokens = highlighter.codeToTokensWithThemes(content4, {
+    lang: language,
+    themes: CODE_HIGHLIGHT_THEMES,
+    grammarState
+  });
+  const nextGrammarState = highlighter.getLastGrammarState(tokens);
+  return { tokens, nextGrammarState };
+}
+function buildDiffTokenLines(lines, language, grammarState) {
+  const output = [];
+  let idx = 0;
+  let nextOldState = grammarState.old;
+  let nextNewState = grammarState.new;
+  while (idx < lines.length) {
+    const line = lines[idx];
+    if (line.kind === "meta" || line.kind === "hunk") {
+      output.push({ spans: [{ t: line.text }] });
+      idx += 1;
+      continue;
+    }
+    const runKind = line.kind;
+    const run = [];
+    while (idx < lines.length && lines[idx].kind === runKind) {
+      run.push(lines[idx]);
+      idx += 1;
+    }
+    try {
+      if (runKind === "add") {
+        const { tokens, nextGrammarState } = tokenizeDiffRun(run, language, nextNewState);
+        nextNewState = nextGrammarState;
+        const tokenLines = renderTokenLines(tokens);
+        for (let i = 0; i < run.length; i++) {
+          const prefixSpan = { t: run[i].prefix };
+          const lineTokens = tokenLines[i] ?? null;
+          if (!lineTokens || lineTokens.spans.length === 0) {
+            const fullText = `${run[i].prefix}${run[i].content}`;
+            output.push(fullText.length > 0 ? { spans: [{ t: fullText }] } : { spans: [] });
+          } else {
+            output.push({ spans: [prefixSpan, ...lineTokens.spans] });
+          }
+        }
+        continue;
+      }
+      if (runKind === "remove") {
+        const { tokens, nextGrammarState } = tokenizeDiffRun(run, language, nextOldState);
+        nextOldState = nextGrammarState;
+        const tokenLines = renderTokenLines(tokens);
+        for (let i = 0; i < run.length; i++) {
+          const prefixSpan = { t: run[i].prefix };
+          const lineTokens = tokenLines[i] ?? null;
+          if (!lineTokens || lineTokens.spans.length === 0) {
+            const fullText = `${run[i].prefix}${run[i].content}`;
+            output.push(fullText.length > 0 ? { spans: [{ t: fullText }] } : { spans: [] });
+          } else {
+            output.push({ spans: [prefixSpan, ...lineTokens.spans] });
+          }
+        }
+        continue;
+      }
+      if (runKind === "context") {
+        const { tokens, nextGrammarState } = tokenizeDiffRun(run, language, nextNewState);
+        nextNewState = nextGrammarState;
+        if (run.length > 0) {
+          const oldResult = tokenizeDiffRun(run, language, nextOldState);
+          nextOldState = oldResult.nextGrammarState;
+        }
+        const tokenLines = renderTokenLines(tokens);
+        for (let i = 0; i < run.length; i++) {
+          const prefixSpan = { t: run[i].prefix };
+          const lineTokens = tokenLines[i] ?? null;
+          if (!lineTokens || lineTokens.spans.length === 0) {
+            const fullText = `${run[i].prefix}${run[i].content}`;
+            output.push(fullText.length > 0 ? { spans: [{ t: fullText }] } : { spans: [] });
+          } else {
+            output.push({ spans: [prefixSpan, ...lineTokens.spans] });
+          }
+        }
+        continue;
+      }
+    } catch (error) {
+      console.warn("Diff tokenization failed for", language, error);
+    }
+    for (let i = 0; i < run.length; i++) {
+      const fullText = run[i].text;
+      output.push(fullText.length > 0 ? { spans: [{ t: fullText }] } : { spans: [] });
+    }
+  }
+  return { tokenLines: output, grammarState: { old: nextOldState, new: nextNewState } };
+}
+function toTokenStyle(style2) {
+  if (!style2) return void 0;
+  const fg = typeof style2.color === "string" ? style2.color : void 0;
+  const fs = typeof style2.fontStyle === "number" && style2.fontStyle > 0 ? style2.fontStyle : void 0;
+  if (!fg && fs === void 0) return void 0;
+  return { fg, fs };
+}
+function toTokenSpan(token2) {
+  const dark = toTokenStyle(token2.variants.dark);
+  const light = toTokenStyle(token2.variants.light);
+  const span = { t: token2.content };
+  if (dark || light) {
+    span.v = {};
+    if (dark) span.v.dark = dark;
+    if (light) span.v.light = light;
+  }
+  return span;
+}
+function renderTokenLines(tokens) {
+  return tokens.map((lineTokens) => ({
+    spans: lineTokens.map((token2) => toTokenSpan(token2))
+  }));
+}
+function resetIncrementalHighlightState(blockId) {
+  if (blockId) {
+    incrementalHighlightStates.delete(blockId);
+    return;
+  }
+  incrementalHighlightStates.clear();
+}
+function getIncrementalHighlightState(blockId, lang236) {
+  const existing = incrementalHighlightStates.get(blockId);
+  if (!existing || existing.lang !== lang236) {
+    const fresh = {
+      lang: lang236,
+      tokenLang: void 0,
+      processedLength: 0,
+      pendingLine: "",
+      highlightedLines: [],
+      tokenLines: [],
+      diffKind: [],
+      oldNo: [],
+      newNo: [],
+      diffCursor: void 0,
+      diffGrammar: void 0,
+      grammarState: void 0
+    };
+    incrementalHighlightStates.set(blockId, fresh);
+    return fresh;
+  }
+  return existing;
+}
+async function resolveHighlightLanguage(requestedLanguage) {
+  if (!highlighter) return "text";
+  const loadedLangs = highlighter.getLoadedLanguages();
+  if (!loadedLangs.includes(requestedLanguage)) {
+    try {
+      await highlighter.loadLanguage(requestedLanguage);
+    } catch (loadError) {
+      console.warn(`Failed to load language ${requestedLanguage}, falling back to text:`, loadError);
+    }
+  }
+  const nextLangs = highlighter.getLoadedLanguages();
+  return nextLangs.includes(requestedLanguage) ? requestedLanguage : "text";
+}
+function makeLazySignature(codeBody, lang236, tokenLang, diffEnabled) {
+  const head2 = codeBody.slice(0, 32);
+  const tail = codeBody.slice(Math.max(0, codeBody.length - 32));
+  return `${lang236}|${tokenLang}|${diffEnabled ? "diff" : "plain"}|${codeBody.length}|${head2}|${tail}`;
+}
+function getOrInitLazyState(blockId, signature, lineCount, lang236, tokenLang, diffEnabled) {
+  const existing = lazyTokenizationStates.get(blockId);
+  if (existing && existing.signature === signature && existing.highlightedLines.length === lineCount) {
+    return existing;
+  }
+  const fresh = {
+    signature,
+    lang: lang236,
+    tokenLang,
+    processedLines: 0,
+    highlightedLines: new Array(lineCount).fill(null),
+    tokenLines: new Array(lineCount).fill(null),
+    diffKind: new Array(lineCount).fill(null),
+    oldNo: new Array(lineCount).fill(null),
+    newNo: new Array(lineCount).fill(null),
+    diffCursor: diffEnabled ? { oldLine: null, newLine: null } : void 0,
+    diffGrammar: diffEnabled ? {} : void 0,
+    grammarState: void 0
+  };
+  lazyTokenizationStates.set(blockId, fresh);
+  return fresh;
+}
+function shouldLazyTokenizeBlock(lineCount, hasHighlighter, hasHighlightableContent) {
+  if (!lazyTokenizationEnabled) return false;
+  if (!hasHighlighter || !hasHighlightableContent) return false;
+  return lineCount >= lazyTokenizationThresholdLines;
+}
+function pruneLazyTokenizationStates(nextBlocks) {
+  if (lazyTokenizationStates.size === 0) return;
+  const ids = new Set(nextBlocks.map((block) => block.id));
+  for (const key2 of lazyTokenizationStates.keys()) {
+    if (!ids.has(key2)) {
+      lazyTokenizationStates.delete(key2);
+    }
+  }
+}
+function enqueueLazyTokenization(request) {
+  if (!lazyTokenizationEnabled) return;
+  const existing = lazyTokenizationQueue.get(request.blockId);
+  const next2 = existing ? mergeLazyRequests(existing, request) : request;
+  lazyTokenizationQueue.set(request.blockId, next2);
+  if (!lazyTokenizationScheduled) {
+    lazyTokenizationScheduled = true;
+    setTimeout(() => {
+      lazyTokenizationScheduled = false;
+      void processLazyTokenizationQueue();
+    }, 0);
+  }
+}
+function selectNextLazyRequest() {
+  let selected = null;
+  for (const request of lazyTokenizationQueue.values()) {
+    if (!selected) {
+      selected = request;
+      continue;
+    }
+    const priorityDiff = compareLazyPriority(request.priority, selected.priority);
+    if (priorityDiff > 0) {
+      selected = request;
+      continue;
+    }
+    if (priorityDiff === 0 && request.requestedAt < selected.requestedAt) {
+      selected = request;
+    }
+  }
+  return selected;
+}
+async function processLazyTokenizationQueue() {
+  if (lazyTokenizationProcessing) return;
+  lazyTokenizationProcessing = true;
+  try {
+    while (lazyTokenizationQueue.size > 0) {
+      const next2 = selectNextLazyRequest();
+      if (!next2) break;
+      lazyTokenizationQueue.delete(next2.blockId);
+      await handleLazyTokenizationRequest(next2);
+    }
+  } finally {
+    lazyTokenizationProcessing = false;
+  }
+}
+async function handleLazyTokenizationRequest(request) {
+  if (!lazyTokenizationEnabled || !highlighter) return;
+  const index2 = blocks.findIndex((block2) => block2.id === request.blockId);
+  if (index2 === -1) return;
+  const block = blocks[index2];
+  if (block.type !== "code" || !block.isFinalized) return;
+  const raw2 = block.payload.raw ?? "";
+  const { code: code4, info, hadFence } = stripCodeFence(raw2);
+  const { lang: lang236, meta: meta2 } = parseCodeFenceInfo(info);
+  const diffInfo = detectDiffLanguage(lang236, meta2);
+  const requestedLanguage = diffInfo.isDiff ? diffInfo.diffLang : lang236 || "text";
+  const tokenLanguage = diffInfo.isDiff ? diffInfo.baseLang ?? "text" : requestedLanguage;
+  const codeBody = hadFence ? code4 : dedentIndentedCode(raw2);
+  const codeLines = extractCodeLines(raw2);
+  const hasHighlightableContent = codeBody.trim().length > 0;
+  if (!shouldLazyTokenizeBlock(codeLines.length, Boolean(highlighter), hasHighlightableContent)) {
+    return;
+  }
+  const wantsHtml = highlightOutputMode === "html" || highlightOutputMode === "both";
+  const wantsTokens = highlightOutputMode === "tokens" || highlightOutputMode === "both";
+  const diffEnabled = diffInfo.isDiff && wantsTokens;
+  if (!wantsHtml && !wantsTokens) return;
+  const { startLine, endLine } = clampLazyRange(request.startLine, request.endLine, codeLines.length);
+  if (endLine <= startLine) return;
+  let resolvedLanguage = requestedLanguage;
+  let resolvedTokenLanguage = tokenLanguage;
+  if (wantsHtml) {
+    resolvedLanguage = await resolveHighlightLanguage(requestedLanguage);
+  }
+  if (wantsTokens) {
+    resolvedTokenLanguage = await resolveHighlightLanguage(tokenLanguage);
+  }
+  const signature = makeLazySignature(codeBody, resolvedLanguage, resolvedTokenLanguage, diffEnabled);
+  const state = getOrInitLazyState(block.id, signature, codeLines.length, resolvedLanguage, resolvedTokenLanguage, diffEnabled);
+  if (state.processedLines < 0 || state.processedLines > codeLines.length) {
+    state.processedLines = 0;
+  }
+  const targetEnd = Math.min(endLine, codeLines.length);
+  if (targetEnd <= state.processedLines) {
+    return;
+  }
+  const metricsCollector = new WorkerMetricsCollector(workerGrammarEngine);
+  setActiveMetricsCollector(metricsCollector);
+  metricsCollector.setBlocksProduced(blocks.length);
+  const prevSnapshot = await blockToNodeSnapshot(block);
+  const tokenizationStart = now();
+  const segmentLines = codeLines.slice(state.processedLines, targetEnd);
+  if (segmentLines.length > 0) {
+    if (wantsHtml || wantsTokens && !diffEnabled) {
+      try {
+        const sharedLanguage = wantsHtml ? resolvedLanguage : resolvedTokenLanguage;
+        const tokens = highlighter.codeToTokensWithThemes(segmentLines.join("\n"), {
+          lang: sharedLanguage,
+          themes: CODE_HIGHLIGHT_THEMES,
+          grammarState: state.grammarState
+        });
+        if (wantsHtml) {
+          const htmlLines = renderShikiLines(tokens);
+          for (let i = 0; i < htmlLines.length; i += 1) {
+            state.highlightedLines[state.processedLines + i] = htmlLines[i] ?? null;
+          }
+        }
+        if (wantsTokens && !diffEnabled) {
+          const tokenLines = renderTokenLines(tokens);
+          for (let i = 0; i < tokenLines.length; i += 1) {
+            state.tokenLines[state.processedLines + i] = tokenLines[i] ?? null;
+          }
+        }
+        state.grammarState = highlighter.getLastGrammarState(tokens);
+      } catch (error) {
+        console.warn("Lazy tokenization failed for", requestedLanguage, error);
+        if (wantsHtml) {
+          for (let i = 0; i < segmentLines.length; i += 1) {
+            state.highlightedLines[state.processedLines + i] = null;
+          }
+        }
+        if (wantsTokens && !diffEnabled) {
+          for (let i = 0; i < segmentLines.length; i += 1) {
+            state.tokenLines[state.processedLines + i] = null;
+          }
+        }
+      }
+    }
+    if (wantsTokens && diffEnabled) {
+      const cursor = state.diffCursor ?? { oldLine: null, newLine: null };
+      const diffLines = segmentLines.map((line) => parseUnifiedDiffLine(line, cursor));
+      for (let i = 0; i < diffLines.length; i += 1) {
+        const line = diffLines[i];
+        const idx = state.processedLines + i;
+        state.diffKind[idx] = line.kind;
+        state.oldNo[idx] = line.oldNo ?? null;
+        state.newNo[idx] = line.newNo ?? null;
+      }
+      try {
+        const { tokenLines, grammarState } = buildDiffTokenLines(diffLines, resolvedTokenLanguage, state.diffGrammar ?? {});
+        for (let i = 0; i < tokenLines.length; i += 1) {
+          state.tokenLines[state.processedLines + i] = tokenLines[i] ?? null;
+        }
+        state.diffGrammar = grammarState;
+      } catch (error) {
+        console.warn("Lazy diff tokenization failed for", tokenLanguage, error);
+        for (let i = 0; i < diffLines.length; i += 1) {
+          const idx = state.processedLines + i;
+          const line = diffLines[i];
+          state.tokenLines[idx] = line.text.length > 0 ? { spans: [{ t: line.text }] } : { spans: [] };
+        }
+      }
+      state.diffCursor = cursor;
+    }
+  }
+  state.processedLines = targetEnd;
+  const tokenizationDuration = now() - tokenizationStart;
+  metricsCollector.recordShiki(tokenizationDuration);
+  metricsCollector.recordHighlightForLanguage(resolvedLanguage, tokenizationDuration);
+  metricsCollector.recordLazyTokenization(
+    lazyRequestRangeSize({ ...request, startLine, endLine }),
+    now() - request.requestedAt,
+    lazyTokenizationQueue.size
+  );
+  const updated = cloneBlock(block);
+  updated.payload.highlightedHtml = void 0;
+  const effectiveLang = wantsHtml ? resolvedLanguage : resolvedTokenLanguage;
+  const nextMeta = { ...updated.payload.meta ?? {}, ...meta2, lang: effectiveLang };
+  nextMeta.lazyTokenization = true;
+  nextMeta.lazyTokenizedUntil = state.processedLines;
+  if (wantsHtml) {
+    nextMeta.highlightedLines = state.highlightedLines;
+  } else if ("highlightedLines" in nextMeta) {
+    delete nextMeta.highlightedLines;
+  }
+  if (wantsTokens) {
+    nextMeta.tokenLines = state.tokenLines;
+    if (diffEnabled) {
+      nextMeta.diffKind = state.diffKind;
+      nextMeta.oldNo = state.oldNo;
+      nextMeta.newNo = state.newNo;
+    } else {
+      if ("diffKind" in nextMeta) delete nextMeta.diffKind;
+      if ("oldNo" in nextMeta) delete nextMeta.oldNo;
+      if ("newNo" in nextMeta) delete nextMeta.newNo;
+    }
+  } else if ("tokenLines" in nextMeta) {
+    delete nextMeta.tokenLines;
+    if ("diffKind" in nextMeta) delete nextMeta.diffKind;
+    if ("oldNo" in nextMeta) delete nextMeta.oldNo;
+    if ("newNo" in nextMeta) delete nextMeta.newNo;
+  }
+  updated.payload.meta = nextMeta;
+  blocks[index2] = updated;
+  const nextSnapshot = await blockToNodeSnapshot(updated);
+  const patches = [];
+  diffNodeSnapshot(updated.id, prevSnapshot, nextSnapshot, patches, metricsCollector);
+  if (patches.length > 0) {
+    dispatchPatchBatch(patches, metricsCollector);
+  } else {
+    emitMetricsSample(metricsCollector);
+    if (getActiveMetricsCollector() === metricsCollector) {
+      setActiveMetricsCollector(null);
+    }
+  }
+}
 async function enrichCodeBlock(block) {
   performanceTimer.mark("highlight-code");
   const metrics = getActiveMetricsCollector();
   const raw2 = block.payload.raw ?? "";
   const { code: code4, info, hadFence } = stripCodeFence(raw2);
   const { lang: lang236, meta: meta2 } = parseCodeFenceInfo(info);
-  const requestedLanguage = lang236 || "text";
+  const diffInfo = detectDiffLanguage(lang236, meta2);
+  const requestedLanguage = diffInfo.isDiff ? diffInfo.diffLang : lang236 || "text";
+  const tokenLanguage = diffInfo.isDiff ? diffInfo.baseLang ?? "text" : requestedLanguage;
   const codeBody = hadFence ? code4 : dedentIndentedCode(raw2);
+  const codeLines = extractCodeLines(raw2);
+  const baseMeta = block.payload.meta ?? {};
   let resolvedLanguage = requestedLanguage;
-  let cachedHighlight = getHighlightCacheEntry(requestedLanguage, codeBody);
-  if (!block.isFinalized && !enableLiveCodeHighlighting) {
+  let resolvedTokenLanguage = tokenLanguage;
+  const hasHighlighter = Boolean(highlighter);
+  const hasHighlightableContent = codeBody.trim().length > 0;
+  const wantsHtml = highlightOutputMode === "html" || highlightOutputMode === "both";
+  const wantsTokens = highlightOutputMode === "tokens" || highlightOutputMode === "both";
+  if (!block.isFinalized) {
+    if (codeHighlightingMode === "final" || !hasHighlighter || !hasHighlightableContent) {
+      resetIncrementalHighlightState(block.id);
+      block.payload.highlightedHtml = void 0;
+      const nextMeta2 = { ...baseMeta, ...meta2, lang: resolvedLanguage };
+      if ("highlightedLines" in nextMeta2) {
+        delete nextMeta2.highlightedLines;
+      }
+      if ("tokenLines" in nextMeta2) {
+        delete nextMeta2.tokenLines;
+      }
+      if ("diffKind" in nextMeta2) {
+        delete nextMeta2.diffKind;
+      }
+      if ("oldNo" in nextMeta2) {
+        delete nextMeta2.oldNo;
+      }
+      if ("newNo" in nextMeta2) {
+        delete nextMeta2.newNo;
+      }
+      block.payload.meta = nextMeta2;
+      return;
+    }
+    if (codeHighlightingMode === "incremental") {
+      if (hasHighlighter) {
+        if (wantsHtml) {
+          resolvedLanguage = await resolveHighlightLanguage(requestedLanguage);
+        }
+        if (wantsTokens) {
+          resolvedTokenLanguage = await resolveHighlightLanguage(tokenLanguage);
+        }
+      }
+      let state = getIncrementalHighlightState(block.id, resolvedLanguage);
+      if (wantsTokens && state.tokenLang && state.tokenLang !== resolvedTokenLanguage) {
+        resetIncrementalHighlightState(block.id);
+        state = getIncrementalHighlightState(block.id, resolvedLanguage);
+      }
+      if (codeBody.length < state.processedLength) {
+        resetIncrementalHighlightState(block.id);
+        state = getIncrementalHighlightState(block.id, resolvedLanguage);
+      }
+      state.tokenLang = wantsTokens ? resolvedTokenLanguage : void 0;
+      const appended = codeBody.slice(state.processedLength);
+      const combined = state.pendingLine + appended;
+      const diffEnabled = diffInfo.isDiff && wantsTokens;
+      if (combined.length > 0) {
+        const parts = combined.split("\n");
+        const completeLines = parts.slice(0, -1);
+        const tail = parts.length > 0 ? parts[parts.length - 1] ?? "" : "";
+        if (completeLines.length > 0) {
+          if (wantsHtml || wantsTokens && !diffEnabled) {
+            try {
+              const sharedLanguage = wantsHtml ? resolvedLanguage : resolvedTokenLanguage;
+              const tokens = highlighter?.codeToTokensWithThemes(completeLines.join("\n"), {
+                lang: sharedLanguage,
+                themes: CODE_HIGHLIGHT_THEMES,
+                grammarState: state.grammarState
+              });
+              if (wantsHtml) {
+                const htmlLines = renderShikiLines(tokens);
+                state.highlightedLines.push(...htmlLines);
+              }
+              if (wantsTokens && !diffEnabled) {
+                const tokenLines2 = renderTokenLines(tokens);
+                state.tokenLines.push(...tokenLines2);
+              }
+              state.grammarState = highlighter?.getLastGrammarState(tokens);
+            } catch (error) {
+              console.warn("Incremental highlighting failed for", requestedLanguage, error);
+              if (wantsHtml) {
+                state.highlightedLines.push(...completeLines.map(() => null));
+              }
+              if (wantsTokens && !diffEnabled) {
+                state.tokenLines.push(...completeLines.map(() => null));
+              }
+            }
+          }
+          if (wantsTokens && diffEnabled) {
+            const cursor = state.diffCursor ?? { oldLine: null, newLine: null };
+            const diffLines = completeLines.map((line) => parseUnifiedDiffLine(line, cursor));
+            state.diffCursor = cursor;
+            state.diffKind.push(...diffLines.map((line) => line.kind));
+            state.oldNo.push(...diffLines.map((line) => line.oldNo ?? null));
+            state.newNo.push(...diffLines.map((line) => line.newNo ?? null));
+            try {
+              const { tokenLines: tokenLines2, grammarState } = buildDiffTokenLines(diffLines, resolvedTokenLanguage, state.diffGrammar ?? {});
+              state.tokenLines.push(...tokenLines2);
+              state.diffGrammar = grammarState;
+            } catch (error) {
+              console.warn("Incremental diff tokenization failed for", tokenLanguage, error);
+              state.tokenLines.push(
+                ...diffLines.map((line) => line.text.length > 0 ? { spans: [{ t: line.text }] } : { spans: [] })
+              );
+            }
+          }
+        }
+        state.pendingLine = tail ?? "";
+      }
+      state.processedLength = codeBody.length;
+      state.lang = resolvedLanguage;
+      block.payload.highlightedHtml = void 0;
+      const effectiveLang2 = wantsHtml ? resolvedLanguage : resolvedTokenLanguage;
+      const nextMeta2 = { ...baseMeta, ...meta2, lang: effectiveLang2 };
+      if (wantsHtml) {
+        nextMeta2.highlightedLines = state.highlightedLines;
+      } else if ("highlightedLines" in nextMeta2) {
+        delete nextMeta2.highlightedLines;
+      }
+      if (wantsTokens) {
+        nextMeta2.tokenLines = state.tokenLines;
+        if (diffEnabled) {
+          nextMeta2.diffKind = state.diffKind;
+          nextMeta2.oldNo = state.oldNo;
+          nextMeta2.newNo = state.newNo;
+        } else {
+          if ("diffKind" in nextMeta2) delete nextMeta2.diffKind;
+          if ("oldNo" in nextMeta2) delete nextMeta2.oldNo;
+          if ("newNo" in nextMeta2) delete nextMeta2.newNo;
+        }
+      } else if ("tokenLines" in nextMeta2) {
+        delete nextMeta2.tokenLines;
+        if ("diffKind" in nextMeta2) delete nextMeta2.diffKind;
+        if ("oldNo" in nextMeta2) delete nextMeta2.oldNo;
+        if ("newNo" in nextMeta2) delete nextMeta2.newNo;
+      }
+      block.payload.meta = nextMeta2;
+      const highlightDuration2 = performanceTimer.measure("highlight-code");
+      metrics?.recordShiki(highlightDuration2);
+      metrics?.recordHighlightForLanguage(resolvedLanguage, highlightDuration2);
+      return;
+    }
+  }
+  resetIncrementalHighlightState(block.id);
+  if (block.isFinalized && shouldLazyTokenizeBlock(codeLines.length, hasHighlighter, hasHighlightableContent) && (wantsHtml || wantsTokens)) {
+    if (hasHighlighter) {
+      if (wantsHtml) {
+        resolvedLanguage = await resolveHighlightLanguage(requestedLanguage);
+      }
+      if (wantsTokens) {
+        resolvedTokenLanguage = await resolveHighlightLanguage(tokenLanguage);
+      }
+    }
+    const diffEnabled = diffInfo.isDiff && wantsTokens;
+    const signature = makeLazySignature(codeBody, resolvedLanguage, resolvedTokenLanguage, diffEnabled);
+    const state = getOrInitLazyState(block.id, signature, codeLines.length, resolvedLanguage, resolvedTokenLanguage, diffEnabled);
+    const effectiveLang2 = wantsHtml ? resolvedLanguage : resolvedTokenLanguage;
+    const nextMeta2 = { ...baseMeta, ...meta2, lang: effectiveLang2 };
+    nextMeta2.lazyTokenization = true;
+    nextMeta2.lazyTokenizedUntil = state.processedLines;
+    if (wantsHtml) {
+      nextMeta2.highlightedLines = state.highlightedLines;
+    } else if ("highlightedLines" in nextMeta2) {
+      delete nextMeta2.highlightedLines;
+    }
+    if (wantsTokens) {
+      nextMeta2.tokenLines = state.tokenLines;
+      if (diffEnabled) {
+        nextMeta2.diffKind = state.diffKind;
+        nextMeta2.oldNo = state.oldNo;
+        nextMeta2.newNo = state.newNo;
+      } else {
+        if ("diffKind" in nextMeta2) delete nextMeta2.diffKind;
+        if ("oldNo" in nextMeta2) delete nextMeta2.oldNo;
+        if ("newNo" in nextMeta2) delete nextMeta2.newNo;
+      }
+    } else if ("tokenLines" in nextMeta2) {
+      delete nextMeta2.tokenLines;
+      if ("diffKind" in nextMeta2) delete nextMeta2.diffKind;
+      if ("oldNo" in nextMeta2) delete nextMeta2.oldNo;
+      if ("newNo" in nextMeta2) delete nextMeta2.newNo;
+    }
     block.payload.highlightedHtml = void 0;
-    block.payload.meta = {
-      ...meta2,
-      lang: resolvedLanguage
-    };
+    block.payload.meta = nextMeta2;
     return;
   }
-  if (!cachedHighlight && highlighter && codeBody.trim()) {
-    try {
-      const loadedLangs = highlighter.getLoadedLanguages();
-      if (!loadedLangs.includes(requestedLanguage)) {
+  let tokenLines;
+  let diffKind;
+  let diffOldNo;
+  let diffNewNo;
+  if (wantsHtml) {
+    let cachedHighlight = block.isFinalized ? getHighlightCacheEntry(requestedLanguage, codeBody) : null;
+    if (!cachedHighlight && highlighter && hasHighlightableContent) {
+      try {
+        resolvedLanguage = await resolveHighlightLanguage(requestedLanguage);
+        const highlighted = highlighter.codeToHtml(codeBody, {
+          lang: resolvedLanguage,
+          themes: CODE_HIGHLIGHT_THEMES,
+          defaultColor: false
+        });
+        const enhanced = enhanceHighlightedHtml(highlighted, resolvedLanguage);
+        block.payload.highlightedHtml = enhanced;
+        if (block.isFinalized) {
+          setHighlightCacheEntry(resolvedLanguage, codeBody, enhanced);
+          if (resolvedLanguage !== requestedLanguage) {
+            setHighlightCacheEntry(requestedLanguage, codeBody, enhanced);
+          }
+        }
+        cachedHighlight = getHighlightCacheEntry(resolvedLanguage, codeBody);
+      } catch (error) {
+        console.warn("Highlighting failed for", requestedLanguage, error);
+        resolvedLanguage = "text";
+      }
+    }
+    if (cachedHighlight) {
+      resolvedLanguage = cachedHighlight.lang;
+      block.payload.highlightedHtml = cachedHighlight.html;
+    }
+  } else {
+    block.payload.highlightedHtml = void 0;
+  }
+  if (wantsTokens) {
+    if (diffInfo.isDiff) {
+      if (hasHighlighter) {
+        resolvedTokenLanguage = await resolveHighlightLanguage(tokenLanguage);
+      }
+      const diffLines = codeBody.length > 0 ? codeBody.split("\n") : [];
+      const cursor = { oldLine: null, newLine: null };
+      const diffInfoLines = diffLines.map((line) => parseUnifiedDiffLine(line, cursor));
+      diffKind = diffInfoLines.map((line) => line.kind);
+      diffOldNo = diffInfoLines.map((line) => line.oldNo ?? null);
+      diffNewNo = diffInfoLines.map((line) => line.newNo ?? null);
+      if (!hasHighlighter || !hasHighlightableContent) {
+        tokenLines = diffInfoLines.map((line) => line.text.length > 0 ? { spans: [{ t: line.text }] } : { spans: [] });
+      } else {
         try {
-          await highlighter.loadLanguage(requestedLanguage);
-        } catch (loadError) {
-          console.warn(`Failed to load language ${requestedLanguage}, falling back to text:`, loadError);
+          const result = buildDiffTokenLines(diffInfoLines, resolvedTokenLanguage, {});
+          tokenLines = result.tokenLines;
+        } catch (error) {
+          console.warn("Diff tokenization failed for", tokenLanguage, error);
+          tokenLines = diffInfoLines.map((line) => line.text.length > 0 ? { spans: [{ t: line.text }] } : { spans: [] });
         }
       }
-      resolvedLanguage = loadedLangs.includes(requestedLanguage) ? requestedLanguage : "text";
-      const highlighted = highlighter.codeToHtml(codeBody, {
-        lang: resolvedLanguage,
-        themes: {
-          dark: "github-dark",
-          light: "github-light"
-        },
-        defaultColor: false
-      });
-      const enhanced = enhanceHighlightedHtml(highlighted, resolvedLanguage);
-      block.payload.highlightedHtml = enhanced;
-      if (block.isFinalized) {
-        setHighlightCacheEntry(resolvedLanguage, codeBody, enhanced);
-        if (resolvedLanguage !== requestedLanguage) {
-          setHighlightCacheEntry(requestedLanguage, codeBody, enhanced);
+    } else {
+      const lineCount = codeBody.length > 0 ? codeBody.split("\n").length : 0;
+      if (!hasHighlighter || !hasHighlightableContent) {
+        tokenLines = new Array(lineCount).fill(null);
+      } else {
+        try {
+          resolvedTokenLanguage = await resolveHighlightLanguage(tokenLanguage);
+          const tokens = highlighter?.codeToTokensWithThemes(codeBody, {
+            lang: resolvedTokenLanguage,
+            themes: CODE_HIGHLIGHT_THEMES
+          });
+          tokenLines = renderTokenLines(tokens);
+        } catch (error) {
+          console.warn("Tokenization failed for", requestedLanguage, error);
+          tokenLines = new Array(lineCount).fill(null);
         }
       }
-      cachedHighlight = getHighlightCacheEntry(resolvedLanguage, codeBody);
-    } catch (error) {
-      console.warn("Highlighting failed for", requestedLanguage, error);
-      resolvedLanguage = "text";
     }
   }
-  if (cachedHighlight) {
-    resolvedLanguage = cachedHighlight.lang;
-    block.payload.highlightedHtml = cachedHighlight.html;
+  const effectiveLang = wantsHtml ? resolvedLanguage : resolvedTokenLanguage;
+  const nextMeta = { ...baseMeta, ...meta2, lang: effectiveLang };
+  if ("highlightedLines" in nextMeta) {
+    delete nextMeta.highlightedLines;
   }
-  block.payload.meta = {
-    ...meta2,
-    lang: resolvedLanguage
-  };
+  if (wantsTokens) {
+    nextMeta.tokenLines = tokenLines ?? [];
+    if (diffKind) {
+      nextMeta.diffKind = diffKind;
+      nextMeta.oldNo = diffOldNo ?? [];
+      nextMeta.newNo = diffNewNo ?? [];
+    } else {
+      if ("diffKind" in nextMeta) delete nextMeta.diffKind;
+      if ("oldNo" in nextMeta) delete nextMeta.oldNo;
+      if ("newNo" in nextMeta) delete nextMeta.newNo;
+    }
+  } else if ("tokenLines" in nextMeta) {
+    delete nextMeta.tokenLines;
+    if ("diffKind" in nextMeta) delete nextMeta.diffKind;
+    if ("oldNo" in nextMeta) delete nextMeta.oldNo;
+    if ("newNo" in nextMeta) delete nextMeta.newNo;
+  }
+  block.payload.meta = nextMeta;
   const highlightDuration = performanceTimer.measure("highlight-code");
   metrics?.recordShiki(highlightDuration);
   metrics?.recordHighlightForLanguage(resolvedLanguage, highlightDuration);
@@ -83149,7 +84255,12 @@ function enrichListBlock(block) {
   }
   const items = itemsRaw.map((raw2) => inlineParser.parse(raw2));
   const isOrdered = /^\d+\./.test(lines[0]?.trimStart() || "");
-  block.payload.meta = { ordered: isOrdered, items };
+  block.payload.meta = {
+    ordered: isOrdered,
+    items,
+    formatAnticipation: block.isFinalized ? void 0 : formatAnticipationConfig,
+    mathEnabled: enableMath
+  };
 }
 function runDocumentPlugins(inputBlocks, content4) {
   const blocks2 = inputBlocks.slice();
@@ -83492,6 +84603,10 @@ function diffNodeSnapshot(blockId, prevNode, nextNode, patches, metrics) {
     if (onlyAppend && nextChildren.length > prevChildren.length) {
       const startIndex = prevChildren.length;
       const appended = nextChildren.slice(startIndex);
+      const hasTokenLines = appended.some((child) => Object.prototype.hasOwnProperty.call(child.props ?? {}, "tokens"));
+      const hasDiffKind = appended.some((child) => Object.prototype.hasOwnProperty.call(child.props ?? {}, "diffKind"));
+      const hasOldNo = appended.some((child) => Object.prototype.hasOwnProperty.call(child.props ?? {}, "oldNo"));
+      const hasNewNo = appended.some((child) => Object.prototype.hasOwnProperty.call(child.props ?? {}, "newNo"));
       patches.push({
         op: "appendLines",
         at: { blockId, nodeId: prevNode.id },
@@ -83500,7 +84615,27 @@ function diffNodeSnapshot(blockId, prevNode, nextNode, patches, metrics) {
           const text12 = typeof child.props?.text === "string" ? child.props?.text : "";
           return text12;
         }),
-        highlight: appended.map((child) => typeof child.props?.html === "string" ? child.props?.html : null)
+        highlight: appended.map((child) => typeof child.props?.html === "string" ? child.props?.html : null),
+        ...hasTokenLines ? {
+          tokens: appended.map(
+            (child) => Object.prototype.hasOwnProperty.call(child.props ?? {}, "tokens") ? child.props?.tokens : null
+          )
+        } : {},
+        ...hasDiffKind ? {
+          diffKind: appended.map(
+            (child) => Object.prototype.hasOwnProperty.call(child.props ?? {}, "diffKind") ? child.props?.diffKind : null
+          )
+        } : {},
+        ...hasOldNo ? {
+          oldNo: appended.map(
+            (child) => Object.prototype.hasOwnProperty.call(child.props ?? {}, "oldNo") ? child.props?.oldNo : null
+          )
+        } : {},
+        ...hasNewNo ? {
+          newNo: appended.map(
+            (child) => Object.prototype.hasOwnProperty.call(child.props ?? {}, "newNo") ? child.props?.newNo : null
+          )
+        } : {}
       });
       metrics?.recordAppendLines(appended.length);
       return;
@@ -84019,6 +85154,17 @@ async function processWorkerMessage(msg) {
       });
       return;
     }
+    case "TOKENIZE_RANGE": {
+      const priority = msg.priority === "prefetch" ? "prefetch" : "visible";
+      enqueueLazyTokenization({
+        blockId: msg.blockId,
+        startLine: msg.startLine,
+        endLine: msg.endLine,
+        priority,
+        requestedAt: now()
+      });
+      return;
+    }
     case "MDX_COMPILED":
       handleMdxStatus(msg.blockId, {
         compiledRef: { id: msg.compiledId },