@stream-mdx/worker 0.2.0 → 0.3.0

package/dist/hosted/markdown-worker.js

@@ -82165,6 +82165,9 @@ var mdxCompileMode = "server";
 var formatAnticipationConfig = normalizeFormatAnticipation(false);
 var codeHighlightingMode = "final";
 var highlightOutputMode = "html";
+var emitHighlightTokens = true;
+var emitDiffBlocks = false;
+var liveTokenizationEnabled = true;
 var enableMath = true;
 var mdxComponentAllowlist = null;
 var WORKER_MDX_CACHE = /* @__PURE__ */ new Map();
@@ -82604,7 +82607,10 @@ async function initialize(initialContent = "", prewarmLangs = [], docPlugins, md
     math: docPlugins?.math ?? true,
     formatAnticipation: docPlugins?.formatAnticipation ?? false,
     codeHighlighting: docPlugins?.codeHighlighting,
-    liveCodeHighlighting: docPlugins?.liveCodeHighlighting ?? false
+    liveCodeHighlighting: docPlugins?.liveCodeHighlighting ?? false,
+    liveTokenization: docPlugins?.liveTokenization ?? true,
+    emitHighlightTokens: docPlugins?.emitHighlightTokens ?? true,
+    emitDiffBlocks: docPlugins?.emitDiffBlocks ?? false
   };
   enableMath = enable.math;
   formatAnticipationConfig = normalizeFormatAnticipation(enable.formatAnticipation);
@@ -82618,6 +82624,9 @@ async function initialize(initialContent = "", prewarmLangs = [], docPlugins, md
   } else {
     highlightOutputMode = "html";
   }
+  emitHighlightTokens = enable.emitHighlightTokens;
+  emitDiffBlocks = enable.emitDiffBlocks;
+  liveTokenizationEnabled = enable.liveTokenization;
   if (docPlugins?.lazyTokenization) {
     lazyTokenizationEnabled = docPlugins.lazyTokenization.enabled ?? true;
     const desiredThreshold = docPlugins.lazyTokenization.thresholdLines ?? DEFAULT_LAZY_TOKENIZATION_THRESHOLD;
@@ -83302,6 +83311,154 @@ function parseUnifiedDiffLine(line, cursor) {
   }
   return { text: line, kind: "meta", prefix: "", content: line, oldNo: null, newNo: null };
 }
+function normalizeDiffPath(value) {
+  if (!value) return null;
+  let path3 = value.trim();
+  if (!path3 || path3 === "/dev/null") return null;
+  if (path3.startsWith('"') && path3.endsWith('"') || path3.startsWith("'") && path3.endsWith("'")) {
+    path3 = path3.slice(1, -1);
+  }
+  if (path3.startsWith("a/") || path3.startsWith("b/")) {
+    path3 = path3.slice(2);
+  }
+  return path3 || null;
+}
+function extractDiffFilePath(line) {
+  if (line.startsWith("diff --git")) {
+    const match = line.match(/^diff --git\\s+(\\S+)\\s+(\\S+)/);
+    if (match) {
+      return normalizeDiffPath(match[2] ?? match[1] ?? null);
+    }
+  }
+  if (line.startsWith("+++ ") || line.startsWith("--- ")) {
+    const path3 = normalizeDiffPath(line.slice(4));
+    return path3;
+  }
+  if (line.startsWith("rename to ")) {
+    return normalizeDiffPath(line.slice("rename to ".length));
+  }
+  return null;
+}
+function looksLikeUnifiedDiff(lines) {
+  if (lines.length === 0) return false;
+  for (const line of lines) {
+    if (line.startsWith("diff --git") || line.startsWith("@@") || line.startsWith("+++ ") || line.startsWith("--- ")) {
+      return true;
+    }
+  }
+  return false;
+}
+function guessLanguageFromPath(filePath) {
+  if (!filePath) return null;
+  const parts = filePath.split(/[\\\\/]/);
+  const file = parts[parts.length - 1] ?? "";
+  const idx = file.lastIndexOf(".");
+  if (idx <= 0 || idx >= file.length - 1) return null;
+  const ext = file.slice(idx + 1);
+  return normalizeLang(ext);
+}
+function guessLanguageFromDiffLines(lines) {
+  for (const line of lines) {
+    const filePath = extractDiffFilePath(line);
+    if (filePath) {
+      const guessed = guessLanguageFromPath(filePath);
+      if (guessed) return guessed;
+    }
+  }
+  return null;
+}
+function toDiffLineKind(kind) {
+  if (kind === "remove") return "del";
+  return kind;
+}
+function toThemedLine(tokenLine, theme) {
+  if (!tokenLine) return null;
+  return tokenLine.spans.map((span) => {
+    const style2 = span.s ?? span.v?.[theme];
+    const color3 = typeof style2?.fg === "string" ? style2.fg : null;
+    const fontStyle = typeof style2?.fs === "number" ? style2.fs : null;
+    return {
+      content: span.t,
+      color: color3,
+      fontStyle
+    };
+  });
+}
+function stripPrefixFromThemedLine(tokens, prefix) {
+  if (!tokens || !prefix) return tokens;
+  if (tokens.length === 0) return tokens;
+  const first = tokens[0];
+  if (first.content === prefix) {
+    return tokens.slice(1);
+  }
+  if (first.content.startsWith(prefix)) {
+    const trimmed = first.content.slice(prefix.length);
+    const next2 = trimmed ? [{ ...first, content: trimmed }, ...tokens.slice(1)] : tokens.slice(1);
+    return next2;
+  }
+  return tokens;
+}
+function buildDiffBlocksFromLines(lines, rawLines, tokenLines, defaultLanguage) {
+  const blocks2 = [];
+  let current2 = null;
+  let currentRaw = [];
+  let additions = 0;
+  let deletions = 0;
+  const finalize = () => {
+    if (!current2) return;
+    current2.additions = additions;
+    current2.deletions = deletions;
+    if (currentRaw.length > 0) {
+      current2.unified = currentRaw.join("\n");
+    }
+    blocks2.push(current2);
+    current2 = null;
+    currentRaw = [];
+    additions = 0;
+    deletions = 0;
+  };
+  for (let i = 0; i < lines.length; i++) {
+    const line = lines[i];
+    const raw2 = rawLines[i] ?? line.text;
+    const filePath = extractDiffFilePath(raw2);
+    if (raw2.startsWith("diff --git")) {
+      finalize();
+    }
+    if (!current2) {
+      current2 = {
+        kind: "diff",
+        filePath: filePath ?? null,
+        language: filePath ? guessLanguageFromPath(filePath) ?? defaultLanguage : defaultLanguage,
+        lines: [],
+        additions: null,
+        deletions: null,
+        unified: null
+      };
+    } else if (filePath && !current2.filePath) {
+      current2.filePath = filePath;
+      if (!current2.language) {
+        current2.language = guessLanguageFromPath(filePath) ?? defaultLanguage;
+      }
+    }
+    if (line.kind === "add") additions += 1;
+    if (line.kind === "remove") deletions += 1;
+    let themed = tokenLines ? toThemedLine(tokenLines[i] ?? null, "dark") : null;
+    if (line.kind === "add" || line.kind === "remove" || line.kind === "context") {
+      themed = stripPrefixFromThemedLine(themed, line.prefix);
+    }
+    const entry = {
+      kind: toDiffLineKind(line.kind),
+      oldNo: line.oldNo ?? null,
+      newNo: line.newNo ?? null,
+      raw: raw2,
+      tokens: themed ?? null
+    };
+    current2.lines.push(entry);
+    currentRaw.push(raw2);
+  }
+  finalize();
+  return blocks2;
+}
 function tokenizeDiffRun(lines, language, grammarState) {
   if (!highlighter) {
     return { tokens: [], nextGrammarState: grammarState };
@@ -83559,17 +83716,24 @@ async function handleLazyTokenizationRequest(request) {
   const raw2 = block.payload.raw ?? "";
   const { code: code4, info, hadFence } = stripCodeFence(raw2);
   const { lang: lang236, meta: meta2 } = parseCodeFenceInfo(info);
-  const diffInfo = detectDiffLanguage(lang236, meta2);
-  const requestedLanguage = diffInfo.isDiff ? diffInfo.diffLang : lang236 || "text";
-  const tokenLanguage = diffInfo.isDiff ? diffInfo.baseLang ?? "text" : requestedLanguage;
   const codeBody = hadFence ? code4 : dedentIndentedCode(raw2);
   const codeLines = extractCodeLines(raw2);
+  let diffInfo = detectDiffLanguage(lang236, meta2);
+  if (!diffInfo.isDiff && emitDiffBlocks && looksLikeUnifiedDiff(codeLines)) {
+    diffInfo = {
+      isDiff: true,
+      diffLang: "diff",
+      baseLang: guessLanguageFromDiffLines(codeLines)
+    };
+  }
+  const requestedLanguage = diffInfo.isDiff ? diffInfo.diffLang : lang236 || "text";
+  const tokenLanguage = diffInfo.isDiff ? diffInfo.baseLang ?? "text" : requestedLanguage;
   const hasHighlightableContent = codeBody.trim().length > 0;
   if (!shouldLazyTokenizeBlock(codeLines.length, Boolean(highlighter), hasHighlightableContent)) {
     return;
   }
   const wantsHtml = highlightOutputMode === "html" || highlightOutputMode === "both";
-  const wantsTokens = highlightOutputMode === "tokens" || highlightOutputMode === "both";
+  const wantsTokens = (highlightOutputMode === "tokens" || highlightOutputMode === "both") && emitHighlightTokens;
   const diffEnabled = diffInfo.isDiff && wantsTokens;
   if (!wantsHtml && !wantsTokens) return;
   const { startLine, endLine } = clampLazyRange(request.startLine, request.endLine, codeLines.length);
@@ -83697,6 +83861,13 @@ async function handleLazyTokenizationRequest(request) {
     if ("oldNo" in nextMeta) delete nextMeta.oldNo;
     if ("newNo" in nextMeta) delete nextMeta.newNo;
   }
+  if (emitDiffBlocks && diffInfo.isDiff) {
+    const cursor = { oldLine: null, newLine: null };
+    const diffLines = codeLines.map((line) => parseUnifiedDiffLine(line, cursor));
+    nextMeta.diffBlocks = buildDiffBlocksFromLines(diffLines, codeLines, wantsTokens ? state.tokenLines : null, diffInfo.baseLang ?? null);
+  } else if ("diffBlocks" in nextMeta) {
+    delete nextMeta.diffBlocks;
+  }
   updated.payload.meta = nextMeta;
   blocks[index2] = updated;
   const nextSnapshot = await blockToNodeSnapshot(updated);
@@ -83717,18 +83888,26 @@ async function enrichCodeBlock(block) {
   const raw2 = block.payload.raw ?? "";
   const { code: code4, info, hadFence } = stripCodeFence(raw2);
   const { lang: lang236, meta: meta2 } = parseCodeFenceInfo(info);
-  const diffInfo = detectDiffLanguage(lang236, meta2);
-  const requestedLanguage = diffInfo.isDiff ? diffInfo.diffLang : lang236 || "text";
-  const tokenLanguage = diffInfo.isDiff ? diffInfo.baseLang ?? "text" : requestedLanguage;
+  let diffInfo = detectDiffLanguage(lang236, meta2);
   const codeBody = hadFence ? code4 : dedentIndentedCode(raw2);
   const codeLines = extractCodeLines(raw2);
+  if (!diffInfo.isDiff && emitDiffBlocks && looksLikeUnifiedDiff(codeLines)) {
+    diffInfo = {
+      isDiff: true,
+      diffLang: "diff",
+      baseLang: guessLanguageFromDiffLines(codeLines)
+    };
+  }
+  const requestedLanguage = diffInfo.isDiff ? diffInfo.diffLang : lang236 || "text";
+  const tokenLanguage = diffInfo.isDiff ? diffInfo.baseLang ?? "text" : requestedLanguage;
   const baseMeta = block.payload.meta ?? {};
   let resolvedLanguage = requestedLanguage;
   let resolvedTokenLanguage = tokenLanguage;
   const hasHighlighter = Boolean(highlighter);
   const hasHighlightableContent = codeBody.trim().length > 0;
   const wantsHtml = highlightOutputMode === "html" || highlightOutputMode === "both";
-  const wantsTokens = highlightOutputMode === "tokens" || highlightOutputMode === "both";
+  const wantsTokens = (highlightOutputMode === "tokens" || highlightOutputMode === "both") && emitHighlightTokens;
+  const wantsTokensNow = wantsTokens && (block.isFinalized || liveTokenizationEnabled);
   if (!block.isFinalized) {
     if (codeHighlightingMode === "final" || !hasHighlighter || !hasHighlightableContent) {
       resetIncrementalHighlightState(block.id);
@@ -83749,6 +83928,9 @@ async function enrichCodeBlock(block) {
       if ("newNo" in nextMeta2) {
         delete nextMeta2.newNo;
       }
+      if ("diffBlocks" in nextMeta2) {
+        delete nextMeta2.diffBlocks;
+      }
       block.payload.meta = nextMeta2;
       return;
     }
@@ -83773,13 +83955,13 @@ async function enrichCodeBlock(block) {
       state.tokenLang = wantsTokens ? resolvedTokenLanguage : void 0;
       const appended = codeBody.slice(state.processedLength);
       const combined = state.pendingLine + appended;
-      const diffEnabled = diffInfo.isDiff && wantsTokens;
+      const diffEnabled = diffInfo.isDiff && wantsTokensNow;
       if (combined.length > 0) {
         const parts = combined.split("\n");
         const completeLines = parts.slice(0, -1);
         const tail = parts.length > 0 ? parts[parts.length - 1] ?? "" : "";
         if (completeLines.length > 0) {
-          if (wantsHtml || wantsTokens && !diffEnabled) {
+          if (wantsHtml || wantsTokensNow && !diffEnabled) {
             try {
               const sharedLanguage = wantsHtml ? resolvedLanguage : resolvedTokenLanguage;
               const tokens = highlighter?.codeToTokensWithThemes(completeLines.join("\n"), {
@@ -83791,7 +83973,7 @@ async function enrichCodeBlock(block) {
                 const htmlLines = renderShikiLines(tokens);
                 state.highlightedLines.push(...htmlLines);
               }
-              if (wantsTokens && !diffEnabled) {
+              if (wantsTokensNow && !diffEnabled) {
                 const tokenLines2 = renderTokenLines(tokens);
                 state.tokenLines.push(...tokenLines2);
               }
@@ -83801,12 +83983,12 @@ async function enrichCodeBlock(block) {
               if (wantsHtml) {
                 state.highlightedLines.push(...completeLines.map(() => null));
               }
-              if (wantsTokens && !diffEnabled) {
+              if (wantsTokensNow && !diffEnabled) {
                 state.tokenLines.push(...completeLines.map(() => null));
               }
             }
           }
-          if (wantsTokens && diffEnabled) {
+          if (wantsTokensNow && diffEnabled) {
             const cursor = state.diffCursor ?? { oldLine: null, newLine: null };
             const diffLines = completeLines.map((line) => parseUnifiedDiffLine(line, cursor));
             state.diffCursor = cursor;
@@ -83837,7 +84019,7 @@ async function enrichCodeBlock(block) {
       } else if ("highlightedLines" in nextMeta2) {
         delete nextMeta2.highlightedLines;
       }
-      if (wantsTokens) {
+      if (wantsTokensNow) {
         nextMeta2.tokenLines = state.tokenLines;
         if (diffEnabled) {
           nextMeta2.diffKind = state.diffKind;
@@ -83854,6 +84036,13 @@ async function enrichCodeBlock(block) {
         if ("oldNo" in nextMeta2) delete nextMeta2.oldNo;
         if ("newNo" in nextMeta2) delete nextMeta2.newNo;
       }
+      if (emitDiffBlocks && diffInfo.isDiff && liveTokenizationEnabled) {
+        const cursor = { oldLine: null, newLine: null };
+        const diffLines = codeLines.map((line) => parseUnifiedDiffLine(line, cursor));
+        nextMeta2.diffBlocks = buildDiffBlocksFromLines(diffLines, codeLines, wantsTokensNow ? state.tokenLines : null, diffInfo.baseLang ?? null);
+      } else if ("diffBlocks" in nextMeta2) {
+        delete nextMeta2.diffBlocks;
+      }
       block.payload.meta = nextMeta2;
       const highlightDuration2 = performanceTimer.measure("highlight-code");
       metrics?.recordShiki(highlightDuration2);
@@ -83900,6 +84089,13 @@ async function enrichCodeBlock(block) {
       if ("oldNo" in nextMeta2) delete nextMeta2.oldNo;
       if ("newNo" in nextMeta2) delete nextMeta2.newNo;
     }
+    if (emitDiffBlocks && diffInfo.isDiff) {
+      const cursor = { oldLine: null, newLine: null };
+      const diffLines = codeLines.map((line) => parseUnifiedDiffLine(line, cursor));
+      nextMeta2.diffBlocks = buildDiffBlocksFromLines(diffLines, codeLines, wantsTokens ? state.tokenLines : null, diffInfo.baseLang ?? null);
+    } else if ("diffBlocks" in nextMeta2) {
+      delete nextMeta2.diffBlocks;
+    }
     block.payload.highlightedHtml = void 0;
     block.payload.meta = nextMeta2;
     return;
@@ -84002,6 +84198,13 @@ async function enrichCodeBlock(block) {
     if ("oldNo" in nextMeta) delete nextMeta.oldNo;
     if ("newNo" in nextMeta) delete nextMeta.newNo;
   }
+  if (emitDiffBlocks && diffInfo.isDiff) {
+    const cursor = { oldLine: null, newLine: null };
+    const diffLines = codeLines.map((line) => parseUnifiedDiffLine(line, cursor));
+    nextMeta.diffBlocks = buildDiffBlocksFromLines(diffLines, codeLines, wantsTokens ? tokenLines ?? null : null, diffInfo.baseLang ?? null);
+  } else if ("diffBlocks" in nextMeta) {
+    delete nextMeta.diffBlocks;
+  }
   block.payload.meta = nextMeta;
   const highlightDuration = performanceTimer.measure("highlight-code");
   metrics?.recordShiki(highlightDuration);

package/dist/worker-client.d.cts

@@ -58,6 +58,9 @@ declare class MarkdownWorkerClient {
         codeHighlighting?: CodeHighlightingMode;
         outputMode?: CodeHighlightOutputMode;
         liveCodeHighlighting?: boolean;
+        liveTokenization?: boolean;
+        emitHighlightTokens?: boolean;
+        emitDiffBlocks?: boolean;
         mdxComponentNames?: string[];
         lazyTokenization?: {
             enabled?: boolean;

package/dist/worker-client.d.ts

@@ -58,6 +58,9 @@ declare class MarkdownWorkerClient {
         codeHighlighting?: CodeHighlightingMode;
         outputMode?: CodeHighlightOutputMode;
         liveCodeHighlighting?: boolean;
+        liveTokenization?: boolean;
+        emitHighlightTokens?: boolean;
+        emitDiffBlocks?: boolean;
         mdxComponentNames?: string[];
         lazyTokenization?: {
             enabled?: boolean;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stream-mdx/worker",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "Worker client utilities and shared worker helpers for the Streaming Markdown V2 pipeline",
   "license": "MIT",
   "repository": {
@@ -63,8 +63,8 @@
     "@lezer/common": "^1.2.3",
     "@lezer/lr": "^1.4.2",
     "@lezer/markdown": "^1.3.0",
-    "@stream-mdx/core": "0.2.0",
-    "@stream-mdx/plugins": "0.2.0",
+    "@stream-mdx/core": "0.3.0",
+    "@stream-mdx/plugins": "0.3.0",
     "@mdx-js/mdx": "^3.1.0",
     "@shikijs/engine-javascript": "^1.29.2",
     "@shikijs/engine-oniguruma": "^1.29.2",