@stream-mdx/worker 0.0.3 → 0.1.1

package/CHANGELOG.md

@@ -1,5 +1,26 @@
 # @stream-mdx/worker
 
+## 0.1.1
+
+### Patch Changes
+
+- Refine streaming scheduling and list layout, add worker append batching/debug state support, and refresh docs/README examples.
+- Updated dependencies
+  - @stream-mdx/core@0.1.1
+  - @stream-mdx/plugins@0.1.1
+
+## 0.1.0
+
+### Minor Changes
+
+- 294e557: Release StreamMDX 0.1.0 across all published packages.
+
+### Patch Changes
+
+- Updated dependencies [294e557]
+  - @stream-mdx/core@0.1.0
+  - @stream-mdx/plugins@0.1.0
+
 ## 0.0.3
 
 ### Patch Changes

package/dist/hosted/markdown-worker.js

@@ -30993,6 +30993,12 @@ var {
   createHighlighter
 );
 
+// ../../node_modules/rehype-parse/index.js
+var rehype_parse_exports = {};
+__export(rehype_parse_exports, {
+  default: () => rehypeParse
+});
+
 // ../../node_modules/devlop/lib/default.js
 function ok() {
 }
@@ -41377,6 +41383,12 @@ function rehypeSanitize(options) {
   };
 }
 
+// ../../node_modules/rehype-stringify/index.js
+var rehype_stringify_exports = {};
+__export(rehype_stringify_exports, {
+  default: () => rehypeStringify
+});
+
 // ../../node_modules/rehype-stringify/lib/index.js
 function rehypeStringify(options) {
   const self2 = this;
@@ -43371,6 +43383,36 @@ function filterAllowedAttributes(attrs) {
   }
   return filtered;
 }
+function ensureGlobal(pattern) {
+  if (pattern.global) return pattern;
+  const flags = pattern.flags.includes("g") ? pattern.flags : `${pattern.flags}g`;
+  return new RegExp(pattern.source, flags);
+}
+function findLastMatch(pattern, value) {
+  const re2 = ensureGlobal(pattern);
+  let match = null;
+  let next2;
+  while ((next2 = re2.exec(value)) !== null) {
+    match = next2;
+  }
+  return match;
+}
+function findMatchAfter(pattern, value, startIndex) {
+  const re2 = ensureGlobal(pattern);
+  re2.lastIndex = Math.max(0, startIndex);
+  return re2.exec(value);
+}
+function isSamePattern(a, b) {
+  return a.source === b.source && a.flags === b.flags;
+}
+function countMatches(pattern, value) {
+  const re2 = ensureGlobal(pattern);
+  let count = 0;
+  while (re2.exec(value)) {
+    count += 1;
+  }
+  return count;
+}
 var InlineParser = class {
   constructor(options = {}) {
     this.plugins = [];
@@ -43418,6 +43460,42 @@ var InlineParser = class {
     }
     return result;
   }
+  /**
+   * Streaming regex anticipation helper. Returns an append string if a plugin
+   * declares an incomplete match at the end of the buffer.
+   */
+  getRegexAnticipationAppend(content4) {
+    if (!content4 || this.plugins.length === 0) {
+      return null;
+    }
+    for (const plugin of this.plugins) {
+      if (!("re" in plugin)) continue;
+      const regexPlugin = plugin;
+      const anticipation = regexPlugin.anticipation;
+      if (!anticipation) continue;
+      const maxScanChars = Number.isFinite(anticipation.maxScanChars ?? Number.NaN) ? Math.max(1, anticipation.maxScanChars ?? 0) : 240;
+      const scan = content4.slice(Math.max(0, content4.length - maxScanChars));
+      if (regexPlugin.fastCheck && !regexPlugin.fastCheck(scan)) {
+        continue;
+      }
+      const lastStart = findLastMatch(anticipation.start, scan);
+      if (!lastStart) continue;
+      if (isSamePattern(anticipation.start, anticipation.end)) {
+        const occurrences = countMatches(anticipation.start, scan);
+        if (occurrences % 2 === 0) {
+          continue;
+        }
+      } else {
+        const startIndex = lastStart.index + lastStart[0].length;
+        const hasEnd = Boolean(findMatchAfter(anticipation.end, scan, startIndex));
+        if (hasEnd) continue;
+      }
+      const appendValue = typeof anticipation.append === "function" ? anticipation.append(lastStart, content4) : anticipation.append;
+      if (!appendValue) continue;
+      return appendValue;
+    }
+    return null;
+  }
   /**
    * Clear the memoization cache
    */
@@ -43546,9 +43624,16 @@ var InlineParser = class {
     this.registerPlugin({
       id: "citations",
       priority: 10,
-      re: /\[\^([^\]]+)\]|@cite\{([^}]+)\}/g,
-      toNode: (match) => ({ kind: "citation", id: match[1] || match[2] }),
-      fastCheck: (text12) => text12.indexOf("@") !== -1 || text12.indexOf("[^") !== -1
+      re: /\[\^([^\]\n]+)\]|@cite\{([^}\n]+)\}|\{cite:([^}\n]+)\}/g,
+      toNode: (match) => ({ kind: "citation", id: match[1] || match[2] || match[3] }),
+      fastCheck: (text12) => text12.indexOf("@cite") !== -1 || text12.indexOf("[^") !== -1 || text12.indexOf("{cite:") !== -1,
+      anticipation: {
+        start: /@cite\{|\{cite:/g,
+        end: /\}/g,
+        full: /@cite\{[^}\n]+?\}|\{cite:[^}\n]+?\}/g,
+        append: "}",
+        maxScanChars: 120
+      }
     });
     this.registerPlugin({
       id: "mentions",
@@ -43612,9 +43697,23 @@ function splitTextByRegexWithPrecedence(text12, regex3, toNode3) {
   }
   return result;
 }
-var { defaultSchema: defaultSchema2 } = rehype_sanitize_exports;
+var rehypeSanitizeModule = rehype_sanitize_exports;
+var defaultSchema2 = rehypeSanitizeModule.defaultSchema;
+var resolvePlugin = (mod) => {
+  if (typeof mod === "function") return mod;
+  if (mod && typeof mod.default === "function") {
+    return mod.default;
+  }
+  if (mod && typeof mod.default?.default === "function") {
+    return mod.default?.default;
+  }
+  return mod;
+};
+var rehypeParsePlugin = resolvePlugin(rehype_parse_exports);
+var rehypeSanitizePlugin = resolvePlugin(rehypeSanitizeModule);
+var rehypeStringifyPlugin = resolvePlugin(rehype_stringify_exports);
 var SANITIZED_SCHEMA = createSchema();
-var sanitizeProcessor = unified().use(rehypeParse, { fragment: true }).use(rehypeSanitize, SANITIZED_SCHEMA).use(rehypeStringify).freeze();
+var sanitizeProcessor = unified().use(rehypeParsePlugin, { fragment: true }).use(rehypeSanitizePlugin, SANITIZED_SCHEMA).use(rehypeStringifyPlugin).freeze();
 function sanitizeHtmlInWorker(html9) {
   if (!html9) return "";
   try {
@@ -43728,9 +43827,27 @@ function mergeAttributes(existing, additions) {
   }
   return next2;
 }
-function extractMixedContentSegments(raw2, baseOffset, parseInline) {
+var DEFAULT_INLINE_HTML_AUTOCLOSE_TAGS = /* @__PURE__ */ new Set([
+  "span",
+  "em",
+  "strong",
+  "code",
+  "kbd",
+  "del",
+  "s",
+  "mark",
+  "sub",
+  "sup",
+  "i",
+  "b",
+  "u",
+  "small",
+  "abbr",
+  "a"
+]);
+function extractMixedContentSegments(raw2, baseOffset, parseInline, options) {
   if (!raw2) return [];
-  const initial = splitByTagSegments(raw2, baseOffset, parseInline);
+  const initial = splitByTagSegments(raw2, baseOffset, parseInline, options);
   const expanded = [];
   for (const segment of initial) {
     if (segment.kind === "text") {
@@ -43741,22 +43858,58 @@ function extractMixedContentSegments(raw2, baseOffset, parseInline) {
   }
   return mergeAdjacentTextSegments(expanded, parseInline);
 }
-function splitByTagSegments(source, baseOffset, parseInline) {
+function splitByTagSegments(source, baseOffset, parseInline, options) {
   const segments = [];
   const lowerSource = source.toLowerCase();
   const tagPattern = /<([A-Za-z][\w:-]*)([^<>]*?)\/?>/g;
   let cursor = 0;
   let match = tagPattern.exec(source);
   const baseIsFinite = typeof baseOffset === "number" && Number.isFinite(baseOffset);
+  const htmlAllowTags = normalizeHtmlAllowlist(options?.html?.allowTags);
+  const htmlAutoClose = options?.html?.autoClose === true;
+  const htmlMaxNewlines = normalizeNewlineLimit(options?.html?.maxNewlines);
+  const mdxAutoClose = options?.mdx?.autoClose === true;
+  const mdxMaxNewlines = normalizeNewlineLimit(options?.mdx?.maxNewlines);
+  const mdxAllowlist = normalizeComponentAllowlist(options?.mdx?.componentAllowlist);
   while (match !== null) {
     const start2 = match.index;
     const tagName = match[1];
     const matchText = match[0];
-    const isSelfClosing = matchText.endsWith("/>") || isVoidHtmlTag(tagName);
+    const tagNameLower = tagName.toLowerCase();
+    const isSelfClosing = matchText.endsWith("/>") || isVoidHtmlTag(tagNameLower);
+    const mdxCandidate = isLikelyMdxComponent(tagName);
+    const mdxAllowed = mdxCandidate && (!mdxAllowlist || mdxAllowlist.has(tagName));
+    if (mdxCandidate && mdxAllowlist && !mdxAllowed) {
+      tagPattern.lastIndex = start2 + 1;
+      match = tagPattern.exec(source);
+      continue;
+    }
     let end = tagPattern.lastIndex;
-    if (!isSelfClosing && !isLikelyMdxComponent(tagName)) {
+    if (!isSelfClosing && !mdxAllowed) {
       const closingIndex = findClosingHtmlTag(lowerSource, tagName.toLowerCase(), end);
       if (closingIndex === -1) {
+        if (htmlAutoClose && htmlAllowTags.has(tagNameLower)) {
+          const tail = source.slice(end);
+          const newlineCount = countNewlines(tail, htmlMaxNewlines + 1);
+          if (newlineCount <= htmlMaxNewlines) {
+            if (start2 > cursor) {
+              const absoluteFrom = baseIsFinite ? baseOffset + cursor : void 0;
+              const absoluteTo = baseIsFinite ? baseOffset + start2 : void 0;
+              pushTextSegment(segments, source.slice(cursor, start2), absoluteFrom, absoluteTo, parseInline);
+            }
+            const rawSegment2 = source.slice(start2);
+            const closedValue = `${rawSegment2}</${tagName}>`;
+            const segment2 = {
+              kind: "html",
+              value: closedValue,
+              range: createSegmentRange(baseOffset, start2, source.length),
+              sanitized: sanitizeHtmlInWorker(closedValue)
+            };
+            segments.push(segment2);
+            cursor = source.length;
+            break;
+          }
+        }
         tagPattern.lastIndex = start2 + 1;
         match = tagPattern.exec(source);
         continue;
@@ -43768,8 +43921,8 @@ function splitByTagSegments(source, baseOffset, parseInline) {
       const absoluteTo = baseIsFinite ? baseOffset + start2 : void 0;
       pushTextSegment(segments, source.slice(cursor, start2), absoluteFrom, absoluteTo, parseInline);
     }
-    const rawSegment = source.slice(start2, end);
-    const kind = isLikelyMdxComponent(tagName) ? "mdx" : "html";
+    let rawSegment = source.slice(start2, end);
+    const kind = mdxAllowed ? "mdx" : "html";
     const segment = {
       kind,
       value: rawSegment,
@@ -43778,6 +43931,17 @@ function splitByTagSegments(source, baseOffset, parseInline) {
     if (kind === "html") {
       segment.sanitized = sanitizeHtmlInWorker(rawSegment);
     } else {
+      const tail = source.slice(end);
+      const newlineCount = countNewlines(tail, mdxMaxNewlines + 1);
+      if (mdxAutoClose && newlineCount > mdxMaxNewlines) {
+        tagPattern.lastIndex = start2 + 1;
+        match = tagPattern.exec(source);
+        continue;
+      }
+      if (mdxAutoClose && !rawSegment.endsWith("/>")) {
+        rawSegment = selfCloseTag(rawSegment);
+        segment.value = rawSegment;
+      }
       segment.status = "pending";
     }
     segments.push(segment);
@@ -43902,6 +44066,48 @@ var VOID_HTML_TAGS = /* @__PURE__ */ new Set(["br", "hr", "img", "meta", "input"
 function isVoidHtmlTag(tagName) {
   return VOID_HTML_TAGS.has(tagName.toLowerCase());
 }
+function normalizeNewlineLimit(value) {
+  if (!Number.isFinite(value ?? Number.NaN)) {
+    return 2;
+  }
+  return Math.max(0, value ?? 0);
+}
+function normalizeHtmlAllowlist(value) {
+  if (!value) return DEFAULT_INLINE_HTML_AUTOCLOSE_TAGS;
+  const tags2 = /* @__PURE__ */ new Set();
+  for (const tag of value) {
+    if (tag) {
+      tags2.add(tag.toLowerCase());
+    }
+  }
+  return tags2.size > 0 ? tags2 : DEFAULT_INLINE_HTML_AUTOCLOSE_TAGS;
+}
+function normalizeComponentAllowlist(value) {
+  if (!value) return null;
+  const tags2 = /* @__PURE__ */ new Set();
+  for (const tag of value) {
+    if (tag) tags2.add(tag);
+  }
+  return tags2.size > 0 ? tags2 : null;
+}
+function countNewlines(value, limit) {
+  let count = 0;
+  for (let i = 0; i < value.length; i++) {
+    if (value.charCodeAt(i) === 10) {
+      count += 1;
+      if (limit !== void 0 && count >= limit) {
+        return count;
+      }
+    }
+  }
+  return count;
+}
+function selfCloseTag(rawTag) {
+  if (rawTag.endsWith("/>")) return rawTag;
+  const closeIndex = rawTag.lastIndexOf(">");
+  if (closeIndex === -1) return rawTag;
+  return `${rawTag.slice(0, closeIndex)}/>`;
+}
 function isLikelyMdxComponent(tagName) {
   const first = tagName.charAt(0);
   return first.toUpperCase() === first && first.toLowerCase() !== first;
@@ -44686,63 +44892,126 @@ function computeHeavyPatchBudget(credit, config = DEFAULT_BACKPRESSURE_CONFIG) {
   return Math.min(config.maxHeavyPatchBudget, budget);
 }
 var USE_LINEAR_COALESCING = typeof process === "undefined" ? true : process.env.V2_USE_LINEAR_COALESCING !== "false" && true;
+var DEFAULT_FORMAT_ANTICIPATION = {
+  inline: false,
+  mathInline: false,
+  mathBlock: false,
+  html: false,
+  mdx: false,
+  regex: false
+};
+function normalizeFormatAnticipation(input) {
+  if (input === true) {
+    return { ...DEFAULT_FORMAT_ANTICIPATION, inline: true };
+  }
+  if (!input) {
+    return { ...DEFAULT_FORMAT_ANTICIPATION };
+  }
+  return {
+    inline: input.inline ?? false,
+    mathInline: input.mathInline ?? false,
+    mathBlock: input.mathBlock ?? false,
+    html: input.html ?? false,
+    mdx: input.mdx ?? false,
+    regex: input.regex ?? false
+  };
+}
 function prepareInlineStreamingContent(content4, options) {
-  const enableAnticipation = Boolean(options?.formatAnticipation);
   const enableMath2 = options?.math !== false;
-  let dollarCount = 0;
-  let backtickCount = 0;
-  let starCount = 0;
-  let doubleStarCount = 0;
-  let tildePairCount = 0;
+  const anticipation = normalizeFormatAnticipation(options?.formatAnticipation);
+  const enableInlineAnticipation = anticipation.inline;
+  const enableMathInlineAnticipation = anticipation.mathInline;
+  const enableMathBlockAnticipation = anticipation.mathBlock;
+  const stack = [];
+  const toggleToken = (token2) => {
+    const last = stack[stack.length - 1];
+    if (last === token2) {
+      stack.pop();
+    } else {
+      stack.push(token2);
+    }
+  };
+  let mathDisplayOpen = false;
+  let mathDisplayCrossedNewline = false;
   for (let i = 0; i < content4.length; i++) {
     const code4 = content4.charCodeAt(i);
-    if (code4 === 36) {
-      dollarCount += 1;
+    if (code4 === 10 || code4 === 13) {
+      if (mathDisplayOpen) {
+        mathDisplayCrossedNewline = true;
+      }
       continue;
     }
     if (code4 === 96) {
-      backtickCount += 1;
+      toggleToken("code");
+      continue;
+    }
+    if (code4 === 126 && i + 1 < content4.length && content4.charCodeAt(i + 1) === 126) {
+      toggleToken("strike");
+      i += 1;
       continue;
     }
     if (code4 === 42) {
       if (i + 1 < content4.length && content4.charCodeAt(i + 1) === 42) {
-        doubleStarCount += 1;
-        starCount += 2;
+        toggleToken("strong");
         i += 1;
       } else {
-        starCount += 1;
+        toggleToken("em");
       }
       continue;
     }
-    if (code4 === 126) {
-      if (i + 1 < content4.length && content4.charCodeAt(i + 1) === 126) {
-        tildePairCount += 1;
+    if (enableMath2 && code4 === 36) {
+      if (i + 1 < content4.length && content4.charCodeAt(i + 1) === 36) {
+        toggleToken("math-display");
+        if (mathDisplayOpen) {
+          mathDisplayOpen = false;
+          mathDisplayCrossedNewline = false;
+        } else {
+          mathDisplayOpen = true;
+          mathDisplayCrossedNewline = false;
+        }
         i += 1;
+      } else {
+        toggleToken("math-inline");
       }
     }
   }
-  const hasIncompleteMath = enableMath2 && dollarCount % 2 !== 0;
-  if (hasIncompleteMath) {
-    return { kind: "raw", status: "raw", reason: "incomplete-math" };
-  }
-  const hasIncompleteCode = backtickCount % 2 !== 0;
-  const hasIncompleteStrong = doubleStarCount % 2 !== 0;
-  const singleStarCount = starCount - doubleStarCount * 2;
-  const hasIncompleteEmphasis = singleStarCount % 2 !== 0;
-  const hasIncompleteStrike = tildePairCount % 2 !== 0;
-  const hasAnyIncomplete = hasIncompleteCode || hasIncompleteStrong || hasIncompleteEmphasis || hasIncompleteStrike;
-  if (!hasAnyIncomplete) {
-    return { kind: "parse", status: "complete", content: content4, appended: "" };
+  const hasIncompleteFormatting = stack.some((token2) => token2 === "code" || token2 === "strike" || token2 === "strong" || token2 === "em");
+  const hasIncompleteMathInline = stack.includes("math-inline");
+  const hasIncompleteMathDisplay = stack.includes("math-display");
+  const hasIncompleteMath = hasIncompleteMathInline || hasIncompleteMathDisplay;
+  if (enableMath2 && hasIncompleteMath) {
+    if (hasIncompleteMathInline && !enableMathInlineAnticipation) {
+      return { kind: "raw", status: "raw", reason: "incomplete-math" };
+    }
+    if (hasIncompleteMathDisplay && (!enableMathBlockAnticipation || mathDisplayCrossedNewline)) {
+      return { kind: "raw", status: "raw", reason: "incomplete-math" };
+    }
   }
-  if (!enableAnticipation) {
+  if (hasIncompleteFormatting && !enableInlineAnticipation) {
     return { kind: "raw", status: "raw", reason: "incomplete-formatting" };
   }
-  let appended = "";
-  if (hasIncompleteCode) appended += "`";
-  if (hasIncompleteStrike) appended += "~~";
-  if (hasIncompleteStrong && hasIncompleteEmphasis) appended += "***";
-  else if (hasIncompleteStrong) appended += "**";
-  else if (hasIncompleteEmphasis) appended += "*";
+  if (!hasIncompleteFormatting && !hasIncompleteMath) {
+    return { kind: "parse", status: "complete", content: content4, appended: "" };
+  }
+  const appendForToken = (token2) => {
+    switch (token2) {
+      case "code":
+        return "`";
+      case "strike":
+        return "~~";
+      case "strong":
+        return "**";
+      case "em":
+        return "*";
+      case "math-inline":
+        return "$";
+      case "math-display":
+        return "$$";
+      default:
+        return "";
+    }
+  };
+  const appended = stack.slice().reverse().map((token2) => appendForToken(token2)).join("");
   return { kind: "parse", status: "anticipated", content: content4 + appended, appended };
 }
 
@@ -81699,20 +81968,72 @@ var documentPluginState = {};
 var txCounter = 0;
 var workerGrammarEngine = "js";
 var workerCredits = 1;
+var lastDiffSummary = null;
+var lastStructuralDiffSummary = null;
+var maxEmittedBlockCount = 0;
+var maxDeferredQueueSize = 0;
+var maxStructuralBlockCount = 0;
+var lastHighCountNoStructural = null;
 var MAX_DEFERRED_PATCHES = 400;
 var deferredPatchQueue = [];
 var MAX_DEFERRED_FLUSH_PATCHES = 120;
 var CODE_HIGHLIGHT_CACHE = /* @__PURE__ */ new Map();
 var MAX_CODE_HIGHLIGHT_CACHE_ENTRIES = 200;
 var mdxCompileMode = "server";
-var enableFormatAnticipation = false;
+var formatAnticipationConfig = normalizeFormatAnticipation(false);
 var enableLiveCodeHighlighting = false;
 var enableMath = true;
+var mdxComponentAllowlist = null;
 var WORKER_MDX_CACHE = /* @__PURE__ */ new Map();
 var WORKER_MDX_INFLIGHT = /* @__PURE__ */ new Map();
 var MAX_WORKER_MDX_CACHE_ENTRIES = 128;
 var loggedMdxSkipCount = 0;
 var MAX_MDX_SKIP_LOGS = 20;
+var MIXED_CONTENT_AUTOCLOSE_NEWLINES = 2;
+var DEFAULT_APPEND_CHUNK_SIZE = 1400;
+var DEFAULT_APPEND_BATCH_MS = 12;
+var MIN_APPEND_CHUNK_SIZE = 256;
+var MAX_APPEND_CHUNK_SIZE = 12e3;
+var MIN_APPEND_BATCH_MS = 4;
+var MAX_APPEND_BATCH_MS = 50;
+var APPEND_NEWLINE_GRACE = 200;
+function clampInt(value, min, max) {
+  if (!Number.isFinite(value)) return min;
+  return Math.min(max, Math.max(min, Math.floor(value)));
+}
+function readNumericEnv(key2) {
+  try {
+    if (typeof process !== "undefined" && process.env && process.env[key2]) {
+      const parsed = Number(process.env[key2]);
+      if (Number.isFinite(parsed)) {
+        return parsed;
+      }
+    }
+  } catch {
+  }
+  return null;
+}
+var APPEND_CHUNK_SIZE = clampInt(
+  readNumericEnv("NEXT_PUBLIC_STREAMING_APPEND_CHUNK") ?? DEFAULT_APPEND_CHUNK_SIZE,
+  MIN_APPEND_CHUNK_SIZE,
+  MAX_APPEND_CHUNK_SIZE
+);
+var APPEND_BATCH_MS = clampInt(
+  readNumericEnv("NEXT_PUBLIC_STREAMING_APPEND_BATCH_MS") ?? DEFAULT_APPEND_BATCH_MS,
+  MIN_APPEND_BATCH_MS,
+  MAX_APPEND_BATCH_MS
+);
+function sliceAppendChunk(text12, maxChars) {
+  if (text12.length <= maxChars) {
+    return { chunk: text12, rest: "" };
+  }
+  const newlineIndex = text12.lastIndexOf("\n", maxChars);
+  const cut = newlineIndex > 0 && maxChars - newlineIndex <= APPEND_NEWLINE_GRACE ? newlineIndex + 1 : maxChars;
+  return { chunk: text12.slice(0, cut), rest: text12.slice(cut) };
+}
+function waitForNextTick() {
+  return new Promise((resolve) => setTimeout(resolve, 0));
+}
 function isDebugEnabled(flag) {
   try {
     if (typeof process !== "undefined" && process.env) {
@@ -81731,6 +82052,8 @@ function isDebugEnabled(flag) {
 }
 var DEBUG_MDX = isDebugEnabled("mdx");
 var sharedTextEncoder = typeof TextEncoder !== "undefined" ? new TextEncoder() : null;
+var shouldTrackInlineStatus = () => formatAnticipationConfig.inline || formatAnticipationConfig.mathInline || formatAnticipationConfig.mathBlock;
+var shouldAllowMixedStreaming = () => formatAnticipationConfig.html || formatAnticipationConfig.mdx;
 function now() {
   return typeof performance !== "undefined" && typeof performance.now === "function" ? performance.now() : Date.now();
 }
@@ -81918,7 +82241,23 @@ function partitionPatchesForCredits(patches, maxImmediate) {
   const immediate = [];
   const deferred = [];
   let heavyBudget = computeHeavyPatchBudget(workerCredits);
+  let nonStructuralImmediate = 0;
+  const isStructuralPatch = (patch5) => {
+    switch (patch5.op) {
+      case "insertChild":
+      case "deleteChild":
+      case "replaceChild":
+      case "reorder":
+        return true;
+      default:
+        return false;
+    }
+  };
   for (const patch5 of combined) {
+    if (isStructuralPatch(patch5)) {
+      immediate.push(patch5);
+      continue;
+    }
     const heavy = isHeavyPatch(patch5);
     if (heavy) {
       if (heavyBudget <= 0) {
@@ -81930,14 +82269,14 @@ function partitionPatchesForCredits(patches, maxImmediate) {
         heavyBudget -= 1;
       }
     }
+    if (typeof maxImmediate === "number" && nonStructuralImmediate >= maxImmediate) {
+      deferred.push(patch5);
+      continue;
+    }
     immediate.push(patch5);
+    nonStructuralImmediate += 1;
   }
-  if (typeof maxImmediate === "number" && immediate.length > maxImmediate) {
-    const overflow = immediate.splice(maxImmediate);
-    deferredPatchQueue = overflow.concat(deferred);
-  } else {
-    deferredPatchQueue = deferred;
-  }
+  deferredPatchQueue = deferred;
   return immediate;
 }
 function flushDeferredPatches() {
@@ -81998,13 +82337,23 @@ function mapToHighlightRecord(map3) {
   return result;
 }
 function parseInlineStreaming(content4) {
-  const prepared = prepareInlineStreamingContent(content4, { formatAnticipation: enableFormatAnticipation, math: enableMath });
+  const prepared = prepareInlineStreamingContent(content4, { formatAnticipation: formatAnticipationConfig, math: enableMath });
   if (prepared.kind === "raw") {
     return { inline: [{ kind: "text", text: content4 }], status: prepared.status };
   }
+  let preparedContent = prepared.content;
+  let appended = prepared.appended;
+  if (formatAnticipationConfig.regex) {
+    const regexAppend = inlineParser.getRegexAnticipationAppend(content4);
+    if (regexAppend) {
+      preparedContent += regexAppend;
+      appended += regexAppend;
+    }
+  }
+  const status = appended.length > 0 ? "anticipated" : prepared.status;
   return {
-    inline: inlineParser.parse(prepared.content, { cache: false }),
-    status: prepared.status
+    inline: inlineParser.parse(preparedContent, { cache: false }),
+    status
   };
 }
 async function initialize(initialContent = "", prewarmLangs = [], docPlugins, mdxOptions) {
@@ -82032,8 +82381,13 @@ async function initialize(initialContent = "", prewarmLangs = [], docPlugins, md
     liveCodeHighlighting: docPlugins?.liveCodeHighlighting ?? false
   };
   enableMath = enable.math;
-  enableFormatAnticipation = enable.formatAnticipation;
+  formatAnticipationConfig = normalizeFormatAnticipation(enable.formatAnticipation);
   enableLiveCodeHighlighting = enable.liveCodeHighlighting;
+  if (Array.isArray(docPlugins?.mdxComponentNames) && docPlugins?.mdxComponentNames.length > 0) {
+    mdxComponentAllowlist = new Set(docPlugins.mdxComponentNames);
+  } else {
+    mdxComponentAllowlist = null;
+  }
   inlineParser = new InlineParser({ enableMath });
   documentPluginState.inlineParser = inlineParser;
   if (enable.footnotes) registerFootnotesPlugin();
@@ -82066,25 +82420,35 @@ async function initialize(initialContent = "", prewarmLangs = [], docPlugins, md
   }
 }
 async function handleAppend(text12) {
-  performanceTimer.mark("append-operation");
-  const metricsCollector = new WorkerMetricsCollector(workerGrammarEngine);
-  setActiveMetricsCollector(metricsCollector);
-  await appendAndReparse(text12, metricsCollector);
-  const hadPatchMetrics = metricsCollector.patchCount > 0;
-  const totalTime = performanceTimer.measure("append-operation");
-  if (getActiveMetricsCollector() === metricsCollector) {
-    setActiveMetricsCollector(null);
-  }
-  if (!hadPatchMetrics && totalTime !== null && Number.isFinite(totalTime)) {
-    postMessage({
-      type: "METRICS",
-      metrics: {
-        parseMs: roundMetric(totalTime),
-        parseTime: roundMetric(totalTime),
-        blocksProduced: blocks.length,
-        grammarEngine: workerGrammarEngine
-      }
-    });
+  let remaining = text12;
+  let batchStartedAt = now();
+  while (remaining.length > 0) {
+    const { chunk, rest } = sliceAppendChunk(remaining, APPEND_CHUNK_SIZE);
+    remaining = rest;
+    performanceTimer.mark("append-operation");
+    const metricsCollector = new WorkerMetricsCollector(workerGrammarEngine);
+    setActiveMetricsCollector(metricsCollector);
+    await appendAndReparse(chunk, metricsCollector);
+    const hadPatchMetrics = metricsCollector.patchCount > 0;
+    const totalTime = performanceTimer.measure("append-operation");
+    if (getActiveMetricsCollector() === metricsCollector) {
+      setActiveMetricsCollector(null);
+    }
+    if (!hadPatchMetrics && totalTime !== null && Number.isFinite(totalTime)) {
+      postMessage({
+        type: "METRICS",
+        metrics: {
+          parseMs: roundMetric(totalTime),
+          parseTime: roundMetric(totalTime),
+          blocksProduced: blocks.length,
+          grammarEngine: workerGrammarEngine
+        }
+      });
+    }
+    if (remaining.length > 0 && now() - batchStartedAt >= APPEND_BATCH_MS) {
+      await waitForNextTick();
+      batchStartedAt = now();
+    }
   }
 }
 async function parseAll(content4, options = {}) {
@@ -82107,9 +82471,24 @@ async function appendAndReparse(appendedText, metrics) {
   metrics?.markParseStart();
   const newContent = currentContent + appendedText;
   const changeRanges = computeChangedRanges(currentContent, newContent);
-  const newTree = lastTree ? parser.parse(newContent, lastTree.fragments) : parser.parse(newContent);
+  let newTree = lastTree ? parser.parse(newContent, lastTree.fragments) : parser.parse(newContent);
   let changedBlocks = await extractBlocks(newTree, newContent);
   changedBlocks = runDocumentPlugins(changedBlocks, newContent);
+  const lastRange = changedBlocks.length > 0 ? changedBlocks[changedBlocks.length - 1]?.payload?.range : void 0;
+  const lastTo = typeof lastRange?.to === "number" ? lastRange.to : 0;
+  if (lastTo < newContent.length - 1) {
+    const tail = newContent.slice(lastTo).trim();
+    if (tail.length > 0) {
+      const fullTree = parser.parse(newContent);
+      let fullBlocks = await extractBlocks(fullTree, newContent);
+      fullBlocks = runDocumentPlugins(fullBlocks, newContent);
+      const fullLast = fullBlocks.length > 0 ? fullBlocks[fullBlocks.length - 1]?.payload?.range?.to ?? 0 : 0;
+      if (fullLast >= newContent.length - 1 || fullBlocks.length >= changedBlocks.length) {
+        newTree = fullTree;
+        changedBlocks = fullBlocks;
+      }
+    }
+  }
   const prevBlocks = blocks;
   blocks = changedBlocks;
   lastTree = newTree;
@@ -82221,13 +82600,13 @@ async function enrichBlock(block) {
       if (block.isFinalized) {
         block.payload.raw = normalizedHeading;
         block.payload.inline = inlineParser.parse(normalizedHeading);
-        if (enableFormatAnticipation && block.payload.meta) {
+        if (shouldTrackInlineStatus() && block.payload.meta) {
           block.payload.meta.inlineStatus = void 0;
         }
       } else {
         const parsed = parseInlineStreaming(normalizedHeading);
         block.payload.inline = parsed.inline;
-        if (enableFormatAnticipation && block.payload.meta) {
+        if (shouldTrackInlineStatus() && block.payload.meta) {
           block.payload.meta.inlineStatus = parsed.status;
         }
       }
@@ -82238,7 +82617,11 @@ async function enrichBlock(block) {
       block.payload.raw = normalized;
       const streamingParsed = !block.isFinalized ? parseInlineStreaming(normalized) : null;
       const inlineParse = block.isFinalized ? (value) => inlineParser.parse(value) : (value) => parseInlineStreaming(value).inline;
-      const segments = extractMixedContentSegments(normalized, void 0, inlineParse);
+      const mixedOptions = !block.isFinalized && shouldAllowMixedStreaming() ? {
+        html: formatAnticipationConfig.html ? { autoClose: true, maxNewlines: MIXED_CONTENT_AUTOCLOSE_NEWLINES } : void 0,
+        mdx: formatAnticipationConfig.mdx ? { autoClose: true, maxNewlines: MIXED_CONTENT_AUTOCLOSE_NEWLINES, componentAllowlist: mdxComponentAllowlist ?? void 0 } : void 0
+      } : void 0;
+      const segments = extractMixedContentSegments(normalized, void 0, inlineParse, mixedOptions);
       const currentMeta = block.payload.meta ?? {};
       const nextMeta = {
         ...currentMeta,
@@ -82249,7 +82632,7 @@ async function enrichBlock(block) {
       } else {
         nextMeta.mixedSegments = void 0;
       }
-      if (enableFormatAnticipation) {
+      if (shouldTrackInlineStatus()) {
         nextMeta.inlineStatus = block.isFinalized ? void 0 : streamingParsed?.status;
       }
       if (Object.keys(nextMeta).length > 0) {
@@ -82272,7 +82655,8 @@ async function enrichBlock(block) {
       const currentMeta = block.payload.meta ?? {};
       const nextMeta = { ...currentMeta };
       let metaChanged = false;
-      if (enableFormatAnticipation) {
+      const allowMixedStreaming = shouldAllowMixedStreaming();
+      if (shouldTrackInlineStatus()) {
         const desired = block.isFinalized ? void 0 : streamingParsed?.status;
         if (desired !== void 0) {
           if (!Object.prototype.hasOwnProperty.call(nextMeta, "inlineStatus") || nextMeta.inlineStatus !== desired) {
@@ -82298,17 +82682,36 @@ async function enrichBlock(block) {
       const shouldExtractSegments = typeof rawParagraph === "string" && (rawParagraph.includes("<") || rawParagraph.includes("{"));
       if (shouldExtractSegments) {
         const baseOffset = typeof block.payload.range?.from === "number" ? block.payload.range.from : void 0;
-        const segments = extractMixedContentSegments(rawParagraph, baseOffset, (value) => inlineParse(value));
+        const mixedOptions = !block.isFinalized && shouldAllowMixedStreaming() ? {
+          html: formatAnticipationConfig.html ? { autoClose: true, maxNewlines: MIXED_CONTENT_AUTOCLOSE_NEWLINES } : void 0,
+          mdx: formatAnticipationConfig.mdx ? { autoClose: true, maxNewlines: MIXED_CONTENT_AUTOCLOSE_NEWLINES, componentAllowlist: mdxComponentAllowlist ?? void 0 } : void 0
+        } : void 0;
+        const segments = extractMixedContentSegments(rawParagraph, baseOffset, (value) => inlineParse(value), mixedOptions);
         if (segments.length > 0) {
           nextMeta.mixedSegments = segments;
           metaChanged = true;
+          if (allowMixedStreaming) {
+            nextMeta.allowMixedStreaming = true;
+            metaChanged = true;
+          } else if (Object.prototype.hasOwnProperty.call(nextMeta, "allowMixedStreaming")) {
+            nextMeta.allowMixedStreaming = void 0;
+            metaChanged = true;
+          }
         } else if (Object.prototype.hasOwnProperty.call(nextMeta, "mixedSegments")) {
           nextMeta.mixedSegments = void 0;
           metaChanged = true;
+          if (Object.prototype.hasOwnProperty.call(nextMeta, "allowMixedStreaming")) {
+            nextMeta.allowMixedStreaming = void 0;
+            metaChanged = true;
+          }
         }
       } else if (Object.prototype.hasOwnProperty.call(nextMeta, "mixedSegments")) {
         nextMeta.mixedSegments = void 0;
         metaChanged = true;
+        if (Object.prototype.hasOwnProperty.call(nextMeta, "allowMixedStreaming")) {
+          nextMeta.allowMixedStreaming = void 0;
+          metaChanged = true;
+        }
       }
       if (metaChanged) {
         if (Object.keys(nextMeta).length > 0) {
@@ -82906,6 +83309,50 @@ async function emitBlockDiffPatches(previousBlocks, nextBlocks, changedRanges, m
     });
   }
   const immediatePatches = partitionPatchesForCredits(combined, paragraphLimit === null ? void 0 : paragraphLimit);
+  if (deferredPatchQueue.length > maxDeferredQueueSize) {
+    maxDeferredQueueSize = deferredPatchQueue.length;
+  }
+  let structuralCount = 0;
+  for (const patch5 of patches) {
+    if (patch5.op === "insertChild" || patch5.op === "deleteChild" || patch5.op === "replaceChild" || patch5.op === "reorder") {
+      structuralCount += 1;
+    }
+  }
+  lastDiffSummary = {
+    prevCount: previousBlocks.length,
+    nextCount: nextBlocks.length,
+    prefix,
+    removeCount,
+    addCount,
+    structuralPatches: structuralCount,
+    contentPatches: contentPatches.length,
+    immediatePatches: immediatePatches.length,
+    deferredQueue: deferredPatchQueue.length
+  };
+  if (addCount > 0 || removeCount > 0 || structuralCount > 0 || deferredPatchQueue.length > 0) {
+    lastStructuralDiffSummary = { ...lastDiffSummary };
+  }
+  if (immediatePatches.length > 0) {
+    maxEmittedBlockCount = Math.max(maxEmittedBlockCount, nextBlocks.length);
+  }
+  if (structuralCount > 0) {
+    maxStructuralBlockCount = Math.max(maxStructuralBlockCount, nextBlocks.length);
+  }
+  if (nextBlocks.length >= 60 && removeCount === 0 && addCount === 0) {
+    const sliceStart = Math.max(0, Math.min(55, nextBlocks.length - 1));
+    const sliceEnd = Math.min(nextBlocks.length, sliceStart + 8);
+    const toPreview = (block) => ({
+      id: block.id,
+      type: block.type,
+      raw: typeof block.payload.raw === "string" ? block.payload.raw.slice(0, 80) : ""
+    });
+    lastHighCountNoStructural = {
+      nextCount: nextBlocks.length,
+      sliceStart,
+      prevSlice: previousBlocks.slice(sliceStart, sliceEnd).map(toPreview),
+      nextSlice: nextBlocks.slice(sliceStart, sliceEnd).map(toPreview)
+    };
+  }
   if (immediatePatches.length === 0) {
     if (metrics) {
       metrics.finalizePatch(txCounter, 0, deferredPatchQueue.length, 0);
@@ -83414,6 +83861,19 @@ async function finalizeAllBlocks() {
       dispatchPatchBatch(finalizeSetProps, metricsCollector);
     }
   }
+  if (deferredPatchQueue.length > 0) {
+    const previousCredits = workerCredits;
+    workerCredits = 1;
+    const maxIterations = Math.ceil(deferredPatchQueue.length / MAX_DEFERRED_FLUSH_PATCHES) + 8;
+    for (let i = 0; i < maxIterations && deferredPatchQueue.length > 0; i++) {
+      const before = deferredPatchQueue.length;
+      flushDeferredPatches();
+      if (deferredPatchQueue.length >= before) {
+        break;
+      }
+    }
+    workerCredits = previousCredits;
+  }
   if (getActiveMetricsCollector() === metricsCollector) {
     setActiveMetricsCollector(null);
   }
@@ -83461,6 +83921,104 @@ async function processWorkerMessage(msg) {
     case "FINALIZE":
       await finalizeAllBlocks();
       return;
+    case "DEBUG_STATE": {
+      const blockTypeCounts = {};
+      for (const block of blocks) {
+        const key2 = block.type ?? "unknown";
+        blockTypeCounts[key2] = (blockTypeCounts[key2] || 0) + 1;
+      }
+      let lastBlockType;
+      let lastBlockRange;
+      let lastBlockRawTail;
+      const headingTexts = [];
+      const tailBlocks = [];
+      const headBlocks = [];
+      const duplicateBlockIds = [];
+      const seenBlockIds = /* @__PURE__ */ new Set();
+      const headingIndices = {};
+      for (const block of blocks) {
+        if (seenBlockIds.has(block.id)) {
+          if (duplicateBlockIds.length < 8) {
+            duplicateBlockIds.push(block.id);
+          }
+        } else {
+          seenBlockIds.add(block.id);
+        }
+      }
+      if (blocks.length > 0) {
+        const lastBlock = blocks[blocks.length - 1];
+        lastBlockType = lastBlock.type;
+        const range2 = lastBlock.payload.range;
+        if (range2 && typeof range2.from === "number" && typeof range2.to === "number") {
+          lastBlockRange = { from: range2.from, to: range2.to };
+        }
+        const raw2 = typeof lastBlock.payload.raw === "string" ? lastBlock.payload.raw : "";
+        lastBlockRawTail = raw2 ? raw2.slice(Math.max(0, raw2.length - 240)) : void 0;
+        for (const block of blocks) {
+          if (block.type === "heading" && typeof block.payload.raw === "string") {
+            headingTexts.push(block.payload.raw);
+          }
+        }
+        const targets = ["HTML and MDX Testing", "Inline Code", "Code Blocks", "Media", "Tables", "Footnotes"];
+        for (let i = 0; i < blocks.length; i++) {
+          const block = blocks[i];
+          if (block.type === "heading" && typeof block.payload.raw === "string") {
+            if (targets.includes(block.payload.raw)) {
+              headingIndices[block.payload.raw] = i;
+            }
+          }
+        }
+        const tailStart = Math.max(0, blocks.length - 8);
+        for (let i = tailStart; i < blocks.length; i++) {
+          const block = blocks[i];
+          const raw3 = typeof block.payload.raw === "string" ? block.payload.raw : "";
+          tailBlocks.push({
+            id: block.id,
+            type: block.type,
+            raw: raw3.slice(0, 120)
+          });
+        }
+        const headEnd = Math.min(8, blocks.length);
+        for (let i = 0; i < headEnd; i++) {
+          const block = blocks[i];
+          const raw3 = typeof block.payload.raw === "string" ? block.payload.raw : "";
+          headBlocks.push({
+            id: block.id,
+            type: block.type,
+            raw: raw3.slice(0, 120)
+          });
+        }
+      }
+      const contentTail = currentContent.slice(Math.max(0, currentContent.length - 500));
+      postMessage({
+        type: "DEBUG_STATE",
+        state: {
+          contentLength: currentContent.length,
+          contentTail,
+          blockCount: blocks.length,
+          blockTypeCounts,
+          lastBlockType,
+          lastBlockRange,
+          lastBlockRawTail,
+          headingTexts,
+          headBlocks,
+          tailBlocks,
+          headingIndices,
+          lastDiffSummary,
+          lastStructuralDiffSummary,
+          maxEmittedBlockCount,
+          maxDeferredQueueSize,
+          maxStructuralBlockCount,
+          lastHighCountNoStructural,
+          duplicateBlockIds,
+          duplicateBlockCount: duplicateBlockIds.length,
+          hasInlineCodeHeading: currentContent.includes("# Inline Code"),
+          hasCodeBlocksHeading: currentContent.includes("# Code Blocks"),
+          hasMediaHeading: currentContent.includes("# Media")
+        }
+      });
+      return;
+    }
     case "MDX_COMPILED":
       handleMdxStatus(msg.blockId, {
         compiledRef: { id: msg.compiledId },

package/dist/worker-client.d.cts

@@ -1,4 +1,4 @@
-import { WorkerOut } from '@stream-mdx/core';
+import { WorkerOut, FormatAnticipationConfig } from '@stream-mdx/core';
 
 type DefaultWorkerMode = "auto" | "hosted" | "blob";
 interface CreateDefaultWorkerOptions {
@@ -54,8 +54,9 @@ declare class MarkdownWorkerClient {
         tables?: boolean;
         callouts?: boolean;
         math?: boolean;
-        formatAnticipation?: boolean;
+        formatAnticipation?: FormatAnticipationConfig;
         liveCodeHighlighting?: boolean;
+        mdxComponentNames?: string[];
     }, mdxOptions?: {
         compileMode?: "server" | "worker";
     }): void;

package/dist/worker-client.d.ts

@@ -1,4 +1,4 @@
-import { WorkerOut } from '@stream-mdx/core';
+import { WorkerOut, FormatAnticipationConfig } from '@stream-mdx/core';
 
 type DefaultWorkerMode = "auto" | "hosted" | "blob";
 interface CreateDefaultWorkerOptions {
@@ -54,8 +54,9 @@ declare class MarkdownWorkerClient {
         tables?: boolean;
         callouts?: boolean;
         math?: boolean;
-        formatAnticipation?: boolean;
+        formatAnticipation?: FormatAnticipationConfig;
         liveCodeHighlighting?: boolean;
+        mdxComponentNames?: string[];
     }, mdxOptions?: {
         compileMode?: "server" | "worker";
     }): void;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stream-mdx/worker",
-  "version": "0.0.3",
+  "version": "0.1.1",
   "description": "Worker client utilities and shared worker helpers for the Streaming Markdown V2 pipeline",
   "license": "MIT",
   "repository": {
@@ -55,7 +55,7 @@
   "scripts": {
     "build": "tsup",
     "build:hosted": "tsup --config tsup.hosted-worker.config.ts",
-    "test": "tsx __tests__/*.test.ts",
+    "test": "tsx ../../scripts/run-tests.ts __tests__",
     "clean": "rm -rf dist",
     "prepack": "npm run build && npm run build:hosted"
   },
@@ -63,8 +63,8 @@
     "@lezer/common": "^1.2.3",
     "@lezer/lr": "^1.4.2",
     "@lezer/markdown": "^1.3.0",
-    "@stream-mdx/core": "0.0.3",
-    "@stream-mdx/plugins": "0.0.3",
+    "@stream-mdx/core": "0.1.1",
+    "@stream-mdx/plugins": "0.1.1",
     "@mdx-js/mdx": "^3.1.0",
     "@shikijs/engine-javascript": "^1.29.2",
     "@shikijs/engine-oniguruma": "^1.29.2",