@stream-mdx/worker 0.1.0 → 0.1.1

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # @stream-mdx/worker
 
+## 0.1.1
+
+### Patch Changes
+
+- Refine streaming scheduling and list layout, add worker append batching/debug state support, and refresh docs/README examples.
+- Updated dependencies
+  - @stream-mdx/core@0.1.1
+  - @stream-mdx/plugins@0.1.1
+
 ## 0.1.0
 
 ### Minor Changes

package/dist/hosted/markdown-worker.js

@@ -81968,6 +81968,12 @@ var documentPluginState = {};
 var txCounter = 0;
 var workerGrammarEngine = "js";
 var workerCredits = 1;
+var lastDiffSummary = null;
+var lastStructuralDiffSummary = null;
+var maxEmittedBlockCount = 0;
+var maxDeferredQueueSize = 0;
+var maxStructuralBlockCount = 0;
+var lastHighCountNoStructural = null;
 var MAX_DEFERRED_PATCHES = 400;
 var deferredPatchQueue = [];
 var MAX_DEFERRED_FLUSH_PATCHES = 120;
@@ -81984,6 +81990,50 @@ var MAX_WORKER_MDX_CACHE_ENTRIES = 128;
 var loggedMdxSkipCount = 0;
 var MAX_MDX_SKIP_LOGS = 20;
 var MIXED_CONTENT_AUTOCLOSE_NEWLINES = 2;
+var DEFAULT_APPEND_CHUNK_SIZE = 1400;
+var DEFAULT_APPEND_BATCH_MS = 12;
+var MIN_APPEND_CHUNK_SIZE = 256;
+var MAX_APPEND_CHUNK_SIZE = 12e3;
+var MIN_APPEND_BATCH_MS = 4;
+var MAX_APPEND_BATCH_MS = 50;
+var APPEND_NEWLINE_GRACE = 200;
+function clampInt(value, min, max) {
+  if (!Number.isFinite(value)) return min;
+  return Math.min(max, Math.max(min, Math.floor(value)));
+}
+function readNumericEnv(key2) {
+  try {
+    if (typeof process !== "undefined" && process.env && process.env[key2]) {
+      const parsed = Number(process.env[key2]);
+      if (Number.isFinite(parsed)) {
+        return parsed;
+      }
+    }
+  } catch {
+  }
+  return null;
+}
+var APPEND_CHUNK_SIZE = clampInt(
+  readNumericEnv("NEXT_PUBLIC_STREAMING_APPEND_CHUNK") ?? DEFAULT_APPEND_CHUNK_SIZE,
+  MIN_APPEND_CHUNK_SIZE,
+  MAX_APPEND_CHUNK_SIZE
+);
+var APPEND_BATCH_MS = clampInt(
+  readNumericEnv("NEXT_PUBLIC_STREAMING_APPEND_BATCH_MS") ?? DEFAULT_APPEND_BATCH_MS,
+  MIN_APPEND_BATCH_MS,
+  MAX_APPEND_BATCH_MS
+);
+function sliceAppendChunk(text12, maxChars) {
+  if (text12.length <= maxChars) {
+    return { chunk: text12, rest: "" };
+  }
+  const newlineIndex = text12.lastIndexOf("\n", maxChars);
+  const cut = newlineIndex > 0 && maxChars - newlineIndex <= APPEND_NEWLINE_GRACE ? newlineIndex + 1 : maxChars;
+  return { chunk: text12.slice(0, cut), rest: text12.slice(cut) };
+}
+function waitForNextTick() {
+  return new Promise((resolve) => setTimeout(resolve, 0));
+}
 function isDebugEnabled(flag) {
   try {
     if (typeof process !== "undefined" && process.env) {
@@ -82191,7 +82241,23 @@ function partitionPatchesForCredits(patches, maxImmediate) {
   const immediate = [];
   const deferred = [];
   let heavyBudget = computeHeavyPatchBudget(workerCredits);
+  let nonStructuralImmediate = 0;
+  const isStructuralPatch = (patch5) => {
+    switch (patch5.op) {
+      case "insertChild":
+      case "deleteChild":
+      case "replaceChild":
+      case "reorder":
+        return true;
+      default:
+        return false;
+    }
+  };
   for (const patch5 of combined) {
+    if (isStructuralPatch(patch5)) {
+      immediate.push(patch5);
+      continue;
+    }
     const heavy = isHeavyPatch(patch5);
     if (heavy) {
       if (heavyBudget <= 0) {
@@ -82203,14 +82269,14 @@ function partitionPatchesForCredits(patches, maxImmediate) {
         heavyBudget -= 1;
       }
     }
+    if (typeof maxImmediate === "number" && nonStructuralImmediate >= maxImmediate) {
+      deferred.push(patch5);
+      continue;
+    }
     immediate.push(patch5);
+    nonStructuralImmediate += 1;
   }
-  if (typeof maxImmediate === "number" && immediate.length > maxImmediate) {
-    const overflow = immediate.splice(maxImmediate);
-    deferredPatchQueue = overflow.concat(deferred);
-  } else {
-    deferredPatchQueue = deferred;
-  }
+  deferredPatchQueue = deferred;
   return immediate;
 }
 function flushDeferredPatches() {
@@ -82354,25 +82420,35 @@ async function initialize(initialContent = "", prewarmLangs = [], docPlugins, md
   }
 }
 async function handleAppend(text12) {
-  performanceTimer.mark("append-operation");
-  const metricsCollector = new WorkerMetricsCollector(workerGrammarEngine);
-  setActiveMetricsCollector(metricsCollector);
-  await appendAndReparse(text12, metricsCollector);
-  const hadPatchMetrics = metricsCollector.patchCount > 0;
-  const totalTime = performanceTimer.measure("append-operation");
-  if (getActiveMetricsCollector() === metricsCollector) {
-    setActiveMetricsCollector(null);
-  }
-  if (!hadPatchMetrics && totalTime !== null && Number.isFinite(totalTime)) {
-    postMessage({
-      type: "METRICS",
-      metrics: {
-        parseMs: roundMetric(totalTime),
-        parseTime: roundMetric(totalTime),
-        blocksProduced: blocks.length,
-        grammarEngine: workerGrammarEngine
-      }
-    });
+  let remaining = text12;
+  let batchStartedAt = now();
+  while (remaining.length > 0) {
+    const { chunk, rest } = sliceAppendChunk(remaining, APPEND_CHUNK_SIZE);
+    remaining = rest;
+    performanceTimer.mark("append-operation");
+    const metricsCollector = new WorkerMetricsCollector(workerGrammarEngine);
+    setActiveMetricsCollector(metricsCollector);
+    await appendAndReparse(chunk, metricsCollector);
+    const hadPatchMetrics = metricsCollector.patchCount > 0;
+    const totalTime = performanceTimer.measure("append-operation");
+    if (getActiveMetricsCollector() === metricsCollector) {
+      setActiveMetricsCollector(null);
+    }
+    if (!hadPatchMetrics && totalTime !== null && Number.isFinite(totalTime)) {
+      postMessage({
+        type: "METRICS",
+        metrics: {
+          parseMs: roundMetric(totalTime),
+          parseTime: roundMetric(totalTime),
+          blocksProduced: blocks.length,
+          grammarEngine: workerGrammarEngine
+        }
+      });
+    }
+    if (remaining.length > 0 && now() - batchStartedAt >= APPEND_BATCH_MS) {
+      await waitForNextTick();
+      batchStartedAt = now();
+    }
   }
 }
 async function parseAll(content4, options = {}) {
@@ -82395,9 +82471,24 @@ async function appendAndReparse(appendedText, metrics) {
   metrics?.markParseStart();
   const newContent = currentContent + appendedText;
   const changeRanges = computeChangedRanges(currentContent, newContent);
-  const newTree = lastTree ? parser.parse(newContent, lastTree.fragments) : parser.parse(newContent);
+  let newTree = lastTree ? parser.parse(newContent, lastTree.fragments) : parser.parse(newContent);
   let changedBlocks = await extractBlocks(newTree, newContent);
   changedBlocks = runDocumentPlugins(changedBlocks, newContent);
+  const lastRange = changedBlocks.length > 0 ? changedBlocks[changedBlocks.length - 1]?.payload?.range : void 0;
+  const lastTo = typeof lastRange?.to === "number" ? lastRange.to : 0;
+  if (lastTo < newContent.length - 1) {
+    const tail = newContent.slice(lastTo).trim();
+    if (tail.length > 0) {
+      const fullTree = parser.parse(newContent);
+      let fullBlocks = await extractBlocks(fullTree, newContent);
+      fullBlocks = runDocumentPlugins(fullBlocks, newContent);
+      const fullLast = fullBlocks.length > 0 ? fullBlocks[fullBlocks.length - 1]?.payload?.range?.to ?? 0 : 0;
+      if (fullLast >= newContent.length - 1 || fullBlocks.length >= changedBlocks.length) {
+        newTree = fullTree;
+        changedBlocks = fullBlocks;
+      }
+    }
+  }
   const prevBlocks = blocks;
   blocks = changedBlocks;
   lastTree = newTree;
@@ -83218,6 +83309,50 @@ async function emitBlockDiffPatches(previousBlocks, nextBlocks, changedRanges, m
     });
   }
   const immediatePatches = partitionPatchesForCredits(combined, paragraphLimit === null ? void 0 : paragraphLimit);
+  if (deferredPatchQueue.length > maxDeferredQueueSize) {
+    maxDeferredQueueSize = deferredPatchQueue.length;
+  }
+  let structuralCount = 0;
+  for (const patch5 of patches) {
+    if (patch5.op === "insertChild" || patch5.op === "deleteChild" || patch5.op === "replaceChild" || patch5.op === "reorder") {
+      structuralCount += 1;
+    }
+  }
+  lastDiffSummary = {
+    prevCount: previousBlocks.length,
+    nextCount: nextBlocks.length,
+    prefix,
+    removeCount,
+    addCount,
+    structuralPatches: structuralCount,
+    contentPatches: contentPatches.length,
+    immediatePatches: immediatePatches.length,
+    deferredQueue: deferredPatchQueue.length
+  };
+  if (addCount > 0 || removeCount > 0 || structuralCount > 0 || deferredPatchQueue.length > 0) {
+    lastStructuralDiffSummary = { ...lastDiffSummary };
+  }
+  if (immediatePatches.length > 0) {
+    maxEmittedBlockCount = Math.max(maxEmittedBlockCount, nextBlocks.length);
+  }
+  if (structuralCount > 0) {
+    maxStructuralBlockCount = Math.max(maxStructuralBlockCount, nextBlocks.length);
+  }
+  if (nextBlocks.length >= 60 && removeCount === 0 && addCount === 0) {
+    const sliceStart = Math.max(0, Math.min(55, nextBlocks.length - 1));
+    const sliceEnd = Math.min(nextBlocks.length, sliceStart + 8);
+    const toPreview = (block) => ({
+      id: block.id,
+      type: block.type,
+      raw: typeof block.payload.raw === "string" ? block.payload.raw.slice(0, 80) : ""
+    });
+    lastHighCountNoStructural = {
+      nextCount: nextBlocks.length,
+      sliceStart,
+      prevSlice: previousBlocks.slice(sliceStart, sliceEnd).map(toPreview),
+      nextSlice: nextBlocks.slice(sliceStart, sliceEnd).map(toPreview)
+    };
+  }
   if (immediatePatches.length === 0) {
     if (metrics) {
       metrics.finalizePatch(txCounter, 0, deferredPatchQueue.length, 0);
@@ -83786,6 +83921,104 @@ async function processWorkerMessage(msg) {
     case "FINALIZE":
       await finalizeAllBlocks();
       return;
+    case "DEBUG_STATE": {
+      const blockTypeCounts = {};
+      for (const block of blocks) {
+        const key2 = block.type ?? "unknown";
+        blockTypeCounts[key2] = (blockTypeCounts[key2] || 0) + 1;
+      }
+      let lastBlockType;
+      let lastBlockRange;
+      let lastBlockRawTail;
+      const headingTexts = [];
+      const tailBlocks = [];
+      const headBlocks = [];
+      const duplicateBlockIds = [];
+      const seenBlockIds = /* @__PURE__ */ new Set();
+      const headingIndices = {};
+      for (const block of blocks) {
+        if (seenBlockIds.has(block.id)) {
+          if (duplicateBlockIds.length < 8) {
+            duplicateBlockIds.push(block.id);
+          }
+        } else {
+          seenBlockIds.add(block.id);
+        }
+      }
+      if (blocks.length > 0) {
+        const lastBlock = blocks[blocks.length - 1];
+        lastBlockType = lastBlock.type;
+        const range2 = lastBlock.payload.range;
+        if (range2 && typeof range2.from === "number" && typeof range2.to === "number") {
+          lastBlockRange = { from: range2.from, to: range2.to };
+        }
+        const raw2 = typeof lastBlock.payload.raw === "string" ? lastBlock.payload.raw : "";
+        lastBlockRawTail = raw2 ? raw2.slice(Math.max(0, raw2.length - 240)) : void 0;
+        for (const block of blocks) {
+          if (block.type === "heading" && typeof block.payload.raw === "string") {
+            headingTexts.push(block.payload.raw);
+          }
+        }
+        const targets = ["HTML and MDX Testing", "Inline Code", "Code Blocks", "Media", "Tables", "Footnotes"];
+        for (let i = 0; i < blocks.length; i++) {
+          const block = blocks[i];
+          if (block.type === "heading" && typeof block.payload.raw === "string") {
+            if (targets.includes(block.payload.raw)) {
+              headingIndices[block.payload.raw] = i;
+            }
+          }
+        }
+        const tailStart = Math.max(0, blocks.length - 8);
+        for (let i = tailStart; i < blocks.length; i++) {
+          const block = blocks[i];
+          const raw3 = typeof block.payload.raw === "string" ? block.payload.raw : "";
+          tailBlocks.push({
+            id: block.id,
+            type: block.type,
+            raw: raw3.slice(0, 120)
+          });
+        }
+        const headEnd = Math.min(8, blocks.length);
+        for (let i = 0; i < headEnd; i++) {
+          const block = blocks[i];
+          const raw3 = typeof block.payload.raw === "string" ? block.payload.raw : "";
+          headBlocks.push({
+            id: block.id,
+            type: block.type,
+            raw: raw3.slice(0, 120)
+          });
+        }
+      }
+      const contentTail = currentContent.slice(Math.max(0, currentContent.length - 500));
+      postMessage({
+        type: "DEBUG_STATE",
+        state: {
+          contentLength: currentContent.length,
+          contentTail,
+          blockCount: blocks.length,
+          blockTypeCounts,
+          lastBlockType,
+          lastBlockRange,
+          lastBlockRawTail,
+          headingTexts,
+          headBlocks,
+          tailBlocks,
+          headingIndices,
+          lastDiffSummary,
+          lastStructuralDiffSummary,
+          maxEmittedBlockCount,
+          maxDeferredQueueSize,
+          maxStructuralBlockCount,
+          lastHighCountNoStructural,
+          duplicateBlockIds,
+          duplicateBlockCount: duplicateBlockIds.length,
+          hasInlineCodeHeading: currentContent.includes("# Inline Code"),
+          hasCodeBlocksHeading: currentContent.includes("# Code Blocks"),
+          hasMediaHeading: currentContent.includes("# Media")
+        }
+      });
+      return;
+    }
     case "MDX_COMPILED":
       handleMdxStatus(msg.blockId, {
         compiledRef: { id: msg.compiledId },

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@stream-mdx/worker",
-  "version": "0.1.0",
+  "version": "0.1.1",
   "description": "Worker client utilities and shared worker helpers for the Streaming Markdown V2 pipeline",
   "license": "MIT",
   "repository": {
@@ -63,8 +63,8 @@
     "@lezer/common": "^1.2.3",
     "@lezer/lr": "^1.4.2",
     "@lezer/markdown": "^1.3.0",
-    "@stream-mdx/core": "0.1.0",
-    "@stream-mdx/plugins": "0.1.0",
+    "@stream-mdx/core": "0.1.1",
+    "@stream-mdx/plugins": "0.1.1",
     "@mdx-js/mdx": "^3.1.0",
     "@shikijs/engine-javascript": "^1.29.2",
     "@shikijs/engine-oniguruma": "^1.29.2",