@mui/internal-docs-infra 0.11.1-canary.9 → 0.11.1

package/pipeline/loadIsomorphicCodeVariant/loadIsomorphicCodeVariant.mjs

@@ -1,14 +1,17 @@
 import * as path from 'path-module';
 import { compressHastAsync } from "../hastUtils/index.mjs";
+import { buildRootFallback, buildCriticalFallback, fallbackToText } from "../../CodeHighlighter/fallbackFormat.mjs";
+import { getInitialVisibleFrames } from "../parseSource/frameVisibility.mjs";
 import { transformSource } from "./transformSource.mjs";
 import { diffHast } from "./diffHast.mjs";
+import { isFrameSpan } from "../parseSource/isFrameSpan.mjs";
 import { getFileNameFromUrl, getLanguageFromExtension, normalizeLanguage } from "../loaderUtils/index.mjs";
-import { convertCommentsToOneIndexed } from "../loaderUtils/convertCommentsToOneIndexed.mjs";
 import { mergeExternals } from "../loaderUtils/mergeExternals.mjs";
 import { applyUrlPrefixToVariant } from "../loaderUtils/applyUrlPrefix.mjs";
 import { performanceMeasure } from "../loadPrecomputedCodeHighlighter/performanceLogger.mjs";
 import { starryNightGutter } from "../parseSource/addLineGutters.mjs";
 import { applyEnhancers } from "./runSourceEnhancers.mjs";
+import { embedTransformsInRoot, splitTransformsForEmbed } from "./embedTransforms.mjs";
 
 /**
  * Check if a path is absolute (either filesystem absolute or URL)
@@ -17,6 +20,23 @@ function isAbsolutePath(filePath) {
   return path.isAbsolute(filePath) || filePath.includes('://');
 }
 
+/**
+ * Removes the per-frame `data.fallback` text from each `span.frame` before the
+ * hast is serialized. The variant-level root fallback already carries this text
+ * (and `redistributeRootFallback` puts it back on decode), so keeping it on the
+ * stored tree would duplicate it in every payload.
+ */
+function stripFrameFallbacks(root) {
+  for (const child of root.children) {
+    if (child.type !== 'element' || child.tagName !== 'span' || !child.data) {
+      continue;
+    }
+    if (isFrameSpan(child) && 'fallback' in child.data) {
+      delete child.data.fallback;
+    }
+  }
+}
+
 /**
  * Generate a conflict-free filename for globalsCode files.
  * Strategy:
@@ -158,9 +178,15 @@ function normalizePathKey(key) {
 async function loadSingleFile(variantName, fileName, source, url, loadSource, sourceParser, sourceTransformers, sourceEnhancers, loadSourceCache, transforms, options = {}, allFilesListed = false, knownExtraFiles = new Set(), language, variantComments) {
   const {
     disableTransforms = false,
-    disableParsing = false
+    disableParsing = false,
+    framePlainFallback = false
   } = options;
   let finalSource = source;
+  let finalFallback;
+  let finalFallbackCritical;
+  let finalTotalLines;
+  let finalFocusedLines;
+  let finalCollapsible;
   let extraFilesFromSource;
   let extraDependenciesFromSource;
   let externalsFromSource;
@@ -249,14 +275,16 @@ async function loadSingleFile(variantName, fileName, source, url, loadSource, so
       if (error instanceof Error && (error.message.startsWith('Invalid extraFiles from loadSource:') || error.message.startsWith('Invalid extraDependencies from loadSource:') || error.message.startsWith('Unexpected files discovered via loadSource when allFilesListed=true'))) {
         throw error;
       }
-      throw new Error(`Failed to load source code (variant: ${variantName}, file: ${fileName}, url: ${url}): ${JSON.stringify(error)}`);
+      throw new Error(`Failed to load source code (variant: ${variantName}, file: ${fileName}, url: ${url}): ${error instanceof Error ? error.message : String(error)}`, {
+        cause: error
+      });
     }
   }
 
   // Apply source transformers if no transforms exist and transforms are not disabled
   let finalTransforms = transforms;
   if (sourceTransformers && !finalTransforms && !disableTransforms && finalSource) {
-    finalTransforms = await transformSource(finalSource, normalizePathKey(fileName), sourceTransformers);
+    finalTransforms = await transformSource(finalSource, normalizePathKey(fileName), sourceTransformers, commentsFromSource);
     currentMark = performanceMeasure(currentMark, {
       mark: 'Transformed File',
       measure: 'File Transforming'
@@ -277,13 +305,18 @@ async function loadSingleFile(variantName, fileName, source, url, loadSource, so
         measure: 'File Parsing'
       }, [functionName, url || fileName]);
 
+      // `commentsFromSource` is already 1-indexed (both `Code` comments and
+      // `parseImportsAndComments`/`loadSource` output use the 1-indexed convention).
+      // Aliased so the diff path (below) can reuse it when wrapping `parseSource` for
+      // transformed sources — the comments live in the code itself and don't shift for
+      // transforms that only blank lines.
+      const oneIndexedComments = commentsFromSource;
+
       // Apply source enhancers if provided (run sequentially as a pipeline).
       // Enhancers with a stable `enhancerName` are recorded on the HAST root
       // and skipped if they have already been applied (e.g. by a previous
       // server-side pass).
       if (sourceEnhancers && sourceEnhancers.length > 0) {
-        // Convert comments from 0-indexed to 1-indexed for HAST compatibility
-        const oneIndexedComments = convertCommentsToOneIndexed(commentsFromSource);
         parsedSource = await applyEnhancers(parsedSource, oneIndexedComments, fileName, sourceEnhancers);
         currentMark = performanceMeasure(currentMark, {
           mark: 'Enhanced File',
@@ -292,16 +325,93 @@ async function loadSingleFile(variantName, fileName, source, url, loadSource, so
       }
       finalSource = parsedSource;
       if (finalTransforms && !disableTransforms) {
-        finalTransforms = await diffHast(sourceString, finalSource, normalizePathKey(fileName), finalTransforms, parseSource);
+        // Wrap parseSource so transformed sources receive the same source
+        // enhancers as the original. The frame structure produced by
+        // enhanceCodeEmphasis depends on `@focus`/`@padding-*` comments;
+        // running enhancers on both sides keeps the per-frame children
+        // layout aligned for a positional diff. Without this the diff
+        // balloons at the frame level (source has N frames, transform
+        // has 1) and jsondiffpatch deletes the extras.
+        const parseSourceForDiff = sourceEnhancers && sourceEnhancers.length > 0 ? async (transformedSourceString, transformedFileName, _language, transformedComments) => {
+          const transformedTree = await parseSource(transformedSourceString, transformedFileName);
+          // Prefer the transform-provided comment map (already
+          // 1-indexed against the transformed source) so enhancers
+          // emit the same frame structure on both sides. Falling
+          // back to the source's `oneIndexedComments` is safe for
+          // transforms that only blank lines in place, where the
+          // comment positions don't shift.
+          return applyEnhancers(transformedTree, transformedComments ?? oneIndexedComments, transformedFileName, sourceEnhancers);
+        } : parseSource;
+        finalTransforms = await diffHast(sourceString, finalSource, normalizePathKey(fileName), finalTransforms, parseSourceForDiff);
         currentMark = performanceMeasure(currentMark, {
           mark: 'Transform Parsed File',
           measure: 'Parsed File Transforming'
         }, [functionName, url || fileName]);
       }
+
+      // When the source is about to be serialized (compressed or stringified to
+      // JSON), embed the transform deltas inside the hast root's `data` field
+      // so they ride along inside the compressed payload — DEFLATE then
+      // shares the dictionary across the tree and the deltas, and the deltas
+      // never appear as plain JSON in the rendered HTML / module graph.
+      // The variant-level `finalTransforms` becomes a manifest (no `delta`).
+      if (finalTransforms && (options.output === 'hastCompressed' || options.output === 'hastJson') && finalSource && typeof finalSource === 'object' && !('hastJson' in finalSource) && !('hastCompressed' in finalSource)) {
+        const root = finalSource;
+        const split = splitTransformsForEmbed(finalTransforms);
+        if (split) {
+          embedTransformsInRoot(root, split.embedded);
+          finalTransforms = split.manifest;
+        } else {
+          // Every entry was empty; drop transforms entirely so we don't emit
+          // an empty manifest.
+          finalTransforms = undefined;
+        }
+      }
+
+      // Derive a variant-level root fallback from the per-frame `data.fallback`
+      // text before any serialization. This fallback is rendered by a
+      // `ContentLoading` component before the hast is decoded, and its text
+      // doubles as the DEFLATE dictionary so the compressed payload can be
+      // decompressed on the client once the fallback travels over via context.
+      if (finalSource && typeof finalSource === 'object' && !('hastJson' in finalSource) && !('hastCompressed' in finalSource)) {
+        finalFallback = buildRootFallback(finalSource);
+        // Sparse highlighted-visible companion (see `VariantCode.fallbackCritical`),
+        // computed here while the source is still a live `HastRoot` (no
+        // decompression) and BEFORE `stripFrameFallbacks` removes the per-frame
+        // text it reuses. `false` builds the `collapseToEmpty: false` form (the only
+        // one carrying highlighting); the boundary skips promoting it under
+        // `collapseToEmpty`. Empty (no visible frames) → omit it entirely.
+        const critical = buildCriticalFallback(finalSource, getInitialVisibleFrames(finalSource, false));
+        finalFallbackCritical = Object.keys(critical).length > 0 ? critical : undefined;
+
+        // Hoist the window counts off `root.data` while the source is still a live
+        // `HastRoot`. They then ride on the variant (see the return below) so every
+        // downstream reader (`prepareInitialSource`, `getVariantFileLineCounts`,
+        // layout-shift classification) gets `totalLines`/`focusedLines`/`collapsible`
+        // WITHOUT decompressing the payload — the compact fallback and the compressed
+        // source both drop `root.data`, so without this the only way to recover the
+        // counts is to decode the hast (the first-render decompression we want to avoid).
+        const rootData = finalSource.data;
+        if (rootData?.totalLines !== undefined) {
+          const total = Number(rootData.totalLines);
+          if (Number.isFinite(total) && total >= 0) {
+            finalTotalLines = total;
+            const focused = Number(rootData.focusedLines);
+            finalFocusedLines = Number.isFinite(focused) && focused >= 0 ? focused : total;
+            finalCollapsible = rootData.collapsible === true;
+          }
+        }
+      }
       if (options.output === 'hastCompressed' && process.env.NODE_ENV === 'production') {
-        const hastCompressed = await compressHastAsync(JSON.stringify(finalSource));
+        if (finalFallback) {
+          stripFrameFallbacks(finalSource);
+        }
+        const json = JSON.stringify(finalSource);
+        // Use the fallback text as a DEFLATE dictionary for better compression.
+        // The same dictionary is rebuilt on decode from the variant `fallback`.
+        const dictionary = finalFallback ? fallbackToText(finalFallback) : undefined;
         finalSource = {
-          hastCompressed
+          hastCompressed: await compressHastAsync(json, dictionary)
         };
         currentMark = performanceMeasure(currentMark, {
           mark: 'Compressed File',
@@ -309,6 +419,9 @@ async function loadSingleFile(variantName, fileName, source, url, loadSource, so
         }, [functionName, url || fileName]);
       } else if (options.output === 'hastJson' || options.output === 'hastCompressed') {
         // in development, we skip compression but still convert to JSON
+        if (finalFallback) {
+          stripFrameFallbacks(finalSource);
+        }
         finalSource = {
           hastJson: JSON.stringify(finalSource)
         };
@@ -321,14 +434,49 @@ async function loadSingleFile(variantName, fileName, source, url, loadSource, so
       throw new Error(`Failed to parse source code (variant: ${variantName}, file: ${fileName}, url: ${url}): ${error instanceof Error ? error.message : ''}`);
     }
   }
+
+  // Parsing disabled (deferred highlight) but a framed fallback was requested: keep
+  // `finalSource` a plain string so the client still knows it needs syntax
+  // highlighting — a HAST source reads as "already loaded" (see `isSourceLoaded`) and
+  // would never re-highlight. But the loading fallback MUST be framed — rendered code
+  // needs frames — so build a line-guttered plain-text HAST (no syntax engine: the
+  // light `starryNightGutter`), run the same enhancers (focus window / truncation),
+  // and derive the root fallback from it. The string source is returned unchanged;
+  // only `fallback` is produced. Gated by `framePlainFallback` so lazy variant loads
+  // (which never paint a fallback) keep skipping the work.
+  if (framePlainFallback && typeof finalSource === 'string' && !finalFallback) {
+    const plainRoot = {
+      type: 'root',
+      children: [{
+        type: 'text',
+        value: finalSource
+      }]
+    };
+    starryNightGutter(plainRoot, finalSource.split(/\r?\n|\r/));
+    let framedRoot = plainRoot;
+    if (sourceEnhancers && sourceEnhancers.length > 0) {
+      framedRoot = await applyEnhancers(framedRoot, commentsFromSource, fileName, sourceEnhancers);
+    }
+    finalFallback = buildRootFallback(framedRoot);
+    // Surface the enhancer's window counts (the compact fallback drops `root.data`,
+    // and a plain-string source can't recompute `focusedLines`/`collapsible` downstream).
+    finalTotalLines = framedRoot.data?.totalLines ?? 0;
+    finalFocusedLines = framedRoot.data?.focusedLines ?? finalTotalLines;
+    finalCollapsible = framedRoot.data?.collapsible === true;
+  }
   return {
     source: finalSource,
+    fallback: finalFallback,
+    fallbackCritical: finalFallbackCritical,
+    totalLines: finalTotalLines,
+    focusedLines: finalFocusedLines,
+    collapsible: finalCollapsible,
     transforms: finalTransforms,
     extraFiles: extraFilesFromSource,
     extraDependencies: extraDependenciesFromSource,
     externals: externalsFromSource,
-    // Convert comments to 1-indexed for HAST compatibility when stored on variant
-    comments: convertCommentsToOneIndexed(commentsFromSource)
+    // `commentsFromSource` is already 1-indexed (the stored `Code` convention).
+    comments: commentsFromSource
   };
 }
 
@@ -356,6 +504,7 @@ loadSource, sourceParser, sourceTransformers, sourceEnhancers, loadSourceCache,
     try {
       let fileUrl;
       let sourceData;
+      let inlineComments;
       let transforms;
       let nextLoadedFiles;
       // True when the entry references an external file to load (string form
@@ -391,6 +540,10 @@ loadSource, sourceParser, sourceTransformers, sourceEnhancers, loadSourceCache,
       } else {
         // fileData is an object with source and/or transforms
         sourceData = fileData.source;
+        // Inline extra files carry their own 1-indexed comments (their marker lines were
+        // stripped from the source upstream); forward them so the enhancers apply the
+        // `@focus`/`@highlight` frames instead of silently dropping them.
+        inlineComments = fileData.comments;
         transforms = fileData.transforms;
         fileUrl = baseUrl; // Use base URL as fallback
         // For inline source, just pass a copy of loadedFiles without adding current file
@@ -406,7 +559,7 @@ loadSource, sourceParser, sourceTransformers, sourceEnhancers, loadSourceCache,
         ...options,
         maxDepth: maxDepth - 1,
         loadedFiles: nextLoadedFiles
-      }, allFilesListed, knownExtraFiles, extraFileLanguage);
+      }, allFilesListed, knownExtraFiles, extraFileLanguage, inlineComments);
 
       // Collect files used from this file load
       const filesUsedFromFile = [];
@@ -481,6 +634,18 @@ loadSource, sourceParser, sourceTransformers, sourceEnhancers, loadSourceCache,
     const extraFileLanguage = getLanguageFromExtension(extraFileExtension);
     processedExtraFiles[normalizedFileName] = {
       source: result.source,
+      ...(result.fallback && {
+        fallback: result.fallback
+      }),
+      ...(result.totalLines !== undefined && {
+        totalLines: result.totalLines
+      }),
+      ...(result.focusedLines !== undefined && {
+        focusedLines: result.focusedLines
+      }),
+      ...(result.collapsible !== undefined && {
+        collapsible: result.collapsible
+      }),
       ...(extraFileLanguage && {
         language: extraFileLanguage
       }),
@@ -606,7 +771,9 @@ export async function loadIsomorphicCodeVariant(url, variantName, variant, optio
       try {
         variant = await loadVariantMeta(variantName, variant);
       } catch (error) {
-        throw new Error(`Failed to load variant code (variant: ${variantName}, url: ${variant}): ${JSON.stringify(error)}`);
+        throw new Error(`Failed to load variant code (variant: ${variantName}, url: ${variant}): ${error instanceof Error ? error.message : String(error)}`, {
+          cause: error
+        });
       }
       currentMark = performanceMeasure(currentMark, {
         mark: 'Loaded Variant Meta',
@@ -643,6 +810,8 @@ export async function loadIsomorphicCodeVariant(url, variantName, variant, optio
   // If we don't have a fileName and no URL, we can still parse if we have language
   if (!fileName && !url) {
     let finalSource = variant.source;
+    let finalFallback;
+    let finalFallbackCritical;
 
     // Parse the source if we have language and sourceParser
     if (typeof finalSource === 'string' && language && sourceParser && !disableParsing) {
@@ -664,13 +833,43 @@ export async function loadIsomorphicCodeVariant(url, variantName, variant, optio
 
     // Apply source enhancers if provided and parsing is not disabled
     if (!disableParsing && sourceEnhancers && sourceEnhancers.length > 0) {
-      const oneIndexedComments = convertCommentsToOneIndexed(variant.comments);
+      // `variant.comments` is already 1-indexed (the stored `Code` convention).
+      const oneIndexedComments = variant.comments;
       finalSource = await applyEnhancers(finalSource, oneIndexedComments, '', sourceEnhancers);
     }
+
+    // Apply output format compression in production. Other format conversions
+    // happen lazily via the loader so tests can inspect the parsed HAST directly.
+    if (finalSource && typeof finalSource === 'object' && 'type' in finalSource) {
+      // Always derive a variant-level root fallback from the per-frame text so a
+      // `ContentLoading` component can render before the hast is decoded.
+      finalFallback = buildRootFallback(finalSource);
+      // Sparse highlighted-visible companion (see `VariantCode.fallbackCritical`),
+      // built from the live `HastRoot` before `stripFrameFallbacks` runs.
+      const critical = buildCriticalFallback(finalSource, getInitialVisibleFrames(finalSource, false));
+      finalFallbackCritical = Object.keys(critical).length > 0 ? critical : undefined;
+      if (options.output === 'hastCompressed' && process.env.NODE_ENV === 'production') {
+        if (finalFallback) {
+          stripFrameFallbacks(finalSource);
+        }
+        const json = JSON.stringify(finalSource);
+        // Use the fallback text as a DEFLATE dictionary; rebuilt on decode.
+        const dictionary = finalFallback ? fallbackToText(finalFallback) : undefined;
+        finalSource = {
+          hastCompressed: await compressHastAsync(json, dictionary)
+        };
+      }
+    }
     const finalVariant = {
       ...variant,
       language,
-      source: finalSource
+      source: finalSource,
+      ...(finalFallback ? {
+        fallback: finalFallback
+      } : {}),
+      ...(finalFallbackCritical ? {
+        fallbackCritical: finalFallbackCritical
+      } : {})
     };
     return {
       code: finalVariant,
@@ -919,6 +1118,21 @@ export async function loadIsomorphicCodeVariant(url, variantName, variant, optio
     ...variant,
     language,
     source: mainFileResult.source,
+    ...(mainFileResult.fallback && {
+      fallback: mainFileResult.fallback
+    }),
+    ...(mainFileResult.fallbackCritical && {
+      fallbackCritical: mainFileResult.fallbackCritical
+    }),
+    ...(mainFileResult.totalLines !== undefined && {
+      totalLines: mainFileResult.totalLines
+    }),
+    ...(mainFileResult.focusedLines !== undefined && {
+      focusedLines: mainFileResult.focusedLines
+    }),
+    ...(mainFileResult.collapsible !== undefined && {
+      collapsible: mainFileResult.collapsible
+    }),
     transforms: mainFileResult.transforms,
     extraFiles: Object.keys(allExtraFiles).length > 0 ? allExtraFiles : undefined,
     externals: Object.keys(allExternals).length > 0 ? Object.keys(allExternals) : undefined,

package/pipeline/loadIsomorphicCodeVariant/transformSource.d.mts

@@ -1,2 +1,2 @@
-import type { VariantSource, SourceTransformers, Transforms } from "../../CodeHighlighter/types.mjs";
-export declare function transformSource(source: VariantSource, fileName: string, sourceTransformers: SourceTransformers): Promise<Transforms | undefined>;
+import type { VariantSource, SourceComments, SourceTransformers, Transforms } from "../../CodeHighlighter/types.mjs";
+export declare function transformSource(source: VariantSource, fileName: string, sourceTransformers: SourceTransformers, comments?: SourceComments): Promise<Transforms | undefined>;

package/pipeline/loadIsomorphicCodeVariant/transformSource.mjs

@@ -5,36 +5,70 @@ const differ = create({
   omitRemovedValues: true,
   cloneDiffValues: true
 });
-export async function transformSource(source, fileName, sourceTransformers) {
-  const transforms = await Promise.all(sourceTransformers.map(async ({
-    extensions,
+export async function transformSource(source, fileName, sourceTransformers, comments) {
+  // Find applicable transformers up front so we can short-circuit before
+  // doing any source decoding work.
+  const applicableTransformers = sourceTransformers.filter(({
+    extensions
+  }) => extensions.some(ext => fileName.endsWith(`.${ext}`)));
+  if (applicableTransformers.length === 0) {
+    return undefined;
+  }
+
+  // Decode the source string and split it once. Both are independent of
+  // which transformer is running, so doing this per-transformer (as the
+  // previous implementation did) wasted work proportional to the number
+  // of registered transformers.
+  let sourceString;
+  if (typeof source === 'string') {
+    sourceString = source;
+  } else if ('hastJson' in source) {
+    sourceString = toText(JSON.parse(source.hastJson));
+  } else if ('hastCompressed' in source) {
+    sourceString = toText(JSON.parse(await decompressHastAsync(source.hastCompressed)));
+  } else {
+    sourceString = toText(source);
+  }
+  const splitSource = sourceString.split('\n');
+  const transforms = await Promise.all(applicableTransformers.map(async ({
     transformer
   }) => {
-    if (!extensions.some(ext => fileName.endsWith(`.${ext}`))) {
-      return undefined;
-    }
     try {
-      let sourceString;
-      if (typeof source === 'string') {
-        sourceString = source;
-      } else if ('hastJson' in source) {
-        sourceString = toText(JSON.parse(source.hastJson));
-      } else if ('hastCompressed' in source) {
-        sourceString = toText(JSON.parse(await decompressHastAsync(source.hastCompressed)));
-      } else {
-        sourceString = toText(source);
-      }
-      const transformed = await transformer(sourceString, fileName);
+      const transformed = await transformer(sourceString, fileName, comments);
       if (transformed) {
-        const splitSource = sourceString.split('\n');
-        return Object.keys(transformed).reduce((acc, key) => {
-          const delta = differ.diff(splitSource, transformed[key].source.split('\n'));
+        const reduced = Object.keys(transformed).reduce((acc, key) => {
+          const entry = transformed[key];
+          const delta = differ.diff(splitSource, entry.source.split('\n'));
+
+          // Comments are 1-indexed everywhere, including transformer input and output
+          // (keyed against the transformed source's 1-indexed lines).
+          const transformedComments = entry.comments;
+          const hasDelta = !!delta && typeof delta === 'object' && Object.keys(delta).length > 0;
+          const renamed = !!entry.fileName && entry.fileName !== fileName;
+
+          // Drop entries that neither change the source nor rename the
+          // file — there's nothing for the runtime to apply.
+          if (!hasDelta && !renamed) {
+            return acc;
+          }
           acc[key] = {
-            delta,
-            fileName: transformed[key].fileName
+            ...(hasDelta && {
+              delta,
+              hasDelta: true
+            }),
+            ...(entry.fileName !== undefined && {
+              fileName: entry.fileName
+            }),
+            ...(transformedComments && {
+              comments: transformedComments
+            })
           };
           return acc;
         }, {});
+        // If every entry was dropped (e.g. a transformer that only
+        // produced noop entries), surface `undefined` so the caller
+        // treats the variant as untransformed.
+        return Object.keys(reduced).length > 0 ? reduced : undefined;
       }
       return undefined;
     } catch (error) {

package/pipeline/loadPrecomputedCodeHighlighter/loadPrecomputedCodeHighlighter.d.mts

@@ -39,6 +39,24 @@ export type LoaderOptions = {
    * to be relative to each generated `client.ts`.
    */
   requireClient?: string;
+  /**
+   * Marker option consumed by `pnpm docs-infra validate` (not by this loader).
+   *
+   * When `true` on a demo `index.ts` rule, the validate command ensures every
+   * matched demo has a sibling `page.tsx` that renders the demo as the route's
+   * default export, so each demo is browsable on its own page.
+   *
+   * Existing `page.tsx`/`page.ts` files are never overwritten.
+   */
+  requirePage?: boolean;
+  /**
+   * When `true`, registers the `TypescriptToJavascriptTransformer` so that
+   * TypeScript variants also produce a JavaScript counterpart at build time.
+   *
+   * Defaults to `false` because the transform is comparatively expensive;
+   * enable it when the rendered demos need both TS and JS sources.
+   */
+  transformTypescriptToJavascript?: boolean;
 };
 /**
  * Webpack loader that processes demo files and precomputes variant data.

package/pipeline/loadPrecomputedCodeHighlighter/loadPrecomputedCodeHighlighter.mjs

@@ -5,9 +5,7 @@ import path from 'path';
 import { fileURLToPath, pathToFileURL } from 'url';
 import { loadIsomorphicCodeVariant } from "../loadIsomorphicCodeVariant/loadIsomorphicCodeVariant.mjs";
 import { createParseSource } from "../parseSource/index.mjs";
-// TODO: re-enable following benchmarking
-// import { TypescriptToJavascriptTransformer } from '../transformTypescriptToJavascript';
-
+import { TypescriptToJavascriptTransformer } from "../transformTypescriptToJavascript/index.mjs";
 import { createEnhanceCodeEmphasis, EMPHASIS_COMMENT_PREFIX, FOCUS_COMMENT_PREFIX } from "../enhanceCodeEmphasis/enhanceCodeEmphasis.mjs";
 import { parseCreateFactoryCall } from "../parseCreateFactoryCall/parseCreateFactoryCall.mjs";
 import { resolveVariantPathsWithFs } from "../loadServerCodeMeta/resolveModulePathWithFs.mjs";
@@ -132,9 +130,7 @@ export async function loadPrecomputedCodeHighlighter(source) {
     });
 
     // Setup source transformers for TypeScript to JavaScript conversion
-    // const sourceTransformers: SourceTransformers = [TypescriptToJavascriptTransformer];
-    // TODO: maybe we should have `loadPrecomputedCodeHighlighterWithJsToTs`
-    const sourceTransformers = [];
+    const sourceTransformers = options.transformTypescriptToJavascript ? [TypescriptToJavascriptTransformer] : [];
 
     // Setup source enhancers for post-parsing modifications
     const sourceEnhancers = [createEnhanceCodeEmphasis(options.emphasisOptions)];
@@ -205,10 +201,18 @@ export async function loadPrecomputedCodeHighlighter(source) {
     });
     const variantResults = await Promise.all(variantPromises);
 
+    // Diagnostic: re-serialize each variant through JSON to sever any
+    // `SlicedString`/`ConsString` references that may pin large parent strings
+    // (e.g. raw source files) alive inside the precomputed hast tree. Enabled
+    // by `DEBUG_DOCS_INFRA_FLATTEN=1`. If memory usage drops noticeably with
+    // this on, the leak is SlicedString retention in variant data and we
+    // should flatten at the source instead.
+    const flattenVariants = typeof process !== 'undefined' && process.env?.DEBUG_DOCS_INFRA_FLATTEN === '1';
+
     // Process results and collect dependencies
     for (const result of variantResults) {
       if (result) {
-        variantData[result.variantName] = result.variantData;
+        variantData[result.variantName] = flattenVariants ? JSON.parse(JSON.stringify(result.variantData)) : result.variantData;
         result.dependencies.forEach(file => {
           allDependencies.push(file);
         });

package/pipeline/loadServerTypes/hastTypeUtils.d.mts

@@ -147,8 +147,8 @@ export type TypesOutputFormat = 'hast' | 'hastJson' | 'hastCompressed';
 /** Converts a HastRoot to a JSON-serialized wrapper. */
 export declare function serializeHastRoot(hast: HastRoot): SerializedHastRoot;
 /** Converts a HastRoot to a dictionary-compressed, base64-encoded wrapper. */
-export declare function compressHastRoot(hast: HastRoot): SerializedHastCompressed;
+export declare function compressHastRoot(hast: HastRoot, textContent?: string): SerializedHastCompressed;
 /** Returns the appropriate serializer function for the given output format. */
-export declare function resolveSerializer(output: TypesOutputFormat): (hast: HastRoot) => HastRoot | SerializedHastRoot | SerializedHastCompressed;
+export declare function resolveSerializer(output: TypesOutputFormat, textContent?: string): (hast: HastRoot) => HastRoot | SerializedHastRoot | SerializedHastCompressed;
 /** No-op passthrough — avoids allocating a fresh closure on every call. */
 export declare function hastIdentity(hast: HastRoot): HastRoot;

package/pipeline/loadServerTypes/hastTypeUtils.mjs

@@ -460,16 +460,16 @@ export function serializeHastRoot(hast) {
 }
 
 /** Converts a HastRoot to a dictionary-compressed, base64-encoded wrapper. */
-export function compressHastRoot(hast) {
+export function compressHastRoot(hast, textContent) {
   return {
-    hastCompressed: compressHast(JSON.stringify(hast))
+    hastCompressed: compressHast(JSON.stringify(hast), textContent)
   };
 }
 
 /** Returns the appropriate serializer function for the given output format. */
-export function resolveSerializer(output) {
+export function resolveSerializer(output, textContent) {
   if (output === 'hastCompressed') {
-    return compressHastRoot;
+    return hast => compressHastRoot(hast, textContent);
   }
   if (output === 'hastJson') {
     return serializeHastRoot;

package/pipeline/loadServerTypes/loadServerTypes.mjs

@@ -60,7 +60,7 @@ export async function loadServerTypes(options) {
         return parts.slice(1).join('.').toLowerCase();
       }
       // No prefix match, use the full name
-      return name.replace(/\./g, '.').toLowerCase();
+      return name.toLowerCase();
     }
     // Non-dotted name: use as-is
     return name.toLowerCase();

package/pipeline/loadServerTypesMeta/extractJSDocText.d.mts

@@ -0,0 +1,14 @@
+/**
+ * Extracts text content from a JSDoc description array.
+ *
+ * typescript-api-extractor returns a description as an array of JSDoc nodes when
+ * the comment contains `{@link}` tags: plain-text nodes carry a `text` property,
+ * while link nodes carry the referenced symbol in `name.escapedText`. Link nodes
+ * are turned into markdown via {@link renderJSDocLink}, gated on `documentedNames`.
+ */
+export declare function extractJSDocText(nodes: unknown[], documentedNames?: Set<string>): string;
+/**
+ * Checks if an array looks like JSDoc description nodes from typescript-api-extractor.
+ * These have properties like 'pos', 'end', 'kind', 'text' from the TypeScript AST.
+ */
+export declare function isJSDocNodeArray(value: unknown[]): boolean;