@docusaurus/mdx-loader 3.5.0 → 3.5.2-canary-6121

package/src/loader.ts

@@ -5,22 +5,22 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import fs from 'fs-extra';
 import logger from '@docusaurus/logger';
 import {
+  aliasedSitePath,
   DEFAULT_PARSE_FRONT_MATTER,
-  escapePath,
   getFileLoaderUtils,
   getWebpackLoaderCompilerName,
 } from '@docusaurus/utils';
 import stringifyObject from 'stringify-object';
-import preprocessor from './preprocessor';
-import {validateMDXFrontMatter} from './frontMatter';
-import {createProcessorCached} from './processor';
-import type {ResolveMarkdownLink} from './remark/resolveMarkdownLinks';
-import type {MDXOptions} from './processor';
-
-import type {MarkdownConfig} from '@docusaurus/types';
+import {
+  compileToJSX,
+  createAssetsExportCode,
+  extractContentTitleData,
+  promiseWithResolvers,
+} from './utils';
+import type {WebpackCompilerName} from '@docusaurus/utils';
+import type {Options} from './options';
 import type {LoaderContext} from 'webpack';
 
 // TODO as of April 2023, no way to import/re-export this ESM type easily :/
@@ -28,235 +28,91 @@ import type {LoaderContext} from 'webpack';
 // See https://github.com/microsoft/TypeScript/issues/49721#issuecomment-1517839391
 type Pluggable = any; // TODO fix this asap
 
-const {
-  loaders: {inlineMarkdownAssetImageFileLoader},
-} = getFileLoaderUtils();
-
 export type MDXPlugin = Pluggable;
 
-export type Options = Partial<MDXOptions> & {
-  markdownConfig: MarkdownConfig;
-  staticDirs: string[];
-  siteDir: string;
-  isMDXPartial?: (filePath: string) => boolean;
-  isMDXPartialFrontMatterWarningDisabled?: boolean;
-  removeContentTitle?: boolean;
-  metadataPath?: string | ((filePath: string) => string);
-  createAssets?: (metadata: {
-    frontMatter: {[key: string]: unknown};
-    metadata: {[key: string]: unknown};
-  }) => {[key: string]: unknown};
-  resolveMarkdownLink?: ResolveMarkdownLink;
-};
-
-/**
- * When this throws, it generally means that there's no metadata file associated
- * with this MDX document. It can happen when using MDX partials (usually
- * starting with _). That's why it's important to provide the `isMDXPartial`
- * function in config
- */
-async function readMetadataPath(metadataPath: string) {
-  try {
-    return await fs.readFile(metadataPath, 'utf8');
-  } catch (err) {
-    logger.error`MDX loader can't read MDX metadata file path=${metadataPath}. Maybe the isMDXPartial option function was not provided?`;
-    throw err;
-  }
-}
-
-/**
- * Converts assets an object with Webpack require calls code.
- * This is useful for mdx files to reference co-located assets using relative
- * paths. Those assets should enter the Webpack assets pipeline and be hashed.
- * For now, we only handle that for images and paths starting with `./`:
- *
- * `{image: "./myImage.png"}` => `{image: require("./myImage.png")}`
- */
-function createAssetsExportCode(assets: unknown) {
-  if (
-    typeof assets !== 'object' ||
-    !assets ||
-    Object.keys(assets).length === 0
-  ) {
-    return 'undefined';
-  }
-
-  // TODO implementation can be completed/enhanced
-  function createAssetValueCode(assetValue: unknown): string | undefined {
-    if (Array.isArray(assetValue)) {
-      const arrayItemCodes = assetValue.map(
-        (item: unknown) => createAssetValueCode(item) ?? 'undefined',
-      );
-      return `[${arrayItemCodes.join(', ')}]`;
-    }
-    // Only process string values starting with ./
-    // We could enhance this logic and check if file exists on disc?
-    if (typeof assetValue === 'string' && assetValue.startsWith('./')) {
-      // TODO do we have other use-cases than image assets?
-      // Probably not worth adding more support, as we want to move to Webpack 5 new asset system (https://github.com/facebook/docusaurus/pull/4708)
-      return `require("${inlineMarkdownAssetImageFileLoader}${escapePath(
-        assetValue,
-      )}").default`;
-    }
-    return undefined;
-  }
-
-  const assetEntries = Object.entries(assets);
-
-  const codeLines = assetEntries
-    .map(([key, value]: [string, unknown]) => {
-      const assetRequireCode = createAssetValueCode(value);
-      return assetRequireCode ? `"${key}": ${assetRequireCode},` : undefined;
-    })
-    .filter(Boolean);
-
-  return `{\n${codeLines.join('\n')}\n}`;
-}
-
-// TODO temporary, remove this after v3.1?
-// Some plugin authors use our mdx-loader, despite it not being public API
-// see https://github.com/facebook/docusaurus/issues/8298
-function ensureMarkdownConfig(reqOptions: Options) {
-  if (!reqOptions.markdownConfig) {
-    throw new Error(
-      'Docusaurus v3+ requires MDX loader options.markdownConfig - plugin authors using the MDX loader should make sure to provide that option',
-    );
-  }
-}
-
-/**
- * data.contentTitle is set by the remark contentTitle plugin
- */
-function extractContentTitleData(data: {
-  [key: string]: unknown;
-}): string | undefined {
-  return data.contentTitle as string | undefined;
-}
-
-export async function mdxLoader(
-  this: LoaderContext<Options>,
-  fileContent: string,
-): Promise<void> {
-  const compilerName = getWebpackLoaderCompilerName(this);
-  const callback = this.async();
-  const filePath = this.resourcePath;
-  const reqOptions: Options = this.getOptions();
-  const {query} = this;
-
-  ensureMarkdownConfig(reqOptions);
-
-  const {frontMatter} = await reqOptions.markdownConfig.parseFrontMatter({
+async function loadMDX({
+  fileContent,
+  filePath,
+  options,
+  compilerName,
+}: {
+  fileContent: string;
+  filePath: string;
+  options: Options;
+  compilerName: WebpackCompilerName;
+}): Promise<string> {
+  const {frontMatter} = await options.markdownConfig.parseFrontMatter({
     filePath,
     fileContent,
     defaultParseFrontMatter: DEFAULT_PARSE_FRONT_MATTER,
   });
-  const mdxFrontMatter = validateMDXFrontMatter(frontMatter.mdx);
-
-  const preprocessedContent = preprocessor({
-    fileContent,
-    filePath,
-    admonitions: reqOptions.admonitions,
-    markdownConfig: reqOptions.markdownConfig,
-  });
 
   const hasFrontMatter = Object.keys(frontMatter).length > 0;
 
-  const processor = await createProcessorCached({
+  const result = await compileToJSX({
+    fileContent,
     filePath,
-    reqOptions,
-    query,
-    mdxFrontMatter,
+    frontMatter,
+    options,
+    compilerName,
   });
 
-  let result: {content: string; data: {[key: string]: unknown}};
-  try {
-    result = await processor.process({
-      content: preprocessedContent,
-      filePath,
-      frontMatter,
-      compilerName,
-    });
-  } catch (errorUnknown) {
-    const error = errorUnknown as Error;
-
-    // MDX can emit errors that have useful extra attributes
-    const errorJSON = JSON.stringify(error, null, 2);
-    const errorDetails =
-      errorJSON === '{}'
-        ? // regular JS error case: print stacktrace
-          error.stack ?? 'N/A'
-        : // MDX error: print extra attributes + stacktrace
-          `${errorJSON}\n${error.stack}`;
-
-    return callback(
-      new Error(
-        `MDX compilation failed for file ${logger.path(filePath)}\nCause: ${
-          error.message
-        }\nDetails:\n${errorDetails}`,
-        // TODO error cause doesn't seem to be used by Webpack stats.errors :s
-        {cause: error},
-      ),
-    );
-  }
-
   const contentTitle = extractContentTitleData(result.data);
 
   // MDX partials are MDX files starting with _ or in a folder starting with _
   // Partial are not expected to have associated metadata files or front matter
-  const isMDXPartial = reqOptions.isMDXPartial?.(filePath);
+  const isMDXPartial = options.isMDXPartial?.(filePath);
   if (isMDXPartial && hasFrontMatter) {
     const errorMessage = `Docusaurus MDX partial files should not contain front matter.
 Those partial files use the _ prefix as a convention by default, but this is configurable.
 File at ${filePath} contains front matter that will be ignored:
 ${JSON.stringify(frontMatter, null, 2)}`;
 
-    if (!reqOptions.isMDXPartialFrontMatterWarningDisabled) {
+    if (!options.isMDXPartialFrontMatterWarningDisabled) {
       const shouldError = process.env.NODE_ENV === 'test' || process.env.CI;
       if (shouldError) {
-        return callback(new Error(errorMessage));
+        throw new Error(errorMessage);
       }
       logger.warn(errorMessage);
     }
   }
 
-  function getMetadataPath(): string | undefined {
+  const metadataPath = (function getMetadataPath() {
     if (!isMDXPartial) {
-      // Read metadata for this MDX and export it.
-      if (
-        reqOptions.metadataPath &&
-        typeof reqOptions.metadataPath === 'function'
-      ) {
-        return reqOptions.metadataPath(filePath);
-      }
+      return options.metadataPath?.(filePath);
     }
     return undefined;
-  }
-
-  const metadataPath = getMetadataPath();
-  if (metadataPath) {
-    this.addDependency(metadataPath);
-  }
-
-  const metadataJsonString = metadataPath
-    ? await readMetadataPath(metadataPath)
-    : undefined;
-
-  const metadata = metadataJsonString
-    ? (JSON.parse(metadataJsonString) as {[key: string]: unknown})
-    : undefined;
+  })();
 
   const assets =
-    reqOptions.createAssets && metadata
-      ? reqOptions.createAssets({frontMatter, metadata})
+    options.createAssets && !isMDXPartial
+      ? options.createAssets({filePath, frontMatter})
       : undefined;
 
+  const fileLoaderUtils = getFileLoaderUtils(compilerName === 'server');
+
   // TODO use remark plugins to insert extra exports instead of string concat?
   // cf how the toc is exported
   const exportsCode = `
 export const frontMatter = ${stringifyObject(frontMatter)};
 export const contentTitle = ${stringifyObject(contentTitle)};
-${metadataJsonString ? `export const metadata = ${metadataJsonString};` : ''}
-${assets ? `export const assets = ${createAssetsExportCode(assets)};` : ''}
+${
+  metadataPath
+    ? `export {default as metadata} from '${aliasedSitePath(
+        metadataPath,
+        options.siteDir,
+      )}'`
+    : ''
+}
+${
+  assets
+    ? `export const assets = ${createAssetsExportCode({
+        assets,
+        inlineMarkdownAssetImageFileLoader:
+          fileLoaderUtils.loaders.inlineMarkdownAssetImageFileLoader,
+      })};`
+    : ''
+}
 `;
 
   const code = `
@@ -264,5 +120,114 @@ ${exportsCode}
 ${result.content}
 `;
 
-  return callback(null, code);
+  return code;
+}
+
+// Note: we cache promises instead of strings
+// This is because client/server compilations might be triggered in parallel
+// When this happens for the same file, we don't want to compile it twice
+async function loadMDXWithCaching({
+  resource,
+  fileContent,
+  filePath,
+  options,
+  compilerName,
+}: {
+  resource: string; // path?query#hash
+  filePath: string; // path
+  fileContent: string;
+  options: Options;
+  compilerName: WebpackCompilerName;
+}): Promise<string> {
+  const {crossCompilerCache} = options;
+  if (!crossCompilerCache) {
+    return loadMDX({
+      fileContent,
+      filePath,
+      options,
+      compilerName,
+    });
+  }
+
+  // Note we "resource" as cache key, not "filePath" nor "fileContent"
+  // This is because:
+  // - the same file can be compiled in different variants (blog.mdx?truncated)
+  // - the same content can be processed differently (versioned docs links)
+  const cacheKey = resource;
+
+  // We can clean up the cache and free memory after cache entry consumption
+  // We know there are only 2 compilations for the same file
+  // Note: once we introduce RSCs we'll probably have 3 compilations
+  // Note: we can't use string keys in WeakMap
+  // But we could eventually use WeakRef for the values
+  const deleteCacheEntry = () => crossCompilerCache.delete(cacheKey);
+
+  const cacheEntry = crossCompilerCache?.get(cacheKey);
+
+  // When deduplicating client/server compilations, we always use the client
+  // compilation and not the server compilation
+  // This is important because the server compilation usually skips some steps
+  // Notably: the server compilation does not emit file-loader assets
+  // Using the server compilation otherwise leads to broken images
+  // See https://github.com/facebook/docusaurus/issues/10544#issuecomment-2390943794
+  // See https://github.com/facebook/docusaurus/pull/10553
+  // TODO a problem with this: server bundle will use client inline loaders
+  //  This means server bundle will use ?emit=true for assets
+  //  We should try to get rid of inline loaders to cleanup this caching logic
+  if (compilerName === 'client') {
+    const promise = loadMDX({
+      fileContent,
+      filePath,
+      options,
+      compilerName,
+    });
+    if (cacheEntry) {
+      promise.then(cacheEntry.resolve, cacheEntry.reject);
+      deleteCacheEntry();
+    } else {
+      const noop = () => {
+        throw new Error('this should never be called');
+      };
+      crossCompilerCache.set(cacheKey, {
+        promise,
+        resolve: noop,
+        reject: noop,
+      });
+    }
+    return promise;
+  }
+  // Server compilation always uses the result of the client compilation above
+  else if (compilerName === 'server') {
+    if (cacheEntry) {
+      deleteCacheEntry();
+      return cacheEntry.promise;
+    } else {
+      const {promise, resolve, reject} = promiseWithResolvers<string>();
+      crossCompilerCache.set(cacheKey, {promise, resolve, reject});
+      return promise;
+    }
+  } else {
+    throw new Error(`Unexpected compilerName=${compilerName}`);
+  }
+}
+
+export async function mdxLoader(
+  this: LoaderContext<Options>,
+  fileContent: string,
+): Promise<void> {
+  const compilerName = getWebpackLoaderCompilerName(this);
+  const callback = this.async();
+  const options: Options = this.getOptions();
+  try {
+    const result = await loadMDXWithCaching({
+      resource: this.resource,
+      filePath: this.resourcePath,
+      fileContent,
+      options,
+      compilerName,
+    });
+    return callback(null, result);
+  } catch (error) {
+    return callback(error as Error);
+  }
 }

package/src/options.ts

@@ -0,0 +1,32 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import type {MDXOptions, SimpleProcessors} from './processor';
+import type {MarkdownConfig} from '@docusaurus/types';
+import type {ResolveMarkdownLink} from './remark/resolveMarkdownLinks';
+import type {PromiseWithResolvers} from './utils';
+
+export type Options = Partial<MDXOptions> & {
+  markdownConfig: MarkdownConfig;
+  staticDirs: string[];
+  siteDir: string;
+  isMDXPartial?: (filePath: string) => boolean;
+  isMDXPartialFrontMatterWarningDisabled?: boolean;
+  removeContentTitle?: boolean;
+  metadataPath?: (filePath: string) => string;
+  createAssets?: (metadata: {
+    filePath: string;
+    frontMatter: {[key: string]: unknown};
+  }) => {[key: string]: unknown};
+  resolveMarkdownLink?: ResolveMarkdownLink;
+
+  // Will usually be created by "createMDXLoaderItem"
+  processors?: SimpleProcessors;
+  crossCompilerCache?: Map<string, CrossCompilerCacheEntry>; // MDX => Promise<JSX> cache
+};
+
+type CrossCompilerCacheEntry = PromiseWithResolvers<string>;

package/src/preprocessor.ts

@@ -11,7 +11,7 @@ import {
   admonitionTitleToDirectiveLabel,
 } from '@docusaurus/utils';
 import {normalizeAdmonitionOptions} from './remark/admonitions';
-import type {Options} from './loader';
+import type {Options} from './options';
 
 /**
  * Preprocess the string before passing it to MDX

package/src/processor.ts

@@ -20,7 +20,7 @@ import codeCompatPlugin from './remark/mdx1Compat/codeCompatPlugin';
 import {getFormat} from './format';
 import type {WebpackCompilerName} from '@docusaurus/utils';
 import type {MDXFrontMatter} from './frontMatter';
-import type {Options} from './loader';
+import type {Options} from './options';
 import type {AdmonitionOptions} from './remark/admonitions';
 
 // @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
@@ -31,10 +31,13 @@ import type {ProcessorOptions} from '@mdx-js/mdx';
 // See https://github.com/microsoft/TypeScript/issues/49721#issuecomment-1517839391
 type Pluggable = any; // TODO fix this asap
 
-type SimpleProcessorResult = {content: string; data: {[key: string]: unknown}};
+export type SimpleProcessorResult = {
+  content: string;
+  data: {[key: string]: unknown};
+};
 
 // TODO alt interface because impossible to import type Processor (ESM + TS :/)
-type SimpleProcessor = {
+export type SimpleProcessor = {
   process: ({
     content,
     filePath,
@@ -219,63 +222,64 @@ export async function createProcessorUncached(parameters: {
 }
 
 // We use different compilers depending on the file type (md vs mdx)
-type ProcessorsCacheEntry = {
+export type SimpleProcessors = {
   mdProcessor: SimpleProcessor;
   mdxProcessor: SimpleProcessor;
 };
 
 // Compilers are cached so that Remark/Rehype plugins can run
 // expensive code during initialization
-const ProcessorsCache = new Map<string | Options, ProcessorsCacheEntry>();
+const ProcessorsCache = new Map<string | Options, SimpleProcessors>();
 
-async function createProcessorsCacheEntry({
-  query,
-  reqOptions,
+export async function createProcessors({
+  options,
 }: {
-  query: string | Options;
-  reqOptions: Options;
-}): Promise<ProcessorsCacheEntry> {
+  options: Options;
+}): Promise<SimpleProcessors> {
   const {createProcessorSync} = await createProcessorFactory();
-
-  const compilers = ProcessorsCache.get(query);
-  if (compilers) {
-    return compilers;
-  }
-
-  const compilerCacheEntry: ProcessorsCacheEntry = {
+  return {
     mdProcessor: createProcessorSync({
-      options: reqOptions,
+      options,
       format: 'md',
     }),
     mdxProcessor: createProcessorSync({
-      options: reqOptions,
+      options,
       format: 'mdx',
     }),
   };
+}
 
-  ProcessorsCache.set(query, compilerCacheEntry);
-
-  return compilerCacheEntry;
+async function createProcessorsCacheEntry({
+  options,
+}: {
+  options: Options;
+}): Promise<SimpleProcessors> {
+  const compilers = ProcessorsCache.get(options);
+  if (compilers) {
+    return compilers;
+  }
+  const processors = await createProcessors({options});
+  ProcessorsCache.set(options, processors);
+  return processors;
 }
 
-export async function createProcessorCached({
+export async function getProcessor({
   filePath,
   mdxFrontMatter,
-  query,
-  reqOptions,
+  options,
 }: {
   filePath: string;
   mdxFrontMatter: MDXFrontMatter;
-  query: string | Options;
-  reqOptions: Options;
+  options: Options;
 }): Promise<SimpleProcessor> {
-  const compilers = await createProcessorsCacheEntry({query, reqOptions});
+  const processors =
+    options.processors ?? (await createProcessorsCacheEntry({options}));
 
   const format = getFormat({
     filePath,
     frontMatterFormat: mdxFrontMatter.format,
-    markdownConfigFormat: reqOptions.markdownConfig.format,
+    markdownConfigFormat: options.markdownConfig.format,
   });
 
-  return format === 'md' ? compilers.mdProcessor : compilers.mdxProcessor;
+  return format === 'md' ? processors.mdProcessor : processors.mdxProcessor;
 }

package/src/remark/transformImage/index.ts

@@ -13,8 +13,8 @@ import {
   toMessageRelativeFilePath,
   posixPath,
   escapePath,
-  getFileLoaderUtils,
   findAsyncSequential,
+  getFileLoaderUtils,
 } from '@docusaurus/utils';
 import escapeHtml from 'escape-html';
 import sizeOf from 'image-size';
@@ -27,10 +27,6 @@ import type {MdxJsxTextElement} from 'mdast-util-mdx';
 import type {Image} from 'mdast';
 import type {Parent} from 'unist';
 
-const {
-  loaders: {inlineMarkdownImageFileLoader},
-} = getFileLoaderUtils();
-
 type PluginOptions = {
   staticDirs: string[];
   siteDir: string;
@@ -38,6 +34,7 @@ type PluginOptions = {
 
 type Context = PluginOptions & {
   filePath: string;
+  inlineMarkdownImageFileLoader: string;
 };
 
 type Target = [node: Image, index: number, parent: Parent];
@@ -45,21 +42,21 @@ type Target = [node: Image, index: number, parent: Parent];
 async function toImageRequireNode(
   [node]: Target,
   imagePath: string,
-  filePath: string,
+  context: Context,
 ) {
   // MdxJsxTextElement => see https://github.com/facebook/docusaurus/pull/8288#discussion_r1125871405
   const jsxNode = node as unknown as MdxJsxTextElement;
   const attributes: MdxJsxTextElement['attributes'] = [];
 
   let relativeImagePath = posixPath(
-    path.relative(path.dirname(filePath), imagePath),
+    path.relative(path.dirname(context.filePath), imagePath),
   );
   relativeImagePath = `./${relativeImagePath}`;
 
   const parsedUrl = url.parse(node.url);
   const hash = parsedUrl.hash ?? '';
   const search = parsedUrl.search ?? '';
-  const requireString = `${inlineMarkdownImageFileLoader}${
+  const requireString = `${context.inlineMarkdownImageFileLoader}${
     escapePath(relativeImagePath) + search
   }`;
   if (node.alt) {
@@ -186,21 +183,26 @@ async function processImageNode(target: Target, context: Context) {
   // We try to convert image urls without protocol to images with require calls
   // going through webpack ensures that image assets exist at build time
   const imagePath = await getImageAbsolutePath(parsedUrl.pathname, context);
-  await toImageRequireNode(target, imagePath, context.filePath);
+  await toImageRequireNode(target, imagePath, context);
 }
 
 export default function plugin(options: PluginOptions): Transformer {
   return async (root, vfile) => {
     const {visit} = await import('unist-util-visit');
 
+    const fileLoaderUtils = getFileLoaderUtils(
+      vfile.data.compilerName === 'server',
+    );
+    const context: Context = {
+      ...options,
+      filePath: vfile.path!,
+      inlineMarkdownImageFileLoader:
+        fileLoaderUtils.loaders.inlineMarkdownImageFileLoader,
+    };
+
     const promises: Promise<void>[] = [];
     visit(root, 'image', (node: Image, index, parent) => {
-      promises.push(
-        processImageNode([node, index, parent!], {
-          ...options,
-          filePath: vfile.path!,
-        }),
-      );
+      promises.push(processImageNode([node, index, parent!], context));
     });
     await Promise.all(promises);
   };