@docusaurus/mdx-loader 2.4.1 → 3.0.0-beta.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@docusaurus/mdx-loader",
-  "version": "2.4.1",
+  "version": "3.0.0-beta.0",
   "description": "Docusaurus Loader for MDX",
   "main": "lib/index.js",
   "types": "lib/index.d.ts",
@@ -18,44 +18,52 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@babel/parser": "^7.18.8",
-    "@babel/traverse": "^7.18.8",
-    "@docusaurus/logger": "2.4.1",
-    "@docusaurus/utils": "2.4.1",
-    "@mdx-js/mdx": "^1.6.22",
+    "@babel/parser": "^7.22.7",
+    "@babel/traverse": "^7.22.8",
+    "@docusaurus/logger": "3.0.0-beta.0",
+    "@docusaurus/utils": "3.0.0-beta.0",
+    "@docusaurus/utils-validation": "3.0.0-beta.0",
+    "@mdx-js/mdx": "^2.1.5",
+    "@slorber/remark-comment": "^1.0.0",
     "escape-html": "^1.0.3",
+    "estree-util-value-to-estree": "^2.1.0",
     "file-loader": "^6.2.0",
-    "fs-extra": "^10.1.0",
-    "image-size": "^1.0.1",
-    "mdast-util-to-string": "^2.0.0",
+    "fs-extra": "^11.1.1",
+    "hastscript": "^7.1.0",
+    "image-size": "^1.0.2",
+    "mdast-util-mdx": "^2.0.0",
+    "mdast-util-to-string": "^3.2.0",
+    "rehype-raw": "^6.1.1",
+    "remark-directive": "^2.0.1",
     "remark-emoji": "^2.2.0",
+    "remark-gfm": "^3.0.1",
     "stringify-object": "^3.3.0",
-    "tslib": "^2.4.0",
-    "unified": "^9.2.2",
+    "tslib": "^2.6.0",
+    "unified": "^10.1.2",
     "unist-util-visit": "^2.0.3",
     "url-loader": "^4.1.1",
-    "webpack": "^5.73.0"
+    "webpack": "^5.88.1"
   },
   "devDependencies": {
-    "@docusaurus/types": "2.4.1",
+    "@docusaurus/types": "3.0.0-beta.0",
     "@types/escape-html": "^1.0.2",
-    "@types/mdast": "^3.0.10",
+    "@types/mdast": "^3.0.12",
     "@types/stringify-object": "^3.3.1",
-    "@types/unist": "^2.0.6",
+    "@types/unist": "^2.0.7",
     "rehype-stringify": "^8.0.0",
-    "remark": "^12.0.1",
-    "remark-mdx": "^1.6.21",
+    "remark": "^14.0.3",
+    "remark-mdx": "^2.1.5",
     "remark-rehype": "^8.1.0",
     "to-vfile": "^6.1.0",
     "unist-builder": "^2.0.3",
     "unist-util-remove-position": "^3.0.0"
   },
   "peerDependencies": {
-    "react": "^16.8.4 || ^17.0.0",
-    "react-dom": "^16.8.4 || ^17.0.0"
+    "react": "^18.0.0",
+    "react-dom": "^18.0.0"
   },
   "engines": {
     "node": ">=16.14"
   },
-  "gitHead": "60e657d8ae5a4a9ed1c2d777f9defd882cc12681"
+  "gitHead": "27a1e90d9fff88af90ecad35bea16d4d7230482a"
 }

package/src/format.ts

@@ -0,0 +1,50 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import path from 'path';
+import type {MDXFrontMatter} from './frontMatter';
+import type {Format, FormatInput} from './index';
+
+// Copied from https://mdxjs.com/packages/mdx/#optionsmdextensions
+// Although we are likely to only use .md / .mdx anyway...
+const mdFormatExtensions = [
+  '.md',
+  '.markdown',
+  '.mdown',
+  '.mkdn',
+  '.mkd',
+  '.mdwn',
+  '.mkdown',
+  '.ron',
+];
+
+function getExtensionFormat(filepath: string): Format {
+  const isMDFormat = mdFormatExtensions.includes(path.extname(filepath));
+  // Bias toward mdx if unknown extension
+  return isMDFormat ? 'md' : 'mdx';
+}
+
+export function getFormat({
+  filePath,
+  frontMatterFormat,
+  markdownConfigFormat,
+}: {
+  filePath: string;
+  frontMatterFormat: MDXFrontMatter['format'];
+  markdownConfigFormat: FormatInput;
+}): Format {
+  if (frontMatterFormat) {
+    if (frontMatterFormat !== 'detect') {
+      return frontMatterFormat;
+    }
+    return getExtensionFormat(filePath);
+  }
+  if (markdownConfigFormat !== 'detect') {
+    return markdownConfigFormat;
+  }
+  return getExtensionFormat(filePath);
+}

package/src/frontMatter.ts

@@ -0,0 +1,31 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {
+  JoiFrontMatter as Joi,
+  validateFrontMatter,
+} from '@docusaurus/utils-validation';
+
+import type {FormatInput} from './index';
+
+export type MDXFrontMatter = {
+  format?: FormatInput;
+};
+
+export const DefaultMDXFrontMatter: MDXFrontMatter = {
+  format: undefined,
+};
+
+const MDXFrontMatterSchema = Joi.object<MDXFrontMatter>({
+  format: Joi.string().equal('md', 'mdx', 'detect').optional(),
+}).default(DefaultMDXFrontMatter);
+
+export function validateMDXFrontMatter(frontMatter: unknown): MDXFrontMatter {
+  return validateFrontMatter(frontMatter, MDXFrontMatterSchema, {
+    allowUnknown: false,
+  });
+}

package/src/index.ts

@@ -7,13 +7,15 @@
 
 import {mdxLoader} from './loader';
 
+import type {TOCItem as TOCItemImported} from './remark/toc';
+
 export default mdxLoader;
 
-export type TOCItem = {
-  readonly value: string;
-  readonly id: string;
-  readonly level: number;
-};
+export type TOCItem = TOCItemImported;
+
+export type Format = 'md' | 'mdx';
+
+export type FormatInput = Format | 'detect';
 
 export type LoadedMDXContent<FrontMatter, Metadata, Assets = undefined> = {
   /** As verbatim declared in the MDX document. */
@@ -32,4 +34,6 @@ export type LoadedMDXContent<FrontMatter, Metadata, Assets = undefined> = {
   readonly assets: Assets;
   (): JSX.Element;
 };
-export type {Options, MDXPlugin, MDXOptions} from './loader';
+
+export type {Options, MDXPlugin} from './loader';
+export type {MDXOptions} from './processor';

package/src/loader.ts

@@ -13,54 +13,25 @@ import {
   escapePath,
   getFileLoaderUtils,
 } from '@docusaurus/utils';
-import {createCompiler} from '@mdx-js/mdx';
-import emoji from 'remark-emoji';
 import stringifyObject from 'stringify-object';
+import preprocessor from './preprocessor';
+import {validateMDXFrontMatter} from './frontMatter';
+import {createProcessorCached} from './processor';
+import type {MDXOptions} from './processor';
 
-import headings from './remark/headings';
-import toc from './remark/toc';
-import unwrapMdxCodeBlocks from './remark/unwrapMdxCodeBlocks';
-import transformImage from './remark/transformImage';
-import transformLinks from './remark/transformLinks';
-import mermaid from './remark/mermaid';
-
-import transformAdmonitions from './remark/admonitions';
 import type {MarkdownConfig} from '@docusaurus/types';
 import type {LoaderContext} from 'webpack';
-import type {Processor, Plugin} from 'unified';
-import type {AdmonitionOptions} from './remark/admonitions';
+
+// TODO as of April 2023, no way to import/re-export this ESM type easily :/
+// This might change soon, likely after TS 5.2
+// See https://github.com/microsoft/TypeScript/issues/49721#issuecomment-1517839391
+type Pluggable = any; // TODO fix this asap
 
 const {
   loaders: {inlineMarkdownImageFileLoader},
 } = getFileLoaderUtils();
 
-const pragma = `
-/* @jsxRuntime classic */
-/* @jsx mdx */
-/* @jsxFrag React.Fragment */
-`;
-
-const DEFAULT_OPTIONS: MDXOptions = {
-  admonitions: true,
-  rehypePlugins: [],
-  remarkPlugins: [unwrapMdxCodeBlocks, emoji, headings, toc],
-  beforeDefaultRemarkPlugins: [],
-  beforeDefaultRehypePlugins: [],
-};
-
-const compilerCache = new Map<string | Options, [Processor, Options]>();
-
-export type MDXPlugin =
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  [Plugin<any[]>, any] | Plugin<any[]>;
-
-export type MDXOptions = {
-  admonitions: boolean | AdmonitionOptions;
-  remarkPlugins: MDXPlugin[];
-  rehypePlugins: MDXPlugin[];
-  beforeDefaultRemarkPlugins: MDXPlugin[];
-  beforeDefaultRehypePlugins: MDXPlugin[];
-};
+export type MDXPlugin = Pluggable;
 
 export type Options = Partial<MDXOptions> & {
   markdownConfig: MarkdownConfig;
@@ -139,17 +110,15 @@ function createAssetsExportCode(assets: unknown) {
   return `{\n${codeLines.join('\n')}\n}`;
 }
 
-function getAdmonitionsPlugins(
-  admonitionsOption: MDXOptions['admonitions'],
-): MDXPlugin[] {
-  if (admonitionsOption) {
-    const plugin: MDXPlugin =
-      admonitionsOption === true
-        ? transformAdmonitions
-        : [transformAdmonitions, admonitionsOption];
-    return [plugin];
+// TODO temporary, remove this after v3.1?
+// Some plugin authors use our mdx-loader, despite it not being public API
+// see https://github.com/facebook/docusaurus/issues/8298
+function ensureMarkdownConfig(reqOptions: Options) {
+  if (!reqOptions.markdownConfig) {
+    throw new Error(
+      'Docusaurus v3+ requires MDX loader options.markdownConfig - plugin authors using the MDX loader should make sure to provide that option',
+    );
   }
-  return [];
 }
 
 export async function mdxLoader(
@@ -158,77 +127,72 @@ export async function mdxLoader(
 ): Promise<void> {
   const callback = this.async();
   const filePath = this.resourcePath;
-  const reqOptions = this.getOptions();
+  const reqOptions: Options = this.getOptions();
+  const {query} = this;
+  ensureMarkdownConfig(reqOptions);
 
   const {frontMatter, content: contentWithTitle} = parseFrontMatter(fileString);
-
-  const {content, contentTitle} = parseMarkdownContentTitle(contentWithTitle, {
-    removeContentTitle: reqOptions.removeContentTitle,
+  const mdxFrontMatter = validateMDXFrontMatter(frontMatter.mdx);
+
+  const {content: contentUnprocessed, contentTitle} = parseMarkdownContentTitle(
+    contentWithTitle,
+    {
+      removeContentTitle: reqOptions.removeContentTitle,
+    },
+  );
+
+  const content = preprocessor({
+    fileContent: contentUnprocessed,
+    filePath,
+    admonitions: reqOptions.admonitions,
+    markdownConfig: reqOptions.markdownConfig,
   });
 
   const hasFrontMatter = Object.keys(frontMatter).length > 0;
 
-  if (!compilerCache.has(this.query)) {
-    const remarkPlugins: MDXPlugin[] = [
-      ...(reqOptions.beforeDefaultRemarkPlugins ?? []),
-      ...getAdmonitionsPlugins(reqOptions.admonitions ?? false),
-      ...DEFAULT_OPTIONS.remarkPlugins,
-      ...(reqOptions.markdownConfig.mermaid ? [mermaid] : []),
-      [
-        transformImage,
-        {
-          staticDirs: reqOptions.staticDirs,
-          siteDir: reqOptions.siteDir,
-        },
-      ],
-      [
-        transformLinks,
-        {
-          staticDirs: reqOptions.staticDirs,
-          siteDir: reqOptions.siteDir,
-        },
-      ],
-      ...(reqOptions.remarkPlugins ?? []),
-    ];
-
-    const rehypePlugins: MDXPlugin[] = [
-      ...(reqOptions.beforeDefaultRehypePlugins ?? []),
-      ...DEFAULT_OPTIONS.rehypePlugins,
-      ...(reqOptions.rehypePlugins ?? []),
-    ];
-
-    const options: Options = {
-      ...reqOptions,
-      remarkPlugins,
-      rehypePlugins,
-    };
-    compilerCache.set(this.query, [createCompiler(options), options]);
-  }
-
-  const [compiler, options] = compilerCache.get(this.query)!;
+  const processor = await createProcessorCached({
+    filePath,
+    reqOptions,
+    query,
+    mdxFrontMatter,
+  });
 
   let result: string;
   try {
-    result = await compiler
-      .process({
-        contents: content,
-        path: this.resourcePath,
-      })
-      .then((res) => res.toString());
-  } catch (err) {
-    return callback(err as Error);
+    result = await processor.process({content, filePath});
+  } catch (errorUnknown) {
+    const error = errorUnknown as Error;
+
+    // MDX can emit errors that have useful extra attributes
+    const errorJSON = JSON.stringify(error, null, 2);
+    const errorDetails =
+      errorJSON === '{}'
+        ? // regular JS error case: print stacktrace
+          error.stack ?? 'N/A'
+        : // MDX error: print extra attributes + stacktrace
+          `${errorJSON}\n${error.stack}`;
+
+    return callback(
+      new Error(
+        `MDX compilation failed for file ${logger.path(filePath)}\nCause: ${
+          error.message
+        }\nDetails:\n${errorDetails}`,
+        // TODO error cause doesn't seem to be used by Webpack stats.errors :s
+        {cause: error},
+      ),
+    );
   }
 
   // MDX partials are MDX files starting with _ or in a folder starting with _
   // Partial are not expected to have associated metadata files or front matter
-  const isMDXPartial = options.isMDXPartial?.(filePath);
+  const isMDXPartial = reqOptions.isMDXPartial?.(filePath);
   if (isMDXPartial && hasFrontMatter) {
     const errorMessage = `Docusaurus MDX partial files should not contain front matter.
 Those partial files use the _ prefix as a convention by default, but this is configurable.
 File at ${filePath} contains front matter that will be ignored:
 ${JSON.stringify(frontMatter, null, 2)}`;
 
-    if (!options.isMDXPartialFrontMatterWarningDisabled) {
+    if (!reqOptions.isMDXPartialFrontMatterWarningDisabled) {
       const shouldError = process.env.NODE_ENV === 'test' || process.env.CI;
       if (shouldError) {
         return callback(new Error(errorMessage));
@@ -240,8 +204,11 @@ ${JSON.stringify(frontMatter, null, 2)}`;
   function getMetadataPath(): string | undefined {
     if (!isMDXPartial) {
       // Read metadata for this MDX and export it.
-      if (options.metadataPath && typeof options.metadataPath === 'function') {
-        return options.metadataPath(filePath);
+      if (
+        reqOptions.metadataPath &&
+        typeof reqOptions.metadataPath === 'function'
+      ) {
+        return reqOptions.metadataPath(filePath);
       }
     }
     return undefined;
@@ -265,6 +232,8 @@ ${JSON.stringify(frontMatter, null, 2)}`;
       ? reqOptions.createAssets({frontMatter, metadata})
       : undefined;
 
+  // TODO use remark plugins to insert extra exports instead of string concat?
+  // cf how the toc is exported
   const exportsCode = `
 export const frontMatter = ${stringifyObject(frontMatter)};
 export const contentTitle = ${stringifyObject(contentTitle)};
@@ -273,10 +242,6 @@ ${assets ? `export const assets = ${createAssetsExportCode(assets)};` : ''}
 `;
 
   const code = `
-${pragma}
-import React from 'react';
-import { mdx } from '@mdx-js/react';
-
 ${exportsCode}
 ${result}
 `;

package/src/preprocessor.ts

@@ -0,0 +1,47 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {
+  escapeMarkdownHeadingIds,
+  unwrapMdxCodeBlocks,
+  admonitionTitleToDirectiveLabel,
+} from '@docusaurus/utils';
+import {normalizeAdmonitionOptions} from './remark/admonitions';
+import type {Options} from './loader';
+
+/**
+ * Preprocess the string before passing it to MDX
+ * This is not particularly recommended but makes it easier to upgrade to MDX 2
+ */
+export default function preprocessContent({
+  fileContent: initialFileContent,
+  filePath,
+  markdownConfig,
+  admonitions,
+}: {
+  fileContent: string;
+  filePath: string;
+  markdownConfig: Options['markdownConfig'];
+  admonitions: Options['admonitions'] | undefined;
+}): string {
+  let fileContent = initialFileContent;
+  if (markdownConfig.preprocessor) {
+    fileContent = markdownConfig.preprocessor({
+      fileContent,
+      filePath,
+    });
+  }
+  fileContent = unwrapMdxCodeBlocks(fileContent);
+  if (markdownConfig.mdx1Compat.headingIds) {
+    fileContent = escapeMarkdownHeadingIds(fileContent);
+  }
+  if (markdownConfig.mdx1Compat.admonitions && admonitions) {
+    const {keywords} = normalizeAdmonitionOptions(admonitions);
+    fileContent = admonitionTitleToDirectiveLabel(fileContent, keywords);
+  }
+  return fileContent;
+}