@docusaurus/mdx-loader 2.4.1 → 3.0.0-beta.0

package/src/processor.ts

@@ -0,0 +1,243 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import emoji from 'remark-emoji';
+import headings from './remark/headings';
+import toc from './remark/toc';
+import transformImage from './remark/transformImage';
+import transformLinks from './remark/transformLinks';
+import details from './remark/details';
+import head from './remark/head';
+import mermaid from './remark/mermaid';
+import transformAdmonitions from './remark/admonitions';
+import codeCompatPlugin from './remark/mdx1Compat/codeCompatPlugin';
+import {getFormat} from './format';
+import type {MDXFrontMatter} from './frontMatter';
+import type {Options} from './loader';
+import type {AdmonitionOptions} from './remark/admonitions';
+
+// @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
+import type {ProcessorOptions} from '@mdx-js/mdx';
+
+// TODO as of April 2023, no way to import/re-export this ESM type easily :/
+// This might change soon, likely after TS 5.2
+// See https://github.com/microsoft/TypeScript/issues/49721#issuecomment-1517839391
+type Pluggable = any; // TODO fix this asap
+
+// TODO alt interface because impossible to import type Processor (ESM + TS :/)
+type SimpleProcessor = {
+  process: ({
+    content,
+    filePath,
+  }: {
+    content: string;
+    filePath: string;
+  }) => Promise<string>;
+};
+
+const DEFAULT_OPTIONS: MDXOptions = {
+  admonitions: true,
+  rehypePlugins: [],
+  remarkPlugins: [headings, emoji, toc],
+  beforeDefaultRemarkPlugins: [],
+  beforeDefaultRehypePlugins: [],
+};
+
+export type MDXPlugin = Pluggable;
+
+export type MDXOptions = {
+  admonitions: boolean | Partial<AdmonitionOptions>;
+  remarkPlugins: MDXPlugin[];
+  rehypePlugins: MDXPlugin[];
+  beforeDefaultRemarkPlugins: MDXPlugin[];
+  beforeDefaultRehypePlugins: MDXPlugin[];
+};
+
+function getAdmonitionsPlugins(
+  admonitionsOption: MDXOptions['admonitions'],
+): MDXPlugin[] {
+  if (admonitionsOption) {
+    const plugin: MDXPlugin =
+      admonitionsOption === true
+        ? transformAdmonitions
+        : [transformAdmonitions, admonitionsOption];
+    return [plugin];
+  }
+
+  return [];
+}
+
+// Need to be async due to ESM dynamic imports...
+async function createProcessorFactory() {
+  const {createProcessor: createMdxProcessor} = await import('@mdx-js/mdx');
+  const {default: rehypeRaw} = await import('rehype-raw');
+  const {default: gfm} = await import('remark-gfm');
+  // TODO using fork until PR merged: https://github.com/leebyron/remark-comment/pull/3
+  const {default: comment} = await import('@slorber/remark-comment');
+  const {default: directive} = await import('remark-directive');
+
+  // /!\ this method is synchronous on purpose
+  // Using async code here can create cache entry race conditions!
+  function createProcessorSync({
+    options,
+    format,
+  }: {
+    options: Options;
+    format: 'md' | 'mdx';
+  }): SimpleProcessor {
+    const remarkPlugins: MDXPlugin[] = [
+      ...(options.beforeDefaultRemarkPlugins ?? []),
+      directive,
+      ...getAdmonitionsPlugins(options.admonitions ?? false),
+      ...DEFAULT_OPTIONS.remarkPlugins,
+      details,
+      head,
+      ...(options.markdownConfig.mermaid ? [mermaid] : []),
+      [
+        transformImage,
+        {
+          staticDirs: options.staticDirs,
+          siteDir: options.siteDir,
+        },
+      ],
+      [
+        transformLinks,
+        {
+          staticDirs: options.staticDirs,
+          siteDir: options.siteDir,
+        },
+      ],
+      gfm,
+      options.markdownConfig.mdx1Compat.comments ? comment : null,
+      ...(options.remarkPlugins ?? []),
+    ].filter((plugin): plugin is MDXPlugin => Boolean(plugin));
+
+    // codeCompatPlugin needs to be applied last after user-provided plugins
+    // (after npm2yarn for example)
+    remarkPlugins.push(codeCompatPlugin);
+
+    const rehypePlugins: MDXPlugin[] = [
+      ...(options.beforeDefaultRehypePlugins ?? []),
+      ...DEFAULT_OPTIONS.rehypePlugins,
+      ...(options.rehypePlugins ?? []),
+    ];
+
+    if (format === 'md') {
+      // This is what permits to embed HTML elements with format 'md'
+      // See https://github.com/facebook/docusaurus/pull/8960
+      // See https://github.com/mdx-js/mdx/pull/2295#issuecomment-1540085960
+      const rehypeRawPlugin: MDXPlugin = [
+        rehypeRaw,
+        {
+          passThrough: [
+            'mdxFlowExpression',
+            'mdxJsxFlowElement',
+            'mdxJsxTextElement',
+            'mdxTextExpression',
+            'mdxjsEsm',
+          ],
+        },
+      ];
+      rehypePlugins.unshift(rehypeRawPlugin);
+    }
+
+    const processorOptions: ProcessorOptions & Options = {
+      ...options,
+      remarkPlugins,
+      rehypePlugins,
+      providerImportSource: '@mdx-js/react',
+    };
+
+    const mdxProcessor = createMdxProcessor({
+      ...processorOptions,
+      format,
+    });
+
+    return {
+      process: async ({content, filePath}) =>
+        mdxProcessor
+          .process({
+            value: content,
+            path: filePath,
+          })
+          .then((res) => res.toString()),
+    };
+  }
+
+  return {createProcessorSync};
+}
+
+// Will be useful for tests
+export async function createProcessorUncached(parameters: {
+  options: Options;
+  format: 'md' | 'mdx';
+}): Promise<SimpleProcessor> {
+  const {createProcessorSync} = await createProcessorFactory();
+  return createProcessorSync(parameters);
+}
+
+// We use different compilers depending on the file type (md vs mdx)
+type ProcessorsCacheEntry = {
+  mdProcessor: SimpleProcessor;
+  mdxProcessor: SimpleProcessor;
+};
+
+// Compilers are cached so that Remark/Rehype plugins can run
+// expensive code during initialization
+const ProcessorsCache = new Map<string | Options, ProcessorsCacheEntry>();
+
+async function createProcessorsCacheEntry({
+  query,
+  reqOptions,
+}: {
+  query: string | Options;
+  reqOptions: Options;
+}): Promise<ProcessorsCacheEntry> {
+  const {createProcessorSync} = await createProcessorFactory();
+
+  const compilers = ProcessorsCache.get(query);
+  if (compilers) {
+    return compilers;
+  }
+
+  const compilerCacheEntry: ProcessorsCacheEntry = {
+    mdProcessor: createProcessorSync({
+      options: reqOptions,
+      format: 'md',
+    }),
+    mdxProcessor: createProcessorSync({
+      options: reqOptions,
+      format: 'mdx',
+    }),
+  };
+
+  ProcessorsCache.set(query, compilerCacheEntry);
+
+  return compilerCacheEntry;
+}
+
+export async function createProcessorCached({
+  filePath,
+  mdxFrontMatter,
+  query,
+  reqOptions,
+}: {
+  filePath: string;
+  mdxFrontMatter: MDXFrontMatter;
+  query: string | Options;
+  reqOptions: Options;
+}): Promise<SimpleProcessor> {
+  const compilers = await createProcessorsCacheEntry({query, reqOptions});
+
+  const format = getFormat({
+    filePath,
+    frontMatterFormat: mdxFrontMatter.format,
+    markdownConfigFormat: reqOptions.markdownConfig.format,
+  });
+
+  return format === 'md' ? compilers.mdProcessor : compilers.mdxProcessor;
+}

package/src/remark/admonitions/README.md

@@ -1,3 +1,3 @@
 # Docusaurus admonitions
 
-Code from [remark-admonitions](https://github.com/remarkjs/remark-directive) (MIT license) has been copied to this folder, and highly customized for Docusaurus needs.
+Code from [remark-admonitions](https://github.com/elviswolcott/remark-admonitions) (MIT license) has been copied to this folder, and highly customized for Docusaurus needs.

package/src/remark/admonitions/index.ts

@@ -4,20 +4,26 @@
  * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  */
-
 import visit from 'unist-util-visit';
-import type {Transformer, Processor, Plugin} from 'unified';
-import type {Literal} from 'mdast';
+// @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
+import type {Transformer, Processor} from 'unified';
+
+// @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
+import type {ContainerDirective} from 'mdast-util-directive';
+import type {Parent} from 'mdast';
 
-const NEWLINE = '\n';
+// TODO as of April 2023, no way to import/re-export this ESM type easily :/
+// This might change soon, likely after TS 5.2
+// See https://github.com/microsoft/TypeScript/issues/49721#issuecomment-1517839391
+// import type {Plugin} from 'unified';
+type Plugin = any; // TODO fix this asap
 
 export type AdmonitionOptions = {
-  tag: string;
   keywords: string[];
+  extendDefaults: boolean;
 };
 
 export const DefaultAdmonitionOptions: AdmonitionOptions = {
-  tag: ':::',
   keywords: [
     'secondary',
     'info',
@@ -29,193 +35,108 @@ export const DefaultAdmonitionOptions: AdmonitionOptions = {
     'important',
     'caution',
   ],
+  extendDefaults: true,
 };
 
-function escapeRegExp(s: string): string {
-  return s.replace(/[-[\]{}()*+?.\\^$|/]/g, '\\$&');
+export function normalizeAdmonitionOptions(
+  providedOptions: Partial<AdmonitionOptions> | true,
+): AdmonitionOptions {
+  if (providedOptions === true) {
+    return DefaultAdmonitionOptions;
+  }
+
+  const options = {...DefaultAdmonitionOptions, ...providedOptions};
+
+  // By default it makes more sense to append keywords to the default ones
+  // Adding custom keywords is more common than disabling existing ones
+  if (options.extendDefaults) {
+    options.keywords = [
+      ...DefaultAdmonitionOptions.keywords,
+      ...options.keywords,
+    ];
+  }
+
+  return options;
 }
 
-function normalizeOptions(
-  options: Partial<AdmonitionOptions>,
-): AdmonitionOptions {
-  return {...DefaultAdmonitionOptions, ...options};
+type DirectiveLabel = Parent;
+type DirectiveContent = ContainerDirective['children'];
+
+function parseDirective(directive: ContainerDirective): {
+  directiveLabel: DirectiveLabel | undefined;
+  contentNodes: DirectiveContent;
+} {
+  const hasDirectiveLabel =
+    directive.children?.[0]?.data?.directiveLabel === true;
+  if (hasDirectiveLabel) {
+    const [directiveLabel, ...contentNodes] = directive.children;
+    return {directiveLabel: directiveLabel as DirectiveLabel, contentNodes};
+  }
+  return {directiveLabel: undefined, contentNodes: directive.children};
 }
 
-// This string value does not matter much
-// It is ignored because nodes are using hName/hProperties coming from HAST
-const admonitionNodeType = 'admonitionHTML';
+function getTextOnlyTitle(directiveLabel: DirectiveLabel): string | undefined {
+  const isTextOnlyTitle =
+    directiveLabel?.children?.length === 1 &&
+    directiveLabel?.children?.[0]?.type === 'text';
+  return isTextOnlyTitle
+    ? // @ts-expect-error: todo type
+      (directiveLabel?.children?.[0].value as string)
+    : undefined;
+}
 
 const plugin: Plugin = function plugin(
   this: Processor,
   optionsInput: Partial<AdmonitionOptions> = {},
 ): Transformer {
-  const options = normalizeOptions(optionsInput);
-
-  const keywords = Object.values(options.keywords).map(escapeRegExp).join('|');
-  const nestingChar = escapeRegExp(options.tag.slice(0, 1));
-  const tag = escapeRegExp(options.tag);
-
-  // resolve th nesting level of an opening tag
-  // ::: -> 0, :::: -> 1, ::::: -> 2 ...
-  const nestingLevelRegex = new RegExp(
-    `^${tag}(?<nestingLevel>${nestingChar}*)`,
-  );
-
-  const regex = new RegExp(`${tag}${nestingChar}*(${keywords})(?: *(.*))?\n`);
-  const escapeTag = new RegExp(
-    escapeRegExp(`\\${options.tag}${options.tag.slice(0, 1)}*`),
-    'g',
-  );
-
-  // The tokenizer is called on blocks to determine if there is an admonition
-  // present and create tags for it
-  function blockTokenizer(this: any, eat: any, value: string, silent: boolean) {
-    // Stop if no match or match does not start at beginning of line
-    const match = regex.exec(value);
-    if (!match || match.index !== 0) {
-      return false;
-    }
-    // If silent return the match
-    if (silent) {
-      return true;
-    }
-
-    const now = eat.now();
-    const [opening, keyword, title] = match as string[] as [
-      string,
-      string,
-      string,
-    ];
-    const food = [];
-    const content = [];
-    // get the nesting level of the opening tag
-    const openingLevel =
-      nestingLevelRegex.exec(opening)!.groups!.nestingLevel!.length;
-    // used as a stack to keep track of nested admonitions
-    const nestingLevels: number[] = [openingLevel];
-
-    let newValue = value;
-    // consume lines until a closing tag
-    let idx = newValue.indexOf(NEWLINE);
-    while (idx !== -1) {
-      // grab this line and eat it
-      const next = newValue.indexOf(NEWLINE, idx + 1);
-      const line =
-        next !== -1 ? newValue.slice(idx + 1, next) : newValue.slice(idx + 1);
-      food.push(line);
-      newValue = newValue.slice(idx + 1);
-      const nesting = nestingLevelRegex.exec(line);
-      idx = newValue.indexOf(NEWLINE);
-      if (!nesting) {
-        content.push(line);
-        continue;
-      }
-      const tagLevel = nesting.groups!.nestingLevel!.length;
-      // first level
-      if (nestingLevels.length === 0) {
-        nestingLevels.push(tagLevel);
-        content.push(line);
-        continue;
-      }
-      const currentLevel = nestingLevels[nestingLevels.length - 1]!;
-      if (tagLevel < currentLevel) {
-        // entering a nested admonition block
-        nestingLevels.push(tagLevel);
-      } else if (tagLevel === currentLevel) {
-        // closing a nested admonition block
-        nestingLevels.pop();
-        // the closing tag is NOT part of the content
-        if (nestingLevels.length === 0) {
-          break;
-        }
-      }
-      content.push(line);
-    }
+  const {keywords} = normalizeAdmonitionOptions(optionsInput);
 
-    // consume the processed tag and replace escape sequences
-    const contentString = content.join(NEWLINE).replace(escapeTag, options.tag);
-    const add = eat(opening + food.join(NEWLINE));
+  return async (root) => {
+    visit(root, (node) => {
+      if (node.type === 'containerDirective') {
+        const directive = node as ContainerDirective;
+        const isAdmonition = keywords.includes(directive.name);
 
-    // parse the content in block mode
-    const exit = this.enterBlock();
-    const contentNodes = this.tokenizeBlock(contentString, now);
-    exit();
+        if (!isAdmonition) {
+          return;
+        }
 
-    const titleNodes = this.tokenizeInline(title, now);
+        const {directiveLabel, contentNodes} = parseDirective(directive);
 
-    const isSimpleTextTitle =
-      titleNodes.length === 1 && titleNodes[0].type === 'text';
+        const textOnlyTitle =
+          directive.attributes?.title ??
+          (directiveLabel ? getTextOnlyTitle(directiveLabel) : undefined);
 
-    const element = {
-      type: admonitionNodeType,
-      data: {
-        // hName/hProperties come from HAST
+        // Transform the mdast directive node to a hast admonition node
         // See https://github.com/syntax-tree/mdast-util-to-hast#fields-on-nodes
-        hName: 'admonition',
-        hProperties: {
-          ...(title && isSimpleTextTitle && {title}),
-          type: keyword,
-        },
-      },
-      children: [
-        // For titles containing MDX syntax: create a custom element. The theme
-        // component will extract it and render it nicely.
-        //
-        // Temporary workaround, because it's complex in MDX v1 to emit
-        // interpolated JSX prop syntax (title={<>my <code>title</code></>}).
-        // For this reason, we use children instead of the title prop.
-        title &&
-          !isSimpleTextTitle && {
-            type: admonitionNodeType,
+        // TODO in MDX v2 we should transform the whole directive to
+        // mdxJsxFlowElement instead of using hast
+        directive.data = {
+          hName: 'admonition',
+          hProperties: {
+            ...(textOnlyTitle && {title: textOnlyTitle}),
+            type: directive.name,
+          },
+        };
+        directive.children = contentNodes;
+
+        // TODO legacy MDX v1 <mdxAdmonitionTitle> workaround
+        // v1: not possible to inject complex JSX elements as props
+        // v2: now possible: use a mdxJsxFlowElement element
+        if (directiveLabel && !textOnlyTitle) {
+          const complexTitleNode = {
+            type: 'mdxAdmonitionTitle',
             data: {
               hName: 'mdxAdmonitionTitle',
               hProperties: {},
             },
-            children: titleNodes,
-          },
-        ...contentNodes,
-      ].filter(Boolean),
-    };
-
-    return add(element);
-  }
-
-  // add tokenizer to parser after fenced code blocks
-  const Parser = this.Parser.prototype;
-  Parser.blockTokenizers.admonition = blockTokenizer;
-  Parser.blockMethods.splice(
-    Parser.blockMethods.indexOf('fencedCode') + 1,
-    0,
-    'admonition',
-  );
-  Parser.interruptParagraph.splice(
-    Parser.interruptParagraph.indexOf('fencedCode') + 1,
-    0,
-    ['admonition'],
-  );
-  Parser.interruptList.splice(
-    Parser.interruptList.indexOf('fencedCode') + 1,
-    0,
-    ['admonition'],
-  );
-  Parser.interruptBlockquote.splice(
-    Parser.interruptBlockquote.indexOf('fencedCode') + 1,
-    0,
-    ['admonition'],
-  );
-
-  return (root) => {
-    // escape everything except admonitionHTML nodes
-    visit(
-      root,
-      (node: unknown): node is Literal =>
-        (node as Literal | undefined)?.type !== admonitionNodeType,
-      (node: Literal) => {
-        if (node.value) {
-          node.value = node.value.replace(escapeTag, options.tag);
+            children: directiveLabel.children,
+          };
+          // @ts-expect-error: invented node type
+          directive.children.unshift(complexTitleNode);
         }
-      },
-    );
+      }
+    });
   };
 };
 

package/src/remark/details/index.ts

@@ -0,0 +1,25 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import visit from 'unist-util-visit';
+// @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
+import type {Transformer} from 'unified';
+
+// @ts-expect-error: ES support...
+import type {MdxJsxFlowElement} from 'mdast-util-mdx';
+
+// Transform <details> to <Details>
+// MDX 2 doesn't allow to substitute html elements with the provider anymore
+export default function plugin(): Transformer {
+  return (root) => {
+    visit(root, 'mdxJsxFlowElement', (node: MdxJsxFlowElement) => {
+      if (node.name === 'details') {
+        node.name = 'Details';
+      }
+    });
+  };
+}

package/src/remark/head/index.ts

@@ -0,0 +1,25 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import visit from 'unist-util-visit';
+// @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
+import type {Transformer} from 'unified';
+
+// @ts-expect-error: ES support...
+import type {MdxJsxFlowElement} from 'mdast-util-mdx';
+
+// Transform <head> to <Head>
+// MDX 2 doesn't allow to substitute html elements with the provider anymore
+export default function plugin(): Transformer {
+  return (root) => {
+    visit(root, 'mdxJsxFlowElement', (node: MdxJsxFlowElement) => {
+      if (node.name === 'head') {
+        node.name = 'Head';
+      }
+    });
+  };
+}

package/src/remark/headings/index.ts

@@ -9,12 +9,14 @@
 
 import {parseMarkdownHeadingId, createSlugger} from '@docusaurus/utils';
 import visit from 'unist-util-visit';
-import mdastToString from 'mdast-util-to-string';
+// @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
 import type {Transformer} from 'unified';
 import type {Heading, Text} from 'mdast';
 
 export default function plugin(): Transformer {
-  return (root) => {
+  return async (root) => {
+    const {toString} = await import('mdast-util-to-string');
+
     const slugs = createSlugger();
     visit(root, 'heading', (headingNode: Heading) => {
       const data = headingNode.data ?? (headingNode.data = {});
@@ -29,7 +31,7 @@ export default function plugin(): Transformer {
         const headingTextNodes = headingNode.children.filter(
           ({type}) => !['html', 'jsx'].includes(type),
         );
-        const heading = mdastToString(
+        const heading = toString(
           headingTextNodes.length > 0 ? headingTextNodes : headingNode,
         );
 

package/src/remark/mdx1Compat/codeCompatPlugin.ts

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import visit from 'unist-util-visit';
+// @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
+import type {Transformer, Processor} from 'unified';
+import type {Code} from 'mdast';
+
+// Solution inspired by https://github.com/pomber/docusaurus-mdx-2/blob/main/packages/mdx-loader/src/remark/codeCompat/index.ts
+// TODO after MDX 2 we probably don't need this - remove soon?
+// Only fenced code blocks are swapped by pre/code MDX components
+// Using <pre><code> in JSX shouldn't use our MDX components anymore
+
+// To make theme-classic/src/theme/MDXComponents/Pre work
+// we need to fill two properties that mdx v2 doesn't provide anymore
+export default function codeCompatPlugin(this: Processor): Transformer {
+  return (root) => {
+    visit(root, 'code', (node: Code) => {
+      node.data = node.data || {};
+      node.data.hProperties = node.data.hProperties || {};
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      (node.data.hProperties as any).metastring = node.meta;
+
+      // Retrocompatible support for live codeblock metastring
+      // Not really the appropriate place to handle that :s
+      // eslint-disable-next-line @typescript-eslint/no-explicit-any
+      (node.data.hProperties as any).live = node.meta
+        ?.split(' ')
+        .includes('live');
+    });
+  };
+}

package/src/remark/mermaid/index.ts

@@ -6,6 +6,7 @@
  */
 
 import visit from 'unist-util-visit';
+// @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721
 import type {Transformer} from 'unified';
 import type {Code} from 'mdast';
 
@@ -17,6 +18,7 @@ export default function plugin(): Transformer {
   return (root) => {
     visit(root, 'code', (node: Code, index, parent) => {
       if (node.lang === 'mermaid') {
+        // TODO migrate to mdxJsxFlowElement? cf admonitions
         parent!.children.splice(index, 1, {
           type: 'mermaidCodeBlock',
           data: {