@docusaurus/mdx-loader 2.0.0-beta.16 → 2.0.0-beta.19

package/src/loader.ts

@@ -0,0 +1,253 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import fs from 'fs-extra';
+import {createCompiler} from '@mdx-js/mdx';
+import logger from '@docusaurus/logger';
+import emoji from 'remark-emoji';
+import {
+  parseFrontMatter,
+  parseMarkdownContentTitle,
+  escapePath,
+  getFileLoaderUtils,
+} from '@docusaurus/utils';
+import stringifyObject from 'stringify-object';
+import headings from './remark/headings';
+import toc from './remark/toc';
+import unwrapMdxCodeBlocks from './remark/unwrapMdxCodeBlocks';
+import transformImage from './remark/transformImage';
+import transformLinks from './remark/transformLinks';
+import type {LoaderContext} from 'webpack';
+import type {Processor, Plugin} from 'unified';
+
+const {
+  loaders: {inlineMarkdownImageFileLoader},
+} = getFileLoaderUtils();
+
+const pragma = `
+/* @jsxRuntime classic */
+/* @jsx mdx */
+/* @jsxFrag mdx.Fragment */
+`;
+
+const DEFAULT_OPTIONS: MDXOptions = {
+  rehypePlugins: [],
+  remarkPlugins: [unwrapMdxCodeBlocks, emoji, headings, toc],
+  beforeDefaultRemarkPlugins: [],
+  beforeDefaultRehypePlugins: [],
+};
+
+const compilerCache = new Map<string | Options, [Processor, Options]>();
+
+export type MDXPlugin =
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  [Plugin<any[]>, any] | Plugin<any[]>;
+export type MDXOptions = {
+  remarkPlugins: MDXPlugin[];
+  rehypePlugins: MDXPlugin[];
+  beforeDefaultRemarkPlugins: MDXPlugin[];
+  beforeDefaultRehypePlugins: MDXPlugin[];
+};
+
+export type Options = MDXOptions & {
+  staticDirs: string[];
+  siteDir: string;
+  isMDXPartial?: (filePath: string) => boolean;
+  isMDXPartialFrontMatterWarningDisabled?: boolean;
+  removeContentTitle?: boolean;
+  metadataPath?: string | ((filePath: string) => string);
+  createAssets?: (metadata: {
+    frontMatter: {[key: string]: unknown};
+    metadata: {[key: string]: unknown};
+  }) => {[key: string]: unknown};
+  filepath: string;
+};
+
+/**
+ * When this throws, it generally means that there's no metadata file associated
+ * with this MDX document. It can happen when using MDX partials (usually
+ * starting with _). That's why it's important to provide the `isMDXPartial`
+ * function in config
+ */
+async function readMetadataPath(metadataPath: string) {
+  try {
+    return await fs.readFile(metadataPath, 'utf8');
+  } catch (err) {
+    logger.error`MDX loader can't read MDX metadata file path=${metadataPath}. Maybe the isMDXPartial option function was not provided?`;
+    throw err;
+  }
+}
+
+/**
+ * Converts assets an object with Webpack require calls code.
+ * This is useful for mdx files to reference co-located assets using relative
+ * paths. Those assets should enter the Webpack assets pipeline and be hashed.
+ * For now, we only handle that for images and paths starting with `./`:
+ *
+ * `{image: "./myImage.png"}` => `{image: require("./myImage.png")}`
+ */
+function createAssetsExportCode(assets: {[key: string]: unknown}) {
+  if (Object.keys(assets).length === 0) {
+    return 'undefined';
+  }
+
+  // TODO implementation can be completed/enhanced
+  function createAssetValueCode(assetValue: unknown): string | undefined {
+    if (Array.isArray(assetValue)) {
+      const arrayItemCodes = assetValue.map(
+        (item) => createAssetValueCode(item) ?? 'undefined',
+      );
+      return `[${arrayItemCodes.join(', ')}]`;
+    }
+    // Only process string values starting with ./
+    // We could enhance this logic and check if file exists on disc?
+    if (typeof assetValue === 'string' && assetValue.startsWith('./')) {
+      // TODO do we have other use-cases than image assets?
+      // Probably not worth adding more support, as we want to move to Webpack 5 new asset system (https://github.com/facebook/docusaurus/pull/4708)
+      const inlineLoader = inlineMarkdownImageFileLoader;
+      return `require("${inlineLoader}${escapePath(assetValue)}").default`;
+    }
+    return undefined;
+  }
+
+  const assetEntries = Object.entries(assets);
+
+  const codeLines = assetEntries
+    .map(([key, value]) => {
+      const assetRequireCode = createAssetValueCode(value);
+      return assetRequireCode ? `"${key}": ${assetRequireCode},` : undefined;
+    })
+    .filter(Boolean);
+
+  return `{\n${codeLines.join('\n')}\n}`;
+}
+
+export async function mdxLoader(
+  this: LoaderContext<Options>,
+  fileString: string,
+): Promise<void> {
+  const callback = this.async();
+  const filePath = this.resourcePath;
+  const reqOptions = this.getOptions() ?? {};
+
+  const {frontMatter, content: contentWithTitle} = parseFrontMatter(fileString);
+
+  const {content, contentTitle} = parseMarkdownContentTitle(contentWithTitle, {
+    removeContentTitle: reqOptions.removeContentTitle,
+  });
+
+  const hasFrontMatter = Object.keys(frontMatter).length > 0;
+
+  if (!compilerCache.has(this.query)) {
+    const options: Options = {
+      ...reqOptions,
+      remarkPlugins: [
+        ...(reqOptions.beforeDefaultRemarkPlugins ?? []),
+        ...DEFAULT_OPTIONS.remarkPlugins,
+        [
+          transformImage,
+          {
+            staticDirs: reqOptions.staticDirs,
+            siteDir: reqOptions.siteDir,
+          },
+        ],
+        [
+          transformLinks,
+          {
+            staticDirs: reqOptions.staticDirs,
+            siteDir: reqOptions.siteDir,
+          },
+        ],
+        ...(reqOptions.remarkPlugins ?? []),
+      ],
+      rehypePlugins: [
+        ...(reqOptions.beforeDefaultRehypePlugins ?? []),
+        ...DEFAULT_OPTIONS.rehypePlugins,
+        ...(reqOptions.rehypePlugins ?? []),
+      ],
+    };
+    compilerCache.set(this.query, [createCompiler(options), options]);
+  }
+
+  const [compiler, options] = compilerCache.get(this.query)!;
+
+  let result: string;
+  try {
+    result = await compiler
+      .process({
+        contents: content,
+        path: this.resourcePath,
+      })
+      .then((res) => res.toString());
+  } catch (err) {
+    return callback(err as Error);
+  }
+
+  // MDX partials are MDX files starting with _ or in a folder starting with _
+  // Partial are not expected to have associated metadata files or front matter
+  const isMDXPartial = options.isMDXPartial?.(filePath);
+  if (isMDXPartial && hasFrontMatter) {
+    const errorMessage = `Docusaurus MDX partial files should not contain front matter.
+Those partial files use the _ prefix as a convention by default, but this is configurable.
+File at ${filePath} contains front matter that will be ignored:
+${JSON.stringify(frontMatter, null, 2)}`;
+
+    if (!options.isMDXPartialFrontMatterWarningDisabled) {
+      const shouldError = process.env.NODE_ENV === 'test' || process.env.CI;
+      if (shouldError) {
+        return callback(new Error(errorMessage));
+      }
+      logger.warn(errorMessage);
+    }
+  }
+
+  function getMetadataPath(): string | undefined {
+    if (!isMDXPartial) {
+      // Read metadata for this MDX and export it.
+      if (options.metadataPath && typeof options.metadataPath === 'function') {
+        return options.metadataPath(filePath);
+      }
+    }
+    return undefined;
+  }
+
+  const metadataPath = getMetadataPath();
+  if (metadataPath) {
+    this.addDependency(metadataPath);
+  }
+
+  const metadataJsonString = metadataPath
+    ? await readMetadataPath(metadataPath)
+    : undefined;
+
+  const metadata = metadataJsonString
+    ? JSON.parse(metadataJsonString)
+    : undefined;
+
+  const assets =
+    reqOptions.createAssets && metadata
+      ? reqOptions.createAssets({frontMatter, metadata})
+      : undefined;
+
+  const exportsCode = `
+export const frontMatter = ${stringifyObject(frontMatter)};
+export const contentTitle = ${stringifyObject(contentTitle)};
+${metadataJsonString ? `export const metadata = ${metadataJsonString};` : ''}
+${assets ? `export const assets = ${createAssetsExportCode(assets)};` : ''}
+`;
+
+  const code = `
+${pragma}
+import React from 'react';
+import { mdx } from '@mdx-js/react';
+
+${exportsCode}
+${result}
+`;
+
+  return callback(null, code);
+}

package/src/remark/headings/index.ts

@@ -9,9 +9,8 @@
 
 import {parseMarkdownHeadingId, createSlugger} from '@docusaurus/utils';
 import visit from 'unist-util-visit';
-import toString from 'mdast-util-to-string';
+import mdastToString from 'mdast-util-to-string';
 import type {Transformer} from 'unified';
-import type {Parent} from 'unist';
 import type {Heading, Text} from 'mdast';
 
 export default function plugin(): Transformer {
@@ -30,10 +29,8 @@ export default function plugin(): Transformer {
         const headingTextNodes = headingNode.children.filter(
           ({type}) => !['html', 'jsx'].includes(type),
         );
-        const heading = toString(
-          headingTextNodes.length > 0
-            ? ({children: headingTextNodes} as Parent)
-            : headingNode,
+        const heading = mdastToString(
+          headingTextNodes.length > 0 ? headingTextNodes : headingNode,
         );
 
         // Support explicit heading IDs

package/src/remark/toc/index.ts

@@ -13,7 +13,7 @@ import toString from 'mdast-util-to-string';
 import visit from 'unist-util-visit';
 import {toValue} from '../utils';
 
-import type {TOCItem} from '@docusaurus/types';
+import type {TOCItem} from '../..';
 import type {Node, Parent} from 'unist';
 import type {Heading, Literal} from 'mdast';
 import type {Transformer} from 'unified';
@@ -27,9 +27,9 @@ const isImport = (child: Node): child is Literal => child.type === 'import';
 const hasImports = (index: number) => index > -1;
 const isExport = (child: Node): child is Literal => child.type === 'export';
 
-interface PluginOptions {
+type PluginOptions = {
   name?: string;
-}
+};
 
 const isTarget = (child: Literal, name: string) => {
   let found = false;
@@ -76,10 +76,10 @@ export default function plugin(options: PluginOptions = {}): Transformer {
   return (root) => {
     const headings: TOCItem[] = [];
 
-    visit(root, 'heading', (child: Heading, _index, parent) => {
+    visit(root, 'heading', (child: Heading, index, parent) => {
       const value = toString(child);
 
-      // depth:1 headings are titles and not included in the TOC
+      // depth: 1 headings are titles and not included in the TOC
       if (parent !== root || !value || child.depth < 2) {
         return;
       }
@@ -93,8 +93,8 @@ export default function plugin(options: PluginOptions = {}): Transformer {
     const {children} = root as Parent<Literal>;
     const targetIndex = getOrCreateExistingTargetIndex(children, name);
 
-    if (headings && headings.length) {
-      children[targetIndex].value = `export const ${name} = ${stringifyObject(
+    if (headings.length) {
+      children[targetIndex]!.value = `export const ${name} = ${stringifyObject(
         headings,
       )};`;
     }

package/src/remark/transformImage/index.ts

@@ -67,8 +67,12 @@ async function toImageRequireNode(
       height = ` height="${size.height}"`;
     }
   } catch (err) {
-    logger.error`The image at path=${imagePath} can't be read correctly. Please ensure it's a valid image.
+    // Workaround for https://github.com/yarnpkg/berry/pull/3889#issuecomment-1034469784
+    // TODO remove this check once fixed in Yarn PnP
+    if (!process.versions.pnp) {
+      logger.warn`The image at path=${imagePath} can't be read correctly. Please ensure it's a valid image.
 ${(err as Error).message}`;
+    }
   }
 
   Object.keys(jsxNode).forEach(
@@ -76,7 +80,7 @@ ${(err as Error).message}`;
   );
 
   (jsxNode as Literal).type = 'jsx';
-  jsxNode.value = `<img loading="lazy" ${alt}src={${src}}${title}${width}${height} />`;
+  jsxNode.value = `<img ${alt}src={${src}}${title}${width}${height} />`;
 }
 
 async function ensureImageFileExist(imagePath: string, sourceFilePath: string) {
@@ -99,7 +103,7 @@ async function getImageAbsolutePath(
     await ensureImageFileExist(imageFilePath, filePath);
     return imageFilePath;
   } else if (path.isAbsolute(imagePath)) {
-    // absolute paths are expected to exist in the static folder
+    // Absolute paths are expected to exist in the static folder.
     const possiblePaths = staticDirs.map((dir) => path.join(dir, imagePath));
     const imageFilePath = await findAsyncSequential(
       possiblePaths,
@@ -116,8 +120,11 @@ async function getImageAbsolutePath(
     }
     return imageFilePath;
   }
-  // relative paths are resolved against the source file's folder
-  const imageFilePath = path.join(path.dirname(filePath), imagePath);
+  // Relative paths are resolved against the source file's folder.
+  const imageFilePath = path.join(
+    path.dirname(filePath),
+    decodeURIComponent(imagePath),
+  );
   await ensureImageFileExist(imageFilePath, filePath);
   return imageFilePath;
 }

package/src/remark/transformLinks/index.ts

@@ -34,7 +34,9 @@ type Context = PluginOptions & {
   filePath: string;
 };
 
-// transform the link node to a jsx link with a require() call
+/**
+ * Transforms the link node to a JSX `<a>` element with a `require()` call.
+ */
 function toAssetRequireNode(node: Link, assetPath: string, filePath: string) {
   const jsxNode = node as Literal & Partial<Link>;
   let relativeAssetPath = posixPath(
@@ -106,7 +108,7 @@ async function getAssetAbsolutePath(
 
 async function processLinkNode(node: Link, context: Context) {
   if (!node.url) {
-    // try to improve error feedback
+    // Try to improve error feedback
     // see https://github.com/facebook/docusaurus/issues/3309#issuecomment-690371675
     const title = node.title || (node.children[0] as Literal)?.value || '?';
     const line = node?.position?.start?.line || '?';
@@ -130,7 +132,10 @@ async function processLinkNode(node: Link, context: Context) {
     return;
   }
 
-  const assetPath = await getAssetAbsolutePath(parsedUrl.pathname, context);
+  const assetPath = await getAssetAbsolutePath(
+    decodeURIComponent(parsedUrl.pathname),
+    context,
+  );
   if (assetPath) {
     toAssetRequireNode(node, assetPath, context.filePath);
   }

package/src/remark/unwrapMdxCodeBlocks/index.ts

@@ -17,7 +17,7 @@ import type {Code, Parent} from 'mdast';
 // See https://github.com/facebook/docusaurus/pull/4278
 export default function plugin(this: Processor): Transformer {
   return (root) => {
-    visit(root, 'code', (node: Code, _index, parent) => {
+    visit(root, 'code', (node: Code, index, parent) => {
       if (node.lang === 'mdx-code-block') {
         const newChildren = (this.parse(node.value) as Parent).children;
 

package/src/remark/utils/index.ts

@@ -31,7 +31,6 @@ export function toValue(node: PhrasingContent | Heading): string {
     case 'link':
       return stringifyContent(node);
     default:
+      return toString(node);
   }
-
-  return toString(node);
 }

package/src/mdx-loader.d.ts

@@ -1,18 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-import type {Plugin} from 'unified';
-
-export type RemarkOrRehypePlugin =
-  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  [Plugin<any[]>, Record<string, unknown>] | Plugin<any[]>;
-export type RemarkAndRehypePluginOptions = {
-  remarkPlugins: RemarkOrRehypePlugin[];
-  rehypePlugins: RemarkOrRehypePlugin[];
-  beforeDefaultRemarkPlugins: RemarkOrRehypePlugin[];
-  beforeDefaultRehypePlugins: RemarkOrRehypePlugin[];
-};