@docusaurus/mdx-loader 2.0.0-beta.fbdeefcac → 2.0.0-rc.1

package/src/loader.ts

@@ -0,0 +1,282 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import fs from 'fs-extra';
+import logger from '@docusaurus/logger';
+import {
+  parseFrontMatter,
+  parseMarkdownContentTitle,
+  escapePath,
+  getFileLoaderUtils,
+} from '@docusaurus/utils';
+import {createCompiler} from '@mdx-js/mdx';
+import emoji from 'remark-emoji';
+import stringifyObject from 'stringify-object';
+
+import headings from './remark/headings';
+import toc from './remark/toc';
+import unwrapMdxCodeBlocks from './remark/unwrapMdxCodeBlocks';
+import transformImage from './remark/transformImage';
+import transformLinks from './remark/transformLinks';
+
+import transformAdmonitions from './remark/admonitions';
+import type {LoaderContext} from 'webpack';
+import type {Processor, Plugin} from 'unified';
+import type {AdmonitionOptions} from './remark/admonitions';
+
+const {
+  loaders: {inlineMarkdownImageFileLoader},
+} = getFileLoaderUtils();
+
+const pragma = `
+/* @jsxRuntime classic */
+/* @jsx mdx */
+/* @jsxFrag React.Fragment */
+`;
+
+const DEFAULT_OPTIONS: MDXOptions = {
+  admonitions: true,
+  rehypePlugins: [],
+  remarkPlugins: [unwrapMdxCodeBlocks, emoji, headings, toc],
+  beforeDefaultRemarkPlugins: [],
+  beforeDefaultRehypePlugins: [],
+};
+
+const compilerCache = new Map<string | Options, [Processor, Options]>();
+
+export type MDXPlugin =
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  [Plugin<any[]>, any] | Plugin<any[]>;
+
+export type MDXOptions = {
+  admonitions: boolean | AdmonitionOptions;
+  remarkPlugins: MDXPlugin[];
+  rehypePlugins: MDXPlugin[];
+  beforeDefaultRemarkPlugins: MDXPlugin[];
+  beforeDefaultRehypePlugins: MDXPlugin[];
+};
+
+export type Options = Partial<MDXOptions> & {
+  staticDirs: string[];
+  siteDir: string;
+  isMDXPartial?: (filePath: string) => boolean;
+  isMDXPartialFrontMatterWarningDisabled?: boolean;
+  removeContentTitle?: boolean;
+  metadataPath?: string | ((filePath: string) => string);
+  createAssets?: (metadata: {
+    frontMatter: {[key: string]: unknown};
+    metadata: {[key: string]: unknown};
+  }) => {[key: string]: unknown};
+  filepath: string;
+};
+
+/**
+ * When this throws, it generally means that there's no metadata file associated
+ * with this MDX document. It can happen when using MDX partials (usually
+ * starting with _). That's why it's important to provide the `isMDXPartial`
+ * function in config
+ */
+async function readMetadataPath(metadataPath: string) {
+  try {
+    return await fs.readFile(metadataPath, 'utf8');
+  } catch (err) {
+    logger.error`MDX loader can't read MDX metadata file path=${metadataPath}. Maybe the isMDXPartial option function was not provided?`;
+    throw err;
+  }
+}
+
+/**
+ * Converts assets an object with Webpack require calls code.
+ * This is useful for mdx files to reference co-located assets using relative
+ * paths. Those assets should enter the Webpack assets pipeline and be hashed.
+ * For now, we only handle that for images and paths starting with `./`:
+ *
+ * `{image: "./myImage.png"}` => `{image: require("./myImage.png")}`
+ */
+function createAssetsExportCode(assets: unknown) {
+  if (
+    typeof assets !== 'object' ||
+    !assets ||
+    Object.keys(assets).length === 0
+  ) {
+    return 'undefined';
+  }
+
+  // TODO implementation can be completed/enhanced
+  function createAssetValueCode(assetValue: unknown): string | undefined {
+    if (Array.isArray(assetValue)) {
+      const arrayItemCodes = assetValue.map(
+        (item: unknown) => createAssetValueCode(item) ?? 'undefined',
+      );
+      return `[${arrayItemCodes.join(', ')}]`;
+    }
+    // Only process string values starting with ./
+    // We could enhance this logic and check if file exists on disc?
+    if (typeof assetValue === 'string' && assetValue.startsWith('./')) {
+      // TODO do we have other use-cases than image assets?
+      // Probably not worth adding more support, as we want to move to Webpack 5 new asset system (https://github.com/facebook/docusaurus/pull/4708)
+      const inlineLoader = inlineMarkdownImageFileLoader;
+      return `require("${inlineLoader}${escapePath(assetValue)}").default`;
+    }
+    return undefined;
+  }
+
+  const assetEntries = Object.entries(assets);
+
+  const codeLines = assetEntries
+    .map(([key, value]: [string, unknown]) => {
+      const assetRequireCode = createAssetValueCode(value);
+      return assetRequireCode ? `"${key}": ${assetRequireCode},` : undefined;
+    })
+    .filter(Boolean);
+
+  return `{\n${codeLines.join('\n')}\n}`;
+}
+
+function getAdmonitionsPlugins(
+  admonitionsOption: MDXOptions['admonitions'],
+): MDXPlugin[] {
+  if (admonitionsOption) {
+    const plugin: MDXPlugin =
+      admonitionsOption === true
+        ? transformAdmonitions
+        : [transformAdmonitions, admonitionsOption];
+    return [plugin];
+  }
+  return [];
+}
+
+export async function mdxLoader(
+  this: LoaderContext<Options>,
+  fileString: string,
+): Promise<void> {
+  const callback = this.async();
+  const filePath = this.resourcePath;
+  const reqOptions = this.getOptions();
+
+  const {frontMatter, content: contentWithTitle} = parseFrontMatter(fileString);
+
+  const {content, contentTitle} = parseMarkdownContentTitle(contentWithTitle, {
+    removeContentTitle: reqOptions.removeContentTitle,
+  });
+
+  const hasFrontMatter = Object.keys(frontMatter).length > 0;
+
+  if (!compilerCache.has(this.query)) {
+    const remarkPlugins: MDXPlugin[] = [
+      ...(reqOptions.beforeDefaultRemarkPlugins ?? []),
+      ...getAdmonitionsPlugins(reqOptions.admonitions ?? false),
+      ...DEFAULT_OPTIONS.remarkPlugins,
+      [
+        transformImage,
+        {
+          staticDirs: reqOptions.staticDirs,
+          siteDir: reqOptions.siteDir,
+        },
+      ],
+      [
+        transformLinks,
+        {
+          staticDirs: reqOptions.staticDirs,
+          siteDir: reqOptions.siteDir,
+        },
+      ],
+      ...(reqOptions.remarkPlugins ?? []),
+    ];
+
+    const rehypePlugins: MDXPlugin[] = [
+      ...(reqOptions.beforeDefaultRehypePlugins ?? []),
+      ...DEFAULT_OPTIONS.rehypePlugins,
+      ...(reqOptions.rehypePlugins ?? []),
+    ];
+
+    const options: Options = {
+      ...reqOptions,
+      remarkPlugins,
+      rehypePlugins,
+    };
+    compilerCache.set(this.query, [createCompiler(options), options]);
+  }
+
+  const [compiler, options] = compilerCache.get(this.query)!;
+
+  let result: string;
+  try {
+    result = await compiler
+      .process({
+        contents: content,
+        path: this.resourcePath,
+      })
+      .then((res) => res.toString());
+  } catch (err) {
+    return callback(err as Error);
+  }
+
+  // MDX partials are MDX files starting with _ or in a folder starting with _
+  // Partial are not expected to have associated metadata files or front matter
+  const isMDXPartial = options.isMDXPartial?.(filePath);
+  if (isMDXPartial && hasFrontMatter) {
+    const errorMessage = `Docusaurus MDX partial files should not contain front matter.
+Those partial files use the _ prefix as a convention by default, but this is configurable.
+File at ${filePath} contains front matter that will be ignored:
+${JSON.stringify(frontMatter, null, 2)}`;
+
+    if (!options.isMDXPartialFrontMatterWarningDisabled) {
+      const shouldError = process.env.NODE_ENV === 'test' || process.env.CI;
+      if (shouldError) {
+        return callback(new Error(errorMessage));
+      }
+      logger.warn(errorMessage);
+    }
+  }
+
+  function getMetadataPath(): string | undefined {
+    if (!isMDXPartial) {
+      // Read metadata for this MDX and export it.
+      if (options.metadataPath && typeof options.metadataPath === 'function') {
+        return options.metadataPath(filePath);
+      }
+    }
+    return undefined;
+  }
+
+  const metadataPath = getMetadataPath();
+  if (metadataPath) {
+    this.addDependency(metadataPath);
+  }
+
+  const metadataJsonString = metadataPath
+    ? await readMetadataPath(metadataPath)
+    : undefined;
+
+  const metadata = metadataJsonString
+    ? (JSON.parse(metadataJsonString) as {[key: string]: unknown})
+    : undefined;
+
+  const assets =
+    reqOptions.createAssets && metadata
+      ? reqOptions.createAssets({frontMatter, metadata})
+      : undefined;
+
+  const exportsCode = `
+export const frontMatter = ${stringifyObject(frontMatter)};
+export const contentTitle = ${stringifyObject(contentTitle)};
+${metadataJsonString ? `export const metadata = ${metadataJsonString};` : ''}
+${assets ? `export const assets = ${createAssetsExportCode(assets)};` : ''}
+`;
+
+  const code = `
+${pragma}
+import React from 'react';
+import { mdx } from '@mdx-js/react';
+
+${exportsCode}
+${result}
+`;
+
+  return callback(null, code);
+}

package/src/remark/admonitions/LICENSE

@@ -0,0 +1,45 @@
+MIT License
+
+Copyright (c) 2020 Elvis Wolcott
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+---
+
+MIT License
+
+Copyright (c) Facebook, Inc. and its affiliates.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/src/remark/admonitions/README.md

@@ -0,0 +1,3 @@
+# Docusaurus admonitions
+
+Code from [remark-admonitions](https://github.com/remarkjs/remark-directive) (MIT license) has been copied to this folder, and highly customized for Docusaurus needs.

package/src/remark/admonitions/index.ts

@@ -0,0 +1,186 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import visit from 'unist-util-visit';
+import type {Transformer, Processor, Plugin} from 'unified';
+import type {Literal} from 'mdast';
+
+const NEWLINE = '\n';
+
+export type AdmonitionOptions = {
+  tag: string;
+  keywords: string[];
+};
+
+export const DefaultAdmonitionOptions: AdmonitionOptions = {
+  tag: ':::',
+  keywords: [
+    'secondary',
+    'info',
+    'success',
+    'danger',
+    'note',
+    'tip',
+    'warning',
+    'important',
+    'caution',
+  ],
+};
+
+function escapeRegExp(s: string): string {
+  return s.replace(/[-[\]{}()*+?.\\^$|/]/g, '\\$&');
+}
+
+function normalizeOptions(
+  options: Partial<AdmonitionOptions>,
+): AdmonitionOptions {
+  return {...DefaultAdmonitionOptions, ...options};
+}
+
+// This string value does not matter much
+// It is ignored because nodes are using hName/hProperties coming from HAST
+const admonitionNodeType = 'admonitionHTML';
+
+const plugin: Plugin = function plugin(
+  this: Processor,
+  optionsInput: Partial<AdmonitionOptions> = {},
+): Transformer {
+  const options = normalizeOptions(optionsInput);
+
+  const keywords = Object.values(options.keywords).map(escapeRegExp).join('|');
+  const tag = escapeRegExp(options.tag);
+  const regex = new RegExp(`${tag}(${keywords})(?: *(.*))?\n`);
+  const escapeTag = new RegExp(escapeRegExp(`\\${options.tag}`), 'g');
+
+  // The tokenizer is called on blocks to determine if there is an admonition
+  // present and create tags for it
+  function blockTokenizer(this: any, eat: any, value: string, silent: boolean) {
+    // Stop if no match or match does not start at beginning of line
+    const match = regex.exec(value);
+    if (!match || match.index !== 0) {
+      return false;
+    }
+    // If silent return the match
+    if (silent) {
+      return true;
+    }
+
+    const now = eat.now();
+    const [opening, keyword, title] = match as string[] as [
+      string,
+      string,
+      string,
+    ];
+    const food = [];
+    const content = [];
+
+    let newValue = value;
+    // consume lines until a closing tag
+    let idx = newValue.indexOf(NEWLINE);
+    while (idx !== -1) {
+      // grab this line and eat it
+      const next = newValue.indexOf(NEWLINE, idx + 1);
+      const line =
+        next !== -1 ? newValue.slice(idx + 1, next) : newValue.slice(idx + 1);
+      food.push(line);
+      newValue = newValue.slice(idx + 1);
+      // the closing tag is NOT part of the content
+      if (line.startsWith(options.tag)) {
+        break;
+      }
+      content.push(line);
+      idx = newValue.indexOf(NEWLINE);
+    }
+
+    // consume the processed tag and replace escape sequences
+    const contentString = content.join(NEWLINE).replace(escapeTag, options.tag);
+    const add = eat(opening + food.join(NEWLINE));
+
+    // parse the content in block mode
+    const exit = this.enterBlock();
+    const contentNodes = this.tokenizeBlock(contentString, now);
+    exit();
+
+    const titleNodes = this.tokenizeInline(title, now);
+
+    const isSimpleTextTitle =
+      titleNodes.length === 1 && titleNodes[0].type === 'text';
+
+    const element = {
+      type: admonitionNodeType,
+      data: {
+        // hName/hProperties come from HAST
+        // See https://github.com/syntax-tree/mdast-util-to-hast#fields-on-nodes
+        hName: 'admonition',
+        hProperties: {
+          ...(title && isSimpleTextTitle && {title}),
+          type: keyword,
+        },
+      },
+      children: [
+        // For titles containing MDX syntax: create a custom element. The theme
+        // component will extract it and render it nicely.
+        //
+        // Temporary workaround, because it's complex in MDX v1 to emit
+        // interpolated JSX prop syntax (title={<>my <code>title</code></>}).
+        // For this reason, we use children instead of the title prop.
+        title &&
+          !isSimpleTextTitle && {
+            type: admonitionNodeType,
+            data: {
+              hName: 'mdxAdmonitionTitle',
+              hProperties: {},
+            },
+            children: titleNodes,
+          },
+        ...contentNodes,
+      ].filter(Boolean),
+    };
+
+    return add(element);
+  }
+
+  // add tokenizer to parser after fenced code blocks
+  const Parser = this.Parser.prototype;
+  Parser.blockTokenizers.admonition = blockTokenizer;
+  Parser.blockMethods.splice(
+    Parser.blockMethods.indexOf('fencedCode') + 1,
+    0,
+    'admonition',
+  );
+  Parser.interruptParagraph.splice(
+    Parser.interruptParagraph.indexOf('fencedCode') + 1,
+    0,
+    ['admonition'],
+  );
+  Parser.interruptList.splice(
+    Parser.interruptList.indexOf('fencedCode') + 1,
+    0,
+    ['admonition'],
+  );
+  Parser.interruptBlockquote.splice(
+    Parser.interruptBlockquote.indexOf('fencedCode') + 1,
+    0,
+    ['admonition'],
+  );
+
+  return (root) => {
+    // escape everything except admonitionHTML nodes
+    visit(
+      root,
+      (node: unknown): node is Literal =>
+        (node as Literal | undefined)?.type !== admonitionNodeType,
+      (node: Literal) => {
+        if (node.value) {
+          node.value = node.value.replace(escapeTag, options.tag);
+        }
+      },
+    );
+  };
+};
+
+export default plugin;

package/src/remark/headings/{index.js → index.ts}

@@ -7,43 +7,45 @@
 
 /* Based on remark-slug (https://github.com/remarkjs/remark-slug) and gatsby-remark-autolink-headers (https://github.com/gatsbyjs/gatsby/blob/master/packages/gatsby-remark-autolink-headers) */
 
-const {parseMarkdownHeadingId} = require('@docusaurus/utils');
-const visit = require('unist-util-visit');
-const toString = require('mdast-util-to-string');
-const slugs = require('github-slugger')();
+import {parseMarkdownHeadingId, createSlugger} from '@docusaurus/utils';
+import visit from 'unist-util-visit';
+import mdastToString from 'mdast-util-to-string';
+import type {Transformer} from 'unified';
+import type {Heading, Text} from 'mdast';
 
-function headings() {
-  const transformer = (ast) => {
-    slugs.reset();
-
-    function visitor(headingNode) {
-      const data = headingNode.data || (headingNode.data = {}); // eslint-disable-line
-      const properties = data.hProperties || (data.hProperties = {});
+export default function plugin(): Transformer {
+  return (root) => {
+    const slugs = createSlugger();
+    visit(root, 'heading', (headingNode: Heading) => {
+      const data = headingNode.data ?? (headingNode.data = {});
+      const properties = (data.hProperties || (data.hProperties = {})) as {
+        id: string;
+      };
       let {id} = properties;
 
       if (id) {
-        id = slugs.slug(id, true);
+        id = slugs.slug(id, {maintainCase: true});
       } else {
         const headingTextNodes = headingNode.children.filter(
           ({type}) => !['html', 'jsx'].includes(type),
         );
-        const heading = toString(
-          headingTextNodes.length > 0
-            ? {children: headingTextNodes}
-            : headingNode,
+        const heading = mdastToString(
+          headingTextNodes.length > 0 ? headingTextNodes : headingNode,
         );
 
         // Support explicit heading IDs
         const parsedHeading = parseMarkdownHeadingId(heading);
 
-        id = parsedHeading.id || slugs.slug(heading);
+        id = parsedHeading.id ?? slugs.slug(heading);
 
         if (parsedHeading.id) {
           // When there's an id, it is always in the last child node
-          // Sometimes heading is in multiple "parts" (** syntax creates a child node):
+          // Sometimes heading is in multiple "parts" (** syntax creates a child
+          // node):
           // ## part1 *part2* part3 {#id}
-          const lastNode =
-            headingNode.children[headingNode.children.length - 1];
+          const lastNode = headingNode.children[
+            headingNode.children.length - 1
+          ] as Text;
 
           if (headingNode.children.length > 1) {
             const lastNodeText = parseMarkdownHeadingId(lastNode.value).text;
@@ -63,12 +65,6 @@ function headings() {
 
       data.id = id;
       properties.id = id;
-    }
-
-    visit(ast, 'heading', visitor);
+    });
   };
-
-  return transformer;
 }
-
-module.exports = headings;

package/src/remark/toc/index.ts

@@ -0,0 +1,98 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {parse, type ParserOptions} from '@babel/parser';
+import traverse from '@babel/traverse';
+import stringifyObject from 'stringify-object';
+import toString from 'mdast-util-to-string';
+import visit from 'unist-util-visit';
+import {toValue} from '../utils';
+
+import type {Identifier} from '@babel/types';
+import type {TOCItem} from '../..';
+import type {Node, Parent} from 'unist';
+import type {Heading, Literal} from 'mdast';
+import type {Transformer} from 'unified';
+
+const parseOptions: ParserOptions = {
+  plugins: ['jsx'],
+  sourceType: 'module',
+};
+
+const name = 'toc';
+
+const isImport = (child: Node): child is Literal => child.type === 'import';
+const hasImports = (index: number) => index > -1;
+const isExport = (child: Node): child is Literal => child.type === 'export';
+
+const isTarget = (child: Literal) => {
+  let found = false;
+  const ast = parse(child.value, parseOptions);
+  traverse(ast, {
+    VariableDeclarator: (path) => {
+      if ((path.node.id as Identifier).name === name) {
+        found = true;
+      }
+    },
+  });
+  return found;
+};
+
+const getOrCreateExistingTargetIndex = (children: Node[]) => {
+  let importsIndex = -1;
+  let targetIndex = -1;
+
+  children.forEach((child, index) => {
+    if (isImport(child)) {
+      importsIndex = index;
+    } else if (isExport(child) && isTarget(child)) {
+      targetIndex = index;
+    }
+  });
+
+  if (targetIndex === -1) {
+    const target = {
+      default: false,
+      type: 'export',
+      value: `export const ${name} = [];`,
+    };
+
+    targetIndex = hasImports(importsIndex) ? importsIndex + 1 : 0;
+    children.splice(targetIndex, 0, target);
+  }
+
+  return targetIndex;
+};
+
+export default function plugin(): Transformer {
+  return (root) => {
+    const headings: TOCItem[] = [];
+
+    visit(root, 'heading', (child: Heading, index, parent) => {
+      const value = toString(child);
+
+      // depth: 1 headings are titles and not included in the TOC
+      if (parent !== root || !value || child.depth < 2) {
+        return;
+      }
+
+      headings.push({
+        value: toValue(child),
+        id: child.data!.id as string,
+        level: child.depth,
+      });
+    });
+    const {children} = root as Parent<Literal>;
+    const targetIndex = getOrCreateExistingTargetIndex(children);
+
+    if (headings.length) {
+      children[targetIndex]!.value = `export const ${name} = ${stringifyObject(
+        headings,
+      )};`;
+    }
+  };
+}