@azerion/bunosaurus-mdx-loader 0.0.0-stage → 4.0.0

package/src/frontMatter.ts

@@ -0,0 +1,31 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {
+  JoiFrontMatter as Joi,
+  validateFrontMatter,
+} from '@docusaurus/utils-validation';
+
+import type {FormatInput} from './index';
+
+export type MDXFrontMatter = {
+  format?: FormatInput;
+};
+
+export const DefaultMDXFrontMatter: MDXFrontMatter = {
+  format: undefined,
+};
+
+const MDXFrontMatterSchema = Joi.object<MDXFrontMatter>({
+  format: Joi.string().equal('md', 'mdx', 'detect').optional(),
+}).default(DefaultMDXFrontMatter);
+
+export function validateMDXFrontMatter(frontMatter: unknown): MDXFrontMatter {
+  return validateFrontMatter(frontMatter, MDXFrontMatterSchema, {
+    allowUnknown: false,
+  });
+}

package/src/index.ts

@@ -0,0 +1,44 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import type {ReactNode} from 'react';
+
+import {mdxLoader} from './loader';
+
+import type {TOCItem as TOCItemImported} from './remark/toc/types';
+
+export {createMDXLoaderRule, createMDXLoaderItem} from './createMDXLoader';
+
+export default mdxLoader;
+
+export type TOCItem = TOCItemImported;
+
+export type Format = 'md' | 'mdx';
+
+export type FormatInput = Format | 'detect';
+
+export type LoadedMDXContent<FrontMatter, Metadata, Assets = undefined> = {
+  /** As verbatim declared in the MDX document. */
+  readonly frontMatter: FrontMatter;
+  /** As provided by the content plugin. */
+  readonly metadata: Metadata;
+  /** A list of TOC items (headings). */
+  readonly toc: readonly TOCItem[];
+  /** First h1 title before any content. */
+  readonly contentTitle: string | undefined;
+  /**
+   * Usually image assets that may be collocated like `./img/thumbnail.png`.
+   * The loader would also bundle these assets and the client should use these
+   * in priority.
+   */
+  readonly assets: Assets;
+  (): ReactNode;
+};
+
+export type {MDXPlugin} from './loader';
+export type {MDXOptions} from './processor';
+export type {Options} from './options';

package/src/loader.ts

@@ -0,0 +1,227 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import logger from '@docusaurus/logger';
+import {
+  aliasedSitePath,
+  DEFAULT_PARSE_FRONT_MATTER,
+  getFileLoaderUtils,
+  getWebpackLoaderCompilerName,
+} from '@docusaurus/utils';
+import {
+  compileToJSX,
+  createAssetsExportCode,
+  extractContentTitleData,
+} from './utils';
+import {serializeJS} from './serializeJS';
+import type {WebpackCompilerName} from '@docusaurus/utils';
+import type {Options} from './options';
+import type {LoaderContext} from 'webpack';
+export type {MDXPlugin} from './processor';
+
+async function loadMDX({
+  fileContent,
+  filePath,
+  options,
+  compilerName,
+}: {
+  fileContent: string;
+  filePath: string;
+  options: Options;
+  compilerName: WebpackCompilerName;
+}): Promise<string> {
+  const {frontMatter} = await options.markdownConfig.parseFrontMatter({
+    filePath,
+    fileContent,
+    defaultParseFrontMatter: DEFAULT_PARSE_FRONT_MATTER,
+  });
+
+  const hasFrontMatter = Object.keys(frontMatter).length > 0;
+
+  const result = await compileToJSX({
+    fileContent,
+    filePath,
+    frontMatter,
+    options,
+    compilerName,
+  });
+
+  const contentTitle = extractContentTitleData(result.data);
+
+  // MDX partials are MDX files starting with _ or in a folder starting with _
+  // Partial are not expected to have associated metadata files or front matter
+  const isMDXPartial = options.isMDXPartial?.(filePath);
+  if (isMDXPartial && hasFrontMatter) {
+    const errorMessage = `Docusaurus MDX partial files should not contain front matter.
+Those partial files use the _ prefix as a convention by default, but this is configurable.
+File at ${filePath} contains front matter that will be ignored:
+${JSON.stringify(frontMatter, null, 2)}`;
+
+    if (!options.isMDXPartialFrontMatterWarningDisabled) {
+      const shouldError = process.env.NODE_ENV === 'test' || process.env.CI;
+      if (shouldError) {
+        throw new Error(errorMessage);
+      }
+      logger.warn(errorMessage);
+    }
+  }
+
+  const metadataPath = (function getMetadataPath() {
+    if (!isMDXPartial) {
+      return options.metadataPath?.(filePath);
+    }
+    return undefined;
+  })();
+
+  const assets =
+    options.createAssets && !isMDXPartial
+      ? options.createAssets({filePath, frontMatter})
+      : undefined;
+
+  const fileLoaderUtils = getFileLoaderUtils(compilerName === 'server');
+
+  // TODO use remark plugins to insert extra exports instead of string concat?
+  // cf how the toc is exported
+  const exportsCode = `
+export const frontMatter = ${serializeJS(frontMatter)};
+export const contentTitle = ${serializeJS(contentTitle)};
+${
+  metadataPath
+    ? `export {default as metadata} from '${aliasedSitePath(
+        metadataPath,
+        options.siteDir,
+      )}'`
+    : ''
+}
+${
+  assets
+    ? `export const assets = ${createAssetsExportCode({
+        assets,
+        inlineMarkdownAssetImageFileLoader:
+          fileLoaderUtils.loaders.inlineMarkdownAssetImageFileLoader,
+      })};`
+    : ''
+}
+`;
+
+  const code = `
+${exportsCode}
+${result.content}
+`;
+
+  return code;
+}
+
+// Note: we cache promises instead of strings
+// This is because client/server compilations might be triggered in parallel
+// When this happens for the same file, we don't want to compile it twice
+async function loadMDXWithCaching({
+  resource,
+  fileContent,
+  filePath,
+  options,
+  compilerName,
+}: {
+  resource: string; // path?query#hash
+  filePath: string; // path
+  fileContent: string;
+  options: Options;
+  compilerName: WebpackCompilerName;
+}): Promise<string> {
+  const {crossCompilerCache} = options;
+  if (!crossCompilerCache) {
+    return loadMDX({
+      fileContent,
+      filePath,
+      options,
+      compilerName,
+    });
+  }
+
+  // Note we "resource" as cache key, not "filePath" nor "fileContent"
+  // This is because:
+  // - the same file can be compiled in different variants (blog.mdx?truncated)
+  // - the same content can be processed differently (versioned docs links)
+  const cacheKey = resource;
+
+  // We can clean up the cache and free memory after cache entry consumption
+  // We know there are only 2 compilations for the same file
+  // Note: once we introduce RSCs we'll probably have 3 compilations
+  // Note: we can't use string keys in WeakMap
+  // But we could eventually use WeakRef for the values
+  const deleteCacheEntry = () => crossCompilerCache.delete(cacheKey);
+
+  const cacheEntry = crossCompilerCache?.get(cacheKey);
+
+  // When deduplicating client/server compilations, we always use the client
+  // compilation and not the server compilation
+  // This is important because the server compilation usually skips some steps
+  // Notably: the server compilation does not emit file-loader assets
+  // Using the server compilation otherwise leads to broken images
+  // See https://github.com/facebook/docusaurus/issues/10544#issuecomment-2390943794
+  // See https://github.com/facebook/docusaurus/pull/10553
+  // TODO a problem with this: server bundle will use client inline loaders
+  //  This means server bundle will use ?emit=true for assets
+  //  We should try to get rid of inline loaders to cleanup this caching logic
+  if (compilerName === 'client') {
+    const promise = loadMDX({
+      fileContent,
+      filePath,
+      options,
+      compilerName,
+    });
+    if (cacheEntry) {
+      promise.then(cacheEntry.resolve, cacheEntry.reject);
+      deleteCacheEntry();
+    } else {
+      const noop = () => {
+        throw new Error('this should never be called');
+      };
+      crossCompilerCache.set(cacheKey, {
+        promise,
+        resolve: noop,
+        reject: noop,
+      });
+    }
+    return promise;
+  }
+  // Server compilation always uses the result of the client compilation above
+  else if (compilerName === 'server') {
+    if (cacheEntry) {
+      deleteCacheEntry();
+      return cacheEntry.promise;
+    } else {
+      const {promise, resolve, reject} = Promise.withResolvers<string>();
+      crossCompilerCache.set(cacheKey, {promise, resolve, reject});
+      return promise;
+    }
+  } else {
+    throw new Error(`Unexpected compilerName=${compilerName}`);
+  }
+}
+
+export async function mdxLoader(
+  this: LoaderContext<Options>,
+  fileContent: string,
+): Promise<void> {
+  const compilerName = getWebpackLoaderCompilerName(this);
+  const callback = this.async();
+  const options: Options = this.getOptions();
+  options.dependencies?.forEach(this.addDependency);
+  try {
+    const result = await loadMDXWithCaching({
+      resource: this.resource,
+      filePath: this.resourcePath,
+      fileContent,
+      options,
+      compilerName,
+    });
+    return callback(null, result);
+  } catch (error) {
+    return callback(error as Error);
+  }
+}

package/src/options.ts

@@ -0,0 +1,37 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import type {MDXOptions, SimpleProcessors} from './processor';
+import type {MarkdownConfig} from '@docusaurus/types';
+import type {ResolveMarkdownLink} from './remark/resolveMarkdownLinks';
+import type {Options as RemarkMdxOptions} from 'remark-mdx';
+
+export type Options = Partial<MDXOptions> & {
+  dependencies?: string[];
+
+  markdownConfig: MarkdownConfig;
+  staticDirs: string[];
+  siteDir: string;
+  isMDXPartial?: (filePath: string) => boolean;
+  isMDXPartialFrontMatterWarningDisabled?: boolean;
+  removeContentTitle?: boolean;
+  metadataPath?: (filePath: string) => string;
+  createAssets?: (metadata: {
+    filePath: string;
+    frontMatter: {[key: string]: unknown};
+  }) => {[key: string]: unknown};
+  resolveMarkdownLink?: ResolveMarkdownLink;
+  // Options passed to remark-mdx (only for the 'mdx' format)
+  // Replaces our default {acornOptions: {ecmaVersion: 2025, sourceType: 'module'}}
+  remarkMdxOptions?: RemarkMdxOptions;
+
+  // Will usually be created by "createMDXLoaderItem"
+  processors?: SimpleProcessors;
+  crossCompilerCache?: Map<string, CrossCompilerCacheEntry>; // MDX => Promise<JSX> cache
+};
+
+type CrossCompilerCacheEntry = PromiseWithResolvers<string>;

package/src/preprocessor.ts

@@ -0,0 +1,47 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {
+  escapeMarkdownHeadingIds,
+  unwrapMdxCodeBlocks,
+  admonitionTitleToDirectiveLabel,
+} from '@docusaurus/utils';
+import {normalizeAdmonitionOptions} from './remark/admonitions';
+import type {Options} from './options';
+
+/**
+ * Preprocess the string before passing it to MDX
+ * This is not particularly recommended but makes it easier to upgrade to MDX 2
+ */
+export default function preprocessContent({
+  fileContent: initialFileContent,
+  filePath,
+  markdownConfig,
+  admonitions,
+}: {
+  fileContent: string;
+  filePath: string;
+  markdownConfig: Options['markdownConfig'];
+  admonitions: Options['admonitions'] | undefined;
+}): string {
+  let fileContent = initialFileContent;
+  if (markdownConfig.preprocessor) {
+    fileContent = markdownConfig.preprocessor({
+      fileContent,
+      filePath,
+    });
+  }
+  fileContent = unwrapMdxCodeBlocks(fileContent);
+  if (markdownConfig.mdx1Compat.headingIds) {
+    fileContent = escapeMarkdownHeadingIds(fileContent);
+  }
+  if (markdownConfig.mdx1Compat.admonitions && admonitions) {
+    const {keywords} = normalizeAdmonitionOptions(admonitions);
+    fileContent = admonitionTitleToDirectiveLabel(fileContent, keywords);
+  }
+  return fileContent;
+}

package/src/processor.ts

@@ -0,0 +1,306 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {createProcessor as createMdxProcessor} from '@mdx-js/mdx';
+import frontmatter from 'remark-frontmatter';
+import rehypeRaw from 'rehype-raw';
+import gfm from 'remark-gfm';
+// TODO using fork until PR merged: https://github.com/leebyron/remark-comment/pull/3
+import remarkComment from '@slorber/remark-comment';
+import directive from 'remark-directive';
+import remarkMdx from 'remark-mdx';
+import {VFile} from 'vfile';
+import emoji from 'remark-emoji';
+import headings from './remark/headings';
+import contentTitle from './remark/contentTitle';
+import toc from './remark/toc';
+import transformImage from './remark/transformImage';
+import transformLinks from './remark/transformLinks';
+import resolveMarkdownLinks from './remark/resolveMarkdownLinks';
+import details from './remark/details';
+import head from './remark/head';
+import mermaid from './remark/mermaid';
+import transformAdmonitions from './remark/admonitions';
+import unusedDirectives from './remark/unusedDirectives';
+import codeCompatPlugin from './remark/mdx1Compat/codeCompatPlugin';
+import {getFormat} from './format';
+import type {WebpackCompilerName} from '@docusaurus/utils';
+import type {MDXFrontMatter} from './frontMatter';
+import type {Options} from './options';
+import type {AdmonitionOptions} from './remark/admonitions';
+import type {PluginOptions as ResolveMarkdownLinksOptions} from './remark/resolveMarkdownLinks';
+import type {PluginOptions as TransformLinksOptions} from './remark/transformLinks';
+import type {PluginOptions as TransformImageOptions} from './remark/transformImage';
+import type {PluginOptions as UnusedDirectivesOptions} from './remark/unusedDirectives';
+import type {ProcessorOptions} from '@mdx-js/mdx';
+import type {Options as RemarkMdxOptions} from 'remark-mdx';
+import type {Pluggable} from 'unified';
+
+export type SimpleProcessorResult = {
+  content: string;
+  data: {[key: string]: unknown};
+};
+
+export type SimpleProcessor = {
+  process: ({
+    content,
+    filePath,
+    frontMatter,
+    compilerName,
+  }: {
+    content: string;
+    filePath: string;
+    frontMatter: {[key: string]: unknown};
+    compilerName: WebpackCompilerName;
+  }) => Promise<SimpleProcessorResult>;
+};
+
+export type MDXPlugin = Pluggable;
+
+// MDX defaults to ES2024, we opt-in for ES2025 syntax (import attributes...)
+const DefaultRemarkMdxOptions: RemarkMdxOptions = {
+  acornOptions: {ecmaVersion: 2025, sourceType: 'module'},
+};
+
+export type MDXOptions = {
+  admonitions: boolean | Partial<AdmonitionOptions>;
+  remarkPlugins: MDXPlugin[];
+  rehypePlugins: MDXPlugin[];
+  recmaPlugins: MDXPlugin[];
+  beforeDefaultRemarkPlugins: MDXPlugin[];
+  beforeDefaultRehypePlugins: MDXPlugin[];
+};
+
+function getAdmonitionsPlugins(
+  admonitionsOption: MDXOptions['admonitions'],
+): MDXPlugin[] {
+  if (admonitionsOption) {
+    const plugin: MDXPlugin =
+      admonitionsOption === true
+        ? transformAdmonitions
+        : [transformAdmonitions, admonitionsOption];
+    return [plugin];
+  }
+
+  return [];
+}
+
+function getDefaultRemarkPlugins({options}: {options: Options}): MDXPlugin[] {
+  return [
+    [
+      headings,
+      {anchorsMaintainCase: options.markdownConfig.anchors.maintainCase},
+    ],
+    ...(options.markdownConfig.emoji ? [emoji] : []),
+    toc,
+  ];
+}
+
+// MDX's createProcessor() registers remark-mdx without options
+// See https://github.com/mdx-js/mdx/issues/2628
+// unified dedupes plugins by reference: using remark-mdx again reconfigures
+// the existing attacher in place, at its original pipeline position
+// This only applies to the 'mdx' format: we must not add remark-mdx for 'md'
+function getRemarkMdxPlugins({
+  options,
+  format,
+}: {
+  options: Options;
+  format: 'md' | 'mdx';
+}): MDXPlugin[] {
+  if (format === 'md') {
+    return [];
+  }
+  return [[remarkMdx, options.remarkMdxOptions ?? DefaultRemarkMdxOptions]];
+}
+
+// /!\ this method is synchronous on purpose
+// Using async code here can create cache entry race conditions!
+function createProcessorUncached({
+  options,
+  format,
+}: {
+  options: Options;
+  format: 'md' | 'mdx';
+}): SimpleProcessor {
+  const remarkPlugins: MDXPlugin[] = [
+    ...getRemarkMdxPlugins({options, format}),
+    ...(options.beforeDefaultRemarkPlugins ?? []),
+    frontmatter,
+    directive,
+    [contentTitle, {removeContentTitle: options.removeContentTitle}],
+    ...getAdmonitionsPlugins(options.admonitions ?? false),
+    ...getDefaultRemarkPlugins({options}),
+    details,
+    head,
+    ...(options.markdownConfig.mermaid ? [mermaid] : []),
+    [
+      transformImage,
+      {
+        staticDirs: options.staticDirs,
+        siteDir: options.siteDir,
+        onBrokenMarkdownImages:
+          options.markdownConfig.hooks.onBrokenMarkdownImages,
+      } satisfies TransformImageOptions,
+    ],
+    // TODO merge this with transformLinks?
+    options.resolveMarkdownLink
+      ? [
+          resolveMarkdownLinks,
+          {
+            resolveMarkdownLink: options.resolveMarkdownLink,
+            onBrokenMarkdownLinks:
+              options.markdownConfig.hooks.onBrokenMarkdownLinks,
+          } satisfies ResolveMarkdownLinksOptions,
+        ]
+      : undefined,
+    [
+      transformLinks,
+      {
+        staticDirs: options.staticDirs,
+        siteDir: options.siteDir,
+        onBrokenMarkdownLinks:
+          options.markdownConfig.hooks.onBrokenMarkdownLinks,
+      } satisfies TransformLinksOptions,
+    ],
+    gfm,
+    options.markdownConfig.mdx1Compat.comments ? remarkComment : null,
+    ...(options.remarkPlugins ?? []),
+    [
+      unusedDirectives,
+      {
+        onUnusedMarkdownDirectives:
+          options.markdownConfig.hooks.onUnusedMarkdownDirectives,
+      } satisfies UnusedDirectivesOptions,
+    ],
+  ].filter((plugin): plugin is MDXPlugin => Boolean(plugin));
+
+  // codeCompatPlugin needs to be applied last after user-provided plugins
+  // (after npm2yarn for example)
+  remarkPlugins.push(codeCompatPlugin);
+
+  const rehypePlugins: MDXPlugin[] = [
+    ...(options.beforeDefaultRehypePlugins ?? []),
+    ...(options.rehypePlugins ?? []),
+  ];
+
+  // Maybe we'll want to introduce default recma plugins later?
+  // For example https://github.com/domdomegg/recma-mdx-displayname ?
+  const recmaPlugins = [...(options.recmaPlugins ?? [])];
+
+  if (format === 'md') {
+    // This is what permits to embed HTML elements with format 'md'
+    // See https://github.com/facebook/docusaurus/pull/8960
+    // See https://github.com/mdx-js/mdx/pull/2295#issuecomment-1540085960
+    const rehypeRawPlugin: MDXPlugin = [
+      rehypeRaw,
+      {
+        passThrough: [
+          'mdxFlowExpression',
+          'mdxJsxFlowElement',
+          'mdxJsxTextElement',
+          'mdxTextExpression',
+          'mdxjsEsm',
+        ],
+      },
+    ];
+    rehypePlugins.unshift(rehypeRawPlugin);
+  }
+
+  const processorOptions: ProcessorOptions & Options = {
+    ...options,
+    remarkPlugins,
+    rehypePlugins,
+    recmaPlugins,
+    providerImportSource: '@mdx-js/react',
+  };
+
+  const mdxProcessor = createMdxProcessor({
+    ...processorOptions,
+    remarkRehypeOptions: options.markdownConfig.remarkRehypeOptions,
+    format,
+  });
+
+  return {
+    process: async ({content, filePath, frontMatter, compilerName}) => {
+      const vfile = new VFile({
+        value: content,
+        path: filePath,
+        data: {
+          frontMatter,
+          compilerName,
+        },
+      });
+      return mdxProcessor.process(vfile).then((result) => ({
+        content: result.toString(),
+        data: result.data,
+      }));
+    },
+  };
+}
+
+// We use different compilers depending on the file type (md vs mdx)
+export type SimpleProcessors = {
+  mdProcessor: SimpleProcessor;
+  mdxProcessor: SimpleProcessor;
+};
+
+// Compilers are cached so that Remark/Rehype plugins can run
+// expensive code during initialization
+const ProcessorsCache = new Map<string | Options, SimpleProcessors>();
+
+export function createProcessors({
+  options,
+}: {
+  options: Options;
+}): SimpleProcessors {
+  return {
+    mdProcessor: createProcessorUncached({
+      options,
+      format: 'md',
+    }),
+    mdxProcessor: createProcessorUncached({
+      options,
+      format: 'mdx',
+    }),
+  };
+}
+
+function createProcessorsCacheEntry({
+  options,
+}: {
+  options: Options;
+}): SimpleProcessors {
+  const compilers = ProcessorsCache.get(options);
+  if (compilers) {
+    return compilers;
+  }
+  const processors = createProcessors({options});
+  ProcessorsCache.set(options, processors);
+  return processors;
+}
+
+export function getProcessor({
+  filePath,
+  mdxFrontMatter,
+  options,
+}: {
+  filePath: string;
+  mdxFrontMatter: MDXFrontMatter;
+  options: Options;
+}): SimpleProcessor {
+  const processors =
+    options.processors ?? createProcessorsCacheEntry({options});
+
+  const format = getFormat({
+    filePath,
+    frontMatterFormat: mdxFrontMatter.format,
+    markdownConfigFormat: options.markdownConfig.format,
+  });
+
+  return format === 'md' ? processors.mdProcessor : processors.mdxProcessor;
+}