@docusaurus/utils 2.0.0-beta.0f144213d → 2.0.0-beta.11

package/src/markdownLinks.ts

@@ -5,7 +5,7 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import {resolve} from 'url';
+import path from 'path';
 import {aliasedSitePath} from './index';
 
 export type ContentPaths = {
@@ -63,16 +63,27 @@ export function replaceMarkdownLinks<T extends ContentPaths>({
       // Replace it to correct html link.
       const mdLink = mdMatch[1];
 
-      const aliasedSource = (source: string) =>
-        aliasedSitePath(source, siteDir);
+      const sourcesToTry = [
+        path.resolve(path.dirname(filePath), decodeURIComponent(mdLink)),
+        `${contentPathLocalized}/${decodeURIComponent(mdLink)}`,
+        `${contentPath}/${decodeURIComponent(mdLink)}`,
+      ];
 
-      const permalink: string | undefined =
-        sourceToPermalink[aliasedSource(resolve(filePath, mdLink))] ||
-        sourceToPermalink[aliasedSource(`${contentPathLocalized}/${mdLink}`)] ||
-        sourceToPermalink[aliasedSource(`${contentPath}/${mdLink}`)];
+      const aliasedSourceMatch = sourcesToTry
+        .map((source) => aliasedSitePath(source, siteDir))
+        .find((source) => sourceToPermalink[source]);
+
+      const permalink: string | undefined = aliasedSourceMatch
+        ? sourceToPermalink[aliasedSourceMatch]
+        : undefined;
 
       if (permalink) {
-        modifiedLine = modifiedLine.replace(mdLink, permalink);
+        // MDX won't be happy if the permalink contains a space, we need to convert it to %20
+        const encodedPermalink = permalink
+          .split('/')
+          .map((part) => part.replace(/\s/g, '%20'))
+          .join('/');
+        modifiedLine = modifiedLine.replace(mdLink, encodedPermalink);
       } else {
         const brokenMarkdownLink: BrokenMarkdownLink<T> = {
           contentPaths,

package/src/markdownParser.ts

@@ -18,6 +18,7 @@ export function createExcerpt(fileString: string): string | undefined {
     // Remove Markdown alternate title
     .replace(/^[^\n]*\n[=]+/g, '')
     .split('\n');
+  let inCode = false;
 
   /* eslint-disable no-continue */
   // eslint-disable-next-line no-restricted-syntax
@@ -32,6 +33,14 @@ export function createExcerpt(fileString: string): string | undefined {
       continue;
     }
 
+    // Skip code block line.
+    if (fileLine.trim().startsWith('```')) {
+      inCode = !inCode;
+      continue;
+    } else if (inCode) {
+      continue;
+    }
+
     const cleanedLine = fileLine
       // Remove HTML tags.
       .replace(/<[^>]*>/g, '')
@@ -67,9 +76,7 @@ export function createExcerpt(fileString: string): string | undefined {
   return undefined;
 }
 
-export function parseFrontMatter(
-  markdownFileContent: string,
-): {
+export function parseFrontMatter(markdownFileContent: string): {
   frontMatter: Record<string, unknown>;
   content: string;
 } {
@@ -98,10 +105,11 @@ export function parseMarkdownContentTitle(
 
   const content = contentUntrimmed.trim();
 
-  const IMPORT_STATEMENT = /import\s+(([\w*{}\s\n,]+)from\s+)?["'\s]([@\w/_.-]+)["'\s];?|\n/
-    .source;
-  const REGULAR_TITLE = /(?<pattern>#\s*(?<title>[^#\n{]*)+[ \t]*(?<suffix>({#*[\w-]+})|#)?\n*?)/
-    .source;
+  const IMPORT_STATEMENT =
+    /import\s+(([\w*{}\s\n,]+)from\s+)?["'\s]([@\w/_.-]+)["'\s];?|\n/.source;
+  const REGULAR_TITLE =
+    /(?<pattern>#\s*(?<title>[^#\n{]*)+[ \t]*(?<suffix>({#*[\w-]+})|#)?\n*?)/
+      .source;
   const ALTERNATE_TITLE = /(?<pattern>\s*(?<title>[^\n]*)\s*\n[=]+)/.source;
 
   const regularTitleMatch = new RegExp(
@@ -141,9 +149,8 @@ export function parseMarkdownString(
   options?: {removeContentTitle?: boolean},
 ): ParsedMarkdown {
   try {
-    const {frontMatter, content: contentWithoutFrontMatter} = parseFrontMatter(
-      markdownFileContent,
-    );
+    const {frontMatter, content: contentWithoutFrontMatter} =
+      parseFrontMatter(markdownFileContent);
 
     const {content, contentTitle} = parseMarkdownContentTitle(
       contentWithoutFrontMatter,
@@ -176,7 +183,7 @@ export async function parseMarkdownFile(
     return parseMarkdownString(markdownString, options);
   } catch (e) {
     throw new Error(
-      `Error while parsing Markdown file ${source}: "${e.message}".`,
+      `Error while parsing Markdown file ${source}: "${(e as Error).message}".`,
     );
   }
 }

package/src/mdxUtils.ts

@@ -0,0 +1,32 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import React from 'react';
+import ReactDOMServer from 'react-dom/server';
+import MDX from '@mdx-js/runtime';
+import removeImports from 'remark-mdx-remove-imports';
+import removeExports from 'remark-mdx-remove-exports';
+
+/**
+ * Transform mdx text to plain html text
+ * Initially created to convert MDX blog posts to HTML for the RSS feed
+ * without import/export nodes
+ *
+ * TODO not ideal implementation, won't work well with MDX elements!
+ * TODO theme+global site config should be able to declare MDX comps in scope for rendering the RSS feeds
+ * see also https://github.com/facebook/docusaurus/issues/4625
+ */
+export function mdxToHtml(
+  mdxStr: string,
+  // TODO allow providing components/scope here, see https://github.com/mdx-js/mdx/tree/v1.6.13/packages/runtime
+): string {
+  return ReactDOMServer.renderToString(
+    React.createElement(MDX, {remarkPlugins: [removeImports, removeExports]}, [
+      mdxStr,
+    ]),
+  );
+}

package/src/normalizeUrl.ts

@@ -0,0 +1,80 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+export function normalizeUrl(rawUrls: string[]): string {
+  const urls = [...rawUrls];
+  const resultArray = [];
+
+  let hasStartingSlash = false;
+  let hasEndingSlash = false;
+
+  // If the first part is a plain protocol, we combine it with the next part.
+  if (urls[0].match(/^[^/:]+:\/*$/) && urls.length > 1) {
+    const first = urls.shift();
+    urls[0] = first + urls[0];
+  }
+
+  // There must be two or three slashes in the file protocol,
+  // two slashes in anything else.
+  const replacement = urls[0].match(/^file:\/\/\//) ? '$1:///' : '$1://';
+  urls[0] = urls[0].replace(/^([^/:]+):\/*/, replacement);
+
+  // eslint-disable-next-line
+  for (let i = 0; i < urls.length; i++) {
+    let component = urls[i];
+
+    if (typeof component !== 'string') {
+      throw new TypeError(`Url must be a string. Received ${typeof component}`);
+    }
+
+    if (component === '') {
+      if (i === urls.length - 1 && hasEndingSlash) {
+        resultArray.push('/');
+      }
+      // eslint-disable-next-line
+      continue;
+    }
+
+    if (component !== '/') {
+      if (i > 0) {
+        // Removing the starting slashes for each component but the first.
+        component = component.replace(
+          /^[/]+/,
+          // Special case where the first element of rawUrls is empty ["", "/hello"] => /hello
+          component[0] === '/' && !hasStartingSlash ? '/' : '',
+        );
+      }
+
+      hasEndingSlash = component[component.length - 1] === '/';
+      // Removing the ending slashes for each component but the last.
+      // For the last component we will combine multiple slashes to a single one.
+      component = component.replace(/[/]+$/, i < urls.length - 1 ? '' : '/');
+    }
+
+    hasStartingSlash = true;
+    resultArray.push(component);
+  }
+
+  let str = resultArray.join('/');
+  // Each input component is now separated by a single slash
+  // except the possible first plain protocol part.
+
+  // Remove trailing slash before parameters or hash.
+  str = str.replace(/\/(\?|&|#[^!])/g, '$1');
+
+  // Replace ? in parameters with &.
+  const parts = str.split('?');
+  str = parts.shift() + (parts.length > 0 ? '?' : '') + parts.join('&');
+
+  // Dedupe forward slashes in the entire path, avoiding protocol slashes.
+  str = str.replace(/([^:]\/)\/+/g, '$1');
+
+  // Dedupe forward slashes at the beginning of the path.
+  str = str.replace(/^\/+/g, '/');
+
+  return str;
+}

package/src/pathUtils.ts

@@ -16,11 +16,10 @@ const SPACE_FOR_APPENDING = 10;
 const isMacOs = process.platform === `darwin`;
 const isWindows = process.platform === `win32`;
 
-export const isNameTooLong = (str: string): boolean => {
-  return isMacOs || isWindows
+export const isNameTooLong = (str: string): boolean =>
+  isMacOs || isWindows
     ? str.length + SPACE_FOR_APPENDING > MAX_PATH_SEGMENT_CHARS // MacOS (APFS) and Windows (NTFS) filename length limit (255 chars)
     : Buffer.from(str).length + SPACE_FOR_APPENDING > MAX_PATH_SEGMENT_BYTES; // Other (255 bytes)
-};
 
 export const shortName = (str: string): string => {
   if (isMacOs || isWindows) {

package/src/slugger.ts

@@ -0,0 +1,24 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import GithubSlugger from 'github-slugger';
+
+// We create our own abstraction on top of the lib:
+// - unify usage everywhere in the codebase
+// - ability to add extra options
+export type SluggerOptions = {maintainCase?: boolean};
+
+export type Slugger = {
+  slug: (value: string, options?: SluggerOptions) => string;
+};
+
+export function createSlugger(): Slugger {
+  const githubSlugger = new GithubSlugger();
+  return {
+    slug: (value, options) => githubSlugger.slug(value, options?.maintainCase),
+  };
+}

package/src/tags.ts

@@ -0,0 +1,100 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {kebabCase, uniq, uniqBy} from 'lodash';
+import {normalizeUrl} from './normalizeUrl';
+
+export type Tag = {
+  label: string;
+  permalink: string;
+};
+
+export type FrontMatterTag = string | Tag;
+
+export function normalizeFrontMatterTag(
+  tagsPath: string,
+  frontMatterTag: FrontMatterTag,
+): Tag {
+  function toTagObject(tagString: string): Tag {
+    return {
+      label: tagString,
+      permalink: kebabCase(tagString),
+    };
+  }
+
+  // TODO maybe make ensure the permalink is valid url path?
+  function normalizeTagPermalink(permalink: string): string {
+    // note: we always apply tagsPath on purpose
+    // for versioned docs, v1/doc.md and v2/doc.md tags with custom permalinks don't lead to the same created page
+    // tagsPath is different for each doc version
+    return normalizeUrl([tagsPath, permalink]);
+  }
+
+  const tag: Tag =
+    typeof frontMatterTag === 'string'
+      ? toTagObject(frontMatterTag)
+      : frontMatterTag;
+
+  return {
+    label: tag.label,
+    permalink: normalizeTagPermalink(tag.permalink),
+  };
+}
+
+export function normalizeFrontMatterTags(
+  tagsPath: string,
+  frontMatterTags: FrontMatterTag[] | undefined,
+): Tag[] {
+  const tags =
+    frontMatterTags?.map((tag) => normalizeFrontMatterTag(tagsPath, tag)) ?? [];
+
+  return uniqBy(tags, (tag) => tag.permalink);
+}
+
+export type TaggedItemGroup<Item> = {
+  tag: Tag;
+  items: Item[];
+};
+
+// Permits to group docs/blogPosts by tag (provided by FrontMatter)
+// Note: groups are indexed by permalink, because routes must be unique in the end
+// Labels may vary on 2 md files but they are normalized.
+// Docs with label='some label' and label='some-label' should end-up in the same group/page in the end
+// We can't create 2 routes /some-label because one would override the other
+export function groupTaggedItems<Item>(
+  items: Item[],
+  getItemTags: (item: Item) => Tag[],
+): Record<string, TaggedItemGroup<Item>> {
+  const result: Record<string, TaggedItemGroup<Item>> = {};
+
+  function handleItemTag(item: Item, tag: Tag) {
+    // Init missing tag groups
+    // TODO: it's not really clear what should be the behavior if 2 items have the same tag but the permalink is different for each
+    // For now, the first tag found wins
+    result[tag.permalink] = result[tag.permalink] ?? {
+      tag,
+      items: [],
+    };
+
+    // Add item to group
+    result[tag.permalink].items.push(item);
+  }
+
+  items.forEach((item) => {
+    getItemTags(item).forEach((tag) => {
+      handleItemTag(item, tag);
+    });
+  });
+
+  // If user add twice the same tag to a md doc (weird but possible),
+  // we don't want the item to appear twice in the list...
+  Object.values(result).forEach((group) => {
+    group.items = uniq(group.items);
+  });
+
+  return result;
+}

package/src/webpackUtils.ts

@@ -0,0 +1,144 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import type {RuleSetRule} from 'webpack';
+import path from 'path';
+import {posixPath} from './posixPath';
+import {
+  WEBPACK_URL_LOADER_LIMIT,
+  OUTPUT_STATIC_ASSETS_DIR_NAME,
+} from './constants';
+
+type AssetFolder = 'images' | 'files' | 'fonts' | 'medias';
+
+type FileLoaderUtils = {
+  loaders: {
+    file: (options: {folder: AssetFolder}) => RuleSetRule;
+    url: (options: {folder: AssetFolder}) => RuleSetRule;
+    inlineMarkdownImageFileLoader: string;
+    inlineMarkdownLinkFileLoader: string;
+  };
+  rules: {
+    images: () => RuleSetRule;
+    fonts: () => RuleSetRule;
+    media: () => RuleSetRule;
+    svg: () => RuleSetRule;
+    otherAssets: () => RuleSetRule;
+  };
+};
+
+// Inspired by https://github.com/gatsbyjs/gatsby/blob/8e6e021014da310b9cc7d02e58c9b3efe938c665/packages/gatsby/src/utils/webpack-utils.ts#L447
+export function getFileLoaderUtils(): FileLoaderUtils {
+  // files/images < urlLoaderLimit will be inlined as base64 strings directly in the html
+  const urlLoaderLimit = WEBPACK_URL_LOADER_LIMIT;
+
+  // defines the path/pattern of the assets handled by webpack
+  const fileLoaderFileName = (folder: AssetFolder) =>
+    `${OUTPUT_STATIC_ASSETS_DIR_NAME}/${folder}/[name]-[hash].[ext]`;
+
+  const loaders: FileLoaderUtils['loaders'] = {
+    file: (options: {folder: AssetFolder}) => ({
+      loader: require.resolve(`file-loader`),
+      options: {
+        name: fileLoaderFileName(options.folder),
+      },
+    }),
+    url: (options: {folder: AssetFolder}) => ({
+      loader: require.resolve('url-loader'),
+      options: {
+        limit: urlLoaderLimit,
+        name: fileLoaderFileName(options.folder),
+        fallback: require.resolve('file-loader'),
+      },
+    }),
+
+    // TODO find a better solution to avoid conflicts with the ideal-image plugin
+    // TODO this may require a little breaking change for ideal-image users?
+    // Maybe with the ideal image plugin, all md images should be "ideal"?
+    // This is used to force url-loader+file-loader on markdown images
+    // https://webpack.js.org/concepts/loaders/#inline
+    inlineMarkdownImageFileLoader: `!${posixPath(
+      require.resolve('url-loader'),
+    )}?limit=${urlLoaderLimit}&name=${fileLoaderFileName(
+      'images',
+    )}&fallback=${posixPath(require.resolve('file-loader'))}!`,
+    inlineMarkdownLinkFileLoader: `!${posixPath(
+      require.resolve('file-loader'),
+    )}?name=${fileLoaderFileName('files')}!`,
+  };
+
+  const rules: FileLoaderUtils['rules'] = {
+    /**
+     * Loads image assets, inlines images via a data URI if they are below
+     * the size threshold
+     */
+    images: () => ({
+      use: [loaders.url({folder: 'images'})],
+      test: /\.(ico|jpg|jpeg|png|gif|webp)(\?.*)?$/,
+    }),
+
+    fonts: () => ({
+      use: [loaders.url({folder: 'fonts'})],
+      test: /\.(woff|woff2|eot|ttf|otf)$/,
+    }),
+
+    /**
+     * Loads audio and video and inlines them via a data URI if they are below
+     * the size threshold
+     */
+    media: () => ({
+      use: [loaders.url({folder: 'medias'})],
+      test: /\.(mp4|webm|ogv|wav|mp3|m4a|aac|oga|flac)$/,
+    }),
+
+    svg: () => ({
+      test: /\.svg?$/,
+      oneOf: [
+        {
+          use: [
+            {
+              loader: require.resolve('@svgr/webpack'),
+              options: {
+                prettier: false,
+                svgo: true,
+                svgoConfig: {
+                  plugins: [
+                    {
+                      name: 'preset-default',
+                      params: {
+                        overrides: {
+                          removeViewBox: false,
+                        },
+                      },
+                    },
+                  ],
+                },
+                titleProp: true,
+                ref: ![path],
+              },
+            },
+          ],
+          // We don't want to use SVGR loader for non-React source code
+          // ie we don't want to use SVGR for CSS files...
+          issuer: {
+            and: [/\.(ts|tsx|js|jsx|md|mdx)$/],
+          },
+        },
+        {
+          use: [loaders.url({folder: 'images'})],
+        },
+      ],
+    }),
+
+    otherAssets: () => ({
+      use: [loaders.file({folder: 'files'})],
+      test: /\.(pdf|doc|docx|xls|xlsx|zip|rar)$/,
+    }),
+  };
+
+  return {loaders, rules};
+}

package/lib/codeTranslationsUtils.d.ts

@@ -1,11 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-export declare function codeTranslationLocalesToTry(locale: string): string[];
-export declare function readDefaultCodeTranslationMessages({ dirPath, locale, }: {
-    dirPath: string;
-    locale: string;
-}): Promise<Record<string, string>>;

package/lib/codeTranslationsUtils.js

@@ -1,50 +0,0 @@
-"use strict";
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.readDefaultCodeTranslationMessages = exports.codeTranslationLocalesToTry = void 0;
-const tslib_1 = require("tslib");
-const path_1 = tslib_1.__importDefault(require("path"));
-const fs_extra_1 = tslib_1.__importDefault(require("fs-extra"));
-// Return an ordered list of locales we should try
-function codeTranslationLocalesToTry(locale) {
-    // @ts-expect-error: TODO until available in TS, see https://github.com/microsoft/TypeScript/issues/37326
-    const intlLocale = Intl.Locale ? new Intl.Locale(locale) : undefined;
-    if (!intlLocale) {
-        return [locale];
-    }
-    // if locale is just a simple language like "pt", we want to fallback to pt-BR (not pt-PT!)
-    // see https://github.com/facebook/docusaurus/pull/4536#issuecomment-810088783
-    if (intlLocale.language === locale) {
-        const maximizedLocale = intlLocale.maximize(); // pt-Latn-BR`
-        // ["pt","pt-BR"]
-        return [locale, `${maximizedLocale.language}-${maximizedLocale.region}`];
-    }
-    // if locale is like "pt-BR", we want to fallback to "pt"
-    else {
-        return [locale, intlLocale.language];
-    }
-}
-exports.codeTranslationLocalesToTry = codeTranslationLocalesToTry;
-// Useful to implement getDefaultCodeTranslationMessages() in themes
-async function readDefaultCodeTranslationMessages({ dirPath, locale, }) {
-    const localesToTry = codeTranslationLocalesToTry(locale);
-    // Return the content of the first file that match
-    // fr_FR.json => fr.json => nothing
-    // eslint-disable-next-line no-restricted-syntax
-    for (const fileName of localesToTry) {
-        const filePath = path_1.default.resolve(dirPath, `${fileName}.json`);
-        // eslint-disable-next-line no-await-in-loop
-        if (await fs_extra_1.default.pathExists(filePath)) {
-            // eslint-disable-next-line no-await-in-loop
-            const fileContent = await fs_extra_1.default.readFile(filePath, 'utf8');
-            return JSON.parse(fileContent);
-        }
-    }
-    return {};
-}
-exports.readDefaultCodeTranslationMessages = readDefaultCodeTranslationMessages;

package/src/__tests__/__fixtures__/defaultCodeTranslations/en.json

@@ -1,4 +0,0 @@
-{
-  "id1": "message 1 en",
-  "id2": "message 2 en"
-}

package/src/__tests__/__fixtures__/defaultCodeTranslations/fr-FR.json

@@ -1,5 +0,0 @@
-{
-  "id1": "message 1 fr_FR",
-  "id2": "message 2 fr_FR",
-  "id3": "message 3 fr_FR"
-}

package/src/__tests__/__fixtures__/defaultCodeTranslations/fr.json

@@ -1,4 +0,0 @@
-{
-  "id1": "message 1 fr",
-  "id2": "message 2 fr"
-}