@docusaurus/utils 2.0.0-beta.15d451942 → 2.0.0-beta.16

package/src/index.ts

@@ -5,32 +5,88 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import chalk from 'chalk';
+import logger from '@docusaurus/logger';
 import path from 'path';
 import {createHash} from 'crypto';
-import {camelCase, kebabCase, mapValues} from 'lodash';
-import escapeStringRegexp from 'escape-string-regexp';
+import _ from 'lodash';
 import fs from 'fs-extra';
 import {URL} from 'url';
-import {
+import type {
   ReportingSeverity,
   TranslationFileContent,
   TranslationFile,
 } from '@docusaurus/types';
 
-// @ts-expect-error: no typedefs :s
 import resolvePathnameUnsafe from 'resolve-pathname';
 
-import {posixPath as posixPathImport} from './posixPath';
-
-export const posixPath = posixPathImport;
-
-export * from './codeTranslationsUtils';
-export * from './markdownParser';
-export * from './markdownLinks';
-export * from './escapePath';
-
-const fileHash = new Map();
+import {simpleHash, docuHash} from './hashUtils';
+import {DEFAULT_PLUGIN_ID} from './constants';
+
+export {
+  NODE_MAJOR_VERSION,
+  NODE_MINOR_VERSION,
+  DEFAULT_BUILD_DIR_NAME,
+  DEFAULT_CONFIG_FILE_NAME,
+  BABEL_CONFIG_FILE_NAME,
+  GENERATED_FILES_DIR_NAME,
+  SRC_DIR_NAME,
+  STATIC_DIR_NAME,
+  OUTPUT_STATIC_ASSETS_DIR_NAME,
+  THEME_PATH,
+  DEFAULT_PORT,
+  DEFAULT_PLUGIN_ID,
+  WEBPACK_URL_LOADER_LIMIT,
+} from './constants';
+export {getFileCommitDate, GitNotFoundError} from './gitUtils';
+export {normalizeUrl, getEditUrl} from './urlUtils';
+export {
+  type Tag,
+  type FrontMatterTag,
+  type TaggedItemGroup,
+  normalizeFrontMatterTag,
+  normalizeFrontMatterTags,
+  groupTaggedItems,
+} from './tags';
+export {
+  parseMarkdownHeadingId,
+  createExcerpt,
+  parseFrontMatter,
+  parseMarkdownContentTitle,
+  parseMarkdownString,
+} from './markdownParser';
+export {
+  type ContentPaths,
+  type BrokenMarkdownLink,
+  type ReplaceMarkdownLinksParams,
+  type ReplaceMarkdownLinksReturn,
+  replaceMarkdownLinks,
+} from './markdownLinks';
+export {type SluggerOptions, type Slugger, createSlugger} from './slugger';
+export {
+  isNameTooLong,
+  shortName,
+  posixPath,
+  toMessageRelativeFilePath,
+  aliasedSitePath,
+  escapePath,
+} from './pathUtils';
+export {md5Hash, simpleHash, docuHash} from './hashUtils';
+export {
+  Globby,
+  GlobExcludeDefault,
+  createMatcher,
+  createAbsoluteFilePathMatcher,
+} from './globUtils';
+export {getFileLoaderUtils} from './webpackUtils';
+export {
+  getDataFilePath,
+  getDataFileData,
+  getContentPathList,
+  findFolderContainingFile,
+  getFolderContainingFile,
+} from './dataFileUtils';
+
+const fileHash = new Map<string, string>();
 export async function generate(
   generatedFilesDir: string,
   file: string,
@@ -50,7 +106,7 @@ export async function generate(
   // If file already exists but its not in runtime cache yet,
   // we try to calculate the content hash and then compare
   // This is to avoid unnecessary overwriting and we can reuse old file.
-  if (!lastHash && fs.existsSync(filepath)) {
+  if (!lastHash && (await fs.pathExists(filepath))) {
     const lastContent = await fs.readFile(filepath, 'utf8');
     lastHash = createHash('md5').update(lastContent).digest('hex');
     fileHash.set(filepath, lastHash);
@@ -65,20 +121,8 @@ export async function generate(
   }
 }
 
-export function objectWithKeySorted<T>(
-  obj: Record<string, T>,
-): Record<string, T> {
-  // https://github.com/lodash/lodash/issues/1459#issuecomment-460941233
-  return Object.keys(obj)
-    .sort()
-    .reduce((acc: Record<string, T>, key: string) => {
-      acc[key] = obj[key];
-      return acc;
-    }, {});
-}
-
-const indexRE = /(^|.*\/)index\.(md|mdx|js|jsx|ts|tsx)$/i;
-const extRE = /\.(md|mdx|js|jsx|ts|tsx)$/;
+const indexRE = /(?<dirname>^|.*\/)index\.(?:mdx?|jsx?|tsx?)$/i;
+const extRE = /\.(?:mdx?|jsx?|tsx?)$/;
 
 /**
  * Convert filepath to url path.
@@ -91,60 +135,13 @@ export function fileToPath(file: string): string {
   return `/${file.replace(extRE, '').replace(/\\/g, '/')}`;
 }
 
-export function encodePath(userpath: string): string {
-  return userpath
+export function encodePath(userPath: string): string {
+  return userPath
     .split('/')
     .map((item) => encodeURIComponent(item))
     .join('/');
 }
 
-export function simpleHash(str: string, length: number): string {
-  return createHash('md5').update(str).digest('hex').substr(0, length);
-}
-
-/**
- * Given an input string, convert to kebab-case and append a hash.
- * Avoid str collision.
- */
-export function docuHash(str: string): string {
-  if (str === '/') {
-    return 'index';
-  }
-  const shortHash = simpleHash(str, 3);
-  return `${kebabCase(str)}-${shortHash}`;
-}
-
-/**
- * Convert first string character to the upper case.
- * E.g: docusaurus -> Docusaurus
- */
-export function upperFirst(str: string): string {
-  return str ? str.charAt(0).toUpperCase() + str.slice(1) : '';
-}
-
-/**
- * Generate unique React Component Name.
- * E.g: /foo-bar -> FooBar096
- */
-export function genComponentName(pagePath: string): string {
-  if (pagePath === '/') {
-    return 'index';
-  }
-  const pageHash = docuHash(pagePath);
-  return upperFirst(camelCase(pageHash));
-}
-
-// When you want to display a path in a message/warning/error,
-// it's more convenient to:
-// - make it relative to cwd()
-// - convert to posix (ie not using windows \ path separator)
-// This way, Jest tests can run more reliably on any computer/CI
-// on both Unix/Windows
-// For Windows users this is not perfect (as they see / instead of \) but it's probably good enough
-export function toMessageRelativeFilePath(filePath: string): string {
-  return posixPath(path.relative(process.cwd(), filePath));
-}
-
 const chunkNameCache = new Map();
 /**
  * Generate unique chunk name given a module path.
@@ -173,126 +170,6 @@ export function genChunkName(
   return chunkName;
 }
 
-// Too dynamic
-// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types, @typescript-eslint/no-explicit-any
-export function idx(target: any, keyPaths?: string | (string | number)[]): any {
-  return (
-    target &&
-    keyPaths &&
-    (Array.isArray(keyPaths)
-      ? keyPaths.reduce((obj, key) => obj && obj[key], target)
-      : target[keyPaths])
-  );
-}
-
-/**
- * Given a filepath and dirpath, get the first directory.
- */
-export function getSubFolder(file: string, refDir: string): string | null {
-  const separator = escapeStringRegexp(path.sep);
-  const baseDir = escapeStringRegexp(path.basename(refDir));
-  const regexSubFolder = new RegExp(
-    `${baseDir}${separator}(.*?)${separator}.*`,
-  );
-  const match = regexSubFolder.exec(file);
-  return match && match[1];
-}
-
-export function normalizeUrl(rawUrls: string[]): string {
-  const urls = [...rawUrls];
-  const resultArray = [];
-
-  let hasStartingSlash = false;
-  let hasEndingSlash = false;
-
-  // If the first part is a plain protocol, we combine it with the next part.
-  if (urls[0].match(/^[^/:]+:\/*$/) && urls.length > 1) {
-    const first = urls.shift();
-    urls[0] = first + urls[0];
-  }
-
-  // There must be two or three slashes in the file protocol,
-  // two slashes in anything else.
-  const replacement = urls[0].match(/^file:\/\/\//) ? '$1:///' : '$1://';
-  urls[0] = urls[0].replace(/^([^/:]+):\/*/, replacement);
-
-  // eslint-disable-next-line
-  for (let i = 0; i < urls.length; i++) {
-    let component = urls[i];
-
-    if (typeof component !== 'string') {
-      throw new TypeError(`Url must be a string. Received ${typeof component}`);
-    }
-
-    if (component === '') {
-      if (i === urls.length - 1 && hasEndingSlash) {
-        resultArray.push('/');
-      }
-      // eslint-disable-next-line
-      continue;
-    }
-
-    if (component !== '/') {
-      if (i > 0) {
-        // Removing the starting slashes for each component but the first.
-        component = component.replace(
-          /^[/]+/,
-          // Special case where the first element of rawUrls is empty ["", "/hello"] => /hello
-          component[0] === '/' && !hasStartingSlash ? '/' : '',
-        );
-      }
-
-      hasEndingSlash = component[component.length - 1] === '/';
-      // Removing the ending slashes for each component but the last.
-      // For the last component we will combine multiple slashes to a single one.
-      component = component.replace(/[/]+$/, i < urls.length - 1 ? '' : '/');
-    }
-
-    hasStartingSlash = true;
-    resultArray.push(component);
-  }
-
-  let str = resultArray.join('/');
-  // Each input component is now separated by a single slash
-  // except the possible first plain protocol part.
-
-  // Remove trailing slash before parameters or hash.
-  str = str.replace(/\/(\?|&|#[^!])/g, '$1');
-
-  // Replace ? in parameters with &.
-  const parts = str.split('?');
-  str = parts.shift() + (parts.length > 0 ? '?' : '') + parts.join('&');
-
-  // Dedupe forward slashes in the entire path, avoiding protocol slashes.
-  str = str.replace(/([^:]\/)\/+/g, '$1');
-
-  // Dedupe forward slashes at the beginning of the path.
-  str = str.replace(/^\/+/g, '/');
-
-  return str;
-}
-
-/**
- * Alias filepath relative to site directory, very useful so that we
- * don't expose user's site structure.
- * Example: some/path/to/website/docs/foo.md -> @site/docs/foo.md
- */
-export function aliasedSitePath(filePath: string, siteDir: string): string {
-  const relativePath = posixPath(path.relative(siteDir, filePath));
-  // Cannot use path.join() as it resolves '../' and removes
-  // the '@site'. Let webpack loader resolve it.
-  return `@site/${relativePath}`;
-}
-
-export function getEditUrl(
-  fileRelativePath: string,
-  editUrl?: string,
-): string | undefined {
-  return editUrl
-    ? normalizeUrl([editUrl, posixPath(fileRelativePath)])
-    : undefined;
-}
-
 export function isValidPathname(str: string): boolean {
   if (!str.startsWith('/')) {
     return false;
@@ -301,7 +178,7 @@ export function isValidPathname(str: string): boolean {
     // weird, but is there a better way?
     const parsedPathname = new URL(str, 'https://domain.com').pathname;
     return parsedPathname === str || parsedPathname === encodeURI(str);
-  } catch (e) {
+  } catch {
     return false;
   }
 }
@@ -313,13 +190,18 @@ export function resolvePathname(to: string, from?: string): string {
 export function addLeadingSlash(str: string): string {
   return str.startsWith('/') ? str : `/${str}`;
 }
-export function addTrailingSlash(str: string): string {
-  return str.endsWith('/') ? str : `${str}/`;
-}
+
 export function addTrailingPathSeparator(str: string): string {
-  return str.endsWith(path.sep) ? str : `${str}${path.sep}`;
+  return str.endsWith(path.sep)
+    ? str
+    : // If this is Windows, we need to change the forward slash to backward
+      `${str.replace(/\/$/, '')}${path.sep}`;
 }
 
+// TODO deduplicate: also present in @docusaurus/utils-common
+export function addTrailingSlash(str: string): string {
+  return str.endsWith('/') ? str : `${str}/`;
+}
 export function removeTrailingSlash(str: string): string {
   return removeSuffix(str, '/');
 }
@@ -335,13 +217,7 @@ export function removePrefix(str: string, prefix: string): string {
   return str.startsWith(prefix) ? str.slice(prefix.length) : str;
 }
 
-export function getFilePathForRoutePath(routePath: string): string {
-  const fileName = path.basename(routePath);
-  const filePath = path.dirname(routePath);
-  return path.join(filePath, `${fileName}/index.html`);
-}
-
-export function getElementsAround<T extends unknown>(
+export function getElementsAround<T>(
   array: T[],
   aroundIndex: number,
 ): {
@@ -352,7 +228,7 @@ export function getElementsAround<T extends unknown>(
   const max = array.length - 1;
   if (aroundIndex < min || aroundIndex > max) {
     throw new Error(
-      `Valid aroundIndex for array (of size ${array.length}) are between ${min} and ${max}, but you provided aroundIndex=${aroundIndex}`,
+      `Valid "aroundIndex" for array (of size ${array.length}) are between ${min} and ${max}, but you provided ${aroundIndex}.`,
     );
   }
   const previous = aroundIndex === min ? undefined : array[aroundIndex - 1];
@@ -364,7 +240,7 @@ export function getPluginI18nPath({
   siteDir,
   locale,
   pluginName,
-  pluginId = 'default', // TODO duplicated constant
+  pluginId = DEFAULT_PLUGIN_ID,
   subPaths = [],
 }: {
   siteDir: string;
@@ -376,26 +252,58 @@ export function getPluginI18nPath({
   return path.join(
     siteDir,
     'i18n',
-    // namespace first by locale: convenient to work in a single folder for a translator
+    // namespace first by locale: convenient to work in a single folder for a
+    // translator
     locale,
     // Make it convenient to use for single-instance
     // ie: return "docs", not "docs-default" nor "docs/default"
-    `${pluginName}${
-      // TODO duplicate constant :(
-      pluginId === 'default' ? '' : `-${pluginId}`
-    }`,
+    `${pluginName}${pluginId === DEFAULT_PLUGIN_ID ? '' : `-${pluginId}`}`,
     ...subPaths,
   );
 }
 
-export async function mapAsyncSequencial<T extends unknown, R extends unknown>(
+/**
+ * @param permalink The URL that the HTML file corresponds to, without base URL
+ * @param outDir Full path to the output directory
+ * @param trailingSlash The site config option. If provided, only one path will
+ * be read.
+ * @returns This returns a buffer, which you have to decode string yourself if
+ * needed. (Not always necessary since the output isn't for human consumption
+ * anyways, and most HTML manipulation libs accept buffers)
+ */
+export async function readOutputHTMLFile(
+  permalink: string,
+  outDir: string,
+  trailingSlash: boolean | undefined,
+): Promise<Buffer> {
+  const withTrailingSlashPath = path.join(outDir, permalink, 'index.html');
+  const withoutTrailingSlashPath = path.join(
+    outDir,
+    `${permalink.replace(/\/$/, '')}.html`,
+  );
+  if (trailingSlash) {
+    return fs.readFile(withTrailingSlashPath);
+  } else if (trailingSlash === false) {
+    return fs.readFile(withoutTrailingSlashPath);
+  }
+  const HTMLPath = await findAsyncSequential(
+    [withTrailingSlashPath, withoutTrailingSlashPath],
+    fs.pathExists,
+  );
+  if (!HTMLPath) {
+    throw new Error(
+      `Expected output HTML file to be found at ${withTrailingSlashPath}`,
+    );
+  }
+  return fs.readFile(HTMLPath);
+}
+
+export async function mapAsyncSequential<T, R>(
   array: T[],
   action: (t: T) => Promise<R>,
 ): Promise<R[]> {
   const results: R[] = [];
-  // eslint-disable-next-line no-restricted-syntax
   for (const t of array) {
-    // eslint-disable-next-line no-await-in-loop
     const result = await action(t);
     results.push(result);
   }
@@ -406,9 +314,7 @@ export async function findAsyncSequential<T>(
   array: T[],
   predicate: (t: T) => Promise<boolean>,
 ): Promise<T | undefined> {
-  // eslint-disable-next-line no-restricted-syntax
   for (const t of array) {
-    // eslint-disable-next-line no-await-in-loop
     if (await predicate(t)) {
       return t;
     }
@@ -416,35 +322,6 @@ export async function findAsyncSequential<T>(
   return undefined;
 }
 
-// return the  first folder path in which the file exists in
-export async function findFolderContainingFile(
-  folderPaths: string[],
-  relativeFilePath: string,
-): Promise<string | undefined> {
-  return findAsyncSequential(folderPaths, (folderPath) =>
-    fs.pathExists(path.join(folderPath, relativeFilePath)),
-  );
-}
-
-export async function getFolderContainingFile(
-  folderPaths: string[],
-  relativeFilePath: string,
-): Promise<string> {
-  const maybeFolderPath = await findFolderContainingFile(
-    folderPaths,
-    relativeFilePath,
-  );
-  // should never happen, as the source was read from the FS anyway...
-  if (!maybeFolderPath) {
-    throw new Error(
-      `relativeFilePath=[${relativeFilePath}] does not exist in any of these folders: \n- ${folderPaths.join(
-        '\n- ',
-      )}]`,
-    );
-  }
-  return maybeFolderPath;
-}
-
 export function reportMessage(
   message: string,
   reportingSeverity: ReportingSeverity,
@@ -453,19 +330,19 @@ export function reportMessage(
     case 'ignore':
       break;
     case 'log':
-      console.log(chalk.bold.blue('info ') + chalk.blue(message));
+      logger.info(message);
       break;
     case 'warn':
-      console.warn(chalk.bold.yellow('warn ') + chalk.yellow(message));
+      logger.warn(message);
       break;
     case 'error':
-      console.error(chalk.bold.red('error ') + chalk.red(message));
+      logger.error(message);
       break;
     case 'throw':
       throw new Error(message);
     default:
       throw new Error(
-        `unexpected reportingSeverity value: ${reportingSeverity}`,
+        `Unexpected "reportingSeverity" value: ${reportingSeverity}.`,
       );
   }
 }
@@ -473,23 +350,7 @@ export function reportMessage(
 export function mergeTranslations(
   contents: TranslationFileContent[],
 ): TranslationFileContent {
-  return contents.reduce((acc, content) => {
-    return {...acc, ...content};
-  }, {});
-}
-
-export function getSwizzledComponent(
-  componentPath: string,
-): string | undefined {
-  const swizzledComponentPath = path.resolve(
-    process.cwd(),
-    'src',
-    componentPath,
-  );
-
-  return fs.existsSync(swizzledComponentPath)
-    ? swizzledComponentPath
-    : undefined;
+  return contents.reduce((acc, content) => ({...acc, ...content}), {});
 }
 
 // Useful to update all the messages of a translation file
@@ -500,29 +361,9 @@ export function updateTranslationFileMessages(
 ): TranslationFile {
   return {
     ...translationFile,
-    content: mapValues(translationFile.content, (translation) => ({
+    content: _.mapValues(translationFile.content, (translation) => ({
       ...translation,
       message: updateMessage(translation.message),
     })),
   };
 }
-
-// Input: ## Some heading {#some-heading}
-// Output: {text: "## Some heading", id: "some-heading"}
-export function parseMarkdownHeadingId(
-  heading: string,
-): {
-  text: string;
-  id?: string;
-} {
-  const customHeadingIdRegex = /^(.*?)\s*\{#([\w-]+)\}$/;
-  const matches = customHeadingIdRegex.exec(heading);
-  if (matches) {
-    return {
-      text: matches[1],
-      id: matches[2],
-    };
-  } else {
-    return {text: heading, id: undefined};
-  }
-}

package/src/markdownLinks.ts

@@ -5,8 +5,8 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import {resolve} from 'url';
-import {aliasedSitePath} from './index';
+import path from 'path';
+import {aliasedSitePath} from './pathUtils';
 
 export type ContentPaths = {
   contentPath: string;
@@ -45,9 +45,17 @@ export function replaceMarkdownLinks<T extends ContentPaths>({
 
   // Replace internal markdown linking (except in fenced blocks).
   let fencedBlock = false;
+  let lastCodeFence = '';
   const lines = fileString.split('\n').map((line) => {
     if (line.trim().startsWith('```')) {
-      fencedBlock = !fencedBlock;
+      if (!fencedBlock) {
+        fencedBlock = true;
+        [lastCodeFence] = line.trim().match(/^`+/)!;
+        // If we are in a ````-fenced block, all ``` would be plain text instead
+        // of fences
+      } else if (line.trim().match(/^`+/)![0].length >= lastCodeFence.length) {
+        fencedBlock = false;
+      }
     }
     if (fencedBlock) {
       return line;
@@ -55,24 +63,38 @@ export function replaceMarkdownLinks<T extends ContentPaths>({
 
     let modifiedLine = line;
     // Replace inline-style links or reference-style links e.g:
-    // This is [Document 1](doc1.md) -> we replace this doc1.md with correct link
+    // This is [Document 1](doc1.md) -> we replace this doc1.md with correct
+    // ink
     // [doc1]: doc1.md -> we replace this doc1.md with correct link
-    const mdRegex = /(?:(?:\]\()|(?:\]:\s?))(?!https)([^'")\]\s>]+\.mdx?)/g;
+    const mdRegex =
+      /(?:(?:\]\()|(?:\]:\s*))(?!https?:\/\/|@site\/)(?<filename>[^'")\]\s>]+\.mdx?)/g;
     let mdMatch = mdRegex.exec(modifiedLine);
     while (mdMatch !== null) {
       // Replace it to correct html link.
-      const mdLink = mdMatch[1];
+      const mdLink = mdMatch.groups!.filename;
+
+      const sourcesToTry = [
+        path.resolve(path.dirname(filePath), decodeURIComponent(mdLink)),
+        `${contentPathLocalized}/${decodeURIComponent(mdLink)}`,
+        `${contentPath}/${decodeURIComponent(mdLink)}`,
+      ];
 
-      const aliasedSource = (source: string) =>
-        aliasedSitePath(source, siteDir);
+      const aliasedSourceMatch = sourcesToTry
+        .map((source) => aliasedSitePath(source, siteDir))
+        .find((source) => sourceToPermalink[source]);
 
-      const permalink: string | undefined =
-        sourceToPermalink[aliasedSource(resolve(filePath, mdLink))] ||
-        sourceToPermalink[aliasedSource(`${contentPathLocalized}/${mdLink}`)] ||
-        sourceToPermalink[aliasedSource(`${contentPath}/${mdLink}`)];
+      const permalink: string | undefined = aliasedSourceMatch
+        ? sourceToPermalink[aliasedSourceMatch]
+        : undefined;
 
       if (permalink) {
-        modifiedLine = modifiedLine.replace(mdLink, permalink);
+        // MDX won't be happy if the permalink contains a space, we need to
+        // convert it to %20
+        const encodedPermalink = permalink
+          .split('/')
+          .map((part) => part.replace(/\s/g, '%20'))
+          .join('/');
+        modifiedLine = modifiedLine.replace(mdLink, encodedPermalink);
       } else {
         const brokenMarkdownLink: BrokenMarkdownLink<T> = {
           contentPaths,