@docusaurus/utils 2.0.0-beta.16 → 2.0.0-beta.19

package/src/index.ts

@@ -5,45 +5,62 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import logger from '@docusaurus/logger';
-import path from 'path';
-import {createHash} from 'crypto';
-import _ from 'lodash';
-import fs from 'fs-extra';
-import {URL} from 'url';
-import type {
-  ReportingSeverity,
-  TranslationFileContent,
-  TranslationFile,
-} from '@docusaurus/types';
-
-import resolvePathnameUnsafe from 'resolve-pathname';
-
-import {simpleHash, docuHash} from './hashUtils';
-import {DEFAULT_PLUGIN_ID} from './constants';
-
 export {
   NODE_MAJOR_VERSION,
   NODE_MINOR_VERSION,
+  DOCUSAURUS_VERSION,
   DEFAULT_BUILD_DIR_NAME,
   DEFAULT_CONFIG_FILE_NAME,
   BABEL_CONFIG_FILE_NAME,
   GENERATED_FILES_DIR_NAME,
   SRC_DIR_NAME,
-  STATIC_DIR_NAME,
+  DEFAULT_STATIC_DIR_NAME,
   OUTPUT_STATIC_ASSETS_DIR_NAME,
   THEME_PATH,
+  I18N_DIR_NAME,
+  CODE_TRANSLATIONS_FILE_NAME,
   DEFAULT_PORT,
   DEFAULT_PLUGIN_ID,
   WEBPACK_URL_LOADER_LIMIT,
 } from './constants';
-export {getFileCommitDate, GitNotFoundError} from './gitUtils';
-export {normalizeUrl, getEditUrl} from './urlUtils';
+export {generate, readOutputHTMLFile} from './emitUtils';
+export {
+  getFileCommitDate,
+  FileNotTrackedError,
+  GitNotFoundError,
+} from './gitUtils';
+export {
+  mergeTranslations,
+  updateTranslationFileMessages,
+  getPluginI18nPath,
+  localizePath,
+} from './i18nUtils';
+export {
+  removeSuffix,
+  removePrefix,
+  mapAsyncSequential,
+  findAsyncSequential,
+  reportMessage,
+} from './jsUtils';
+export {
+  normalizeUrl,
+  getEditUrl,
+  fileToPath,
+  encodePath,
+  isValidPathname,
+  resolvePathname,
+  addLeadingSlash,
+  addTrailingSlash,
+  removeTrailingSlash,
+  hasSSHProtocol,
+  buildHttpsUrl,
+  buildSshUrl,
+} from './urlUtils';
 export {
   type Tag,
+  type TagsListItem,
+  type TagModule,
   type FrontMatterTag,
-  type TaggedItemGroup,
-  normalizeFrontMatterTag,
   normalizeFrontMatterTags,
   groupTaggedItems,
 } from './tags';
@@ -53,12 +70,12 @@ export {
   parseFrontMatter,
   parseMarkdownContentTitle,
   parseMarkdownString,
-} from './markdownParser';
+  writeMarkdownHeadingId,
+  type WriteHeadingIDOptions,
+} from './markdownUtils';
 export {
   type ContentPaths,
   type BrokenMarkdownLink,
-  type ReplaceMarkdownLinksParams,
-  type ReplaceMarkdownLinksReturn,
   replaceMarkdownLinks,
 } from './markdownLinks';
 export {type SluggerOptions, type Slugger, createSlugger} from './slugger';
@@ -69,6 +86,7 @@ export {
   toMessageRelativeFilePath,
   aliasedSitePath,
   escapePath,
+  addTrailingPathSeparator,
 } from './pathUtils';
 export {md5Hash, simpleHash, docuHash} from './hashUtils';
 export {
@@ -85,285 +103,3 @@ export {
   findFolderContainingFile,
   getFolderContainingFile,
 } from './dataFileUtils';
-
-const fileHash = new Map<string, string>();
-export async function generate(
-  generatedFilesDir: string,
-  file: string,
-  content: string,
-  skipCache: boolean = process.env.NODE_ENV === 'production',
-): Promise<void> {
-  const filepath = path.join(generatedFilesDir, file);
-
-  if (skipCache) {
-    await fs.ensureDir(path.dirname(filepath));
-    await fs.writeFile(filepath, content);
-    return;
-  }
-
-  let lastHash = fileHash.get(filepath);
-
-  // If file already exists but its not in runtime cache yet,
-  // we try to calculate the content hash and then compare
-  // This is to avoid unnecessary overwriting and we can reuse old file.
-  if (!lastHash && (await fs.pathExists(filepath))) {
-    const lastContent = await fs.readFile(filepath, 'utf8');
-    lastHash = createHash('md5').update(lastContent).digest('hex');
-    fileHash.set(filepath, lastHash);
-  }
-
-  const currentHash = createHash('md5').update(content).digest('hex');
-
-  if (lastHash !== currentHash) {
-    await fs.ensureDir(path.dirname(filepath));
-    await fs.writeFile(filepath, content);
-    fileHash.set(filepath, currentHash);
-  }
-}
-
-const indexRE = /(?<dirname>^|.*\/)index\.(?:mdx?|jsx?|tsx?)$/i;
-const extRE = /\.(?:mdx?|jsx?|tsx?)$/;
-
-/**
- * Convert filepath to url path.
- * Example: 'index.md' -> '/', 'foo/bar.js' -> '/foo/bar',
- */
-export function fileToPath(file: string): string {
-  if (indexRE.test(file)) {
-    return file.replace(indexRE, '/$1');
-  }
-  return `/${file.replace(extRE, '').replace(/\\/g, '/')}`;
-}
-
-export function encodePath(userPath: string): string {
-  return userPath
-    .split('/')
-    .map((item) => encodeURIComponent(item))
-    .join('/');
-}
-
-const chunkNameCache = new Map();
-/**
- * Generate unique chunk name given a module path.
- */
-export function genChunkName(
-  modulePath: string,
-  prefix?: string,
-  preferredName?: string,
-  shortId: boolean = process.env.NODE_ENV === 'production',
-): string {
-  let chunkName: string | undefined = chunkNameCache.get(modulePath);
-  if (!chunkName) {
-    if (shortId) {
-      chunkName = simpleHash(modulePath, 8);
-    } else {
-      let str = modulePath;
-      if (preferredName) {
-        const shortHash = simpleHash(modulePath, 3);
-        str = `${preferredName}${shortHash}`;
-      }
-      const name = str === '/' ? 'index' : docuHash(str);
-      chunkName = prefix ? `${prefix}---${name}` : name;
-    }
-    chunkNameCache.set(modulePath, chunkName);
-  }
-  return chunkName;
-}
-
-export function isValidPathname(str: string): boolean {
-  if (!str.startsWith('/')) {
-    return false;
-  }
-  try {
-    // weird, but is there a better way?
-    const parsedPathname = new URL(str, 'https://domain.com').pathname;
-    return parsedPathname === str || parsedPathname === encodeURI(str);
-  } catch {
-    return false;
-  }
-}
-
-// resolve pathname and fail fast if resolution fails
-export function resolvePathname(to: string, from?: string): string {
-  return resolvePathnameUnsafe(to, from);
-}
-export function addLeadingSlash(str: string): string {
-  return str.startsWith('/') ? str : `/${str}`;
-}
-
-export function addTrailingPathSeparator(str: string): string {
-  return str.endsWith(path.sep)
-    ? str
-    : // If this is Windows, we need to change the forward slash to backward
-      `${str.replace(/\/$/, '')}${path.sep}`;
-}
-
-// TODO deduplicate: also present in @docusaurus/utils-common
-export function addTrailingSlash(str: string): string {
-  return str.endsWith('/') ? str : `${str}/`;
-}
-export function removeTrailingSlash(str: string): string {
-  return removeSuffix(str, '/');
-}
-
-export function removeSuffix(str: string, suffix: string): string {
-  if (suffix === '') {
-    return str; // always returns "" otherwise!
-  }
-  return str.endsWith(suffix) ? str.slice(0, -suffix.length) : str;
-}
-
-export function removePrefix(str: string, prefix: string): string {
-  return str.startsWith(prefix) ? str.slice(prefix.length) : str;
-}
-
-export function getElementsAround<T>(
-  array: T[],
-  aroundIndex: number,
-): {
-  next: T | undefined;
-  previous: T | undefined;
-} {
-  const min = 0;
-  const max = array.length - 1;
-  if (aroundIndex < min || aroundIndex > max) {
-    throw new Error(
-      `Valid "aroundIndex" for array (of size ${array.length}) are between ${min} and ${max}, but you provided ${aroundIndex}.`,
-    );
-  }
-  const previous = aroundIndex === min ? undefined : array[aroundIndex - 1];
-  const next = aroundIndex === max ? undefined : array[aroundIndex + 1];
-  return {previous, next};
-}
-
-export function getPluginI18nPath({
-  siteDir,
-  locale,
-  pluginName,
-  pluginId = DEFAULT_PLUGIN_ID,
-  subPaths = [],
-}: {
-  siteDir: string;
-  locale: string;
-  pluginName: string;
-  pluginId?: string | undefined;
-  subPaths?: string[];
-}): string {
-  return path.join(
-    siteDir,
-    'i18n',
-    // namespace first by locale: convenient to work in a single folder for a
-    // translator
-    locale,
-    // Make it convenient to use for single-instance
-    // ie: return "docs", not "docs-default" nor "docs/default"
-    `${pluginName}${pluginId === DEFAULT_PLUGIN_ID ? '' : `-${pluginId}`}`,
-    ...subPaths,
-  );
-}
-
-/**
- * @param permalink The URL that the HTML file corresponds to, without base URL
- * @param outDir Full path to the output directory
- * @param trailingSlash The site config option. If provided, only one path will
- * be read.
- * @returns This returns a buffer, which you have to decode string yourself if
- * needed. (Not always necessary since the output isn't for human consumption
- * anyways, and most HTML manipulation libs accept buffers)
- */
-export async function readOutputHTMLFile(
-  permalink: string,
-  outDir: string,
-  trailingSlash: boolean | undefined,
-): Promise<Buffer> {
-  const withTrailingSlashPath = path.join(outDir, permalink, 'index.html');
-  const withoutTrailingSlashPath = path.join(
-    outDir,
-    `${permalink.replace(/\/$/, '')}.html`,
-  );
-  if (trailingSlash) {
-    return fs.readFile(withTrailingSlashPath);
-  } else if (trailingSlash === false) {
-    return fs.readFile(withoutTrailingSlashPath);
-  }
-  const HTMLPath = await findAsyncSequential(
-    [withTrailingSlashPath, withoutTrailingSlashPath],
-    fs.pathExists,
-  );
-  if (!HTMLPath) {
-    throw new Error(
-      `Expected output HTML file to be found at ${withTrailingSlashPath}`,
-    );
-  }
-  return fs.readFile(HTMLPath);
-}
-
-export async function mapAsyncSequential<T, R>(
-  array: T[],
-  action: (t: T) => Promise<R>,
-): Promise<R[]> {
-  const results: R[] = [];
-  for (const t of array) {
-    const result = await action(t);
-    results.push(result);
-  }
-  return results;
-}
-
-export async function findAsyncSequential<T>(
-  array: T[],
-  predicate: (t: T) => Promise<boolean>,
-): Promise<T | undefined> {
-  for (const t of array) {
-    if (await predicate(t)) {
-      return t;
-    }
-  }
-  return undefined;
-}
-
-export function reportMessage(
-  message: string,
-  reportingSeverity: ReportingSeverity,
-): void {
-  switch (reportingSeverity) {
-    case 'ignore':
-      break;
-    case 'log':
-      logger.info(message);
-      break;
-    case 'warn':
-      logger.warn(message);
-      break;
-    case 'error':
-      logger.error(message);
-      break;
-    case 'throw':
-      throw new Error(message);
-    default:
-      throw new Error(
-        `Unexpected "reportingSeverity" value: ${reportingSeverity}.`,
-      );
-  }
-}
-
-export function mergeTranslations(
-  contents: TranslationFileContent[],
-): TranslationFileContent {
-  return contents.reduce((acc, content) => ({...acc, ...content}), {});
-}
-
-// Useful to update all the messages of a translation file
-// Used in tests to simulate translations
-export function updateTranslationFileMessages(
-  translationFile: TranslationFile,
-  updateMessage: (message: string) => string,
-): TranslationFile {
-  return {
-    ...translationFile,
-    content: _.mapValues(translationFile.content, (translation) => ({
-      ...translation,
-      message: updateMessage(translation.message),
-    })),
-  };
-}

package/src/jsUtils.ts

@@ -0,0 +1,102 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import type {ReportingSeverity} from '@docusaurus/types';
+import logger from '@docusaurus/logger';
+
+/** Removes a given string suffix from `str`. */
+export function removeSuffix(str: string, suffix: string): string {
+  if (suffix === '') {
+    // str.slice(0, 0) is ""
+    return str;
+  }
+  return str.endsWith(suffix) ? str.slice(0, -suffix.length) : str;
+}
+
+/** Removes a given string prefix from `str`. */
+export function removePrefix(str: string, prefix: string): string {
+  return str.startsWith(prefix) ? str.slice(prefix.length) : str;
+}
+
+/**
+ * `Array#map` for async operations where order matters.
+ * @param array The array to traverse.
+ * @param action An async action to be performed on every array item. Will be
+ * awaited before working on the next.
+ * @returns The list of results returned from every `action(item)`
+ */
+export async function mapAsyncSequential<T, R>(
+  array: T[],
+  action: (t: T) => Promise<R>,
+): Promise<R[]> {
+  const results: R[] = [];
+  for (const t of array) {
+    const result = await action(t);
+    results.push(result);
+  }
+  return results;
+}
+
+/**
+ * `Array#find` for async operations where order matters.
+ * @param array The array to traverse.
+ * @param predicate An async predicate to be called on every array item. Should
+ * return a boolean indicating whether the currently element should be returned.
+ * @returns The function immediately returns the first item on which `predicate`
+ * returns `true`, or `undefined` if none matches the predicate.
+ */
+export async function findAsyncSequential<T>(
+  array: T[],
+  predicate: (t: T) => Promise<boolean>,
+): Promise<T | undefined> {
+  for (const t of array) {
+    if (await predicate(t)) {
+      return t;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Takes a message and reports it according to the severity that the user wants.
+ *
+ * - `ignore`: completely no-op
+ * - `log`: uses the `INFO` log level
+ * - `warn`: uses the `WARN` log level
+ * - `error`: uses the `ERROR` log level
+ * - `throw`: aborts the process, throws the error.
+ *
+ * Since the logger doesn't have logging level filters yet, these severities
+ * mostly just differ by their colors.
+ *
+ * @throws In addition to throwing when `reportingSeverity === "throw"`, this
+ * function also throws if `reportingSeverity` is not one of the above.
+ */
+export function reportMessage(
+  message: string,
+  reportingSeverity: ReportingSeverity,
+): void {
+  switch (reportingSeverity) {
+    case 'ignore':
+      break;
+    case 'log':
+      logger.info(message);
+      break;
+    case 'warn':
+      logger.warn(message);
+      break;
+    case 'error':
+      logger.error(message);
+      break;
+    case 'throw':
+      throw new Error(message);
+    default:
+      throw new Error(
+        `Unexpected "reportingSeverity" value: ${reportingSeverity}.`,
+      );
+  }
+}

package/src/markdownLinks.ts

@@ -6,41 +6,79 @@
  */
 
 import path from 'path';
+import {getContentPathList} from './dataFileUtils';
 import {aliasedSitePath} from './pathUtils';
 
+/**
+ * Content plugins have a base path and a localized path to source content from.
+ * We will look into the localized path in priority.
+ */
 export type ContentPaths = {
+  /**
+   * The absolute path to the base content directory, like `"<siteDir>/docs"`.
+   */
   contentPath: string;
+  /**
+   * The absolute path to the localized content directory, like
+   * `"<siteDir>/i18n/zh-Hans/plugin-content-docs"`.
+   */
   contentPathLocalized: string;
 };
 
+/** Data structure representing each broken Markdown link to be reported. */
 export type BrokenMarkdownLink<T extends ContentPaths> = {
+  /** Absolute path to the file containing this link. */
   filePath: string;
+  /**
+   * This is generic because it may contain extra metadata like version name,
+   * which the reporter can provide for context.
+   */
   contentPaths: T;
+  /**
+   * The content of the link, like `"./brokenFile.md"`
+   */
   link: string;
 };
 
-export type ReplaceMarkdownLinksParams<T extends ContentPaths> = {
-  siteDir: string;
-  fileString: string;
-  filePath: string;
-  contentPaths: T;
-  sourceToPermalink: Record<string, string>;
-};
-
-export type ReplaceMarkdownLinksReturn<T extends ContentPaths> = {
-  newContent: string;
-  brokenMarkdownLinks: BrokenMarkdownLink<T>[];
-};
-
+/**
+ * Takes a Markdown file and replaces relative file references with their URL
+ * counterparts, e.g. `[link](./intro.md)` => `[link](/docs/intro)`, preserving
+ * everything else.
+ *
+ * This method uses best effort to find a matching file. The file reference can
+ * be relative to the directory of the current file (most likely) or any of the
+ * content paths (so `/tutorials/intro.md` can be resolved as
+ * `<siteDir>/docs/tutorials/intro.md`). Links that contain the `http(s):` or
+ * `@site/` prefix will always be ignored.
+ */
 export function replaceMarkdownLinks<T extends ContentPaths>({
   siteDir,
   fileString,
   filePath,
   contentPaths,
   sourceToPermalink,
-}: ReplaceMarkdownLinksParams<T>): ReplaceMarkdownLinksReturn<T> {
-  const {contentPath, contentPathLocalized} = contentPaths;
-
+}: {
+  /** Absolute path to the site directory, used to resolve aliased paths. */
+  siteDir: string;
+  /** The Markdown file content to be processed. */
+  fileString: string;
+  /** Absolute path to the current file containing `fileString`. */
+  filePath: string;
+  /** The content paths which the file reference may live in. */
+  contentPaths: T;
+  /**
+   * A map from source paths to their URLs. Source paths are `@site` aliased.
+   */
+  sourceToPermalink: {[aliasedPath: string]: string};
+}): {
+  /**
+   * The content with all Markdown file references replaced with their URLs.
+   * Unresolved links are left as-is.
+   */
+  newContent: string;
+  /** The list of broken links,  */
+  brokenMarkdownLinks: BrokenMarkdownLink<T>[];
+} {
   const brokenMarkdownLinks: BrokenMarkdownLink<T>[] = [];
 
   // Replace internal markdown linking (except in fenced blocks).
@@ -48,12 +86,13 @@ export function replaceMarkdownLinks<T extends ContentPaths>({
   let lastCodeFence = '';
   const lines = fileString.split('\n').map((line) => {
     if (line.trim().startsWith('```')) {
+      const codeFence = line.trim().match(/^`+/)![0]!;
       if (!fencedBlock) {
         fencedBlock = true;
-        [lastCodeFence] = line.trim().match(/^`+/)!;
+        lastCodeFence = codeFence;
         // If we are in a ````-fenced block, all ``` would be plain text instead
         // of fences
-      } else if (line.trim().match(/^`+/)![0].length >= lastCodeFence.length) {
+      } else if (codeFence.length >= lastCodeFence.length) {
         fencedBlock = false;
       }
     }
@@ -63,23 +102,27 @@ export function replaceMarkdownLinks<T extends ContentPaths>({
 
     let modifiedLine = line;
     // Replace inline-style links or reference-style links e.g:
-    // This is [Document 1](doc1.md) -> we replace this doc1.md with correct
-    // ink
-    // [doc1]: doc1.md -> we replace this doc1.md with correct link
+    // This is [Document 1](doc1.md)
+    // [doc1]: doc1.md
     const mdRegex =
-      /(?:(?:\]\()|(?:\]:\s*))(?!https?:\/\/|@site\/)(?<filename>[^'")\]\s>]+\.mdx?)/g;
+      /(?:\]\(|\]:\s*)(?!https?:\/\/|@site\/)(?<filename>[^'")\]\s>]+\.mdx?)/g;
     let mdMatch = mdRegex.exec(modifiedLine);
     while (mdMatch !== null) {
       // Replace it to correct html link.
-      const mdLink = mdMatch.groups!.filename;
+      const mdLink = mdMatch.groups!.filename!;
 
-      const sourcesToTry = [
-        path.resolve(path.dirname(filePath), decodeURIComponent(mdLink)),
-        `${contentPathLocalized}/${decodeURIComponent(mdLink)}`,
-        `${contentPath}/${decodeURIComponent(mdLink)}`,
-      ];
+      const sourcesToTry: string[] = [];
+      // ./file.md and ../file.md are always relative to the current file
+      if (!mdLink.startsWith('./') && !mdLink.startsWith('../')) {
+        sourcesToTry.push(...getContentPathList(contentPaths), siteDir);
+      }
+      // /file.md is always relative to the content path
+      if (!mdLink.startsWith('/')) {
+        sourcesToTry.push(path.dirname(filePath));
+      }
 
       const aliasedSourceMatch = sourcesToTry
+        .map((p) => path.join(p, decodeURIComponent(mdLink)))
         .map((source) => aliasedSitePath(source, siteDir))
         .find((source) => sourceToPermalink[source]);