@docusaurus/utils 2.0.0-beta.208c08983 → 2.0.0-beta.21

package/src/jsUtils.ts

@@ -0,0 +1,102 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import logger from '@docusaurus/logger';
+import type {ReportingSeverity} from '@docusaurus/types';
+
+/** Removes a given string suffix from `str`. */
+export function removeSuffix(str: string, suffix: string): string {
+  if (suffix === '') {
+    // str.slice(0, 0) is ""
+    return str;
+  }
+  return str.endsWith(suffix) ? str.slice(0, -suffix.length) : str;
+}
+
+/** Removes a given string prefix from `str`. */
+export function removePrefix(str: string, prefix: string): string {
+  return str.startsWith(prefix) ? str.slice(prefix.length) : str;
+}
+
+/**
+ * `Array#map` for async operations where order matters.
+ * @param array The array to traverse.
+ * @param action An async action to be performed on every array item. Will be
+ * awaited before working on the next.
+ * @returns The list of results returned from every `action(item)`
+ */
+export async function mapAsyncSequential<T, R>(
+  array: T[],
+  action: (t: T) => Promise<R>,
+): Promise<R[]> {
+  const results: R[] = [];
+  for (const t of array) {
+    const result = await action(t);
+    results.push(result);
+  }
+  return results;
+}
+
+/**
+ * `Array#find` for async operations where order matters.
+ * @param array The array to traverse.
+ * @param predicate An async predicate to be called on every array item. Should
+ * return a boolean indicating whether the currently element should be returned.
+ * @returns The function immediately returns the first item on which `predicate`
+ * returns `true`, or `undefined` if none matches the predicate.
+ */
+export async function findAsyncSequential<T>(
+  array: T[],
+  predicate: (t: T) => Promise<boolean>,
+): Promise<T | undefined> {
+  for (const t of array) {
+    if (await predicate(t)) {
+      return t;
+    }
+  }
+  return undefined;
+}
+
+/**
+ * Takes a message and reports it according to the severity that the user wants.
+ *
+ * - `ignore`: completely no-op
+ * - `log`: uses the `INFO` log level
+ * - `warn`: uses the `WARN` log level
+ * - `error`: uses the `ERROR` log level
+ * - `throw`: aborts the process, throws the error.
+ *
+ * Since the logger doesn't have logging level filters yet, these severities
+ * mostly just differ by their colors.
+ *
+ * @throws In addition to throwing when `reportingSeverity === "throw"`, this
+ * function also throws if `reportingSeverity` is not one of the above.
+ */
+export function reportMessage(
+  message: string,
+  reportingSeverity: ReportingSeverity,
+): void {
+  switch (reportingSeverity) {
+    case 'ignore':
+      break;
+    case 'log':
+      logger.info(message);
+      break;
+    case 'warn':
+      logger.warn(message);
+      break;
+    case 'error':
+      logger.error(message);
+      break;
+    case 'throw':
+      throw new Error(message);
+    default:
+      throw new Error(
+        `Unexpected "reportingSeverity" value: ${reportingSeverity}.`,
+      );
+  }
+}

package/src/markdownLinks.ts

@@ -5,49 +5,96 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import {resolve} from 'url';
-import {aliasedSitePath} from './index';
+import path from 'path';
+import {getContentPathList} from './dataFileUtils';
+import {aliasedSitePath} from './pathUtils';
 
+/**
+ * Content plugins have a base path and a localized path to source content from.
+ * We will look into the localized path in priority.
+ */
 export type ContentPaths = {
+  /**
+   * The absolute path to the base content directory, like `"<siteDir>/docs"`.
+   */
   contentPath: string;
+  /**
+   * The absolute path to the localized content directory, like
+   * `"<siteDir>/i18n/zh-Hans/plugin-content-docs"`.
+   */
   contentPathLocalized: string;
 };
 
+/** Data structure representing each broken Markdown link to be reported. */
 export type BrokenMarkdownLink<T extends ContentPaths> = {
+  /** Absolute path to the file containing this link. */
   filePath: string;
+  /**
+   * This is generic because it may contain extra metadata like version name,
+   * which the reporter can provide for context.
+   */
   contentPaths: T;
+  /**
+   * The content of the link, like `"./brokenFile.md"`
+   */
   link: string;
 };
 
-export type ReplaceMarkdownLinksParams<T extends ContentPaths> = {
-  siteDir: string;
-  fileString: string;
-  filePath: string;
-  contentPaths: T;
-  sourceToPermalink: Record<string, string>;
-};
-
-export type ReplaceMarkdownLinksReturn<T extends ContentPaths> = {
-  newContent: string;
-  brokenMarkdownLinks: BrokenMarkdownLink<T>[];
-};
-
+/**
+ * Takes a Markdown file and replaces relative file references with their URL
+ * counterparts, e.g. `[link](./intro.md)` => `[link](/docs/intro)`, preserving
+ * everything else.
+ *
+ * This method uses best effort to find a matching file. The file reference can
+ * be relative to the directory of the current file (most likely) or any of the
+ * content paths (so `/tutorials/intro.md` can be resolved as
+ * `<siteDir>/docs/tutorials/intro.md`). Links that contain the `http(s):` or
+ * `@site/` prefix will always be ignored.
+ */
 export function replaceMarkdownLinks<T extends ContentPaths>({
   siteDir,
   fileString,
   filePath,
   contentPaths,
   sourceToPermalink,
-}: ReplaceMarkdownLinksParams<T>): ReplaceMarkdownLinksReturn<T> {
-  const {contentPath, contentPathLocalized} = contentPaths;
-
+}: {
+  /** Absolute path to the site directory, used to resolve aliased paths. */
+  siteDir: string;
+  /** The Markdown file content to be processed. */
+  fileString: string;
+  /** Absolute path to the current file containing `fileString`. */
+  filePath: string;
+  /** The content paths which the file reference may live in. */
+  contentPaths: T;
+  /**
+   * A map from source paths to their URLs. Source paths are `@site` aliased.
+   */
+  sourceToPermalink: {[aliasedPath: string]: string};
+}): {
+  /**
+   * The content with all Markdown file references replaced with their URLs.
+   * Unresolved links are left as-is.
+   */
+  newContent: string;
+  /** The list of broken links,  */
+  brokenMarkdownLinks: BrokenMarkdownLink<T>[];
+} {
   const brokenMarkdownLinks: BrokenMarkdownLink<T>[] = [];
 
   // Replace internal markdown linking (except in fenced blocks).
   let fencedBlock = false;
+  let lastCodeFence = '';
   const lines = fileString.split('\n').map((line) => {
     if (line.trim().startsWith('```')) {
-      fencedBlock = !fencedBlock;
+      const codeFence = line.trim().match(/^`+/)![0]!;
+      if (!fencedBlock) {
+        fencedBlock = true;
+        lastCodeFence = codeFence;
+        // If we are in a ````-fenced block, all ``` would be plain text instead
+        // of fences
+      } else if (codeFence.length >= lastCodeFence.length) {
+        fencedBlock = false;
+      }
     }
     if (fencedBlock) {
       return line;
@@ -55,24 +102,48 @@ export function replaceMarkdownLinks<T extends ContentPaths>({
 
     let modifiedLine = line;
     // Replace inline-style links or reference-style links e.g:
-    // This is [Document 1](doc1.md) -> we replace this doc1.md with correct link
-    // [doc1]: doc1.md -> we replace this doc1.md with correct link
-    const mdRegex = /(?:(?:\]\()|(?:\]:\s?))(?!https)([^'")\]\s>]+\.mdx?)/g;
+    // This is [Document 1](doc1.md)
+    // [doc1]: doc1.md
+    const mdRegex =
+      /(?:\]\(|\]:\s*)(?!https?:\/\/|@site\/)(?<filename>[^'")\]\s>]+\.mdx?)/g;
     let mdMatch = mdRegex.exec(modifiedLine);
     while (mdMatch !== null) {
       // Replace it to correct html link.
-      const mdLink = mdMatch[1];
+      const mdLink = mdMatch.groups!.filename!;
+
+      const sourcesToTry: string[] = [];
+      // ./file.md and ../file.md are always relative to the current file
+      if (!mdLink.startsWith('./') && !mdLink.startsWith('../')) {
+        sourcesToTry.push(...getContentPathList(contentPaths), siteDir);
+      }
+      // /file.md is always relative to the content path
+      if (!mdLink.startsWith('/')) {
+        sourcesToTry.push(path.dirname(filePath));
+      }
 
-      const aliasedSource = (source: string) =>
-        aliasedSitePath(source, siteDir);
+      const aliasedSourceMatch = sourcesToTry
+        .map((p) => path.join(p, decodeURIComponent(mdLink)))
+        .map((source) => aliasedSitePath(source, siteDir))
+        .find((source) => sourceToPermalink[source]);
 
-      const permalink: string | undefined =
-        sourceToPermalink[aliasedSource(resolve(filePath, mdLink))] ||
-        sourceToPermalink[aliasedSource(`${contentPathLocalized}/${mdLink}`)] ||
-        sourceToPermalink[aliasedSource(`${contentPath}/${mdLink}`)];
+      const permalink: string | undefined = aliasedSourceMatch
+        ? sourceToPermalink[aliasedSourceMatch]
+        : undefined;
 
       if (permalink) {
-        modifiedLine = modifiedLine.replace(mdLink, permalink);
+        // MDX won't be happy if the permalink contains a space, we need to
+        // convert it to %20
+        const encodedPermalink = permalink
+          .split('/')
+          .map((part) => part.replace(/\s/g, '%20'))
+          .join('/');
+        modifiedLine = modifiedLine.replace(
+          mdMatch[0]!,
+          mdMatch[0]!.replace(mdLink, encodedPermalink),
+        );
+        // Adjust the lastIndex to avoid passing over the next link if the
+        // newly replaced URL is shorter.
+        mdRegex.lastIndex += encodedPermalink.length - mdLink.length;
       } else {
         const brokenMarkdownLink: BrokenMarkdownLink<T> = {
           contentPaths,

package/src/markdownUtils.ts

@@ -0,0 +1,357 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import logger from '@docusaurus/logger';
+import matter from 'gray-matter';
+import {createSlugger, type Slugger, type SluggerOptions} from './slugger';
+
+// Some utilities for parsing Markdown content. These things are only used on
+// server-side when we infer metadata like `title` and `description` from the
+// content. Most parsing is still done in MDX through the mdx-loader.
+
+/**
+ * Parses custom ID from a heading. The ID must be composed of letters,
+ * underscores, and dashes only.
+ *
+ * @param heading e.g. `## Some heading {#some-heading}` where the last
+ * character must be `}` for the ID to be recognized
+ */
+export function parseMarkdownHeadingId(heading: string): {
+  /**
+   * The heading content sans the ID part, right-trimmed. e.g. `## Some heading`
+   */
+  text: string;
+  /** The heading ID. e.g. `some-heading` */
+  id?: string;
+} {
+  const customHeadingIdRegex = /\s*\{#(?<id>[\w-]+)\}$/;
+  const matches = customHeadingIdRegex.exec(heading);
+  if (matches) {
+    return {
+      text: heading.replace(matches[0]!, ''),
+      id: matches.groups!.id!,
+    };
+  }
+  return {text: heading, id: undefined};
+}
+
+// TODO: Find a better way to do so, possibly by compiling the Markdown content,
+// stripping out HTML tags and obtaining the first line.
+/**
+ * Creates an excerpt of a Markdown file. This function will:
+ *
+ * - Ignore h1 headings (setext or atx)
+ * - Ignore import/export
+ * - Ignore code blocks
+ *
+ * And for the first contentful line, it will strip away most Markdown
+ * syntax, including HTML tags, emphasis, links (keeping the text), etc.
+ */
+export function createExcerpt(fileString: string): string | undefined {
+  const fileLines = fileString
+    .trimStart()
+    // Remove Markdown alternate title
+    .replace(/^[^\n]*\n[=]+/g, '')
+    .split('\n');
+  let inCode = false;
+  let inImport = false;
+  let lastCodeFence = '';
+
+  for (const fileLine of fileLines) {
+    if (fileLine === '' && inImport) {
+      inImport = false;
+    }
+    // Skip empty line.
+    if (!fileLine.trim()) {
+      continue;
+    }
+
+    // Skip import/export declaration.
+    if ((/^(?:import|export)\s.*/.test(fileLine) || inImport) && !inCode) {
+      inImport = true;
+      continue;
+    }
+
+    // Skip code block line.
+    if (fileLine.trim().startsWith('```')) {
+      const codeFence = fileLine.trim().match(/^`+/)![0]!;
+      if (!inCode) {
+        inCode = true;
+        lastCodeFence = codeFence;
+        // If we are in a ````-fenced block, all ``` would be plain text instead
+        // of fences
+      } else if (codeFence.length >= lastCodeFence.length) {
+        inCode = false;
+      }
+      continue;
+    } else if (inCode) {
+      continue;
+    }
+
+    const cleanedLine = fileLine
+      // Remove HTML tags.
+      .replace(/<[^>]*>/g, '')
+      // Remove Title headers
+      .replace(/^#[^#]+#?/gm, '')
+      // Remove Markdown + ATX-style headers
+      .replace(/^#{1,6}\s*(?<text>[^#]*)\s*#{0,6}/gm, '$1')
+      // Remove emphasis.
+      .replace(/(?<opening>[*_]{1,3})(?<text>.*?)\1/g, '$2')
+      // Remove strikethroughs.
+      .replace(/~~(?<text>\S.*\S)~~/g, '$1')
+      // Remove images.
+      .replace(/!\[(?<alt>.*?)\][[(].*?[\])]/g, '$1')
+      // Remove footnotes.
+      .replace(/\[\^.+?\](?:: .*$)?/g, '')
+      // Remove inline links.
+      .replace(/\[(?<alt>.*?)\][[(].*?[\])]/g, '$1')
+      // Remove inline code.
+      .replace(/`(?<text>.+?)`/g, '$1')
+      // Remove blockquotes.
+      .replace(/^\s{0,3}>\s?/g, '')
+      // Remove admonition definition.
+      .replace(/:::.*/, '')
+      // Remove Emoji names within colons include preceding whitespace.
+      .replace(/\s?:(?:::|[^:\n])+:/g, '')
+      // Remove custom Markdown heading id.
+      .replace(/\{#*[\w-]+\}/, '')
+      .trim();
+
+    if (cleanedLine) {
+      return cleanedLine;
+    }
+  }
+
+  return undefined;
+}
+
+/**
+ * Takes a raw Markdown file content, and parses the front matter using
+ * gray-matter. Worth noting that gray-matter accepts TOML and other markup
+ * languages as well.
+ *
+ * @throws Throws when gray-matter throws. e.g.:
+ * ```md
+ * ---
+ * foo: : bar
+ * ---
+ * ```
+ */
+export function parseFrontMatter(markdownFileContent: string): {
+  /** Front matter as parsed by gray-matter. */
+  frontMatter: {[key: string]: unknown};
+  /** The remaining content, trimmed. */
+  content: string;
+} {
+  const {data, content} = matter(markdownFileContent);
+  return {
+    frontMatter: data,
+    content: content.trim(),
+  };
+}
+
+function toTextContentTitle(contentTitle: string): string {
+  if (contentTitle.startsWith('`') && contentTitle.endsWith('`')) {
+    return contentTitle.substring(1, contentTitle.length - 1);
+  }
+  return contentTitle;
+}
+
+type ParseMarkdownContentTitleOptions = {
+  /**
+   * If `true`, the matching title will be removed from the returned content.
+   * We can promise that at least one empty line will be left between the
+   * content before and after, but you shouldn't make too much assumption
+   * about what's left.
+   */
+  removeContentTitle?: boolean;
+};
+
+/**
+ * Takes the raw Markdown content, without front matter, and tries to find an h1
+ * title (setext or atx) to be used as metadata.
+ *
+ * It only searches until the first contentful paragraph, ignoring import/export
+ * declarations.
+ *
+ * It will try to convert markdown to reasonable text, but won't be best effort,
+ * since it's only used as a fallback when `frontMatter.title` is not provided.
+ * For now, we just unwrap inline code (``# `config.js` `` => `config.js`).
+ */
+export function parseMarkdownContentTitle(
+  contentUntrimmed: string,
+  options?: ParseMarkdownContentTitleOptions,
+): {
+  /** The content, optionally without the content title. */
+  content: string;
+  /** The title, trimmed and without the `#`. */
+  contentTitle: string | undefined;
+} {
+  const removeContentTitleOption = options?.removeContentTitle ?? false;
+
+  const content = contentUntrimmed.trim();
+  // We only need to detect import statements that will be parsed by MDX as
+  // `import` nodes, as broken syntax can't render anyways. That means any block
+  // that has `import` at the very beginning and surrounded by empty lines.
+  const contentWithoutImport = content
+    .replace(/^(?:import\s(?:.|\r?\n(?!\r?\n))*(?:\r?\n){2,})*/, '')
+    .trim();
+
+  const regularTitleMatch = /^#[ \t]+(?<title>[^ \t].*)(?:\r?\n|$)/.exec(
+    contentWithoutImport,
+  );
+  const alternateTitleMatch = /^(?<title>.*)\r?\n=+(?:\r?\n|$)/.exec(
+    contentWithoutImport,
+  );
+
+  const titleMatch = regularTitleMatch ?? alternateTitleMatch;
+  if (!titleMatch) {
+    return {content, contentTitle: undefined};
+  }
+  const newContent = removeContentTitleOption
+    ? content.replace(titleMatch[0]!, '')
+    : content;
+  if (regularTitleMatch) {
+    return {
+      content: newContent.trim(),
+      contentTitle: toTextContentTitle(
+        regularTitleMatch
+          .groups!.title!.trim()
+          .replace(/\s*(?:\{#*[\w-]+\}|#+)$/, ''),
+      ).trim(),
+    };
+  }
+  return {
+    content: newContent.trim(),
+    contentTitle: toTextContentTitle(
+      alternateTitleMatch!.groups!.title!.trim().replace(/\s*=+$/, ''),
+    ).trim(),
+  };
+}
+
+/**
+ * Makes a full-round parse.
+ *
+ * @throws Throws when `parseFrontMatter` throws, usually because of invalid
+ * syntax.
+ */
+export function parseMarkdownString(
+  markdownFileContent: string,
+  options?: ParseMarkdownContentTitleOptions,
+): {
+  /** @see {@link parseFrontMatter} */
+  frontMatter: {[key: string]: unknown};
+  /** @see {@link parseMarkdownContentTitle} */
+  contentTitle: string | undefined;
+  /** @see {@link createExcerpt} */
+  excerpt: string | undefined;
+  /**
+   * Content without front matter and (optionally) without title, depending on
+   * the `removeContentTitle` option.
+   */
+  content: string;
+} {
+  try {
+    const {frontMatter, content: contentWithoutFrontMatter} =
+      parseFrontMatter(markdownFileContent);
+
+    const {content, contentTitle} = parseMarkdownContentTitle(
+      contentWithoutFrontMatter,
+      options,
+    );
+
+    const excerpt = createExcerpt(content);
+
+    return {
+      frontMatter,
+      content,
+      contentTitle,
+      excerpt,
+    };
+  } catch (err) {
+    logger.error(`Error while parsing Markdown front matter.
+This can happen if you use special characters in front matter values (try using double quotes around that value).`);
+    throw err;
+  }
+}
+
+function unwrapMarkdownLinks(line: string): string {
+  return line.replace(
+    /\[(?<alt>[^\]]+)\]\([^)]+\)/g,
+    (match, p1: string) => p1,
+  );
+}
+
+function addHeadingId(
+  line: string,
+  slugger: Slugger,
+  maintainCase: boolean,
+): string {
+  let headingLevel = 0;
+  while (line.charAt(headingLevel) === '#') {
+    headingLevel += 1;
+  }
+
+  const headingText = line.slice(headingLevel).trimEnd();
+  const headingHashes = line.slice(0, headingLevel);
+  const slug = slugger.slug(unwrapMarkdownLinks(headingText).trim(), {
+    maintainCase,
+  });
+
+  return `${headingHashes}${headingText} {#${slug}}`;
+}
+
+export type WriteHeadingIDOptions = SluggerOptions & {
+  /** Overwrite existing heading IDs. */
+  overwrite?: boolean;
+};
+
+/**
+ * Takes Markdown content, returns new content with heading IDs written.
+ * Respects existing IDs (unless `overwrite=true`) and never generates colliding
+ * IDs (through the slugger).
+ */
+export function writeMarkdownHeadingId(
+  content: string,
+  options: WriteHeadingIDOptions = {maintainCase: false, overwrite: false},
+): string {
+  const {maintainCase = false, overwrite = false} = options;
+  const lines = content.split('\n');
+  const slugger = createSlugger();
+
+  // If we can't overwrite existing slugs, make sure other headings don't
+  // generate colliding slugs by first marking these slugs as occupied
+  if (!overwrite) {
+    lines.forEach((line) => {
+      const parsedHeading = parseMarkdownHeadingId(line);
+      if (parsedHeading.id) {
+        slugger.slug(parsedHeading.id);
+      }
+    });
+  }
+
+  let inCode = false;
+  return lines
+    .map((line) => {
+      if (line.startsWith('```')) {
+        inCode = !inCode;
+        return line;
+      }
+      // Ignore h1 headings, as we don't create anchor links for those
+      if (inCode || !line.startsWith('##')) {
+        return line;
+      }
+      const parsedHeading = parseMarkdownHeadingId(line);
+
+      // Do not process if id is already there
+      if (parsedHeading.id && !overwrite) {
+        return line;
+      }
+      return addHeadingId(parsedHeading.text, slugger, maintainCase);
+    })
+    .join('\n');
+}