@docusaurus/utils 2.0.0-beta.ff31de0ff → 2.0.1

package/src/markdownLinks.ts

@@ -5,49 +5,96 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import {resolve} from 'url';
-import {aliasedSitePath} from './index';
+import path from 'path';
+import {getContentPathList} from './dataFileUtils';
+import {aliasedSitePath} from './pathUtils';
 
+/**
+ * Content plugins have a base path and a localized path to source content from.
+ * We will look into the localized path in priority.
+ */
 export type ContentPaths = {
+  /**
+   * The absolute path to the base content directory, like `"<siteDir>/docs"`.
+   */
   contentPath: string;
+  /**
+   * The absolute path to the localized content directory, like
+   * `"<siteDir>/i18n/zh-Hans/plugin-content-docs"`.
+   */
   contentPathLocalized: string;
 };
 
+/** Data structure representing each broken Markdown link to be reported. */
 export type BrokenMarkdownLink<T extends ContentPaths> = {
+  /** Absolute path to the file containing this link. */
   filePath: string;
+  /**
+   * This is generic because it may contain extra metadata like version name,
+   * which the reporter can provide for context.
+   */
   contentPaths: T;
+  /**
+   * The content of the link, like `"./brokenFile.md"`
+   */
   link: string;
 };
 
-export type ReplaceMarkdownLinksParams<T extends ContentPaths> = {
-  siteDir: string;
-  fileString: string;
-  filePath: string;
-  contentPaths: T;
-  sourceToPermalink: Record<string, string>;
-};
-
-export type ReplaceMarkdownLinksReturn<T extends ContentPaths> = {
-  newContent: string;
-  brokenMarkdownLinks: BrokenMarkdownLink<T>[];
-};
-
+/**
+ * Takes a Markdown file and replaces relative file references with their URL
+ * counterparts, e.g. `[link](./intro.md)` => `[link](/docs/intro)`, preserving
+ * everything else.
+ *
+ * This method uses best effort to find a matching file. The file reference can
+ * be relative to the directory of the current file (most likely) or any of the
+ * content paths (so `/tutorials/intro.md` can be resolved as
+ * `<siteDir>/docs/tutorials/intro.md`). Links that contain the `http(s):` or
+ * `@site/` prefix will always be ignored.
+ */
 export function replaceMarkdownLinks<T extends ContentPaths>({
   siteDir,
   fileString,
   filePath,
   contentPaths,
   sourceToPermalink,
-}: ReplaceMarkdownLinksParams<T>): ReplaceMarkdownLinksReturn<T> {
-  const {contentPath, contentPathLocalized} = contentPaths;
-
+}: {
+  /** Absolute path to the site directory, used to resolve aliased paths. */
+  siteDir: string;
+  /** The Markdown file content to be processed. */
+  fileString: string;
+  /** Absolute path to the current file containing `fileString`. */
+  filePath: string;
+  /** The content paths which the file reference may live in. */
+  contentPaths: T;
+  /**
+   * A map from source paths to their URLs. Source paths are `@site` aliased.
+   */
+  sourceToPermalink: {[aliasedPath: string]: string};
+}): {
+  /**
+   * The content with all Markdown file references replaced with their URLs.
+   * Unresolved links are left as-is.
+   */
+  newContent: string;
+  /** The list of broken links,  */
+  brokenMarkdownLinks: BrokenMarkdownLink<T>[];
+} {
   const brokenMarkdownLinks: BrokenMarkdownLink<T>[] = [];
 
   // Replace internal markdown linking (except in fenced blocks).
   let fencedBlock = false;
+  let lastCodeFence = '';
   const lines = fileString.split('\n').map((line) => {
     if (line.trim().startsWith('```')) {
-      fencedBlock = !fencedBlock;
+      const codeFence = line.trim().match(/^`+/)![0]!;
+      if (!fencedBlock) {
+        fencedBlock = true;
+        lastCodeFence = codeFence;
+        // If we are in a ````-fenced block, all ``` would be plain text instead
+        // of fences
+      } else if (codeFence.length >= lastCodeFence.length) {
+        fencedBlock = false;
+      }
     }
     if (fencedBlock) {
       return line;
@@ -55,24 +102,48 @@ export function replaceMarkdownLinks<T extends ContentPaths>({
 
     let modifiedLine = line;
     // Replace inline-style links or reference-style links e.g:
-    // This is [Document 1](doc1.md) -> we replace this doc1.md with correct link
-    // [doc1]: doc1.md -> we replace this doc1.md with correct link
-    const mdRegex = /(?:(?:\]\()|(?:\]:\s?))(?!https)([^'")\]\s>]+\.mdx?)/g;
+    // This is [Document 1](doc1.md)
+    // [doc1]: doc1.md
+    const mdRegex =
+      /(?:\]\(|\]:\s*)(?!https?:\/\/|@site\/)(?<filename>[^'")\]\s>]+\.mdx?)/g;
     let mdMatch = mdRegex.exec(modifiedLine);
     while (mdMatch !== null) {
       // Replace it to correct html link.
-      const mdLink = mdMatch[1];
+      const mdLink = mdMatch.groups!.filename!;
+
+      const sourcesToTry: string[] = [];
+      // ./file.md and ../file.md are always relative to the current file
+      if (!mdLink.startsWith('./') && !mdLink.startsWith('../')) {
+        sourcesToTry.push(...getContentPathList(contentPaths), siteDir);
+      }
+      // /file.md is always relative to the content path
+      if (!mdLink.startsWith('/')) {
+        sourcesToTry.push(path.dirname(filePath));
+      }
 
-      const aliasedSource = (source: string) =>
-        aliasedSitePath(source, siteDir);
+      const aliasedSourceMatch = sourcesToTry
+        .map((p) => path.join(p, decodeURIComponent(mdLink)))
+        .map((source) => aliasedSitePath(source, siteDir))
+        .find((source) => sourceToPermalink[source]);
 
-      const permalink: string | undefined =
-        sourceToPermalink[aliasedSource(resolve(filePath, mdLink))] ||
-        sourceToPermalink[aliasedSource(`${contentPathLocalized}/${mdLink}`)] ||
-        sourceToPermalink[aliasedSource(`${contentPath}/${mdLink}`)];
+      const permalink: string | undefined = aliasedSourceMatch
+        ? sourceToPermalink[aliasedSourceMatch]
+        : undefined;
 
       if (permalink) {
-        modifiedLine = modifiedLine.replace(mdLink, permalink);
+        // MDX won't be happy if the permalink contains a space, we need to
+        // convert it to %20
+        const encodedPermalink = permalink
+          .split('/')
+          .map((part) => part.replace(/\s/g, '%20'))
+          .join('/');
+        modifiedLine = modifiedLine.replace(
+          mdMatch[0]!,
+          mdMatch[0]!.replace(mdLink, encodedPermalink),
+        );
+        // Adjust the lastIndex to avoid passing over the next link if the
+        // newly replaced URL is shorter.
+        mdRegex.lastIndex += encodedPermalink.length - mdLink.length;
       } else {
         const brokenMarkdownLink: BrokenMarkdownLink<T> = {
           contentPaths,

package/src/markdownUtils.ts

@@ -0,0 +1,357 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import logger from '@docusaurus/logger';
+import matter from 'gray-matter';
+import {createSlugger, type Slugger, type SluggerOptions} from './slugger';
+
+// Some utilities for parsing Markdown content. These things are only used on
+// server-side when we infer metadata like `title` and `description` from the
+// content. Most parsing is still done in MDX through the mdx-loader.
+
+/**
+ * Parses custom ID from a heading. The ID can contain any characters except
+ * `{#` and `}`.
+ *
+ * @param heading e.g. `## Some heading {#some-heading}` where the last
+ * character must be `}` for the ID to be recognized
+ */
+export function parseMarkdownHeadingId(heading: string): {
+  /**
+   * The heading content sans the ID part, right-trimmed. e.g. `## Some heading`
+   */
+  text: string;
+  /** The heading ID. e.g. `some-heading` */
+  id: string | undefined;
+} {
+  const customHeadingIdRegex = /\s*\{#(?<id>(?:.(?!\{#|\}))*.)\}$/;
+  const matches = customHeadingIdRegex.exec(heading);
+  if (matches) {
+    return {
+      text: heading.replace(matches[0]!, ''),
+      id: matches.groups!.id!,
+    };
+  }
+  return {text: heading, id: undefined};
+}
+
+// TODO: Find a better way to do so, possibly by compiling the Markdown content,
+// stripping out HTML tags and obtaining the first line.
+/**
+ * Creates an excerpt of a Markdown file. This function will:
+ *
+ * - Ignore h1 headings (setext or atx)
+ * - Ignore import/export
+ * - Ignore code blocks
+ *
+ * And for the first contentful line, it will strip away most Markdown
+ * syntax, including HTML tags, emphasis, links (keeping the text), etc.
+ */
+export function createExcerpt(fileString: string): string | undefined {
+  const fileLines = fileString
+    .trimStart()
+    // Remove Markdown alternate title
+    .replace(/^[^\n]*\n[=]+/g, '')
+    .split('\n');
+  let inCode = false;
+  let inImport = false;
+  let lastCodeFence = '';
+
+  for (const fileLine of fileLines) {
+    if (fileLine === '' && inImport) {
+      inImport = false;
+    }
+    // Skip empty line.
+    if (!fileLine.trim()) {
+      continue;
+    }
+
+    // Skip import/export declaration.
+    if ((/^(?:import|export)\s.*/.test(fileLine) || inImport) && !inCode) {
+      inImport = true;
+      continue;
+    }
+
+    // Skip code block line.
+    if (fileLine.trim().startsWith('```')) {
+      const codeFence = fileLine.trim().match(/^`+/)![0]!;
+      if (!inCode) {
+        inCode = true;
+        lastCodeFence = codeFence;
+        // If we are in a ````-fenced block, all ``` would be plain text instead
+        // of fences
+      } else if (codeFence.length >= lastCodeFence.length) {
+        inCode = false;
+      }
+      continue;
+    } else if (inCode) {
+      continue;
+    }
+
+    const cleanedLine = fileLine
+      // Remove HTML tags.
+      .replace(/<[^>]*>/g, '')
+      // Remove Title headers
+      .replace(/^#[^#]+#?/gm, '')
+      // Remove Markdown + ATX-style headers
+      .replace(/^#{1,6}\s*(?<text>[^#]*)\s*#{0,6}/gm, '$1')
+      // Remove emphasis.
+      .replace(/(?<opening>[*_]{1,3})(?<text>.*?)\1/g, '$2')
+      // Remove strikethroughs.
+      .replace(/~~(?<text>\S.*\S)~~/g, '$1')
+      // Remove images.
+      .replace(/!\[(?<alt>.*?)\][[(].*?[\])]/g, '$1')
+      // Remove footnotes.
+      .replace(/\[\^.+?\](?:: .*$)?/g, '')
+      // Remove inline links.
+      .replace(/\[(?<alt>.*?)\][[(].*?[\])]/g, '$1')
+      // Remove inline code.
+      .replace(/`(?<text>.+?)`/g, '$1')
+      // Remove blockquotes.
+      .replace(/^\s{0,3}>\s?/g, '')
+      // Remove admonition definition.
+      .replace(/:::.*/, '')
+      // Remove Emoji names within colons include preceding whitespace.
+      .replace(/\s?:(?:::|[^:\n])+:/g, '')
+      // Remove custom Markdown heading id.
+      .replace(/\{#*[\w-]+\}/, '')
+      .trim();
+
+    if (cleanedLine) {
+      return cleanedLine;
+    }
+  }
+
+  return undefined;
+}
+
+/**
+ * Takes a raw Markdown file content, and parses the front matter using
+ * gray-matter. Worth noting that gray-matter accepts TOML and other markup
+ * languages as well.
+ *
+ * @throws Throws when gray-matter throws. e.g.:
+ * ```md
+ * ---
+ * foo: : bar
+ * ---
+ * ```
+ */
+export function parseFrontMatter(markdownFileContent: string): {
+  /** Front matter as parsed by gray-matter. */
+  frontMatter: {[key: string]: unknown};
+  /** The remaining content, trimmed. */
+  content: string;
+} {
+  const {data, content} = matter(markdownFileContent);
+  return {
+    frontMatter: data,
+    content: content.trim(),
+  };
+}
+
+function toTextContentTitle(contentTitle: string): string {
+  if (contentTitle.startsWith('`') && contentTitle.endsWith('`')) {
+    return contentTitle.substring(1, contentTitle.length - 1);
+  }
+  return contentTitle;
+}
+
+type ParseMarkdownContentTitleOptions = {
+  /**
+   * If `true`, the matching title will be removed from the returned content.
+   * We can promise that at least one empty line will be left between the
+   * content before and after, but you shouldn't make too much assumption
+   * about what's left.
+   */
+  removeContentTitle?: boolean;
+};
+
+/**
+ * Takes the raw Markdown content, without front matter, and tries to find an h1
+ * title (setext or atx) to be used as metadata.
+ *
+ * It only searches until the first contentful paragraph, ignoring import/export
+ * declarations.
+ *
+ * It will try to convert markdown to reasonable text, but won't be best effort,
+ * since it's only used as a fallback when `frontMatter.title` is not provided.
+ * For now, we just unwrap inline code (``# `config.js` `` => `config.js`).
+ */
+export function parseMarkdownContentTitle(
+  contentUntrimmed: string,
+  options?: ParseMarkdownContentTitleOptions,
+): {
+  /** The content, optionally without the content title. */
+  content: string;
+  /** The title, trimmed and without the `#`. */
+  contentTitle: string | undefined;
+} {
+  const removeContentTitleOption = options?.removeContentTitle ?? false;
+
+  const content = contentUntrimmed.trim();
+  // We only need to detect import statements that will be parsed by MDX as
+  // `import` nodes, as broken syntax can't render anyways. That means any block
+  // that has `import` at the very beginning and surrounded by empty lines.
+  const contentWithoutImport = content
+    .replace(/^(?:import\s(?:.|\r?\n(?!\r?\n))*(?:\r?\n){2,})*/, '')
+    .trim();
+
+  const regularTitleMatch = /^#[ \t]+(?<title>[^ \t].*)(?:\r?\n|$)/.exec(
+    contentWithoutImport,
+  );
+  const alternateTitleMatch = /^(?<title>.*)\r?\n=+(?:\r?\n|$)/.exec(
+    contentWithoutImport,
+  );
+
+  const titleMatch = regularTitleMatch ?? alternateTitleMatch;
+  if (!titleMatch) {
+    return {content, contentTitle: undefined};
+  }
+  const newContent = removeContentTitleOption
+    ? content.replace(titleMatch[0]!, '')
+    : content;
+  if (regularTitleMatch) {
+    return {
+      content: newContent.trim(),
+      contentTitle: toTextContentTitle(
+        regularTitleMatch
+          .groups!.title!.trim()
+          .replace(/\s*(?:\{#*[\w-]+\}|#+)$/, ''),
+      ).trim(),
+    };
+  }
+  return {
+    content: newContent.trim(),
+    contentTitle: toTextContentTitle(
+      alternateTitleMatch!.groups!.title!.trim().replace(/\s*=+$/, ''),
+    ).trim(),
+  };
+}
+
+/**
+ * Makes a full-round parse.
+ *
+ * @throws Throws when `parseFrontMatter` throws, usually because of invalid
+ * syntax.
+ */
+export function parseMarkdownString(
+  markdownFileContent: string,
+  options?: ParseMarkdownContentTitleOptions,
+): {
+  /** @see {@link parseFrontMatter} */
+  frontMatter: {[key: string]: unknown};
+  /** @see {@link parseMarkdownContentTitle} */
+  contentTitle: string | undefined;
+  /** @see {@link createExcerpt} */
+  excerpt: string | undefined;
+  /**
+   * Content without front matter and (optionally) without title, depending on
+   * the `removeContentTitle` option.
+   */
+  content: string;
+} {
+  try {
+    const {frontMatter, content: contentWithoutFrontMatter} =
+      parseFrontMatter(markdownFileContent);
+
+    const {content, contentTitle} = parseMarkdownContentTitle(
+      contentWithoutFrontMatter,
+      options,
+    );
+
+    const excerpt = createExcerpt(content);
+
+    return {
+      frontMatter,
+      content,
+      contentTitle,
+      excerpt,
+    };
+  } catch (err) {
+    logger.error(`Error while parsing Markdown front matter.
+This can happen if you use special characters in front matter values (try using double quotes around that value).`);
+    throw err;
+  }
+}
+
+function unwrapMarkdownLinks(line: string): string {
+  return line.replace(
+    /\[(?<alt>[^\]]+)\]\([^)]+\)/g,
+    (match, p1: string) => p1,
+  );
+}
+
+function addHeadingId(
+  line: string,
+  slugger: Slugger,
+  maintainCase: boolean,
+): string {
+  let headingLevel = 0;
+  while (line.charAt(headingLevel) === '#') {
+    headingLevel += 1;
+  }
+
+  const headingText = line.slice(headingLevel).trimEnd();
+  const headingHashes = line.slice(0, headingLevel);
+  const slug = slugger.slug(unwrapMarkdownLinks(headingText).trim(), {
+    maintainCase,
+  });
+
+  return `${headingHashes}${headingText} {#${slug}}`;
+}
+
+export type WriteHeadingIDOptions = SluggerOptions & {
+  /** Overwrite existing heading IDs. */
+  overwrite?: boolean;
+};
+
+/**
+ * Takes Markdown content, returns new content with heading IDs written.
+ * Respects existing IDs (unless `overwrite=true`) and never generates colliding
+ * IDs (through the slugger).
+ */
+export function writeMarkdownHeadingId(
+  content: string,
+  options: WriteHeadingIDOptions = {maintainCase: false, overwrite: false},
+): string {
+  const {maintainCase = false, overwrite = false} = options;
+  const lines = content.split('\n');
+  const slugger = createSlugger();
+
+  // If we can't overwrite existing slugs, make sure other headings don't
+  // generate colliding slugs by first marking these slugs as occupied
+  if (!overwrite) {
+    lines.forEach((line) => {
+      const parsedHeading = parseMarkdownHeadingId(line);
+      if (parsedHeading.id) {
+        slugger.slug(parsedHeading.id);
+      }
+    });
+  }
+
+  let inCode = false;
+  return lines
+    .map((line) => {
+      if (line.startsWith('```')) {
+        inCode = !inCode;
+        return line;
+      }
+      // Ignore h1 headings, as we don't create anchor links for those
+      if (inCode || !line.startsWith('##')) {
+        return line;
+      }
+      const parsedHeading = parseMarkdownHeadingId(line);
+
+      // Do not process if id is already there
+      if (parsedHeading.id && !overwrite) {
+        return line;
+      }
+      return addHeadingId(parsedHeading.text, slugger, maintainCase);
+    })
+    .join('\n');
+}

package/src/pathUtils.ts

@@ -0,0 +1,123 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import path from 'path';
+
+// Based on https://github.com/gatsbyjs/gatsby/pull/21518/files
+// macOS (APFS) and Windows (NTFS) filename length limit = 255 chars,
+// Others = 255 bytes
+const MAX_PATH_SEGMENT_CHARS = 255;
+const MAX_PATH_SEGMENT_BYTES = 255;
+// Space for appending things to the string like file extensions and so on
+const SPACE_FOR_APPENDING = 10;
+
+const isMacOs = () => process.platform === 'darwin';
+const isWindows = () => process.platform === 'win32';
+
+export const isNameTooLong = (str: string): boolean =>
+  // Not entirely correct: we can't assume FS from OS. But good enough?
+  isMacOs() || isWindows()
+    ? // Windows (NTFS) and macOS (APFS) filename length limit (255 chars)
+      str.length + SPACE_FOR_APPENDING > MAX_PATH_SEGMENT_CHARS
+    : // Other (255 bytes)
+      Buffer.from(str).length + SPACE_FOR_APPENDING > MAX_PATH_SEGMENT_BYTES;
+
+export function shortName(str: string): string {
+  if (isMacOs() || isWindows()) {
+    const overflowingChars = str.length - MAX_PATH_SEGMENT_CHARS;
+    return str.slice(
+      0,
+      str.length - overflowingChars - SPACE_FOR_APPENDING - 1,
+    );
+  }
+  const strBuffer = Buffer.from(str);
+  const overflowingBytes =
+    Buffer.byteLength(strBuffer) - MAX_PATH_SEGMENT_BYTES;
+  return strBuffer
+    .slice(
+      0,
+      Buffer.byteLength(strBuffer) - overflowingBytes - SPACE_FOR_APPENDING - 1,
+    )
+    .toString();
+}
+
+/**
+ * Convert Windows backslash paths to posix style paths.
+ * E.g: endi\lie -> endi/lie
+ *
+ * Returns original path if the posix counterpart is not valid Windows path.
+ * This makes the legacy code that uses posixPath safe; but also makes it less
+ * useful when you actually want a path with forward slashes (e.g. for URL)
+ *
+ * Adopted from https://github.com/sindresorhus/slash/blob/main/index.js
+ */
+export function posixPath(str: string): string {
+  const isExtendedLengthPath = str.startsWith('\\\\?\\');
+
+  // Forward slashes are only valid Windows paths when they don't contain non-
+  // ascii characters.
+  // eslint-disable-next-line no-control-regex
+  const hasNonAscii = /[^\u0000-\u0080]+/.test(str);
+
+  if (isExtendedLengthPath || hasNonAscii) {
+    return str;
+  }
+  return str.replace(/\\/g, '/');
+}
+
+/**
+ * When you want to display a path in a message/warning/error, it's more
+ * convenient to:
+ *
+ * - make it relative to `cwd()`
+ * - convert to posix (ie not using windows \ path separator)
+ *
+ * This way, Jest tests can run more reliably on any computer/CI on both
+ * Unix/Windows
+ * For Windows users this is not perfect (as they see / instead of \) but it's
+ * probably good enough
+ */
+export function toMessageRelativeFilePath(filePath: string): string {
+  return posixPath(path.relative(process.cwd(), filePath));
+}
+
+/**
+ * Alias filepath relative to site directory, very useful so that we
+ * don't expose user's site structure.
+ * Example: some/path/to/website/docs/foo.md -> @site/docs/foo.md
+ */
+export function aliasedSitePath(filePath: string, siteDir: string): string {
+  const relativePath = posixPath(path.relative(siteDir, filePath));
+  // Cannot use path.join() as it resolves '../' and removes
+  // the '@site'. Let webpack loader resolve it.
+  return `@site/${relativePath}`;
+}
+
+/**
+ * When you have a path like C:\X\Y
+ * It is not safe to use directly when generating code
+ * For example, this would fail due to unescaped \:
+ * `<img src={require('${filePath}')} />`
+ * But this would work: `<img src={require('${escapePath(filePath)}')} />`
+ *
+ * posixPath can't be used in all cases, because forward slashes are only valid
+ * Windows paths when they don't contain non-ascii characters, and posixPath
+ * doesn't escape those that fail to be converted.
+ */
+export function escapePath(str: string): string {
+  const escaped = JSON.stringify(str);
+
+  // Remove the " around the json string;
+  return escaped.substring(1, escaped.length - 1);
+}
+
+export function addTrailingPathSeparator(str: string): string {
+  return str.endsWith(path.sep)
+    ? str
+    : // If this is Windows, we need to change the forward slash to backward
+      `${str.replace(/\/$/, '')}${path.sep}`;
+}

package/src/shellUtils.ts

@@ -0,0 +1,18 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// TODO move from shelljs to execa later?
+// Execa is well maintained and widely used
+// Even shelljs recommends execa for security / escaping:
+// https://github.com/shelljs/shelljs/wiki/Security-guidelines
+
+// Inspired by https://github.com/xxorax/node-shell-escape/blob/master/shell-escape.js
+export function escapeShellArg(s: string): string {
+  let res = `'${s.replace(/'/g, "'\\''")}'`;
+  res = res.replace(/^(?:'')+/g, '').replace(/\\'''/g, "\\'");
+  return res;
+}