@docusaurus/utils 2.0.0-beta.17 → 2.0.0-beta.18

package/src/markdownLinks.ts

@@ -6,41 +6,79 @@
  */
 
 import path from 'path';
+import {getContentPathList} from './dataFileUtils';
 import {aliasedSitePath} from './pathUtils';
 
+/**
+ * Content plugins have a base path and a localized path to source content from.
+ * We will look into the localized path in priority.
+ */
 export type ContentPaths = {
+  /**
+   * The absolute path to the base content directory, like `"<siteDir>/docs"`.
+   */
   contentPath: string;
+  /**
+   * The absolute path to the localized content directory, like
+   * `"<siteDir>/i18n/zh-Hans/plugin-content-docs"`.
+   */
   contentPathLocalized: string;
 };
 
+/** Data structure representing each broken Markdown link to be reported. */
 export type BrokenMarkdownLink<T extends ContentPaths> = {
+  /** Absolute path to the file containing this link. */
   filePath: string;
+  /**
+   * This is generic because it may contain extra metadata like version name,
+   * which the reporter can provide for context.
+   */
   contentPaths: T;
+  /**
+   * The content of the link, like `"./brokenFile.md"`
+   */
   link: string;
 };
 
-export type ReplaceMarkdownLinksParams<T extends ContentPaths> = {
-  siteDir: string;
-  fileString: string;
-  filePath: string;
-  contentPaths: T;
-  sourceToPermalink: Record<string, string>;
-};
-
-export type ReplaceMarkdownLinksReturn<T extends ContentPaths> = {
-  newContent: string;
-  brokenMarkdownLinks: BrokenMarkdownLink<T>[];
-};
-
+/**
+ * Takes a Markdown file and replaces relative file references with their URL
+ * counterparts, e.g. `[link](./intro.md)` => `[link](/docs/intro)`, preserving
+ * everything else.
+ *
+ * This method uses best effort to find a matching file. The file reference can
+ * be relative to the directory of the current file (most likely) or any of the
+ * content paths (so `/tutorials/intro.md` can be resolved as
+ * `<siteDir>/docs/tutorials/intro.md`). Links that contain the `http(s):` or
+ * `@site/` prefix will always be ignored.
+ */
 export function replaceMarkdownLinks<T extends ContentPaths>({
   siteDir,
   fileString,
   filePath,
   contentPaths,
   sourceToPermalink,
-}: ReplaceMarkdownLinksParams<T>): ReplaceMarkdownLinksReturn<T> {
-  const {contentPath, contentPathLocalized} = contentPaths;
-
+}: {
+  /** Absolute path to the site directory, used to resolve aliased paths. */
+  siteDir: string;
+  /** The Markdown file content to be processed. */
+  fileString: string;
+  /** Absolute path to the current file containing `fileString`. */
+  filePath: string;
+  /** The content paths which the file reference may live in. */
+  contentPaths: T;
+  /**
+   * A map from source paths to their URLs. Source paths are `@site` aliased.
+   */
+  sourceToPermalink: {[aliasedPath: string]: string};
+}): {
+  /**
+   * The content with all Markdown file references replaced with their URLs.
+   * Unresolved links are left as-is.
+   */
+  newContent: string;
+  /** The list of broken links,  */
+  brokenMarkdownLinks: BrokenMarkdownLink<T>[];
+} {
   const brokenMarkdownLinks: BrokenMarkdownLink<T>[] = [];
 
   // Replace internal markdown linking (except in fenced blocks).
@@ -48,12 +86,13 @@ export function replaceMarkdownLinks<T extends ContentPaths>({
   let lastCodeFence = '';
   const lines = fileString.split('\n').map((line) => {
     if (line.trim().startsWith('```')) {
+      const codeFence = line.trim().match(/^`+/)![0]!;
       if (!fencedBlock) {
         fencedBlock = true;
-        [lastCodeFence] = line.trim().match(/^`+/)!;
+        lastCodeFence = codeFence;
         // If we are in a ````-fenced block, all ``` would be plain text instead
         // of fences
-      } else if (line.trim().match(/^`+/)![0].length >= lastCodeFence.length) {
+      } else if (codeFence.length >= lastCodeFence.length) {
         fencedBlock = false;
       }
     }
@@ -63,21 +102,19 @@ export function replaceMarkdownLinks<T extends ContentPaths>({
 
     let modifiedLine = line;
     // Replace inline-style links or reference-style links e.g:
-    // This is [Document 1](doc1.md) -> we replace this doc1.md with correct
-    // ink
-    // [doc1]: doc1.md -> we replace this doc1.md with correct link
+    // This is [Document 1](doc1.md)
+    // [doc1]: doc1.md
     const mdRegex =
-      /(?:(?:\]\()|(?:\]:\s*))(?!https?:\/\/|@site\/)(?<filename>[^'")\]\s>]+\.mdx?)/g;
+      /(?:\]\(|\]:\s*)(?!https?:\/\/|@site\/)(?<filename>[^'")\]\s>]+\.mdx?)/g;
     let mdMatch = mdRegex.exec(modifiedLine);
     while (mdMatch !== null) {
       // Replace it to correct html link.
-      const mdLink = mdMatch.groups!.filename;
+      const mdLink = mdMatch.groups!.filename!;
 
       const sourcesToTry = [
-        path.resolve(path.dirname(filePath), decodeURIComponent(mdLink)),
-        `${contentPathLocalized}/${decodeURIComponent(mdLink)}`,
-        `${contentPath}/${decodeURIComponent(mdLink)}`,
-      ];
+        path.dirname(filePath),
+        ...getContentPathList(contentPaths),
+      ].map((p) => path.join(p, decodeURIComponent(mdLink)));
 
       const aliasedSourceMatch = sourcesToTry
         .map((source) => aliasedSitePath(source, siteDir))

package/src/markdownUtils.ts

@@ -0,0 +1,354 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import logger from '@docusaurus/logger';
+import matter from 'gray-matter';
+import {createSlugger, type Slugger, type SluggerOptions} from './slugger';
+
+// Some utilities for parsing Markdown content. These things are only used on
+// server-side when we infer metadata like `title` and `description` from the
+// content. Most parsing is still done in MDX through the mdx-loader.
+
+/**
+ * Parses custom ID from a heading. The ID must be composed of letters,
+ * underscores, and dashes only.
+ *
+ * @param heading e.g. `## Some heading {#some-heading}` where the last
+ * character must be `}` for the ID to be recognized
+ */
+export function parseMarkdownHeadingId(heading: string): {
+  /**
+   * The heading content sans the ID part, right-trimmed. e.g. `## Some heading`
+   */
+  text: string;
+  /** The heading ID. e.g. `some-heading` */
+  id?: string;
+} {
+  const customHeadingIdRegex = /\s*\{#(?<id>[\w-]+)\}$/;
+  const matches = customHeadingIdRegex.exec(heading);
+  if (matches) {
+    return {
+      text: heading.replace(matches[0]!, ''),
+      id: matches.groups!.id!,
+    };
+  }
+  return {text: heading, id: undefined};
+}
+
+// TODO: Find a better way to do so, possibly by compiling the Markdown content,
+// stripping out HTML tags and obtaining the first line.
+/**
+ * Creates an excerpt of a Markdown file. This function will:
+ *
+ * - Ignore h1 headings (setext or atx)
+ * - Ignore import/export
+ * - Ignore code blocks
+ *
+ * And for the first contentful line, it will strip away most Markdown
+ * syntax, including HTML tags, emphasis, links (keeping the text), etc.
+ */
+export function createExcerpt(fileString: string): string | undefined {
+  const fileLines = fileString
+    .trimStart()
+    // Remove Markdown alternate title
+    .replace(/^[^\n]*\n[=]+/g, '')
+    .split('\n');
+  let inCode = false;
+  let inImport = false;
+  let lastCodeFence = '';
+
+  for (const fileLine of fileLines) {
+    if (fileLine === '' && inImport) {
+      inImport = false;
+    }
+    // Skip empty line.
+    if (!fileLine.trim()) {
+      continue;
+    }
+
+    // Skip import/export declaration.
+    if ((/^(?:import|export)\s.*/.test(fileLine) || inImport) && !inCode) {
+      inImport = true;
+      continue;
+    }
+
+    // Skip code block line.
+    if (fileLine.trim().startsWith('```')) {
+      const codeFence = fileLine.trim().match(/^`+/)![0]!;
+      if (!inCode) {
+        inCode = true;
+        lastCodeFence = codeFence;
+        // If we are in a ````-fenced block, all ``` would be plain text instead
+        // of fences
+      } else if (codeFence.length >= lastCodeFence.length) {
+        inCode = false;
+      }
+      continue;
+    } else if (inCode) {
+      continue;
+    }
+
+    const cleanedLine = fileLine
+      // Remove HTML tags.
+      .replace(/<[^>]*>/g, '')
+      // Remove Title headers
+      .replace(/^#[^#]+#?/gm, '')
+      // Remove Markdown + ATX-style headers
+      .replace(/^#{1,6}\s*(?<text>[^#]*)\s*#{0,6}/gm, '$1')
+      // Remove emphasis.
+      .replace(/(?<opening>[*_]{1,3})(?<text>.*?)\1/g, '$2')
+      // Remove strikethroughs.
+      .replace(/~~(?<text>\S.*\S)~~/g, '$1')
+      // Remove images.
+      .replace(/!\[(?<alt>.*?)\][[(].*?[\])]/g, '$1')
+      // Remove footnotes.
+      .replace(/\[\^.+?\](?:: .*$)?/g, '')
+      // Remove inline links.
+      .replace(/\[(?<alt>.*?)\][[(].*?[\])]/g, '$1')
+      // Remove inline code.
+      .replace(/`(?<text>.+?)`/g, '$1')
+      // Remove blockquotes.
+      .replace(/^\s{0,3}>\s?/g, '')
+      // Remove admonition definition.
+      .replace(/:::.*/, '')
+      // Remove Emoji names within colons include preceding whitespace.
+      .replace(/\s?:(?:::|[^:\n])+:/g, '')
+      // Remove custom Markdown heading id.
+      .replace(/\{#*[\w-]+\}/, '')
+      .trim();
+
+    if (cleanedLine) {
+      return cleanedLine;
+    }
+  }
+
+  return undefined;
+}
+
+/**
+ * Takes a raw Markdown file content, and parses the front matter using
+ * gray-matter. Worth noting that gray-matter accepts TOML and other markup
+ * languages as well.
+ *
+ * @throws Throws when gray-matter throws. e.g.:
+ * ```md
+ * ---
+ * foo: : bar
+ * ---
+ * ```
+ */
+export function parseFrontMatter(markdownFileContent: string): {
+  /** Front matter as parsed by gray-matter. */
+  frontMatter: {[key: string]: unknown};
+  /** The remaining content, trimmed. */
+  content: string;
+} {
+  const {data, content} = matter(markdownFileContent);
+  return {
+    frontMatter: data,
+    content: content.trim(),
+  };
+}
+
+function toTextContentTitle(contentTitle: string): string {
+  if (contentTitle.startsWith('`') && contentTitle.endsWith('`')) {
+    return contentTitle.substring(1, contentTitle.length - 1);
+  }
+  return contentTitle;
+}
+
+type ParseMarkdownContentTitleOptions = {
+  /**
+   * If `true`, the matching title will be removed from the returned content.
+   * We can promise that at least one empty line will be left between the
+   * content before and after, but you shouldn't make too much assumption
+   * about what's left.
+   */
+  removeContentTitle?: boolean;
+};
+
+/**
+ * Takes the raw Markdown content, without front matter, and tries to find an h1
+ * title (setext or atx) to be used as metadata.
+ *
+ * It only searches until the first contentful paragraph, ignoring import/export
+ * declarations.
+ *
+ * It will try to convert markdown to reasonable text, but won't be best effort,
+ * since it's only used as a fallback when `frontMatter.title` is not provided.
+ * For now, we just unwrap inline code (``# `config.js` `` => `config.js`).
+ */
+export function parseMarkdownContentTitle(
+  contentUntrimmed: string,
+  options?: ParseMarkdownContentTitleOptions,
+): {
+  /** The content, optionally without the content title. */
+  content: string;
+  /** The title, trimmed and without the `#`. */
+  contentTitle: string | undefined;
+} {
+  const removeContentTitleOption = options?.removeContentTitle ?? false;
+
+  const content = contentUntrimmed.trim();
+  // We only need to detect import statements that will be parsed by MDX as
+  // `import` nodes, as broken syntax can't render anyways. That means any block
+  // that has `import` at the very beginning and surrounded by empty lines.
+  const contentWithoutImport = content
+    .replace(/^(?:import\s(?:.|\n(?!\n))*\n{2,})*/, '')
+    .trim();
+
+  const regularTitleMatch = /^#[ \t]+(?<title>[^ \t].*)(?:\n|$)/.exec(
+    contentWithoutImport,
+  );
+  const alternateTitleMatch = /^(?<title>.*)\n=+(?:\n|$)/.exec(
+    contentWithoutImport,
+  );
+
+  const titleMatch = regularTitleMatch ?? alternateTitleMatch;
+  if (!titleMatch) {
+    return {content, contentTitle: undefined};
+  }
+  const newContent = removeContentTitleOption
+    ? content.replace(titleMatch[0]!, '')
+    : content;
+  if (regularTitleMatch) {
+    return {
+      content: newContent.trim(),
+      contentTitle: toTextContentTitle(
+        regularTitleMatch
+          .groups!.title!.trim()
+          .replace(/\s*(?:\{#*[\w-]+\}|#+)$/, ''),
+      ).trim(),
+    };
+  }
+  return {
+    content: newContent.trim(),
+    contentTitle: toTextContentTitle(
+      alternateTitleMatch!.groups!.title!.trim().replace(/\s*=+$/, ''),
+    ).trim(),
+  };
+}
+
+/**
+ * Makes a full-round parse.
+ *
+ * @throws Throws when `parseFrontMatter` throws, usually because of invalid
+ * syntax.
+ */
+export function parseMarkdownString(
+  markdownFileContent: string,
+  options?: ParseMarkdownContentTitleOptions,
+): {
+  /** @see {@link parseFrontMatter} */
+  frontMatter: {[key: string]: unknown};
+  /** @see {@link parseMarkdownContentTitle} */
+  contentTitle: string | undefined;
+  /** @see {@link createExcerpt} */
+  excerpt: string | undefined;
+  /**
+   * Content without front matter and (optionally) without title, depending on
+   * the `removeContentTitle` option.
+   */
+  content: string;
+} {
+  try {
+    const {frontMatter, content: contentWithoutFrontMatter} =
+      parseFrontMatter(markdownFileContent);
+
+    const {content, contentTitle} = parseMarkdownContentTitle(
+      contentWithoutFrontMatter,
+      options,
+    );
+
+    const excerpt = createExcerpt(content);
+
+    return {
+      frontMatter,
+      content,
+      contentTitle,
+      excerpt,
+    };
+  } catch (err) {
+    logger.error(`Error while parsing Markdown front matter.
+This can happen if you use special characters in front matter values (try using double quotes around that value).`);
+    throw err;
+  }
+}
+
+function unwrapMarkdownLinks(line: string): string {
+  return line.replace(/\[(?<alt>[^\]]+)\]\([^)]+\)/g, (match, p1) => p1);
+}
+
+function addHeadingId(
+  line: string,
+  slugger: Slugger,
+  maintainCase: boolean,
+): string {
+  let headingLevel = 0;
+  while (line.charAt(headingLevel) === '#') {
+    headingLevel += 1;
+  }
+
+  const headingText = line.slice(headingLevel).trimEnd();
+  const headingHashes = line.slice(0, headingLevel);
+  const slug = slugger.slug(unwrapMarkdownLinks(headingText).trim(), {
+    maintainCase,
+  });
+
+  return `${headingHashes}${headingText} {#${slug}}`;
+}
+
+export type WriteHeadingIDOptions = SluggerOptions & {
+  /** Overwrite existing heading IDs. */
+  overwrite?: boolean;
+};
+
+/**
+ * Takes Markdown content, returns new content with heading IDs written.
+ * Respects existing IDs (unless `overwrite=true`) and never generates colliding
+ * IDs (through the slugger).
+ */
+export function writeMarkdownHeadingId(
+  content: string,
+  options: WriteHeadingIDOptions = {maintainCase: false, overwrite: false},
+): string {
+  const {maintainCase = false, overwrite = false} = options;
+  const lines = content.split('\n');
+  const slugger = createSlugger();
+
+  // If we can't overwrite existing slugs, make sure other headings don't
+  // generate colliding slugs by first marking these slugs as occupied
+  if (!overwrite) {
+    lines.forEach((line) => {
+      const parsedHeading = parseMarkdownHeadingId(line);
+      if (parsedHeading.id) {
+        slugger.slug(parsedHeading.id);
+      }
+    });
+  }
+
+  let inCode = false;
+  return lines
+    .map((line) => {
+      if (line.startsWith('```')) {
+        inCode = !inCode;
+        return line;
+      }
+      // Ignore h1 headings, as we don't create anchor links for those
+      if (inCode || !line.startsWith('##')) {
+        return line;
+      }
+      const parsedHeading = parseMarkdownHeadingId(line);
+
+      // Do not process if id is already there
+      if (parsedHeading.id && !overwrite) {
+        return line;
+      }
+      return addHeadingId(parsedHeading.text, slugger, maintainCase);
+    })
+    .join('\n');
+}

package/src/pathUtils.ts

@@ -24,7 +24,7 @@ export const isNameTooLong = (str: string): boolean =>
     ? str.length + SPACE_FOR_APPENDING > MAX_PATH_SEGMENT_CHARS // MacOS (APFS) and Windows (NTFS) filename length limit (255 chars)
     : Buffer.from(str).length + SPACE_FOR_APPENDING > MAX_PATH_SEGMENT_BYTES; // Other (255 bytes)
 
-export const shortName = (str: string): string => {
+export function shortName(str: string): string {
   if (isMacOs() || isWindows()) {
     const overflowingChars = str.length - MAX_PATH_SEGMENT_CHARS;
     return str.slice(
@@ -41,7 +41,7 @@ export const shortName = (str: string): string => {
       Buffer.byteLength(strBuffer) - overflowingBytes - SPACE_FOR_APPENDING - 1,
     )
     .toString();
-};
+}
 
 /**
  * Convert Windows backslash paths to posix style paths.

package/src/slugger.ts

@@ -10,12 +10,24 @@ import GithubSlugger from 'github-slugger';
 // We create our own abstraction on top of the lib:
 // - unify usage everywhere in the codebase
 // - ability to add extra options
-export type SluggerOptions = {maintainCase?: boolean};
+export type SluggerOptions = {
+  /** Keep the headings' casing, otherwise make all lowercase. */
+  maintainCase?: boolean;
+};
 
 export type Slugger = {
+  /**
+   * Takes a Markdown heading like "Josh Cena" and sluggifies it according to
+   * GitHub semantics (in this case `josh-cena`). Stateful, because if you try
+   * to sluggify "Josh Cena" again it would return `josh-cena-1`.
+   */
   slug: (value: string, options?: SluggerOptions) => string;
 };
 
+/**
+ * A thin wrapper around github-slugger. This is a factory function that returns
+ * a stateful Slugger object.
+ */
 export function createSlugger(): Slugger {
   const githubSlugger = new GithubSlugger();
   return {

package/src/tags.ts

@@ -10,12 +10,13 @@ import {normalizeUrl} from './urlUtils';
 
 export type Tag = {
   label: string;
+  /** Permalink to this tag's page, without the `/tags/` base path. */
   permalink: string;
 };
 
 export type FrontMatterTag = string | Tag;
 
-export function normalizeFrontMatterTag(
+function normalizeFrontMatterTag(
   tagsPath: string,
   frontMatterTag: FrontMatterTag,
 ): Tag {
@@ -45,8 +46,19 @@ export function normalizeFrontMatterTag(
   };
 }
 
+/**
+ * Takes tag objects as they are defined in front matter, and normalizes each
+ * into a standard tag object. The permalink is created by appending the
+ * sluggified label to `tagsPath`. Front matter tags already containing
+ * permalinks would still have `tagsPath` prepended.
+ *
+ * The result will always be unique by permalinks. The behavior with colliding
+ * permalinks is undetermined.
+ */
 export function normalizeFrontMatterTags(
+  /** Base path to append the tag permalinks to. */
   tagsPath: string,
+  /** Can be `undefined`, so that we can directly pipe in `frontMatter.tags`. */
   frontMatterTags: FrontMatterTag[] | undefined = [],
 ): Tag[] {
   const tags = frontMatterTags.map((tag) =>
@@ -56,42 +68,42 @@ export function normalizeFrontMatterTags(
   return _.uniqBy(tags, (tag) => tag.permalink);
 }
 
-export type TaggedItemGroup<Item> = {
+type TaggedItemGroup<Item> = {
   tag: Tag;
   items: Item[];
 };
 
 /**
- * Permits to group docs/blogPosts by tag (provided by front matter)
- * Note: groups are indexed by permalink, because routes must be unique in the
- * end. Labels may vary on 2 md files but they are normalized. Docs with
- * label='some label' and label='some-label' should end-up in the same
- * group/page in the end. We can't create 2 routes /some-label because one would
- * override the other
+ * Permits to group docs/blog posts by tag (provided by front matter).
+ *
+ * @returns a map from tag permalink to the items and other relevant tag data.
+ * The record is indexed by permalink, because routes must be unique in the end.
+ * Labels may vary on 2 MD files but they are normalized. Docs with
+ * label='some label' and label='some-label' should end up in the same page.
  */
 export function groupTaggedItems<Item>(
-  items: Item[],
-  getItemTags: (item: Item) => Tag[],
-): Record<string, TaggedItemGroup<Item>> {
-  const result: Record<string, TaggedItemGroup<Item>> = {};
-
-  function handleItemTag(item: Item, tag: Tag) {
-    // Init missing tag groups
-    // TODO: it's not really clear what should be the behavior if 2 items have
-    // the same tag but the permalink is different for each
-    // For now, the first tag found wins
-    result[tag.permalink] = result[tag.permalink] ?? {
-      tag,
-      items: [],
-    };
-
-    // Add item to group
-    result[tag.permalink].items.push(item);
-  }
+  items: readonly Item[],
+  /**
+   * A callback telling me how to get the tags list of the current item. Usually
+   * simply getting it from some metadata of the current item.
+   */
+  getItemTags: (item: Item) => readonly Tag[],
+): {[permalink: string]: TaggedItemGroup<Item>} {
+  const result: {[permalink: string]: TaggedItemGroup<Item>} = {};
 
   items.forEach((item) => {
     getItemTags(item).forEach((tag) => {
-      handleItemTag(item, tag);
+      // Init missing tag groups
+      // TODO: it's not really clear what should be the behavior if 2 tags have
+      // the same permalink but the label is different for each
+      // For now, the first tag found wins
+      result[tag.permalink] ??= {
+        tag,
+        items: [],
+      };
+
+      // Add item to group
+      result[tag.permalink]!.items.push(item);
     });
   });