@docusaurus/utils 2.0.0-beta.16 → 2.0.0-beta.19

package/src/markdownUtils.ts

@@ -0,0 +1,354 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import logger from '@docusaurus/logger';
+import matter from 'gray-matter';
+import {createSlugger, type Slugger, type SluggerOptions} from './slugger';
+
+// Some utilities for parsing Markdown content. These things are only used on
+// server-side when we infer metadata like `title` and `description` from the
+// content. Most parsing is still done in MDX through the mdx-loader.
+
+/**
+ * Parses custom ID from a heading. The ID must be composed of letters,
+ * underscores, and dashes only.
+ *
+ * @param heading e.g. `## Some heading {#some-heading}` where the last
+ * character must be `}` for the ID to be recognized
+ */
+export function parseMarkdownHeadingId(heading: string): {
+  /**
+   * The heading content sans the ID part, right-trimmed. e.g. `## Some heading`
+   */
+  text: string;
+  /** The heading ID. e.g. `some-heading` */
+  id?: string;
+} {
+  const customHeadingIdRegex = /\s*\{#(?<id>[\w-]+)\}$/;
+  const matches = customHeadingIdRegex.exec(heading);
+  if (matches) {
+    return {
+      text: heading.replace(matches[0]!, ''),
+      id: matches.groups!.id!,
+    };
+  }
+  return {text: heading, id: undefined};
+}
+
+// TODO: Find a better way to do so, possibly by compiling the Markdown content,
+// stripping out HTML tags and obtaining the first line.
+/**
+ * Creates an excerpt of a Markdown file. This function will:
+ *
+ * - Ignore h1 headings (setext or atx)
+ * - Ignore import/export
+ * - Ignore code blocks
+ *
+ * And for the first contentful line, it will strip away most Markdown
+ * syntax, including HTML tags, emphasis, links (keeping the text), etc.
+ */
+export function createExcerpt(fileString: string): string | undefined {
+  const fileLines = fileString
+    .trimStart()
+    // Remove Markdown alternate title
+    .replace(/^[^\n]*\n[=]+/g, '')
+    .split('\n');
+  let inCode = false;
+  let inImport = false;
+  let lastCodeFence = '';
+
+  for (const fileLine of fileLines) {
+    if (fileLine === '' && inImport) {
+      inImport = false;
+    }
+    // Skip empty line.
+    if (!fileLine.trim()) {
+      continue;
+    }
+
+    // Skip import/export declaration.
+    if ((/^(?:import|export)\s.*/.test(fileLine) || inImport) && !inCode) {
+      inImport = true;
+      continue;
+    }
+
+    // Skip code block line.
+    if (fileLine.trim().startsWith('```')) {
+      const codeFence = fileLine.trim().match(/^`+/)![0]!;
+      if (!inCode) {
+        inCode = true;
+        lastCodeFence = codeFence;
+        // If we are in a ````-fenced block, all ``` would be plain text instead
+        // of fences
+      } else if (codeFence.length >= lastCodeFence.length) {
+        inCode = false;
+      }
+      continue;
+    } else if (inCode) {
+      continue;
+    }
+
+    const cleanedLine = fileLine
+      // Remove HTML tags.
+      .replace(/<[^>]*>/g, '')
+      // Remove Title headers
+      .replace(/^#[^#]+#?/gm, '')
+      // Remove Markdown + ATX-style headers
+      .replace(/^#{1,6}\s*(?<text>[^#]*)\s*#{0,6}/gm, '$1')
+      // Remove emphasis.
+      .replace(/(?<opening>[*_]{1,3})(?<text>.*?)\1/g, '$2')
+      // Remove strikethroughs.
+      .replace(/~~(?<text>\S.*\S)~~/g, '$1')
+      // Remove images.
+      .replace(/!\[(?<alt>.*?)\][[(].*?[\])]/g, '$1')
+      // Remove footnotes.
+      .replace(/\[\^.+?\](?:: .*$)?/g, '')
+      // Remove inline links.
+      .replace(/\[(?<alt>.*?)\][[(].*?[\])]/g, '$1')
+      // Remove inline code.
+      .replace(/`(?<text>.+?)`/g, '$1')
+      // Remove blockquotes.
+      .replace(/^\s{0,3}>\s?/g, '')
+      // Remove admonition definition.
+      .replace(/:::.*/, '')
+      // Remove Emoji names within colons include preceding whitespace.
+      .replace(/\s?:(?:::|[^:\n])+:/g, '')
+      // Remove custom Markdown heading id.
+      .replace(/\{#*[\w-]+\}/, '')
+      .trim();
+
+    if (cleanedLine) {
+      return cleanedLine;
+    }
+  }
+
+  return undefined;
+}
+
+/**
+ * Takes a raw Markdown file content, and parses the front matter using
+ * gray-matter. Worth noting that gray-matter accepts TOML and other markup
+ * languages as well.
+ *
+ * @throws Throws when gray-matter throws. e.g.:
+ * ```md
+ * ---
+ * foo: : bar
+ * ---
+ * ```
+ */
+export function parseFrontMatter(markdownFileContent: string): {
+  /** Front matter as parsed by gray-matter. */
+  frontMatter: {[key: string]: unknown};
+  /** The remaining content, trimmed. */
+  content: string;
+} {
+  const {data, content} = matter(markdownFileContent);
+  return {
+    frontMatter: data,
+    content: content.trim(),
+  };
+}
+
+function toTextContentTitle(contentTitle: string): string {
+  if (contentTitle.startsWith('`') && contentTitle.endsWith('`')) {
+    return contentTitle.substring(1, contentTitle.length - 1);
+  }
+  return contentTitle;
+}
+
+type ParseMarkdownContentTitleOptions = {
+  /**
+   * If `true`, the matching title will be removed from the returned content.
+   * We can promise that at least one empty line will be left between the
+   * content before and after, but you shouldn't make too much assumption
+   * about what's left.
+   */
+  removeContentTitle?: boolean;
+};
+
+/**
+ * Takes the raw Markdown content, without front matter, and tries to find an h1
+ * title (setext or atx) to be used as metadata.
+ *
+ * It only searches until the first contentful paragraph, ignoring import/export
+ * declarations.
+ *
+ * It will try to convert markdown to reasonable text, but won't be best effort,
+ * since it's only used as a fallback when `frontMatter.title` is not provided.
+ * For now, we just unwrap inline code (``# `config.js` `` => `config.js`).
+ */
+export function parseMarkdownContentTitle(
+  contentUntrimmed: string,
+  options?: ParseMarkdownContentTitleOptions,
+): {
+  /** The content, optionally without the content title. */
+  content: string;
+  /** The title, trimmed and without the `#`. */
+  contentTitle: string | undefined;
+} {
+  const removeContentTitleOption = options?.removeContentTitle ?? false;
+
+  const content = contentUntrimmed.trim();
+  // We only need to detect import statements that will be parsed by MDX as
+  // `import` nodes, as broken syntax can't render anyways. That means any block
+  // that has `import` at the very beginning and surrounded by empty lines.
+  const contentWithoutImport = content
+    .replace(/^(?:import\s(?:.|\r?\n(?!\r?\n))*(?:\r?\n){2,})*/, '')
+    .trim();
+
+  const regularTitleMatch = /^#[ \t]+(?<title>[^ \t].*)(?:\r?\n|$)/.exec(
+    contentWithoutImport,
+  );
+  const alternateTitleMatch = /^(?<title>.*)\r?\n=+(?:\r?\n|$)/.exec(
+    contentWithoutImport,
+  );
+
+  const titleMatch = regularTitleMatch ?? alternateTitleMatch;
+  if (!titleMatch) {
+    return {content, contentTitle: undefined};
+  }
+  const newContent = removeContentTitleOption
+    ? content.replace(titleMatch[0]!, '')
+    : content;
+  if (regularTitleMatch) {
+    return {
+      content: newContent.trim(),
+      contentTitle: toTextContentTitle(
+        regularTitleMatch
+          .groups!.title!.trim()
+          .replace(/\s*(?:\{#*[\w-]+\}|#+)$/, ''),
+      ).trim(),
+    };
+  }
+  return {
+    content: newContent.trim(),
+    contentTitle: toTextContentTitle(
+      alternateTitleMatch!.groups!.title!.trim().replace(/\s*=+$/, ''),
+    ).trim(),
+  };
+}
+
+/**
+ * Makes a full-round parse.
+ *
+ * @throws Throws when `parseFrontMatter` throws, usually because of invalid
+ * syntax.
+ */
+export function parseMarkdownString(
+  markdownFileContent: string,
+  options?: ParseMarkdownContentTitleOptions,
+): {
+  /** @see {@link parseFrontMatter} */
+  frontMatter: {[key: string]: unknown};
+  /** @see {@link parseMarkdownContentTitle} */
+  contentTitle: string | undefined;
+  /** @see {@link createExcerpt} */
+  excerpt: string | undefined;
+  /**
+   * Content without front matter and (optionally) without title, depending on
+   * the `removeContentTitle` option.
+   */
+  content: string;
+} {
+  try {
+    const {frontMatter, content: contentWithoutFrontMatter} =
+      parseFrontMatter(markdownFileContent);
+
+    const {content, contentTitle} = parseMarkdownContentTitle(
+      contentWithoutFrontMatter,
+      options,
+    );
+
+    const excerpt = createExcerpt(content);
+
+    return {
+      frontMatter,
+      content,
+      contentTitle,
+      excerpt,
+    };
+  } catch (err) {
+    logger.error(`Error while parsing Markdown front matter.
+This can happen if you use special characters in front matter values (try using double quotes around that value).`);
+    throw err;
+  }
+}
+
+function unwrapMarkdownLinks(line: string): string {
+  return line.replace(/\[(?<alt>[^\]]+)\]\([^)]+\)/g, (match, p1) => p1);
+}
+
+function addHeadingId(
+  line: string,
+  slugger: Slugger,
+  maintainCase: boolean,
+): string {
+  let headingLevel = 0;
+  while (line.charAt(headingLevel) === '#') {
+    headingLevel += 1;
+  }
+
+  const headingText = line.slice(headingLevel).trimEnd();
+  const headingHashes = line.slice(0, headingLevel);
+  const slug = slugger.slug(unwrapMarkdownLinks(headingText).trim(), {
+    maintainCase,
+  });
+
+  return `${headingHashes}${headingText} {#${slug}}`;
+}
+
+export type WriteHeadingIDOptions = SluggerOptions & {
+  /** Overwrite existing heading IDs. */
+  overwrite?: boolean;
+};
+
+/**
+ * Takes Markdown content, returns new content with heading IDs written.
+ * Respects existing IDs (unless `overwrite=true`) and never generates colliding
+ * IDs (through the slugger).
+ */
+export function writeMarkdownHeadingId(
+  content: string,
+  options: WriteHeadingIDOptions = {maintainCase: false, overwrite: false},
+): string {
+  const {maintainCase = false, overwrite = false} = options;
+  const lines = content.split('\n');
+  const slugger = createSlugger();
+
+  // If we can't overwrite existing slugs, make sure other headings don't
+  // generate colliding slugs by first marking these slugs as occupied
+  if (!overwrite) {
+    lines.forEach((line) => {
+      const parsedHeading = parseMarkdownHeadingId(line);
+      if (parsedHeading.id) {
+        slugger.slug(parsedHeading.id);
+      }
+    });
+  }
+
+  let inCode = false;
+  return lines
+    .map((line) => {
+      if (line.startsWith('```')) {
+        inCode = !inCode;
+        return line;
+      }
+      // Ignore h1 headings, as we don't create anchor links for those
+      if (inCode || !line.startsWith('##')) {
+        return line;
+      }
+      const parsedHeading = parseMarkdownHeadingId(line);
+
+      // Do not process if id is already there
+      if (parsedHeading.id && !overwrite) {
+        return line;
+      }
+      return addHeadingId(parsedHeading.text, slugger, maintainCase);
+    })
+    .join('\n');
+}

package/src/pathUtils.ts

@@ -5,11 +5,10 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-// Based on https://github.com/gatsbyjs/gatsby/pull/21518/files
-
 import path from 'path';
 
-// MacOS (APFS) and Windows (NTFS) filename length limit = 255 chars,
+// Based on https://github.com/gatsbyjs/gatsby/pull/21518/files
+// macOS (APFS) and Windows (NTFS) filename length limit = 255 chars,
 // Others = 255 bytes
 const MAX_PATH_SEGMENT_CHARS = 255;
 const MAX_PATH_SEGMENT_BYTES = 255;
@@ -22,10 +21,12 @@ const isWindows = () => process.platform === 'win32';
 export const isNameTooLong = (str: string): boolean =>
   // Not entirely correct: we can't assume FS from OS. But good enough?
   isMacOs() || isWindows()
-    ? str.length + SPACE_FOR_APPENDING > MAX_PATH_SEGMENT_CHARS // MacOS (APFS) and Windows (NTFS) filename length limit (255 chars)
-    : Buffer.from(str).length + SPACE_FOR_APPENDING > MAX_PATH_SEGMENT_BYTES; // Other (255 bytes)
+    ? // Windows (NTFS) and macOS (APFS) filename length limit (255 chars)
+      str.length + SPACE_FOR_APPENDING > MAX_PATH_SEGMENT_CHARS
+    : // Other (255 bytes)
+      Buffer.from(str).length + SPACE_FOR_APPENDING > MAX_PATH_SEGMENT_BYTES;
 
-export const shortName = (str: string): string => {
+export function shortName(str: string): string {
   if (isMacOs() || isWindows()) {
     const overflowingChars = str.length - MAX_PATH_SEGMENT_CHARS;
     return str.slice(
@@ -42,7 +43,7 @@ export const shortName = (str: string): string => {
       Buffer.byteLength(strBuffer) - overflowingBytes - SPACE_FOR_APPENDING - 1,
     )
     .toString();
-};
+}
 
 /**
  * Convert Windows backslash paths to posix style paths.
@@ -113,3 +114,10 @@ export function escapePath(str: string): string {
   // Remove the " around the json string;
   return escaped.substring(1, escaped.length - 1);
 }
+
+export function addTrailingPathSeparator(str: string): string {
+  return str.endsWith(path.sep)
+    ? str
+    : // If this is Windows, we need to change the forward slash to backward
+      `${str.replace(/\/$/, '')}${path.sep}`;
+}

package/src/slugger.ts

@@ -10,12 +10,24 @@ import GithubSlugger from 'github-slugger';
 // We create our own abstraction on top of the lib:
 // - unify usage everywhere in the codebase
 // - ability to add extra options
-export type SluggerOptions = {maintainCase?: boolean};
+export type SluggerOptions = {
+  /** Keep the headings' casing, otherwise make all lowercase. */
+  maintainCase?: boolean;
+};
 
 export type Slugger = {
+  /**
+   * Takes a Markdown heading like "Josh Cena" and sluggifies it according to
+   * GitHub semantics (in this case `josh-cena`). Stateful, because if you try
+   * to sluggify "Josh Cena" again it would return `josh-cena-1`.
+   */
   slug: (value: string, options?: SluggerOptions) => string;
 };
 
+/**
+ * A thin wrapper around github-slugger. This is a factory function that returns
+ * a stateful Slugger object.
+ */
 export function createSlugger(): Slugger {
   const githubSlugger = new GithubSlugger();
   return {

package/src/tags.ts

@@ -8,14 +8,28 @@
 import _ from 'lodash';
 import {normalizeUrl} from './urlUtils';
 
+/** What the user configures. */
 export type Tag = {
   label: string;
+  /** Permalink to this tag's page, without the `/tags/` base path. */
   permalink: string;
 };
 
+/** What the tags list page should know about each tag. */
+export type TagsListItem = Tag & {
+  /** Number of posts/docs with this tag. */
+  count: number;
+};
+
+/** What the tag's own page should know about the tag. */
+export type TagModule = TagsListItem & {
+  /** The tags list page's permalink. */
+  allTagsPath: string;
+};
+
 export type FrontMatterTag = string | Tag;
 
-export function normalizeFrontMatterTag(
+function normalizeFrontMatterTag(
   tagsPath: string,
   frontMatterTag: FrontMatterTag,
 ): Tag {
@@ -28,7 +42,7 @@ export function normalizeFrontMatterTag(
 
   // TODO maybe make ensure the permalink is valid url path?
   function normalizeTagPermalink(permalink: string): string {
-    // note: we always apply tagsPath on purpose. For versioned docs, v1/doc.md
+    // Note: we always apply tagsPath on purpose. For versioned docs, v1/doc.md
     // and v2/doc.md tags with custom permalinks don't lead to the same created
     // page. tagsPath is different for each doc version
     return normalizeUrl([tagsPath, permalink]);
@@ -45,8 +59,19 @@ export function normalizeFrontMatterTag(
   };
 }
 
+/**
+ * Takes tag objects as they are defined in front matter, and normalizes each
+ * into a standard tag object. The permalink is created by appending the
+ * sluggified label to `tagsPath`. Front matter tags already containing
+ * permalinks would still have `tagsPath` prepended.
+ *
+ * The result will always be unique by permalinks. The behavior with colliding
+ * permalinks is undetermined.
+ */
 export function normalizeFrontMatterTags(
+  /** Base path to append the tag permalinks to. */
   tagsPath: string,
+  /** Can be `undefined`, so that we can directly pipe in `frontMatter.tags`. */
   frontMatterTags: FrontMatterTag[] | undefined = [],
 ): Tag[] {
   const tags = frontMatterTags.map((tag) =>
@@ -56,42 +81,42 @@ export function normalizeFrontMatterTags(
   return _.uniqBy(tags, (tag) => tag.permalink);
 }
 
-export type TaggedItemGroup<Item> = {
+type TaggedItemGroup<Item> = {
   tag: Tag;
   items: Item[];
 };
 
 /**
- * Permits to group docs/blogPosts by tag (provided by front matter)
- * Note: groups are indexed by permalink, because routes must be unique in the
- * end. Labels may vary on 2 md files but they are normalized. Docs with
- * label='some label' and label='some-label' should end-up in the same
- * group/page in the end. We can't create 2 routes /some-label because one would
- * override the other
+ * Permits to group docs/blog posts by tag (provided by front matter).
+ *
+ * @returns a map from tag permalink to the items and other relevant tag data.
+ * The record is indexed by permalink, because routes must be unique in the end.
+ * Labels may vary on 2 MD files but they are normalized. Docs with
+ * label='some label' and label='some-label' should end up in the same page.
  */
 export function groupTaggedItems<Item>(
-  items: Item[],
-  getItemTags: (item: Item) => Tag[],
-): Record<string, TaggedItemGroup<Item>> {
-  const result: Record<string, TaggedItemGroup<Item>> = {};
-
-  function handleItemTag(item: Item, tag: Tag) {
-    // Init missing tag groups
-    // TODO: it's not really clear what should be the behavior if 2 items have
-    // the same tag but the permalink is different for each
-    // For now, the first tag found wins
-    result[tag.permalink] = result[tag.permalink] ?? {
-      tag,
-      items: [],
-    };
-
-    // Add item to group
-    result[tag.permalink].items.push(item);
-  }
+  items: readonly Item[],
+  /**
+   * A callback telling me how to get the tags list of the current item. Usually
+   * simply getting it from some metadata of the current item.
+   */
+  getItemTags: (item: Item) => readonly Tag[],
+): {[permalink: string]: TaggedItemGroup<Item>} {
+  const result: {[permalink: string]: TaggedItemGroup<Item>} = {};
 
   items.forEach((item) => {
     getItemTags(item).forEach((tag) => {
-      handleItemTag(item, tag);
+      // Init missing tag groups
+      // TODO: it's not really clear what should be the behavior if 2 tags have
+      // the same permalink but the label is different for each
+      // For now, the first tag found wins
+      result[tag.permalink] ??= {
+        tag,
+        items: [],
+      };
+
+      // Add item to group
+      result[tag.permalink]!.items.push(item);
     });
   });