@docusaurus/utils 2.0.0-beta.15d451942 → 2.0.0-beta.16

package/src/markdownParser.ts

@@ -5,10 +5,26 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import chalk from 'chalk';
-import fs from 'fs-extra';
+import logger from '@docusaurus/logger';
 import matter from 'gray-matter';
 
+// Input: ## Some heading {#some-heading}
+// Output: {text: "## Some heading", id: "some-heading"}
+export function parseMarkdownHeadingId(heading: string): {
+  text: string;
+  id?: string;
+} {
+  const customHeadingIdRegex = /^(?<text>.*?)\s*\{#(?<id>[\w-]+)\}$/;
+  const matches = customHeadingIdRegex.exec(heading);
+  if (matches) {
+    return {
+      text: matches.groups!.text,
+      id: matches.groups!.id,
+    };
+  }
+  return {text: heading, id: undefined};
+}
+
 // Hacky way of stripping out import statements from the excerpt
 // TODO: Find a better way to do so, possibly by compiling the Markdown content,
 // stripping out HTML tags and obtaining the first line.
@@ -18,9 +34,10 @@ export function createExcerpt(fileString: string): string | undefined {
     // Remove Markdown alternate title
     .replace(/^[^\n]*\n[=]+/g, '')
     .split('\n');
+  let inCode = false;
+  let lastCodeFence = '';
 
   /* eslint-disable no-continue */
-  // eslint-disable-next-line no-restricted-syntax
   for (const fileLine of fileLines) {
     // Skip empty line.
     if (!fileLine.trim()) {
@@ -28,7 +45,24 @@ export function createExcerpt(fileString: string): string | undefined {
     }
 
     // Skip import/export declaration.
-    if (/^\s*?import\s.*(from.*)?;?|export\s.*{.*};?/.test(fileLine)) {
+    if (/^(?:import|export)\s.*/.test(fileLine)) {
+      continue;
+    }
+
+    // Skip code block line.
+    if (fileLine.trim().startsWith('```')) {
+      if (!inCode) {
+        inCode = true;
+        [lastCodeFence] = fileLine.trim().match(/^`+/)!;
+        // If we are in a ````-fenced block, all ``` would be plain text instead
+        // of fences
+      } else if (
+        fileLine.trim().match(/^`+/)![0].length >= lastCodeFence.length
+      ) {
+        inCode = false;
+      }
+      continue;
+    } else if (inCode) {
       continue;
     }
 
@@ -36,25 +70,29 @@ export function createExcerpt(fileString: string): string | undefined {
       // Remove HTML tags.
       .replace(/<[^>]*>/g, '')
       // Remove Title headers
-      .replace(/^\#\s*([^#]*)\s*\#?/gm, '')
+      .replace(/^#\s*[^#]*\s*#?/gm, '')
       // Remove Markdown + ATX-style headers
-      .replace(/^\#{1,6}\s*([^#]*)\s*(\#{1,6})?/gm, '$1')
-      // Remove emphasis and strikethroughs.
-      .replace(/([\*_~]{1,3})(\S.*?\S{0,1})\1/g, '$2')
+      .replace(/^#{1,6}\s*(?<text>[^#]*)\s*(?:#{1,6})?/gm, '$1')
+      // Remove emphasis.
+      .replace(/(?<opening>[*_]{1,3})(?<text>.*?)\1/g, '$2')
+      // Remove strikethroughs.
+      .replace(/~~(?<text>\S.*\S)~~/g, '$1')
       // Remove images.
-      .replace(/\!\[(.*?)\][\[\(].*?[\]\)]/g, '$1')
+      .replace(/!\[(?<alt>.*?)\][[(].*?[\])]/g, '$1')
       // Remove footnotes.
-      .replace(/\[\^.+?\](\: .*?$)?/g, '')
+      .replace(/\[\^.+?\](?:: .*?$)?/g, '')
       // Remove inline links.
-      .replace(/\[(.*?)\][\[\(].*?[\]\)]/g, '$1')
+      .replace(/\[(?<alt>.*?)\][[(].*?[\])]/g, '$1')
       // Remove inline code.
-      .replace(/`(.+?)`/g, '$1')
+      .replace(/`(?<text>.+?)`/g, '$1')
       // Remove blockquotes.
       .replace(/^\s{0,3}>\s?/g, '')
       // Remove admonition definition.
-      .replace(/(:{3}.*)/, '')
+      .replace(/:::.*/, '')
       // Remove Emoji names within colons include preceding whitespace.
-      .replace(/\s?(:(::|[^:\n])+:)/g, '')
+      .replace(/\s?:(?:::|[^:\n])+:/g, '')
+      // Remove custom Markdown heading id.
+      .replace(/{#*[\w-]+}/, '')
       .trim();
 
     if (cleanedLine) {
@@ -65,31 +103,42 @@ export function createExcerpt(fileString: string): string | undefined {
   return undefined;
 }
 
-export function parseFrontMatter(
-  markdownFileContent: string,
-): {
+export function parseFrontMatter(markdownFileContent: string): {
   frontMatter: Record<string, unknown>;
   content: string;
 } {
   const {data, content} = matter(markdownFileContent);
   return {
-    frontMatter: data ?? {},
-    content: content?.trim() ?? '',
+    frontMatter: data,
+    content: content.trim(),
   };
 }
 
+/**
+ * Try to convert markdown heading to text. Does not need to be perfect, it is
+ * only used as a fallback when frontMatter.title is not provided. For now, we
+ * just unwrap possible inline code blocks (# `config.js`)
+ */
+function toTextContentTitle(contentTitle: string): string {
+  if (contentTitle.startsWith('`') && contentTitle.endsWith('`')) {
+    return contentTitle.substring(1, contentTitle.length - 1);
+  }
+  return contentTitle;
+}
+
 export function parseMarkdownContentTitle(
   contentUntrimmed: string,
-  options?: {keepContentTitle?: boolean},
+  options?: {removeContentTitle?: boolean},
 ): {content: string; contentTitle: string | undefined} {
-  const keepContentTitleOption = options?.keepContentTitle ?? false;
+  const removeContentTitleOption = options?.removeContentTitle ?? false;
 
   const content = contentUntrimmed.trim();
 
-  const IMPORT_STATEMENT = /import\s+(([\w*{}\s\n,]+)from\s+)?["'\s]([@\w/_.-]+)["'\s];?|\n/
-    .source;
-  const REGULAR_TITLE = /(?<pattern>#\s*(?<title>[^#\n{]*)+[ \t]*(?<suffix>({#*[\w-]+})|#)?\n*?)/
-    .source;
+  const IMPORT_STATEMENT =
+    /import\s+(?:[\w*{}\s\n,]+from\s+)?["'\s][@\w/_.-]+["'\s];?|\n/.source;
+  const REGULAR_TITLE =
+    /(?<pattern>#\s*(?<title>[^#\n{]*)+[ \t]*(?<suffix>(?:{#*[\w-]+})|#)?\n*?)/
+      .source;
   const ALTERNATE_TITLE = /(?<pattern>\s*(?<title>[^\n]*)\s*\n[=]+)/.source;
 
   const regularTitleMatch = new RegExp(
@@ -107,14 +156,12 @@ export function parseMarkdownContentTitle(
   if (!pattern || !title) {
     return {content, contentTitle: undefined};
   }
-
-  const newContent = keepContentTitleOption
-    ? content
-    : content.replace(pattern, '');
-
+  const newContent = removeContentTitleOption
+    ? content.replace(pattern, '')
+    : content;
   return {
     content: newContent.trim(),
-    contentTitle: title.trim(),
+    contentTitle: toTextContentTitle(title.trim()).trim(),
   };
 }
 
@@ -127,22 +174,15 @@ type ParsedMarkdown = {
 
 export function parseMarkdownString(
   markdownFileContent: string,
-  options?: {
-    keepContentTitle?: boolean;
-  },
+  options?: {removeContentTitle?: boolean},
 ): ParsedMarkdown {
   try {
-    const keepContentTitle = options?.keepContentTitle ?? false;
-
-    const {frontMatter, content: contentWithoutFrontMatter} = parseFrontMatter(
-      markdownFileContent,
-    );
+    const {frontMatter, content: contentWithoutFrontMatter} =
+      parseFrontMatter(markdownFileContent);
 
     const {content, contentTitle} = parseMarkdownContentTitle(
       contentWithoutFrontMatter,
-      {
-        keepContentTitle,
-      },
+      options,
     );
 
     const excerpt = createExcerpt(content);
@@ -153,25 +193,9 @@ export function parseMarkdownString(
       contentTitle,
       excerpt,
     };
-  } catch (e) {
-    console.error(
-      chalk.red(`Error while parsing markdown front matter.
-This can happen if you use special characters like : in frontmatter values (try using "" around that value)`),
-    );
-    throw e;
-  }
-}
-
-export async function parseMarkdownFile(
-  source: string,
-): Promise<ParsedMarkdown> {
-  const markdownString = await fs.readFile(source, 'utf-8');
-  try {
-    return parseMarkdownString(markdownString);
-  } catch (e) {
-    throw new Error(
-      `Error while parsing markdown file ${source}
-${e.message}`,
-    );
+  } catch (err) {
+    logger.error(`Error while parsing Markdown front matter.
+This can happen if you use special characters in front matter values (try using double quotes around that value).`);
+    throw err;
   }
 }

package/src/pathUtils.ts

@@ -0,0 +1,115 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Based on https://github.com/gatsbyjs/gatsby/pull/21518/files
+
+import path from 'path';
+
+// MacOS (APFS) and Windows (NTFS) filename length limit = 255 chars,
+// Others = 255 bytes
+const MAX_PATH_SEGMENT_CHARS = 255;
+const MAX_PATH_SEGMENT_BYTES = 255;
+// Space for appending things to the string like file extensions and so on
+const SPACE_FOR_APPENDING = 10;
+
+const isMacOs = () => process.platform === 'darwin';
+const isWindows = () => process.platform === 'win32';
+
+export const isNameTooLong = (str: string): boolean =>
+  // Not entirely correct: we can't assume FS from OS. But good enough?
+  isMacOs() || isWindows()
+    ? str.length + SPACE_FOR_APPENDING > MAX_PATH_SEGMENT_CHARS // MacOS (APFS) and Windows (NTFS) filename length limit (255 chars)
+    : Buffer.from(str).length + SPACE_FOR_APPENDING > MAX_PATH_SEGMENT_BYTES; // Other (255 bytes)
+
+export const shortName = (str: string): string => {
+  if (isMacOs() || isWindows()) {
+    const overflowingChars = str.length - MAX_PATH_SEGMENT_CHARS;
+    return str.slice(
+      0,
+      str.length - overflowingChars - SPACE_FOR_APPENDING - 1,
+    );
+  }
+  const strBuffer = Buffer.from(str);
+  const overflowingBytes =
+    Buffer.byteLength(strBuffer) - MAX_PATH_SEGMENT_BYTES;
+  return strBuffer
+    .slice(
+      0,
+      Buffer.byteLength(strBuffer) - overflowingBytes - SPACE_FOR_APPENDING - 1,
+    )
+    .toString();
+};
+
+/**
+ * Convert Windows backslash paths to posix style paths.
+ * E.g: endi\lie -> endi/lie
+ *
+ * Returns original path if the posix counterpart is not valid Windows path.
+ * This makes the legacy code that uses posixPath safe; but also makes it less
+ * useful when you actually want a path with forward slashes (e.g. for URL)
+ *
+ * Adopted from https://github.com/sindresorhus/slash/blob/main/index.js
+ */
+export function posixPath(str: string): string {
+  const isExtendedLengthPath = /^\\\\\?\\/.test(str);
+
+  // Forward slashes are only valid Windows paths when they don't contain non-
+  // ascii characters.
+  // eslint-disable-next-line no-control-regex
+  const hasNonAscii = /[^\u0000-\u0080]+/.test(str);
+
+  if (isExtendedLengthPath || hasNonAscii) {
+    return str;
+  }
+  return str.replace(/\\/g, '/');
+}
+
+/**
+ * When you want to display a path in a message/warning/error, it's more
+ * convenient to:
+ *
+ * - make it relative to `cwd()`
+ * - convert to posix (ie not using windows \ path separator)
+ *
+ * This way, Jest tests can run more reliably on any computer/CI on both
+ * Unix/Windows
+ * For Windows users this is not perfect (as they see / instead of \) but it's
+ * probably good enough
+ */
+export function toMessageRelativeFilePath(filePath: string): string {
+  return posixPath(path.relative(process.cwd(), filePath));
+}
+
+/**
+ * Alias filepath relative to site directory, very useful so that we
+ * don't expose user's site structure.
+ * Example: some/path/to/website/docs/foo.md -> @site/docs/foo.md
+ */
+export function aliasedSitePath(filePath: string, siteDir: string): string {
+  const relativePath = posixPath(path.relative(siteDir, filePath));
+  // Cannot use path.join() as it resolves '../' and removes
+  // the '@site'. Let webpack loader resolve it.
+  return `@site/${relativePath}`;
+}
+
+/**
+ * When you have a path like C:\X\Y
+ * It is not safe to use directly when generating code
+ * For example, this would fail due to unescaped \:
+ * `<img src={require('${filePath}')} />`
+ * But this would work: `<img src={require('${escapePath(filePath)}')} />`
+ *
+ * posixPath can't be used in all cases, because forward slashes are only valid
+ * Windows paths when they don't contain non-ascii characters, and posixPath
+ * doesn't escape those that fail to be converted.
+ */
+export function escapePath(str: string): string {
+  const escaped = JSON.stringify(str);
+
+  // Remove the " around the json string;
+  return escaped.substring(1, escaped.length - 1);
+}

package/src/slugger.ts

@@ -0,0 +1,24 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import GithubSlugger from 'github-slugger';
+
+// We create our own abstraction on top of the lib:
+// - unify usage everywhere in the codebase
+// - ability to add extra options
+export type SluggerOptions = {maintainCase?: boolean};
+
+export type Slugger = {
+  slug: (value: string, options?: SluggerOptions) => string;
+};
+
+export function createSlugger(): Slugger {
+  const githubSlugger = new GithubSlugger();
+  return {
+    slug: (value, options) => githubSlugger.slug(value, options?.maintainCase),
+  };
+}

package/src/tags.ts

@@ -0,0 +1,105 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import _ from 'lodash';
+import {normalizeUrl} from './urlUtils';
+
+export type Tag = {
+  label: string;
+  permalink: string;
+};
+
+export type FrontMatterTag = string | Tag;
+
+export function normalizeFrontMatterTag(
+  tagsPath: string,
+  frontMatterTag: FrontMatterTag,
+): Tag {
+  function toTagObject(tagString: string): Tag {
+    return {
+      label: tagString,
+      permalink: _.kebabCase(tagString),
+    };
+  }
+
+  // TODO maybe make ensure the permalink is valid url path?
+  function normalizeTagPermalink(permalink: string): string {
+    // note: we always apply tagsPath on purpose. For versioned docs, v1/doc.md
+    // and v2/doc.md tags with custom permalinks don't lead to the same created
+    // page. tagsPath is different for each doc version
+    return normalizeUrl([tagsPath, permalink]);
+  }
+
+  const tag: Tag =
+    typeof frontMatterTag === 'string'
+      ? toTagObject(frontMatterTag)
+      : frontMatterTag;
+
+  return {
+    label: tag.label,
+    permalink: normalizeTagPermalink(tag.permalink),
+  };
+}
+
+export function normalizeFrontMatterTags(
+  tagsPath: string,
+  frontMatterTags: FrontMatterTag[] | undefined = [],
+): Tag[] {
+  const tags = frontMatterTags.map((tag) =>
+    normalizeFrontMatterTag(tagsPath, tag),
+  );
+
+  return _.uniqBy(tags, (tag) => tag.permalink);
+}
+
+export type TaggedItemGroup<Item> = {
+  tag: Tag;
+  items: Item[];
+};
+
+/**
+ * Permits to group docs/blogPosts by tag (provided by front matter)
+ * Note: groups are indexed by permalink, because routes must be unique in the
+ * end. Labels may vary on 2 md files but they are normalized. Docs with
+ * label='some label' and label='some-label' should end-up in the same
+ * group/page in the end. We can't create 2 routes /some-label because one would
+ * override the other
+ */
+export function groupTaggedItems<Item>(
+  items: Item[],
+  getItemTags: (item: Item) => Tag[],
+): Record<string, TaggedItemGroup<Item>> {
+  const result: Record<string, TaggedItemGroup<Item>> = {};
+
+  function handleItemTag(item: Item, tag: Tag) {
+    // Init missing tag groups
+    // TODO: it's not really clear what should be the behavior if 2 items have
+    // the same tag but the permalink is different for each
+    // For now, the first tag found wins
+    result[tag.permalink] = result[tag.permalink] ?? {
+      tag,
+      items: [],
+    };
+
+    // Add item to group
+    result[tag.permalink].items.push(item);
+  }
+
+  items.forEach((item) => {
+    getItemTags(item).forEach((tag) => {
+      handleItemTag(item, tag);
+    });
+  });
+
+  // If user add twice the same tag to a md doc (weird but possible),
+  // we don't want the item to appear twice in the list...
+  Object.values(result).forEach((group) => {
+    group.items = _.uniq(group.items);
+  });
+
+  return result;
+}

package/src/urlUtils.ts

@@ -0,0 +1,96 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+export function normalizeUrl(rawUrls: string[]): string {
+  const urls = [...rawUrls];
+  const resultArray = [];
+
+  let hasStartingSlash = false;
+  let hasEndingSlash = false;
+
+  // If the first part is a plain protocol, we combine it with the next part.
+  if (urls[0].match(/^[^/:]+:\/*$/) && urls.length > 1) {
+    const first = urls.shift();
+    if (first!.startsWith('file:') && urls[0].startsWith('/')) {
+      // Force a double slash here, else we lose the information that the next
+      // segment is an absolute path
+      urls[0] = `${first}//${urls[0]}`;
+    } else {
+      urls[0] = first + urls[0];
+    }
+  }
+
+  // There must be two or three slashes in the file protocol,
+  // two slashes in anything else.
+  const replacement = urls[0].match(/^file:\/\/\//) ? '$1:///' : '$1://';
+  urls[0] = urls[0].replace(/^(?<protocol>[^/:]+):\/*/, replacement);
+
+  for (let i = 0; i < urls.length; i += 1) {
+    let component = urls[i];
+
+    if (typeof component !== 'string') {
+      throw new TypeError(`Url must be a string. Received ${typeof component}`);
+    }
+
+    if (component === '') {
+      if (i === urls.length - 1 && hasEndingSlash) {
+        resultArray.push('/');
+      }
+      // eslint-disable-next-line no-continue
+      continue;
+    }
+
+    if (component !== '/') {
+      if (i > 0) {
+        // Removing the starting slashes for each component but the first.
+        component = component.replace(
+          /^[/]+/,
+          // Special case where the first element of rawUrls is empty
+          // ["", "/hello"] => /hello
+          component[0] === '/' && !hasStartingSlash ? '/' : '',
+        );
+      }
+
+      hasEndingSlash = component[component.length - 1] === '/';
+      // Removing the ending slashes for each component but the last. For the
+      // last component we will combine multiple slashes to a single one.
+      component = component.replace(/[/]+$/, i < urls.length - 1 ? '' : '/');
+    }
+
+    hasStartingSlash = true;
+    resultArray.push(component);
+  }
+
+  let str = resultArray.join('/');
+  // Each input component is now separated by a single slash
+  // except the possible first plain protocol part.
+
+  // Remove trailing slash before parameters or hash.
+  str = str.replace(/\/(?<search>\?|&|#[^!])/g, '$1');
+
+  // Replace ? in parameters with &.
+  const parts = str.split('?');
+  str = parts.shift() + (parts.length > 0 ? '?' : '') + parts.join('&');
+
+  // Dedupe forward slashes in the entire path, avoiding protocol slashes.
+  str = str.replace(/(?<textBefore>[^:/]\/)\/+/g, '$1');
+
+  // Dedupe forward slashes at the beginning of the path.
+  str = str.replace(/^\/+/g, '/');
+
+  return str;
+}
+
+export function getEditUrl(
+  fileRelativePath: string,
+  editUrl?: string,
+): string | undefined {
+  return editUrl
+    ? // Don't use posixPath for this: we need to force a forward slash path
+      normalizeUrl([editUrl, fileRelativePath.replace(/\\/g, '/')])
+    : undefined;
+}