@docusaurus/utils 2.0.0-beta.8bda3b2db → 2.0.0-beta.9

package/src/__tests__/tags.test.ts

@@ -0,0 +1,183 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {
+  normalizeFrontMatterTag,
+  normalizeFrontMatterTags,
+  groupTaggedItems,
+  Tag,
+} from '../tags';
+
+describe('normalizeFrontMatterTag', () => {
+  type Input = Parameters<typeof normalizeFrontMatterTag>[1];
+  type Output = ReturnType<typeof normalizeFrontMatterTag>;
+
+  test('should normalize simple string tag', () => {
+    const tagsPath = '/all/tags';
+    const input: Input = 'tag';
+    const expectedOutput: Output = {
+      label: 'tag',
+      permalink: `${tagsPath}/tag`,
+    };
+    expect(normalizeFrontMatterTag(tagsPath, input)).toEqual(expectedOutput);
+  });
+
+  test('should normalize complex string tag', () => {
+    const tagsPath = '/all/tags';
+    const input: Input = 'some more Complex_tag';
+    const expectedOutput: Output = {
+      label: 'some more Complex_tag',
+      permalink: `${tagsPath}/some-more-complex-tag`,
+    };
+    expect(normalizeFrontMatterTag(tagsPath, input)).toEqual(expectedOutput);
+  });
+
+  test('should normalize simple object tag', () => {
+    const tagsPath = '/all/tags';
+    const input: Input = {label: 'tag', permalink: 'tagPermalink'};
+    const expectedOutput: Output = {
+      label: 'tag',
+      permalink: `${tagsPath}/tagPermalink`,
+    };
+    expect(normalizeFrontMatterTag(tagsPath, input)).toEqual(expectedOutput);
+  });
+
+  test('should normalize complex string tag', () => {
+    const tagsPath = '/all/tags';
+    const input: Input = {
+      label: 'tag complex Label',
+      permalink: '/MoreComplex/Permalink',
+    };
+    const expectedOutput: Output = {
+      label: 'tag complex Label',
+      permalink: `${tagsPath}/MoreComplex/Permalink`,
+    };
+    expect(normalizeFrontMatterTag(tagsPath, input)).toEqual(expectedOutput);
+  });
+});
+
+describe('normalizeFrontMatterTags', () => {
+  type Input = Parameters<typeof normalizeFrontMatterTags>[1];
+  type Output = ReturnType<typeof normalizeFrontMatterTags>;
+
+  test('should normalize string list', () => {
+    const tagsPath = '/all/tags';
+    const input: Input = ['tag 1', 'tag-1', 'tag 3', 'tag1', 'tag-2'];
+    // Keep user input order but remove tags that lead to same permalink
+    const expectedOutput: Output = [
+      {
+        label: 'tag 1',
+        permalink: `${tagsPath}/tag-1`,
+      },
+      {
+        label: 'tag 3',
+        permalink: `${tagsPath}/tag-3`,
+      },
+      {
+        label: 'tag-2',
+        permalink: `${tagsPath}/tag-2`,
+      },
+    ];
+    expect(normalizeFrontMatterTags(tagsPath, input)).toEqual(expectedOutput);
+  });
+
+  test('should normalize complex mixed list', () => {
+    const tagsPath = '/all/tags';
+    const input: Input = [
+      'tag 1',
+      {label: 'tag-1', permalink: '/tag-1'},
+      'tag 3',
+      'tag1',
+      {label: 'tag 4', permalink: '/tag4Permalink'},
+    ];
+    // Keep user input order but remove tags that lead to same permalink
+    const expectedOutput: Output = [
+      {
+        label: 'tag 1',
+        permalink: `${tagsPath}/tag-1`,
+      },
+      {
+        label: 'tag 3',
+        permalink: `${tagsPath}/tag-3`,
+      },
+      {
+        label: 'tag 4',
+        permalink: `${tagsPath}/tag4Permalink`,
+      },
+    ];
+    expect(normalizeFrontMatterTags(tagsPath, input)).toEqual(expectedOutput);
+  });
+});
+
+describe('groupTaggedItems', () => {
+  type SomeTaggedItem = {
+    id: string;
+    nested: {
+      tags: Tag[];
+    };
+  };
+  function groupItems(items: SomeTaggedItem[]) {
+    return groupTaggedItems(items, (item) => item.nested.tags);
+  }
+
+  type Input = Parameters<typeof groupItems>[0];
+  type Output = ReturnType<typeof groupItems>;
+
+  test('should group items by tag permalink', () => {
+    const tagGuide = {label: 'Guide', permalink: '/guide'};
+    const tagTutorial = {label: 'Tutorial', permalink: '/tutorial'};
+    const tagAPI = {label: 'API', permalink: '/api'};
+
+    // This one will be grouped under same permalink and label is ignored
+    const tagTutorialOtherLabel = {
+      label: 'TutorialOtherLabel',
+      permalink: '/tutorial',
+    };
+
+    const item1: SomeTaggedItem = {
+      id: '1',
+      nested: {
+        tags: [
+          tagGuide,
+          tagTutorial,
+          tagAPI,
+          // Add some duplicates on purpose: they should be filtered
+          tagGuide,
+          tagTutorialOtherLabel,
+        ],
+      },
+    };
+    const item2: SomeTaggedItem = {
+      id: '2',
+      nested: {
+        tags: [tagAPI],
+      },
+    };
+    const item3: SomeTaggedItem = {
+      id: '3',
+      nested: {
+        tags: [tagTutorial],
+      },
+    };
+    const item4: SomeTaggedItem = {
+      id: '4',
+      nested: {
+        tags: [tagTutorialOtherLabel],
+      },
+    };
+
+    const input: Input = [item1, item2, item3, item4];
+
+    const expectedOutput: Output = {
+      '/guide': {tag: tagGuide, items: [item1]},
+      '/tutorial': {tag: tagTutorial, items: [item1, item3, item4]},
+      '/api': {tag: tagAPI, items: [item1, item2]},
+    };
+
+    expect(groupItems(input)).toEqual(expectedOutput);
+  });
+});

package/src/dependencies.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Dependencies with missing typedefs
+
+declare module '@mdx-js/runtime';
+declare module 'remark-mdx-remove-imports';
+declare module 'remark-mdx-remove-exports';

package/src/globUtils.ts

@@ -0,0 +1,63 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+// Globby/Micromatch are the 2 libs we use in Docusaurus consistently
+
+export {default as Globby} from 'globby';
+import Micromatch from 'micromatch'; // Note: Micromatch is used by Globby
+import path from 'path';
+
+// The default patterns we ignore when globbing
+// using _ prefix for exclusion by convention
+export const GlobExcludeDefault = [
+  // Ignore files starting with _
+  '**/_*.{js,jsx,ts,tsx,md,mdx}',
+
+  // Ignore folders starting with _ (including folder content)
+  '**/_*/**',
+
+  // Ignore tests
+  '**/*.test.{js,jsx,ts,tsx}',
+  '**/__tests__/**',
+];
+
+type Matcher = (str: string) => boolean;
+
+export function createMatcher(patterns: string[]): Matcher {
+  const regexp = new RegExp(
+    patterns.map((pattern) => Micromatch.makeRe(pattern).source).join('|'),
+  );
+  return (str) => regexp.test(str);
+}
+
+// We use match patterns like '**/_*/**',
+// This function permits to help to:
+// Match /user/sebastien/website/docs/_partials/xyz.md
+// Ignore /user/_sebastien/website/docs/partials/xyz.md
+export function createAbsoluteFilePathMatcher(
+  patterns: string[],
+  rootFolders: string[],
+): Matcher {
+  const matcher = createMatcher(patterns);
+
+  function getRelativeFilePath(absoluteFilePath: string) {
+    const rootFolder = rootFolders.find((folderPath) =>
+      absoluteFilePath.startsWith(folderPath),
+    );
+    if (!rootFolder) {
+      throw new Error(
+        `createAbsoluteFilePathMatcher unexpected error, absoluteFilePath=${absoluteFilePath} was not contained in any of the root folders ${JSON.stringify(
+          rootFolders,
+        )}`,
+      );
+    }
+    return path.relative(rootFolder, absoluteFilePath);
+  }
+
+  return (absoluteFilePath: string) =>
+    matcher(getRelativeFilePath(absoluteFilePath));
+}

package/src/{docuHash.ts → hashUtils.ts}

@@ -5,10 +5,19 @@
  * LICENSE file in the root directory of this source tree.
  */
 
+import {createHash} from 'crypto';
 import {kebabCase} from 'lodash';
+import {shortName, isNameTooLong} from './pathUtils';
 
-import {shortName, isNameTooLong, simpleHash} from './pathUtils';
+export function md5Hash(str: string): string {
+  return createHash('md5').update(str).digest('hex');
+}
+
+export function simpleHash(str: string, length: number): string {
+  return md5Hash(str).substr(0, length);
+}
 
+// Based on https://github.com/gatsbyjs/gatsby/pull/21518/files
 /**
  * Given an input string, convert to kebab-case and append a hash.
  * Avoid str collision.

package/src/index.ts

@@ -18,22 +18,29 @@ import {
   TranslationFile,
 } from '@docusaurus/types';
 
-// @ts-expect-error: no typedefs :s
 import resolvePathnameUnsafe from 'resolve-pathname';
 
 import {posixPath as posixPathImport} from './posixPath';
-import {simpleHash} from './pathUtils';
-import {docuHash} from './docuHash';
+import {simpleHash, docuHash} from './hashUtils';
+import {normalizeUrl} from './normalizeUrl';
+
+export * from './mdxUtils';
+export * from './normalizeUrl';
+export * from './tags';
 
 export const posixPath = posixPathImport;
 
-export * from './getFilePathForRoutePath';
 export * from './codeTranslationsUtils';
 export * from './markdownParser';
 export * from './markdownLinks';
 export * from './escapePath';
-export * from './docuHash';
-export {simpleHash} from './pathUtils';
+export {md5Hash, simpleHash, docuHash} from './hashUtils';
+export {
+  Globby,
+  GlobExcludeDefault,
+  createMatcher,
+  createAbsoluteFilePathMatcher,
+} from './globUtils';
 
 const fileHash = new Map();
 export async function generate(
@@ -187,80 +194,6 @@ export function getSubFolder(file: string, refDir: string): string | null {
   return match && match[1];
 }
 
-export function normalizeUrl(rawUrls: string[]): string {
-  const urls = [...rawUrls];
-  const resultArray = [];
-
-  let hasStartingSlash = false;
-  let hasEndingSlash = false;
-
-  // If the first part is a plain protocol, we combine it with the next part.
-  if (urls[0].match(/^[^/:]+:\/*$/) && urls.length > 1) {
-    const first = urls.shift();
-    urls[0] = first + urls[0];
-  }
-
-  // There must be two or three slashes in the file protocol,
-  // two slashes in anything else.
-  const replacement = urls[0].match(/^file:\/\/\//) ? '$1:///' : '$1://';
-  urls[0] = urls[0].replace(/^([^/:]+):\/*/, replacement);
-
-  // eslint-disable-next-line
-  for (let i = 0; i < urls.length; i++) {
-    let component = urls[i];
-
-    if (typeof component !== 'string') {
-      throw new TypeError(`Url must be a string. Received ${typeof component}`);
-    }
-
-    if (component === '') {
-      if (i === urls.length - 1 && hasEndingSlash) {
-        resultArray.push('/');
-      }
-      // eslint-disable-next-line
-      continue;
-    }
-
-    if (component !== '/') {
-      if (i > 0) {
-        // Removing the starting slashes for each component but the first.
-        component = component.replace(
-          /^[/]+/,
-          // Special case where the first element of rawUrls is empty ["", "/hello"] => /hello
-          component[0] === '/' && !hasStartingSlash ? '/' : '',
-        );
-      }
-
-      hasEndingSlash = component[component.length - 1] === '/';
-      // Removing the ending slashes for each component but the last.
-      // For the last component we will combine multiple slashes to a single one.
-      component = component.replace(/[/]+$/, i < urls.length - 1 ? '' : '/');
-    }
-
-    hasStartingSlash = true;
-    resultArray.push(component);
-  }
-
-  let str = resultArray.join('/');
-  // Each input component is now separated by a single slash
-  // except the possible first plain protocol part.
-
-  // Remove trailing slash before parameters or hash.
-  str = str.replace(/\/(\?|&|#[^!])/g, '$1');
-
-  // Replace ? in parameters with &.
-  const parts = str.split('?');
-  str = parts.shift() + (parts.length > 0 ? '?' : '') + parts.join('&');
-
-  // Dedupe forward slashes in the entire path, avoiding protocol slashes.
-  str = str.replace(/([^:]\/)\/+/g, '$1');
-
-  // Dedupe forward slashes at the beginning of the path.
-  str = str.replace(/^\/+/g, '/');
-
-  return str;
-}
-
 /**
  * Alias filepath relative to site directory, very useful so that we
  * don't expose user's site structure.
@@ -494,9 +427,7 @@ export function updateTranslationFileMessages(
 
 // Input: ## Some heading {#some-heading}
 // Output: {text: "## Some heading", id: "some-heading"}
-export function parseMarkdownHeadingId(
-  heading: string,
-): {
+export function parseMarkdownHeadingId(heading: string): {
   text: string;
   id?: string;
 } {

package/src/markdownLinks.ts

@@ -5,7 +5,7 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import {resolve} from 'url';
+import path from 'path';
 import {aliasedSitePath} from './index';
 
 export type ContentPaths = {
@@ -63,16 +63,27 @@ export function replaceMarkdownLinks<T extends ContentPaths>({
       // Replace it to correct html link.
       const mdLink = mdMatch[1];
 
-      const aliasedSource = (source: string) =>
-        aliasedSitePath(source, siteDir);
+      const sourcesToTry = [
+        path.resolve(path.dirname(filePath), decodeURIComponent(mdLink)),
+        `${contentPathLocalized}/${decodeURIComponent(mdLink)}`,
+        `${contentPath}/${decodeURIComponent(mdLink)}`,
+      ];
 
-      const permalink: string | undefined =
-        sourceToPermalink[aliasedSource(resolve(filePath, mdLink))] ||
-        sourceToPermalink[aliasedSource(`${contentPathLocalized}/${mdLink}`)] ||
-        sourceToPermalink[aliasedSource(`${contentPath}/${mdLink}`)];
+      const aliasedSourceMatch = sourcesToTry
+        .map((source) => aliasedSitePath(source, siteDir))
+        .find((source) => sourceToPermalink[source]);
+
+      const permalink: string | undefined = aliasedSourceMatch
+        ? sourceToPermalink[aliasedSourceMatch]
+        : undefined;
 
       if (permalink) {
-        modifiedLine = modifiedLine.replace(mdLink, permalink);
+        // MDX won't be happy if the permalink contains a space, we need to convert it to %20
+        const encodedPermalink = permalink
+          .split('/')
+          .map((part) => part.replace(/\s/g, '%20'))
+          .join('/');
+        modifiedLine = modifiedLine.replace(mdLink, encodedPermalink);
       } else {
         const brokenMarkdownLink: BrokenMarkdownLink<T> = {
           contentPaths,

package/src/markdownParser.ts

@@ -18,6 +18,7 @@ export function createExcerpt(fileString: string): string | undefined {
     // Remove Markdown alternate title
     .replace(/^[^\n]*\n[=]+/g, '')
     .split('\n');
+  let inCode = false;
 
   /* eslint-disable no-continue */
   // eslint-disable-next-line no-restricted-syntax
@@ -32,21 +33,29 @@ export function createExcerpt(fileString: string): string | undefined {
       continue;
     }
 
+    // Skip code block line.
+    if (fileLine.trim().startsWith('```')) {
+      inCode = !inCode;
+      continue;
+    } else if (inCode) {
+      continue;
+    }
+
     const cleanedLine = fileLine
       // Remove HTML tags.
       .replace(/<[^>]*>/g, '')
       // Remove Title headers
-      .replace(/^\#\s*([^#]*)\s*\#?/gm, '')
+      .replace(/^#\s*([^#]*)\s*#?/gm, '')
       // Remove Markdown + ATX-style headers
-      .replace(/^\#{1,6}\s*([^#]*)\s*(\#{1,6})?/gm, '$1')
+      .replace(/^#{1,6}\s*([^#]*)\s*(#{1,6})?/gm, '$1')
       // Remove emphasis and strikethroughs.
-      .replace(/([\*_~]{1,3})(\S.*?\S{0,1})\1/g, '$2')
+      .replace(/([*_~]{1,3})(\S.*?\S{0,1})\1/g, '$2')
       // Remove images.
-      .replace(/\!\[(.*?)\][\[\(].*?[\]\)]/g, '$1')
+      .replace(/!\[(.*?)\][[(].*?[\])]/g, '$1')
       // Remove footnotes.
-      .replace(/\[\^.+?\](\: .*?$)?/g, '')
+      .replace(/\[\^.+?\](: .*?$)?/g, '')
       // Remove inline links.
-      .replace(/\[(.*?)\][\[\(].*?[\]\)]/g, '$1')
+      .replace(/\[(.*?)\][[(].*?[\])]/g, '$1')
       // Remove inline code.
       .replace(/`(.+?)`/g, '$1')
       // Remove blockquotes.
@@ -67,9 +76,7 @@ export function createExcerpt(fileString: string): string | undefined {
   return undefined;
 }
 
-export function parseFrontMatter(
-  markdownFileContent: string,
-): {
+export function parseFrontMatter(markdownFileContent: string): {
   frontMatter: Record<string, unknown>;
   content: string;
 } {
@@ -98,10 +105,11 @@ export function parseMarkdownContentTitle(
 
   const content = contentUntrimmed.trim();
 
-  const IMPORT_STATEMENT = /import\s+(([\w*{}\s\n,]+)from\s+)?["'\s]([@\w/_.-]+)["'\s];?|\n/
-    .source;
-  const REGULAR_TITLE = /(?<pattern>#\s*(?<title>[^#\n{]*)+[ \t]*(?<suffix>({#*[\w-]+})|#)?\n*?)/
-    .source;
+  const IMPORT_STATEMENT =
+    /import\s+(([\w*{}\s\n,]+)from\s+)?["'\s]([@\w/_.-]+)["'\s];?|\n/.source;
+  const REGULAR_TITLE =
+    /(?<pattern>#\s*(?<title>[^#\n{]*)+[ \t]*(?<suffix>({#*[\w-]+})|#)?\n*?)/
+      .source;
   const ALTERNATE_TITLE = /(?<pattern>\s*(?<title>[^\n]*)\s*\n[=]+)/.source;
 
   const regularTitleMatch = new RegExp(
@@ -141,9 +149,8 @@ export function parseMarkdownString(
   options?: {removeContentTitle?: boolean},
 ): ParsedMarkdown {
   try {
-    const {frontMatter, content: contentWithoutFrontMatter} = parseFrontMatter(
-      markdownFileContent,
-    );
+    const {frontMatter, content: contentWithoutFrontMatter} =
+      parseFrontMatter(markdownFileContent);
 
     const {content, contentTitle} = parseMarkdownContentTitle(
       contentWithoutFrontMatter,
@@ -176,7 +183,7 @@ export async function parseMarkdownFile(
     return parseMarkdownString(markdownString, options);
   } catch (e) {
     throw new Error(
-      `Error while parsing Markdown file ${source}: "${e.message}".`,
+      `Error while parsing Markdown file ${source}: "${(e as Error).message}".`,
     );
   }
 }

package/src/mdxUtils.ts

@@ -0,0 +1,32 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import React from 'react';
+import ReactDOMServer from 'react-dom/server';
+import MDX from '@mdx-js/runtime';
+import removeImports from 'remark-mdx-remove-imports';
+import removeExports from 'remark-mdx-remove-exports';
+
+/**
+ * Transform mdx text to plain html text
+ * Initially created to convert MDX blog posts to HTML for the RSS feed
+ * without import/export nodes
+ *
+ * TODO not ideal implementation, won't work well with MDX elements!
+ * TODO theme+global site config should be able to declare MDX comps in scope for rendering the RSS feeds
+ * see also https://github.com/facebook/docusaurus/issues/4625
+ */
+export function mdxToHtml(
+  mdxStr: string,
+  // TODO allow providing components/scope here, see https://github.com/mdx-js/mdx/tree/v1.6.13/packages/runtime
+): string {
+  return ReactDOMServer.renderToString(
+    React.createElement(MDX, {remarkPlugins: [removeImports, removeExports]}, [
+      mdxStr,
+    ]),
+  );
+}

package/src/normalizeUrl.ts

@@ -0,0 +1,80 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+export function normalizeUrl(rawUrls: string[]): string {
+  const urls = [...rawUrls];
+  const resultArray = [];
+
+  let hasStartingSlash = false;
+  let hasEndingSlash = false;
+
+  // If the first part is a plain protocol, we combine it with the next part.
+  if (urls[0].match(/^[^/:]+:\/*$/) && urls.length > 1) {
+    const first = urls.shift();
+    urls[0] = first + urls[0];
+  }
+
+  // There must be two or three slashes in the file protocol,
+  // two slashes in anything else.
+  const replacement = urls[0].match(/^file:\/\/\//) ? '$1:///' : '$1://';
+  urls[0] = urls[0].replace(/^([^/:]+):\/*/, replacement);
+
+  // eslint-disable-next-line
+  for (let i = 0; i < urls.length; i++) {
+    let component = urls[i];
+
+    if (typeof component !== 'string') {
+      throw new TypeError(`Url must be a string. Received ${typeof component}`);
+    }
+
+    if (component === '') {
+      if (i === urls.length - 1 && hasEndingSlash) {
+        resultArray.push('/');
+      }
+      // eslint-disable-next-line
+      continue;
+    }
+
+    if (component !== '/') {
+      if (i > 0) {
+        // Removing the starting slashes for each component but the first.
+        component = component.replace(
+          /^[/]+/,
+          // Special case where the first element of rawUrls is empty ["", "/hello"] => /hello
+          component[0] === '/' && !hasStartingSlash ? '/' : '',
+        );
+      }
+
+      hasEndingSlash = component[component.length - 1] === '/';
+      // Removing the ending slashes for each component but the last.
+      // For the last component we will combine multiple slashes to a single one.
+      component = component.replace(/[/]+$/, i < urls.length - 1 ? '' : '/');
+    }
+
+    hasStartingSlash = true;
+    resultArray.push(component);
+  }
+
+  let str = resultArray.join('/');
+  // Each input component is now separated by a single slash
+  // except the possible first plain protocol part.
+
+  // Remove trailing slash before parameters or hash.
+  str = str.replace(/\/(\?|&|#[^!])/g, '$1');
+
+  // Replace ? in parameters with &.
+  const parts = str.split('?');
+  str = parts.shift() + (parts.length > 0 ? '?' : '') + parts.join('&');
+
+  // Dedupe forward slashes in the entire path, avoiding protocol slashes.
+  str = str.replace(/([^:]\/)\/+/g, '$1');
+
+  // Dedupe forward slashes at the beginning of the path.
+  str = str.replace(/^\/+/g, '/');
+
+  return str;
+}

package/src/pathUtils.ts

@@ -7,8 +7,6 @@
 
 // Based on https://github.com/gatsbyjs/gatsby/pull/21518/files
 
-import {createHash} from 'crypto';
-
 // MacOS (APFS) and Windows (NTFS) filename length limit = 255 chars, Others = 255 bytes
 const MAX_PATH_SEGMENT_CHARS = 255;
 const MAX_PATH_SEGMENT_BYTES = 255;
@@ -42,7 +40,3 @@ export const shortName = (str: string): string => {
     )
     .toString();
 };
-
-export function simpleHash(str: string, length: number): string {
-  return createHash('md5').update(str).digest('hex').substr(0, length);
-}