@docusaurus/utils 3.3.2 → 3.5.0

package/src/markdownLinks.ts

@@ -40,159 +40,40 @@ export type BrokenMarkdownLink<T extends ContentPaths> = {
   link: string;
 };
 
-type CodeFence = {
-  type: '`' | '~';
-  definitelyOpen: boolean;
-  count: number;
-};
+export type SourceToPermalink = Map<
+  string, // Aliased source path: "@site/docs/content.mdx"
+  string // Permalink: "/docs/content"
+>;
 
-function parseCodeFence(line: string): CodeFence | null {
-  const match = line.trim().match(/^(?<fence>`{3,}|~{3,})(?<rest>.*)/);
-  if (!match) {
-    return null;
+// Note this is historical logic extracted during a 2024 refactor
+// The algo has been kept exactly as before for retro compatibility
+// See also https://github.com/facebook/docusaurus/pull/10168
+export function resolveMarkdownLinkPathname(
+  linkPathname: string,
+  context: {
+    sourceFilePath: string;
+    sourceToPermalink: SourceToPermalink;
+    contentPaths: ContentPaths;
+    siteDir: string;
+  },
+): string | null {
+  const {sourceFilePath, sourceToPermalink, contentPaths, siteDir} = context;
+  const sourceDirsToTry: string[] = [];
+  // ./file.md and ../file.md are always relative to the current file
+  if (!linkPathname.startsWith('./') && !linkPathname.startsWith('../')) {
+    sourceDirsToTry.push(...getContentPathList(contentPaths), siteDir);
+  }
+  // /file.md is never relative to the source file path
+  if (!linkPathname.startsWith('/')) {
+    sourceDirsToTry.push(path.dirname(sourceFilePath));
   }
-  return {
-    type: match.groups!.fence![0]! as '`' | '~',
-    definitelyOpen: !!match.groups!.rest!,
-    count: match.groups!.fence!.length,
-  };
-}
-
-/**
- * Takes a Markdown file and replaces relative file references with their URL
- * counterparts, e.g. `[link](./intro.md)` => `[link](/docs/intro)`, preserving
- * everything else.
- *
- * This method uses best effort to find a matching file. The file reference can
- * be relative to the directory of the current file (most likely) or any of the
- * content paths (so `/tutorials/intro.md` can be resolved as
- * `<siteDir>/docs/tutorials/intro.md`). Links that contain the `http(s):` or
- * `@site/` prefix will always be ignored.
- */
-export function replaceMarkdownLinks<T extends ContentPaths>({
-  siteDir,
-  fileString,
-  filePath,
-  contentPaths,
-  sourceToPermalink,
-}: {
-  /** Absolute path to the site directory, used to resolve aliased paths. */
-  siteDir: string;
-  /** The Markdown file content to be processed. */
-  fileString: string;
-  /** Absolute path to the current file containing `fileString`. */
-  filePath: string;
-  /** The content paths which the file reference may live in. */
-  contentPaths: T;
-  /**
-   * A map from source paths to their URLs. Source paths are `@site` aliased.
-   */
-  sourceToPermalink: {[aliasedPath: string]: string};
-}): {
-  /**
-   * The content with all Markdown file references replaced with their URLs.
-   * Unresolved links are left as-is.
-   */
-  newContent: string;
-  /** The list of broken links,  */
-  brokenMarkdownLinks: BrokenMarkdownLink<T>[];
-} {
-  const brokenMarkdownLinks: BrokenMarkdownLink<T>[] = [];
-
-  // Replace internal markdown linking (except in fenced blocks).
-  let lastOpenCodeFence: CodeFence | null = null;
-  const lines = fileString.split('\n').map((line) => {
-    const codeFence = parseCodeFence(line);
-    if (codeFence) {
-      if (!lastOpenCodeFence) {
-        lastOpenCodeFence = codeFence;
-      } else if (
-        !codeFence.definitelyOpen &&
-        lastOpenCodeFence.type === codeFence.type &&
-        lastOpenCodeFence.count <= codeFence.count
-      ) {
-        // All three conditions must be met in order for this to be considered
-        // a closing fence.
-        lastOpenCodeFence = null;
-      }
-    }
-    if (lastOpenCodeFence) {
-      return line;
-    }
-
-    let modifiedLine = line;
-    // Replace inline-style links or reference-style links e.g:
-    // This is [Document 1](doc1.md)
-    // [doc1]: doc1.md
-    const linkTitlePattern = '(?:\\s+(?:\'.*?\'|".*?"|\\(.*?\\)))?';
-    const linkSuffixPattern = '(?:\\?[^#>\\s]+)?(?:#[^>\\s]+)?';
-    const linkCapture = (forbidden: string) =>
-      `((?!https?://|@site/)[^${forbidden}#?]+)`;
-    const linkURLPattern = `(?:(?!<)${linkCapture(
-      '()\\s',
-    )}${linkSuffixPattern}|<${linkCapture('>')}${linkSuffixPattern}>)`;
-    const linkPattern = new RegExp(
-      `\\[(?:(?!\\]\\().)*\\]\\(\\s*${linkURLPattern}${linkTitlePattern}\\s*\\)|^\\s*\\[[^[\\]]*[^[\\]\\s][^[\\]]*\\]:\\s*${linkURLPattern}${linkTitlePattern}$`,
-      'dgm',
-    );
-    let mdMatch = linkPattern.exec(modifiedLine);
-    while (mdMatch !== null) {
-      // Replace it to correct html link.
-      const mdLink = mdMatch.slice(1, 5).find(Boolean)!;
-      const mdLinkRange = mdMatch.indices!.slice(1, 5).find(Boolean)!;
-      if (!/\.mdx?$/.test(mdLink)) {
-        mdMatch = linkPattern.exec(modifiedLine);
-        continue;
-      }
-
-      const sourcesToTry: string[] = [];
-      // ./file.md and ../file.md are always relative to the current file
-      if (!mdLink.startsWith('./') && !mdLink.startsWith('../')) {
-        sourcesToTry.push(...getContentPathList(contentPaths), siteDir);
-      }
-      // /file.md is always relative to the content path
-      if (!mdLink.startsWith('/')) {
-        sourcesToTry.push(path.dirname(filePath));
-      }
-
-      const aliasedSourceMatch = sourcesToTry
-        .map((p) => path.join(p, decodeURIComponent(mdLink)))
-        .map((source) => aliasedSitePath(source, siteDir))
-        .find((source) => sourceToPermalink[source]);
-
-      const permalink: string | undefined = aliasedSourceMatch
-        ? sourceToPermalink[aliasedSourceMatch]
-        : undefined;
-
-      if (permalink) {
-        // MDX won't be happy if the permalink contains a space, we need to
-        // convert it to %20
-        const encodedPermalink = permalink
-          .split('/')
-          .map((part) => part.replace(/\s/g, '%20'))
-          .join('/');
-        modifiedLine = `${modifiedLine.slice(
-          0,
-          mdLinkRange[0],
-        )}${encodedPermalink}${modifiedLine.slice(mdLinkRange[1])}`;
-        // Adjust the lastIndex to avoid passing over the next link if the
-        // newly replaced URL is shorter.
-        linkPattern.lastIndex += encodedPermalink.length - mdLink.length;
-      } else {
-        const brokenMarkdownLink: BrokenMarkdownLink<T> = {
-          contentPaths,
-          filePath,
-          link: mdLink,
-        };
-
-        brokenMarkdownLinks.push(brokenMarkdownLink);
-      }
-      mdMatch = linkPattern.exec(modifiedLine);
-    }
-    return modifiedLine;
-  });
 
-  const newContent = lines.join('\n');
+  const aliasedSourceMatch = sourceDirsToTry
+    .map((sourceDir) => path.join(sourceDir, decodeURIComponent(linkPathname)))
+    .map((source) => aliasedSitePath(source, siteDir))
+    .find((source) => sourceToPermalink.has(source));
 
-  return {newContent, brokenMarkdownLinks};
+  return aliasedSourceMatch
+    ? sourceToPermalink.get(aliasedSourceMatch) ?? null
+    : null;
 }

package/src/markdownUtils.ts

@@ -70,9 +70,9 @@ export function escapeMarkdownHeadingIds(content: string): string {
 export function unwrapMdxCodeBlocks(content: string): string {
   // We only support 3/4 backticks on purpose, should be good enough
   const regexp3 =
-    /(?<begin>^|\r?\n)```(?<spaces>\x20*)mdx-code-block\r?\n(?<children>.*?)\r?\n```(?<end>\r?\n|$)/gs;
+    /(?<begin>^|\r?\n)(?<indentStart>\x20*)```(?<spaces>\x20*)mdx-code-block\r?\n(?<children>.*?)\r?\n(?<indentEnd>\x20*)```(?<end>\r?\n|$)/gs;
   const regexp4 =
-    /(?<begin>^|\r?\n)````(?<spaces>\x20*)mdx-code-block\r?\n(?<children>.*?)\r?\n````(?<end>\r?\n|$)/gs;
+    /(?<begin>^|\r?\n)(?<indentStart>\x20*)````(?<spaces>\x20*)mdx-code-block\r?\n(?<children>.*?)\r?\n(?<indentEnd>\x20*)````(?<end>\r?\n|$)/gs;
 
   // eslint-disable-next-line @typescript-eslint/no-explicit-any
   const replacer = (substring: string, ...args: any[]) => {

package/src/tags.ts

@@ -6,13 +6,34 @@
  */
 
 import _ from 'lodash';
+import logger from '@docusaurus/logger';
 import {normalizeUrl} from './urlUtils';
+import type {Optional} from 'utility-types';
 
-/** What the user configures. */
 export type Tag = {
+  /** The display label of a tag */
   label: string;
   /** Permalink to this tag's page, without the `/tags/` base path. */
   permalink: string;
+  /** An optional description of the tag */
+  description: string | undefined;
+};
+
+export type TagsFileInput = Record<string, Partial<Tag> | null>;
+
+export type TagsFile = Record<string, Tag>;
+
+// Tags plugins options shared between docs/blog
+export type TagsPluginOptions = {
+  // TODO allow option tags later? | TagsFile;
+  /** Path to the tags file. */
+  tags: string | false | null | undefined;
+  /** The behavior of Docusaurus when it finds inline tags. */
+  onInlineTags: 'ignore' | 'log' | 'warn' | 'throw';
+};
+
+export type TagMetadata = Tag & {
+  inline: boolean;
 };
 
 /** What the tags list page should know about each tag. */
@@ -24,67 +45,132 @@ export type TagsListItem = Tag & {
 /** What the tag's own page should know about the tag. */
 export type TagModule = TagsListItem & {
   /** The tags list page's permalink. */
+  // TODO move this global value to a shared docs/blog bundle
   allTagsPath: string;
   /** Is this tag unlisted? (when it only contains unlisted items) */
   unlisted: boolean;
 };
 
-export type FrontMatterTag = string | Tag;
+export type FrontMatterTag = string | Optional<Tag, 'description'>;
 
-function normalizeFrontMatterTag(
-  tagsPath: string,
+// We always apply tagsBaseRoutePath on purpose. For versioned docs, v1/doc.md
+// and v2/doc.md tags with custom permalinks don't lead to the same created
+// page. tagsBaseRoutePath is different for each doc version
+function normalizeTagPermalink({
+  tagsBaseRoutePath,
+  permalink,
+}: {
+  tagsBaseRoutePath: string;
+  permalink: string;
+}): string {
+  return normalizeUrl([tagsBaseRoutePath, permalink]);
+}
+
+function normalizeInlineTag(
+  tagsBaseRoutePath: string,
   frontMatterTag: FrontMatterTag,
-): Tag {
-  function toTagObject(tagString: string): Tag {
+): TagMetadata {
+  function toTagObject(tagString: string): TagMetadata {
     return {
+      inline: true,
       label: tagString,
       permalink: _.kebabCase(tagString),
+      description: undefined,
     };
   }
 
-  // TODO maybe make ensure the permalink is valid url path?
-  function normalizeTagPermalink(permalink: string): string {
-    // Note: we always apply tagsPath on purpose. For versioned docs, v1/doc.md
-    // and v2/doc.md tags with custom permalinks don't lead to the same created
-    // page. tagsPath is different for each doc version
-    return normalizeUrl([tagsPath, permalink]);
-  }
-
   const tag: Tag =
     typeof frontMatterTag === 'string'
       ? toTagObject(frontMatterTag)
-      : frontMatterTag;
+      : {...frontMatterTag, description: frontMatterTag.description};
 
   return {
+    inline: true,
     label: tag.label,
-    permalink: normalizeTagPermalink(tag.permalink),
+    permalink: normalizeTagPermalink({
+      permalink: tag.permalink,
+      tagsBaseRoutePath,
+    }),
+    description: tag.description,
   };
 }
 
-/**
- * Takes tag objects as they are defined in front matter, and normalizes each
- * into a standard tag object. The permalink is created by appending the
- * sluggified label to `tagsPath`. Front matter tags already containing
- * permalinks would still have `tagsPath` prepended.
- *
- * The result will always be unique by permalinks. The behavior with colliding
- * permalinks is undetermined.
- */
-export function normalizeFrontMatterTags(
-  /** Base path to append the tag permalinks to. */
-  tagsPath: string,
-  /** Can be `undefined`, so that we can directly pipe in `frontMatter.tags`. */
-  frontMatterTags: FrontMatterTag[] | undefined = [],
-): Tag[] {
-  const tags = frontMatterTags.map((tag) =>
-    normalizeFrontMatterTag(tagsPath, tag),
+export function normalizeTag({
+  tag,
+  tagsFile,
+  tagsBaseRoutePath,
+}: {
+  tag: FrontMatterTag;
+  tagsBaseRoutePath: string;
+  tagsFile: TagsFile | null;
+}): TagMetadata {
+  if (typeof tag === 'string') {
+    const tagDescription = tagsFile?.[tag];
+    if (tagDescription) {
+      // pre-defined tag from tags.yml
+      return {
+        inline: false,
+        label: tagDescription.label,
+        permalink: normalizeTagPermalink({
+          permalink: tagDescription.permalink,
+          tagsBaseRoutePath,
+        }),
+        description: tagDescription.description,
+      };
+    }
+  }
+  // legacy inline tag object, always inline, unknown because isn't a string
+  return normalizeInlineTag(tagsBaseRoutePath, tag);
+}
+
+export function normalizeTags({
+  options,
+  source,
+  frontMatterTags,
+  tagsBaseRoutePath,
+  tagsFile,
+}: {
+  options: TagsPluginOptions;
+  source: string;
+  frontMatterTags: FrontMatterTag[] | undefined;
+  tagsBaseRoutePath: string;
+  tagsFile: TagsFile | null;
+}): TagMetadata[] {
+  const tags = (frontMatterTags ?? []).map((tag) =>
+    normalizeTag({tag, tagsBaseRoutePath, tagsFile}),
   );
+  if (tagsFile !== null) {
+    reportInlineTags({tags, source, options});
+  }
+  return tags;
+}
 
-  return _.uniqBy(tags, (tag) => tag.permalink);
+export function reportInlineTags({
+  tags,
+  source,
+  options,
+}: {
+  tags: TagMetadata[];
+  source: string;
+  options: TagsPluginOptions;
+}): void {
+  if (options.onInlineTags === 'ignore') {
+    return;
+  }
+  const inlineTags = tags.filter((tag) => tag.inline);
+  if (inlineTags.length > 0) {
+    const uniqueUnknownTags = [...new Set(inlineTags.map((tag) => tag.label))];
+    const tagListString = uniqueUnknownTags.join(', ');
+    logger.report(options.onInlineTags)(
+      `Tags [${tagListString}] used in ${source} are not defined in ${
+        options.tags ?? 'tags.yml'
+      }`,
+    );
+  }
 }
 
 type TaggedItemGroup<Item> = {
-  tag: Tag;
+  tag: TagMetadata;
   items: Item[];
 };
 
@@ -102,7 +188,7 @@ export function groupTaggedItems<Item>(
    * A callback telling me how to get the tags list of the current item. Usually
    * simply getting it from some metadata of the current item.
    */
-  getItemTags: (item: Item) => readonly Tag[],
+  getItemTags: (item: Item) => readonly TagMetadata[],
 ): {[permalink: string]: TaggedItemGroup<Item>} {
   const result: {[permalink: string]: TaggedItemGroup<Item>} = {};
 

package/src/urlUtils.ts

@@ -90,7 +90,7 @@ export function normalizeUrl(rawUrls: string[]): string {
   // first plain protocol part.
 
   // Remove trailing slash before parameters or hash.
-  str = str.replace(/\/(?<search>\?|&|#[^!])/g, '$1');
+  str = str.replace(/\/(?<search>\?|&|#[^!/])/g, '$1');
 
   // Replace ? in parameters with &.
   const parts = str.split('?');
@@ -164,27 +164,22 @@ export function isValidPathname(str: string): boolean {
   }
 }
 
-export type URLPath = {pathname: string; search?: string; hash?: string};
-
-// Let's name the concept of (pathname + search + hash) as URLPath
-// See also https://twitter.com/kettanaito/status/1741768992866308120
-// Note: this function also resolves relative pathnames while parsing!
-export function parseURLPath(urlPath: string, fromPath?: string): URLPath {
-  function parseURL(url: string, base?: string | URL): URL {
-    try {
-      // A possible alternative? https://github.com/unjs/ufo#url
-      return new URL(url, base ?? 'https://example.com');
-    } catch (e) {
-      throw new Error(
-        `Can't parse URL ${url}${base ? ` with base ${base}` : ''}`,
-        {cause: e},
-      );
-    }
+export function parseURLOrPath(url: string, base?: string | URL): URL {
+  try {
+    // TODO when Node supports it, use URL.parse could be faster?
+    //  see https://kilianvalkhof.com/2024/javascript/the-problem-with-new-url-and-how-url-parse-fixes-that/
+    return new URL(url, base ?? 'https://example.com');
+  } catch (e) {
+    throw new Error(
+      `Can't parse URL ${url}${base ? ` with base ${base}` : ''}`,
+      {cause: e},
+    );
   }
+}
 
-  const base = fromPath ? parseURL(fromPath) : undefined;
-  const url = parseURL(urlPath, base);
+export type URLPath = {pathname: string; search?: string; hash?: string};
 
+export function toURLPath(url: URL): URLPath {
   const {pathname} = url;
 
   // Fixes annoying url.search behavior
@@ -193,17 +188,17 @@ export function parseURLPath(urlPath: string, fromPath?: string): URLPath {
   // "?param => "param"
   const search = url.search
     ? url.search.slice(1)
-    : urlPath.includes('?')
+    : url.href.includes('?')
     ? ''
     : undefined;
 
   // Fixes annoying url.hash behavior
   // "" => undefined
   // "#" => ""
-  // "?param => "param"
+  // "#param => "param"
   const hash = url.hash
     ? url.hash.slice(1)
-    : urlPath.includes('#')
+    : url.href.includes('#')
     ? ''
     : undefined;
 
@@ -214,6 +209,65 @@ export function parseURLPath(urlPath: string, fromPath?: string): URLPath {
   };
 }
 
+/**
+ * Let's name the concept of (pathname + search + hash) as URLPath
+ * See also https://twitter.com/kettanaito/status/1741768992866308120
+ * Note: this function also resolves relative pathnames while parsing!
+ */
+export function parseURLPath(urlPath: string, fromPath?: string): URLPath {
+  const base = fromPath ? parseURLOrPath(fromPath) : undefined;
+  const url = parseURLOrPath(urlPath, base);
+  return toURLPath(url);
+}
+
+/**
+ * This returns results for strings like "foo", "../foo", "./foo.mdx?qs#hash"
+ * Unlike "parseURLPath()" above, this will not resolve the pathnames
+ * Te returned pathname of "../../foo.mdx" will be "../../foo.mdx", not "/foo"
+ * This returns null if the url is not "local" (contains domain/protocol etc)
+ */
+export function parseLocalURLPath(urlPath: string): URLPath | null {
+  // Workaround because URL("") requires a protocol
+  const unspecifiedProtocol = 'unspecified:';
+
+  const url = parseURLOrPath(urlPath, `${unspecifiedProtocol}//`);
+  // Ignore links with specified protocol / host
+  // (usually fully qualified links starting with https://)
+  if (
+    url.protocol !== unspecifiedProtocol ||
+    url.host !== '' ||
+    url.username !== '' ||
+    url.password !== ''
+  ) {
+    return null;
+  }
+
+  // We can't use "new URL()" result because it always tries to resolve urls
+  // IE it will remove any "./" or "../" in the pathname, which we don't want
+  // We have to parse it manually...
+  let localUrlPath = urlPath;
+
+  // Extract and remove the #hash part
+  const hashIndex = localUrlPath.indexOf('#');
+  const hash =
+    hashIndex !== -1 ? localUrlPath.substring(hashIndex + 1) : undefined;
+  localUrlPath =
+    hashIndex !== -1 ? localUrlPath.substring(0, hashIndex) : localUrlPath;
+
+  // Extract and remove ?search part
+  const searchIndex = localUrlPath.indexOf('?');
+  const search =
+    searchIndex !== -1 ? localUrlPath.substring(searchIndex + 1) : undefined;
+  localUrlPath =
+    searchIndex !== -1 ? localUrlPath.substring(0, searchIndex) : localUrlPath;
+
+  return {
+    pathname: localUrlPath,
+    search,
+    hash,
+  };
+}
+
 export function serializeURLPath(urlPath: URLPath): string {
   const search = urlPath.search === undefined ? '' : `?${urlPath.search}`;
   const hash = urlPath.hash === undefined ? '' : `#${urlPath.hash}`;