@docusaurus/utils 2.0.0-beta.1 → 2.0.0-beta.10

package/lib/tags.js

@@ -0,0 +1,72 @@
+"use strict";
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.groupTaggedItems = exports.normalizeFrontMatterTags = exports.normalizeFrontMatterTag = void 0;
+const lodash_1 = require("lodash");
+const normalizeUrl_1 = require("./normalizeUrl");
+function normalizeFrontMatterTag(tagsPath, frontMatterTag) {
+    function toTagObject(tagString) {
+        return {
+            label: tagString,
+            permalink: (0, lodash_1.kebabCase)(tagString),
+        };
+    }
+    // TODO maybe make ensure the permalink is valid url path?
+    function normalizeTagPermalink(permalink) {
+        // note: we always apply tagsPath on purpose
+        // for versioned docs, v1/doc.md and v2/doc.md tags with custom permalinks don't lead to the same created page
+        // tagsPath is different for each doc version
+        return (0, normalizeUrl_1.normalizeUrl)([tagsPath, permalink]);
+    }
+    const tag = typeof frontMatterTag === 'string'
+        ? toTagObject(frontMatterTag)
+        : frontMatterTag;
+    return {
+        label: tag.label,
+        permalink: normalizeTagPermalink(tag.permalink),
+    };
+}
+exports.normalizeFrontMatterTag = normalizeFrontMatterTag;
+function normalizeFrontMatterTags(tagsPath, frontMatterTags) {
+    var _a;
+    const tags = (_a = frontMatterTags === null || frontMatterTags === void 0 ? void 0 : frontMatterTags.map((tag) => normalizeFrontMatterTag(tagsPath, tag))) !== null && _a !== void 0 ? _a : [];
+    return (0, lodash_1.uniqBy)(tags, (tag) => tag.permalink);
+}
+exports.normalizeFrontMatterTags = normalizeFrontMatterTags;
+// Permits to group docs/blogPosts by tag (provided by FrontMatter)
+// Note: groups are indexed by permalink, because routes must be unique in the end
+// Labels may vary on 2 md files but they are normalized.
+// Docs with label='some label' and label='some-label' should end-up in the same group/page in the end
+// We can't create 2 routes /some-label because one would override the other
+function groupTaggedItems(items, getItemTags) {
+    const result = {};
+    function handleItemTag(item, tag) {
+        var _a;
+        // Init missing tag groups
+        // TODO: it's not really clear what should be the behavior if 2 items have the same tag but the permalink is different for each
+        // For now, the first tag found wins
+        result[tag.permalink] = (_a = result[tag.permalink]) !== null && _a !== void 0 ? _a : {
+            tag,
+            items: [],
+        };
+        // Add item to group
+        result[tag.permalink].items.push(item);
+    }
+    items.forEach((item) => {
+        getItemTags(item).forEach((tag) => {
+            handleItemTag(item, tag);
+        });
+    });
+    // If user add twice the same tag to a md doc (weird but possible),
+    // we don't want the item to appear twice in the list...
+    Object.values(result).forEach((group) => {
+        group.items = (0, lodash_1.uniq)(group.items);
+    });
+    return result;
+}
+exports.groupTaggedItems = groupTaggedItems;

package/lib/webpackUtils.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+import type { RuleSetRule } from 'webpack';
+declare type AssetFolder = 'images' | 'files' | 'fonts' | 'medias';
+declare type FileLoaderUtils = {
+    loaders: {
+        file: (options: {
+            folder: AssetFolder;
+        }) => RuleSetRule;
+        url: (options: {
+            folder: AssetFolder;
+        }) => RuleSetRule;
+        inlineMarkdownImageFileLoader: string;
+        inlineMarkdownLinkFileLoader: string;
+    };
+    rules: {
+        images: () => RuleSetRule;
+        fonts: () => RuleSetRule;
+        media: () => RuleSetRule;
+        svg: () => RuleSetRule;
+        otherAssets: () => RuleSetRule;
+    };
+};
+export declare function getFileLoaderUtils(): FileLoaderUtils;
+export {};

package/lib/webpackUtils.js

@@ -0,0 +1,109 @@
+"use strict";
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getFileLoaderUtils = void 0;
+const tslib_1 = require("tslib");
+const path_1 = (0, tslib_1.__importDefault)(require("path"));
+const posixPath_1 = require("./posixPath");
+const constants_1 = require("./constants");
+// Inspired by https://github.com/gatsbyjs/gatsby/blob/8e6e021014da310b9cc7d02e58c9b3efe938c665/packages/gatsby/src/utils/webpack-utils.ts#L447
+function getFileLoaderUtils() {
+    // files/images < urlLoaderLimit will be inlined as base64 strings directly in the html
+    const urlLoaderLimit = constants_1.WEBPACK_URL_LOADER_LIMIT;
+    // defines the path/pattern of the assets handled by webpack
+    const fileLoaderFileName = (folder) => `${constants_1.OUTPUT_STATIC_ASSETS_DIR_NAME}/${folder}/[name]-[hash].[ext]`;
+    const loaders = {
+        file: (options) => ({
+            loader: require.resolve(`file-loader`),
+            options: {
+                name: fileLoaderFileName(options.folder),
+            },
+        }),
+        url: (options) => ({
+            loader: require.resolve('url-loader'),
+            options: {
+                limit: urlLoaderLimit,
+                name: fileLoaderFileName(options.folder),
+                fallback: require.resolve('file-loader'),
+            },
+        }),
+        // TODO find a better solution to avoid conflicts with the ideal-image plugin
+        // TODO this may require a little breaking change for ideal-image users?
+        // Maybe with the ideal image plugin, all md images should be "ideal"?
+        // This is used to force url-loader+file-loader on markdown images
+        // https://webpack.js.org/concepts/loaders/#inline
+        inlineMarkdownImageFileLoader: `!${(0, posixPath_1.posixPath)(require.resolve('url-loader'))}?limit=${urlLoaderLimit}&name=${fileLoaderFileName('images')}&fallback=${(0, posixPath_1.posixPath)(require.resolve('file-loader'))}!`,
+        inlineMarkdownLinkFileLoader: `!${(0, posixPath_1.posixPath)(require.resolve('file-loader'))}?name=${fileLoaderFileName('files')}!`,
+    };
+    const rules = {
+        /**
+         * Loads image assets, inlines images via a data URI if they are below
+         * the size threshold
+         */
+        images: () => ({
+            use: [loaders.url({ folder: 'images' })],
+            test: /\.(ico|jpg|jpeg|png|gif|webp)(\?.*)?$/,
+        }),
+        fonts: () => ({
+            use: [loaders.url({ folder: 'fonts' })],
+            test: /\.(woff|woff2|eot|ttf|otf)$/,
+        }),
+        /**
+         * Loads audio and video and inlines them via a data URI if they are below
+         * the size threshold
+         */
+        media: () => ({
+            use: [loaders.url({ folder: 'medias' })],
+            test: /\.(mp4|webm|ogv|wav|mp3|m4a|aac|oga|flac)$/,
+        }),
+        svg: () => ({
+            test: /\.svg?$/,
+            oneOf: [
+                {
+                    use: [
+                        {
+                            loader: require.resolve('@svgr/webpack'),
+                            options: {
+                                prettier: false,
+                                svgo: true,
+                                svgoConfig: {
+                                    plugins: [
+                                        {
+                                            name: 'preset-default',
+                                            params: {
+                                                overrides: {
+                                                    removeViewBox: false,
+                                                },
+                                            },
+                                        },
+                                    ],
+                                },
+                                titleProp: true,
+                                ref: ![path_1.default],
+                            },
+                        },
+                    ],
+                    // We don't want to use SVGR loader for non-React source code
+                    // ie we don't want to use SVGR for CSS files...
+                    issuer: {
+                        and: [/\.(ts|tsx|js|jsx|md|mdx)$/],
+                    },
+                },
+                {
+                    use: [loaders.url({ folder: 'images' })],
+                },
+            ],
+        }),
+        otherAssets: () => ({
+            use: [loaders.file({ folder: 'files' })],
+            test: /\.(pdf|doc|docx|xls|xlsx|zip|rar)$/,
+        }),
+    };
+    return { loaders, rules };
+}
+exports.getFileLoaderUtils = getFileLoaderUtils;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@docusaurus/utils",
-  "version": "2.0.0-beta.1",
+  "version": "2.0.0-beta.10",
   "description": "Node utility functions for Docusaurus packages.",
   "main": "./lib/index.js",
   "types": "./lib/index.d.ts",
@@ -18,22 +18,38 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@docusaurus/types": "2.0.0-beta.1",
-    "@types/github-slugger": "^1.3.0",
-    "chalk": "^4.1.1",
+    "@mdx-js/runtime": "^1.6.22",
+    "@svgr/webpack": "^6.0.0",
+    "chalk": "^4.1.2",
     "escape-string-regexp": "^4.0.0",
+    "file-loader": "^6.2.0",
     "fs-extra": "^10.0.0",
+    "github-slugger": "^1.4.0",
+    "globby": "^11.0.4",
     "gray-matter": "^4.0.3",
     "lodash": "^4.17.20",
+    "micromatch": "^4.0.4",
+    "remark-mdx-remove-exports": "^1.6.22",
+    "remark-mdx-remove-imports": "^1.6.22",
     "resolve-pathname": "^3.0.0",
-    "tslib": "^2.2.0"
+    "tslib": "^2.3.1",
+    "url-loader": "^4.1.1"
   },
   "engines": {
-    "node": ">=12.13.0"
+    "node": ">=14"
   },
   "devDependencies": {
+    "@docusaurus/types": "2.0.0-beta.10",
     "@types/dedent": "^0.7.0",
+    "@types/github-slugger": "^1.3.0",
+    "@types/micromatch": "^4.0.2",
+    "@types/react-dom": "^17.0.1",
     "dedent": "^0.7.0"
   },
-  "gitHead": "72ed3476bf4ffe1fc2d1025dd2def536a2de8f12"
+  "peerDependencies": {
+    "react": "*",
+    "react-dom": "*",
+    "webpack": "5.x"
+  },
+  "gitHead": "6f4869af5721435d6995e63e2f24ac41646e34ea"
 }

package/src/__tests__/globUtils.test.ts

@@ -0,0 +1,109 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {
+  GlobExcludeDefault,
+  createMatcher,
+  createAbsoluteFilePathMatcher,
+} from '../globUtils';
+
+describe('createMatcher', () => {
+  const matcher = createMatcher(GlobExcludeDefault);
+
+  test('match default exclude MD/MDX partials correctly', () => {
+    expect(matcher('doc.md')).toEqual(false);
+    expect(matcher('category/doc.md')).toEqual(false);
+    expect(matcher('category/subcategory/doc.md')).toEqual(false);
+    //
+    expect(matcher('doc.mdx')).toEqual(false);
+    expect(matcher('category/doc.mdx')).toEqual(false);
+    expect(matcher('category/subcategory/doc.mdx')).toEqual(false);
+    //
+    expect(matcher('_doc.md')).toEqual(true);
+    expect(matcher('category/_doc.md')).toEqual(true);
+    expect(matcher('category/subcategory/_doc.md')).toEqual(true);
+    expect(matcher('_category/doc.md')).toEqual(true);
+    expect(matcher('_category/subcategory/doc.md')).toEqual(true);
+    expect(matcher('category/_subcategory/doc.md')).toEqual(true);
+  });
+
+  test('match default exclude tests correctly', () => {
+    expect(matcher('xyz.js')).toEqual(false);
+    expect(matcher('xyz.ts')).toEqual(false);
+    expect(matcher('xyz.jsx')).toEqual(false);
+    expect(matcher('xyz.tsx')).toEqual(false);
+    expect(matcher('folder/xyz.js')).toEqual(false);
+    expect(matcher('folder/xyz.ts')).toEqual(false);
+    expect(matcher('folder/xyz.jsx')).toEqual(false);
+    expect(matcher('folder/xyz.tsx')).toEqual(false);
+    //
+    expect(matcher('xyz.test.js')).toEqual(true);
+    expect(matcher('xyz.test.ts')).toEqual(true);
+    expect(matcher('xyz.test.jsx')).toEqual(true);
+    expect(matcher('xyz.test.tsx')).toEqual(true);
+    expect(matcher('folder/xyz.test.js')).toEqual(true);
+    expect(matcher('folder/xyz.test.ts')).toEqual(true);
+    expect(matcher('folder/xyz.test.jsx')).toEqual(true);
+    expect(matcher('folder/xyz.test.tsx')).toEqual(true);
+    expect(matcher('folder/subfolder/xyz.test.js')).toEqual(true);
+    expect(matcher('folder/subfolder/xyz.test.ts')).toEqual(true);
+    expect(matcher('folder/subfolder/xyz.test.jsx')).toEqual(true);
+    expect(matcher('folder/subfolder/xyz.test.tsx')).toEqual(true);
+    //
+    expect(matcher('__tests__/subfolder/xyz.js')).toEqual(true);
+    expect(matcher('__tests__/subfolder/xyz.ts')).toEqual(true);
+    expect(matcher('__tests__/subfolder/xyz.jsx')).toEqual(true);
+    expect(matcher('__tests__/subfolder/xyz.tsx')).toEqual(true);
+    expect(matcher('folder/__tests__/xyz.js')).toEqual(true);
+    expect(matcher('folder/__tests__/xyz.ts')).toEqual(true);
+    expect(matcher('folder/__tests__/xyz.jsx')).toEqual(true);
+    expect(matcher('folder/__tests__/xyz.tsx')).toEqual(true);
+  });
+});
+
+describe('createAbsoluteFilePathMatcher', () => {
+  const rootFolders = ['/_root/docs', '/root/_docs/', '/__test__/website/src'];
+
+  const matcher = createAbsoluteFilePathMatcher(
+    GlobExcludeDefault,
+    rootFolders,
+  );
+
+  test('match default exclude MD/MDX partials correctly', () => {
+    expect(matcher('/_root/docs/myDoc.md')).toEqual(false);
+    expect(matcher('/_root/docs/myDoc.mdx')).toEqual(false);
+    expect(matcher('/root/_docs/myDoc.md')).toEqual(false);
+    expect(matcher('/root/_docs/myDoc.mdx')).toEqual(false);
+    expect(matcher('/_root/docs/category/myDoc.md')).toEqual(false);
+    expect(matcher('/_root/docs/category/myDoc.mdx')).toEqual(false);
+    expect(matcher('/root/_docs/category/myDoc.md')).toEqual(false);
+    expect(matcher('/root/_docs/category/myDoc.mdx')).toEqual(false);
+    //
+    expect(matcher('/_root/docs/_myDoc.md')).toEqual(true);
+    expect(matcher('/_root/docs/_myDoc.mdx')).toEqual(true);
+    expect(matcher('/root/_docs/_myDoc.md')).toEqual(true);
+    expect(matcher('/root/_docs/_myDoc.mdx')).toEqual(true);
+    expect(matcher('/_root/docs/_category/myDoc.md')).toEqual(true);
+    expect(matcher('/_root/docs/_category/myDoc.mdx')).toEqual(true);
+    expect(matcher('/root/_docs/_category/myDoc.md')).toEqual(true);
+    expect(matcher('/root/_docs/_category/myDoc.mdx')).toEqual(true);
+  });
+
+  test('match default exclude tests correctly', () => {
+    expect(matcher('/__test__/website/src/xyz.js')).toEqual(false);
+    expect(matcher('/__test__/website/src/__test__/xyz.js')).toEqual(true);
+    expect(matcher('/__test__/website/src/xyz.test.js')).toEqual(true);
+  });
+
+  test('throw if file is not contained in any root doc', () => {
+    expect(() =>
+      matcher('/bad/path/myDoc.md'),
+    ).toThrowErrorMatchingInlineSnapshot(
+      `"createAbsoluteFilePathMatcher unexpected error, absoluteFilePath=/bad/path/myDoc.md was not contained in any of the root folders [\\"/_root/docs\\",\\"/root/_docs/\\",\\"/__test__/website/src\\"]"`,
+    );
+  });
+});

package/src/__tests__/{docuHash.test.ts → hashUtils.test.ts}

@@ -5,7 +5,28 @@
  * LICENSE file in the root directory of this source tree.
  */
 
-import {docuHash} from '../docuHash';
+import {simpleHash, docuHash} from '../hashUtils';
+
+describe('hashUtils', () => {
+  test('simpleHash', () => {
+    const asserts: Record<string, string> = {
+      '': 'd41',
+      '/foo-bar': '096',
+      '/foo/bar': '1df',
+      '/endi/lie': '9fa',
+      '/endi-lie': 'fd3',
+      '/yangshun/tay': '48d',
+      '/yangshun-tay': 'f3b',
+      '/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar':
+        'd46',
+      '/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/foo/bar/test1-test2':
+        '787',
+    };
+    Object.keys(asserts).forEach((str) => {
+      expect(simpleHash(str, 3)).toBe(asserts[str]);
+    });
+  });
+});
 
 describe('docuHash', () => {
   test('docuHash works', () => {

package/src/__tests__/index.test.ts

@@ -12,7 +12,6 @@ import {
   genChunkName,
   idx,
   getSubFolder,
-  normalizeUrl,
   posixPath,
   objectWithKeySorted,
   aliasedSitePath,
@@ -218,113 +217,6 @@ describe('load utils', () => {
     expect(getSubFolder(testE, 'docs')).toBeNull();
   });
 
-  test('normalizeUrl', () => {
-    const asserts = [
-      {
-        input: ['/', ''],
-        output: '/',
-      },
-      {
-        input: ['', '/'],
-        output: '/',
-      },
-      {
-        input: ['/'],
-        output: '/',
-      },
-      {
-        input: [''],
-        output: '',
-      },
-      {
-        input: ['/', '/'],
-        output: '/',
-      },
-      {
-        input: ['/', 'docs'],
-        output: '/docs',
-      },
-      {
-        input: ['/', 'docs', 'en', 'next', 'blog'],
-        output: '/docs/en/next/blog',
-      },
-      {
-        input: ['/test/', '/docs', 'ro', 'doc1'],
-        output: '/test/docs/ro/doc1',
-      },
-      {
-        input: ['/test/', '/', 'ro', 'doc1'],
-        output: '/test/ro/doc1',
-      },
-      {
-        input: ['/', '/', '2020/02/29/leap-day'],
-        output: '/2020/02/29/leap-day',
-      },
-      {
-        input: ['', '/', 'ko', 'hello'],
-        output: '/ko/hello',
-      },
-      {
-        input: ['hello', 'world'],
-        output: 'hello/world',
-      },
-      {
-        input: ['http://www.google.com/', 'foo/bar', '?test=123'],
-        output: 'http://www.google.com/foo/bar?test=123',
-      },
-      {
-        input: ['http:', 'www.google.com///', 'foo/bar', '?test=123'],
-        output: 'http://www.google.com/foo/bar?test=123',
-      },
-      {
-        input: ['http://foobar.com', '', 'test'],
-        output: 'http://foobar.com/test',
-      },
-      {
-        input: ['http://foobar.com', '', 'test', '/'],
-        output: 'http://foobar.com/test/',
-      },
-      {
-        input: ['/', '', 'hello', '', '/', '/', '', '/', '/world'],
-        output: '/hello/world',
-      },
-      {
-        input: ['', '', '/tt', 'ko', 'hello'],
-        output: '/tt/ko/hello',
-      },
-      {
-        input: ['', '///hello///', '', '///world'],
-        output: '/hello/world',
-      },
-      {
-        input: ['', '/hello/', ''],
-        output: '/hello/',
-      },
-      {
-        input: ['', '/', ''],
-        output: '/',
-      },
-      {
-        input: ['///', '///'],
-        output: '/',
-      },
-      {
-        input: ['/', '/hello/world/', '///'],
-        output: '/hello/world/',
-      },
-    ];
-    asserts.forEach((testCase) => {
-      expect(normalizeUrl(testCase.input)).toBe(testCase.output);
-    });
-
-    expect(() =>
-      // @ts-expect-error undefined for test
-      normalizeUrl(['http:example.com', undefined]),
-    ).toThrowErrorMatchingInlineSnapshot(
-      `"Url must be a string. Received undefined"`,
-    );
-  });
-
   test('isValidPathname', () => {
     expect(isValidPathname('/')).toBe(true);
     expect(isValidPathname('/hey')).toBe(true);
@@ -459,7 +351,9 @@ describe('mergeTranslations', () => {
 
 describe('mapAsyncSequencial', () => {
   function sleep(timeout: number): Promise<void> {
-    return new Promise((resolve) => setTimeout(resolve, timeout));
+    return new Promise((resolve) => {
+      setTimeout(resolve, timeout);
+    });
   }
 
   test('map sequentially', async () => {
@@ -498,7 +392,9 @@ describe('mapAsyncSequencial', () => {
 
 describe('findAsyncSequencial', () => {
   function sleep(timeout: number): Promise<void> {
-    return new Promise((resolve) => setTimeout(resolve, timeout));
+    return new Promise((resolve) => {
+      setTimeout(resolve, timeout);
+    });
   }
 
   test('find sequentially', async () => {

package/src/__tests__/markdownParser.test.ts

@@ -36,7 +36,7 @@ describe('createExcerpt', () => {
           Nunc porttitor libero nec vulputate venenatis. Nam nec rhoncus mauris. Morbi tempus est et nibh maximus, tempus venenatis arcu lobortis.
         `),
     ).toEqual(
-      // h1 title is skipped on purpose, because we don't want the page to have SEO metadatas title === description
+      // h1 title is skipped on purpose, because we don't want the page to have SEO metadata title === description
       'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vestibulum ex urna, molestie et sagittis ut, varius ac justo.',
     );
   });
@@ -53,7 +53,7 @@ describe('createExcerpt', () => {
           Nunc porttitor libero nec vulputate venenatis. Nam nec rhoncus mauris. Morbi tempus est et nibh maximus, tempus venenatis arcu lobortis.
         `),
     ).toEqual(
-      // h1 title is skipped on purpose, because we don't want the page to have SEO metadatas title === description
+      // h1 title is skipped on purpose, because we don't want the page to have SEO metadata title === description
       'Lorem ipsum dolor sit amet, consectetur adipiscing elit. Vestibulum ex urna, molestie et sagittis ut, varius ac justo.',
     );
   });
@@ -129,6 +129,19 @@ describe('createExcerpt', () => {
         `),
     ).toEqual('Markdown title');
   });
+
+  test('should create excerpt for content with various code blocks', () => {
+    expect(
+      createExcerpt(dedent`
+          \`\`\`jsx
+          import React from 'react';
+          import Layout from '@theme/Layout';
+          \`\`\`
+
+          Lorem \`ipsum\` dolor sit amet, consectetur \`adipiscing elit\`.
+        `),
+    ).toEqual('Lorem ipsum dolor sit amet, consectetur adipiscing elit.');
+  });
 });
 
 describe('parseMarkdownContentTitle', () => {