@docusaurus/plugin-sitemap 2.0.0-beta.16 → 2.0.0-beta.19

package/lib/createSitemap.d.ts

@@ -4,6 +4,9 @@
  * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  */
-import type { Options } from '@docusaurus/plugin-sitemap';
 import type { DocusaurusConfig } from '@docusaurus/types';
-export default function createSitemap(siteConfig: DocusaurusConfig, routesPaths: string[], options: Options): Promise<string>;
+import type { HelmetServerState } from 'react-helmet-async';
+import type { PluginOptions } from './options';
+export default function createSitemap(siteConfig: DocusaurusConfig, routesPaths: string[], head: {
+    [location: string]: HelmetServerState;
+}, options: PluginOptions): Promise<string>;

package/lib/createSitemap.js

@@ -8,16 +8,24 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 const sitemap_1 = require("sitemap");
 const utils_common_1 = require("@docusaurus/utils-common");
-async function createSitemap(siteConfig, routesPaths, options) {
+const utils_1 = require("@docusaurus/utils");
+async function createSitemap(siteConfig, routesPaths, head, options) {
     const { url: hostname } = siteConfig;
     if (!hostname) {
         throw new Error('URL in docusaurus.config.js cannot be empty/undefined.');
     }
-    const { changefreq, priority } = options;
+    const { changefreq, priority, ignorePatterns } = options;
+    const ignoreMatcher = (0, utils_1.createMatcher)(ignorePatterns);
     const sitemapStream = new sitemap_1.SitemapStream({ hostname });
-    routesPaths
-        .filter((route) => !route.endsWith('404.html'))
-        .forEach((routePath) => sitemapStream.write({
+    function routeShouldBeIncluded(route) {
+        if (route.endsWith('404.html') || ignoreMatcher(route)) {
+            return false;
+        }
+        // https://github.com/staylor/react-helmet-async/pull/167
+        const meta = head[route]?.meta.toComponent();
+        return !meta?.some((tag) => tag.props.name === 'robots' && tag.props.content === 'noindex');
+    }
+    routesPaths.filter(routeShouldBeIncluded).forEach((routePath) => sitemapStream.write({
         url: (0, utils_common_1.applyTrailingSlash)(routePath, {
             trailingSlash: siteConfig.trailingSlash,
             baseUrl: siteConfig.baseUrl,

package/lib/index.d.ts

@@ -4,7 +4,8 @@
  * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  */
-import type { Options } from '@docusaurus/plugin-sitemap';
-import type { LoadContext, OptionValidationContext, ValidationResult, Plugin } from '@docusaurus/types';
-export default function pluginSitemap(_context: LoadContext, options: Options): Plugin<void>;
-export declare function validateOptions({ validate, options, }: OptionValidationContext<Options>): ValidationResult<Options>;
+import type { PluginOptions, Options } from './options';
+import type { LoadContext, Plugin } from '@docusaurus/types';
+export default function pluginSitemap(context: LoadContext, options: PluginOptions): Plugin<void>;
+export { validateOptions } from './options';
+export type { PluginOptions, Options };

package/lib/index.js

@@ -8,19 +8,18 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.validateOptions = void 0;
 const tslib_1 = require("tslib");
-const fs_extra_1 = (0, tslib_1.__importDefault)(require("fs-extra"));
-const path_1 = (0, tslib_1.__importDefault)(require("path"));
-const createSitemap_1 = (0, tslib_1.__importDefault)(require("./createSitemap"));
-const pluginOptionSchema_1 = require("./pluginOptionSchema");
-function pluginSitemap(_context, options) {
+const fs_extra_1 = tslib_1.__importDefault(require("fs-extra"));
+const path_1 = tslib_1.__importDefault(require("path"));
+const createSitemap_1 = tslib_1.__importDefault(require("./createSitemap"));
+function pluginSitemap(context, options) {
     return {
         name: 'docusaurus-plugin-sitemap',
-        async postBuild({ siteConfig, routesPaths, outDir }) {
+        async postBuild({ siteConfig, routesPaths, outDir, head }) {
             if (siteConfig.noIndex) {
                 return;
             }
             // Generate sitemap.
-            const generatedSitemap = await (0, createSitemap_1.default)(siteConfig, routesPaths, options);
+            const generatedSitemap = await (0, createSitemap_1.default)(siteConfig, routesPaths, head, options);
             // Write sitemap file.
             const sitemapPath = path_1.default.join(outDir, 'sitemap.xml');
             try {
@@ -33,8 +32,5 @@ function pluginSitemap(_context, options) {
     };
 }
 exports.default = pluginSitemap;
-function validateOptions({ validate, options, }) {
-    const validatedOptions = validate(pluginOptionSchema_1.PluginOptionSchema, options);
-    return validatedOptions;
-}
-exports.validateOptions = validateOptions;
+var options_1 = require("./options");
+Object.defineProperty(exports, "validateOptions", { enumerable: true, get: function () { return options_1.validateOptions; } });

package/lib/options.d.ts

@@ -0,0 +1,22 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+import { EnumChangefreq } from 'sitemap';
+import type { OptionValidationContext } from '@docusaurus/types';
+export declare type PluginOptions = {
+    /** @see https://www.sitemaps.org/protocol.html#xmlTagDefinitions */
+    changefreq: EnumChangefreq;
+    /** @see https://www.sitemaps.org/protocol.html#xmlTagDefinitions */
+    priority: number;
+    /**
+     * A list of glob patterns; matching route paths will be filtered from the
+     * sitemap. Note that you may need to include the base URL in here.
+     */
+    ignorePatterns: string[];
+};
+export declare type Options = Partial<PluginOptions>;
+export declare const DEFAULT_OPTIONS: PluginOptions;
+export declare function validateOptions({ validate, options, }: OptionValidationContext<Options, PluginOptions>): PluginOptions;

package/lib/{pluginOptionSchema.js → options.js}

@@ -6,15 +6,15 @@
  * LICENSE file in the root directory of this source tree.
  */
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.PluginOptionSchema = exports.DEFAULT_OPTIONS = void 0;
+exports.validateOptions = exports.DEFAULT_OPTIONS = void 0;
 const utils_validation_1 = require("@docusaurus/utils-validation");
 const sitemap_1 = require("sitemap");
 exports.DEFAULT_OPTIONS = {
     changefreq: sitemap_1.EnumChangefreq.WEEKLY,
     priority: 0.5,
+    ignorePatterns: [],
 };
-exports.PluginOptionSchema = utils_validation_1.Joi.object({
-    // TODO temporary (@alpha-71)
+const PluginOptionSchema = utils_validation_1.Joi.object({
     cacheTime: utils_validation_1.Joi.forbidden().messages({
         'any.unknown': 'Option `cacheTime` in sitemap config is deprecated. Please remove it.',
     }),
@@ -22,7 +22,15 @@ exports.PluginOptionSchema = utils_validation_1.Joi.object({
         .valid(...Object.values(sitemap_1.EnumChangefreq))
         .default(exports.DEFAULT_OPTIONS.changefreq),
     priority: utils_validation_1.Joi.number().min(0).max(1).default(exports.DEFAULT_OPTIONS.priority),
+    ignorePatterns: utils_validation_1.Joi.array()
+        .items(utils_validation_1.Joi.string())
+        .default(exports.DEFAULT_OPTIONS.ignorePatterns),
     trailingSlash: utils_validation_1.Joi.forbidden().messages({
         'any.unknown': 'Please use the new Docusaurus global trailingSlash config instead, and the sitemaps plugin will use it.',
     }),
 });
+function validateOptions({ validate, options, }) {
+    const validatedOptions = validate(PluginOptionSchema, options);
+    return validatedOptions;
+}
+exports.validateOptions = validateOptions;

package/package.json

@@ -1,9 +1,9 @@
 {
   "name": "@docusaurus/plugin-sitemap",
-  "version": "2.0.0-beta.16",
+  "version": "2.0.0-beta.19",
   "description": "Simple sitemap generation plugin for Docusaurus.",
   "main": "lib/index.js",
-  "types": "src/plugin-sitemap.d.ts",
+  "types": "lib/index.d.ts",
   "scripts": {
     "build": "tsc",
     "watch": "tsc --watch"
@@ -18,16 +18,16 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@docusaurus/core": "2.0.0-beta.16",
-    "@docusaurus/utils": "2.0.0-beta.16",
-    "@docusaurus/utils-common": "2.0.0-beta.16",
-    "@docusaurus/utils-validation": "2.0.0-beta.16",
-    "fs-extra": "^10.0.1",
+    "@docusaurus/core": "2.0.0-beta.19",
+    "@docusaurus/utils": "2.0.0-beta.19",
+    "@docusaurus/utils-common": "2.0.0-beta.19",
+    "@docusaurus/utils-validation": "2.0.0-beta.19",
+    "fs-extra": "^10.1.0",
     "sitemap": "^7.1.1",
-    "tslib": "^2.3.1"
+    "tslib": "^2.4.0"
   },
   "devDependencies": {
-    "@docusaurus/types": "2.0.0-beta.16"
+    "@docusaurus/types": "2.0.0-beta.19"
   },
   "peerDependencies": {
     "react": "^16.8.4 || ^17.0.0",
@@ -36,5 +36,5 @@
   "engines": {
     "node": ">=14"
   },
-  "gitHead": "eb43c4d4f95a4fb97dc9bb9dc615413e0dc2e1e7"
+  "gitHead": "a71e60a49cce93c1006ef10c41ac03187f057102"
 }

package/src/createSitemap.ts

@@ -6,35 +6,52 @@
  */
 
 import {SitemapStream, streamToPromise} from 'sitemap';
-import type {Options} from '@docusaurus/plugin-sitemap';
-import type {DocusaurusConfig} from '@docusaurus/types';
 import {applyTrailingSlash} from '@docusaurus/utils-common';
+import {createMatcher} from '@docusaurus/utils';
+import type {DocusaurusConfig} from '@docusaurus/types';
+import type {HelmetServerState} from 'react-helmet-async';
+import type {PluginOptions} from './options';
+import type {ReactElement} from 'react';
 
 export default async function createSitemap(
   siteConfig: DocusaurusConfig,
   routesPaths: string[],
-  options: Options,
+  head: {[location: string]: HelmetServerState},
+  options: PluginOptions,
 ): Promise<string> {
   const {url: hostname} = siteConfig;
   if (!hostname) {
     throw new Error('URL in docusaurus.config.js cannot be empty/undefined.');
   }
-  const {changefreq, priority} = options;
+  const {changefreq, priority, ignorePatterns} = options;
+
+  const ignoreMatcher = createMatcher(ignorePatterns);
 
   const sitemapStream = new SitemapStream({hostname});
 
-  routesPaths
-    .filter((route) => !route.endsWith('404.html'))
-    .forEach((routePath) =>
-      sitemapStream.write({
-        url: applyTrailingSlash(routePath, {
-          trailingSlash: siteConfig.trailingSlash,
-          baseUrl: siteConfig.baseUrl,
-        }),
-        changefreq,
-        priority,
-      }),
+  function routeShouldBeIncluded(route: string) {
+    if (route.endsWith('404.html') || ignoreMatcher(route)) {
+      return false;
+    }
+    // https://github.com/staylor/react-helmet-async/pull/167
+    const meta = head[route]?.meta.toComponent() as unknown as
+      | ReactElement[]
+      | undefined;
+    return !meta?.some(
+      (tag) => tag.props.name === 'robots' && tag.props.content === 'noindex',
     );
+  }
+
+  routesPaths.filter(routeShouldBeIncluded).forEach((routePath) =>
+    sitemapStream.write({
+      url: applyTrailingSlash(routePath, {
+        trailingSlash: siteConfig.trailingSlash,
+        baseUrl: siteConfig.baseUrl,
+      }),
+      changefreq,
+      priority,
+    }),
+  );
 
   sitemapStream.end();
 

package/src/index.ts

@@ -7,25 +7,18 @@
 
 import fs from 'fs-extra';
 import path from 'path';
-import type {Options} from '@docusaurus/plugin-sitemap';
 import createSitemap from './createSitemap';
-import type {
-  LoadContext,
-  Props,
-  OptionValidationContext,
-  ValidationResult,
-  Plugin,
-} from '@docusaurus/types';
-import {PluginOptionSchema} from './pluginOptionSchema';
+import type {PluginOptions, Options} from './options';
+import type {LoadContext, Plugin} from '@docusaurus/types';
 
 export default function pluginSitemap(
-  _context: LoadContext,
-  options: Options,
+  context: LoadContext,
+  options: PluginOptions,
 ): Plugin<void> {
   return {
     name: 'docusaurus-plugin-sitemap',
 
-    async postBuild({siteConfig, routesPaths, outDir}: Props) {
+    async postBuild({siteConfig, routesPaths, outDir, head}) {
       if (siteConfig.noIndex) {
         return;
       }
@@ -33,6 +26,7 @@ export default function pluginSitemap(
       const generatedSitemap = await createSitemap(
         siteConfig,
         routesPaths,
+        head,
         options,
       );
 
@@ -47,10 +41,5 @@ export default function pluginSitemap(
   };
 }
 
-export function validateOptions({
-  validate,
-  options,
-}: OptionValidationContext<Options>): ValidationResult<Options> {
-  const validatedOptions = validate(PluginOptionSchema, options);
-  return validatedOptions;
-}
+export {validateOptions} from './options';
+export type {PluginOptions, Options};

package/src/options.ts

@@ -0,0 +1,56 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import {Joi} from '@docusaurus/utils-validation';
+import {EnumChangefreq} from 'sitemap';
+import type {OptionValidationContext} from '@docusaurus/types';
+
+export type PluginOptions = {
+  /** @see https://www.sitemaps.org/protocol.html#xmlTagDefinitions */
+  changefreq: EnumChangefreq;
+  /** @see https://www.sitemaps.org/protocol.html#xmlTagDefinitions */
+  priority: number;
+  /**
+   * A list of glob patterns; matching route paths will be filtered from the
+   * sitemap. Note that you may need to include the base URL in here.
+   */
+  ignorePatterns: string[];
+};
+
+export type Options = Partial<PluginOptions>;
+
+export const DEFAULT_OPTIONS: PluginOptions = {
+  changefreq: EnumChangefreq.WEEKLY,
+  priority: 0.5,
+  ignorePatterns: [],
+};
+
+const PluginOptionSchema = Joi.object({
+  cacheTime: Joi.forbidden().messages({
+    'any.unknown':
+      'Option `cacheTime` in sitemap config is deprecated. Please remove it.',
+  }),
+  changefreq: Joi.string()
+    .valid(...Object.values(EnumChangefreq))
+    .default(DEFAULT_OPTIONS.changefreq),
+  priority: Joi.number().min(0).max(1).default(DEFAULT_OPTIONS.priority),
+  ignorePatterns: Joi.array()
+    .items(Joi.string())
+    .default(DEFAULT_OPTIONS.ignorePatterns),
+  trailingSlash: Joi.forbidden().messages({
+    'any.unknown':
+      'Please use the new Docusaurus global trailingSlash config instead, and the sitemaps plugin will use it.',
+  }),
+});
+
+export function validateOptions({
+  validate,
+  options,
+}: OptionValidationContext<Options, PluginOptions>): PluginOptions {
+  const validatedOptions = validate(PluginOptionSchema, options);
+  return validatedOptions;
+}

package/lib/pluginOptionSchema.d.ts

@@ -1,10 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-import { Joi } from '@docusaurus/utils-validation';
-import type { Options } from '@docusaurus/plugin-sitemap';
-export declare const DEFAULT_OPTIONS: Required<Options>;
-export declare const PluginOptionSchema: Joi.ObjectSchema<any>;

package/src/plugin-sitemap.d.ts

@@ -1,13 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-import type {EnumChangefreq} from 'sitemap';
-
-export type Options = {
-  changefreq?: EnumChangefreq;
-  priority?: number;
-};

package/src/pluginOptionSchema.ts

@@ -1,31 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-import {Joi} from '@docusaurus/utils-validation';
-import {EnumChangefreq} from 'sitemap';
-import type {Options} from '@docusaurus/plugin-sitemap';
-
-export const DEFAULT_OPTIONS: Required<Options> = {
-  changefreq: EnumChangefreq.WEEKLY,
-  priority: 0.5,
-};
-
-export const PluginOptionSchema = Joi.object({
-  // TODO temporary (@alpha-71)
-  cacheTime: Joi.forbidden().messages({
-    'any.unknown':
-      'Option `cacheTime` in sitemap config is deprecated. Please remove it.',
-  }),
-  changefreq: Joi.string()
-    .valid(...Object.values(EnumChangefreq))
-    .default(DEFAULT_OPTIONS.changefreq),
-  priority: Joi.number().min(0).max(1).default(DEFAULT_OPTIONS.priority),
-  trailingSlash: Joi.forbidden().messages({
-    'any.unknown':
-      'Please use the new Docusaurus global trailingSlash config instead, and the sitemaps plugin will use it.',
-  }),
-});