@docusaurus/plugin-sitemap 2.0.0-beta.18 → 2.0.0-beta.19

package/lib/createSitemap.d.ts

@@ -4,6 +4,9 @@
  * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  */
-import type { Options } from '@docusaurus/plugin-sitemap';
 import type { DocusaurusConfig } from '@docusaurus/types';
-export default function createSitemap(siteConfig: DocusaurusConfig, routesPaths: string[], options: Options): Promise<string>;
+import type { HelmetServerState } from 'react-helmet-async';
+import type { PluginOptions } from './options';
+export default function createSitemap(siteConfig: DocusaurusConfig, routesPaths: string[], head: {
+    [location: string]: HelmetServerState;
+}, options: PluginOptions): Promise<string>;

package/lib/createSitemap.js

@@ -8,16 +8,24 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 const sitemap_1 = require("sitemap");
 const utils_common_1 = require("@docusaurus/utils-common");
-async function createSitemap(siteConfig, routesPaths, options) {
+const utils_1 = require("@docusaurus/utils");
+async function createSitemap(siteConfig, routesPaths, head, options) {
     const { url: hostname } = siteConfig;
     if (!hostname) {
         throw new Error('URL in docusaurus.config.js cannot be empty/undefined.');
     }
-    const { changefreq, priority } = options;
+    const { changefreq, priority, ignorePatterns } = options;
+    const ignoreMatcher = (0, utils_1.createMatcher)(ignorePatterns);
     const sitemapStream = new sitemap_1.SitemapStream({ hostname });
-    routesPaths
-        .filter((route) => !route.endsWith('404.html'))
-        .forEach((routePath) => sitemapStream.write({
+    function routeShouldBeIncluded(route) {
+        if (route.endsWith('404.html') || ignoreMatcher(route)) {
+            return false;
+        }
+        // https://github.com/staylor/react-helmet-async/pull/167
+        const meta = head[route]?.meta.toComponent();
+        return !meta?.some((tag) => tag.props.name === 'robots' && tag.props.content === 'noindex');
+    }
+    routesPaths.filter(routeShouldBeIncluded).forEach((routePath) => sitemapStream.write({
         url: (0, utils_common_1.applyTrailingSlash)(routePath, {
             trailingSlash: siteConfig.trailingSlash,
             baseUrl: siteConfig.baseUrl,

package/lib/index.d.ts

@@ -4,7 +4,8 @@
  * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  */
-import type { Options } from '@docusaurus/plugin-sitemap';
+import type { PluginOptions, Options } from './options';
 import type { LoadContext, Plugin } from '@docusaurus/types';
-export default function pluginSitemap(context: LoadContext, options: Options): Plugin<void>;
+export default function pluginSitemap(context: LoadContext, options: PluginOptions): Plugin<void>;
 export { validateOptions } from './options';
+export type { PluginOptions, Options };

package/lib/index.js

@@ -14,12 +14,12 @@ const createSitemap_1 = tslib_1.__importDefault(require("./createSitemap"));
 function pluginSitemap(context, options) {
     return {
         name: 'docusaurus-plugin-sitemap',
-        async postBuild({ siteConfig, routesPaths, outDir }) {
+        async postBuild({ siteConfig, routesPaths, outDir, head }) {
             if (siteConfig.noIndex) {
                 return;
             }
             // Generate sitemap.
-            const generatedSitemap = await (0, createSitemap_1.default)(siteConfig, routesPaths, options);
+            const generatedSitemap = await (0, createSitemap_1.default)(siteConfig, routesPaths, head, options);
             // Write sitemap file.
             const sitemapPath = path_1.default.join(outDir, 'sitemap.xml');
             try {

package/lib/options.d.ts

@@ -4,7 +4,19 @@
  * This source code is licensed under the MIT license found in the
  * LICENSE file in the root directory of this source tree.
  */
-import type { Options } from '@docusaurus/plugin-sitemap';
+import { EnumChangefreq } from 'sitemap';
 import type { OptionValidationContext } from '@docusaurus/types';
-export declare const DEFAULT_OPTIONS: Options;
-export declare function validateOptions({ validate, options, }: OptionValidationContext<Options, Options>): Options;
+export declare type PluginOptions = {
+    /** @see https://www.sitemaps.org/protocol.html#xmlTagDefinitions */
+    changefreq: EnumChangefreq;
+    /** @see https://www.sitemaps.org/protocol.html#xmlTagDefinitions */
+    priority: number;
+    /**
+     * A list of glob patterns; matching route paths will be filtered from the
+     * sitemap. Note that you may need to include the base URL in here.
+     */
+    ignorePatterns: string[];
+};
+export declare type Options = Partial<PluginOptions>;
+export declare const DEFAULT_OPTIONS: PluginOptions;
+export declare function validateOptions({ validate, options, }: OptionValidationContext<Options, PluginOptions>): PluginOptions;

package/lib/options.js

@@ -12,6 +12,7 @@ const sitemap_1 = require("sitemap");
 exports.DEFAULT_OPTIONS = {
     changefreq: sitemap_1.EnumChangefreq.WEEKLY,
     priority: 0.5,
+    ignorePatterns: [],
 };
 const PluginOptionSchema = utils_validation_1.Joi.object({
     cacheTime: utils_validation_1.Joi.forbidden().messages({
@@ -21,6 +22,9 @@ const PluginOptionSchema = utils_validation_1.Joi.object({
         .valid(...Object.values(sitemap_1.EnumChangefreq))
         .default(exports.DEFAULT_OPTIONS.changefreq),
     priority: utils_validation_1.Joi.number().min(0).max(1).default(exports.DEFAULT_OPTIONS.priority),
+    ignorePatterns: utils_validation_1.Joi.array()
+        .items(utils_validation_1.Joi.string())
+        .default(exports.DEFAULT_OPTIONS.ignorePatterns),
     trailingSlash: utils_validation_1.Joi.forbidden().messages({
         'any.unknown': 'Please use the new Docusaurus global trailingSlash config instead, and the sitemaps plugin will use it.',
     }),

package/package.json

@@ -1,9 +1,9 @@
 {
   "name": "@docusaurus/plugin-sitemap",
-  "version": "2.0.0-beta.18",
+  "version": "2.0.0-beta.19",
   "description": "Simple sitemap generation plugin for Docusaurus.",
   "main": "lib/index.js",
-  "types": "src/plugin-sitemap.d.ts",
+  "types": "lib/index.d.ts",
   "scripts": {
     "build": "tsc",
     "watch": "tsc --watch"
@@ -18,16 +18,16 @@
   },
   "license": "MIT",
   "dependencies": {
-    "@docusaurus/core": "2.0.0-beta.18",
-    "@docusaurus/utils": "2.0.0-beta.18",
-    "@docusaurus/utils-common": "2.0.0-beta.18",
-    "@docusaurus/utils-validation": "2.0.0-beta.18",
-    "fs-extra": "^10.0.1",
+    "@docusaurus/core": "2.0.0-beta.19",
+    "@docusaurus/utils": "2.0.0-beta.19",
+    "@docusaurus/utils-common": "2.0.0-beta.19",
+    "@docusaurus/utils-validation": "2.0.0-beta.19",
+    "fs-extra": "^10.1.0",
     "sitemap": "^7.1.1",
-    "tslib": "^2.3.1"
+    "tslib": "^2.4.0"
   },
   "devDependencies": {
-    "@docusaurus/types": "2.0.0-beta.18"
+    "@docusaurus/types": "2.0.0-beta.19"
   },
   "peerDependencies": {
     "react": "^16.8.4 || ^17.0.0",
@@ -36,5 +36,5 @@
   "engines": {
     "node": ">=14"
   },
-  "gitHead": "1a945d06993d53376e61bed2c942799fe07dc336"
+  "gitHead": "a71e60a49cce93c1006ef10c41ac03187f057102"
 }

package/src/createSitemap.ts

@@ -6,35 +6,52 @@
  */
 
 import {SitemapStream, streamToPromise} from 'sitemap';
-import type {Options} from '@docusaurus/plugin-sitemap';
-import type {DocusaurusConfig} from '@docusaurus/types';
 import {applyTrailingSlash} from '@docusaurus/utils-common';
+import {createMatcher} from '@docusaurus/utils';
+import type {DocusaurusConfig} from '@docusaurus/types';
+import type {HelmetServerState} from 'react-helmet-async';
+import type {PluginOptions} from './options';
+import type {ReactElement} from 'react';
 
 export default async function createSitemap(
   siteConfig: DocusaurusConfig,
   routesPaths: string[],
-  options: Options,
+  head: {[location: string]: HelmetServerState},
+  options: PluginOptions,
 ): Promise<string> {
   const {url: hostname} = siteConfig;
   if (!hostname) {
     throw new Error('URL in docusaurus.config.js cannot be empty/undefined.');
   }
-  const {changefreq, priority} = options;
+  const {changefreq, priority, ignorePatterns} = options;
+
+  const ignoreMatcher = createMatcher(ignorePatterns);
 
   const sitemapStream = new SitemapStream({hostname});
 
-  routesPaths
-    .filter((route) => !route.endsWith('404.html'))
-    .forEach((routePath) =>
-      sitemapStream.write({
-        url: applyTrailingSlash(routePath, {
-          trailingSlash: siteConfig.trailingSlash,
-          baseUrl: siteConfig.baseUrl,
-        }),
-        changefreq,
-        priority,
-      }),
+  function routeShouldBeIncluded(route: string) {
+    if (route.endsWith('404.html') || ignoreMatcher(route)) {
+      return false;
+    }
+    // https://github.com/staylor/react-helmet-async/pull/167
+    const meta = head[route]?.meta.toComponent() as unknown as
+      | ReactElement[]
+      | undefined;
+    return !meta?.some(
+      (tag) => tag.props.name === 'robots' && tag.props.content === 'noindex',
     );
+  }
+
+  routesPaths.filter(routeShouldBeIncluded).forEach((routePath) =>
+    sitemapStream.write({
+      url: applyTrailingSlash(routePath, {
+        trailingSlash: siteConfig.trailingSlash,
+        baseUrl: siteConfig.baseUrl,
+      }),
+      changefreq,
+      priority,
+    }),
+  );
 
   sitemapStream.end();
 

package/src/index.ts

@@ -7,18 +7,18 @@
 
 import fs from 'fs-extra';
 import path from 'path';
-import type {Options} from '@docusaurus/plugin-sitemap';
 import createSitemap from './createSitemap';
+import type {PluginOptions, Options} from './options';
 import type {LoadContext, Plugin} from '@docusaurus/types';
 
 export default function pluginSitemap(
   context: LoadContext,
-  options: Options,
+  options: PluginOptions,
 ): Plugin<void> {
   return {
     name: 'docusaurus-plugin-sitemap',
 
-    async postBuild({siteConfig, routesPaths, outDir}) {
+    async postBuild({siteConfig, routesPaths, outDir, head}) {
       if (siteConfig.noIndex) {
         return;
       }
@@ -26,6 +26,7 @@ export default function pluginSitemap(
       const generatedSitemap = await createSitemap(
         siteConfig,
         routesPaths,
+        head,
         options,
       );
 
@@ -41,3 +42,4 @@ export default function pluginSitemap(
 }
 
 export {validateOptions} from './options';
+export type {PluginOptions, Options};

package/src/options.ts

@@ -7,12 +7,26 @@
 
 import {Joi} from '@docusaurus/utils-validation';
 import {EnumChangefreq} from 'sitemap';
-import type {Options} from '@docusaurus/plugin-sitemap';
 import type {OptionValidationContext} from '@docusaurus/types';
 
-export const DEFAULT_OPTIONS: Options = {
+export type PluginOptions = {
+  /** @see https://www.sitemaps.org/protocol.html#xmlTagDefinitions */
+  changefreq: EnumChangefreq;
+  /** @see https://www.sitemaps.org/protocol.html#xmlTagDefinitions */
+  priority: number;
+  /**
+   * A list of glob patterns; matching route paths will be filtered from the
+   * sitemap. Note that you may need to include the base URL in here.
+   */
+  ignorePatterns: string[];
+};
+
+export type Options = Partial<PluginOptions>;
+
+export const DEFAULT_OPTIONS: PluginOptions = {
   changefreq: EnumChangefreq.WEEKLY,
   priority: 0.5,
+  ignorePatterns: [],
 };
 
 const PluginOptionSchema = Joi.object({
@@ -24,6 +38,9 @@ const PluginOptionSchema = Joi.object({
     .valid(...Object.values(EnumChangefreq))
     .default(DEFAULT_OPTIONS.changefreq),
   priority: Joi.number().min(0).max(1).default(DEFAULT_OPTIONS.priority),
+  ignorePatterns: Joi.array()
+    .items(Joi.string())
+    .default(DEFAULT_OPTIONS.ignorePatterns),
   trailingSlash: Joi.forbidden().messages({
     'any.unknown':
       'Please use the new Docusaurus global trailingSlash config instead, and the sitemaps plugin will use it.',
@@ -33,7 +50,7 @@ const PluginOptionSchema = Joi.object({
 export function validateOptions({
   validate,
   options,
-}: OptionValidationContext<Options, Options>): Options {
+}: OptionValidationContext<Options, PluginOptions>): PluginOptions {
   const validatedOptions = validate(PluginOptionSchema, options);
   return validatedOptions;
 }

package/src/plugin-sitemap.d.ts

@@ -1,16 +0,0 @@
-/**
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-import type {EnumChangefreq} from 'sitemap';
-
-export type Options = {
-  id?: string;
-  /** @see https://www.sitemaps.org/protocol.html#xmlTagDefinitions */
-  changefreq?: EnumChangefreq;
-  /** @see https://www.sitemaps.org/protocol.html#xmlTagDefinitions */
-  priority?: number;
-};