ghost 6.64.0 → 6.65.0

package/core/frontend/services/rendering/format-response.js

@@ -90,7 +90,7 @@ function formatResponse(post, context, locals = {}) {
 
   prepareContextResource(post);
 
-  let entry = {
+  const entry = {
     post: post,
   };
 

package/core/frontend/services/rendering/renderer.js

@@ -20,6 +20,8 @@ module.exports = function renderer(req, res, data) {
   if (res.destroyed && !res.writableEnded) {
     debug('Client gone before render, skipping: ' + req.originalUrl);
     res.statusCode = 499;
+    // end() anyway: the request queue only frees a slot on res.end()
+    res.end();
     return;
   }
 
@@ -56,6 +58,7 @@ module.exports = function renderer(req, res, data) {
     if (res.destroyed && !res.writableEnded) {
       debug('Client gone during render, discarding: ' + req.originalUrl);
       res.statusCode = 499;
+      res.end();
       return;
     }
 

package/core/frontend/services/rendering/templates.js

@@ -118,7 +118,7 @@ templates.getEntryTemplateHierarchy = function getEntryTemplateHierarchy(postObj
 templates.pickTemplate = function pickTemplate(templateList, fallback) {
   let template;
 
-  if (!_.isArray(templateList)) {
+  if (!Array.isArray(templateList)) {
     templateList = [templateList];
   }
 

package/core/frontend/services/routing/controllers/static.js

@@ -44,7 +44,7 @@ module.exports = function staticController(req, res, next) {
 
   return Promise.all(promises)
     .then(function handleResult(result) {
-      let response = {};
+      const response = {};
 
       if (res.routerOptions.data) {
         response.data = {};

package/core/frontend/services/routing/parent-router.js

@@ -83,7 +83,7 @@ class ParentRouter {
    * @private
    */
   _respectDominantRouter(req, res, next, slug) {
-    let siteRouter = this._getSiteRouter(req);
+    const siteRouter = this._getSiteRouter(req);
     let targetRoute = null;
 
     // CASE: iterate over routers and check whether a router has a redirect for the target slug enabled.

package/core/frontend/services/routing/registry.js

@@ -71,7 +71,7 @@ class Registry {
    * @returns {ParentRouter|undefined}
    */
   getRouterByName(name) {
-    for (let routerKey in this.routers) {
+    for (const routerKey in this.routers) {
       if (this.routers[routerKey].name === name) {
         return this.routers[routerKey];
       }

package/core/frontend/services/sitemap/base-site-map-generator.js

@@ -1,27 +1,28 @@
-const _ = require('lodash');
-const xml = require('xml');
 const moment = require('moment');
-const path = require('path');
 const urlUtils = require('../../../shared/url-utils').default;
-const localUtils = require('./utils');
-
-// Sitemap specific xml namespace declarations that should not change
-const XMLNS_DECLS = {
-  _attr: {
-    xmlns: 'http://www.sitemaps.org/schemas/sitemap/0.9',
-    'xmlns:image': 'http://www.google.com/schemas/sitemap-image/1.1',
-  },
-};
+const sitemapXml = require('./sitemap-xml');
 
 class BaseSiteMapGenerator {
   constructor() {
-    this.nodeLookup = {};
-    this.nodeTimeLookup = {};
+    // id -> {loc, ts, imageLoc}: one flat record per resource, rather than
+    // the nested element tree the xml package used to take plus a parallel
+    // map of Moments. Both were held for the lifetime of the index, and at
+    // 10k posts they measured ~1.1 kB per resource against ~300 B for this.
+    this.nodeLookup = new Map();
     this.siteMapContent = new Map();
     this.lastModified = 0;
     this.maxPerPage = 50000;
   }
 
+  /**
+   * How many resources this generator holds. Read by the index generator to
+   * work out how many pages the type needs, so it does not have to know how
+   * they are stored.
+   */
+  get size() {
+    return this.nodeLookup.size;
+  }
+
   hasCanonicalUrl(datum, url) {
     if (!datum?.canonical_url) {
       return false;
@@ -46,72 +47,58 @@ class BaseSiteMapGenerator {
   }
 
   generateXmlFromNodes(page) {
-    // Get a mapping of node to timestamp
-    let nodesToProcess = _.map(this.nodeLookup, (node, id) => {
-      return {
-        id: id,
-        // Using negative here to sort newest to oldest
-        ts: -(this.nodeTimeLookup[id] || 0),
-        node: node,
-      };
-    });
-
-    // Sort nodes by timestamp
-    nodesToProcess = _.sortBy(nodesToProcess, 'ts');
+    // Sort newest to oldest. The records are sorted in place of a wrapper
+    // object per resource, so a render allocates one array of references.
+    const records = [...this.nodeLookup.values()];
+    records.sort((a, b) => b.ts - a.ts);
 
     // Get the page of nodes that was requested
-    nodesToProcess = nodesToProcess.slice((page - 1) * this.maxPerPage, page * this.maxPerPage);
+    const pageRecords = records.slice((page - 1) * this.maxPerPage, page * this.maxPerPage);
 
     // Do not generate empty sitemaps
-    if (nodesToProcess.length === 0) {
+    if (pageRecords.length === 0) {
       return null;
     }
 
-    // Grab just the nodes
-    const nodes = _.map(nodesToProcess, 'node');
-
-    const data = {
-      // Concat the elements to the _attr declaration
-      urlset: [XMLNS_DECLS].concat(nodes),
-    };
-
-    // Generate full xml
-    let sitemapXml = localUtils.getDeclarations() + xml(data);
-
-    // Perform url transformations
-    // - Necessary because sitemap data is supplied by the router which
-    //   uses knex directly bypassing model-layer attribute transforms
-    sitemapXml = urlUtils.transformReadyToAbsolute(sitemapXml);
-
-    return sitemapXml;
+    return sitemapXml.renderUrlSet(pageRecords);
   }
 
   addUrl(url, datum) {
-    // Computed once and threaded through: the three consumers below each
-    // used to construct their own moment() from the same datum, which
-    // dominates the cost of bulk adds.
+    if (this.hasCanonicalUrl(datum, url)) {
+      return;
+    }
+
+    // Computed once and threaded through: the consumers below each used to
+    // construct their own moment() from the same datum, which dominates the
+    // cost of bulk adds.
     const lastModified = this.getLastModifiedForDatum(datum);
-    const node = this.createUrlNodeFromDatum(url, datum, lastModified);
 
-    if (node && !this.hasCanonicalUrl(datum, url)) {
-      this.updateLastModified(datum, lastModified);
-      this.updateLookups(datum, node, lastModified);
-      // force regeneration of xml
-      this.siteMapContent.clear();
-    }
+    this.updateLastModified(datum, lastModified);
+    this.updateLookups(datum, this.createRecordFromDatum(url, datum, lastModified));
+    // force regeneration of xml
+    this.siteMapContent.clear();
   }
 
   /**
-   * @returns {moment.Moment}
+   * The parse stays with moment, which is more forgiving than `new Date` of
+   * the shapes the database hands back; only what is stored changes, from a
+   * Moment to the epoch milliseconds the sort and the lastmod both want.
+   *
+   * @returns {number} epoch milliseconds
    */
   getLastModifiedForDatum(datum) {
-    if (datum.updated_at || datum.published_at || datum.created_at) {
-      const modifiedDate = datum.updated_at || datum.published_at || datum.created_at;
+    const modifiedDate = datum.updated_at || datum.published_at || datum.created_at;
 
-      return moment(modifiedDate);
-    } else {
-      return moment();
+    if (!modifiedDate) {
+      return Date.now();
     }
+
+    const lastModified = moment(modifiedDate).valueOf();
+
+    // An unparseable date used to render an empty <lastmod>; now it would
+    // throw out of `new Date(NaN).toISOString()` and take the whole sitemap
+    // with it, so fall back to now.
+    return Number.isNaN(lastModified) ? Date.now() : lastModified;
   }
 
   updateLastModified(datum, lastModified = this.getLastModifiedForDatum(datum)) {
@@ -121,54 +108,40 @@ class BaseSiteMapGenerator {
   }
 
   /**
-   *
    * @param {string} url
    * @param {Object} datum
-   * @returns
+   * @param {number} [lastModified] epoch milliseconds
+   * @returns {{loc: string, ts: number, imageLoc: string|null}}
    */
-  createUrlNodeFromDatum(url, datum, lastModified = this.getLastModifiedForDatum(datum)) {
-    let node;
-    let imgNode;
-
-    node = {
-      url: [{ loc: url }, { lastmod: lastModified.toISOString() }],
+  createRecordFromDatum(url, datum, lastModified = this.getLastModifiedForDatum(datum)) {
+    return {
+      // Transformed here rather than over the finished document: the two
+      // urls are the only transform-ready values a sitemap carries, and a
+      // regex across several megabytes of xml both costs and leaves a rope.
+      loc: urlUtils.transformReadyToAbsolute(url),
+      ts: lastModified,
+      imageLoc: this.createImageLocFromDatum(datum),
     };
-
-    imgNode = this.createImageNodeFromDatum(datum);
-
-    if (imgNode) {
-      node.url.push(imgNode);
-    }
-
-    return node;
   }
 
-  createImageNodeFromDatum(datum) {
+  createImageLocFromDatum(datum) {
     // Check for cover first because user has cover but the rest only have image
     const image = datum.cover_image || datum.profile_image || datum.feature_image;
 
-    let imageUrl;
-    let imageEl;
-
     if (!image) {
-      return;
+      return null;
     }
 
     // Grab the image url
-    imageUrl = urlUtils.urlFor('image', { image: image }, true);
+    const imageUrl = urlUtils.urlFor('image', { image: image }, true);
 
-    // Verify the url structure
+    // Verify the url structure. Checked before the transform, as it always
+    // has been: the url the validator sees is the one urlFor returned.
     if (!this.validateImageUrl(imageUrl)) {
-      return;
+      return null;
     }
 
-    // Create the weird xml node syntax structure that is expected
-    imageEl = [{ 'image:loc': imageUrl }, { 'image:caption': path.basename(imageUrl) }];
-
-    // Return the node to be added to the url xml node
-    return {
-      'image:image': imageEl,
-    };
+    return urlUtils.transformReadyToAbsolute(imageUrl);
   }
 
   validateImageUrl(imageUrl) {
@@ -185,14 +158,12 @@ class BaseSiteMapGenerator {
     return content;
   }
 
-  updateLookups(datum, node, lastModified = this.getLastModifiedForDatum(datum)) {
-    this.nodeLookup[datum.id] = node;
-    this.nodeTimeLookup[datum.id] = lastModified;
+  updateLookups(datum, record) {
+    this.nodeLookup.set(datum.id, record);
   }
 
   reset() {
-    this.nodeLookup = {};
-    this.nodeTimeLookup = {};
+    this.nodeLookup.clear();
     this.siteMapContent.clear();
     this.lastModified = 0;
   }

package/core/frontend/services/sitemap/site-map-index-generator.js

@@ -1,37 +1,31 @@
 const _ = require('lodash');
-const xml = require('xml');
-const moment = require('moment');
 const urlUtils = require('../../../shared/url-utils').default;
-const localUtils = require('./utils');
-
-const XMLNS_DECLS = {
-  _attr: {
-    xmlns: 'http://www.sitemaps.org/schemas/sitemap/0.9',
-  },
-};
+const sitemapXml = require('./sitemap-xml');
 
 class SiteMapIndexGenerator {
   constructor(options) {
     options = options || {};
     this.types = options.types;
     this.maxPerPage = options.maxPerPage;
+    // The index is the one sitemap crawlers fetch repeatedly, and rendering
+    // it counts every resource of every type. Cached until something
+    // invalidates the index; the manager resets this when it does.
+    this.siteMapContent = null;
   }
 
   getXml() {
-    const urlElements = this.generateSiteMapUrlElements();
+    if (this.siteMapContent !== null) {
+      return this.siteMapContent;
+    }
 
-    const data = {
-      // Concat the elements to the _attr declaration
-      sitemapindex: [XMLNS_DECLS].concat(urlElements),
-    };
+    this.siteMapContent = sitemapXml.renderSiteMapIndex(this.generateSiteMapUrlElements());
 
-    // Return the xml
-    return localUtils.getDeclarations() + xml(data);
+    return this.siteMapContent;
   }
 
   generateSiteMapUrlElements() {
     return _.map(this.types, (resourceType) => {
-      const noOfPages = Math.ceil(Object.keys(resourceType.nodeLookup).length / this.maxPerPage);
+      const noOfPages = Math.ceil(resourceType.size / this.maxPerPage);
       const pages = [];
       for (let i = 0; i < noOfPages; i++) {
         const page = i === 0 ? '' : `-${i + 1}`;
@@ -39,16 +33,17 @@ class SiteMapIndexGenerator {
           { relativeUrl: '/sitemap-' + resourceType.name + page + '.xml' },
           true,
         );
-        const lastModified = resourceType.lastModified;
 
-        pages.push({
-          sitemap: [{ loc: url }, { lastmod: moment(lastModified).toISOString() }],
-        });
+        pages.push({ loc: url, ts: resourceType.lastModified });
       }
 
       return pages;
     }).flat();
   }
+
+  reset() {
+    this.siteMapContent = null;
+  }
 }
 
 module.exports = SiteMapIndexGenerator;

package/core/frontend/services/sitemap/site-map-manager.js

@@ -1,3 +1,5 @@
+const { performance } = require('node:perf_hooks');
+const { setImmediate: yieldToEventLoop } = require('node:timers/promises');
 const errors = require('@tryghost/errors');
 const urlUtils = require('../../../shared/url-utils').default;
 const IndexMapGenerator = require('./site-map-index-generator');
@@ -22,11 +24,22 @@ const SITEMAP_COLUMNS = [
   'canonical_url',
 ];
 
+const RESOURCE_TYPES = ['posts', 'pages', 'tags', 'authors'];
+
+// Longest stretch a build runs before yielding to the event loop.
+const DEFAULT_BUILD_SLICE_MS = 10;
+
+// Builds a reader will start before giving up on a site that keeps changing.
+const MAX_BUILD_ATTEMPTS = 3;
+
 class SiteMapManager {
   constructor(options) {
     options = options || {};
 
     options.maxPerPage = options.maxPerPage || 50000;
+    // Every build creates fresh generators from these.
+    this._options = options;
+    this._buildSliceMs = options.buildSliceMs ?? DEFAULT_BUILD_SLICE_MS;
 
     this.pages = options.pages || this.createPagesGenerator(options);
     this.posts = options.posts || this.createPostsGenerator(options);
@@ -54,7 +67,7 @@ class SiteMapManager {
     this._indexEpoch = 0;
     // Static/collection route entries only arrive via RouteRegistered,
     // which fires at boot and routes reload. They are recorded here so
-    // every rebuild can replay them after resetting the generators.
+    // every rebuild can replay them into its fresh generators.
     this._routerEntries = [];
 
     routingEvents.on('RouteRegistered', ({ path, type, id }) => {
@@ -66,7 +79,6 @@ class SiteMapManager {
         datum: { id, staticRoute: type === 'StaticRoutesRouter' },
       };
       this._routerEntries.push(entry);
-      this.pages.addUrl(entry.url, entry.datum);
       // A router registering after a build must not leave a
       // zero-router index marked built — the CDN would pin it.
       this._invalidateIndex();
@@ -90,13 +102,13 @@ class SiteMapManager {
     });
   }
 
-  createIndexGenerator(options) {
+  createIndexGenerator(options, types = this) {
     return new IndexMapGenerator({
       types: {
-        pages: this.pages,
-        posts: this.posts,
-        authors: this.authors,
-        tags: this.tags,
+        pages: types.pages,
+        posts: types.posts,
+        authors: types.authors,
+        tags: types.tags,
       },
       maxPerPage: options.maxPerPage,
     });
@@ -133,81 +145,108 @@ class SiteMapManager {
    * no caller can render from an unbuilt index. The index is built on first
    * read; the invalidation signals empty it and the next read rebuilds.
    *
-   * Concurrent readers share one build. A build whose result was
-   * invalidated while it ran is discarded and the read fails — the index
-   * must never serve pre-invalidation data (the CDN would pin it for the
-   * full cache maxAge), and a 503 is retried by crawlers and stored by
-   * nobody. Deliberately no retry; if SITEMAP_BUILD_SUPERSEDED shows up in
-   * the logs at any rate worth caring about, add one then.
+   * Reads wait for the rebuild rather than being served the previous index.
+   * site.changed is emitted by the same response that purges the CDN, so the
+   * first read after an invalidation is the one the CDN stores for the full
+   * cache maxAge: serving the previous index there would pin a sitemap
+   * missing the change on every publish.
+   *
+   * Concurrent readers share one build. A build invalidated while it runs is
+   * abandoned and a fresh one started; a site changing faster than the index
+   * can be built fails the read with a 503, which crawlers retry and nobody
+   * stores.
    */
   async _ensureIndexReady() {
-    if (this._indexBuilt) {
-      return;
-    }
-    if (!this._buildInFlight) {
-      this._buildInFlight = this._buildIndex().finally(() => {
-        this._buildInFlight = null;
-      });
+    for (let attempt = 0; attempt < MAX_BUILD_ATTEMPTS && !this._indexBuilt; attempt++) {
+      if (!this._buildInFlight) {
+        this._buildInFlight = this._buildIndex().finally(() => {
+          this._buildInFlight = null;
+        });
+      }
+      await this._buildInFlight;
     }
-    await this._buildInFlight;
 
     if (!this._indexBuilt) {
       throw new errors.MaintenanceError({
-        message: 'Sitemap index build was invalidated by a concurrent site change',
+        message: 'Sitemap index build was repeatedly invalidated by concurrent site changes',
         code: 'SITEMAP_BUILD_SUPERSEDED',
       });
     }
   }
 
+  /**
+   * Builds into fresh generators and swaps them in once complete, so the
+   * build can yield to the event loop without any reader seeing a partial
+   * index. The epoch is checked after every yield: an invalidation (including
+   * a RouteRegistered mid-build) abandons the build without swapping.
+   */
   async _buildIndex() {
     const epoch = this._indexEpoch;
     const urlService = this._getUrlService();
     const fetch = (type) => urlService.getRoutableResources(type, { columns: SITEMAP_COLUMNS });
 
-    const [posts, pages, tags, authors] = await Promise.all([
-      fetch('posts'),
-      fetch('pages'),
-      fetch('tags'),
-      fetch('authors'),
-    ]);
+    const [posts, pages, tags, authors] = await Promise.all(RESOURCE_TYPES.map(fetch));
     const resources = { posts, pages, tags, authors };
 
     if (epoch !== this._indexEpoch) {
-      // Invalidated while fetching: leave the generators alone and let
-      // _ensureIndexReady start over.
       return;
     }
-    // Everything from here on is synchronous, so no request can observe
-    // a half-applied index.
-    this.posts.reset();
-    this.pages.reset();
-    this.tags.reset();
-    this.users.reset();
+
+    const next = {
+      posts: this.createPostsGenerator(this._options),
+      pages: this.createPagesGenerator(this._options),
+      tags: this.createTagsGenerator(this._options),
+      authors: this.createUsersGenerator(this._options),
+    };
     for (const entry of this._routerEntries) {
-      this.pages.addUrl(entry.url, entry.datum);
+      next.pages.addUrl(entry.url, entry.datum);
     }
-    for (const type of ['posts', 'pages', 'tags', 'authors']) {
-      for (const datum of resources[type]) {
-        this._applyResource(type, datum);
+
+    let sliceStart = performance.now();
+    for (const type of RESOURCE_TYPES) {
+      const rows = resources[type];
+      for (let i = 0; i < rows.length; i++) {
+        this._applyResource(next, type, rows[i]);
+        // Release each row once applied, so rows and records are not both
+        // fully resident.
+        rows[i] = undefined;
+
+        if (performance.now() - sliceStart >= this._buildSliceMs) {
+          await yieldToEventLoop();
+          if (epoch !== this._indexEpoch) {
+            return;
+          }
+          sliceStart = performance.now();
+        }
       }
     }
+
+    this.posts = next.posts;
+    this.pages = next.pages;
+    this.tags = next.tags;
+    this.users = this.authors = next.authors;
+    // The index generator holds references to the generators it counts.
+    this.index = this.createIndexGenerator(this._options, next);
     this._indexBuilt = true;
   }
 
   _invalidateIndex() {
     this._indexBuilt = false;
     this._indexEpoch += 1;
+    // The index generator's cached xml is valid exactly while _indexBuilt
+    // is, so the two are dropped together.
+    this.index.reset();
   }
 
   /**
-   * Add a single resource to the index.
+   * Add a single resource to a set of generators being built.
    */
-  _applyResource(type, datum) {
+  _applyResource(generators, type, datum) {
     const url = this._getUrlService().getUrlForResource({ ...datum, type }, { absolute: true });
     // Exact match on the not-found sentinel: a real resource can carry
     // a slug like "404" (/tag/404/) and must stay in the sitemap.
     if (url && url !== this._notFoundUrl()) {
-      this[type].addUrl(url, datum);
+      generators[type].addUrl(url, datum);
     }
   }
 

package/core/frontend/services/sitemap/sitemap-xml.js

@@ -0,0 +1,72 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.escapeXml = escapeXml;
+exports.getDeclarations = getDeclarations;
+exports.renderUrlSet = renderUrlSet;
+exports.renderSiteMapIndex = renderSiteMapIndex;
+const node_path_1 = __importDefault(require("node:path"));
+const url_utils_1 = __importDefault(require("../../../shared/url-utils"));
+// Ghost renders exactly two sitemap documents, both with a shape the
+// sitemaps.org schema fixes: a <urlset> per resource type, and the
+// <sitemapindex> that lists them. They are emitted here directly rather than
+// through a generic xml library — for a shape this fixed a library spends
+// several times as long walking an object tree as writing the tags costs, and
+// keeping both documents in one module keeps escaping in one auditable place.
+const URLSET_OPEN = '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"' +
+    ' xmlns:image="http://www.google.com/schemas/sitemap-image/1.1">';
+const URLSET_CLOSE = '</urlset>';
+const SITEMAPINDEX_OPEN = '<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">';
+const SITEMAPINDEX_CLOSE = '</sitemapindex>';
+// The characters the xml package escaped in text content, kept character for
+// character so the rendered sitemaps do not change.
+const XML_ESCAPES = {
+    '&': '&amp;',
+    '"': '&quot;',
+    "'": '&apos;',
+    '<': '&lt;',
+    '>': '&gt;',
+};
+function escapeXml(value) {
+    return value.replace(/[&"'<>]/g, (char) => XML_ESCAPES[char]);
+}
+function getDeclarations() {
+    const baseUrl = url_utils_1.default.urlFor('sitemap_xsl', true).replace(/^(http:|https:)/, '');
+    return ('<?xml version="1.0" encoding="UTF-8"?>' +
+        '<?xml-stylesheet type="text/xsl" href="' +
+        baseUrl +
+        '"?>');
+}
+/**
+ * The <urlset> document for one page of one resource type.
+ *
+ * @param entries in the order they should appear
+ */
+function renderUrlSet(entries) {
+    // Joined rather than concatenated: join returns a flat string, where
+    // repeated concatenation leaves a rope whose fragments are held for as long
+    // as the cached sitemap is.
+    const parts = [getDeclarations(), URLSET_OPEN];
+    for (const entry of entries) {
+        parts.push('<url><loc>', escapeXml(entry.loc), '</loc><lastmod>', new Date(entry.ts).toISOString(), '</lastmod>');
+        if (entry.imageLoc) {
+            parts.push('<image:image><image:loc>', escapeXml(entry.imageLoc), '</image:loc><image:caption>', escapeXml(node_path_1.default.basename(entry.imageLoc)), '</image:caption></image:image>');
+        }
+        parts.push('</url>');
+    }
+    parts.push(URLSET_CLOSE);
+    return parts.join('');
+}
+/**
+ * The <sitemapindex> document listing every page of every resource type.
+ */
+function renderSiteMapIndex(entries) {
+    const parts = [getDeclarations(), SITEMAPINDEX_OPEN];
+    for (const entry of entries) {
+        parts.push('<sitemap><loc>', escapeXml(entry.loc), '</loc><lastmod>', new Date(entry.ts).toISOString(), '</lastmod></sitemap>');
+    }
+    parts.push(SITEMAPINDEX_CLOSE);
+    return parts.join('');
+}