ghost 6.63.0 → 6.65.0

package/core/frontend/services/asset-hash/index.js

@@ -5,7 +5,9 @@ const config = require('../../../shared/config');
 /**
  * Asset Hash Service
  *
- * Provides SHA256-based hashing for theme asset files.
+ * Provides SHA256-based hashing for theme asset files, plus the global fallback
+ * hash used for assets we can't hash by content.
+ *
  * Hashes are cached in memory and invalidated when file mtime changes.
  */
 
@@ -14,9 +16,15 @@ const config = require('../../../shared/config');
 // This is longer than the legacy 10-char global hash, making it easy to distinguish
 const HASH_LENGTH = 16;
 
+// The global hash keeps its legacy 10-char hex length so existing asset URLs are unchanged
+const GLOBAL_HASH_LENGTH = 10;
+
 // Cache structure: { filePath: { hash: string, mtimeMs: number } }
 const hashCache = new Map();
 
+// Lazily generated once per boot, and reset whenever the caches are cleared
+let globalHash = null;
+
 /**
  * Calculate SHA256 hash of a file's contents
  * @param {string} filePath - Absolute path to the file
@@ -61,14 +69,36 @@ function getHashForFile(filePath) {
 }
 
 /**
- * Clear all cached hashes
+ * Get the global fallback hash, used for assets whose contents we can't hash -
+ * a missing file, or content-based hashing being turned off.
+ *
+ * Generated lazily on first use rather than at boot, so that nothing pays for it
+ * on a site that never falls back.
+ * @returns {string} - 10 character hex hash
+ */
+function getGlobalHash() {
+  if (!globalHash) {
+    globalHash = crypto
+      .createHash('md5')
+      .update(Date.now().toString())
+      .digest('hex')
+      .substring(0, GLOBAL_HASH_LENGTH);
+  }
+
+  return globalHash;
+}
+
+/**
+ * Clear all cached hashes, including the global fallback hash
  * Should be called when theme is changed/remounted
  */
 function clearCache() {
   hashCache.clear();
+  globalHash = null;
 }
 
 module.exports = {
   getHashForFile,
+  getGlobalHash,
   clearCache,
 };

package/core/frontend/services/assets-minification/minifier.js

@@ -126,7 +126,7 @@ class Minifier {
 
   async writeFile(contents, dest) {
     if (contents) {
-      let writePath = this.getFullDest(dest);
+      const writePath = this.getFullDest(dest);
       // Ensure the output folder exists
       await fs.mkdir(this.destPath, { recursive: true });
       // Create the file

package/core/frontend/services/data/entry-lookup.js

@@ -38,7 +38,7 @@ function entryLookup(postUrl, routerOptions, locals, { giftToken } = {}) {
     isEditURL = true;
   }
 
-  let options = {
+  const options = {
     include: 'authors,tags,tiers',
   };
 

package/core/frontend/services/data/fetch-data.js

@@ -64,8 +64,8 @@ async function fetchData(pathOptions, routerOptions, locals) {
   pathOptions = pathOptions || {};
   routerOptions = routerOptions || {};
 
-  let postQuery = _.cloneDeep(defaultPostQuery);
-  let promises = [];
+  const postQuery = _.cloneDeep(defaultPostQuery);
+  const promises = [];
 
   if (routerOptions.filter) {
     postQuery.options.filter = routerOptions.filter;

package/core/frontend/services/rendering/format-response.js

@@ -90,7 +90,7 @@ function formatResponse(post, context, locals = {}) {
 
   prepareContextResource(post);
 
-  let entry = {
+  const entry = {
     post: post,
   };
 

package/core/frontend/services/rendering/renderer.js

@@ -20,6 +20,8 @@ module.exports = function renderer(req, res, data) {
   if (res.destroyed && !res.writableEnded) {
     debug('Client gone before render, skipping: ' + req.originalUrl);
     res.statusCode = 499;
+    // end() anyway: the request queue only frees a slot on res.end()
+    res.end();
     return;
   }
 
@@ -56,6 +58,7 @@ module.exports = function renderer(req, res, data) {
     if (res.destroyed && !res.writableEnded) {
       debug('Client gone during render, discarding: ' + req.originalUrl);
       res.statusCode = 499;
+      res.end();
       return;
     }
 

package/core/frontend/services/rendering/templates.js

@@ -118,7 +118,7 @@ templates.getEntryTemplateHierarchy = function getEntryTemplateHierarchy(postObj
 templates.pickTemplate = function pickTemplate(templateList, fallback) {
   let template;
 
-  if (!_.isArray(templateList)) {
+  if (!Array.isArray(templateList)) {
     templateList = [templateList];
   }
 

package/core/frontend/services/routing/controllers/static.js

@@ -44,7 +44,7 @@ module.exports = function staticController(req, res, next) {
 
   return Promise.all(promises)
     .then(function handleResult(result) {
-      let response = {};
+      const response = {};
 
       if (res.routerOptions.data) {
         response.data = {};

package/core/frontend/services/routing/controllers/unsubscribe.js

@@ -5,6 +5,7 @@ const { settingsHelpers } = proxy;
 const urlUtils = require('../../../../shared/url-utils').default;
 const logging = require('@tryghost/logging');
 const crypto = require('crypto');
+const { timingSafeStringEqual } = require('../../../../shared/timing-safe-string-equal');
 
 module.exports = async function unsubscribeController(req, res) {
   debug('unsubscribeController');
@@ -42,7 +43,7 @@ module.exports = async function unsubscribeController(req, res) {
       }
       const membersKey = settingsHelpers.getMembersValidationKey();
       const memberHmac = crypto.createHmac('sha256', membersKey).update(query.uuid).digest('hex');
-      if (memberHmac !== query.key) {
+      if (!timingSafeStringEqual(memberHmac, query.key)) {
         logging.warn('[List-Unsubscribe] Unsubscribe failed due to invalid key for ' + query.uuid);
         return res.status(400).end();
       }

package/core/frontend/services/routing/parent-router.js

@@ -83,7 +83,7 @@ class ParentRouter {
    * @private
    */
   _respectDominantRouter(req, res, next, slug) {
-    let siteRouter = this._getSiteRouter(req);
+    const siteRouter = this._getSiteRouter(req);
     let targetRoute = null;
 
     // CASE: iterate over routers and check whether a router has a redirect for the target slug enabled.

package/core/frontend/services/routing/registry.js

@@ -71,7 +71,7 @@ class Registry {
    * @returns {ParentRouter|undefined}
    */
   getRouterByName(name) {
-    for (let routerKey in this.routers) {
+    for (const routerKey in this.routers) {
       if (this.routers[routerKey].name === name) {
         return this.routers[routerKey];
       }

package/core/frontend/services/rss/generate-feed.js

@@ -17,7 +17,7 @@ const generateTags = function generateTags(data) {
 };
 
 const generateItem = function generateItem(post) {
-  const cheerio = require('cheerio');
+  const cheerio = require('cheerio/slim');
 
   // RSS feeds carry posts (pages don't appear in RSS), so the router-level
   // type is always 'posts'. The post object on the public API has its DB

package/core/frontend/services/sitemap/base-site-map-generator.js

@@ -1,27 +1,28 @@
-const _ = require('lodash');
-const xml = require('xml');
 const moment = require('moment');
-const path = require('path');
 const urlUtils = require('../../../shared/url-utils').default;
-const localUtils = require('./utils');
-
-// Sitemap specific xml namespace declarations that should not change
-const XMLNS_DECLS = {
-  _attr: {
-    xmlns: 'http://www.sitemaps.org/schemas/sitemap/0.9',
-    'xmlns:image': 'http://www.google.com/schemas/sitemap-image/1.1',
-  },
-};
+const sitemapXml = require('./sitemap-xml');
 
 class BaseSiteMapGenerator {
   constructor() {
-    this.nodeLookup = {};
-    this.nodeTimeLookup = {};
+    // id -> {loc, ts, imageLoc}: one flat record per resource, rather than
+    // the nested element tree the xml package used to take plus a parallel
+    // map of Moments. Both were held for the lifetime of the index, and at
+    // 10k posts they measured ~1.1 kB per resource against ~300 B for this.
+    this.nodeLookup = new Map();
     this.siteMapContent = new Map();
     this.lastModified = 0;
     this.maxPerPage = 50000;
   }
 
+  /**
+   * How many resources this generator holds. Read by the index generator to
+   * work out how many pages the type needs, so it does not have to know how
+   * they are stored.
+   */
+  get size() {
+    return this.nodeLookup.size;
+  }
+
   hasCanonicalUrl(datum, url) {
     if (!datum?.canonical_url) {
       return false;
@@ -46,72 +47,58 @@ class BaseSiteMapGenerator {
   }
 
   generateXmlFromNodes(page) {
-    // Get a mapping of node to timestamp
-    let nodesToProcess = _.map(this.nodeLookup, (node, id) => {
-      return {
-        id: id,
-        // Using negative here to sort newest to oldest
-        ts: -(this.nodeTimeLookup[id] || 0),
-        node: node,
-      };
-    });
-
-    // Sort nodes by timestamp
-    nodesToProcess = _.sortBy(nodesToProcess, 'ts');
+    // Sort newest to oldest. The records are sorted in place of a wrapper
+    // object per resource, so a render allocates one array of references.
+    const records = [...this.nodeLookup.values()];
+    records.sort((a, b) => b.ts - a.ts);
 
     // Get the page of nodes that was requested
-    nodesToProcess = nodesToProcess.slice((page - 1) * this.maxPerPage, page * this.maxPerPage);
+    const pageRecords = records.slice((page - 1) * this.maxPerPage, page * this.maxPerPage);
 
     // Do not generate empty sitemaps
-    if (nodesToProcess.length === 0) {
+    if (pageRecords.length === 0) {
       return null;
     }
 
-    // Grab just the nodes
-    const nodes = _.map(nodesToProcess, 'node');
-
-    const data = {
-      // Concat the elements to the _attr declaration
-      urlset: [XMLNS_DECLS].concat(nodes),
-    };
-
-    // Generate full xml
-    let sitemapXml = localUtils.getDeclarations() + xml(data);
-
-    // Perform url transformations
-    // - Necessary because sitemap data is supplied by the router which
-    //   uses knex directly bypassing model-layer attribute transforms
-    sitemapXml = urlUtils.transformReadyToAbsolute(sitemapXml);
-
-    return sitemapXml;
+    return sitemapXml.renderUrlSet(pageRecords);
   }
 
   addUrl(url, datum) {
-    // Computed once and threaded through: the three consumers below each
-    // used to construct their own moment() from the same datum, which
-    // dominates the cost of bulk adds.
+    if (this.hasCanonicalUrl(datum, url)) {
+      return;
+    }
+
+    // Computed once and threaded through: the consumers below each used to
+    // construct their own moment() from the same datum, which dominates the
+    // cost of bulk adds.
     const lastModified = this.getLastModifiedForDatum(datum);
-    const node = this.createUrlNodeFromDatum(url, datum, lastModified);
 
-    if (node && !this.hasCanonicalUrl(datum, url)) {
-      this.updateLastModified(datum, lastModified);
-      this.updateLookups(datum, node, lastModified);
-      // force regeneration of xml
-      this.siteMapContent.clear();
-    }
+    this.updateLastModified(datum, lastModified);
+    this.updateLookups(datum, this.createRecordFromDatum(url, datum, lastModified));
+    // force regeneration of xml
+    this.siteMapContent.clear();
   }
 
   /**
-   * @returns {moment.Moment}
+   * The parse stays with moment, which is more forgiving than `new Date` of
+   * the shapes the database hands back; only what is stored changes, from a
+   * Moment to the epoch milliseconds the sort and the lastmod both want.
+   *
+   * @returns {number} epoch milliseconds
    */
   getLastModifiedForDatum(datum) {
-    if (datum.updated_at || datum.published_at || datum.created_at) {
-      const modifiedDate = datum.updated_at || datum.published_at || datum.created_at;
+    const modifiedDate = datum.updated_at || datum.published_at || datum.created_at;
 
-      return moment(modifiedDate);
-    } else {
-      return moment();
+    if (!modifiedDate) {
+      return Date.now();
     }
+
+    const lastModified = moment(modifiedDate).valueOf();
+
+    // An unparseable date used to render an empty <lastmod>; now it would
+    // throw out of `new Date(NaN).toISOString()` and take the whole sitemap
+    // with it, so fall back to now.
+    return Number.isNaN(lastModified) ? Date.now() : lastModified;
   }
 
   updateLastModified(datum, lastModified = this.getLastModifiedForDatum(datum)) {
@@ -121,54 +108,40 @@ class BaseSiteMapGenerator {
   }
 
   /**
-   *
    * @param {string} url
    * @param {Object} datum
-   * @returns
+   * @param {number} [lastModified] epoch milliseconds
+   * @returns {{loc: string, ts: number, imageLoc: string|null}}
    */
-  createUrlNodeFromDatum(url, datum, lastModified = this.getLastModifiedForDatum(datum)) {
-    let node;
-    let imgNode;
-
-    node = {
-      url: [{ loc: url }, { lastmod: lastModified.toISOString() }],
+  createRecordFromDatum(url, datum, lastModified = this.getLastModifiedForDatum(datum)) {
+    return {
+      // Transformed here rather than over the finished document: the two
+      // urls are the only transform-ready values a sitemap carries, and a
+      // regex across several megabytes of xml both costs and leaves a rope.
+      loc: urlUtils.transformReadyToAbsolute(url),
+      ts: lastModified,
+      imageLoc: this.createImageLocFromDatum(datum),
     };
-
-    imgNode = this.createImageNodeFromDatum(datum);
-
-    if (imgNode) {
-      node.url.push(imgNode);
-    }
-
-    return node;
   }
 
-  createImageNodeFromDatum(datum) {
+  createImageLocFromDatum(datum) {
     // Check for cover first because user has cover but the rest only have image
     const image = datum.cover_image || datum.profile_image || datum.feature_image;
 
-    let imageUrl;
-    let imageEl;
-
     if (!image) {
-      return;
+      return null;
     }
 
     // Grab the image url
-    imageUrl = urlUtils.urlFor('image', { image: image }, true);
+    const imageUrl = urlUtils.urlFor('image', { image: image }, true);
 
-    // Verify the url structure
+    // Verify the url structure. Checked before the transform, as it always
+    // has been: the url the validator sees is the one urlFor returned.
     if (!this.validateImageUrl(imageUrl)) {
-      return;
+      return null;
     }
 
-    // Create the weird xml node syntax structure that is expected
-    imageEl = [{ 'image:loc': imageUrl }, { 'image:caption': path.basename(imageUrl) }];
-
-    // Return the node to be added to the url xml node
-    return {
-      'image:image': imageEl,
-    };
+    return urlUtils.transformReadyToAbsolute(imageUrl);
   }
 
   validateImageUrl(imageUrl) {
@@ -185,14 +158,12 @@ class BaseSiteMapGenerator {
     return content;
   }
 
-  updateLookups(datum, node, lastModified = this.getLastModifiedForDatum(datum)) {
-    this.nodeLookup[datum.id] = node;
-    this.nodeTimeLookup[datum.id] = lastModified;
+  updateLookups(datum, record) {
+    this.nodeLookup.set(datum.id, record);
   }
 
   reset() {
-    this.nodeLookup = {};
-    this.nodeTimeLookup = {};
+    this.nodeLookup.clear();
     this.siteMapContent.clear();
     this.lastModified = 0;
   }

package/core/frontend/services/sitemap/site-map-index-generator.js

@@ -1,37 +1,31 @@
 const _ = require('lodash');
-const xml = require('xml');
-const moment = require('moment');
 const urlUtils = require('../../../shared/url-utils').default;
-const localUtils = require('./utils');
-
-const XMLNS_DECLS = {
-  _attr: {
-    xmlns: 'http://www.sitemaps.org/schemas/sitemap/0.9',
-  },
-};
+const sitemapXml = require('./sitemap-xml');
 
 class SiteMapIndexGenerator {
   constructor(options) {
     options = options || {};
     this.types = options.types;
     this.maxPerPage = options.maxPerPage;
+    // The index is the one sitemap crawlers fetch repeatedly, and rendering
+    // it counts every resource of every type. Cached until something
+    // invalidates the index; the manager resets this when it does.
+    this.siteMapContent = null;
   }
 
   getXml() {
-    const urlElements = this.generateSiteMapUrlElements();
+    if (this.siteMapContent !== null) {
+      return this.siteMapContent;
+    }
 
-    const data = {
-      // Concat the elements to the _attr declaration
-      sitemapindex: [XMLNS_DECLS].concat(urlElements),
-    };
+    this.siteMapContent = sitemapXml.renderSiteMapIndex(this.generateSiteMapUrlElements());
 
-    // Return the xml
-    return localUtils.getDeclarations() + xml(data);
+    return this.siteMapContent;
   }
 
   generateSiteMapUrlElements() {
     return _.map(this.types, (resourceType) => {
-      const noOfPages = Math.ceil(Object.keys(resourceType.nodeLookup).length / this.maxPerPage);
+      const noOfPages = Math.ceil(resourceType.size / this.maxPerPage);
       const pages = [];
       for (let i = 0; i < noOfPages; i++) {
         const page = i === 0 ? '' : `-${i + 1}`;
@@ -39,16 +33,17 @@ class SiteMapIndexGenerator {
           { relativeUrl: '/sitemap-' + resourceType.name + page + '.xml' },
           true,
         );
-        const lastModified = resourceType.lastModified;
 
-        pages.push({
-          sitemap: [{ loc: url }, { lastmod: moment(lastModified).toISOString() }],
-        });
+        pages.push({ loc: url, ts: resourceType.lastModified });
       }
 
       return pages;
     }).flat();
   }
+
+  reset() {
+    this.siteMapContent = null;
+  }
 }
 
 module.exports = SiteMapIndexGenerator;