ghost 5.112.0 → 5.113.0

package/core/server/services/email-analytics/lib/queries.js

@@ -99,7 +99,7 @@ module.exports = {
     /**
      * Sets the timestamp of the last seen event for the specified email analytics events.
      * @param {EmailAnalyticsJobName} jobName - The name of the job to update.
-     * @param {'completed'|'started'} field - The field to update.
+     * @param {'finished'|'started'} field - The field to update.
      * @param {Date} date - The timestamp of the last seen event.
      * @returns {Promise<void>}
      * @description
@@ -110,8 +110,8 @@ module.exports = {
         // Convert string dates to Date objects for SQLite compatibility
         try {
             debug(`Setting ${field} timestamp for job ${jobName} to ${date}`);
-            const updateField = field === 'completed' ? 'finished_at' : 'started_at';
-            const status = field === 'completed' ? 'finished' : 'started';
+            const updateField = field === 'finished' ? 'finished_at' : 'started_at';
+            const status = field === 'finished' ? 'finished' : 'started';
             const result = await db.knex('jobs').update({[updateField]: date, updated_at: new Date(), status: status}).where('name', jobName);
             if (result === 0) {
                 await db.knex('jobs').insert({

package/core/server/services/media-inliner/ExternalMediaInliner.js

@@ -0,0 +1,346 @@
+const mime = require('mime-types');
+const FileType = require('file-type');
+const request = require('@tryghost/request');
+const errors = require('@tryghost/errors');
+const logging = require('@tryghost/logging');
+const string = require('@tryghost/string');
+const path = require('path');
+
+class ExternalMediaInliner {
+    /** @type {object} */
+    #PostModel;
+
+    /** @type {object} */
+    #PostMetaModel;
+
+    /** @type {object} */
+    #TagModel;
+
+    /** @type {object} */
+    #UserModel;
+
+    /**
+     *
+     * @param {Object} deps
+     * @param {Object} deps.PostModel - Post model
+     * @param {Object} deps.PostMetaModel - PostMeta model
+     * @param {Object} deps.TagModel - Tag model
+     * @param {Object} deps.UserModel - User model
+     * @param {(extension) => import('ghost-storage-base')} deps.getMediaStorage - getMediaStorage
+     */
+    constructor(deps) {
+        this.#PostModel = deps.PostModel;
+        this.#PostMetaModel = deps.PostMetaModel;
+        this.#TagModel = deps.TagModel;
+        this.#UserModel = deps.UserModel;
+        this.getMediaStorage = deps.getMediaStorage;
+    }
+
+    /**
+     *
+     * @param {string} requestURL - url of remote media
+     * @returns {Promise<Object>}
+     */
+    async getRemoteMedia(requestURL) {
+        // @NOTE: this is the most expensive operation in the whole inlining process
+        //        we should consider caching the results to improve performance
+
+        // Enforce http - http > https redirects are commonplace
+        requestURL = requestURL.replace(/^\/\//g, 'http://');
+
+        // Encode to handle special characters in URLs
+        requestURL = encodeURI(requestURL);
+        try {
+            const response = await request(requestURL, {
+                followRedirect: true,
+                responseType: 'buffer'
+            });
+
+            return response;
+        } catch (error) {
+            // NOTE: add special case for 404s
+            logging.error(`Error downloading remote media: ${requestURL}`);
+            logging.error(new errors.DataImportError({
+                err: error
+            }));
+
+            return null;
+        }
+    }
+
+    /**
+     *
+     * @param {Object} response - response from request
+     * @returns {Object}
+     */
+    async extractFileDataFromResponse(requestURL, response) {
+        let extension;
+
+        // Attempt to get the file extension from the file itself
+        // If that fails, or if `.ext` is undefined, get the extension from the file path in the catch
+        try {
+            const fileInfo = await FileType.fromBuffer(response.body);
+            extension = fileInfo.ext;
+        } catch {
+            const headers = response.headers;
+            const contentType = headers['content-type'];
+            const extensionFromPath = path.parse(requestURL).ext.split(/[^a-z]/i).filter(Boolean)[0];
+            extension = mime.extension(contentType) || extensionFromPath;
+        }
+
+        const removeExtRegExp = new RegExp(`.${extension}`, '');
+        const fileNameNoExt = path.parse(requestURL).base.replace(removeExtRegExp, '');
+
+        // CASE: Query strings _can_ form part of the unique image URL, so rather that strip them include the in the file name
+        // Then trim to last 248 chars (this will be more unique than the first 248), and trim leading & trailing dashes.
+        // 248 is on the lower end of limits from various OSes and file systems
+        const fileName = string.slugify(path.parse(fileNameNoExt).base, {
+            requiredChangesOnly: true
+        }).slice(-248).replace(/^-|-$/, '');
+
+        return {
+            fileBuffer: response.body,
+            filename: `${fileName}.${extension}`,
+            extension: `.${extension}`
+        };
+    }
+
+    /**
+     *
+     * @param {Object} media - media to store locally
+     * @returns {Promise<string>} - path to stored media
+     */
+    async storeMediaLocally(media) {
+        const storage = this.getMediaStorage(media.extension);
+
+        if (!storage) {
+            logging.warn(`No storage adapter found for file extension: ${media.extension}`);
+            return null;
+        } else {
+            // @NOTE: this is extremely convoluted and should live on a
+            //        storage adapter level
+            const targetDir = storage.getTargetDir(storage.storagePath);
+            const uniqueFileName = await storage.getUniqueFileName({
+                name: media.filename
+            }, targetDir);
+            const targetPath = path.relative(storage.storagePath, uniqueFileName);
+            const filePath = await storage.saveRaw(media.fileBuffer, targetPath);
+            return filePath;
+        }
+    }
+
+    static findMatches(content, domain) {
+        // NOTE: the src could end with a quote, bracket, apostrophe, double-backslash, or encoded quote.
+        //     Backlashes are added to content as an escape character
+        const srcTerminationSymbols = `("|\\)|'|(?=(?:,https?))| |<|\\\\|&quot;|$)`;
+        const regex = new RegExp(`(${domain}.*?)(${srcTerminationSymbols})`, 'igm');
+        const matches = content.matchAll(regex);
+
+        // Simplify the matches so we only get the result needed
+        let matchesArray = Array.from(matches, m => m[1]);
+
+        // Trim trailing commas from each match
+        matchesArray = matchesArray.map((item) => {
+            return item.replace(/,$/, '');
+        });
+
+        return matchesArray;
+    }
+
+    /**
+     * Find & inline external media from a JSON sting.
+     * This works with both Lexical & Mobiledoc, so no separate methods are needed here.
+     *
+     * @param {string} content - stringified JSON of post Lexical or Mobiledoc content
+     * @param {String[]} domains - domains to inline media from
+     * @returns {Promise<string>} - updated stringified JSON of post content
+     */
+    async inlineContent(content, domains) {
+        for (const domain of domains) {
+            const matches = this.constructor.findMatches(content, domain);
+
+            for (const src of matches) {
+                const response = await this.getRemoteMedia(src);
+
+                let media;
+                if (response) {
+                    media = await this.extractFileDataFromResponse(src, response);
+                }
+
+                if (media) {
+                    const filePath = await this.storeMediaLocally(media);
+
+                    if (filePath) {
+                        const inlinedSrc = `__GHOST_URL__${filePath}`;
+
+                        // NOTE: does not account for duplicate images in content
+                        //       in those cases would be processed twice
+                        content = content.replace(src, inlinedSrc);
+                        logging.info(`Inlined media: ${src} -> ${inlinedSrc}`);
+                    }
+                }
+            }
+        }
+
+        return content;
+    }
+
+    /**
+     *
+     * @param {Object} resourceModel - one of PostModel, TagModel, UserModel instances
+     * @param {String[]} fields - fields to inline
+     * @param {String[]} domains - domains to inline media from
+     * @returns Promise<Object> - updated fields map with local media paths
+     */
+    async inlineFields(resourceModel, fields, domains) {
+        const updatedFields = {};
+
+        for (const field of fields) {
+            for (const domain of domains) {
+                const src = resourceModel.get(field);
+
+                if (src && src.startsWith(domain)) {
+                    const response = await this.getRemoteMedia(src);
+
+                    let media;
+                    if (response) {
+                        media = await this.extractFileDataFromResponse(src, response);
+                    }
+
+                    if (media) {
+                        const filePath = await this.storeMediaLocally(media);
+
+                        if (filePath) {
+                            const inlinedSrc = `__GHOST_URL__${filePath}`;
+
+                            updatedFields[field] = inlinedSrc;
+                            logging.info(`Added media to inline: ${src} -> ${inlinedSrc}`);
+                        }
+                    }
+                }
+            }
+        }
+
+        return updatedFields;
+    }
+
+    /**
+     *
+     * @param {Object[]} resources - array of model instances
+     * @param {Object} model - resource model
+     * @param {string[]} fields - fields to inline
+     * @param {string[]} domains - domains to inline media from
+     */
+    async inlineSimpleFields(resources, model, fields, domains) {
+        logging.info(`Starting inlining external media for ${resources?.length} resources and with ${fields.join(', ')} fields`);
+
+        for (const resource of resources) {
+            try {
+                const updatedFields = await this.inlineFields(resource, fields, domains);
+
+                if (Object.keys(updatedFields).length > 0) {
+                    await model.edit(updatedFields, {
+                        id: resource.id,
+                        context: {
+                            internal: true
+                        }
+                    });
+                }
+            } catch (err) {
+                logging.error(`Error inlining media for: ${resource.id}`);
+                logging.error(new errors.DataImportError({
+                    err
+                }));
+            }
+        }
+    }
+
+    /**
+     *
+     * @param {string[]} domains domains to inline media from
+     */
+    async inline(domains) {
+        const posts = await this.#PostModel.findAll({context: {internal: true}});
+        const postsInilingFields = [
+            'feature_image'
+        ];
+
+        logging.info(`Starting inlining external media for posts: ${posts?.length}`);
+
+        for (const post of posts) {
+            try {
+                const mobiledocContent = post.get('mobiledoc');
+                const lexicalContent = post.get('lexical');
+
+                const updatedFields = await this.inlineFields(post, postsInilingFields, domains);
+
+                if (mobiledocContent) {
+                    const inlinedContent = await this.inlineContent(mobiledocContent, domains);
+
+                    // If content has changed, update the post
+                    if (inlinedContent !== mobiledocContent) {
+                        updatedFields.mobiledoc = inlinedContent;
+                    }
+                }
+
+                if (lexicalContent) {
+                    const inlinedContent = await this.inlineContent(lexicalContent, domains);
+
+                    // If content has changed, update the post
+                    if (inlinedContent !== lexicalContent) {
+                        updatedFields.lexical = inlinedContent;
+                    }
+                }
+
+                if (Object.keys(updatedFields).length > 0) {
+                    await this.#PostModel.edit(updatedFields, {
+                        id: post.id,
+                        context: {
+                            internal: true
+                        }
+                    });
+                }
+            } catch (err) {
+                logging.error(`Error inlining media for post: ${post.id}`);
+                logging.error(new errors.DataImportError({
+                    err
+                }));
+            }
+        }
+
+        const {data: postsMetas} = await this.#PostMetaModel.findPage({
+            limit: 'all'
+        });
+        const postsMetaInilingFields = [
+            'og_image',
+            'twitter_image'
+        ];
+
+        await this.inlineSimpleFields(postsMetas, this.#PostMetaModel, postsMetaInilingFields, domains);
+
+        const {data: tags} = await this.#TagModel.findPage({
+            limit: 'all'
+        });
+        const tagInliningFields = [
+            'feature_image',
+            'og_image',
+            'twitter_image'
+        ];
+
+        await this.inlineSimpleFields(tags, this.#TagModel, tagInliningFields, domains);
+
+        const {data: users} = await this.#UserModel.findPage({
+            limit: 'all'
+        });
+        const userInliningFields = [
+            'profile_image',
+            'cover_image'
+        ];
+
+        await this.inlineSimpleFields(users, this.#UserModel, userInliningFields, domains);
+
+        logging.info('Finished inlining external media for posts, tags, and users');
+    }
+}
+
+module.exports = ExternalMediaInliner;

package/core/server/services/media-inliner/service.js

@@ -1,7 +1,7 @@
 module.exports = {
     async init() {
         const debug = require('@tryghost/debug')('mediaInliner');
-        const MediaInliner = require('@tryghost/external-media-inliner');
+        const MediaInliner = require('./ExternalMediaInliner');
         const models = require('../../models');
         const jobsService = require('../jobs');
 

package/core/server/services/permissions/can-this.js

@@ -5,6 +5,7 @@ const tpl = require('@tryghost/tpl');
 const providers = require('./providers');
 const parseContext = require('./parse-context');
 const actionsMap = require('./actions-map-cache');
+const {setIsRoles} = require('../../models/role-utils');
 
 const messages = {
     noPermissionToAction: 'You do not have permission to perform this action',
@@ -66,8 +67,8 @@ class CanThisResult {
 
                         return true;
                     };
-
-                    if (loadedPermissions.user && _.some(loadedPermissions.user.roles, {name: 'Owner'})) {
+                    const {isOwner} = setIsRoles(loadedPermissions);
+                    if (isOwner) {
                         hasUserPermission = true;
                     } else if (!_.isEmpty(userPermissions)) {
                         hasUserPermission = _.some(userPermissions, checkPermission);

package/core/server/services/url/Resources.js

@@ -9,6 +9,8 @@ const models = require('../../models');
 // This listens to all manner of model events to find new content that needs a URL...
 const events = require('../../lib/common/events');
 
+/** @typedef {'posts' | 'pages' | 'tags' | 'authors'} ResourceType */
+
 /**
  * @description At the moment the resources class is directly responsible for data population
  * for URLs...but because it's actually a storage cache of all published
@@ -173,7 +175,7 @@ class Resources {
      * If we resolve (https://github.com/TryGhost/Ghost/issues/10360) and talk to the Content API,
      * we could pass on e.g. `?include=authors&fields=authors.id,authors.slug`, but the API has to support it.
      *
-     * @param {Bookshelf-Model} model
+     * @param {import('bookshelf').Model} model
      * @param {Object} resourceConfig
      * @private
      */
@@ -227,8 +229,8 @@ class Resources {
      * all subscribers sequentially. The first generator, where the conditions match the resource, will
      * own the resource and it's url.
      *
-     * @param {String} type (post,user...)
-     * @param {Bookshelf-Model} model
+     * @param {ResourceType} type
+     * @param {import('bookshelf').Model} model
      * @private
      */
     async _onResourceAdded(type, model) {
@@ -306,8 +308,8 @@ class Resources {
      *   - but the data changed and is maybe no longer owned?
      *   - e.g. featured:false changes and your filter requires featured posts
      *
-     * @param {String} type (post,user...)
-     * @param {Bookshelf-Model} model
+     * @param {ResourceType} type
+     * @param {import('bookshelf').Model} model
      * @private
      */
     async _onResourceUpdated(type, model) {
@@ -389,37 +391,25 @@ class Resources {
 
     /**
      * @description Listener for "model removed" event.
-     * @param {String} type (post,user...)
-     * @param {Bookshelf-Model} model
+     * @param {ResourceType} type
+     * @param {import('bookshelf').Model} model
      * @private
      */
     _onResourceRemoved(type, model) {
         debug('_onResourceRemoved', type);
 
-        let index = null;
-        let resource;
-
-        // CASE: search for the cached resource and stop if it was found
-        this.data[type].every((_resource, _index) => {
-            if (_resource.data.id === model._previousAttributes.id) {
-                resource = _resource;
-                index = _index;
-                // break!
-                return false;
-            }
-
-            return true;
-        });
+        const resourceId = model._previousAttributes.id;
+        const index = this.data[type].findIndex(resource => resource.data.id === resourceId);
 
-        // CASE: there are possible cases that the resource was not fetched e.g. visibility is internal
-        if (index === null) {
-            debug('can\'t find resource', model._previousAttributes.id);
+        // Resource might not be in cache if e.g. visibility was internal
+        if (index === -1) {
+            debug('Resource not found in cache', resourceId);
             return;
         }
 
         // remove the resource from cache
-        this.data[type].splice(index, 1);
-        resource.remove();
+        const [removedResource] = this.data[type].splice(index, 1);
+        removedResource.remove();
     }
 
     /**
@@ -432,7 +422,7 @@ class Resources {
 
     /**
      * @description Get all cached resourced by type.
-     * @param {String} type (post, user...)
+     * @param {ResourceType} type
      * @returns {Object}
      */
     getAllByType(type) {
@@ -441,8 +431,8 @@ class Resources {
 
     /**
      * @description Get all cached resourced by resource id and type.
-     * @param {String} type (post, user...)
-     * @param {String} id
+     * @param {ResourceType} type
+     * @param {string} id
      * @returns {Object}
      */
     getByIdAndType(type, id) {

package/core/server/services/url/UrlService.js

@@ -54,7 +54,7 @@ class UrlService {
         this.queue.addListener('started', this._onQueueStartedListener);
 
         this._onQueueEndedListener = this._onQueueEnded.bind(this);
-        this.queue.addListener('ended', this._onQueueEnded.bind(this));
+        this.queue.addListener('ended', this._onQueueEndedListener);
     }
 
     /**
@@ -263,17 +263,7 @@ class UrlService {
     owns(routerId, id) {
         debug('owns', routerId, id);
 
-        let urlGenerator;
-
-        this.urlGenerators.every((_urlGenerator) => {
-            if (_urlGenerator.identifier === routerId) {
-                urlGenerator = _urlGenerator;
-                return false;
-            }
-
-            return true;
-        });
-
+        const urlGenerator = this.urlGenerators.find(g => g.identifier === routerId);
         if (!urlGenerator) {
             return false;
         }

package/core/server/services/url/Urls.js

@@ -1,4 +1,3 @@
-const _ = require('lodash');
 const debug = require('@tryghost/debug')('services:url:urls');
 const urlUtils = require('../../../shared/url-utils');
 const logging = require('@tryghost/logging');
@@ -7,6 +6,10 @@ const errors = require('@tryghost/errors');
 // This emits its own url added/removed events
 const events = require('../../lib/common/events');
 
+/**
+ * @typedef {{url: string, generatorId: string, resource: import('./Resource')}} Url
+ */
+
 /**
  * This class keeps track of all urls in the system.
  * Each resource has exactly one url. Each url is owned by exactly one url generator id.
@@ -20,24 +23,14 @@ const events = require('../../lib/common/events');
  * You can easily ask `this.urls[resourceId]`.
  */
 class Urls {
-    /**
-     *
-     * @param {Object} [options]
-     * @param {Object} [options.urls] map of available URLs with their resources
-     */
-    constructor({urls = {}} = {}) {
-        this.urls = urls;
-    }
+    /** @type {Object<string, Url>} */
+    urls = {};
 
     /**
      * @description Add a url to the system.
-     * @param {Object} options
-     * @param {import('./Resource')} options.resource - instance of the Resource class
-     * @param {string} options.generatorId
-     * @param {string} options.url
+     * @param {Url} options
      */
-    add(options) {
-        const {url, generatorId, resource} = options;
+    add({url, generatorId, resource}) {
         debug('add', resource.data.id, url);
 
         if (this.urls[resource.data.id]) {
@@ -76,7 +69,7 @@ class Urls {
     /**
      * @description Get url by resource id.
      * @param {String} id
-     * @returns {Object}
+     * @returns {Url}
      */
     getByResourceId(id) {
         return this.urls[id];
@@ -85,16 +78,10 @@ class Urls {
     /**
      * @description Get all urls by generator id.
      * @param {String} generatorId
-     * @returns {Array}
+     * @returns {Url[]}
      */
     getByGeneratorId(generatorId) {
-        return _.reduce(Object.keys(this.urls), (toReturn, resourceId) => {
-            if (this.urls[resourceId].generatorId === generatorId) {
-                toReturn.push(this.urls[resourceId]);
-            }
-
-            return toReturn;
-        }, []);
+        return Object.values(this.urls).filter(url => url.generatorId === generatorId);
     }
 
     /**
@@ -108,20 +95,17 @@ class Urls {
      *
      *  But depending on the routing registration, you will always serve e.g. resource1,
      *  because the router it belongs to was registered first.
+     *
+     *  @param {string} urlToLookup
+     *  @returns {Url[]}
      */
-    getByUrl(url) {
-        return _.reduce(Object.keys(this.urls), (toReturn, resourceId) => {
-            if (this.urls[resourceId].url === url) {
-                toReturn.push(this.urls[resourceId]);
-            }
-
-            return toReturn;
-        }, []);
+    getByUrl(urlToLookup) {
+        return Object.values(this.urls).filter(url => url.url === urlToLookup);
     }
 
     /**
      * @description Remove url.
-     * @param id
+     * @param {string} id
      */
     removeResourceId(id) {
         if (!this.urls[id]) {

package/core/shared/config/defaults.json

@@ -214,7 +214,7 @@
     },
     "comments": {
         "url": "https://cdn.jsdelivr.net/ghost/comments-ui@~{version}/umd/comments-ui.min.js",
-        "version": "1.0"
+        "version": "1.1"
     },
     "signupForm": {
         "url": "https://cdn.jsdelivr.net/ghost/signup-form@~{version}/umd/signup-form.min.js",

package/core/shared/labs.js

@@ -38,7 +38,8 @@ const BETA_FEATURES = [
     'ActivityPub',
     'importMemberTier',
     'staff2fa',
-    'contentVisibility'
+    'contentVisibility',
+    'superEditors'
 ];
 
 const ALPHA_FEATURES = [

package/core/shared/settings-cache/CacheManager.js

@@ -54,7 +54,7 @@ class CacheManager {
         if (!this.settingsCache) {
             return;
         }
-        
+
         let override;
         if (this.settingsOverrides && Object.keys(this.settingsOverrides).includes(key)) {
             // Wrap the override value in an object in case it's a boolean
@@ -181,8 +181,8 @@ class CacheManager {
      *
      * Optionally takes a collection of settings & can populate the cache with these.
      *
-     * @param {EventEmitter} events
-     * @param {Bookshelf.Collection<Settings>} settingsCollection
+     * @param {import('events').EventEmitter} events
+     * @param {import('bookshelf').Collection<import('bookshelf').Model>} settingsCollection
      * @param {Array} calculatedFields
      * @param {Object} cacheStore - cache storage instance base on Cache Base Adapter
      * @param {Object} settingsOverrides - key/value pairs of settings which are overridden (i.e. via config)
@@ -219,7 +219,7 @@ class CacheManager {
 
     /**
      * Reset both the cache and the listeners, must be called during init
-     * @param {EventEmitter} events
+     * @param {import('events').EventEmitter} events
      */
     reset(events) {
         if (this.settingsCache) {