ghost 5.37.0 → 5.38.0

package/core/built/admin/index.html

@@ -8,7 +8,7 @@
     <title>Ghost Admin</title>
 
     
-<meta name="ghost-admin/config/environment" content="%7B%22modulePrefix%22%3A%22ghost-admin%22%2C%22environment%22%3A%22production%22%2C%22rootURL%22%3A%22%22%2C%22locationType%22%3A%22trailing-hash%22%2C%22EmberENV%22%3A%7B%22FEATURES%22%3A%7B%7D%2C%22EXTEND_PROTOTYPES%22%3A%7B%22Date%22%3Afalse%2C%22Array%22%3Atrue%2C%22String%22%3Atrue%2C%22Function%22%3Afalse%7D%2C%22_APPLICATION_TEMPLATE_WRAPPER%22%3Afalse%2C%22_JQUERY_INTEGRATION%22%3Atrue%2C%22_TEMPLATE_ONLY_GLIMMER_COMPONENTS%22%3Atrue%7D%2C%22APP%22%3A%7B%22version%22%3A%225.37%22%2C%22name%22%3A%22ghost-admin%22%7D%2C%22ember-simple-auth%22%3A%7B%7D%2C%22ember-websockets%22%3A%7B%22socketIO%22%3Atrue%7D%2C%22%40sentry%2Fember%22%3A%7B%22disablePerformance%22%3Atrue%2C%22sentry%22%3A%7B%7D%7D%2C%22ember-cli-mirage%22%3A%7B%22usingProxy%22%3Afalse%2C%22useDefaultPassthroughs%22%3Atrue%7D%2C%22exportApplicationGlobal%22%3Afalse%2C%22ember-load%22%3A%7B%22loadingIndicatorClass%22%3A%22ember-load-indicator%22%7D%7D" />
+<meta name="ghost-admin/config/environment" content="%7B%22modulePrefix%22%3A%22ghost-admin%22%2C%22environment%22%3A%22production%22%2C%22rootURL%22%3A%22%22%2C%22locationType%22%3A%22trailing-hash%22%2C%22EmberENV%22%3A%7B%22FEATURES%22%3A%7B%7D%2C%22EXTEND_PROTOTYPES%22%3A%7B%22Date%22%3Afalse%2C%22Array%22%3Atrue%2C%22String%22%3Atrue%2C%22Function%22%3Afalse%7D%2C%22_APPLICATION_TEMPLATE_WRAPPER%22%3Afalse%2C%22_JQUERY_INTEGRATION%22%3Atrue%2C%22_TEMPLATE_ONLY_GLIMMER_COMPONENTS%22%3Atrue%7D%2C%22APP%22%3A%7B%22version%22%3A%225.38%22%2C%22name%22%3A%22ghost-admin%22%7D%2C%22ember-simple-auth%22%3A%7B%7D%2C%22ember-websockets%22%3A%7B%22socketIO%22%3Atrue%7D%2C%22%40sentry%2Fember%22%3A%7B%22disablePerformance%22%3Atrue%2C%22sentry%22%3A%7B%7D%7D%2C%22ember-cli-mirage%22%3A%7B%22usingProxy%22%3Afalse%2C%22useDefaultPassthroughs%22%3Atrue%7D%2C%22exportApplicationGlobal%22%3Afalse%2C%22ember-load%22%3A%7B%22loadingIndicatorClass%22%3A%22ember-load-indicator%22%7D%7D" />
 
     <meta name="HandheldFriendly" content="True" />
     <meta name="MobileOptimized" content="320" />
@@ -37,7 +37,7 @@
     </style>
 
     <link integrity="" rel="stylesheet" href="assets/vendor-3e6947aa681f0fb82b193090e520dc73.css">
-    <link integrity="" rel="stylesheet" href="assets/ghost-efbe4dcc249d119a955b038aae5c980d.css" title="light">
+    <link integrity="" rel="stylesheet" href="assets/ghost-a9307c9cfe26a4bc621e02cd3bae421a.css" title="light">
 
     
 </head>
@@ -57,8 +57,8 @@
     <div id="ember-basic-dropdown-wormhole"></div>
 
     <script src="assets/vendor-b982e3bf1020bff77b2a3c44d5f59e55.js"></script>
-<script src="assets/chunk.652.bb618bc5abf23bed4e87.js"></script>
-<script src="assets/chunk.143.27cd10a38f877e715b35.js"></script>
-    <script src="assets/ghost-2948791640be026b987b88f89034bc85.js"></script>
+<script src="assets/chunk.220.9ca2950240aba3fced21.js"></script>
+<script src="assets/chunk.143.c6802c882a911797ce4f.js"></script>
+    <script src="assets/ghost-35103ff053c43f1dfa7f35821c3c2412.js"></script>
 </body>
 </html>

package/core/frontend/helpers/ghost_head.js

@@ -64,7 +64,10 @@ function getMembersHelper(data, frontendKey) {
     let membersHelper = `<script defer src="${scriptUrl}" ${dataAttributes} crossorigin="anonymous"></script>`;
     membersHelper += (`<style id="gh-members-styles">${templateStyles}</style>`);
     if (settingsCache.get('paid_members_enabled')) {
-        membersHelper += '<script async src="https://js.stripe.com/v3/"></script>';
+        // disable fraud detection for e2e tests to reduce waiting time
+        const isFraudSignalsEnabled = process.env.NODE_ENV === 'testing-browser' ? '?advancedFraudSignals=false' : '';
+
+        membersHelper += `<script async src="https://js.stripe.com/v3/${isFraudSignalsEnabled}"></script>`;
     }
     return membersHelper;
 }

package/core/frontend/services/routing/StaticPagesRouter.js

@@ -52,7 +52,7 @@ class StaticPagesRouter extends ParentRouter {
     }
 
     /**
-     * @description Prepare context for futher middleware/controllers.
+     * @description Prepare context for further middleware/controllers.
      * @param {Object} req
      * @param {Object} res
      * @param {Function} next

package/core/frontend/services/sitemap/base-generator.js

@@ -22,6 +22,10 @@ class BaseSiteMapGenerator {
         this.maxPerPage = 50000;
     }
 
+    hasCanonicalUrl(datum) {
+        return Boolean(datum?.canonical_url);
+    }
+
     generateXmlFromNodes(page) {
         // Get a mapping of node to timestamp
         let nodesToProcess = _.map(this.nodeLookup, (node, id) => {
@@ -75,7 +79,7 @@ class BaseSiteMapGenerator {
     addUrl(url, datum) {
         const node = this.createUrlNodeFromDatum(url, datum);
 
-        if (node) {
+        if (node && !this.hasCanonicalUrl(datum)) {
             this.updateLastModified(datum);
             this.updateLookups(datum, node);
             // force regeneration of xml

package/core/server/adapters/storage/LocalImagesStorage.js

@@ -27,7 +27,7 @@ class LocalImagesStorage extends LocalStorageBase {
     /**
      * Saves a buffer in the targetPath
      * @param {Buffer} buffer is an instance of Buffer
-     * @param {String} targetPath path to which the buffer should be written
+     * @param {String} targetPath relative path NOT including storage path to which the buffer should be written
      * @returns {Promise<String>} a URL to retrieve the data
      */
     async saveRaw(buffer, targetPath) {

package/core/server/api/endpoints/email-previews.js

@@ -1,14 +1,4 @@
-const models = require('../../models');
-const tpl = require('@tryghost/tpl');
-const errors = require('@tryghost/errors');
-const mega = require('../../services/mega');
 const emailService = require('../../services/email-service');
-const labs = require('../../../shared/labs');
-const messages = {
-    postNotFound: 'Post not found.'
-};
-
-const emailPreview = new mega.EmailPreview();
 
 module.exports = {
     docName: 'email_previews',
@@ -30,25 +20,7 @@ module.exports = {
         ],
         permissions: true,
         async query(frame) {
-            if (labs.isSet('emailStability')) {
-                return await emailService.controller.previewEmail(frame);
-            }
-
-            const options = Object.assign(frame.options, {formats: 'html,plaintext', withRelated: ['authors', 'posts_meta']});
-            const data = Object.assign(frame.data, {status: 'all'});
-
-            const model = await models.Post.findOne(data, options);
-
-            if (!model) {
-                throw new errors.NotFoundError({
-                    message: tpl(messages.postNotFound)
-                });
-            }
-
-            return emailPreview.generateEmailContent(model, {
-                newsletter: frame.options.newsletter,
-                memberSegment: frame.options.memberSegment
-            });
+            return await emailService.controller.previewEmail(frame);
         }
     },
     sendTestEmail: {
@@ -66,20 +38,7 @@ module.exports = {
         },
         permissions: true,
         async query(frame) {
-            if (labs.isSet('emailStability')) {
-                return await emailService.controller.sendTestEmail(frame);
-            }
-
-            const options = Object.assign(frame.options, {status: 'all'});
-            let model = await models.Post.findOne(options, {withRelated: ['authors']});
-
-            if (!model) {
-                throw new errors.NotFoundError({
-                    message: tpl(messages.postNotFound)
-                });
-            }
-            const {emails = [], memberSegment, newsletter = ''} = frame.data;
-            return await mega.mega.sendTestEmail(model, emails, memberSegment, newsletter);
+            return await emailService.controller.sendTestEmail(frame);
         }
     }
 };

package/core/server/api/endpoints/emails.js

@@ -1,9 +1,7 @@
 const models = require('../../models');
 const tpl = require('@tryghost/tpl');
 const errors = require('@tryghost/errors');
-const megaService = require('../../services/mega');
 const emailService = require('../../services/email-service');
-const labs = require('../../../shared/labs');
 const emailAnalytics = require('../../services/email-analytics');
 
 const messages = {
@@ -63,27 +61,8 @@ module.exports = {
             'id'
         ],
         permissions: true,
-        // (complexity removed with new labs flag)
-        // eslint-disable-next-line ghost/ghost-custom/max-api-complexity
         async query(frame) {
-            if (labs.isSet('emailStability')) {
-                return await emailService.controller.retryFailedEmail(frame);
-            }
-
-            const model = await models.Email.findOne(frame.data, frame.options);
-            if (!model) {
-                throw new errors.NotFoundError({
-                    message: tpl(messages.emailNotFound)
-                });
-            }
-
-            if (model.get('status') !== 'failed') {
-                throw new errors.IncorrectUsageError({
-                    message: tpl(messages.retryNotAllowed)
-                });
-            }
-
-            return await megaService.mega.retryFailedEmail(model);
+            return await emailService.controller.retryFailedEmail(frame);
         }
     },
 

package/core/server/api/endpoints/utils/serializers/output/mappers/emails.js

@@ -1,19 +1,22 @@
-const mega = require('../../../../../../services/mega');
 const labs = require('../../../../../../../shared/labs');
 const config = require('../../../../../../../shared/config');
+const emailService = require('../../../../../../services/email-service');
 
 module.exports = (model, frame) => {
     const jsonModel = model.toJSON ? model.toJSON(frame.options) : model;
 
     // Ensure we're not outputting unwanted replacement strings when viewing email contents
     // TODO: extract this to a utility, it's duplicated in the email-preview API controller
-    const replacements = mega.postEmailSerializer.parseReplacements(jsonModel);
-    replacements.forEach((replacement) => {
-        jsonModel[replacement.format] = jsonModel[replacement.format].replace(
-            replacement.regexp,
-            replacement.fallback || ''
-        );
-    });
+    if (jsonModel.html) {
+        const replacements = emailService.renderer.buildReplacementDefinitions({html: jsonModel.html, newsletterUuid: 'preview'});
+        const exampleMember = emailService.service.getDefaultExampleMember();
+
+        jsonModel.html = emailService.service.replaceDefinitions(jsonModel.html, replacements, exampleMember);
+
+        if (jsonModel.plaintext) {
+            jsonModel.plaintext = emailService.service.replaceDefinitions(jsonModel.plaintext, replacements, exampleMember);
+        }
+    }
 
     if (!labs.isSet('emailErrors') && !!(config.get('bulkEmail') && config.get('bulkEmail').mailgun)) {
         if (jsonModel.status === 'failed') {
@@ -21,5 +24,8 @@ module.exports = (model, frame) => {
         }
     }
 
+    // Removed loaded post relation if set
+    delete jsonModel.post;
+
     return jsonModel;
 };

package/core/server/data/importer/import-manager.js

@@ -301,7 +301,14 @@ class ImportManager {
         const baseDir = this.getBaseDirectory(zipDirectory);
 
         for (const handler of this.handlers) {
-            const files = this.getFilesFromZip(handler, zipDirectory);
+            let files = [];
+            if (handler.directories?.length > 0) {
+                for (const dir of handler.directories) {
+                    files.push(...this.getFilesFromZip(handler, path.join(zipDirectory, (baseDir || ''), dir)));
+                }
+            } else {
+                files.push(...this.getFilesFromZip(handler, zipDirectory));
+            }
 
             debug('handler', handler.type, files);
 

package/core/server/data/migrations/versions/4.9/05-fix-missed-mobiledoc-url-transforms.js

@@ -4,7 +4,7 @@ const htmlToPlaintext = require('@tryghost/html-to-plaintext');
 const mobiledocLib = require('../../../../lib/mobiledoc');
 const {createTransactionalMigration} = require('../../utils');
 
-// in Ghost versions 4.6.1-4.8.4 the 4.0 migration that transfored URLs had a bug
+// in Ghost versions 4.6.1-4.8.4 the 4.0 migration that transformed URLs had a bug
 // that meant urls inside cards in mobiledoc content was not being transformed
 //
 // if the migrations table indicates an upgrade was made from 3.x to 4.6-4.8 then

package/core/server/lib/common/events.js

@@ -12,29 +12,22 @@
  */
 
 const events = require('events');
-const util = require('util');
-let EventRegistry;
-let EventRegistryInstance;
 
-EventRegistry = function () {
-    events.EventEmitter.call(this);
-};
+class EventRegistry extends events.EventEmitter {
+    /**
+     * This is method is semi-hack to make sure listeners are only registered once
+     * during the lifetime of the process. And example problem it solves is
+     * registering duplicate listeners between Ghost instance reboots when running tests.
+     * @param {String} eventName
+     * @param {String} listenerName named function name registered as a listener for the event
+     * @returns {Boolean}
+     */
+    hasRegisteredListener(eventName, listenerName) {
+        return !!(this.listeners(eventName).find(listener => (listener.name === listenerName)));
+    }
+}
 
-util.inherits(EventRegistry, events.EventEmitter);
+const eventRegistryInstance = new EventRegistry();
+eventRegistryInstance.setMaxListeners(100);
 
-/**
- * This is method is semi-hack to make sure listeners are only registered once
- * during the lifetime of the process. And example problem it solves is
- * registering duplicate listeners between Ghost instance reboots when running tests.
- * @param {String} eventName
- * @param {String} listenerName named function name registered as a listener for the event
- * @returns {Boolean}
- */
-EventRegistry.prototype.hasRegisteredListener = function (eventName, listenerName) {
-    return !!(this.listeners(eventName).find(listener => (listener.name === listenerName)));
-};
-
-EventRegistryInstance = new EventRegistry();
-EventRegistryInstance.setMaxListeners(100);
-
-module.exports = EventRegistryInstance;
+module.exports = eventRegistryInstance;

package/core/server/models/base/plugins/relations.js

@@ -7,7 +7,7 @@ module.exports = function (Bookshelf) {
          * Return a relation, and load it if it hasn't been loaded already (or force a refresh with the forceRefresh option).
          * refs https://github.com/TryGhost/Team/issues/1626
          * @param {string} name Name of the relation to load
-         * @param {Object} [options] Options to pass to the fetch when not yet loaded (or when force refreshing) 
+         * @param {Object} [options] Options to pass to the fetch when not yet loaded (or when force refreshing)
          * @param {boolean} [options.forceRefresh] If true, the relation will be fetched again even if it has already been loaded.
          * @returns {Promise<import('bookshelf').Model|import('bookshelf').Collection|null>}
          */
@@ -23,8 +23,10 @@ module.exports = function (Bookshelf) {
             // Not yet loaded, or force refresh
             // Note that we don't use .refresh on the relation on options.forceRefresh
             // Because the relation can also be a collection, which doesn't have a refresh method
-            this.relations[name] = this[name]();
-            return this.relations[name].fetch(options);
+            const instance = this[name]();
+            await instance.fetch(options);
+            this.relations[name] = instance;
+            return instance;
         }
     });
 };

package/core/server/models/index.js

@@ -3,6 +3,7 @@
  */
 
 const _ = require('lodash');
+const debug = require('@tryghost/debug')('models');
 const glob = require('glob');
 
 // enable event listeners
@@ -14,12 +15,16 @@ require('./base/listeners');
 exports = module.exports;
 
 function init() {
+    const baseNow = Date.now();
     exports.Base = require('./base');
+    debug(`${Date.now() - baseNow}ms - Base.js require`);
 
     let modelsFiles = glob.sync('!(index).js', {cwd: __dirname});
     modelsFiles.forEach((model) => {
         const name = model.replace(/.js$/, '');
+        const modelNow = Date.now();
         _.extend(exports, require('./' + name));
+        debug(`${Date.now() - modelNow}ms - ${model} require`);
     });
 }
 

package/core/server/services/comments/emails.js

@@ -2,7 +2,7 @@ const {promises: fs} = require('fs');
 const path = require('path');
 const moment = require('moment');
 const htmlToPlaintext = require('@tryghost/html-to-plaintext');
-const postEmailSerializer = require('../mega/post-email-serializer');
+const emailService = require('../email-service');
 
 class CommentsServiceEmails {
     constructor({config, logging, models, mailer, settingsCache, settingsHelpers, urlService, urlUtils}) {
@@ -95,7 +95,7 @@ class CommentsServiceEmails {
             accentColor: this.settingsCache.get('accent_color'),
             fromEmail: this.notificationFromAddress,
             toEmail: to,
-            profileUrl: postEmailSerializer.createUnsubscribeUrl(member.get('uuid'), {comments: true})
+            profileUrl: emailService.renderer.createUnsubscribeUrl(member.get('uuid'), {comments: true})
         };
 
         const {html, text} = await this.renderEmailTemplate('new-comment-reply', templateData);

package/core/server/services/email-service/wrapper.js

@@ -94,6 +94,8 @@ class EmailServiceWrapper {
             sentry
         });
 
+        this.renderer = emailRenderer;
+
         this.service = new EmailService({
             batchSendingService,
             sendingService,

package/core/server/services/link-tracking/LinkClickRepository.js

@@ -56,7 +56,7 @@ module.exports = class LinkClickRepository {
         }
 
         const model = await this.#MemberLinkClickEventModel.add({
-            // Only store the parthname (no support for variable query strings)
+            // Only store the pathname (no support for variable query strings)
             redirect_id: linkClick.link_id.toHexString(),
             member_id: member.id
         }, {});

package/core/server/services/media-inliner/service.js

@@ -1,12 +1,58 @@
 module.exports = {
     async init() {
         const debug = require('@tryghost/debug')('mediaInliner');
+        const MediaInliner = require('@tryghost/external-media-inliner');
+        const models = require('../../models');
+        const jobsService = require('../jobs');
+
+        const mediaStorage = require('../../adapters/storage').getStorage('media');
+        const imageStorage = require('../../adapters/storage').getStorage('images');
+        const fileStorage = require('../../adapters/storage').getStorage('files');
+
+        const config = require('../../../shared/config');
+
+        const mediaInliner = new MediaInliner({
+            PostModel: models.Post,
+            TagModel: models.Tag,
+            UserModel: models.User,
+            PostMetaModel: models.PostsMeta,
+            getMediaStorage: (extension) => {
+                if (config.get('uploads').images.extensions.includes(extension)) {
+                    return imageStorage;
+                } else if (config.get('uploads').media.extensions.includes(extension)) {
+                    return mediaStorage;
+                } else if (config.get('uploads').files.extensions.includes(extension)) {
+                    return fileStorage;
+                } else {
+                    return null;
+                }
+            }
+        });
 
         this.api = {
-            // @NOTE: the inlining should become an offloaded job
-            // startMediaInliner: mediaInliner.inlineMedia
-            startMediaInliner: (domains) => {
+
+            startMediaInliner: async (domains) => {
+                if (!domains || !domains.length) {
+                    // default domains to inline from if none are provided
+                    domains = [
+                        'https://s3.amazonaws.com/revue',
+                        'https://substackcdn.com'
+                    ];
+                }
+
                 debug('[Inliner] Starting media inlining job for domains: ', domains);
+
+                // @NOTE: the job is "inline" (aka non-offloaded into a thread), because usecases are currently
+                //        limited to migrational, so there is no expectations for site's availability etc.
+                await jobsService.addJob({
+                    name: 'external-media-inliner',
+                    job: (data) => {
+                        return mediaInliner.inline(data.domains);
+                    },
+                    data: {domains},
+                    offloaded: false
+                });
+
                 return {
                     status: 'success'
                 };

package/core/server/services/mentions/service.js

@@ -26,7 +26,12 @@ function getPostUrl(post) {
 
 module.exports = {
     controller: new MentionController(),
+    didInit: false,
     async init() {
+        if (this.didInit) {
+            return;
+        }
+        this.didInit = true;
         const repository = new BookshelfMentionRepository({
             MentionModel: models.Mention,
             DomainEvents
@@ -97,6 +102,6 @@ module.exports = {
                 }
             }
         });
-        sendingService.listen(events); 
+        sendingService.listen(events);
     }
 };

package/core/server/services/posts/posts-service.js

@@ -8,8 +8,7 @@ const messages = {
 };
 
 class PostsService {
-    constructor({mega, urlUtils, models, isSet, stats, emailService}) {
-        this.mega = mega;
+    constructor({urlUtils, models, isSet, stats, emailService}) {
         this.urlUtils = urlUtils;
         this.models = models;
         this.isSet = isSet;
@@ -45,17 +44,9 @@ class PostsService {
                 let email;
 
                 if (!postEmail) {
-                    if (this.isSet('emailStability')) {
-                        email = await this.emailService.createEmail(model);
-                    } else {
-                        email = await this.mega.addEmail(model, frame.options);
-                    }
+                    email = await this.emailService.createEmail(model);
                 } else if (postEmail && postEmail.get('status') === 'failed') {
-                    if (this.isSet('emailStability')) {
-                        email = await this.emailService.retryEmail(postEmail);
-                    } else {
-                        email = await this.mega.retryFailedEmail(postEmail);
-                    }
+                    email = await this.emailService.retryEmail(postEmail);
                 }
                 if (email) {
                     model.set('email', email);
@@ -130,7 +121,6 @@ class PostsService {
  */
 const getPostServiceInstance = () => {
     const urlUtils = require('../../../shared/url-utils');
-    const {mega} = require('../mega');
     const labs = require('../../../shared/labs');
     const models = require('../../models');
     const PostStats = require('./stats/post-stats');
@@ -139,7 +129,6 @@ const getPostServiceInstance = () => {
     const postStats = new PostStats();
 
     return new PostsService({
-        mega: mega,
         urlUtils: urlUtils,
         models: models,
         isSet: flag => labs.isSet(flag), // don't use bind, that breaks test subbing of labs

package/core/server/services/staff/index.js

@@ -12,6 +12,7 @@ class StaffServiceWrapper {
 
         const logging = require('@tryghost/logging');
         const models = require('../../models');
+        const memberAttribution = require('../member-attribution');
         const {GhostMailer} = require('../mail');
         const mailer = new GhostMailer();
         const settingsCache = require('../../../shared/settings-cache');
@@ -26,6 +27,7 @@ class StaffServiceWrapper {
             settingsCache,
             urlUtils,
             DomainEvents,
+            memberAttributionService: memberAttribution.service,
             labs
         });
 

package/core/server/services/url/Urls.js

@@ -44,10 +44,18 @@ class Urls {
         debug('cache', url);
 
         if (this.urls[resource.data.id]) {
-            logging.error(new errors.InternalServerError({
+            const error = new errors.InternalServerError({
                 message: 'This should not happen.',
                 code: 'URLSERVICE_RESOURCE_DUPLICATE'
-            }));
+            });
+            if (process.env.NODE_ENV.startsWith('test')) {
+                logging.warn({
+                    message: 'Duplicate URL',
+                    err: error
+                });
+            } else {
+                logging.error(error);
+            }
 
             this.removeResourceId(resource.data.id);
         }

package/core/shared/labs.js

@@ -20,7 +20,6 @@ const GA_FEATURES = [
     'memberAttribution',
     'audienceFeedback',
     'themeErrorsNotification',
-    'emailStability',
     'emailErrors',
     'outboundLinkTagging'
 ];