ghost 5.22.11 → 5.24.0

package/core/server/services/mega/mega.js

@@ -15,6 +15,8 @@ const db = require('../../data/db');
 const models = require('../../models');
 const postEmailSerializer = require('./post-email-serializer');
 const {getSegmentsFromHtml} = require('./segment-parser');
+const emailSuppressionList = require('../email-suppression-list');
+const labs = require('../../../shared/labs');
 
 // Used to listen to email.added and email.edited model events originally, I think to offload this - ideally would just use jobs now if possible
 const events = require('../../lib/common/events');
@@ -236,6 +238,8 @@ const addEmail = async (postModel, options) => {
             from: emailData.from,
             reply_to: emailData.replyTo,
             html: emailData.html,
+            source: emailData.html,
+            source_type: 'html',
             plaintext: emailData.plaintext,
             submitted_at: moment().toDate(),
             track_opens: !!settingsCache.get('email_track_opens'),
@@ -265,6 +269,10 @@ const retryFailedEmail = async (emailModel) => {
 };
 
 async function pendingEmailHandler(emailModel, options) {
+    if (labs.isSet('emailStability')) {
+        return;
+    }
+
     // CASE: do not send email if we import a database
     // TODO: refactor post.published events to never fire on importing
     if (options && options.importing) {
@@ -283,13 +291,14 @@ async function pendingEmailHandler(emailModel, options) {
     if (!process.env.NODE_ENV.startsWith('test')) {
         return jobsService.addJob({
             job: sendEmailJob,
-            data: {emailModel},
+            data: {emailId: emailModel.id},
             offloaded: false
         });
     }
 }
 
-async function sendEmailJob({emailModel, options}) {
+async function sendEmailJob({emailId, options}) {
+    logging.info('[sendEmailJob] Started for ' + emailId);
     let startEmailSend = null;
 
     try {
@@ -304,10 +313,45 @@ async function sendEmailJob({emailModel, options}) {
             await limitService.errorIfWouldGoOverLimit('emails');
         }
 
+        // Check if the email is still pending. And set the status to submitting in one transaction.
+        let hasSingleAccess = false;
+        let emailModel;
+        await models.Base.transaction(async (transacting) => {
+            const knexOptions = {...options, transacting, forUpdate: true};
+            emailModel = await models.Email.findOne({id: emailId}, knexOptions);
+
+            if (!emailModel) {
+                throw new errors.IncorrectUsageError({
+                    message: 'Provided email id does not match a known email record',
+                    context: {
+                        id: emailId
+                    }
+                });
+            }
+
+            if (emailModel.get('status') !== 'pending') {
+                // We don't throw this, because we don't want to mark this email as failed
+                logging.error(new errors.IncorrectUsageError({
+                    message: 'Emails can only be processed when in the "pending" state',
+                    context: `Email "${emailId}" has state "${emailModel.get('status')}"`,
+                    code: 'EMAIL_NOT_PENDING'
+                }));
+                return;
+            }
+
+            await emailModel.save({status: 'submitting'}, Object.assign({}, knexOptions, {patch: true}));
+            hasSingleAccess = true;
+        });
+
+        if (!hasSingleAccess || !emailModel) {
+            return;
+        }
+
         // Create email batch and recipient rows unless this is a retry and they already exist
         const existingBatchCount = await emailModel.related('emailBatches').count('id');
 
         if (existingBatchCount === 0) {
+            logging.info('[sendEmailJob] Creating new batches for ' + emailId);
             let newBatchCount = 0;
 
             await models.Base.transaction(async (transacting) => {
@@ -316,15 +360,23 @@ async function sendEmailJob({emailModel, options}) {
             });
 
             if (newBatchCount === 0) {
+                logging.info('[sendEmailJob] No batches created for ' + emailId);
+                await emailModel.save({status: 'submitted'}, {patch: true});
                 return;
             }
         }
 
         debug('sendEmailJob: sending email');
         startEmailSend = Date.now();
-        await bulkEmailService.processEmail({emailId: emailModel.get('id'), options});
+        await bulkEmailService.processEmail({emailModel, options});
         debug(`sendEmailJob: sent email (${Date.now() - startEmailSend}ms)`);
     } catch (error) {
+        if (startEmailSend) {
+            logging.info(`[sendEmailJob] Failed sending ${emailId} (${Date.now() - startEmailSend}ms)`);
+        } else {
+            logging.info(`[sendEmailJob] Failed sending ${emailId}`);
+        }
+
         if (startEmailSend) {
             debug(`sendEmailJob: send email failed (${Date.now() - startEmailSend}ms)`);
         }
@@ -334,10 +386,10 @@ async function sendEmailJob({emailModel, options}) {
             errorMessage = errorMessage.substring(0, 2000);
         }
 
-        await emailModel.save({
+        await models.Email.edit({
             status: 'failed',
             error: errorMessage
-        }, {patch: true});
+        }, {id: emailId});
 
         throw new errors.InternalServerError({
             err: error,
@@ -511,9 +563,16 @@ async function createEmailBatches({emailModel, memberRows, memberSegment, option
 
     debug('createEmailBatches: storing recipient list');
     const startOfRecipientStorage = Date.now();
-    const batches = _.chunk(memberRows, bulkEmailService.BATCH_SIZE);
+    const emails = memberRows.map(row => row.email);
+    const emailSuppressionData = await emailSuppressionList.getBulkSuppressionData(emails);
+    const emailSuppressedLookup = _.zipObject(emails, emailSuppressionData);
+    const filteredRows = memberRows.filter((row) => {
+        return emailSuppressedLookup[row.email].suppressed === false;
+    });
+    const batches = _.chunk(filteredRows, bulkEmailService.BATCH_SIZE);
     const batchIds = await Promise.mapSeries(batches, storeRecipientBatch);
     debug(`createEmailBatches: stored recipient list (${Date.now() - startOfRecipientStorage}ms)`);
+    logging.info(`[createEmailBatches] stored recipient list (${Date.now() - startOfRecipientStorage}ms)`);
 
     return batchIds;
 }

package/core/server/services/mega/segment-parser.js

@@ -1,5 +1,3 @@
-const labs = require('../../../shared/labs');
-
 const getSegmentsFromHtml = (html) => {
     const cheerio = require('cheerio');
     const $ = cheerio.load(html);
@@ -11,7 +9,7 @@ const getSegmentsFromHtml = (html) => {
     /**
      * Always add free and paid segments if email has paywall card
      */
-    if (labs.isSet('newsletterPaywall') && html.indexOf('<!--members-only-->') !== -1) {
+    if (html.indexOf('<!--members-only-->') !== -1) {
         allSegments = allSegments.concat(['status:free', 'status:-free']);
     }
 

package/core/server/services/members/api.js

@@ -16,6 +16,7 @@ const offersService = require('../offers');
 const tiersService = require('../tiers');
 const newslettersService = require('../newsletters');
 const memberAttributionService = require('../member-attribution');
+const emailSuppressionList = require('../email-suppression-list');
 
 const MAGIC_LINK_TOKEN_VALIDITY = 24 * 60 * 60 * 1000;
 
@@ -177,6 +178,7 @@ function createApiInstance(config) {
             StripeCustomer: models.MemberStripeCustomer,
             StripeCustomerSubscription: models.StripeCustomerSubscription,
             Member: models.Member,
+            MemberNewsletter: models.MemberNewsletter,
             MemberCancelEvent: models.MemberCancelEvent,
             MemberSubscribeEvent: models.MemberSubscribeEvent,
             MemberPaidSubscriptionEvent: models.MemberPaidSubscriptionEvent,
@@ -185,7 +187,6 @@ function createApiInstance(config) {
             MemberPaymentEvent: models.MemberPaymentEvent,
             MemberStatusEvent: models.MemberStatusEvent,
             MemberProductEvent: models.MemberProductEvent,
-            MemberAnalyticEvent: models.MemberAnalyticEvent,
             MemberCreatedEvent: models.MemberCreatedEvent,
             SubscriptionCreatedEvent: models.SubscriptionCreatedEvent,
             MemberLinkClickEvent: models.MemberClickEvent,
@@ -203,7 +204,8 @@ function createApiInstance(config) {
         offersAPI: offersService.api,
         labsService: labsService,
         newslettersService: newslettersService,
-        memberAttributionService: memberAttributionService.service
+        memberAttributionService: memberAttributionService.service,
+        emailSuppressionList
     });
 
     return membersApiInstance;

package/core/server/services/members/middleware.js

@@ -1,6 +1,7 @@
 const _ = require('lodash');
 const logging = require('@tryghost/logging');
 const membersService = require('./service');
+const emailSuppressionList = require('../email-suppression-list');
 const models = require('../../models');
 const urlUtils = require('../../../shared/url-utils');
 const spamPrevention = require('../../web/shared/middleware/api/spam-prevention');
@@ -29,7 +30,7 @@ const loadMemberSession = async function (req, res, next) {
 
 /**
  * Require member authentication, and make it possible to authenticate via uuid.
- * You can chain this after loadMemberSession to make it possible to authetnicate via both the uuid and the session.
+ * You can chain this after loadMemberSession to make it possible to authenticate via both the uuid and the session.
  */
 const authMemberByUuid = async function (req, res, next) {
     try {
@@ -39,7 +40,7 @@ const authMemberByUuid = async function (req, res, next) {
                 // Already authenticated via session
                 return next();
             }
-            
+
             throw new errors.UnauthorizedError({
                 messsage: tpl(messages.missingUuid)
             });
@@ -97,6 +98,20 @@ const getMemberData = async function (req, res) {
     }
 };
 
+const deleteSuppression = async function (req, res) {
+    try {
+        const member = await membersService.ssr.getMemberDataFromSession(req, res);
+        await emailSuppressionList.removeEmail(member.email);
+        res.writeHead(204);
+        res.end();
+    } catch (err) {
+        res.writeHead(err.statusCode, {
+            'Content-Type': 'text/plain;charset=UTF-8'
+        });
+        res.end(err.message);
+    }
+};
+
 const getMemberNewsletters = async function (req, res) {
     try {
         const memberUuid = req.query.uuid;
@@ -262,5 +277,6 @@ module.exports = {
     getMemberData,
     updateMemberData,
     updateMemberNewsletters,
-    deleteSession
+    deleteSession,
+    deleteSuppression
 };

package/core/server/services/members/service.js

@@ -56,7 +56,7 @@ const membersImporter = new MembersCSVImporter({
         return tiersService.api.readDefaultTier();
     },
     sendEmail: ghostMailer.send.bind(ghostMailer),
-    isSet: labsService.isSet.bind(labsService),
+    isSet: flag => labsService.isSet(flag),
     addJob: jobsService.addJob.bind(jobsService),
     knex: db.knex,
     urlFor: urlUtils.urlFor.bind(urlUtils),
@@ -74,6 +74,36 @@ const processImport = async (options) => {
     return result;
 };
 
+const updateVerificationTrigger = () => {
+    verificationTrigger = new VerificationTrigger({
+        apiTriggerThreshold: _.get(config.get('hostSettings'), 'emailVerification.apiThreshold'),
+        adminTriggerThreshold: _.get(config.get('hostSettings'), 'emailVerification.adminThreshold'),
+        importTriggerThreshold: _.get(config.get('hostSettings'), 'emailVerification.importThreshold'),
+        isVerified: () => config.get('hostSettings:emailVerification:verified') === true,
+        isVerificationRequired: () => settingsCache.get('email_verification_required') === true,
+        sendVerificationEmail: async ({subject, message, amountTriggered}) => {
+            const escalationAddress = config.get('hostSettings:emailVerification:escalationAddress');
+            const fromAddress = config.get('user_email');
+
+            if (escalationAddress) {
+                await ghostMailer.send({
+                    subject,
+                    html: tpl(message, {
+                        amountTriggered: amountTriggered,
+                        siteUrl: urlUtils.getSiteUrl()
+                    }),
+                    forceTextContent: true,
+                    from: fromAddress,
+                    to: escalationAddress
+                });
+            }
+        },
+        membersStats,
+        Settings: models.Settings,
+        eventRepository: membersApi.events
+    });
+};
+
 module.exports = {
     async init() {
         const stripeService = require('../stripe');
@@ -110,33 +140,7 @@ module.exports = {
             getMembersApi: () => module.exports.api
         });
 
-        verificationTrigger = new VerificationTrigger({
-            apiTriggerThreshold: _.get(config.get('hostSettings'), 'emailVerification.apiThreshold'),
-            adminTriggerThreshold: _.get(config.get('hostSettings'), 'emailVerification.adminThreshold'),
-            importTriggerThreshold: _.get(config.get('hostSettings'), 'emailVerification.importThreshold'),
-            isVerified: () => config.get('hostSettings:emailVerification:verified') === true,
-            isVerificationRequired: () => settingsCache.get('email_verification_required') === true,
-            sendVerificationEmail: ({subject, message, amountTriggered}) => {
-                const escalationAddress = config.get('hostSettings:emailVerification:escalationAddress');
-                const fromAddress = config.get('user_email');
-
-                if (escalationAddress) {
-                    ghostMailer.send({
-                        subject,
-                        html: tpl(message, {
-                            amountTriggered: amountTriggered,
-                            siteUrl: urlUtils.getSiteUrl()
-                        }),
-                        forceTextContent: true,
-                        from: fromAddress,
-                        to: escalationAddress
-                    });
-                }
-            },
-            membersStats,
-            Settings: models.Settings,
-            eventRepository: membersApi.events
-        });
+        updateVerificationTrigger();
 
         (async () => {
             try {
@@ -185,7 +189,10 @@ module.exports = {
     processImport: processImport,
 
     stats: membersStats,
-    export: require('./exporter/query')
+    export: require('./exporter/query'),
+
+    // Only for tests
+    _updateVerificationTrigger: updateVerificationTrigger
 };
 
 module.exports.middleware = require('./middleware');

package/core/server/services/members/utils.js

@@ -1,3 +1,5 @@
+const labsService = require('../../../shared/labs');
+
 function formatNewsletterResponse(newsletters) {
     return newsletters.map(({id, name, description, sort_order: sortOrder}) => {
         return {id, name, description, sort_order: sortOrder};
@@ -23,5 +25,10 @@ module.exports.formattedMemberResponse = function formattedMemberResponse(member
     if (member.newsletters) {
         data.newsletters = formatNewsletterResponse(member.newsletters);
     }
+
+    if (labsService.isSet('suppressionList') && member.email_suppression) {
+        data.email_suppression = member.email_suppression;
+    }
+
     return data;
 };

package/core/server/services/posts/posts-service.js

@@ -8,12 +8,13 @@ const messages = {
 };
 
 class PostsService {
-    constructor({mega, urlUtils, models, isSet, stats}) {
+    constructor({mega, urlUtils, models, isSet, stats, emailService}) {
         this.mega = mega;
         this.urlUtils = urlUtils;
         this.models = models;
         this.isSet = isSet;
         this.stats = stats;
+        this.emailService = emailService;
     }
 
     async editPost(frame) {
@@ -41,12 +42,22 @@ class PostsService {
 
             if (sendEmail) {
                 let postEmail = model.relations.email;
+                let email;
 
                 if (!postEmail) {
-                    const email = await this.mega.addEmail(model, frame.options);
-                    model.set('email', email);
+                    if (this.isSet('emailStability')) {
+                        email = await this.emailService.createEmail(model);
+                    } else {
+                        email = await this.mega.addEmail(model, frame.options);
+                    }
                 } else if (postEmail && postEmail.get('status') === 'failed') {
-                    const email = await this.mega.retryFailedEmail(postEmail);
+                    if (this.isSet('emailStability')) {
+                        email = await this.emailService.retryEmail(postEmail);
+                    } else {
+                        email = await this.mega.retryFailedEmail(postEmail);
+                    }
+                }
+                if (email) {
                     model.set('email', email);
                 }
             }
@@ -123,6 +134,7 @@ const getPostServiceInstance = () => {
     const labs = require('../../../shared/labs');
     const models = require('../../models');
     const PostStats = require('./stats/post-stats');
+    const emailService = require('../email-service');
 
     const postStats = new PostStats();
 
@@ -130,8 +142,9 @@ const getPostServiceInstance = () => {
         mega: mega,
         urlUtils: urlUtils,
         models: models,
-        isSet: labs.isSet.bind(labs),
-        stats: postStats
+        isSet: flag => labs.isSet(flag), // don't use bind, that breaks test subbing of labs
+        stats: postStats,
+        emailService: emailService.service
     });
 };
 

package/core/server/web/members/app.js

@@ -45,35 +45,37 @@ module.exports = function setupMembersApp() {
     membersApp.put('/api/member', bodyParser.json({limit: '50mb'}), middleware.updateMemberData);
     membersApp.post('/api/member/email', bodyParser.json({limit: '50mb'}), (req, res) => membersService.api.middleware.updateEmailAddress(req, res));
 
+    // Remove email from suppression list
+    membersApp.delete('/api/member/suppression', labs.enabledMiddleware('suppressionList'), middleware.deleteSuppression);
+
     // Manage session
     membersApp.get('/api/session', middleware.getIdentityToken);
     membersApp.delete('/api/session', middleware.deleteSession);
 
     // NOTE: this is wrapped in a function to ensure we always go via the getter
     membersApp.post(
-        '/api/send-magic-link', 
-        bodyParser.json(), 
+        '/api/send-magic-link',
+        bodyParser.json(),
         // Prevent brute forcing email addresses (user enumeration)
-        shared.middleware.brute.membersAuthEnumeration, 
+        shared.middleware.brute.membersAuthEnumeration,
         // Prevent brute forcing passwords for the same email address
-        shared.middleware.brute.membersAuth, 
+        shared.middleware.brute.membersAuth,
         (req, res, next) => membersService.api.middleware.sendMagicLink(req, res, next)
     );
     membersApp.post('/api/create-stripe-checkout-session', (req, res, next) => membersService.api.middleware.createCheckoutSession(req, res, next));
     membersApp.post('/api/create-stripe-update-session', (req, res, next) => membersService.api.middleware.createCheckoutSetupSession(req, res, next));
     membersApp.put('/api/subscriptions/:id', (req, res, next) => membersService.api.middleware.updateSubscription(req, res, next));
-    membersApp.post('/api/events', labs.enabledMiddleware('membersActivity'), middleware.loadMemberSession, (req, res, next) => membersService.api.middleware.createEvents(req, res, next));
 
     // Comments
     membersApp.use('/api/comments', commentRouter());
 
     // Feedback
     membersApp.post(
-        '/api/feedback', 
-        labs.enabledMiddleware('audienceFeedback'), 
-        bodyParser.json({limit: '50mb'}), 
-        middleware.loadMemberSession, 
-        middleware.authMemberByUuid, 
+        '/api/feedback',
+        labs.enabledMiddleware('audienceFeedback'),
+        bodyParser.json({limit: '50mb'}),
+        middleware.loadMemberSession,
+        middleware.authMemberByUuid,
         http(api.feedbackMembers.add)
     );
 

package/core/shared/config/defaults.json

@@ -161,7 +161,7 @@
     },
     "portal": {
         "url": "https://cdn.jsdelivr.net/ghost/portal@~{version}/umd/portal.min.js",
-        "version": "2.19"
+        "version": "2.20"
     },
     "sodoSearch": {
         "url": "https://cdn.jsdelivr.net/ghost/sodo-search@~{version}/umd/sodo-search.min.js",

package/core/shared/labs.js

@@ -15,12 +15,6 @@ const messages = {
 
 // flags in this list always return `true`, allows quick global enable prior to full flag removal
 const GA_FEATURES = [
-    'newsletterPaywall',
-    'freeTrial',
-    'compExpiring',
-    'searchHelper',
-    'emailAlerts',
-    'fixNewsletterLinks',
     'sourceAttribution',
     'memberAttribution',
     'audienceFeedback'
@@ -35,7 +29,9 @@ const BETA_FEATURES = [
 const ALPHA_FEATURES = [
     'urlCache',
     'beforeAfterCard',
-    'lexicalEditor'
+    'lexicalEditor',
+    'suppressionList',
+    'emailStability'
 ];
 
 module.exports.GA_KEYS = [...GA_FEATURES];

package/core/shared/sentry.js

@@ -4,12 +4,32 @@ const errors = require('@tryghost/errors');
 
 if (sentryConfig && !sentryConfig.disabled) {
     const Sentry = require('@sentry/node');
-    const version = require('../../package.json').version;
+    const version = require('@tryghost/version').full;
     const environment = config.get('env');
     Sentry.init({
         dsn: sentryConfig.dsn,
         release: 'ghost@' + version,
-        environment: environment
+        environment: environment,
+        beforeSend: function (event, hint) {
+            const exception = hint.originalException;
+
+            event.tags = event.tags || {};
+
+            if (errors.utils.isGhostError(exception)) {
+                // Unexpected errors have a generic error message, set it back to context if there is one
+                if (exception.code === 'UNEXPECTED_ERROR' && exception.context !== null) {
+                    event.exception.values[0].type = exception.context;
+                }
+
+                // This is a Ghost Error, copy all our extra data to tags
+                event.tags.type = exception.errorType;
+                event.tags.code = exception.code;
+                event.tags.id = exception.id;
+                event.tags.statusCode = exception.statusCode;
+            }
+
+            return event;
+        }
     });
 
     module.exports = {
@@ -34,9 +54,11 @@ if (sentryConfig && !sentryConfig.disabled) {
         next();
     };
 
+    const noop = () => {};
+
     module.exports = {
         requestHandler: expressNoop,
         errorHandler: expressNoop,
-        captureException: () => {}
+        captureException: noop
     };
 }

package/ghost.js

@@ -19,6 +19,7 @@ switch (mode) {
 case 'repl':
 case 'timetravel':
 case 'generate-data':
+case 'record-test':
     command.run(mode);
     break;
 default: