ghost 5.119.2 → 5.120.0

package/core/server/services/email-service/EmailBodyCache.js

@@ -0,0 +1,20 @@
+/**
+ * This is a cache provider that lives very short in memory, there is no need for persistence.
+ * It is created when scheduling an email in the batch sending service, and is then passed to the sending service. The sending service
+ * can optionally use a passed cache provider to reuse the email body for each batch with the same segment.
+ */
+class EmailBodyCache {
+    constructor() {
+        this.cache = new Map();
+    }
+
+    get(key) {
+        return this.cache.get(key) ?? null;
+    }
+
+    set(key, value) {
+        this.cache.set(key, value);
+    }
+}
+
+module.exports = EmailBodyCache;

package/core/server/services/email-service/EmailController.js

@@ -0,0 +1,94 @@
+const errors = require('@tryghost/errors');
+const tpl = require('@tryghost/tpl');
+
+const messages = {
+    postNotFound: 'Post not found.',
+    noEmailsProvided: 'No emails provided.',
+    emailNotFound: 'Email not found.',
+    tooManyEmailsProvided: 'Too many emails provided. Maximum of 1 test email can be sent at once.'
+};
+
+class EmailController {
+    service;
+    models;
+
+    /**
+     *
+     * @param {EmailService} service
+     * @param {{models: {Post: any, Newsletter: any, Email: any}}} dependencies
+     */
+    constructor(service, {models}) {
+        this.service = service;
+        this.models = models;
+    }
+
+    async _getFrameData(frame) {
+        // Bit absurd situation in email-previews endpoints that one endpoint is using options and other one is using data.
+        // So we need to handle both cases.
+        let post;
+        if (frame.options.id) {
+            post = await this.models.Post.findOne({...frame.options, status: 'all'}, {withRelated: ['posts_meta', 'authors']});
+        } else {
+            post = await this.models.Post.findOne({...frame.data, status: 'all'}, {...frame.options, withRelated: ['posts_meta', 'authors']});
+        }
+
+        if (!post) {
+            throw new errors.NotFoundError({
+                message: tpl(messages.postNotFound)
+            });
+        }
+
+        let newsletter;
+        const slug = frame?.options?.newsletter ?? frame?.data?.newsletter ?? null;
+        if (slug) {
+            newsletter = await this.models.Newsletter.findOne({slug}, {require: true});
+        } else {
+            newsletter = (await post.getLazyRelation('newsletter')) ?? (await this.models.Newsletter.getDefaultNewsletter());
+        }
+        return {
+            post,
+            newsletter,
+            segment: frame.options.memberSegment ?? frame.data.memberSegment ?? null
+        };
+    }
+
+    async previewEmail(frame) {
+        const {post, newsletter, segment} = await this._getFrameData(frame);
+        return await this.service.previewEmail(post, newsletter, segment);
+    }
+
+    async sendTestEmail(frame) {
+        const {post, newsletter, segment} = await this._getFrameData(frame);
+
+        const emails = frame.data.emails ?? [];
+
+        if (emails.length === 0) {
+            throw new errors.ValidationError({
+                message: tpl(messages.noEmailsProvided)
+            });
+        }
+
+        // test emails are limited to 1
+        if (emails.length > 1) {
+            throw new errors.ValidationError({
+                message: tpl(messages.tooManyEmailsProvided)
+            });
+        }
+
+        await this.service.sendTestEmail(post, newsletter, segment, emails);
+    }
+
+    async retryFailedEmail(frame) {
+        const email = await this.models.Email.findOne(frame.data, {require: false});
+
+        if (!email) {
+            throw new errors.NotFoundError({
+                message: tpl(messages.emailNotFound)
+            });
+        }
+
+        return await this.service.retryEmail(email);
+    }
+}
+
+module.exports = EmailController;

package/core/server/services/email-service/EmailEventProcessor.js

@@ -0,0 +1,267 @@
+const logging = require('@tryghost/logging');
+
+const EmailDeliveredEvent = require('./events/EmailDeliveredEvent');
+const EmailOpenedEvent = require('./events/EmailOpenedEvent');
+const EmailBouncedEvent = require('./events/EmailBouncedEvent');
+const EmailTemporaryBouncedEvent = require('./events/EmailTemporaryBouncedEvent');
+const EmailUnsubscribedEvent = require('./events/EmailUnsubscribedEvent');
+const SpamComplaintEvent = require('./events/SpamComplaintEvent');
+
+async function waitForEvent() {
+    return new Promise((resolve) => {
+        setTimeout(resolve, 70);
+    });
+}
+
+/**
+ * @typedef EmailIdentification
+ * @property {string} email
+ * @property {string} providerId
+ * @property {string} [emailId] Optional email id
+ */
+
+/**
+ * @typedef EmailRecipientInformation
+ * @property {string} emailRecipientId
+ * @property {string} memberId
+ * @property {string} emailId
+ */
+
+/**
+ * @typedef EmailEventStorage
+ * @property {(event: EmailDeliveredEvent) => Promise<void>} handleDelivered
+ * @property {(event: EmailOpenedEvent) => Promise<void>} handleOpened
+ * @property {(event: EmailBouncedEvent) => Promise<void>} handlePermanentFailed
+ * @property {(event: EmailTemporaryBouncedEvent) => Promise<void>} handleTemporaryFailed
+ * @property {(event: EmailUnsubscribedEvent) => Promise<void>} handleUnsubscribed
+ * @property {(event: SpamComplaintEvent) => Promise<void>} handleComplained
+ */
+
+/**
+ * WARNING: this class is used in a separate thread (an offloaded job). Be careful when working with settings and models.
+ */
+class EmailEventProcessor {
+    #domainEvents;
+    #db;
+    #eventStorage;
+    #prometheusClient;
+    constructor({domainEvents, db, eventStorage, prometheusClient}) {
+        this.#domainEvents = domainEvents;
+        this.#db = db;
+        this.#eventStorage = eventStorage;
+        this.#prometheusClient = prometheusClient;
+        // Avoid having to query email_batch by provider_id for every event
+        this.providerIdEmailIdMap = {};
+
+        if (this.#prometheusClient) {
+            this.#prometheusClient.registerCounter({
+                name: 'email_analytics_events_processed',
+                help: 'Number of email analytics events processed',
+                labelNames: ['event']
+            });
+        }
+    }
+
+    /**
+     * @param {EmailIdentification} emailIdentification
+     * @param {Date} timestamp
+     */
+    async handleDelivered(emailIdentification, timestamp) {
+        const recipient = await this.getRecipient(emailIdentification);
+        if (recipient) {
+            const event = EmailDeliveredEvent.create({
+                email: emailIdentification.email,
+                emailRecipientId: recipient.emailRecipientId,
+                memberId: recipient.memberId,
+                emailId: recipient.emailId,
+                timestamp
+            });
+            await this.#eventStorage.handleDelivered(event);
+
+            this.#domainEvents.dispatch(event);
+            this.recordEventProcessed('delivered');
+        }
+        return recipient;
+    }
+
+    /**
+     * @param {EmailIdentification} emailIdentification
+     * @param {Date} timestamp
+     */
+    async handleOpened(emailIdentification, timestamp) {
+        const recipient = await this.getRecipient(emailIdentification);
+        if (recipient) {
+            const event = EmailOpenedEvent.create({
+                email: emailIdentification.email,
+                emailRecipientId: recipient.emailRecipientId,
+                memberId: recipient.memberId,
+                emailId: recipient.emailId,
+                timestamp
+            });
+            this.#domainEvents.dispatch(event);
+            await this.#eventStorage.handleOpened(event);
+            this.recordEventProcessed('opened');
+        }
+        return recipient;
+    }
+
+    /**
+     * @param {EmailIdentification} emailIdentification
+     * @param {{id: string, timestamp: Date, error: {code: number; message: string; enhandedCode: string|number} | null}} event
+     */
+    async handleTemporaryFailed(emailIdentification, {timestamp, error, id}) {
+        const recipient = await this.getRecipient(emailIdentification);
+        if (recipient) {
+            const event = EmailTemporaryBouncedEvent.create({
+                id,
+                error,
+                email: emailIdentification.email,
+                memberId: recipient.memberId,
+                emailId: recipient.emailId,
+                emailRecipientId: recipient.emailRecipientId,
+                timestamp
+            });
+            await this.#eventStorage.handleTemporaryFailed(event);
+
+            this.#domainEvents.dispatch(event);
+        }
+        return recipient;
+    }
+
+    /**
+     * @param {EmailIdentification} emailIdentification
+     * @param {{id: string, timestamp: Date, error: {code: number; message: string; enhandedCode: string|number} | null}} event
+     */
+    async handlePermanentFailed(emailIdentification, {timestamp, error, id}) {
+        const recipient = await this.getRecipient(emailIdentification);
+        if (recipient) {
+            const event = EmailBouncedEvent.create({
+                id,
+                error,
+                email: emailIdentification.email,
+                memberId: recipient.memberId,
+                emailId: recipient.emailId,
+                emailRecipientId: recipient.emailRecipientId,
+                timestamp
+            });
+            await this.#eventStorage.handlePermanentFailed(event);
+
+            this.#domainEvents.dispatch(event);
+            await waitForEvent(); // Avoids knex connection pool to run dry
+        }
+        return recipient;
+    }
+
+    /**
+     * @param {EmailIdentification} emailIdentification
+     * @param {Date} timestamp
+     */
+    async handleUnsubscribed(emailIdentification, timestamp) {
+        const recipient = await this.getRecipient(emailIdentification);
+        if (recipient) {
+            const event = EmailUnsubscribedEvent.create({
+                email: emailIdentification.email,
+                memberId: recipient.memberId,
+                emailId: recipient.emailId,
+                timestamp
+            });
+            await this.#eventStorage.handleUnsubscribed(event);
+
+            this.#domainEvents.dispatch(event);
+        }
+        return recipient;
+    }
+
+    /**
+     * @param {EmailIdentification} emailIdentification
+     * @param {Date} timestamp
+     */
+    async handleComplained(emailIdentification, timestamp) {
+        const recipient = await this.getRecipient(emailIdentification);
+        if (recipient) {
+            const event = SpamComplaintEvent.create({
+                email: emailIdentification.email,
+                memberId: recipient.memberId,
+                emailId: recipient.emailId,
+                timestamp
+            });
+            await this.#eventStorage.handleComplained(event);
+
+            this.#domainEvents.dispatch(event);
+            await waitForEvent(); // Avoids knex connection pool to run dry
+        }
+        return recipient;
+    }
+
+    /**
+     * @private
+     * @param {EmailIdentification} emailIdentification
+     * @returns {Promise<EmailRecipientInformation|undefined>}
+     */
+    async getRecipient(emailIdentification) {
+        if (!emailIdentification.emailId && !emailIdentification.providerId) {
+            // Protection if both are null or undefined
+            return;
+        }
+
+        // With the provider_id and email address we can look for the EmailRecipient
+        const emailId = emailIdentification.emailId ?? await this.getEmailId(emailIdentification.providerId);
+        if (!emailId) {
+            // Invalid
+            return;
+        }
+
+        const {id: emailRecipientId, member_id: memberId} = await this.#db.knex('email_recipients')
+            .select('id', 'member_id')
+            .where('member_email', emailIdentification.email)
+            .where('email_id', emailId)
+            .first() || {};
+
+        if (emailRecipientId && memberId) {
+            return {
+                emailRecipientId,
+                memberId,
+                emailId
+            };
+        }
+    }
+
+    /**
+     * Record event processed
+     * @param {string} event
+     */
+    recordEventProcessed(event) {
+        try {
+            if (this.#prometheusClient) {
+                this.#prometheusClient.getMetric('email_analytics_events_processed')?.inc({event});
+            }
+        } catch (err) {
+            logging.error('Error recording email analytics event processed', err);
+        }
+    }
+
+    /**
+     * @private
+     * @param {string} providerId
+     * @returns {Promise<string|undefined>}
+     */
+    async getEmailId(providerId) {
+        if (this.providerIdEmailIdMap[providerId]) {
+            return this.providerIdEmailIdMap[providerId];
+        }
+
+        const {emailId} = await this.#db.knex('email_batches')
+            .select('email_id as emailId')
+            .where('provider_id', providerId)
+            .first() || {};
+
+        if (!emailId) {
+            return;
+        }
+
+        this.providerIdEmailIdMap[providerId] = emailId;
+        return emailId;
+    }
+}
+
+module.exports = EmailEventProcessor;

package/core/server/services/email-service/EmailEventStorage.js

@@ -0,0 +1,187 @@
+const moment = require('moment-timezone');
+const logging = require('@tryghost/logging');
+
+class EmailEventStorage {
+    #db;
+    #membersRepository;
+    #models;
+    #emailSuppressionList;
+    #prometheusClient;
+
+    constructor({db, models, membersRepository, emailSuppressionList, prometheusClient}) {
+        this.#db = db;
+        this.#models = models;
+        this.#membersRepository = membersRepository;
+        this.#emailSuppressionList = emailSuppressionList;
+        this.#prometheusClient = prometheusClient;
+
+        if (this.#prometheusClient) {
+            this.#prometheusClient.registerCounter({
+                name: 'email_analytics_events_stored',
+                help: 'Number of email analytics events stored',
+                labelNames: ['event']
+            });
+        }
+    }
+
+    async handleDelivered(event) {
+        // To properly handle events that are received out of order (this happens because of polling)
+        // only set if delivered_at is null
+        const rowCount = await this.#db.knex('email_recipients')
+            .where('id', '=', event.emailRecipientId)
+            .whereNull('delivered_at')
+            .update({
+                delivered_at: moment.utc(event.timestamp).format('YYYY-MM-DD HH:mm:ss')
+            });
+        this.recordEventStored('delivered', rowCount);
+    }
+
+    async handleOpened(event) {
+        // To properly handle events that are received out of order (this happens because of polling)
+        // only set if opened_at is null
+        const rowCount = await this.#db.knex('email_recipients')
+            .where('id', '=', event.emailRecipientId)
+            .whereNull('opened_at')
+            .update({
+                opened_at: moment.utc(event.timestamp).format('YYYY-MM-DD HH:mm:ss')
+            });
+        this.recordEventStored('opened', rowCount);
+    }
+
+    async handlePermanentFailed(event) {
+        // To properly handle events that are received out of order (this happens because of polling)
+        // only set if failed_at is null
+        await this.#db.knex('email_recipients')
+            .where('id', '=', event.emailRecipientId)
+            .whereNull('failed_at')
+            .update({
+                failed_at: moment.utc(event.timestamp).format('YYYY-MM-DD HH:mm:ss')
+            });
+        await this.saveFailure('permanent', event);
+    }
+
+    async handleTemporaryFailed(event) {
+        await this.saveFailure('temporary', event);
+    }
+
+    /**
+     * @private
+     * @param {'temporary'|'permanent'} severity
+     * @param {import('./events/EmailTemporaryBouncedEvent')|import('./events/EmailBouncedEvent')} event
+     * @param {{transacting?: any}} options
+     * @returns
+     */
+    async saveFailure(severity, event, options = {}) {
+        if (!event.error) {
+            logging.warn(`Missing error information provided for ${severity} failure event with id ${event.id}`);
+            return;
+        }
+
+        if (!options || !options.transacting) {
+            return await this.#models.EmailRecipientFailure.transaction(async (transacting) => {
+                await this.saveFailure(severity, event, {transacting});
+            });
+        }
+
+        // Create a forUpdate transaction
+        const existing = await this.#models.EmailRecipientFailure.findOne({
+            email_recipient_id: event.emailRecipientId
+        }, {...options, require: false, forUpdate: true});
+
+        if (!existing) {
+            // Create a new failure
+            await this.#models.EmailRecipientFailure.add({
+                email_id: event.emailId,
+                member_id: event.memberId,
+                email_recipient_id: event.emailRecipientId,
+                severity,
+                message: event.error.message || `Error ${event.error.enhancedCode ?? event.error.code}`,
+                code: event.error.code,
+                enhanced_code: event.error.enhancedCode,
+                failed_at: event.timestamp,
+                event_id: event.id
+            }, {...options, autoRefresh: false});
+        } else {
+            if (existing.get('severity') === 'permanent') {
+                // Already marked as failed, no need to change anything here
+                return;
+            }
+
+            if (existing.get('failed_at') > event.timestamp) {
+                /// We can get events out of order, so only save the last one
+                return;
+            }
+
+            // Update the existing failure
+            await existing.save({
+                severity,
+                message: event.error.message || `Error ${event.error.enhancedCode ?? event.error.code}`,
+                code: event.error.code,
+                enhanced_code: event.error.enhancedCode ?? null,
+                failed_at: event.timestamp,
+                event_id: event.id
+            }, {...options, patch: true, autoRefresh: false});
+        }
+    }
+
+    async handleUnsubscribed(event) {
+        try {
+            // Unsubscribe member from the specific newsletter
+            const newsletters = await this.findNewslettersToKeep(event);
+            await this.#membersRepository.update({newsletters}, {id: event.memberId});
+
+            // Remove member from Mailgun's suppression list
+            await this.#emailSuppressionList.removeUnsubscribe(event.email);
+        } catch (err) {
+            logging.error(err);
+        }
+    }
+
+    async handleComplained(event) {
+        try {
+            await this.#models.EmailSpamComplaintEvent.add({
+                member_id: event.memberId,
+                email_id: event.emailId,
+                email_address: event.email
+            });
+        } catch (err) {
+            if (err.code !== 'ER_DUP_ENTRY' && err.code !== 'SQLITE_CONSTRAINT') {
+                logging.error(err);
+            }
+        }
+    }
+
+    async findNewslettersToKeep(event) {
+        try {
+            const member = await this.#membersRepository.get({email: event.email}, {
+                withRelated: ['newsletters']
+            });
+            const existingNewsletters = member.related('newsletters');
+
+            const email = await this.#models.Email.findOne({id: event.emailId});
+            const newsletterToRemove = email.get('newsletter_id');
+
+            return existingNewsletters.models.filter(newsletter => newsletter.id !== newsletterToRemove).map((n) => {
+                return {id: n.id};
+            });
+        } catch (err) {
+            logging.error(err);
+            return [];
+        }
+    }
+
+    /**
+     * Record event stored
+     * @param {string} event
+     * @param {number} count
+     */
+    recordEventStored(event, count = 1) {
+        try {
+            this.#prometheusClient?.getMetric('email_analytics_events_stored')?.inc({event}, count);
+        } catch (err) {
+            logging.error('Error recording email analytics event stored', err);
+        }
+    }
+}
+
+module.exports = EmailEventStorage;