ghost 5.119.3 → 5.120.0

package/core/server/services/email-service/EmailSegmenter.js

@@ -0,0 +1,74 @@
+const tpl = require('@tryghost/tpl');
+const errors = require('@tryghost/errors');
+
+const messages = {
+    noneFilterError: 'Cannot send email to "none" recipient filter',
+    newsletterVisibilityError: 'Unexpected visibility value "{value}". Use one of the valid: "members", "paid".'
+};
+
+/**
+ * @typedef {object} MembersRepository
+ * @prop {(options) => Promise<any>} list
+ */
+
+class EmailSegmenter {
+    #membersRepository;
+
+    /**
+     *
+     * @param {object} dependencies
+     * @param {MembersRepository} dependencies.membersRepository
+     */
+    constructor({
+        membersRepository
+    }) {
+        this.#membersRepository = membersRepository;
+    }
+
+    getMemberFilterForSegment(newsletter, emailRecipientFilter, segment) {
+        const filter = [`newsletters.id:'${newsletter.id}'`, 'email_disabled:0'];
+
+        switch (emailRecipientFilter) {
+        case 'all':
+            break;
+        case 'none':
+            throw new errors.InternalServerError({
+                message: tpl(messages.noneFilterError)
+            });
+        default:
+            filter.push(`(${emailRecipientFilter})`);
+            break;
+        }
+
+        const visibility = newsletter.get('visibility');
+        switch (visibility) {
+        case 'members':
+            // No need to add a member status filter as the email is available to all members
+            break;
+        case 'paid':
+            filter.push(`status:-free`);
+            break;
+        default:
+            throw new errors.InternalServerError({
+                message: tpl(messages.newsletterVisibilityError, {
+                    value: visibility
+                })
+            });
+        }
+
+        if (segment) {
+            filter.push(`(${segment})`);
+        }
+
+        return filter.join('+');
+    }
+
+    async getMembersCount(newsletter, emailRecipientFilter, segment) {
+        const filter = this.getMemberFilterForSegment(newsletter, emailRecipientFilter, segment);
+        const {meta: {pagination: {total: membersCount}}} = await this.#membersRepository.list({filter});
+
+        return membersCount;
+    }
+}
+
+module.exports = EmailSegmenter;

package/core/server/services/email-service/EmailService.js

@@ -0,0 +1,310 @@
+/* eslint-disable no-unused-vars */
+
+/**
+ * @typedef {object} Post
+ * @typedef {object} Email
+ * @typedef {object} LimitService
+ * @typedef {{checkVerificationRequired(): Promise<boolean>}} VerificationTrigger
+ */
+
+const BatchSendingService = require('./BatchSendingService');
+const errors = require('@tryghost/errors');
+const tpl = require('@tryghost/tpl');
+const EmailRenderer = require('./EmailRenderer');
+const EmailSegmenter = require('./EmailSegmenter');
+const SendingService = require('./SendingService');
+const logging = require('@tryghost/logging');
+
+const messages = {
+    archivedNewsletterError: 'Cannot send email to archived newsletters',
+    missingNewsletterError: 'The post does not have a newsletter relation',
+    emailSendingDisabled: `Email sending is temporarily disabled because your account is currently in review. You should have an email about this from us already, but you can also reach us any time at support@ghost.org`,
+    retryEmailStatusError: 'Can only retry emails for published posts'
+};
+
+class EmailService {
+    #batchSendingService;
+    #sendingService;
+    #models;
+    #settingsCache;
+    #emailRenderer;
+    #emailSegmenter;
+    #limitService;
+    #membersRepository;
+    #verificationTrigger;
+    #emailAnalyticsJobs;
+
+    /**
+     *
+     * @param {object} dependencies
+     * @param {BatchSendingService} dependencies.batchSendingService
+     * @param {SendingService} dependencies.sendingService
+     * @param {object} dependencies.models
+     * @param {object} dependencies.models.Email
+     * @param {object} dependencies.settingsCache
+     * @param {EmailRenderer} dependencies.emailRenderer
+     * @param {EmailSegmenter} dependencies.emailSegmenter
+     * @param {LimitService} dependencies.limitService
+     * @param {object} dependencies.membersRepository
+     * @param {VerificationTrigger} dependencies.verificationTrigger
+     * @param {object} dependencies.emailAnalyticsJobs
+     */
+    constructor({
+        batchSendingService,
+        sendingService,
+        models,
+        settingsCache,
+        emailRenderer,
+        emailSegmenter,
+        limitService,
+        membersRepository,
+        verificationTrigger,
+        emailAnalyticsJobs
+    }) {
+        this.#batchSendingService = batchSendingService;
+        this.#models = models;
+        this.#settingsCache = settingsCache;
+        this.#emailRenderer = emailRenderer;
+        this.#emailSegmenter = emailSegmenter;
+        this.#limitService = limitService;
+        this.#membersRepository = membersRepository;
+        this.#sendingService = sendingService;
+        this.#verificationTrigger = verificationTrigger;
+        this.#emailAnalyticsJobs = emailAnalyticsJobs;
+    }
+
+    /**
+     * @private
+     */
+    async checkLimits(addedCount = 0) {
+        // Check host limit for allowed member count and throw error if over limit
+        // - do this even if it's a retry so that there's no way around the limit
+        if (this.#limitService.isLimited('members')) {
+            await this.#limitService.errorIfIsOverLimit('members');
+        }
+
+        // Check host limit for disabled emails or going over emails limit
+        if (this.#limitService.isLimited('emails')) {
+            await this.#limitService.errorIfWouldGoOverLimit('emails', {addedCount});
+        }
+
+        // Check if email verification is required
+        if (await this.#verificationTrigger.checkVerificationRequired()) {
+            throw new errors.HostLimitError({
+                message: tpl(messages.emailSendingDisabled)
+            });
+        }
+    }
+
+    /**
+     *
+     * @param {Post} post
+     * @returns {Promise<Email>}
+     */
+    async createEmail(post) {
+        let newsletter = await post.getLazyRelation('newsletter');
+        if (!newsletter) {
+            throw new errors.EmailError({
+                message: tpl(messages.missingNewsletterError)
+            });
+        }
+
+        if (newsletter.get('status') !== 'active') {
+            // A post might have been scheduled to an archived newsletter.
+            // Don't send it (people can't unsubscribe any longer).
+            throw new errors.EmailError({
+                message: tpl(messages.archivedNewsletterError)
+            });
+        }
+
+        const emailRecipientFilter = post.get('email_recipient_filter');
+        const emailCount = await this.#emailSegmenter.getMembersCount(newsletter, emailRecipientFilter);
+        await this.checkLimits(emailCount);
+
+        const email = await this.#models.Email.add({
+            post_id: post.id,
+            newsletter_id: newsletter.id,
+            status: 'pending',
+            submitted_at: new Date(),
+            track_opens: !!this.#settingsCache.get('email_track_opens'),
+            track_clicks: !!this.#settingsCache.get('email_track_clicks'),
+            feedback_enabled: !!newsletter.get('feedback_enabled'),
+            recipient_filter: emailRecipientFilter,
+            subject: this.#emailRenderer.getSubject(post),
+            from: this.#emailRenderer.getFromAddress(post, newsletter),
+            replyTo: this.#emailRenderer.getReplyToAddress(post, newsletter),
+            email_count: emailCount,
+            source: post.get('lexical') || post.get('mobiledoc'),
+            source_type: post.get('lexical') ? 'lexical' : 'mobiledoc'
+        });
+
+        try {
+            this.#batchSendingService.scheduleEmail(email);
+        } catch (e) {
+            await email.save({
+                status: 'failed',
+                error: e.message || 'Something went wrong while scheduling the email'
+            }, {patch: true});
+        }
+
+        // make sure recurring background analytics jobs are running once we have emails
+        try {
+            await this.#emailAnalyticsJobs.scheduleRecurringJobs(true);
+        } catch (e) {
+            logging.error(e);
+        }
+
+        return email;
+    }
+    async retryEmail(email) {
+        // Block accidentaly retrying non-published posts (can happen due to bugs in frontend)
+        const post = await email.getLazyRelation('post');
+        if (post.get('status') !== 'published' && post.get('status') !== 'sent') {
+            throw new errors.IncorrectUsageError({
+                message: tpl(messages.retryEmailStatusError)
+            });
+        }
+
+        await this.checkLimits();
+
+        // Change email status back to 'pending' before scheduling
+        // so we have a immediate response when retrying an email (schedule can take a while to kick off sometimes)
+        if (email.get('status') === 'failed') {
+            await email.save({status: 'pending'}, {patch: true});
+        }
+
+        this.#batchSendingService.scheduleEmail(email);
+        return email;
+    }
+
+    /**
+     * @params {string} [segment]
+     * @return {import('./EmailRenderer').MemberLike}
+     */
+    getDefaultExampleMember(segment) {
+        /**
+         * @type {import('./EmailRenderer').MemberLike}
+         */
+        return {
+            id: 'example-id',
+            uuid: 'example-uuid',
+            email: 'jamie@example.com',
+            name: 'Jamie Larson',
+            createdAt: new Date(),
+            status: segment === 'status:free' ? 'free' : 'paid',
+            subscriptions: segment === 'status:free' ? [] : [
+                {
+                    cancel_at_period_end: false,
+                    trial_end_at: null,
+                    current_period_end: new Date(Date.now() + 1000 * 60 * 60 * 24 * 7),
+                    status: 'active'
+                }
+            ],
+            tiers: []
+        };
+    }
+
+    /**
+     * @private
+     * @param {string} [email] (optional) Search for a member with this email address and use it as the example. If not found, defaults to the default but still uses the provided email address.
+     * @param {string} [segment] (optional) The segment to use for the example member
+     * @return {Promise<import('./EmailRenderer').MemberLike>}
+     */
+    async getExampleMember(email, segment) {
+        /**
+         * @type {import('./EmailRenderer').MemberLike}
+         */
+        const exampleMember = this.getDefaultExampleMember(segment);
+
+        // fetch any matching members so that replacements use expected values
+        if (email) {
+            const member = await this.#membersRepository.get({email});
+            if (member) {
+                exampleMember.id = member.id;
+                exampleMember.uuid = member.get('uuid');
+                exampleMember.email = member.get('email');
+                exampleMember.name = member.get('name');
+                exampleMember.createdAt = member.get('created_at');
+
+                if (segment === 'status:-free' && member.get('status') !== 'free') {
+                    // Make sure the example member matches the chosen segment (otherwise we'll send an email to free segment, but include a paid member details, which looks like a bug)
+                    exampleMember.status = member.get('status');
+                    const subscriptions = (await member.getLazyRelation('stripeSubscriptions')).toJSON();
+                    exampleMember.subscriptions = subscriptions;
+
+                    const tiers = (await member.getLazyRelation('products')).toJSON();
+                    exampleMember.tiers = tiers;
+                }
+            } else {
+                exampleMember.name = ''; // Force empty name to simulate name fallbacks
+                exampleMember.email = email;
+            }
+        }
+
+        return exampleMember;
+    }
+
+    /**
+     * Do a manual replacement of tokens with values for a member (normally only used for previews)
+     *
+     * @param {string} htmlOrPlaintext
+     * @param {import('./EmailRenderer').ReplacementDefinition[]} replacements
+     * @param {import('./EmailRenderer').MemberLike} member
+     * @return {string}
+     */
+    replaceDefinitions(htmlOrPlaintext, replacements, member) {
+        // Do manual replacements with an example member
+        for (const replacement of replacements) {
+            htmlOrPlaintext = htmlOrPlaintext.replace(replacement.token, replacement.getValue(member));
+        }
+        return htmlOrPlaintext;
+    }
+
+    /**
+     *
+     * @param {*} post
+     * @param {*} newsletter
+     * @param {import('./EmailRenderer').Segment} segment
+     * @returns {Promise<{subject: string, html: string, plaintext: string}>} Email preview
+     */
+    async previewEmail(post, newsletter, segment) {
+        const exampleMember = await this.getExampleMember(null, segment);
+
+        const subject = this.#emailRenderer.getSubject(post);
+        let {html, plaintext, replacements} = await this.#emailRenderer.renderBody(post, newsletter, segment, {clickTrackingEnabled: false});
+
+        return {
+            subject,
+            html: this.replaceDefinitions(html, replacements, exampleMember),
+            plaintext: this.replaceDefinitions(plaintext, replacements, exampleMember)
+        };
+    }
+
+    /**
+     *
+     * @param {*} post
+     * @param {*} newsletter
+     * @param {import('./EmailRenderer').Segment} segment
+     * @param {string[]} emails
+     */
+    async sendTestEmail(post, newsletter, segment, emails) {
+        const members = [];
+        for (const email of emails) {
+            members.push(await this.getExampleMember(email, segment));
+        }
+
+        await this.#sendingService.send({
+            post,
+            newsletter,
+            segment,
+            members,
+            emailId: null
+        }, {
+            clickTrackingEnabled: false,
+            openTrackingEnabled: false,
+            isTestEmail: true
+        });
+    }
+}
+
+module.exports = EmailService;

package/core/server/services/email-service/EmailServiceWrapper.js

@@ -15,7 +15,14 @@ class EmailServiceWrapper {
             return;
         }
 
-        const {EmailService, EmailController, EmailRenderer, SendingService, BatchSendingService, EmailSegmenter, MailgunEmailProvider} = require('@tryghost/email-service');
+        const EmailService = require('./EmailService');
+        const EmailController = require('./EmailController');
+        const EmailRenderer = require('./EmailRenderer');
+        const SendingService = require('./SendingService');
+        const BatchSendingService = require('./BatchSendingService');
+        const EmailSegmenter = require('./EmailSegmenter');
+        const MailgunEmailProvider = require('./MailgunEmailProvider');
+
         const {Post, Newsletter, Email, EmailBatch, EmailRecipient, Member} = require('../../models');
         const MailgunClient = require('../lib/MailgunClient');
         const configService = require('../../../shared/config');
@@ -52,7 +59,7 @@ class EmailServiceWrapper {
             config: configService, settings: settingsCache
         });
         const i18nLanguage = labs.isSet('i18n') ? settingsCache.get('locale') || 'en' : 'en';
-        const i18n = i18nLib(i18nLanguage, 'newsletter');
+        const i18n = i18nLib(i18nLanguage, 'ghost');
 
         events.on('settings.labs.edited', () => {
             if (labs.isSet('i18n')) {

package/core/server/services/email-service/MailgunEmailProvider.js

@@ -0,0 +1,191 @@
+const logging = require('@tryghost/logging');
+const errors = require('@tryghost/errors');
+const debug = require('@tryghost/debug')('email-service:mailgun-provider-service');
+
+/**
+ * @typedef {object} Recipient
+ * @prop {string} email
+ * @prop {Replacement[]} replacements
+ */
+
+/**
+ * @typedef {object} Replacement
+ * @prop {string} token
+ * @prop {string} value
+ * @prop {string} id
+ */
+
+/**
+ * @typedef {object} EmailSendingOptions
+ * @prop {boolean} clickTrackingEnabled
+ * @prop {boolean} openTrackingEnabled
+ * @prop {Date} deliveryTime
+ */
+
+/**
+ * @typedef {object} EmailProviderSuccessResponse
+ * @prop {string} id
+ */
+
+class MailgunEmailProvider {
+    #mailgunClient;
+    #errorHandler;
+
+    /**
+     * @param {object} dependencies
+     * @param {import('@tryghost/mailgun-client/lib/MailgunClient')} dependencies.mailgunClient - mailgun client to send emails
+     * @param {Function} [dependencies.errorHandler] - custom error handler for logging exceptions
+     */
+    constructor({
+        mailgunClient,
+        errorHandler
+    }) {
+        this.#mailgunClient = mailgunClient;
+        this.#errorHandler = errorHandler;
+    }
+
+    #createRecipientData(replacements) {
+        let recipientData = {};
+
+        recipientData = replacements.reduce((acc, replacement) => {
+            const {id, value} = replacement;
+            acc[id] = value;
+            return acc;
+        }, {});
+
+        return recipientData;
+    }
+
+    #updateRecipientVariables(data, replacementDefinitions) {
+        for (const def of replacementDefinitions) {
+            data = data.replace(
+                def.token,
+                `%recipient.${def.id}%`
+            );
+        }
+        return data;
+    }
+
+    /**
+     * Create mailgun error message for storing in the database
+     * @param {Object} error
+     * @param {string} error.message
+     * @param {string} error.details
+     * @returns {string}
+    */
+    #createMailgunErrorMessage(error) {
+        const message = (error?.message || 'Mailgun Error') + (error?.details ? (': ' + error.details) : '');
+        return message.slice(0, 2000);
+    }
+
+    /**
+     * Send an email using the Mailgun API
+     * @param {import('./SendingService').EmailData} data
+     * @param {EmailSendingOptions} options
+     * @returns {Promise<EmailProviderSuccessResponse>}
+    */
+    async send(data, options) {
+        const {
+            subject,
+            html,
+            plaintext,
+            from,
+            replyTo,
+            emailId,
+            recipients,
+            replacementDefinitions
+        } = data;
+
+        logging.info(`Sending email to ${recipients.length} recipients`);
+        const startTime = Date.now();
+        debug(`sending message to ${recipients.length} recipients`);
+
+        try {
+            const messageData = {
+                subject,
+                html,
+                plaintext,
+                from,
+                replyTo,
+                id: emailId,
+                track_opens: !!options.openTrackingEnabled,
+                track_clicks: !!options.clickTrackingEnabled
+            };
+
+            if (options.deliveryTime && options.deliveryTime instanceof Date) {
+                messageData.deliveryTime = options.deliveryTime;
+            }
+
+            // create recipient data for Mailgun using replacement definitions
+            const recipientData = recipients.reduce((acc, recipient) => {
+                acc[recipient.email] = this.#createRecipientData(recipient.replacements);
+                return acc;
+            }, {});
+
+            // update content to use Mailgun variable syntax for all replacements
+            ['html', 'plaintext'].forEach((key) => {
+                if (messageData[key]) {
+                    messageData[key] = this.#updateRecipientVariables(messageData[key], replacementDefinitions);
+                }
+            });
+
+            // send the email using Mailgun
+            // uses empty replacements array as we've already replaced all tokens with Mailgun variables
+            const response = await this.#mailgunClient.send(
+                messageData,
+                recipientData,
+                []
+            );
+
+            debug(`sent message (${Date.now() - startTime}ms)`);
+            logging.info(`Sent message (${Date.now() - startTime}ms)`);
+
+            // Return mailgun provider id, trim <> from response
+            return {
+                id: response.id.trim().replace(/^<|>$/g, '')
+            };
+        } catch (e) {
+            let ghostError;
+            if (e.error && e.messageData) {
+                const {error, messageData} = e;
+
+                // REF: possible mailgun errors https://documentation.mailgun.com/en/latest/api-intro.html#status-codes
+                ghostError = new errors.EmailError({
+                    statusCode: error.status,
+                    message: this.#createMailgunErrorMessage(error),
+                    errorDetails: JSON.stringify({error, messageData}),
+                    context: `Mailgun Error ${error.status}: ${error.details}`,
+                    help: `https://ghost.org/docs/newsletters/#bulk-email-configuration`,
+                    code: 'BULK_EMAIL_SEND_FAILED'
+                });
+            } else {
+                ghostError = new errors.EmailError({
+                    statusCode: undefined,
+                    message: this.#createMailgunErrorMessage(e),
+                    errorDetails: undefined,
+                    context: e.context || 'Mailgun Error',
+                    code: 'BULK_EMAIL_SEND_FAILED'
+                });
+            }
+
+            debug(`failed to send message (${Date.now() - startTime}ms)`);
+
+            throw ghostError;
+        }
+    }
+
+    getMaximumRecipients() {
+        return this.#mailgunClient.getBatchSize();
+    }
+
+    /**
+     * Returns the configured delay between batches in milliseconds
+     * 
+     * @returns {number}
+     */
+    getTargetDeliveryWindow() {
+        return this.#mailgunClient.getTargetDeliveryWindow();
+    }
+}
+
+module.exports = MailgunEmailProvider;