ghost 5.115.0 → 5.116.0

package/core/server/services/lib/MailgunClient.js

@@ -0,0 +1,364 @@
+const _ = require('lodash');
+const debug = require('@tryghost/debug');
+const logging = require('@tryghost/logging');
+const metrics = require('@tryghost/metrics');
+const errors = require('@tryghost/errors');
+
+module.exports = class MailgunClient {
+    #config;
+    #settings;
+
+    static DEFAULT_BATCH_SIZE = 1000;
+
+    constructor({config, settings}) {
+        this.#config = config;
+        this.#settings = settings;
+    }
+
+    /**
+     * Creates the data payload and sends to Mailgun
+     *
+     * @param {Object} message
+     * @param {Object} recipientData
+     * @param {Array<Object>} replacements
+     *
+     * recipientData format:
+     * {
+     *     'test@example.com': {
+     *         name: 'Test User',
+     *         unsubscribe_url: 'https://example.com/unsub/me',
+     *         list_unsubscribe: 'https://example.com/unsub/me'
+     *     }
+     * }
+     */
+    async send(message, recipientData, replacements) {
+        const mailgunInstance = this.getInstance();
+        if (!mailgunInstance) {
+            logging.warn(`Mailgun is not configured`);
+            return null;
+        }
+
+        const batchSize = this.getBatchSize();
+        if (Object.keys(recipientData).length > batchSize) {
+            throw new errors.IncorrectUsageError({
+                message: `Mailgun only supports sending to ${batchSize} recipients at a time`
+            });
+        }
+
+        let messageData = {};
+
+        let startTime;
+        try {
+            const bulkEmailConfig = this.#config.get('bulkEmail');
+            const messageContent = _.pick(message, 'subject', 'html', 'plaintext');
+
+            // update content to use Mailgun variable syntax for replacements
+            replacements.forEach((replacement) => {
+                messageContent[replacement.format] = messageContent[replacement.format].replace(
+                    replacement.regexp,
+                    `%recipient.${replacement.id}%`
+                );
+            });
+
+            messageData = {
+                to: Object.keys(recipientData),
+                from: message.from,
+                'h:Reply-To': message.replyTo || message.reply_to,
+                subject: messageContent.subject,
+                html: messageContent.html,
+                text: messageContent.plaintext,
+                'recipient-variables': JSON.stringify(recipientData)
+            };
+
+            // Do we have a custom List-Unsubscribe header set?
+            // (we need a variable for this, as this is a per-email setting)
+            if (Object.keys(recipientData)[0] && recipientData[Object.keys(recipientData)[0]].list_unsubscribe) {
+                messageData['h:List-Unsubscribe'] = '<%recipient.list_unsubscribe%>, <%tag_unsubscribe_email%>';
+                messageData['h:List-Unsubscribe-Post'] = 'List-Unsubscribe=One-Click';
+            }
+
+            // add a reference to the original email record for easier mapping of mailgun event -> email
+            if (message.id) {
+                messageData['v:email-id'] = message.id;
+            }
+
+            const tags = ['bulk-email', 'ghost-email'];
+            if (bulkEmailConfig?.mailgun?.tag) {
+                tags.push(bulkEmailConfig.mailgun.tag);
+            }
+            messageData['o:tag'] = tags;
+
+            if (bulkEmailConfig?.mailgun?.testmode) {
+                messageData['o:testmode'] = true;
+            }
+
+            // enable tracking if turned on for this email
+            if (message.track_opens) {
+                messageData['o:tracking-opens'] = true;
+            }
+
+            // set the delivery time if specified
+            if (message.deliveryTime && message.deliveryTime instanceof Date) {
+                messageData['o:deliverytime'] = message.deliveryTime.toUTCString();
+            }
+
+            const mailgunConfig = this.#getConfig();
+            startTime = Date.now();
+            const response = await mailgunInstance.messages.create(mailgunConfig.domain, messageData);
+            metrics.metric('mailgun-send-mail', {
+                value: Date.now() - startTime,
+                statusCode: 200
+            });
+
+            return {
+                id: response.id
+            };
+        } catch (error) {
+            logging.error(error);
+            metrics.metric('mailgun-send-mail', {
+                value: Date.now() - startTime,
+                statusCode: error.status
+            });
+            return Promise.reject({error, messageData});
+        }
+    }
+
+    /**
+     * @param {import('mailgun.js').default} mailgunInstance
+     * @param {Object} mailgunConfig
+     * @param {Object} mailgunOptions
+     */
+    async getEventsFromMailgun(mailgunInstance, mailgunConfig, mailgunOptions) {
+        const startTime = Date.now();
+        try {
+            const page = await mailgunInstance.events.get(mailgunConfig.domain, mailgunOptions);
+            metrics.metric('mailgun-get-events', {
+                value: Date.now() - startTime,
+                statusCode: 200
+            });
+            return page;
+        } catch (error) {
+            metrics.metric('mailgun-get-events', {
+                value: Date.now() - startTime,
+                statusCode: error.status
+            });
+            throw error;
+        }
+    }
+
+    /**
+     * Fetches events from Mailgun
+     * @param {Object} mailgunOptions
+     * @param {Function} batchHandler
+     * @param {Object} options
+     * @param {Number} options.maxEvents Not a strict maximum. We stop fetching after we reached the maximum AND received at least one event after begin (not equal) to prevent deadlocks.
+     * @returns {Promise<void>}
+     */
+    async fetchEvents(mailgunOptions, batchHandler, {maxEvents = Infinity} = {}) {
+        const mailgunInstance = this.getInstance();
+        if (!mailgunInstance) {
+            logging.warn(`Mailgun is not configured`);
+            return;
+        }
+
+        debug(`[MailgunClient fetchEvents]: starting fetching first events page`);
+        const mailgunConfig = this.#getConfig();
+        const startDate = new Date();
+        const overallStartTime = Date.now();
+
+        let batchCount = 0;
+        let totalBatchTime = 0;
+
+        try {
+            let page = await this.getEventsFromMailgun(mailgunInstance, mailgunConfig, mailgunOptions);
+
+            // By limiting the processed events to ones created before this job started we cancel early ready for the next job run.
+            // Avoids chance of events being missed in long job runs due to mailgun's eventual-consistency creating events outside of our 30min sliding re-check window
+            let events = (page?.items?.map(this.normalizeEvent) || []).filter(e => !!e && e.timestamp <= startDate);
+            debug(`[MailgunClient fetchEvents]: finished fetching first page with ${events.length} events`);
+
+            let eventCount = 0;
+            const beginTimestamp = mailgunOptions.begin ? Math.ceil(mailgunOptions.begin * 1000) : undefined; // ceil here if we have rounding errors
+
+            while (events.length !== 0) {
+                const batchStartTime = Date.now();
+                await batchHandler(events);
+                const batchEndTime = Date.now();
+                const batchDuration = batchEndTime - batchStartTime;
+
+                batchCount += 1;
+                totalBatchTime += batchDuration;
+
+                eventCount += events.length;
+
+                if (eventCount >= maxEvents && (!beginTimestamp || !events[events.length - 1].timestamp || (events[events.length - 1].timestamp.getTime() > beginTimestamp))) {
+                    break;
+                }
+
+                const nextPageId = page.pages.next.page;
+                debug(`[MailgunClient fetchEvents]: starting fetching next page ${nextPageId}`);
+                page = await this.getEventsFromMailgun(mailgunInstance, mailgunConfig, {
+                    page: nextPageId,
+                    ...mailgunOptions
+                });
+
+                // We need to cap events at the time we started fetching them (see comment above)
+                events = (page?.items?.map(this.normalizeEvent) || []).filter(e => !!e && e.timestamp <= startDate);
+                debug(`[MailgunClient fetchEvents]: finished fetching next page with ${events.length} events`);
+            }
+
+            const overallEndTime = Date.now();
+            const totalDuration = overallEndTime - overallStartTime;
+            const averageBatchTime = batchCount > 0 ? totalBatchTime / batchCount : 0;
+
+            logging.info(`[MailgunClient fetchEvents]: Processed ${batchCount} batches in ${(totalDuration / 1000).toFixed(2)}s. Average batch time: ${(averageBatchTime / 1000).toFixed(2)}s`);
+        } catch (error) {
+            logging.error(error);
+            throw error;
+        }
+    }
+
+    async removeSuppression(type, email) {
+        if (!this.isConfigured()) {
+            return false;
+        }
+        const instance = this.getInstance();
+        const config = this.#getConfig();
+
+        try {
+            await instance.suppressions.destroy(
+                config.domain,
+                type,
+                email
+            );
+            return true;
+        } catch (err) {
+            logging.error(err);
+            return false;
+        }
+    }
+
+    async removeBounce(email) {
+        return this.removeSuppression('bounces', email);
+    }
+
+    async removeComplaint(email) {
+        return this.removeSuppression('complaints', email);
+    }
+
+    async removeUnsubscribe(email) {
+        return this.removeSuppression('unsubscribes', email);
+    }
+
+    normalizeEvent(event) {
+        const providerId = event?.message?.headers['message-id'];
+
+        if (!providerId && !(event['user-variables'] && event['user-variables']['email-id'])) {
+            logging.error('Received invalid event from Mailgun');
+            logging.error(event);
+            return null;
+        }
+
+        return {
+            id: event.id,
+            type: event.event,
+            severity: event.severity,
+            recipientEmail: event.recipient,
+            emailId: event['user-variables'] && event['user-variables']['email-id'],
+            providerId: providerId,
+            timestamp: new Date(event.timestamp * 1000),
+
+            error: event['delivery-status'] && (typeof (event['delivery-status'].message || event['delivery-status'].description) === 'string') ? {
+                code: event['delivery-status'].code,
+                message: (event['delivery-status'].message || event['delivery-status'].description).substring(0, 2000),
+                enhancedCode: event['delivery-status']['enhanced-code']?.toString()?.substring(0, 50) ?? null
+            } : null
+        };
+    }
+
+    #getConfig() {
+        const bulkEmailConfig = this.#config.get('bulkEmail');
+        const bulkEmailSetting = {
+            apiKey: this.#settings.get('mailgun_api_key'),
+            domain: this.#settings.get('mailgun_domain'),
+            baseUrl: this.#settings.get('mailgun_base_url')
+        };
+
+        const hasMailgunConfig = !!(bulkEmailConfig?.mailgun);
+        const hasMailgunSetting = !!(bulkEmailSetting && bulkEmailSetting.apiKey && bulkEmailSetting.baseUrl && bulkEmailSetting.domain);
+
+        if (!hasMailgunConfig && !hasMailgunSetting) {
+            return null;
+        }
+
+        const mailgunConfig = hasMailgunConfig ? bulkEmailConfig.mailgun : bulkEmailSetting;
+        return mailgunConfig;
+    }
+
+    /**
+     * Returns an instance of the Mailgun client based upon the config or settings values
+     *
+     * We don't cache the instance so we can always get a fresh one based upon changed settings
+     * or config values over time
+     *
+     * Note: if the credentials are not configure, this method returns `null` and it is down to the
+     * consumer to act upon this/log this out
+     *
+     * @returns {import('mailgun.js')|null} the Mailgun client instance
+     */
+    getInstance() {
+        const mailgunConfig = this.#getConfig();
+        if (!mailgunConfig) {
+            return null;
+        }
+
+        const formData = require('form-data');
+        const Mailgun = require('mailgun.js');
+
+        const baseUrl = new URL(mailgunConfig.baseUrl);
+        const mailgun = new Mailgun(formData);
+
+        return mailgun.client({
+            username: 'api',
+            key: mailgunConfig.apiKey,
+            url: baseUrl.origin,
+            timeout: 60000
+        });
+    }
+
+    /**
+     * Returns whether the Mailgun instance is configured via config/settings
+     *
+     * @returns {boolean}
+     */
+    isConfigured() {
+        const instance = this.getInstance();
+        return !!instance;
+    }
+
+    /**
+     * Returns configured batch size
+     *
+     * @returns {number}
+     */
+    getBatchSize() {
+        return this.#config.get('bulkEmail')?.batchSize ?? this.DEFAULT_BATCH_SIZE;
+    }
+
+    /**
+     * Returns the configured target delivery window in seconds
+     * Ghost will attempt to deliver emails evenly distributed over this window
+     * 
+     * Defaults to 0 (no delay) if not set
+     * 
+     * @returns {number}
+     */
+    getTargetDeliveryWindow() {
+        const targetDeliveryWindow = this.#config.get('bulkEmail')?.targetDeliveryWindow;
+        // If targetDeliveryWindow is not set or is not a positive integer, return 0
+        if (targetDeliveryWindow === undefined || !Number.isInteger(parseInt(targetDeliveryWindow)) || parseInt(targetDeliveryWindow) < 0) {
+            return 0;
+        }
+        return parseInt(targetDeliveryWindow);
+    }
+};

package/core/server/services/link-redirection/LinkRedirect.js

@@ -0,0 +1,26 @@
+const ObjectID = require('bson-objectid').default;
+
+module.exports = class LinkRedirect {
+    /** @type {ObjectID} */
+    link_id;
+    /** @type {URL} */
+    from;
+    /** @type {URL} */
+    to;
+    /** @type {boolean} */
+    edited;
+
+    constructor(data) {
+        if (!data.id) {
+            this.link_id = new ObjectID();
+        }
+
+        if (typeof data.id === 'string') {
+            this.link_id = ObjectID.createFromHexString(data.id);
+        }
+
+        this.from = data.from;
+        this.to = data.to;
+        this.edited = !!data.edited;
+    }
+};

package/core/server/services/link-redirection/LinkRedirectRepository.js

@@ -1,4 +1,4 @@
-const LinkRedirect = require('@tryghost/link-redirects').LinkRedirect;
+const LinkRedirect = require('./LinkRedirect');
 const ObjectID = require('bson-objectid').default;
 const debug = require('@tryghost/debug')('LinkRedirectRepository');
 
@@ -17,7 +17,7 @@ module.exports = class LinkRedirectRepository {
      * @param {object} deps.LinkRedirect - Bookshelf Model
      * @param {object} deps.urlUtils
      * @param {object} deps.cacheAdapter - Cache Adapter instance, or null if cache is disabled
-     * @param {object} deps.EventRegistry 
+     * @param {object} deps.EventRegistry
      */
     constructor(deps) {
         debug('Creating LinkRedirectRepository');
@@ -61,7 +61,7 @@ module.exports = class LinkRedirectRepository {
 
     /**
      * Trim the leading slash from a URL path
-     * @param {string} url 
+     * @param {string} url
      * @returns {string} url without leading slash
      */
     #trimLeadingSlash(url) {
@@ -70,14 +70,14 @@ module.exports = class LinkRedirectRepository {
 
     /**
      * Returns a LinkRedirect object from a model
-     * @param {object} model - Bookshelf model instance 
+     * @param {object} model - Bookshelf model instance
      * @returns {InstanceType<LinkRedirect>} LinkRedirect
      */
     fromModel(model) {
         // Store if link has been edited
         // Note: in some edge cases updated_at is set directly after created_at, sometimes with a second difference, so we need to check for that
         const edited = model.get('updated_at')?.getTime() > (model.get('created_at')?.getTime() + 1000);
-        
+
         return new LinkRedirect({
             id: model.id,
             from: new URL(this.#trimLeadingSlash(model.get('from')), this.#urlUtils.urlFor('home', true)),
@@ -106,7 +106,7 @@ module.exports = class LinkRedirectRepository {
 
     /**
      * Serialize a LinkRedirect object to a plain object (e.g. for caching)
-     * @param {InstanceType<LinkRedirect>} linkRedirect 
+     * @param {InstanceType<LinkRedirect>} linkRedirect
      * @returns {object} - serialized LinkRedirect
      */
     #serialize(linkRedirect) {
@@ -120,7 +120,7 @@ module.exports = class LinkRedirectRepository {
 
     /**
      * Get all LinkRedirects from the DB, with optional filters
-     * @param {object} options - options passed directly to LinkRedirect.findAll 
+     * @param {object} options - options passed directly to LinkRedirect.findAll
      * @returns {Promise<InstanceType<LinkRedirect>[]>} array of LinkRedirects
      */
     async getAll(options) {

package/core/server/services/link-redirection/LinkRedirectsService.js

@@ -0,0 +1,123 @@
+const crypto = require('crypto');
+const DomainEvents = require('@tryghost/domain-events');
+const RedirectEvent = require('./RedirectEvent');
+const LinkRedirect = require('./LinkRedirect');
+
+/**
+ * @typedef {object} ILinkRedirectRepository
+ * @prop {(url: URL) => Promise<LinkRedirect|undefined>} getByURL
+ * @prop {({filter: string}) => Promise<LinkRedirect[]>} getAll
+ * @prop {({filter: string}) => Promise<String[]>} getFilteredIds
+ * @prop {(linkRedirect: LinkRedirect) => Promise<void>} save
+ */
+
+class LinkRedirectsService {
+    /** @type ILinkRedirectRepository */
+    #linkRedirectRepository;
+    /** @type URL */
+    #baseURL;
+
+    /** @type String */
+    #redirectURLPrefix = 'r/';
+
+    /**
+     * @param {object} deps
+     * @param {ILinkRedirectRepository} deps.linkRedirectRepository
+     * @param {object} deps.config
+     * @param {URL} deps.config.baseURL
+     */
+    constructor(deps) {
+        this.#linkRedirectRepository = deps.linkRedirectRepository;
+        if (!deps.config.baseURL.pathname.endsWith('/')) {
+            this.#baseURL = new URL(deps.config.baseURL);
+            this.#baseURL.pathname += '/';
+        } else {
+            this.#baseURL = deps.config.baseURL;
+        }
+        this.handleRequest = this.handleRequest.bind(this);
+    }
+
+    /**
+     * Get a unique URL with slug for creating unique redirects
+     *
+     * @returns {Promise<URL>}
+     */
+    async getSlugUrl() {
+        let url;
+        while (!url || await this.#linkRedirectRepository.getByURL(url)) {
+            const slug = crypto.randomBytes(4).toString('hex');
+            url = new URL(`${this.#redirectURLPrefix}${slug}`, this.#baseURL);
+        }
+        return url;
+    }
+
+    /**
+     * @param {Object} options
+     *
+     * @returns {Promise<String[]>}
+     */
+    async getFilteredIds(options) {
+        return await this.#linkRedirectRepository.getFilteredIds(options);
+    }
+
+    /**
+     * @param {URL} from
+     * @param {URL} to
+     *
+     * @returns {Promise<LinkRedirect>}
+     */
+    async addRedirect(from, to) {
+        const link = new LinkRedirect({
+            from,
+            to
+        });
+
+        await this.#linkRedirectRepository.save(link);
+
+        return link;
+    }
+
+    /**
+     * @param {import('express').Request} req
+     * @param {import('express').Response} res
+     * @param {import('express').NextFunction} next
+     *
+     * @returns {Promise<void>}
+     */
+    async handleRequest(req, res, next) {
+        try {
+            // skip handling if original url doesn't match the prefix
+            const fullURLWithRedirectPrefix = `${this.#baseURL.pathname}${this.#redirectURLPrefix}`;
+            // @NOTE: below is equivalent to doing:
+            //          router.get('/r/'), (req, res) ...
+            //        To make it cleaner we should rework it to:
+            //          linkRedirects.service.handleRequest(router);
+            //        and mount routes on top like for example sitemapHandler does
+            //        Cleanup issue: https://github.com/TryGhost/Toolbox/issues/516
+            if (!req.originalUrl.startsWith(fullURLWithRedirectPrefix)) {
+                return next();
+            }
+
+            const url = new URL(req.originalUrl, this.#baseURL);
+            const link = await this.#linkRedirectRepository.getByURL(url);
+
+            if (!link) {
+                return next();
+            }
+
+            const event = RedirectEvent.create({
+                url,
+                link
+            });
+
+            DomainEvents.dispatch(event);
+
+            res.setHeader('X-Robots-Tag', 'noindex, nofollow');
+            return res.redirect(link.to.href);
+        } catch (e) {
+            return next(e);
+        }
+    }
+}
+
+module.exports = LinkRedirectsService;