ghost 5.115.1 → 5.116.0

package/core/server/services/lib/EmailContentGenerator.js

@@ -0,0 +1,54 @@
+const _ = require('lodash').runInContext();
+const fs = require('fs-extra');
+const path = require('path');
+
+_.templateSettings.interpolate = /{{([\s\S]+?)}}/g;
+
+class EmailContentGenerator {
+    /**
+     *
+     * @param {Object} options
+     * @param {function} options.getSiteUrl
+     * @param {function} options.getSiteTitle
+     * @param {string} options.templatesDir - path to the directory containing email templates
+     */
+    constructor({getSiteUrl, getSiteTitle, templatesDir}) {
+        this.getSiteUrl = getSiteUrl;
+        this.getSiteTitle = getSiteTitle;
+        this.templatesDir = templatesDir;
+    }
+
+    /**
+     *
+     * @param {Object} options
+     * @param {string} options.template - HTML template name to use for generation
+     * @param {Object} [options.data] - variable data to use during HTML template compilation
+     * @returns {Promise<{html: String, text: String}>} resolves with an object containing html and text properties
+     */
+    async getContent(options) {
+        const defaults = {
+            siteUrl: this.getSiteUrl(),
+            siteTitle: this.getSiteTitle()
+        };
+
+        const data = _.defaults(defaults, options.data);
+
+        // read the proper email body template
+        const content = await fs.readFile(path.join(this.templatesDir, options.template + '.html'), 'utf8');
+
+        // insert user-specific data into the email
+        const compiled = _.template(content);
+        const htmlContent = compiled(data);
+
+        // lazyload the lib, and generate a plain-text version of the same email
+        const htmlToText = require('html-to-text');
+        const textContent = htmlToText.fromString(htmlContent);
+
+        return {
+            html: htmlContent,
+            text: textContent
+        };
+    }
+}
+
+module.exports = EmailContentGenerator;

package/core/server/services/lib/InMemoryRepository.js

@@ -0,0 +1,62 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.InMemoryRepository = void 0;
+const nql_1 = __importDefault(require("@tryghost/nql"));
+class InMemoryRepository {
+    store = [];
+    ids = new Map();
+    async save(entity) {
+        if (entity.deleted) {
+            this.store = this.store.filter(item => item.id !== entity.id);
+            this.ids.delete(entity.id);
+            return;
+        }
+        if (this.ids.has(entity.id)) {
+            this.store = this.store.map((item) => {
+                if (item.id === entity.id) {
+                    return entity;
+                }
+                return item;
+            });
+        }
+        else {
+            this.store.push(entity);
+            this.ids.set(entity.id, true);
+        }
+    }
+    async getById(id) {
+        return this.store.find(item => item.id === id) || null;
+    }
+    async getAll(options = {}) {
+        const filter = (0, nql_1.default)(options.filter);
+        const results = this.store.slice().filter(item => filter.queryJSON(this.toPrimitive(item)));
+        if (options.order) {
+            for (const order of options.order) {
+                results.sort((a, b) => {
+                    if (order.direction === 'asc') {
+                        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                        return a[order.field] > b[order.field] ? 1 : -1;
+                    }
+                    else {
+                        return a[order.field] < b[order.field] ? 1 : -1;
+                    }
+                });
+            }
+        }
+        return results;
+    }
+    async getPage(options = { page: 1, limit: 15 }) {
+        const results = await this.getAll(options);
+        const start = (options.page - 1) * options.limit;
+        const end = start + options.limit;
+        return results.slice(start, end);
+    }
+    async getCount(options) {
+        const results = await this.getAll(options);
+        return results.length;
+    }
+}
+exports.InMemoryRepository = InMemoryRepository;

package/core/server/services/lib/InMemoryRepository.ts

@@ -0,0 +1,80 @@
+import nql from '@tryghost/nql';
+
+type Entity<T> = {
+    id: T;
+    deleted: boolean;
+}
+
+type Order<T> = {
+    field: keyof T;
+    direction: 'asc' | 'desc';
+}
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+type OrderOption<T extends Entity<any>> = Order<T>[];
+
+export abstract class InMemoryRepository<IDType, T extends Entity<IDType>> {
+    protected store: T[] = [];
+    private ids: Map<IDType, true> = new Map();
+
+    protected abstract toPrimitive(entity: T): object;
+
+    public async save(entity: T): Promise<void> {
+        if (entity.deleted) {
+            this.store = this.store.filter(item => item.id !== entity.id);
+            this.ids.delete(entity.id);
+            return;
+        }
+
+        if (this.ids.has(entity.id)) {
+            this.store = this.store.map((item) => {
+                if (item.id === entity.id) {
+                    return entity;
+                }
+                return item;
+            });
+        } else {
+            this.store.push(entity);
+            this.ids.set(entity.id, true);
+        }
+    }
+
+    public async getById(id: string): Promise<T | null> {
+        return this.store.find(item => item.id === id) || null;
+    }
+
+    public async getAll(options: { filter?: string; order?: OrderOption<T> } = {}): Promise<T[]> {
+        const filter = nql(options.filter);
+
+        const results = this.store.slice().filter(item => filter.queryJSON(this.toPrimitive(item)));
+
+        if (options.order) {
+            for (const order of options.order) {
+                results.sort((a, b) => {
+                    if (order.direction === 'asc') {
+                        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                        return a[order.field] as any > (b[order.field] as any) ? 1 : -1;
+                    } else {
+                        return a[order.field] < b[order.field] ? 1 : -1;
+                    }
+                });
+            }
+        }
+
+        return results;
+    }
+
+    public async getPage(options: { filter?: string; page: number; limit: number; order?: Order<T>[] } = {page: 1, limit: 15}): Promise<T[]> {
+        const results = await this.getAll(options);
+
+        const start = (options.page - 1) * options.limit;
+        const end = start + options.limit;
+
+        return results.slice(start, end);
+    }
+
+    public async getCount(options: { filter?: string }): Promise<number> {
+        const results = await this.getAll(options);
+        return results.length;
+    }
+}

package/core/server/services/lib/MailgunClient.js

@@ -0,0 +1,364 @@
+const _ = require('lodash');
+const debug = require('@tryghost/debug');
+const logging = require('@tryghost/logging');
+const metrics = require('@tryghost/metrics');
+const errors = require('@tryghost/errors');
+
+module.exports = class MailgunClient {
+    #config;
+    #settings;
+
+    static DEFAULT_BATCH_SIZE = 1000;
+
+    constructor({config, settings}) {
+        this.#config = config;
+        this.#settings = settings;
+    }
+
+    /**
+     * Creates the data payload and sends to Mailgun
+     *
+     * @param {Object} message
+     * @param {Object} recipientData
+     * @param {Array<Object>} replacements
+     *
+     * recipientData format:
+     * {
+     *     'test@example.com': {
+     *         name: 'Test User',
+     *         unsubscribe_url: 'https://example.com/unsub/me',
+     *         list_unsubscribe: 'https://example.com/unsub/me'
+     *     }
+     * }
+     */
+    async send(message, recipientData, replacements) {
+        const mailgunInstance = this.getInstance();
+        if (!mailgunInstance) {
+            logging.warn(`Mailgun is not configured`);
+            return null;
+        }
+
+        const batchSize = this.getBatchSize();
+        if (Object.keys(recipientData).length > batchSize) {
+            throw new errors.IncorrectUsageError({
+                message: `Mailgun only supports sending to ${batchSize} recipients at a time`
+            });
+        }
+
+        let messageData = {};
+
+        let startTime;
+        try {
+            const bulkEmailConfig = this.#config.get('bulkEmail');
+            const messageContent = _.pick(message, 'subject', 'html', 'plaintext');
+
+            // update content to use Mailgun variable syntax for replacements
+            replacements.forEach((replacement) => {
+                messageContent[replacement.format] = messageContent[replacement.format].replace(
+                    replacement.regexp,
+                    `%recipient.${replacement.id}%`
+                );
+            });
+
+            messageData = {
+                to: Object.keys(recipientData),
+                from: message.from,
+                'h:Reply-To': message.replyTo || message.reply_to,
+                subject: messageContent.subject,
+                html: messageContent.html,
+                text: messageContent.plaintext,
+                'recipient-variables': JSON.stringify(recipientData)
+            };
+
+            // Do we have a custom List-Unsubscribe header set?
+            // (we need a variable for this, as this is a per-email setting)
+            if (Object.keys(recipientData)[0] && recipientData[Object.keys(recipientData)[0]].list_unsubscribe) {
+                messageData['h:List-Unsubscribe'] = '<%recipient.list_unsubscribe%>, <%tag_unsubscribe_email%>';
+                messageData['h:List-Unsubscribe-Post'] = 'List-Unsubscribe=One-Click';
+            }
+
+            // add a reference to the original email record for easier mapping of mailgun event -> email
+            if (message.id) {
+                messageData['v:email-id'] = message.id;
+            }
+
+            const tags = ['bulk-email', 'ghost-email'];
+            if (bulkEmailConfig?.mailgun?.tag) {
+                tags.push(bulkEmailConfig.mailgun.tag);
+            }
+            messageData['o:tag'] = tags;
+
+            if (bulkEmailConfig?.mailgun?.testmode) {
+                messageData['o:testmode'] = true;
+            }
+
+            // enable tracking if turned on for this email
+            if (message.track_opens) {
+                messageData['o:tracking-opens'] = true;
+            }
+
+            // set the delivery time if specified
+            if (message.deliveryTime && message.deliveryTime instanceof Date) {
+                messageData['o:deliverytime'] = message.deliveryTime.toUTCString();
+            }
+
+            const mailgunConfig = this.#getConfig();
+            startTime = Date.now();
+            const response = await mailgunInstance.messages.create(mailgunConfig.domain, messageData);
+            metrics.metric('mailgun-send-mail', {
+                value: Date.now() - startTime,
+                statusCode: 200
+            });
+
+            return {
+                id: response.id
+            };
+        } catch (error) {
+            logging.error(error);
+            metrics.metric('mailgun-send-mail', {
+                value: Date.now() - startTime,
+                statusCode: error.status
+            });
+            return Promise.reject({error, messageData});
+        }
+    }
+
+    /**
+     * @param {import('mailgun.js').default} mailgunInstance
+     * @param {Object} mailgunConfig
+     * @param {Object} mailgunOptions
+     */
+    async getEventsFromMailgun(mailgunInstance, mailgunConfig, mailgunOptions) {
+        const startTime = Date.now();
+        try {
+            const page = await mailgunInstance.events.get(mailgunConfig.domain, mailgunOptions);
+            metrics.metric('mailgun-get-events', {
+                value: Date.now() - startTime,
+                statusCode: 200
+            });
+            return page;
+        } catch (error) {
+            metrics.metric('mailgun-get-events', {
+                value: Date.now() - startTime,
+                statusCode: error.status
+            });
+            throw error;
+        }
+    }
+
+    /**
+     * Fetches events from Mailgun
+     * @param {Object} mailgunOptions
+     * @param {Function} batchHandler
+     * @param {Object} options
+     * @param {Number} options.maxEvents Not a strict maximum. We stop fetching after we reached the maximum AND received at least one event after begin (not equal) to prevent deadlocks.
+     * @returns {Promise<void>}
+     */
+    async fetchEvents(mailgunOptions, batchHandler, {maxEvents = Infinity} = {}) {
+        const mailgunInstance = this.getInstance();
+        if (!mailgunInstance) {
+            logging.warn(`Mailgun is not configured`);
+            return;
+        }
+
+        debug(`[MailgunClient fetchEvents]: starting fetching first events page`);
+        const mailgunConfig = this.#getConfig();
+        const startDate = new Date();
+        const overallStartTime = Date.now();
+
+        let batchCount = 0;
+        let totalBatchTime = 0;
+
+        try {
+            let page = await this.getEventsFromMailgun(mailgunInstance, mailgunConfig, mailgunOptions);
+
+            // By limiting the processed events to ones created before this job started we cancel early ready for the next job run.
+            // Avoids chance of events being missed in long job runs due to mailgun's eventual-consistency creating events outside of our 30min sliding re-check window
+            let events = (page?.items?.map(this.normalizeEvent) || []).filter(e => !!e && e.timestamp <= startDate);
+            debug(`[MailgunClient fetchEvents]: finished fetching first page with ${events.length} events`);
+
+            let eventCount = 0;
+            const beginTimestamp = mailgunOptions.begin ? Math.ceil(mailgunOptions.begin * 1000) : undefined; // ceil here if we have rounding errors
+
+            while (events.length !== 0) {
+                const batchStartTime = Date.now();
+                await batchHandler(events);
+                const batchEndTime = Date.now();
+                const batchDuration = batchEndTime - batchStartTime;
+
+                batchCount += 1;
+                totalBatchTime += batchDuration;
+
+                eventCount += events.length;
+
+                if (eventCount >= maxEvents && (!beginTimestamp || !events[events.length - 1].timestamp || (events[events.length - 1].timestamp.getTime() > beginTimestamp))) {
+                    break;
+                }
+
+                const nextPageId = page.pages.next.page;
+                debug(`[MailgunClient fetchEvents]: starting fetching next page ${nextPageId}`);
+                page = await this.getEventsFromMailgun(mailgunInstance, mailgunConfig, {
+                    page: nextPageId,
+                    ...mailgunOptions
+                });
+
+                // We need to cap events at the time we started fetching them (see comment above)
+                events = (page?.items?.map(this.normalizeEvent) || []).filter(e => !!e && e.timestamp <= startDate);
+                debug(`[MailgunClient fetchEvents]: finished fetching next page with ${events.length} events`);
+            }
+
+            const overallEndTime = Date.now();
+            const totalDuration = overallEndTime - overallStartTime;
+            const averageBatchTime = batchCount > 0 ? totalBatchTime / batchCount : 0;
+
+            logging.info(`[MailgunClient fetchEvents]: Processed ${batchCount} batches in ${(totalDuration / 1000).toFixed(2)}s. Average batch time: ${(averageBatchTime / 1000).toFixed(2)}s`);
+        } catch (error) {
+            logging.error(error);
+            throw error;
+        }
+    }
+
+    async removeSuppression(type, email) {
+        if (!this.isConfigured()) {
+            return false;
+        }
+        const instance = this.getInstance();
+        const config = this.#getConfig();
+
+        try {
+            await instance.suppressions.destroy(
+                config.domain,
+                type,
+                email
+            );
+            return true;
+        } catch (err) {
+            logging.error(err);
+            return false;
+        }
+    }
+
+    async removeBounce(email) {
+        return this.removeSuppression('bounces', email);
+    }
+
+    async removeComplaint(email) {
+        return this.removeSuppression('complaints', email);
+    }
+
+    async removeUnsubscribe(email) {
+        return this.removeSuppression('unsubscribes', email);
+    }
+
+    normalizeEvent(event) {
+        const providerId = event?.message?.headers['message-id'];
+
+        if (!providerId && !(event['user-variables'] && event['user-variables']['email-id'])) {
+            logging.error('Received invalid event from Mailgun');
+            logging.error(event);
+            return null;
+        }
+
+        return {
+            id: event.id,
+            type: event.event,
+            severity: event.severity,
+            recipientEmail: event.recipient,
+            emailId: event['user-variables'] && event['user-variables']['email-id'],
+            providerId: providerId,
+            timestamp: new Date(event.timestamp * 1000),
+
+            error: event['delivery-status'] && (typeof (event['delivery-status'].message || event['delivery-status'].description) === 'string') ? {
+                code: event['delivery-status'].code,
+                message: (event['delivery-status'].message || event['delivery-status'].description).substring(0, 2000),
+                enhancedCode: event['delivery-status']['enhanced-code']?.toString()?.substring(0, 50) ?? null
+            } : null
+        };
+    }
+
+    #getConfig() {
+        const bulkEmailConfig = this.#config.get('bulkEmail');
+        const bulkEmailSetting = {
+            apiKey: this.#settings.get('mailgun_api_key'),
+            domain: this.#settings.get('mailgun_domain'),
+            baseUrl: this.#settings.get('mailgun_base_url')
+        };
+
+        const hasMailgunConfig = !!(bulkEmailConfig?.mailgun);
+        const hasMailgunSetting = !!(bulkEmailSetting && bulkEmailSetting.apiKey && bulkEmailSetting.baseUrl && bulkEmailSetting.domain);
+
+        if (!hasMailgunConfig && !hasMailgunSetting) {
+            return null;
+        }
+
+        const mailgunConfig = hasMailgunConfig ? bulkEmailConfig.mailgun : bulkEmailSetting;
+        return mailgunConfig;
+    }
+
+    /**
+     * Returns an instance of the Mailgun client based upon the config or settings values
+     *
+     * We don't cache the instance so we can always get a fresh one based upon changed settings
+     * or config values over time
+     *
+     * Note: if the credentials are not configure, this method returns `null` and it is down to the
+     * consumer to act upon this/log this out
+     *
+     * @returns {import('mailgun.js')|null} the Mailgun client instance
+     */
+    getInstance() {
+        const mailgunConfig = this.#getConfig();
+        if (!mailgunConfig) {
+            return null;
+        }
+
+        const formData = require('form-data');
+        const Mailgun = require('mailgun.js');
+
+        const baseUrl = new URL(mailgunConfig.baseUrl);
+        const mailgun = new Mailgun(formData);
+
+        return mailgun.client({
+            username: 'api',
+            key: mailgunConfig.apiKey,
+            url: baseUrl.origin,
+            timeout: 60000
+        });
+    }
+
+    /**
+     * Returns whether the Mailgun instance is configured via config/settings
+     *
+     * @returns {boolean}
+     */
+    isConfigured() {
+        const instance = this.getInstance();
+        return !!instance;
+    }
+
+    /**
+     * Returns configured batch size
+     *
+     * @returns {number}
+     */
+    getBatchSize() {
+        return this.#config.get('bulkEmail')?.batchSize ?? this.DEFAULT_BATCH_SIZE;
+    }
+
+    /**
+     * Returns the configured target delivery window in seconds
+     * Ghost will attempt to deliver emails evenly distributed over this window
+     * 
+     * Defaults to 0 (no delay) if not set
+     * 
+     * @returns {number}
+     */
+    getTargetDeliveryWindow() {
+        const targetDeliveryWindow = this.#config.get('bulkEmail')?.targetDeliveryWindow;
+        // If targetDeliveryWindow is not set or is not a positive integer, return 0
+        if (targetDeliveryWindow === undefined || !Number.isInteger(parseInt(targetDeliveryWindow)) || parseInt(targetDeliveryWindow) < 0) {
+            return 0;
+        }
+        return parseInt(targetDeliveryWindow);
+    }
+};

package/core/server/services/link-redirection/LinkRedirect.js

@@ -0,0 +1,26 @@
+const ObjectID = require('bson-objectid').default;
+
+module.exports = class LinkRedirect {
+    /** @type {ObjectID} */
+    link_id;
+    /** @type {URL} */
+    from;
+    /** @type {URL} */
+    to;
+    /** @type {boolean} */
+    edited;
+
+    constructor(data) {
+        if (!data.id) {
+            this.link_id = new ObjectID();
+        }
+
+        if (typeof data.id === 'string') {
+            this.link_id = ObjectID.createFromHexString(data.id);
+        }
+
+        this.from = data.from;
+        this.to = data.to;
+        this.edited = !!data.edited;
+    }
+};

package/core/server/services/link-redirection/LinkRedirectRepository.js

@@ -1,4 +1,4 @@
-const LinkRedirect = require('@tryghost/link-redirects').LinkRedirect;
+const LinkRedirect = require('./LinkRedirect');
 const ObjectID = require('bson-objectid').default;
 const debug = require('@tryghost/debug')('LinkRedirectRepository');
 
@@ -17,7 +17,7 @@ module.exports = class LinkRedirectRepository {
      * @param {object} deps.LinkRedirect - Bookshelf Model
      * @param {object} deps.urlUtils
      * @param {object} deps.cacheAdapter - Cache Adapter instance, or null if cache is disabled
-     * @param {object} deps.EventRegistry 
+     * @param {object} deps.EventRegistry
      */
     constructor(deps) {
         debug('Creating LinkRedirectRepository');
@@ -61,7 +61,7 @@ module.exports = class LinkRedirectRepository {
 
     /**
      * Trim the leading slash from a URL path
-     * @param {string} url 
+     * @param {string} url
      * @returns {string} url without leading slash
      */
     #trimLeadingSlash(url) {
@@ -70,14 +70,14 @@ module.exports = class LinkRedirectRepository {
 
     /**
      * Returns a LinkRedirect object from a model
-     * @param {object} model - Bookshelf model instance 
+     * @param {object} model - Bookshelf model instance
      * @returns {InstanceType<LinkRedirect>} LinkRedirect
      */
     fromModel(model) {
         // Store if link has been edited
         // Note: in some edge cases updated_at is set directly after created_at, sometimes with a second difference, so we need to check for that
         const edited = model.get('updated_at')?.getTime() > (model.get('created_at')?.getTime() + 1000);
-        
+
         return new LinkRedirect({
             id: model.id,
             from: new URL(this.#trimLeadingSlash(model.get('from')), this.#urlUtils.urlFor('home', true)),
@@ -106,7 +106,7 @@ module.exports = class LinkRedirectRepository {
 
     /**
      * Serialize a LinkRedirect object to a plain object (e.g. for caching)
-     * @param {InstanceType<LinkRedirect>} linkRedirect 
+     * @param {InstanceType<LinkRedirect>} linkRedirect
      * @returns {object} - serialized LinkRedirect
      */
     #serialize(linkRedirect) {
@@ -120,7 +120,7 @@ module.exports = class LinkRedirectRepository {
 
     /**
      * Get all LinkRedirects from the DB, with optional filters
-     * @param {object} options - options passed directly to LinkRedirect.findAll 
+     * @param {object} options - options passed directly to LinkRedirect.findAll
      * @returns {Promise<InstanceType<LinkRedirect>[]>} array of LinkRedirects
      */
     async getAll(options) {