ghost 5.24.1 → 5.25.0

package/core/server/data/schema/schema.js

@@ -835,6 +835,24 @@ module.exports = {
             ['email_id', 'member_email']
         ]
     },
+    email_recipient_failures: {
+        id: {type: 'string', maxlength: 24, nullable: false, primary: true},
+        email_id: {type: 'string', maxlength: 24, nullable: false, references: 'emails.id'},
+        member_id: {type: 'string', maxlength: 24, nullable: true},
+        email_recipient_id: {type: 'string', maxlength: 24, nullable: false, references: 'email_recipients.id'},
+        code: {type: 'integer', nullable: false, unsigned: true},
+        enhanced_code: {type: 'string', maxlength: 50, nullable: true},
+        message: {type: 'string', maxlength: 2000, nullable: false},
+        severity: {
+            type: 'string',
+            maxlength: 50,
+            nullable: false,
+            defaultTo: 'permanent',
+            validations: {isIn: [['temporary', 'permanent']]}
+        },
+        failed_at: {type: 'dateTime', nullable: false},
+        event_id: {type: 'string', maxlength: 255, nullable: true}
+    },
     tokens: {
         id: {type: 'string', maxlength: 24, nullable: false, primary: true},
         token: {type: 'string', maxlength: 32, nullable: false, index: true},
@@ -931,5 +949,32 @@ module.exports = {
         post_id: {type: 'string', maxlength: 24, nullable: false, references: 'posts.id', cascadeDelete: true},
         created_at: {type: 'dateTime', nullable: false},
         updated_at: {type: 'dateTime', nullable: true}
+    },
+    suppressions: {
+        id: {type: 'string', maxlength: 24, nullable: false, primary: true},
+        email_address: {type: 'string', maxlength: 191, nullable: false, unique: true, validations: {isEmail: true}},
+        email_id: {type: 'string', maxlength: 24, nullable: true, references: 'emails.id'},
+        reason: {
+            type: 'string',
+            maxlength: 50,
+            nullable: false,
+            validations: {
+                isIn: [[
+                    'spam',
+                    'bounce'
+                ]]
+            }
+        },
+        created_at: {type: 'dateTime', nullable: false}
+    },
+    email_spam_complaint_events: {
+        id: {type: 'string', maxlength: 24, nullable: false, primary: true},
+        member_id: {type: 'string', maxlength: 24, nullable: false, references: 'members.id', cascadeDelete: true},
+        email_id: {type: 'string', maxlength: 24, nullable: false, references: 'emails.id'},
+        email_address: {type: 'string', maxlength: 191, nullable: false, unique: false, validations: {isEmail: true}},
+        created_at: {type: 'dateTime', nullable: false},
+        '@@UNIQUE_CONSTRAINTS@@': [
+            ['email_id', 'member_id']
+        ]
     }
 };

package/core/server/lib/lexical.js

@@ -1,4 +1,8 @@
+const urlUtils = require('../../shared/url-utils');
+
+let nodes;
 let lexicalHtmlRenderer;
+let urlTransformMap;
 
 module.exports = {
     get lexicalHtmlRenderer() {
@@ -8,5 +12,38 @@ module.exports = {
         }
 
         return lexicalHtmlRenderer;
+    },
+
+    get nodes() {
+        if (!nodes) {
+            const {DEFAULT_NODES} = require('@tryghost/kg-default-nodes');
+            nodes = DEFAULT_NODES;
+        }
+
+        return nodes;
+    },
+
+    get urlTransformMap() {
+        if (!urlTransformMap) {
+            urlTransformMap = {
+                absoluteToRelative: {
+                    url: urlUtils.absoluteToRelative.bind(urlUtils),
+                    html: urlUtils.htmlAbsoluteToRelative.bind(urlUtils),
+                    markdown: urlUtils.markdownAbsoluteToRelative.bind(urlUtils)
+                },
+                relativeToAbsolute: {
+                    url: urlUtils.relativeToAbsolute.bind(urlUtils),
+                    html: urlUtils.htmlRelativeToAbsolute.bind(urlUtils),
+                    markdown: urlUtils.markdownRelativeToAbsolute.bind(urlUtils)
+                },
+                toTransformReady: {
+                    url: urlUtils.toTransformReady.bind(urlUtils),
+                    html: urlUtils.htmlToTransformReady.bind(urlUtils),
+                    markdown: urlUtils.markdownToTransformReady.bind(urlUtils)
+                }
+            };
+        }
+
+        return urlTransformMap;
     }
 };

package/core/server/models/email-batch.js

@@ -18,6 +18,19 @@ const EmailBatch = ghostBookshelf.Model.extend({
     members() {
         return this.belongsToMany('Member', 'email_recipients', 'batch_id', 'member_id');
     }
+}, {
+    countRelations() {
+        return {
+            recipients(modelOrCollection) {
+                modelOrCollection.query('columns', 'email_batches.*', (qb) => {
+                    qb.count('email_recipients.id')
+                        .from('email_recipients')
+                        .whereRaw('email_batches.id = email_recipients.batch_id')
+                        .as('count__recipients');
+                });
+            }
+        };
+    }
 });
 
 const EmailBatches = ghostBookshelf.Collection.extend({

package/core/server/models/email-recipient-failure.js

@@ -0,0 +1,29 @@
+const ghostBookshelf = require('./base');
+
+const EmailRecipientFailure = ghostBookshelf.Model.extend({
+    tableName: 'email_recipient_failures',
+    hasTimestamps: false,
+
+    defaults() {
+        return {
+        };
+    },
+
+    email() {
+        return this.belongsTo('Email', 'email_id');
+    },
+
+    member() {
+        return this.belongsTo('Member', 'member_id');
+    },
+
+    emailRecipient() {
+        return this.belongsTo('EmailRecipient', 'email_recipient_id');
+    }
+}, {
+
+});
+
+module.exports = {
+    EmailRecipientFailure: ghostBookshelf.model('EmailRecipientFailure', EmailRecipientFailure)
+};

package/core/server/models/email-spam-complaint-event.js

@@ -0,0 +1,44 @@
+const errors = require('@tryghost/errors');
+const ghostBookshelf = require('./base');
+
+const EmailSpamComplaintEvent = ghostBookshelf.Model.extend({
+    tableName: 'email_spam_complaint_events',
+
+    filterRelations: function filterRelations() {
+        return {
+            email: {
+                // Mongo-knex doesn't support belongsTo relations
+                tableName: 'emails',
+                tableNameAs: 'email',
+                type: 'manyToMany',
+                joinTable: 'email_spam_complaint_events',
+                joinFrom: 'id',
+                joinTo: 'email_id'
+            }
+        };
+    },
+
+    email() {
+        return this.belongsTo('Email', 'email_id');
+    },
+
+    member() {
+        return this.belongsTo('Member', 'member_id');
+    }
+}, {
+    async edit() {
+        throw new errors.IncorrectUsageError({
+            message: 'Cannot edit EmailSpamComplaintEvent'
+        });
+    },
+
+    async destroy() {
+        throw new errors.IncorrectUsageError({
+            message: 'Cannot destroy EmailSpamComplaintEvent'
+        });
+    }
+});
+
+module.exports = {
+    EmailSpamComplaintEvent: ghostBookshelf.model('EmailSpamComplaintEvent', EmailSpamComplaintEvent)
+};

package/core/server/models/post.js

@@ -180,7 +180,13 @@ Post = ghostBookshelf.Model.extend({
                     cardTransformers: mobiledocLib.cards
                 }
             },
-            lexical: 'lexicalToTransformReady',
+            lexical: {
+                method: 'lexicalToTransformReady',
+                options: {
+                    nodes: lexicalLib.nodes,
+                    transformMap: lexicalLib.urlTransformMap
+                }
+            },
             html: 'htmlToTransformReady',
             plaintext: 'plaintextToTransformReady',
             custom_excerpt: 'htmlToTransformReady',

package/core/server/models/suppression.js

@@ -0,0 +1,9 @@
+const ghostBookshelf = require('./base');
+
+const Suppression = ghostBookshelf.Model.extend({
+    tableName: 'suppressions'
+});
+
+module.exports = {
+    Suppression: ghostBookshelf.model('Suppression', Suppression)
+};

package/core/server/services/email-analytics/index.js

@@ -2,14 +2,20 @@ const config = require('../../../shared/config');
 const db = require('../../data/db');
 const settings = require('../../../shared/settings-cache');
 const {EmailAnalyticsService} = require('@tryghost/email-analytics-service');
-const EventProcessor = require('./lib/event-processor');
+const {EmailEventProcessor} = require('@tryghost/email-service');
 const MailgunProvider = require('@tryghost/email-analytics-provider-mailgun');
 const queries = require('./lib/queries');
+const DomainEvents = require('@tryghost/domain-events');
+
+const eventProcessor = new EmailEventProcessor({
+    domainEvents: DomainEvents,
+    db
+});
 
 module.exports = new EmailAnalyticsService({
     config,
     settings,
-    eventProcessor: new EventProcessor({db}),
+    eventProcessor,
     providers: [
         new MailgunProvider({config, settings})
     ],

package/core/server/services/email-analytics/jobs/fetch-latest/index.js

@@ -0,0 +1,50 @@
+const {parentPort} = require('worker_threads');
+
+// recurring job to fetch analytics since the most recently seen event timestamp
+
+// Exit early when cancelled to prevent stalling shutdown. No cleanup needed when cancelling as everything is idempotent and will pick up
+// where it left off on next run
+function cancel() {
+    if (parentPort) {
+        parentPort.postMessage('Email analytics fetch-latest job cancelled before completion');
+        parentPort.postMessage('cancelled');
+    } else {
+        setTimeout(() => {
+            process.exit(0);
+        }, 1000);
+    }
+}
+
+if (parentPort) {
+    parentPort.once('message', (message) => {
+        if (message === 'cancel') {
+            return cancel();
+        }
+    });
+}
+
+(async () => {
+    const {run} = require('./run');
+    const {eventStats, aggregateEndDate, fetchStartDate} = await run({
+        domainEvents: {
+            dispatch(event) {
+                parentPort.postMessage({
+                    event: {
+                        type: event.constructor.name, 
+                        data: event
+                    }
+                });
+            }
+        }
+    });
+
+    if (parentPort) {
+        parentPort.postMessage(`Fetched ${eventStats.totalEvents} events and aggregated stats for ${eventStats.emailIds.length} emails in ${aggregateEndDate - fetchStartDate}ms`);
+        parentPort.postMessage('done');
+    } else {
+        // give the logging pipes time finish writing before exit
+        setTimeout(() => {
+            process.exit(0);
+        }, 1000);
+    }
+})();

package/core/server/services/email-analytics/jobs/{fetch-latest.js → fetch-latest/run.js}

@@ -1,32 +1,8 @@
-const {parentPort} = require('worker_threads');
 const debug = require('@tryghost/debug')('jobs:email-analytics:fetch-latest');
 
-// recurring job to fetch analytics since the most recently seen event timestamp
-
-// Exit early when cancelled to prevent stalling shutdown. No cleanup needed when cancelling as everything is idempotent and will pick up
-// where it left off on next run
-function cancel() {
-    if (parentPort) {
-        parentPort.postMessage('Email analytics fetch-latest job cancelled before completion');
-        parentPort.postMessage('cancelled');
-    } else {
-        setTimeout(() => {
-            process.exit(0);
-        }, 1000);
-    }
-}
-
-if (parentPort) {
-    parentPort.once('message', (message) => {
-        if (message === 'cancel') {
-            return cancel();
-        }
-    });
-}
-
-(async () => {
-    const config = require('../../../../shared/config');
-    const db = require('../../../data/db');
+async function run({domainEvents}) {
+    const config = require('../../../../../shared/config');
+    const db = require('../../../../data/db');
 
     const settingsRows = await db.knex('settings')
         .whereIn('key', ['mailgun_api_key', 'mailgun_domain', 'mailgun_base_url']);
@@ -44,14 +20,21 @@ if (parentPort) {
     };
 
     const {EmailAnalyticsService} = require('@tryghost/email-analytics-service');
-    const EventProcessor = require('../lib/event-processor');
     const MailgunProvider = require('@tryghost/email-analytics-provider-mailgun');
-    const queries = require('../lib/queries');
+    const queries = require('../../lib/queries');
+    const {EmailEventProcessor} = require('@tryghost/email-service');
+
+    // Since this is running in a worker thread, we cant dispatch directly
+    // So we post the events as a message to the job manager
+    const eventProcessor = new EmailEventProcessor({
+        domainEvents,
+        db
+    });
 
     const emailAnalyticsService = new EmailAnalyticsService({
         config,
         settings,
-        eventProcessor: new EventProcessor({db}),
+        eventProcessor,
         providers: [
             new MailgunProvider({config, settings})
         ],
@@ -69,14 +52,6 @@ if (parentPort) {
     await emailAnalyticsService.aggregateStats(eventStats);
     const aggregateEndDate = new Date();
     debug(`Finished aggregating email analytics in ${aggregateEndDate - aggregateStartDate}ms`);
-
-    if (parentPort) {
-        parentPort.postMessage(`Fetched ${eventStats.totalEvents} events and aggregated stats for ${eventStats.emailIds.length} emails in ${aggregateEndDate - fetchStartDate}ms`);
-        parentPort.postMessage('done');
-    } else {
-        // give the logging pipes time finish writing before exit
-        setTimeout(() => {
-            process.exit(0);
-        }, 1000);
-    }
-})();
+    return {eventStats, fetchStartDate, fetchEndDate, aggregateStartDate, aggregateEndDate};
+}
+module.exports.run = run;

package/core/server/services/email-analytics/jobs/index.js

@@ -30,7 +30,7 @@ module.exports = {
 
                 jobsService.addJob({
                     at: `${s} ${m}/5 * * * *`,
-                    job: path.resolve(__dirname, 'fetch-latest.js'),
+                    job: path.resolve(__dirname, 'fetch-latest/index.js'),
                     name: 'email-analytics-fetch-latest'
                 });
 

package/core/server/services/email-service/wrapper.js

@@ -1,25 +1,74 @@
 const logging = require('@tryghost/logging');
-const ObjectID = require('bson-objectid').default;
+const url = require('../../../server/api/endpoints/utils/serializers/output/utils/url');
 
 class EmailServiceWrapper {
+    getPostUrl(post) {
+        const jsonModel = post.toJSON();
+        url.forPost(post.id, jsonModel, {options: {}});
+        return jsonModel.url;
+    }
+
     init() {
-        const {EmailService, EmailController, EmailRenderer, SendingService, BatchSendingService, EmailSegmenter} = require('@tryghost/email-service');
-        const {Post, Newsletter, Email, EmailBatch, EmailRecipient, Member} = require('../../models');
+        if (this.service) {
+            return;
+        }
+
+        const {EmailService, EmailController, EmailRenderer, SendingService, BatchSendingService, EmailSegmenter, EmailEventStorage, MailgunEmailProvider} = require('@tryghost/email-service');
+        const {Post, Newsletter, Email, EmailBatch, EmailRecipient, Member, EmailRecipientFailure, EmailSpamComplaintEvent} = require('../../models');
+        const MailgunClient = require('@tryghost/mailgun-client');
+        const configService = require('../../../shared/config');
         const settingsCache = require('../../../shared/settings-cache');
+        const settingsHelpers = require('../../services/settings-helpers');
         const jobsService = require('../jobs');
         const membersService = require('../members');
         const db = require('../../data/db');
+        const sentry = require('../../../shared/sentry');
         const membersRepository = membersService.api.members;
         const limitService = require('../limits');
+        const domainEvents = require('@tryghost/domain-events');
 
-        const emailRenderer = new EmailRenderer();
-        const sendingService = new SendingService({
-            emailProvider: {
-                send: ({plaintext, subject, from, replyTo, recipients}) => {
-                    logging.info(`Sending email\nSubject: ${subject}\nFrom: ${from}\nReplyTo: ${replyTo}\nRecipients: ${recipients.length}\n\n${plaintext}`);
-                    return Promise.resolve({id: 'fake_provider_id_' + ObjectID().toHexString()});
-                }
+        const mobiledocLib = require('../../lib/mobiledoc');
+        const lexicalLib = require('../../lib/lexical');
+        const urlUtils = require('../../../shared/url-utils');
+        const memberAttribution = require('../member-attribution');
+        const linkReplacer = require('@tryghost/link-replacer');
+        const linkTracking = require('../link-tracking');
+        const audienceFeedback = require('../audience-feedback');
+
+        // capture errors from mailgun client and log them in sentry
+        const errorHandler = (error) => {
+            logging.info(`Capturing error for mailgun email provider service`);
+            sentry.captureException(error);
+        };
+
+        // Mailgun client instance for email provider
+        const mailgunClient = new MailgunClient({
+            config: configService, settings: settingsCache
+        });
+
+        const mailgunEmailProvider = new MailgunEmailProvider({
+            mailgunClient,
+            errorHandler
+        });
+
+        const emailRenderer = new EmailRenderer({
+            settingsCache,
+            settingsHelpers,
+            renderers: {
+                mobiledoc: mobiledocLib.mobiledocHtmlRenderer,
+                lexical: lexicalLib.lexicalHtmlRenderer
             },
+            imageSize: null,
+            urlUtils,
+            getPostUrl: this.getPostUrl,
+            linkReplacer,
+            linkTracking,
+            memberAttributionService: memberAttribution.service,
+            audienceFeedbackService: audienceFeedback.service
+        });
+
+        const sendingService = new SendingService({
+            emailProvider: mailgunEmailProvider,
             emailRenderer
         });
 
@@ -42,15 +91,17 @@ class EmailServiceWrapper {
 
         this.service = new EmailService({
             batchSendingService,
+            sendingService,
             models: {
                 Email
             },
             settingsCache,
             emailRenderer,
             emailSegmenter,
-            limitService
+            limitService,
+            membersRepository
         });
-        
+
         this.controller = new EmailController(this.service, {
             models: {
                 Post,
@@ -58,6 +109,16 @@ class EmailServiceWrapper {
                 Email
             }
         });
+
+        this.eventStorage = new EmailEventStorage({
+            db,
+            membersRepository,
+            models: {
+                EmailRecipientFailure,
+                EmailSpamComplaintEvent
+            }
+        });
+        this.eventStorage.listen(domainEvents);
     }
 }
 

package/core/server/services/email-suppression-list/InMemoryEmailSuppressionList.js

@@ -0,0 +1,36 @@
+const {AbstractEmailSuppressionList, EmailSuppressionData} = require('@tryghost/email-suppression-list');
+
+module.exports = class InMemoryEmailSuppressionList extends AbstractEmailSuppressionList {
+    store = ['spam@member.test', 'fail@member.test'];
+
+    async removeEmail(email) {
+        if ((email === 'fail@member.test' || email === 'spam@member.test') && this.store.includes(email)) {
+            this.store = this.store.filter(el => el !== email);
+
+            setTimeout(() => this.store.push(email), 3000);
+            return true;
+        }
+
+        return false;
+    }
+
+    async getSuppressionData(email) {
+        if (email === 'spam@member.test' && this.store.includes(email)) {
+            return new EmailSuppressionData(true, {
+                timestamp: new Date(),
+                reason: 'spam'
+            });
+        }
+        if (email === 'fail@member.test' && this.store.includes(email)) {
+            return new EmailSuppressionData(true, {
+                timestamp: new Date(),
+                reason: 'fail'
+            });
+        }
+        return new EmailSuppressionData(false);
+    }
+
+    async init() {
+        return;
+    }
+};

package/core/server/services/email-suppression-list/MailgunEmailSuppressionList.js

@@ -0,0 +1,117 @@
+const {AbstractEmailSuppressionList, EmailSuppressionData} = require('@tryghost/email-suppression-list');
+const {SpamComplaintEvent, EmailBouncedEvent} = require('@tryghost/email-events');
+const DomainEvents = require('@tryghost/domain-events');
+const logging = require('@tryghost/logging');
+
+/**
+ * @typedef {object} IMailgunAPIClient
+ * @prop {(email: string) => Promise<any>} removeBounce
+ * @prop {(email: string) => Promise<any>} removeComplaint
+ * @prop {(email: string) => Promise<any>} removeUnsubscribe
+ */
+
+class MailgunEmailSuppressionList extends AbstractEmailSuppressionList {
+    /**
+     * @param {object} deps
+     * @param {import('bookshelf').Model} deps.Suppression
+     * @param {IMailgunAPIClient} deps.apiClient
+     */
+    constructor(deps) {
+        super();
+        this.Suppression = deps.Suppression;
+        this.apiClient = deps.apiClient;
+    }
+
+    async removeEmail(email) {
+        try {
+            await this.apiClient.removeBounce(email);
+            await this.apiClient.removeComplaint(email);
+            await this.apiClient.removeUnsubscribe(email);
+        } catch (err) {
+            logging.error(err);
+            return false;
+        }
+
+        try {
+            await this.Suppression.destroy({
+                destroyBy: {
+                    email_address: email
+                }
+            });
+        } catch (err) {
+            logging.error(err);
+            return false;
+        }
+
+        return true;
+    }
+
+    async getSuppressionData(email) {
+        try {
+            const model = await this.Suppression.findOne({
+                email_address: email
+            });
+
+            if (!model) {
+                return new EmailSuppressionData(false);
+            }
+
+            return new EmailSuppressionData(true, {
+                timestamp: model.get('created_at'),
+                reason: model.get('reason') === 'spam' ? 'spam' : 'fail'
+            });
+        } catch (err) {
+            logging.error(err);
+            return new EmailSuppressionData(false);
+        }
+    }
+
+    async getBulkSuppressionData(emails) {
+        if (emails.length === 0) {
+            return [];
+        }
+
+        try {
+            const collection = await this.Suppression.findAll({
+                filter: `email_address:[${emails.join(',')}]`
+            });
+
+            return emails.map((email) => {
+                const model = collection.models.find(m => m.get('email_address') === email);
+
+                if (!model) {
+                    return new EmailSuppressionData(false);
+                }
+
+                return new EmailSuppressionData(true, {
+                    timestamp: model.get('created_at'),
+                    reason: model.get('reason') === 'spam' ? 'spam' : 'fail'
+                });
+            });
+        } catch (err) {
+            logging.error(err);
+            return emails.map(() => new EmailSuppressionData(false));
+        }
+    }
+
+    async init() {
+        const handleEvent = async (event) => {
+            try {
+                await this.Suppression.add({
+                    email_address: event.email,
+                    email_id: event.emailId,
+                    reason: 'bounce',
+                    created_at: event.timestamp
+                });
+            } catch (err) {
+                if (err.code !== 'ER_DUP_ENTRY') {
+                    logging.error(err);
+                }
+            }
+        };
+        DomainEvents.subscribe(EmailBouncedEvent, handleEvent);
+        DomainEvents.subscribe(SpamComplaintEvent, handleEvent);
+    }
+}
+
+module.exports = MailgunEmailSuppressionList;