ghost 5.33.7 → 5.34.0

package/core/server/services/email-analytics/jobs/fetch-latest/index.js

@@ -1,4 +1,5 @@
 const {parentPort} = require('worker_threads');
+const StartEmailAnalyticsJobEvent = require('../../events/StartEmailAnalyticsJobEvent');
 
 // recurring job to fetch analytics since the most recently seen event timestamp
 
@@ -24,22 +25,15 @@ if (parentPort) {
 }
 
 (async () => {
-    const {run} = require('./run');
-    const {eventStats, aggregateEndDate, fetchStartDate} = await run({
-        domainEvents: {
-            dispatch(event) {
-                parentPort.postMessage({
-                    event: {
-                        type: event.constructor.name, 
-                        data: event
-                    }
-                });
-            }
+    // We send an evnet message, so that it is emitted on the main thread by the job manager
+    // This will start the email analytics job on the main thread (the wrapper service is listening for this event)
+    parentPort.postMessage({
+        event: {
+            type: StartEmailAnalyticsJobEvent.name
         }
     });
 
     if (parentPort) {
-        parentPort.postMessage(`Fetched ${eventStats.totalEvents} events and aggregated stats for ${eventStats.emailIds.length} emails in ${aggregateEndDate - fetchStartDate}ms`);
         parentPort.postMessage('done');
     } else {
         // give the logging pipes time finish writing before exit

package/core/server/services/email-analytics/wrapper.js

@@ -0,0 +1,99 @@
+const logging = require('@tryghost/logging');
+const debug = require('@tryghost/debug')('jobs:email-analytics:fetch-latest');
+
+class EmailAnalyticsServiceWrapper {
+    init() {
+        if (this.service) {
+            return;
+        }
+
+        const {EmailAnalyticsService} = require('@tryghost/email-analytics-service');
+        const {EmailEventStorage, EmailEventProcessor} = require('@tryghost/email-service');
+        const MailgunProvider = require('@tryghost/email-analytics-provider-mailgun');
+        const {EmailRecipientFailure, EmailSpamComplaintEvent} = require('../../models');
+        const StartEmailAnalyticsJobEvent = require('./events/StartEmailAnalyticsJobEvent');
+
+        const domainEvents = require('@tryghost/domain-events');
+        const config = require('../../../shared/config');
+        const settings = require('../../../shared/settings-cache');
+        const db = require('../../data/db');
+        const queries = require('./lib/queries');
+        const membersService = require('../members');
+        const membersRepository = membersService.api.members;
+
+        this.eventStorage = new EmailEventStorage({
+            db,
+            membersRepository,
+            models: {
+                EmailRecipientFailure,
+                EmailSpamComplaintEvent
+            }
+        });
+
+        // Since this is running in a worker thread, we cant dispatch directly
+        // So we post the events as a message to the job manager
+        const eventProcessor = new EmailEventProcessor({
+            domainEvents,
+            db,
+            eventStorage: this.eventStorage
+        });
+
+        this.service = new EmailAnalyticsService({
+            config,
+            settings,
+            eventProcessor,
+            providers: [
+                new MailgunProvider({config, settings})
+            ],
+            queries
+        });
+
+        // We currently cannot trigger a non-offloaded job from the job manager
+        // So the email analytics jobs simply emits an event.
+        domainEvents.subscribe(StartEmailAnalyticsJobEvent, async () => {
+            await this.startFetch();
+        });
+    }
+
+    async fetchLatest() {
+        const fetchStartDate = new Date();
+        debug('Starting email analytics fetch of latest events');
+        const eventStats = await this.service.fetchLatest();
+        const fetchEndDate = new Date();
+        debug(`Finished fetching ${eventStats.totalEvents} analytics events in ${fetchEndDate.getTime() - fetchStartDate.getTime()}ms`);
+
+        const aggregateStartDate = new Date();
+        debug(`Starting email analytics aggregation for ${eventStats.emailIds.length} emails`);
+        await this.service.aggregateStats(eventStats);
+        const aggregateEndDate = new Date();
+        debug(`Finished aggregating email analytics in ${aggregateEndDate.getTime() - aggregateStartDate.getTime()}ms`);
+
+        logging.info(`Fetched ${eventStats.totalEvents} events and aggregated stats for ${eventStats.emailIds.length} emails in ${aggregateEndDate.getTime() - fetchStartDate.getTime()}ms`);
+
+        return eventStats;
+    }
+
+    async startFetch() {
+        if (this.fetching) {
+            logging.info('Email analytics fetch already running, skipping');
+            return;
+        }
+        this.fetching = true;
+
+        logging.info('Email analytics fetch started');
+        try {
+            const eventStats = await this.fetchLatest();
+
+            this.fetching = false;
+            return eventStats;
+        } catch (e) {
+            logging.error(e, 'Error while fetching email analytics');
+
+            // Log again only the error, otherwise we lose the stack trace
+            logging.error(e);
+        }
+        this.fetching = false;
+    }
+}
+
+module.exports = EmailAnalyticsServiceWrapper;

package/core/server/services/email-service/wrapper.js

@@ -13,8 +13,8 @@ class EmailServiceWrapper {
             return;
         }
 
-        const {EmailService, EmailController, EmailRenderer, SendingService, BatchSendingService, EmailSegmenter, EmailEventStorage, MailgunEmailProvider} = require('@tryghost/email-service');
-        const {Post, Newsletter, Email, EmailBatch, EmailRecipient, Member, EmailRecipientFailure, EmailSpamComplaintEvent} = require('../../models');
+        const {EmailService, EmailController, EmailRenderer, SendingService, BatchSendingService, EmailSegmenter, MailgunEmailProvider} = require('@tryghost/email-service');
+        const {Post, Newsletter, Email, EmailBatch, EmailRecipient, Member} = require('../../models');
         const MailgunClient = require('@tryghost/mailgun-client');
         const configService = require('../../../shared/config');
         const settingsCache = require('../../../shared/settings-cache');
@@ -25,7 +25,6 @@ class EmailServiceWrapper {
         const sentry = require('../../../shared/sentry');
         const membersRepository = membersService.api.members;
         const limitService = require('../limits');
-        const domainEvents = require('@tryghost/domain-events');
 
         const mobiledocLib = require('../../lib/mobiledoc');
         const lexicalLib = require('../../lib/lexical');
@@ -59,7 +58,7 @@ class EmailServiceWrapper {
             settingsHelpers,
             renderers: {
                 mobiledoc: mobiledocLib.mobiledocHtmlRenderer,
-                lexical: lexicalLib.lexicalHtmlRenderer
+                lexical: lexicalLib
             },
             imageSize,
             urlUtils,
@@ -116,16 +115,6 @@ class EmailServiceWrapper {
                 Email
             }
         });
-
-        this.eventStorage = new EmailEventStorage({
-            db,
-            membersRepository,
-            models: {
-                EmailRecipientFailure,
-                EmailSpamComplaintEvent
-            }
-        });
-        this.eventStorage.listen(domainEvents);
     }
 }
 

package/core/server/services/mega/post-email-serializer.js

@@ -303,7 +303,7 @@ const PostEmailSerializer = {
         }
 
         if (post.lexical) {
-            post.html = lexicalLib.lexicalHtmlRenderer.render(
+            post.html = lexicalLib.render(
                 post.lexical, {target: 'email', postUrl: post.url}
             );
         } else {

package/core/server/services/members/jobs/index.js

@@ -14,10 +14,12 @@ module.exports = {
         ) {
             // use a random seconds value to avoid spikes to external APIs on the minute
             const s = Math.floor(Math.random() * 60); // 0-59
+            const m = Math.floor(Math.random() * 60); // 0-59
+            const h = Math.floor(Math.random() * 6); // 0-5
 
-            // Run everyday at 12:05:X AM to clean all expired complimentary subscriptions
+            // Run everyday at {0-5}:XX:XX AM to clean all expired complimentary subscriptions
             jobsService.addJob({
-                at: `${s} 5 0 * * *`,
+                at: `${s} ${m} ${h} * * *`,
                 job: path.resolve(__dirname, 'clean-expired-comped.js'),
                 name: 'clean-expired-comped'
             });

package/core/server/services/mentions/BookshelfMentionRepository.js

@@ -54,7 +54,7 @@ module.exports = class BookshelfMentionRepository {
             sourceAuthor: model.get('source_author'),
             sourceExcerpt: model.get('source_excerpt'),
             sourceFavicon: model.get('source_favicon'),
-            sourceFeaturedImaged: model.get('source_featured_image')
+            sourceFeaturedImage: model.get('source_featured_image')
         });
     }
 
@@ -106,7 +106,8 @@ module.exports = class BookshelfMentionRepository {
             target: mention.target.href,
             resource_id: mention.resourceId?.toHexString(),
             resource_type: mention.resourceId ? 'post' : null,
-            payload: mention.payload ? JSON.stringify(mention.payload) : null
+            payload: mention.payload ? JSON.stringify(mention.payload) : null,
+            deleted: Mention.isDeleted(mention)
         };
 
         const existing = await this.#MentionModel.findOne({id: data.id}, {require: false});

package/core/server/services/mentions/MentionController.js

@@ -10,17 +10,53 @@ const logging = require('@tryghost/logging');
  * @typedef {import('@tryghost/webmentions/lib/MentionsAPI').Page} Page<Model>
  */
 
+/**
+ * @typedef {object} MentionResource
+ * @prop {ObjectID} id
+ * @prop {string} type
+ * @prop {string} name
+ */
+
+/**
+ * @typedef {Mention} MentionDTO
+ * @prop {Resource} resource
+ */
+
+/**
+ * @typedef {object} IJobService
+ * @prop {(name: string, fn: Function) => void} addJob
+ */
+
+/**
+ * @typedef {object} IMentionResourceService
+ * @prop {(id: ObjectID)  => Promise<MentionResource>} getByID
+ */
+
 module.exports = class MentionController {
     /** @type {import('@tryghost/webmentions/lib/MentionsAPI')} */
     #api;
 
+    /** @type {IJobService} */
+    #jobService;
+
+    /** @type {IMentionResourceService} */
+    #mentionResourceService;
+
+    /**
+     * @param {object} deps
+     * @param {import('@tryghost/webmentions/lib/MentionsAPI')} deps.api
+     * @param {IJobService} deps.jobService
+     * @param {IMentionResourceService} deps.mentionResourceService
+     */
     async init(deps) {
         this.#api = deps.api;
+        this.#jobService = deps.jobService;
+        this.#mentionResourceService = deps.mentionResourceService;
     }
 
     /**
      * @param {import('@tryghost/api-framework').Frame} frame
-     * @returns {Promise<Page<Mention>>}
+     * @returns {Promise<Page<MentionDTO>>}
      */
     async browse(frame) {
         let limit;
@@ -37,13 +73,41 @@ module.exports = class MentionController {
             page = 1;
         }
 
-        const results = await this.#api.listMentions({
+        let order;
+        if (frame.options.order && frame.options.order === 'created_at desc') {
+            order = 'created_at desc';
+        } else {
+            order = 'created_at asc';
+        }
+
+        const mentions = await this.#api.listMentions({
             filter: frame.options.filter,
+            order,
             limit,
             page
         });
 
-        return results;
+        const resources = await Promise.all(mentions.data.map((mention) => {
+            return this.#mentionResourceService.getByID(mention.resourceId);
+        }));
+
+        /** @type {Page<MentionDTO>} */
+        const result = {
+            data: mentions.data.map((mention, index) => {
+                const mentionDTO = {
+                    ...mention.toJSON(),
+                    resource: resources[index],
+                    toJSON() {
+                        return mentionDTO;
+                    }
+                };
+                delete mentionDTO.resourceId;
+                return mentionDTO;
+            }),
+            meta: mentions.meta
+        };
+
+        return result;
     }
 
     /**
@@ -52,15 +116,17 @@ module.exports = class MentionController {
      */
     async receive(frame) {
         logging.info('[Webmention] ' + JSON.stringify(frame.data));
-        const {source, target, ...payload} = frame.data;
-        const result = this.#api.processWebmention({
-            source: new URL(source),
-            target: new URL(target),
-            payload
-        });
-
-        result.catch(function rejected(err) {
-            logging.error(err);
+        this.#jobService.addJob('processWebmention', async () => {
+            const {source, target, ...payload} = frame.data;
+            try {
+                await this.#api.processWebmention({
+                    source: new URL(source),
+                    target: new URL(target),
+                    payload
+                });
+            } catch (err) {
+                logging.error(err);
+            }
         });
     }
 };

package/core/server/services/mentions/service.js

@@ -13,9 +13,10 @@ const events = require('../../lib/common/events');
 const externalRequest = require('../../../server/lib/request-external.js');
 const urlUtils = require('../../../shared/url-utils');
 const outputSerializerUrlUtil = require('../../../server/api/endpoints/utils/serializers/output/utils/url');
-const labs = require('../../../shared/labs');
 const urlService = require('../url');
+const settingsCache = require('../../../shared/settings-cache');
 const DomainEvents = require('@tryghost/domain-events');
+const jobsService = require('../jobs');
 
 function getPostUrl(post) {
     const jsonModel = {};
@@ -50,14 +51,53 @@ module.exports = {
             routingService
         });
 
-        this.controller.init({api});
+        this.controller.init({
+            api,
+            jobService: {
+                async addJob(name, fn) {
+                    jobsService.addJob({
+                        name,
+                        job: fn,
+                        offloaded: false
+                    });
+                }
+            },
+            mentionResourceService: {
+                async getByID(id) {
+                    if (!id) {
+                        return null;
+                    }
+
+                    const post = await models.Post.findOne({id: id.toHexString()});
+
+                    if (!post) {
+                        return null;
+                    }
+
+                    return {
+                        id: id,
+                        name: post.get('title'),
+                        type: 'post'
+                    };
+                }
+            }
+        });
 
         const sendingService = new MentionSendingService({
             discoveryService,
             externalRequest,
             getSiteUrl: () => urlUtils.urlFor('home', true),
             getPostUrl: post => getPostUrl(post),
-            isEnabled: () => labs.isSet('webmentions')
+            isEnabled: () => !settingsCache.get('is_private'),
+            jobService: {
+                async addJob(name, fn) {
+                    jobsService.addJob({
+                        name,
+                        job: fn,
+                        offloaded: false
+                    });
+                }
+            }
         });
         sendingService.listen(events);
     }

package/core/server/services/milestone-emails/MilestoneQueries.js

@@ -0,0 +1,58 @@
+const MIN_DAYS_SINCE_IMPORTED = 7;
+
+module.exports = class MilestoneQueries {
+    #db;
+
+    constructor(deps) {
+        this.#db = deps.db;
+    }
+
+    /**
+     * @returns {Promise<number>}
+     */
+    async getMembersCount() {
+        const [membersCount] = await this.#db.knex('members').count('id as count');
+
+        return membersCount?.count || 0;
+    }
+
+    /**
+     * @returns {Promise<Array>}
+     */
+    async getARR() {
+        const currentARR = await this.#db.knex('members_paid_subscription_events as stripe')
+            .select(this.#db.knex.raw('ROUND(SUM(stripe.mrr_delta) * 12) / 100 AS arr, stripe.currency as currency'))
+            .groupBy('stripe.currency');
+
+        return currentARR;
+    }
+
+    /**
+     * @returns {Promise<boolean>}
+     */
+    async hasImportedMembersInPeriod() {
+        const [hasImportedMembers] = await this.#db.knex('members_subscribe_events')
+            .count('id as count')
+            .where('source', '=', 'import')
+            .where('created_at', '>=', MIN_DAYS_SINCE_IMPORTED);
+
+        return hasImportedMembers?.count > 0;
+    }
+
+    /**
+     * @returns {Promise<string>}
+     */
+    async getDefaultCurrency() {
+        const currentARR = await this.getARR();
+
+        // Set the default currency as the one with the highest value
+        if (currentARR.length > 1) {
+            const highestValues = currentARR.sort((a, b) => b.arr - a.arr);
+            return highestValues?.[0]?.currency;
+        } else if (currentARR?.[0]?.currency) {
+            return currentARR[0].currency;
+        } else {
+            return 'usd';
+        }
+    }
+};

package/core/server/services/milestone-emails/index.js

@@ -0,0 +1 @@
+module.exports = require('./service');

package/core/server/services/milestone-emails/service.js

@@ -0,0 +1,58 @@
+// Stubbing stripe in test was causing issues. Moved it
+// into this function to be able to rewire and stub the
+// expected return value.
+const getStripeLiveEnabled = () => {
+    const stripeService = require('../stripe');
+    // This seems to be the only true way to check if Stripe is configured in live mode
+    // settingsCache only cares if Stripe is enabled
+    return stripeService.api.configured && stripeService.api.mode === 'live';
+};
+
+/**
+ *
+ * @returns {Promise<any>}
+ */
+module.exports = {
+    async initAndRun() {
+        const labs = require('../../../shared/labs');
+
+        if (labs.isSet('milestoneEmails')) {
+            const db = require('../../data/db');
+            const MilestoneQueries = require('./MilestoneQueries');
+
+            const {
+                MilestonesEmailService,
+                InMemoryMilestoneRepository
+            } = require('@tryghost/milestone-emails');
+            const config = require('../../../shared/config');
+            const milestonesConfig = config.get('milestones');
+            const {GhostMailer} = require('../mail');
+
+            const mailer = new GhostMailer();
+            const repository = new InMemoryMilestoneRepository();
+            const queries = new MilestoneQueries({db});
+
+            const milestonesEmailService = new MilestonesEmailService({
+                mailer,
+                repository,
+                milestonesConfig, // avoid using getters and pass as JSON
+                queries
+            });
+
+            let arrResult;
+
+            // @TODO: schedule recurring jobs instead
+            const membersResult = await milestonesEmailService.checkMilestones('members');
+            const stripeLiveEnabled = getStripeLiveEnabled();
+
+            if (stripeLiveEnabled) {
+                arrResult = await milestonesEmailService.checkMilestones('arr');
+            }
+
+            return {
+                members: membersResult,
+                arr: arrResult
+            };
+        }
+    }
+};

package/core/server/services/tags-public/index.js

@@ -0,0 +1 @@
+module.exports = require('./service');

package/core/server/services/tags-public/service.js

@@ -0,0 +1,31 @@
+class TagsPublicServiceWrapper {
+    async init() {
+        if (this.api) {
+            // Already done
+            return;
+        }
+
+        // Wire up all the dependencies
+        const models = require('../../models');
+        const adapterManager = require('../adapter-manager');
+        const config = require('../../../shared/config');
+
+        let tagsCache;
+        if (config.get('hostSettings:tagsPublicCache:enabled')) {
+            tagsCache = adapterManager.getAdapter('cache:tagsPublic');
+        }
+
+        const {TagsPublicRepository} = require('@tryghost/tags-public');
+
+        this.linkRedirectRepository = new TagsPublicRepository({
+            Tag: models.TagPublic,
+            cache: tagsCache
+        });
+
+        this.api = {
+            browse: this.linkRedirectRepository.getAll.bind(this.linkRedirectRepository)
+        };
+    }
+}
+
+module.exports = new TagsPublicServiceWrapper();

package/core/server/web/api/endpoints/admin/routes.js

@@ -240,7 +240,6 @@ module.exports = function apiRoutes() {
         mw.authAdminApi,
         apiMw.upload.single('file'),
         apiMw.upload.validation({type: 'images'}),
-        apiMw.normalizeImage,
         http(api.images.upload)
     );
 

package/core/server/web/api/middleware/index.js

@@ -1,6 +1,5 @@
 module.exports = {
     cors: require('./cors'),
-    normalizeImage: require('./normalize-image'),
     updateUserLastSeen: require('./update-user-last-seen'),
     upload: require('./upload'),
     versionMatch: require('./version-match')

package/core/server/web/shared/middleware/api/spam-prevention.js

@@ -20,7 +20,8 @@ const messages = {
         error: 'Only {rateSigninAttempts} tries per IP address every {rateSigninPeriod} seconds.',
         context: 'Too many login attempts.'
     },
-    tooManyAttempts: 'Too many attempts.'
+    tooManyAttempts: 'Too many attempts.',
+    webmentionsBlock: 'Too many mention attempts'
 };
 let spamPrivateBlock = spam.private_block || {};
 let spamGlobalBlock = spam.global_block || {};
@@ -29,12 +30,14 @@ let spamUserReset = spam.user_reset || {};
 let spamUserLogin = spam.user_login || {};
 let spamMemberLogin = spam.member_login || {};
 let spamContentApiKey = spam.content_api_key || {};
+let spamWebmentionsBlock = spam.webmentions_block || {};
 
 let store;
 let memoryStore;
 let privateBlogInstance;
 let globalResetInstance;
 let globalBlockInstance;
+let webmentionsBlockInstance;
 let userLoginInstance;
 let membersAuthInstance;
 let membersAuthEnumerationInstance;
@@ -123,6 +126,32 @@ const globalReset = () => {
     return globalResetInstance;
 };
 
+const webmentionsBlock = () => {
+    const ExpressBrute = require('express-brute');
+    const BruteKnex = require('brute-knex');
+    const db = require('../../../../data/db');
+
+    store = store || new BruteKnex({
+        tablename: 'brute',
+        createTable: false,
+        knex: db.knex
+    });
+
+    webmentionsBlockInstance = webmentionsBlockInstance || new ExpressBrute(store,
+        extend({
+            attachResetToRequest: false,
+            failCallback(req, res, next) {
+                return next(new errors.TooManyRequestsError({
+                    message: messages.webmentionsBlock
+                }));
+            },
+            handleStoreError: handleStoreError
+        }, pick(spamWebmentionsBlock, spamConfigKeys))
+    );
+
+    return webmentionsBlockInstance;
+};
+
 const membersAuth = () => {
     const ExpressBrute = require('express-brute');
     const BruteKnex = require('brute-knex');
@@ -319,6 +348,7 @@ module.exports = {
     userReset: userReset,
     privateBlog: privateBlog,
     contentApiKey: contentApiKey,
+    webmentionsBlock: webmentionsBlock,
     reset: () => {
         store = undefined;
         memoryStore = undefined;

package/core/server/web/shared/middleware/brute.js

@@ -104,5 +104,18 @@ module.exports = {
      */
     membersAuthEnumeration(req, res, next) {
         return spamPrevention.membersAuthEnumeration().prevent(req, res, next);
+    },
+
+    /**
+     * Blocks webmention spam
+     */
+
+    webmentionsLimiter(req, res, next) {
+        return spamPrevention.webmentionsBlock().getMiddleware({
+            ignoreIP: false,
+            key(_req, _res, _next) {
+                return _next('webmention_blocked');
+            }
+        })(req, res, next);
     }
 };