ghost 5.35.0 → 5.36.0

package/core/server/services/email-analytics/lib/queries.js

@@ -15,9 +15,24 @@ module.exports = {
         const startDate = new Date();
 
         // three separate queries is much faster than using max/greatest (with coalesce to handle nulls) across columns
-        const {maxDeliveredAt} = await db.knex('email_recipients').select(db.knex.raw('MAX(delivered_at) as maxDeliveredAt')).first() || {};
-        const {maxOpenedAt} = await db.knex('email_recipients').select(db.knex.raw('MAX(opened_at) as maxOpenedAt')).first() || {};
-        const {maxFailedAt} = await db.knex('email_recipients').select(db.knex.raw('MAX(failed_at) as maxFailedAt')).first() || {};
+        let {maxDeliveredAt} = await db.knex('email_recipients').select(db.knex.raw('MAX(delivered_at) as maxDeliveredAt')).first() || {};
+        let {maxOpenedAt} = await db.knex('email_recipients').select(db.knex.raw('MAX(opened_at) as maxOpenedAt')).first() || {};
+        let {maxFailedAt} = await db.knex('email_recipients').select(db.knex.raw('MAX(failed_at) as maxFailedAt')).first() || {};
+
+        if (maxDeliveredAt && !(maxDeliveredAt instanceof Date)) {
+            // SQLite returns a string instead of a Date
+            maxDeliveredAt = new Date(maxDeliveredAt);
+        }
+
+        if (maxOpenedAt && !(maxOpenedAt instanceof Date)) {
+            // SQLite returns a string instead of a Date
+            maxOpenedAt = new Date(maxOpenedAt);
+        }
+
+        if (maxFailedAt && !(maxFailedAt instanceof Date)) {
+            // SQLite returns a string instead of a Date
+            maxFailedAt = new Date(maxFailedAt);
+        }
 
         const lastSeenEventTimestamp = _.max([maxDeliveredAt, maxOpenedAt, maxFailedAt]);
         debug(`getLastSeenEventTimestamp: finished in ${Date.now() - startDate}ms`);

package/core/server/services/email-analytics/wrapper.js

@@ -1,5 +1,4 @@
 const logging = require('@tryghost/logging');
-const debug = require('@tryghost/debug')('jobs:email-analytics:fetch-latest');
 
 class EmailAnalyticsServiceWrapper {
     init() {
@@ -55,22 +54,40 @@ class EmailAnalyticsServiceWrapper {
         });
     }
 
-    async fetchLatest() {
+    async fetchLatest({maxEvents} = {maxEvents: Infinity}) {
+        logging.info('[EmailAnalytics] Fetch latest started');
+
         const fetchStartDate = new Date();
-        debug('Starting email analytics fetch of latest events');
-        const eventStats = await this.service.fetchLatest();
+        const totalEvents = await this.service.fetchLatest({maxEvents});
         const fetchEndDate = new Date();
-        debug(`Finished fetching ${eventStats.totalEvents} analytics events in ${fetchEndDate.getTime() - fetchStartDate.getTime()}ms`);
 
-        const aggregateStartDate = new Date();
-        debug(`Starting email analytics aggregation for ${eventStats.emailIds.length} emails`);
-        await this.service.aggregateStats(eventStats);
-        const aggregateEndDate = new Date();
-        debug(`Finished aggregating email analytics in ${aggregateEndDate.getTime() - aggregateStartDate.getTime()}ms`);
+        logging.info(`[EmailAnalytics] Fetched ${totalEvents} events and aggregated stats in ${fetchEndDate.getTime() - fetchStartDate.getTime()}ms (latest)`);
+        return totalEvents;
+    }
 
-        logging.info(`Fetched ${eventStats.totalEvents} events and aggregated stats for ${eventStats.emailIds.length} emails in ${aggregateEndDate.getTime() - fetchStartDate.getTime()}ms`);
+    async fetchMissing({maxEvents} = {maxEvents: Infinity}) {
+        logging.info('[EmailAnalytics] Fetch missing started');
 
-        return eventStats;
+        const fetchStartDate = new Date();
+        const totalEvents = await this.service.fetchMissing({maxEvents});
+        const fetchEndDate = new Date();
+
+        logging.info(`[EmailAnalytics] Fetched ${totalEvents} events and aggregated stats in ${fetchEndDate.getTime() - fetchStartDate.getTime()}ms (missing)`);
+        return totalEvents;
+    }
+
+    async fetchScheduled({maxEvents}) {
+        if (maxEvents < 300) {
+            return 0;
+        }
+        logging.info('[EmailAnalytics] Fetch scheduled started');
+
+        const fetchStartDate = new Date();
+        const totalEvents = await this.service.fetchScheduled({maxEvents});
+        const fetchEndDate = new Date();
+
+        logging.info(`[EmailAnalytics] Fetched ${totalEvents} events and aggregated stats in ${fetchEndDate.getTime() - fetchStartDate.getTime()}ms (scheduled)`);
+        return totalEvents;
     }
 
     async startFetch() {
@@ -80,12 +97,14 @@ class EmailAnalyticsServiceWrapper {
         }
         this.fetching = true;
 
-        logging.info('Email analytics fetch started');
         try {
-            const eventStats = await this.fetchLatest();
+            const c1 = await this.fetchLatest({maxEvents: Infinity});
+            const c2 = await this.fetchMissing({maxEvents: Infinity});
+
+            // Only fetch scheduled if we didn't fetch a lot of normal events
+            await this.fetchScheduled({maxEvents: 20000 - c1 - c2});
 
             this.fetching = false;
-            return eventStats;
         } catch (e) {
             logging.error(e, 'Error while fetching email analytics');
 

package/core/server/services/email-suppression-list/MailgunEmailSuppressionList.js

@@ -2,6 +2,7 @@ const {AbstractEmailSuppressionList, EmailSuppressionData, EmailSuppressedEvent}
 const {SpamComplaintEvent, EmailBouncedEvent} = require('@tryghost/email-events');
 const DomainEvents = require('@tryghost/domain-events');
 const logging = require('@tryghost/logging');
+const models = require('../../models');
 
 /**
  * @typedef {object} IMailgunAPIClient
@@ -95,6 +96,7 @@ class MailgunEmailSuppressionList extends AbstractEmailSuppressionList {
     }
 
     async init() {
+        this.Suppression = models.Suppression;
         const handleEvent = reason => async (event) => {
             try {
                 if (reason === 'bounce') {

package/core/server/services/mentions/BookshelfMentionRepository.js

@@ -49,6 +49,7 @@ module.exports = class BookshelfMentionRepository {
             timestamp: model.get('created_at'),
             payload,
             resourceId: model.get('resource_id'),
+            resourceType: model.get('resource_type'),
             sourceTitle: model.get('source_title'),
             sourceSiteTitle: model.get('source_site_title'),
             sourceAuthor: model.get('source_author'),
@@ -106,7 +107,7 @@ module.exports = class BookshelfMentionRepository {
             source_favicon: mention.sourceFavicon?.href,
             target: mention.target.href,
             resource_id: mention.resourceId?.toHexString(),
-            resource_type: mention.resourceId ? 'post' : null,
+            resource_type: mention.resourceType,
             payload: mention.payload ? JSON.stringify(mention.payload) : null,
             deleted: Mention.isDeleted(mention),
             verified: mention.verified

package/core/server/services/mentions/ResourceService.js

@@ -37,6 +37,12 @@ module.exports = class ResourceService {
                 id: ObjectID.createFromHexString(resource.data.id)
             };
         }
+        if (resource?.config?.type === 'pages') {
+            return {
+                type: 'page',
+                id: ObjectID.createFromHexString(resource.data.id)
+            };
+        }
         return {
             type: null,
             id: null

package/core/server/services/mentions/RoutingService.js

@@ -56,7 +56,8 @@ module.exports = class RoutingService {
 
         try {
             const response = await this.#externalRequest.head(url, {
-                followRedirect: false
+                followRedirect: false,
+                throwHttpErrors: false
             });
             if (response.statusCode < 400 && response.statusCode > 199) {
                 return true;

package/core/server/services/mentions/service.js

@@ -70,17 +70,15 @@ module.exports = {
                     if (!id) {
                         return null;
                     }
-
                     const post = await models.Post.findOne({id: id.toHexString()});
 
                     if (!post) {
                         return null;
                     }
-
                     return {
                         id: id,
                         name: post.get('title'),
-                        type: 'post'
+                        type: post.get('type')
                     };
                 }
             }

package/core/server/services/milestones/BookshelfMilestoneRepository.js

@@ -0,0 +1,136 @@
+const {Milestone} = require('@tryghost/milestones');
+
+/**
+ * @typedef {import('@tryghost/milestones/lib/MilestonesService').IMilestoneRepository} IMilestoneRepository
+ * @typedef {import('@tryghost/milestones/lib/MilestonesService')} Milestone
+ */
+
+/**
+ * @implements {IMilestoneRepository}
+ */
+module.exports = class BookshelfMilestoneRepository {
+    /** @type {Object} */
+    #MilestoneModel;
+
+    /** @type {import('@tryghost/domain-events')} */
+    #DomainEvents;
+
+    /**
+     * @param {object} deps
+     * @param {object} deps.MilestoneModel Bookshelf Model
+     * @param {import('@tryghost/domain-events')} deps.DomainEvents
+     */
+    constructor(deps) {
+        this.#MilestoneModel = deps.MilestoneModel;
+        this.#DomainEvents = deps.DomainEvents;
+    }
+
+    #modelToMilestone(model) {
+        return Milestone.create({
+            id: model.get('id'),
+            type: model.get('type'),
+            value: model.get('value'),
+            currency: model.get('currency'),
+            createdAt: model.get('created_at'),
+            emailSentAt: model.get('email_sent_at')
+        });
+    }
+
+    /**
+     * @param {import('@tryghost/milestones/lib/Milestone')} milestone
+     * @returns {Promise<void>}
+     */
+    async save(milestone) {
+        const data = {
+            id: milestone.id.toHexString(),
+            type: milestone.type,
+            value: milestone.value,
+            currency: milestone?.currency,
+            created_at: milestone?.createdAt,
+            email_sent_at: milestone?.emailSentAt
+        };
+
+        const existing = await this.#MilestoneModel.findOne({id: data.id}, {require: false});
+
+        if (!existing) {
+            await this.#MilestoneModel.add(data);
+        } else {
+            await this.#MilestoneModel.edit(data, {
+                id: data.id
+            });
+        }
+        for (const event of milestone.events) {
+            this.#DomainEvents.dispatch(event);
+        }
+    }
+
+    /**
+     * @param {'arr'|'members'} type
+     * @param {string} [currency]
+     *
+     * @returns {Promise<import('@tryghost/milestones/lib/Milestone')|null>}
+     */
+    async getLatestByType(type, currency = 'usd') {
+        let milestone = null;
+
+        if (type === 'arr') {
+            milestone = await this.#MilestoneModel.findAll({filter: `currency:${currency}+type:arr`, order: 'created_at ASC, value DESC'}, {require: false});
+        } else {
+            milestone = await this.#MilestoneModel.findAll({filter: 'type:members', order: 'created_at ASC, value DESC'}, {require: false});
+        }
+
+        if (!milestone || !milestone?.models?.length) {
+            return null;
+        } else {
+            milestone = milestone.models?.[0];
+        }
+
+        return this.#modelToMilestone(milestone);
+    }
+
+    /**
+     * @returns {Promise<import('@tryghost/milestones/lib/Milestone')|null>}
+     */
+    async getLastEmailSent() {
+        let milestone = await this.#MilestoneModel.findAll({filter: 'email_sent_at:-null', order: 'email_sent_at ASC'}, {require: false});
+
+        if (!milestone || !milestone?.models?.length) {
+            return null;
+        } else {
+            milestone = milestone.models?.[0];
+        }
+
+        return this.#modelToMilestone(milestone);
+    }
+
+    /**
+     * @param {number} value
+     * @param {string} [currency]
+     *
+     * @returns {Promise<import('@tryghost/milestones/lib/Milestone')|null>}
+     */
+    async getByARR(value, currency = 'usd') {
+        // find a milestone of the ARR type by a given value
+        const milestone = await this.#MilestoneModel.findOne({type: 'arr', currency: currency, value: value}, {require: false});
+
+        if (!milestone) {
+            return null;
+        }
+        return this.#modelToMilestone(milestone);
+    }
+
+    /**
+     * @param {number} value
+     *
+     * @returns {Promise<import('@tryghost/milestones/lib/Milestone')|null>}
+     */
+    async getByCount(value) {
+        // find a milestone of the members type by a given value
+        const milestone = await this.#MilestoneModel.findOne({type: 'members', value: value}, {require: false});
+
+        if (!milestone) {
+            return null;
+        }
+        return this.#modelToMilestone(milestone);
+    }
+};

package/core/server/services/milestones/MilestoneQueries.js

@@ -1,10 +1,12 @@
-const MIN_DAYS_SINCE_IMPORTED = 7;
-
 module.exports = class MilestoneQueries {
     #db;
 
+    /** @type {number} */
+    #minDaysSinceImported;
+
     constructor(deps) {
         this.#db = deps.db;
+        this.#minDaysSinceImported = deps.minDaysSinceImported;
     }
 
     /**
@@ -31,10 +33,13 @@ module.exports = class MilestoneQueries {
      * @returns {Promise<boolean>}
      */
     async hasImportedMembersInPeriod() {
+        const importedThreshold = new Date();
+        importedThreshold.setDate(importedThreshold.getDate() - this.#minDaysSinceImported);
+
         const [hasImportedMembers] = await this.#db.knex('members_subscribe_events')
             .count('id as count')
             .where('source', '=', 'import')
-            .where('created_at', '>=', MIN_DAYS_SINCE_IMPORTED);
+            .where('created_at', '>=', importedThreshold);
 
         return hasImportedMembers?.count > 0;
     }

package/core/server/services/milestones/service.js

@@ -1,4 +1,9 @@
 const DomainEvents = require('@tryghost/domain-events');
+const logging = require('@tryghost/logging');
+const models = require('../../models');
+const BookshelfMilestoneRepository = require('./BookshelfMilestoneRepository');
+
+const JOB_TIMEOUT = 1000 * 60 * 60 * 24 * (Math.floor(Math.random() * 4)); // 0 - 4 days;
 
 const getStripeLiveEnabled = () => {
     const settingsCache = require('../../../shared/settings-cache');
@@ -28,19 +33,23 @@ module.exports = {
             const db = require('../../data/db');
             const MilestoneQueries = require('./MilestoneQueries');
 
-            const {
-                MilestonesService,
-                InMemoryMilestoneRepository
-            } = require('@tryghost/milestones');
+            const {MilestonesService} = require('@tryghost/milestones');
             const config = require('../../../shared/config');
             const milestonesConfig = config.get('milestones');
 
-            const repository = new InMemoryMilestoneRepository({DomainEvents});
-            const queries = new MilestoneQueries({db});
+            const repository = new BookshelfMilestoneRepository({
+                DomainEvents,
+                MilestoneModel: models.Milestone
+            });
+
+            const queries = new MilestoneQueries({
+                db,
+                minDaysSinceImported: milestonesConfig?.minDaysSinceImported || 7
+            });
 
             this.api = new MilestonesService({
                 repository,
-                milestonesConfig, // avoid using getters and pass as JSON
+                milestonesConfig,
                 queries
             });
         }
@@ -69,10 +78,39 @@ module.exports = {
     },
 
     /**
+     *
+     * @param {number} [customTimeout]
+     *
+     *  @returns {Promise<object>}
+     */
+    async scheduleRun(customTimeout) {
+        const timeOut = customTimeout || JOB_TIMEOUT;
+
+        const today = new Date();
+        const msNow = today.getMilliseconds();
+        const newMs = msNow + timeOut;
+        const jobDate = today.setMilliseconds(newMs);
+
+        logging.info(`Running milestone emails job on ${new Date(jobDate).toString()}`);
+
+        return new Promise((resolve) => {
+            setTimeout(async () => {
+                const result = await this.run();
+                return resolve(result);
+            }, timeOut);
+        });
+    },
+
+    /**
+     * @param {number} [customTimeout]
+     * Only used temporary for testing purposes.
+     * Will be removed, after job scheduling implementation.
+     *
      * @returns {Promise<object>}
      */
-    async initAndRun() {
+    async initAndRun(customTimeout) {
         await this.init();
-        return await this.run();
+
+        return this.scheduleRun(customTimeout);
     }
 };

package/core/server/services/oembed/nft-oembed.js

@@ -42,9 +42,8 @@ class NFTOEmbedProvider {
             headers['X-API-KEY'] = this.dependencies.config.apiKey;
         }
         const result = await externalRequest(`https://api.opensea.io/api/v1/asset/${transaction}/${asset}/?format=json`, {
-            json: true,
             headers
-        });
+        }).json();
         return {
             version: '1.0',
             type: 'nft',

package/core/server/services/posts-public/service.js

@@ -6,24 +6,36 @@ class PostsPublicServiceWrapper {
         }
 
         // Wire up all the dependencies
-        const {Post} = require('../../models');
         const adapterManager = require('../adapter-manager');
         const config = require('../../../shared/config');
+        const EventAwareCacheWrapper = require('@tryghost/event-aware-cache-wrapper');
+        const EventRegistry = require('../../lib/common/events');
 
         let postsCache;
         if (config.get('hostSettings:postsPublicCache:enabled')) {
-            postsCache = adapterManager.getAdapter('cache:postsPublic');
+            const cache = adapterManager.getAdapter('cache:postsPublic');
+            postsCache = new EventAwareCacheWrapper({
+                cache: cache,
+                resetEvents: ['site.changed'],
+                eventRegistry: EventRegistry
+            });
         }
 
-        const {PublicResourcesRepository} = require('@tryghost/public-resource-repository');
-
-        this.postsRepository = new PublicResourcesRepository({
-            Model: Post,
-            cache: postsCache
-        });
+        let cache;
+        if (postsCache) {
+            // @NOTE: exposing cache through getter and setter to not loose the context of "this"
+            cache = {
+                get() {
+                    return postsCache.get(...arguments);
+                },
+                set() {
+                    return postsCache.set(...arguments);
+                }
+            };
+        }
 
         this.api = {
-            browse: this.postsRepository.getAll.bind(this.postsRepository)
+            cache: cache
         };
     }
 }

package/core/server/services/tags-public/service.js

@@ -6,24 +6,35 @@ class TagsPublicServiceWrapper {
         }
 
         // Wire up all the dependencies
-        const {TagPublic} = require('../../models');
         const adapterManager = require('../adapter-manager');
         const config = require('../../../shared/config');
+        const EventAwareCacheWrapper = require('@tryghost/event-aware-cache-wrapper');
+        const EventRegistry = require('../../lib/common/events');
 
         let tagsCache;
         if (config.get('hostSettings:tagsPublicCache:enabled')) {
-            tagsCache = adapterManager.getAdapter('cache:tagsPublic');
+            let tagsPublicCache = adapterManager.getAdapter('cache:tagsPublic');
+            tagsCache = new EventAwareCacheWrapper({
+                cache: tagsPublicCache,
+                resetEvents: ['site.changed'],
+                eventRegistry: EventRegistry
+            });
         }
 
-        const {PublicResourcesRepository} = require('@tryghost/public-resource-repository');
-
-        this.tagsPublicRepository = new PublicResourcesRepository({
-            Model: TagPublic,
-            cache: tagsCache
-        });
-
+        let cache;
+        if (tagsCache) {
+            // @NOTE: exposing cache through getter and setter to not loose the context of "this"
+            cache = {
+                get() {
+                    return tagsCache.get(...arguments);
+                },
+                set() {
+                    return tagsCache.set(...arguments);
+                }
+            };
+        }
         this.api = {
-            browse: this.tagsPublicRepository.getAll.bind(this.tagsPublicRepository)
+            cache: cache
         };
     }
 }

package/core/server/services/websockets/service.js

@@ -15,7 +15,8 @@ module.exports = {
             let count = 0;
 
             io.on(`connection`, (socket) => {
-                debug(`[Websockets] Client connected`);
+                logging.info(`Websockets client connected (id: ${socket.id})`);
+
                 // on connect, send current value
                 socket.emit('addCount', count);
                 // listen to to changes in value from client

package/core/server/web/api/endpoints/admin/routes.js

@@ -300,6 +300,9 @@ module.exports = function apiRoutes() {
     router.put('/emails/:id/retry', mw.authAdminApi, http(api.emails.retry));
     router.get('/emails/:id/batches', mw.authAdminApi, http(api.emails.browseBatches));
     router.get('/emails/:id/recipient-failures', mw.authAdminApi, http(api.emails.browseFailures));
+    router.get('/emails/:id/analytics', mw.authAdminApi, http(api.emails.analyticsStatus));
+    router.put('/emails/:id/analytics', mw.authAdminApi, http(api.emails.scheduleAnalytics));
+    router.delete('/emails/analytics', mw.authAdminApi, http(api.emails.cancelScheduledAnalytics));
 
     // ## Snippets
     router.get('/snippets', mw.authAdminApi, http(api.snippets.browse));

package/core/shared/config/defaults.json

@@ -176,7 +176,7 @@
     },
     "portal": {
         "url": "https://cdn.jsdelivr.net/ghost/portal@~{version}/umd/portal.min.js",
-        "version": "2.24"
+        "version": "2.25"
     },
     "sodoSearch": {
         "url": "https://cdn.jsdelivr.net/ghost/sodo-search@~{version}/umd/sodo-search.min.js",
@@ -213,6 +213,9 @@
                     "values": [100, 1000, 10000, 50000, 100000, 250000, 500000, 1000000]
                 }
             ],
-            "members": [100, 1000, 10000, 25000, 50000, 100000, 250000, 500000, 1000000]
+            "members": [100, 1000, 10000, 25000, 50000, 100000, 250000, 500000, 1000000],
+            "minDaysSinceImported": 7,
+            "minDaysSinceLastEmail": 14,
+            "maxPercentageFromMilestone": 0.1
         }
 }

package/core/shared/labs.js

@@ -20,7 +20,9 @@ const GA_FEATURES = [
     'memberAttribution',
     'audienceFeedback',
     'themeErrorsNotification',
-    'emailStability'
+    'emailStability',
+    'emailErrors',
+    'outboundLinkTagging'
 ];
 
 // NOTE: this allowlist is meant to be used to filter out any unexpected
@@ -28,7 +30,6 @@ const GA_FEATURES = [
 const BETA_FEATURES = [
     'activitypub',
     'webmentions',
-    'emailErrors',
     'milestoneEmails'
 ];
 
@@ -36,8 +37,8 @@ const ALPHA_FEATURES = [
     'urlCache',
     'beforeAfterCard',
     'lexicalEditor',
-    'outboundLinkTagging',
-    'websockets'
+    'websockets',
+    'webmentionEmails'
 ];
 
 module.exports.GA_KEYS = [...GA_FEATURES];