ghost 6.9.0 → 6.9.3

package/core/server/services/email-analytics/EmailAnalyticsService.js

@@ -330,22 +330,56 @@ module.exports = class EmailAnalyticsService {
 
         // We keep the processing result here, so we also have a result in case of failures
         let processingResult = new EventProcessingResult();
+        // Track cumulative event counts separately since processingResult gets reset during intermediate aggregations
+        const cumulativeResult = new EventProcessingResult();
+        // Track all unique emailIds and memberIds that need aggregation
+        const allEmailIds = new Set();
+        const allMemberIds = new Set();
         let error = null;
 
         /**
          * Process a batch of events
          * @param {Array<Object>} events - Array of event objects to process
-         * @param {EventProcessingResult} processingResult - Object to store the processing results
-         * @param {FetchData} fetchData - Object containing fetch operation data
          * @returns {Promise<void>}
          */
         const processBatch = async (events) => {
             // Even if the fetching is interrupted because of an error, we still store the last event timestamp
             const processingStart = Date.now();
+            // Capture the state before processing to calculate delta
+            const beforeCounts = {
+                opened: processingResult.opened,
+                delivered: processingResult.delivered,
+                temporaryFailed: processingResult.temporaryFailed,
+                permanentFailed: processingResult.permanentFailed,
+                unsubscribed: processingResult.unsubscribed,
+                complained: processingResult.complained,
+                unhandled: processingResult.unhandled,
+                unprocessable: processingResult.unprocessable
+            };
+            const beforeEmailIds = new Set(processingResult.emailIds);
+            const beforeMemberIds = new Set(processingResult.memberIds);
+
             await this.processEventBatch(events, processingResult, fetchData);
             processingTimeMs += (Date.now() - processingStart);
             eventCount += events.length;
 
+            // Calculate delta (only new counts from this batch) and accumulate for final reporting
+            const batchDelta = new EventProcessingResult({
+                opened: processingResult.opened - beforeCounts.opened,
+                delivered: processingResult.delivered - beforeCounts.delivered,
+                temporaryFailed: processingResult.temporaryFailed - beforeCounts.temporaryFailed,
+                permanentFailed: processingResult.permanentFailed - beforeCounts.permanentFailed,
+                unsubscribed: processingResult.unsubscribed - beforeCounts.unsubscribed,
+                complained: processingResult.complained - beforeCounts.complained,
+                unhandled: processingResult.unhandled - beforeCounts.unhandled,
+                unprocessable: processingResult.unprocessable - beforeCounts.unprocessable,
+                emailIds: processingResult.emailIds.filter(id => !beforeEmailIds.has(id)),
+                memberIds: processingResult.memberIds.filter(id => !beforeMemberIds.has(id))
+            });
+            cumulativeResult.merge(batchDelta);
+            batchDelta.emailIds.forEach(id => allEmailIds.add(id));
+            batchDelta.memberIds.forEach(id => allMemberIds.add(id));
+
             // Every 5 minutes or 5000 members we do an aggregation and clear the processingResult
             // Otherwise we need to loop a lot of members afterwards, and this takes too long without updating the stat counts in between
             if ((Date.now() - lastAggregation > 5 * 60 * 1000 || processingResult.memberIds.length > 5000) && eventCount > 0) {
@@ -356,6 +390,9 @@ module.exports = class EmailAnalyticsService {
                     await this.aggregateStats(processingResult, includeOpenedEvents);
                     aggregationTimeMs += (Date.now() - aggregationStart);
                     lastAggregation = Date.now();
+                    // Remove aggregated emailIds and memberIds from tracking sets to avoid re-aggregating at the end
+                    processingResult.emailIds.forEach(id => allEmailIds.delete(id));
+                    processingResult.memberIds.forEach(id => allMemberIds.delete(id));
                     processingResult = new EventProcessingResult();
                 } catch (err) {
                     logging.error('[EmailAnalytics] Error while aggregating stats');
@@ -386,10 +423,20 @@ module.exports = class EmailAnalyticsService {
             }
         }
 
-        if (processingResult.memberIds.length > 0 || processingResult.emailIds.length > 0) {
+        // Final aggregation: aggregate any remaining events and ensure all emailIds are aggregated
+        // We need to aggregate all unique emailIds to ensure the emails table is updated
+        const finalEmailIds = Array.from(new Set([...processingResult.emailIds, ...allEmailIds]));
+        const finalMemberIds = Array.from(new Set([...processingResult.memberIds, ...allMemberIds]));
+
+        if (finalMemberIds.length > 0 || finalEmailIds.length > 0) {
             try {
                 const aggregationStart = Date.now();
-                await this.aggregateStats(processingResult, includeOpenedEvents);
+                // Create a result object with all emailIds and memberIds for final aggregation
+                const finalAggregationResult = {
+                    emailIds: finalEmailIds,
+                    memberIds: finalMemberIds
+                };
+                await this.aggregateStats(finalAggregationResult, includeOpenedEvents);
                 aggregationTimeMs += (Date.now() - aggregationStart);
             } catch (err) {
                 logging.error('[EmailAnalytics] Error while aggregating stats');
@@ -404,7 +451,7 @@ module.exports = class EmailAnalyticsService {
         // Small trick: if reached the end of new events, we are going to keep
         // fetching the same events because 'begin' won't change
         // So if we didn't have errors while fetching, and total events < maxEvents, increase lastEventTimestamp with one second
-        if (!error && eventCount > 0 && eventCount < maxEvents && fetchData.lastEventTimestamp && fetchData.lastEventTimestamp.getTime() < Date.now() - 2000) {
+        if (!error && eventCount > 0 && fetchData.lastEventTimestamp && fetchData.lastEventTimestamp.getTime() < Date.now() - 2000) {
             // set the data on the db so we can store it for fetching after reboot
             await this.queries.setJobTimestamp(fetchData.jobName, 'finished', new Date(fetchData.lastEventTimestamp.getTime()));
             // increment and store in local memory
@@ -425,7 +472,7 @@ module.exports = class EmailAnalyticsService {
             apiPollingTimeMs,
             processingTimeMs,
             aggregationTimeMs,
-            result: processingResult
+            result: cumulativeResult
         };
     }
 

package/core/server/services/email-service/DomainWarmingService.js

@@ -1,32 +1,62 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.DomainWarmingService = void 0;
+/**
+ * Configuration for domain warming email volume scaling.
+ *
+ * | Volume Range | Multiplier                                       |
+ * |--------------|--------------------------------------------------|
+ * | ≤100 (base)  | 200 messages                                     |
+ * | 101 – 1k     | 1.25× (conservative early ramp)                  |
+ * | 1k – 5k      | 1.5× (moderate increase)                         |
+ * | 5k – 100k    | 1.75× (faster ramp after proving deliverability) |
+ * | 100k – 400k  | 2×                                               |
+ * | 400k+        | min(1.2×, +75k) cap                              |
+ */
 const WARMUP_SCALING_TABLE = {
     base: {
         limit: 100,
         value: 200
     },
     thresholds: [{
+            limit: 1_000,
+            scale: 1.25
+        }, {
+            limit: 5_000,
+            scale: 1.5
+        }, {
             limit: 100_000,
-            scale: 2
+            scale: 1.75
         }, {
             limit: 400_000,
-            scale: 1.5
+            scale: 2
         }],
-    defaultScale: 1.25
+    highVolume: {
+        threshold: 400_000,
+        maxScale: 1.2,
+        maxAbsoluteIncrease: 75_000
+    }
 };
 class DomainWarmingService {
     #emailModel;
     #labs;
+    #config;
     constructor(dependencies) {
         this.#emailModel = dependencies.models.Email;
         this.#labs = dependencies.labs;
+        this.#config = dependencies.config;
     }
     /**
      * @returns Whether the domain warming feature is enabled
      */
     isEnabled() {
-        return this.#labs.isSet('domainWarmup');
+        const hasLabsFlag = this.#labs.isSet('domainWarmup');
+        if (!hasLabsFlag) {
+            return false;
+        }
+        const fallbackDomain = this.#config.get('hostSettings:managedEmail:fallbackDomain');
+        const fallbackAddress = this.#config.get('hostSettings:managedEmail:fallbackAddress');
+        return Boolean(fallbackDomain && fallbackAddress);
     }
     /**
      * Get the maximum amount of emails that should be sent from the warming sending domain in today's newsletter
@@ -41,14 +71,15 @@ class DomainWarmingService {
      * @returns The highest number of messages sent from the CSD in a single email (excluding today)
      */
     async #getHighestCount() {
-        const email = await this.#emailModel.findOne({
+        const result = await this.#emailModel.findPage({
             filter: `created_at:<${new Date().toISOString().split('T')[0]}`,
-            order: 'csd_email_count DESC'
+            order: 'csd_email_count DESC',
+            limit: 1
         });
-        if (!email) {
+        if (!result.data.length) {
             return 0;
         }
-        const count = email.get('csd_email_count');
+        const count = result.data[0].get('csd_email_count');
         return count || 0;
     }
     /**
@@ -59,12 +90,19 @@ class DomainWarmingService {
         if (lastCount <= WARMUP_SCALING_TABLE.base.limit) {
             return WARMUP_SCALING_TABLE.base.value;
         }
+        // For high volume senders (400k+), cap the increase at 20% or 75k absolute
+        if (lastCount > WARMUP_SCALING_TABLE.highVolume.threshold) {
+            const scaledIncrease = Math.ceil(lastCount * WARMUP_SCALING_TABLE.highVolume.maxScale);
+            const absoluteIncrease = lastCount + WARMUP_SCALING_TABLE.highVolume.maxAbsoluteIncrease;
+            return Math.min(scaledIncrease, absoluteIncrease);
+        }
         for (const threshold of WARMUP_SCALING_TABLE.thresholds.sort((a, b) => a.limit - b.limit)) {
             if (lastCount <= threshold.limit) {
                 return Math.ceil(lastCount * threshold.scale);
             }
         }
-        return Math.ceil(lastCount * WARMUP_SCALING_TABLE.defaultScale);
+        // This should not be reached given the thresholds cover all cases up to highVolume.threshold
+        return Math.ceil(lastCount * WARMUP_SCALING_TABLE.highVolume.maxScale);
     }
 }
 exports.DomainWarmingService = DomainWarmingService;

package/core/server/services/email-service/DomainWarmingService.ts

@@ -2,8 +2,12 @@ type LabsService = {
     isSet: (flag: string) => boolean;
 };
 
+type ConfigService = {
+    get: (key: string) => string | undefined;
+}
+
 type EmailModel = {
-    findOne: (options: {filter: string; order: string}) => Promise<EmailRecord | null>;
+    findPage: (options: {filter: string; order: string; limit: number}) => Promise<{data: EmailRecord[]}>;
 };
 
 type EmailRecord = {
@@ -20,41 +24,79 @@ type WarmupScalingTable = {
         limit: number;
         scale: number;
     }[];
-    defaultScale: number;
+    highVolume: {
+        threshold: number;
+        maxScale: number;
+        maxAbsoluteIncrease: number;
+    };
 }
 
+/**
+ * Configuration for domain warming email volume scaling.
+ *
+ * | Volume Range | Multiplier                                       |
+ * |--------------|--------------------------------------------------|
+ * | ≤100 (base)  | 200 messages                                     |
+ * | 101 – 1k     | 1.25× (conservative early ramp)                  |
+ * | 1k – 5k      | 1.5× (moderate increase)                         |
+ * | 5k – 100k    | 1.75× (faster ramp after proving deliverability) |
+ * | 100k – 400k  | 2×                                               |
+ * | 400k+        | min(1.2×, +75k) cap                              |
+ */
 const WARMUP_SCALING_TABLE: WarmupScalingTable = {
     base: {
         limit: 100,
         value: 200
     },
     thresholds: [{
+        limit: 1_000,
+        scale: 1.25
+    }, {
+        limit: 5_000,
+        scale: 1.5
+    }, {
         limit: 100_000,
-        scale: 2
+        scale: 1.75
     }, {
         limit: 400_000,
-        scale: 1.5
+        scale: 2
     }],
-    defaultScale: 1.25
+    highVolume: {
+        threshold: 400_000,
+        maxScale: 1.2,
+        maxAbsoluteIncrease: 75_000
+    }
 };
 
 export class DomainWarmingService {
     #emailModel: EmailModel;
     #labs: LabsService;
+    #config: ConfigService;
 
     constructor(dependencies: {
         models: {Email: EmailModel};
         labs: LabsService;
+        config: ConfigService;
     }) {
         this.#emailModel = dependencies.models.Email;
         this.#labs = dependencies.labs;
+        this.#config = dependencies.config;
     }
 
     /**
      * @returns Whether the domain warming feature is enabled
      */
     isEnabled(): boolean {
-        return this.#labs.isSet('domainWarmup');
+        const hasLabsFlag = this.#labs.isSet('domainWarmup');
+
+        if (!hasLabsFlag) {
+            return false;
+        }
+
+        const fallbackDomain = this.#config.get('hostSettings:managedEmail:fallbackDomain');
+        const fallbackAddress = this.#config.get('hostSettings:managedEmail:fallbackAddress');
+
+        return Boolean(fallbackDomain && fallbackAddress);
     }
 
     /**
@@ -72,16 +114,17 @@ export class DomainWarmingService {
      * @returns The highest number of messages sent from the CSD in a single email (excluding today)
      */
     async #getHighestCount(): Promise<number> {
-        const email = await this.#emailModel.findOne({
+        const result = await this.#emailModel.findPage({
             filter: `created_at:<${new Date().toISOString().split('T')[0]}`,
-            order: 'csd_email_count DESC'
+            order: 'csd_email_count DESC',
+            limit: 1
         });
 
-        if (!email) {
+        if (!result.data.length) {
             return 0;
         }
 
-        const count = email.get('csd_email_count');
+        const count = result.data[0].get('csd_email_count');
         return count || 0;
     }
 
@@ -94,12 +137,20 @@ export class DomainWarmingService {
             return WARMUP_SCALING_TABLE.base.value;
         }
 
+        // For high volume senders (400k+), cap the increase at 20% or 75k absolute
+        if (lastCount > WARMUP_SCALING_TABLE.highVolume.threshold) {
+            const scaledIncrease = Math.ceil(lastCount * WARMUP_SCALING_TABLE.highVolume.maxScale);
+            const absoluteIncrease = lastCount + WARMUP_SCALING_TABLE.highVolume.maxAbsoluteIncrease;
+            return Math.min(scaledIncrease, absoluteIncrease);
+        }
+
         for (const threshold of WARMUP_SCALING_TABLE.thresholds.sort((a, b) => a.limit - b.limit)) {
             if (lastCount <= threshold.limit) {
                 return Math.ceil(lastCount * threshold.scale);
             }
         }
 
-        return Math.ceil(lastCount * WARMUP_SCALING_TABLE.defaultScale);
+        // This should not be reached given the thresholds cover all cases up to highVolume.threshold
+        return Math.ceil(lastCount * WARMUP_SCALING_TABLE.highVolume.maxScale);
     }
 }

package/core/server/services/email-service/EmailServiceWrapper.js

@@ -118,7 +118,8 @@ class EmailServiceWrapper {
 
         const domainWarmingService = new DomainWarmingService({
             models: {Email},
-            labs
+            labs,
+            config: configService
         });
 
         const batchSendingService = new BatchSendingService({

package/core/server/services/member-welcome-emails/jobs/index.js

@@ -8,20 +8,24 @@ let hasScheduled = {
 
 module.exports = {
     async scheduleMemberWelcomeEmailJob() {
-        if (!config.get('memberWelcomeEmailTestInbox')) {
+        const testInboxDisabled = !config.get('memberWelcomeEmailTestInbox');
+        const alreadyScheduledProcessing = hasScheduled.processOutbox;
+
+        if (testInboxDisabled || alreadyScheduledProcessing) {
             return false;
         }
 
-        if (!hasScheduled.processOutbox && !process.env.NODE_ENV.startsWith('test')) {
-            jobsService.addJob({
-                at: '0 */5 * * * *',
-                job: path.resolve(__dirname, 'process-outbox.js'),
-                name: 'process-member-welcome-emails'
-            });
+        const configValue = config.get('memberWelcomeEmailSendInstantly');
+        const testEmailSendInstantly = configValue === true || configValue === 'true';
+        const cronSchedule = testEmailSendInstantly ? '*/3 * * * * *' : '0 */5 * * * *';
 
-            hasScheduled.processOutbox = true;
-        }
+        jobsService.addJob({
+            at: cronSchedule,
+            job: path.resolve(__dirname, 'process-outbox.js'),
+            name: 'process-member-welcome-emails'
+        });
 
+        hasScheduled.processOutbox = true;
         return hasScheduled.processOutbox;
     }
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "ghost",
-  "version": "6.9.0",
+  "version": "6.9.3",
   "description": "The professional publishing platform",
   "author": "Ghost Foundation",
   "homepage": "https://ghost.org",
@@ -87,7 +87,7 @@
     "@tryghost/helpers": "1.1.97",
     "@tryghost/html-to-plaintext": "1.0.4",
     "@tryghost/http-cache-utils": "0.1.20",
-    "@tryghost/i18n": "file:components/tryghost-i18n-6.9.0.tgz",
+    "@tryghost/i18n": "file:components/tryghost-i18n-6.9.3.tgz",
     "@tryghost/image-transform": "1.4.6",
     "@tryghost/job-manager": "1.0.3",
     "@tryghost/kg-card-factory": "5.1.2",
@@ -274,7 +274,7 @@
     "jackspeak": "2.3.6",
     "moment": "2.24.0",
     "moment-timezone": "0.5.45",
-    "@tryghost/i18n": "file:components/tryghost-i18n-6.9.0.tgz"
+    "@tryghost/i18n": "file:components/tryghost-i18n-6.9.3.tgz"
   },
   "nx": {
     "targets": {