ghost 6.47.0 → 6.48.0

package/core/server/services/email-service/batch-sending-service.js

@@ -8,6 +8,7 @@ const messages = {
 };
 
 const MAX_SENDING_CONCURRENCY = 2;
+const SHUTDOWN_CODE = 'BULK_EMAIL_SHUTDOWN_IN_PROGRESS';
 
 /**
  * @typedef {import('./sending-service')} SendingService
@@ -33,6 +34,8 @@ class BatchSendingService {
     #sentry;
     #debugStorageFilePath;
     #getRequiredUrlRelations;
+    #shuttingDown = false;
+    #inFlight = new Set();
 
     // Retry database queries happening before sending the email
     #BEFORE_RETRY_CONFIG = {maxRetries: 10, maxTime: 10 * 60 * 1000, sleep: 2000};
@@ -116,6 +119,19 @@ class BatchSendingService {
         return this.#BEFORE_RETRY_CONFIG;
     }
 
+    /**
+     * Stops the batch workers and waits for any in-flight sends to finish.
+     * Called by the cleanup pipeline when the container is shutting down. Idempotent.
+     */
+    async onShutdown() {
+        this.#shuttingDown = true;
+        if (this.#inFlight.size > 0) {
+            logging.warn(`Email send shutdown: awaiting ${this.#inFlight.size} in-flight sendBatches call(s) to settle`);
+        }
+        await Promise.allSettled([...this.#inFlight]);
+        logging.warn(`Email send shutdown: drain complete`);
+    }
+
     /**
      * Schedules a background job that sends the email in the background if it is pending or failed.
      * @param {Email} email
@@ -171,6 +187,10 @@ class BatchSendingService {
                 }, {patch: true, autoRefresh: false});
             }, {...this.#AFTER_RETRY_CONFIG, description: `email ${emailId} -> submitted`});
         } catch (e) {
+            if (e && e.code === SHUTDOWN_CODE) {
+                logging.info(`Email ${email.id} send stopped because the container is shutting down — leaving status=submitting so it can resume on next boot`);
+                return;
+            }
             const ghostError = new errors.EmailError({
                 err: e,
                 code: 'BULK_EMAIL_SEND_FAILED',
@@ -432,6 +452,20 @@ class BatchSendingService {
     }
 
     async sendBatches({email, batches, post, newsletter}) {
+        // Track the in-flight call so onShutdown can await it. The cleanup task
+        // must wait for the Mailgun POST + EmailBatch DB write to settle before
+        // ghost-server schedules process.exit, otherwise mid-flight requests get
+        // killed and EmailBatch rows never record what Mailgun actually accepted.
+        const work = this.#sendBatchesInner({email, batches, post, newsletter});
+        this.#inFlight.add(work);
+        try {
+            return await work;
+        } finally {
+            this.#inFlight.delete(work);
+        }
+    }
+
+    async #sendBatchesInner({email, batches, post, newsletter}) {
         logging.info(`Sending ${batches.length} batches for email ${email.id}`);
         const deadline = this.getDeliveryDeadline(email);
 
@@ -442,21 +476,28 @@ class BatchSendingService {
         /** @type {Map<string, import('./email-renderer').EmailBody>} */
         const emailBodyCache = new Map();
 
-        // Calculate deliverytimes for the batches
+        // Spread batches across the target window if one is configured. `deliveryTimes`
+        // handles the past-deadline case internally (resume of an interrupted send or a
+        // delayed job): if the original `created_at + targetDeliveryWindow` deadline has
+        // passed, the function respreads remaining batches over a fresh window starting
+        // now instead of returning undefined for every batch (which would dump every
+        // remaining batch into Mailgun in the same second and break the rate-spread).
+        const targetDeliveryWindow = this.#sendingService.getTargetDeliveryWindow();
+        const shouldApplyDeliveryTimes = targetDeliveryWindow !== undefined && targetDeliveryWindow > 0;
         const deliveryTimes = this.calculateDeliveryTimes(email, batches.length);
 
         // Loop batches and send them via the EmailProvider
         let succeededCount = 0;
         const queue = batches.slice();
 
-        // Bind this
-        let runNext;
-        runNext = async () => {
-            const batch = queue.shift();
-            if (batch) {
+        const runWorker = async () => {
+            while (!this.#shuttingDown) {
+                const batch = queue.shift();
+                if (!batch) {
+                    return;
+                }
                 const batchData = {email, batch, post, newsletter, emailBodyCache, deliveryTime: undefined};
-                // Only set a delivery time if we have a deadline and it hasn't past yet
-                if (deadline && deadline.getTime() > Date.now()) {
+                if (shouldApplyDeliveryTimes) {
                     const deliveryTime = deliveryTimes.shift();
                     if (deliveryTime && deliveryTime >= Date.now()) {
                         batchData.deliveryTime = deliveryTime;
@@ -465,12 +506,20 @@ class BatchSendingService {
                 if (await this.sendBatch(batchData)) {
                     succeededCount += 1;
                 }
-                await runNext();
             }
         };
 
         // Run maximum MAX_SENDING_CONCURRENCY at the same time
-        await Promise.all(new Array(MAX_SENDING_CONCURRENCY).fill(0).map(() => runNext()));
+        await Promise.all(new Array(MAX_SENDING_CONCURRENCY).fill(0).map(() => runWorker()));
+
+        logging.info(`Email ${email.id} send done: ${succeededCount}/${batches.length} batches succeeded, ${queue.length} unstarted`);
+
+        if (this.#shuttingDown && queue.length > 0) {
+            throw new errors.InternalServerError({
+                code: SHUTDOWN_CODE,
+                message: 'Email send stopped because the container is shutting down'
+            });
+        }
 
         if (succeededCount < batches.length) {
             if (succeededCount > 0) {
@@ -501,8 +550,24 @@ class BatchSendingService {
             {...this.#getBeforeRetryConfig(email), description: `updateStatusLock batch ${originalBatch.id} -> submitting`}
         );
         if (!batch) {
-            logging.error(`Tried sending email batch that is not pending or failed ${originalBatch.id}`);
-            return true;
+            // updateStatusLock returned undefined: the batch's current status is neither
+            // `pending` nor `failed`, so the lock didn't engage. Two distinct cases, and
+            // they need different handling — collapsing them is the bug this branch fixes.
+            const currentStatus = originalBatch.get('status');
+            if (currentStatus === 'submitted') {
+                // Mailgun accepted this batch on a prior run. Nothing to do; return true so
+                // the parent email's success counter stays accurate. Expected path during
+                // resume of an interrupted send where some batches finished before the crash.
+                logging.info(`Email batch ${originalBatch.id} already submitted on a prior run; skipping`);
+                return true;
+            }
+            // Otherwise currentStatus is `submitting`: orphan from a worker that crashed
+            // mid-batch. We have no record of Mailgun accepting it, and re-sending risks
+            // duplicates. Return false so the parent email is promoted to `failed` and an
+            // operator can reconcile against the Mailgun dashboard before retrying.
+            // Runbook: docs/newsletter-send-plan-v9.md.
+            logging.error(`Email batch ${originalBatch.id} is stuck in status=${currentStatus} (orphan from a crashed worker); not re-sending — marking parent email as failed for operator review`);
+            return false;
         }
 
         let succeeded = false;
@@ -760,23 +825,31 @@ class BatchSendingService {
      * @param {number} numBatches - the number of batches to be sent
      */
     calculateDeliveryTimes(email, numBatches) {
-        const deadline = this.getDeliveryDeadline(email);
-        const now = new Date();
-        // If there is no deadline (target delivery window is not set) or the deadline is in the past, delivery immediately
-        if (!deadline || now >= deadline) {
+        let deadline = this.getDeliveryDeadline(email);
+        if (!deadline) {
             return new Array(numBatches).fill(undefined);
-        } else {
-            const timeToDeadline = deadline.getTime() - now.getTime();
-            const batchDelay = timeToDeadline / numBatches;
-            const deliveryTimes = [];
-            for (let i = 0; i < numBatches; i++) {
-                const delay = batchDelay * i;
-                const deliveryTime = new Date(now.getTime() + delay);
-                deliveryTimes.push(deliveryTime);
-            }
-            return deliveryTimes;
         }
+        const now = new Date();
+        // If the original `created_at + targetDeliveryWindow` deadline has passed (resume
+        // of an interrupted send, or a job that was delayed for any reason), respread
+        // batches over a fresh window of the same size starting now. Otherwise a
+        // 50%-resumed 10-minute send would dump every remaining batch into Mailgun in
+        // the same second and defeat the rate-spread.
+        if (now >= deadline) {
+            const targetDeliveryWindow = this.#sendingService.getTargetDeliveryWindow();
+            deadline = new Date(now.getTime() + targetDeliveryWindow);
+        }
+        const timeToDeadline = deadline.getTime() - now.getTime();
+        const batchDelay = timeToDeadline / numBatches;
+        const deliveryTimes = [];
+        for (let i = 0; i < numBatches; i++) {
+            const delay = batchDelay * i;
+            const deliveryTime = new Date(now.getTime() + delay);
+            deliveryTimes.push(deliveryTime);
+        }
+        return deliveryTimes;
     }
 }
 
 module.exports = BatchSendingService;
+module.exports.SHUTDOWN_CODE = SHUTDOWN_CODE;

package/core/server/services/email-service/email-service-wrapper.js

@@ -10,7 +10,7 @@ class EmailServiceWrapper {
         return jsonModel.url;
     }
 
-    init() {
+    init({ghostServer} = {}) {
         if (this.service) {
             return;
         }
@@ -131,13 +131,18 @@ class EmailServiceWrapper {
             debugStorageFilePath: configService.getContentPath('data')
         });
 
+        if (ghostServer) {
+            ghostServer.registerCleanupTask(() => batchSendingService.onShutdown());
+        }
+
         this.renderer = emailRenderer;
 
         this.service = new EmailService({
             batchSendingService,
             sendingService,
             models: {
-                Email
+                Email,
+                EmailBatch
             },
             settingsCache,
             emailRenderer,

package/core/server/services/email-service/email-service.js

@@ -20,9 +20,16 @@ const messages = {
     archivedNewsletterError: 'Cannot send email to archived newsletters',
     missingNewsletterError: 'The post does not have a newsletter relation',
     emailSendingDisabled: `Email sending is temporarily disabled because your account is currently in review. You should have an email about this from us already, but you can also reach us any time at support@ghost.org`,
-    retryEmailStatusError: 'Can only retry emails for published posts'
+    retryEmailStatusError: 'Can only retry emails for published posts',
+    retryEmailNotFailed: 'Only failed emails can be retried'
 };
 
+// Resume scanner won't pick up `submitting` rows older than this. Rows beyond the cutoff
+// are flipped to `failed` on first boot so they surface in admin UI for operator review
+// rather than being silently resumed (and sending stale newsletters to current members).
+// Override via `bulkEmail:resumeMaxAgeMs` in config.
+const DEFAULT_RESUME_MAX_AGE_MS = 24 * 60 * 60 * 1000;
+
 class EmailService {
     #batchSendingService;
     #sendingService;
@@ -44,6 +51,7 @@ class EmailService {
      * @param {SendingService} dependencies.sendingService
      * @param {object} dependencies.models
      * @param {object} dependencies.models.Email
+     * @param {object} [dependencies.models.EmailBatch] - Required for resumeInterruptedSends breadcrumbs
      * @param {object} dependencies.settingsCache
      * @param {EmailRenderer} dependencies.emailRenderer
      * @param {EmailSegmenter} dependencies.emailSegmenter
@@ -189,6 +197,136 @@ class EmailService {
         return email;
     }
 
+    /**
+     * Boot-time scanner: resumes newsletter emails left in `submitting` after a
+     * previous container's interrupted send. Iterates sequentially; one failure
+     * does not skip others. Rows older than the configured max-age are flipped
+     * to `failed` (not resumed) so stale content does not get sent to current members.
+     */
+    async resumeInterruptedSends() {
+        const maxAgeMs = this.#config?.get?.('bulkEmail:resumeMaxAgeMs') ?? DEFAULT_RESUME_MAX_AGE_MS;
+        const cutoffIso = new Date(Date.now() - maxAgeMs).toISOString();
+
+        // Stale rows: too old to safely resume. Flip to `failed` so they surface in admin UI
+        // for operator review instead of being silently left in `submitting` forever.
+        const stale = await this.#models.Email.findAll({
+            filter: `status:submitting+created_at:<'${cutoffIso}'`
+        });
+        const staleList = stale.models || stale;
+        for (const email of staleList) {
+            try {
+                const locked = await this.#batchSendingService.updateStatusLock(
+                    this.#models.Email,
+                    email.id,
+                    'failed',
+                    ['submitting']
+                );
+                if (locked) {
+                    logging.warn(`Email resume: ${email.id} created_at=${email.get('created_at') && new Date(email.get('created_at')).toISOString()} exceeds max age (${maxAgeMs}ms) — flipped to failed for operator review`);
+                }
+            } catch (e) {
+                logging.error(e);
+            }
+        }
+
+        // Fresh rows: within the cutoff. Resume through the normal emailJob path.
+        const emails = await this.#models.Email.findAll({
+            filter: `status:submitting+created_at:>'${cutoffIso}'`
+        });
+        const list = emails.models || emails;
+        if (staleList.length === 0 && list.length === 0) {
+            return;
+        }
+        if (list.length > 0) {
+            logging.info(`Email resume: found ${list.length} email(s) in submitting status within max age (${maxAgeMs}ms)`);
+        }
+
+        for (const email of list) {
+            try {
+                await this.#resumeOneEmail(email);
+            } catch (e) {
+                logging.error(e);
+            }
+        }
+
+        logging.info(`Email resume scan complete: ${staleList.length} stale email(s) flipped to failed, ${list.length} fresh email(s) rescheduled`);
+    }
+
+    async #resumeOneEmail(email) {
+        const post = await email.getLazyRelation('post');
+        const postStatus = post ? post.get('status') : null;
+        const sendable = postStatus === 'published' || postStatus === 'sent';
+
+        if (!sendable) {
+            // Parent post was unpublished or deleted while the email was in flight.
+            // Can't resume — mark the email as failed so it stops showing as "submitting".
+            const locked = await this.#batchSendingService.updateStatusLock(
+                this.#models.Email,
+                email.id,
+                'failed',
+                ['submitting']
+            );
+            if (locked) {
+                logging.warn(`Email resume: ${email.id} parent post status=${postStatus} is not sendable — marked email as failed`);
+            }
+            return;
+        }
+
+        // Flip submitting -> pending so emailJob's status lock (['pending', 'failed']) admits it.
+        // If updateStatusLock returns undefined, another path raced us — just continue.
+        const locked = await this.#batchSendingService.updateStatusLock(
+            this.#models.Email,
+            email.id,
+            'pending',
+            ['submitting']
+        );
+        if (!locked) {
+            logging.info(`Email resume: ${email.id} status changed before lock could be taken — skipping`);
+            return;
+        }
+
+        // Structured breadcrumb so post-incident timing/batch state is recoverable from logs.
+        const breadcrumb = await this.#buildResumeBreadcrumb(email);
+        logging.warn(`Email resume: scheduling ${email.id} for re-send ${JSON.stringify(breadcrumb)}`);
+
+        // Skip checkLimits — this email already passed limits when first sent.
+        this.#batchSendingService.scheduleEmail(email);
+    }
+
+    async #buildResumeBreadcrumb(email) {
+        const counts = {};
+        let latestStatusWrite = email.get('updated_at') || email.get('created_at');
+        if (this.#models.EmailBatch) {
+            try {
+                const batches = await this.#models.EmailBatch.findAll({
+                    filter: `email_id:'${email.id}'`
+                });
+                for (const batch of batches.models || batches) {
+                    const status = batch.get('status');
+                    counts[status] = (counts[status] || 0) + 1;
+                    const updatedAt = batch.get('updated_at');
+                    if (updatedAt && (!latestStatusWrite || updatedAt > latestStatusWrite)) {
+                        latestStatusWrite = updatedAt;
+                    }
+                }
+            } catch (e) {
+                // Breadcrumb is best-effort; never block resume on it.
+                logging.warn(`Email resume: could not build breadcrumb for ${email.id}: ${e.message}`);
+            }
+        }
+        const msSinceLastStatusWrite = latestStatusWrite
+            ? Date.now() - new Date(latestStatusWrite).getTime()
+            : null;
+        const targetDeliveryWindowMs = this.#config?.get?.('bulkEmail:targetDeliveryWindow') ?? 0;
+        return {
+            email_id: email.id,
+            post_id: email.get('post_id'),
+            batch_counts_by_status: counts,
+            ms_since_last_status_write: msSinceLastStatusWrite,
+            target_delivery_window_ms: targetDeliveryWindowMs
+        };
+    }
+
     async retryEmail(email) {
         // Block accidentaly retrying non-published posts (can happen due to bugs in frontend)
         const post = await email.getLazyRelation('post');
@@ -198,13 +336,17 @@ class EmailService {
             });
         }
 
+        if (email.get('status') !== 'failed') {
+            throw new errors.BadRequestError({
+                message: tpl(messages.retryEmailNotFailed)
+            });
+        }
+
         await this.checkLimits();
 
         // Change email status back to 'pending' before scheduling
         // so we have a immediate response when retrying an email (schedule can take a while to kick off sometimes)
-        if (email.get('status') === 'failed') {
-            await email.save({status: 'pending'}, {patch: true});
-        }
+        await email.save({status: 'pending'}, {patch: true});
 
         this.#batchSendingService.scheduleEmail(email);
         return email;

package/core/server/services/gift-links/database.js

@@ -13,9 +13,6 @@ const DbDate = zod_1.z.codec(zod_1.z.union([zod_1.z.date(), zod_1.z.string(), zo
 exports.DbGiftLink = zod_1.z.object({
     token: zod_1.z.string(),
     post_id: zod_1.z.string(),
-    redeemed_count: zod_1.z.number().int().nonnegative(),
-    last_redeemed_at: DbDate.nullable(),
-    revoked_at: DbDate.nullable(),
     created_at: DbDate,
     updated_at: DbDate.nullable()
 });

package/core/server/services/gift-links/database.ts

@@ -13,9 +13,6 @@ const DbDate = z.codec(z.union([z.date(), z.string(), z.number()]), z.date(), {
 export const DbGiftLink = z.object({
     token: z.string(),
     post_id: z.string(),
-    redeemed_count: z.number().int().nonnegative(),
-    last_redeemed_at: DbDate.nullable(),
-    revoked_at: DbDate.nullable(),
     created_at: DbDate,
     updated_at: DbDate.nullable()
 });
@@ -33,7 +30,7 @@ declare module 'knex/types/tables' {
     interface Tables {
         gift_links: Knex.CompositeTableType<
             z.infer<typeof DbGiftLink>,
-            Omit<z.input<typeof DbGiftLink>, 'revoked_at' | 'updated_at'>,
+            Omit<z.input<typeof DbGiftLink>, 'updated_at'>,
             Partial<z.infer<typeof DbGiftLink>>
         >;
         post_gift_links: Knex.CompositeTableType<

package/core/server/services/gift-links/index.js

@@ -7,7 +7,7 @@ function init() {
     if (exports.service) {
         return;
     }
-    // eslint-disable-next-line @typescript-eslint/no-require-imports
     const { knex } = require('../../data/db');
-    exports.service = new service_1.GiftLinksService({ knex });
+    const models = require('../../models');
+    exports.service = new service_1.GiftLinksService({ knex, Action: models.Action });
 }

package/core/server/services/gift-links/index.ts

@@ -1,5 +1,7 @@
 import {GiftLinksService} from './service';
 
+export type {RequestContext} from './service';
+
 // Set by init() at boot, not at import: knex only exists once the DB has connected.
 export let service: GiftLinksService | undefined;
 
@@ -8,8 +10,8 @@ export function init(): void {
         return;
     }
 
-    // eslint-disable-next-line @typescript-eslint/no-require-imports
     const {knex} = require('../../data/db');
+    const models = require('../../models');
 
-    service = new GiftLinksService({knex});
+    service = new GiftLinksService({knex, Action: models.Action});
 }

package/core/server/services/gift-links/models.js

@@ -5,7 +5,5 @@ const zod_1 = require("zod");
 exports.GiftLinkToken = zod_1.z.string().brand('GiftLinkToken');
 exports.GiftLink = zod_1.z.object({
     token: exports.GiftLinkToken,
-    redeemedCount: zod_1.z.number().int().nonnegative(),
-    lastRedeemedAt: zod_1.z.date().nullable(),
     createdAt: zod_1.z.date()
 });

package/core/server/services/gift-links/models.ts

@@ -5,8 +5,6 @@ export type GiftLinkToken = z.infer<typeof GiftLinkToken>;
 
 export const GiftLink = z.object({
     token: GiftLinkToken,
-    redeemedCount: z.number().int().nonnegative(),
-    lastRedeemedAt: z.date().nullable(),
     createdAt: z.date()
 });
 export type GiftLink = z.infer<typeof GiftLink>;

package/core/server/services/gift-links/queries.js

@@ -10,8 +10,6 @@ const models_1 = require("./models");
 // The columns the read path selects and the codec decodes into a GiftLink.
 exports.GiftLinkRow = database_1.DbGiftLink.pick({
     token: true,
-    redeemed_count: true,
-    last_redeemed_at: true,
     created_at: true
 });
 // Maps a selected row to/from the domain GiftLink (snake_case to camelCase, token branding).
@@ -20,6 +18,8 @@ exports.giftLinkCodec = zod_1.z.codec(exports.GiftLinkRow, models_1.GiftLink, {
     encode: link => (0, case_keys_1.snakeKeys)(link)
 });
 const giftLinkColumns = Object.keys(exports.GiftLinkRow.shape).map(column => `gift_links.${column}`);
+// LEFT JOINs from posts: a missing post yields zero rows, while a post with no live link yields one
+// row with null link columns. So zero rows means "no such post".
 function liveLinksForPost(postId) {
     return (knex) => knex('posts')
         .where('posts.id', postId)

package/core/server/services/gift-links/queries.ts

@@ -7,8 +7,6 @@ import {GiftLink} from './models';
 // The columns the read path selects and the codec decodes into a GiftLink.
 export const GiftLinkRow = DbGiftLink.pick({
     token: true,
-    redeemed_count: true,
-    last_redeemed_at: true,
     created_at: true
 });
 
@@ -28,6 +26,8 @@ const giftLinkColumns = Object.keys(GiftLinkRow.shape).map(column => `gift_links
 // post exists with no live link (hence the nullable columns).
 type LiveLinkRow = {[K in keyof z.input<typeof GiftLinkRow>]: z.input<typeof GiftLinkRow>[K] | null};
 
+// LEFT JOINs from posts: a missing post yields zero rows, while a post with no live link yields one
+// row with null link columns. So zero rows means "no such post".
 export function liveLinksForPost(postId: string) {
     return (knex: Knex) => knex('posts')
         .where('posts.id', postId)

package/core/server/services/gift-links/serializers.js

@@ -7,8 +7,6 @@ const models_1 = require("./models");
 // Response schemas — the shapes the admin endpoints emit.
 const GiftLinkResource = zod_1.z.object({
     token: zod_1.z.string(),
-    redeemed_count: zod_1.z.number(),
-    last_redeemed_at: zod_1.z.date().nullable(),
     created_at: zod_1.z.date()
 });
 const GiftLinksResponse = zod_1.z.object({ gift_links: zod_1.z.array(GiftLinkResource) });

package/core/server/services/gift-links/serializers.ts

@@ -5,8 +5,6 @@ import {GiftLink} from './models';
 // Response schemas — the shapes the admin endpoints emit.
 const GiftLinkResource = z.object({
     token: z.string(),
-    redeemed_count: z.number(),
-    last_redeemed_at: z.date().nullable(),
     created_at: z.date()
 });
 const GiftLinksResponse = z.object({gift_links: z.array(GiftLinkResource)});