@stamhoofd/models 2.92.0 → 2.93.0

package/src/models/Email.ts

@@ -3,16 +3,44 @@ import { EmailAttachment, EmailPreview, EmailRecipientFilter, EmailRecipientFilt
 import { v4 as uuidv4 } from 'uuid';
 
 import { AnyDecoder, ArrayDecoder } from '@simonbackx/simple-encoding';
-import { SimpleError } from '@simonbackx/simple-errors';
+import { isSimpleError, isSimpleErrors, SimpleError, SimpleErrors } from '@simonbackx/simple-errors';
 import { I18n } from '@stamhoofd/backend-i18n';
 import { Email as EmailClass, EmailInterfaceRecipient } from '@stamhoofd/email';
-import { QueueHandler } from '@stamhoofd/queues';
-import { QueryableModel, SQL, SQLWhereSign } from '@stamhoofd/sql';
+import { isAbortedError, QueueHandler, QueueHandlerOptions } from '@stamhoofd/queues';
+import { QueryableModel, readDynamicSQLExpression, SQL, SQLAlias, SQLCalculation, SQLCount, SQLPlusSign, SQLSelectAs, SQLWhereSign } from '@stamhoofd/sql';
 import { canSendFromEmail, fillRecipientReplacements, getEmailBuilder } from '../helpers/EmailBuilder';
 import { EmailRecipient } from './EmailRecipient';
 import { EmailTemplate } from './EmailTemplate';
 import { Organization } from './Organization';
 
+function errorToSimpleErrors(e: unknown) {
+    if (isSimpleErrors(e)) {
+        return e;
+    }
+    else if (isSimpleError(e)) {
+        return new SimpleErrors(e);
+    }
+    else {
+        return new SimpleErrors(
+            new SimpleError({
+                code: 'unknown_error',
+                message: ((typeof e === 'object' && e !== null && 'message' in e && typeof e.message === 'string') ? e.message : 'Unknown error'),
+                human: $t(`Onbekende fout`),
+            }),
+        );
+    }
+}
+
+export function isSoftEmailRecipientError(error: SimpleErrors) {
+    if (error.hasCode('email_skipped_unsubscribed')) {
+        return true;
+    }
+    if (error.hasCode('email_skipped_duplicate_recipient')) {
+        return true;
+    }
+    return false;
+}
+
 export class Email extends QueryableModel {
     static table = 'emails';
 
@@ -61,8 +89,64 @@ export class Email extends QueryableModel {
     @column({ type: 'string', nullable: true })
     fromName: string | null = null;
 
+    /**
+     * Amount of recipients with an email address
+     */
     @column({ type: 'integer', nullable: true })
-    recipientCount: number | null = null;
+    emailRecipientsCount: number | null = null;
+
+    /**
+     * Amount of recipients without an email address
+     */
+    @column({ type: 'integer', nullable: true })
+    otherRecipientsCount: number | null = null;
+
+    /**
+     * Amount of recipients that have successfully received the email.
+     */
+    @column({ type: 'integer' })
+    succeededCount = 0;
+
+    /**
+     * Amount of recipients that somehow failed to receive the email,
+     * but with a soft error that doesn't require action.
+     * - Duplicate email in recipient list
+     * - Unsubscribed
+     */
+    @column({ type: 'integer' })
+    softFailedCount = 0;
+
+    /**
+     * Amount of recipients that somehow failed to receive the email:
+     * - Invalid email address
+     * - Full email inbox
+     */
+    @column({ type: 'integer' })
+    failedCount = 0;
+
+    /**
+     * Unique amount of members that are in the recipients list.
+     */
+    @column({ type: 'integer' })
+    membersCount = 0;
+
+    /**
+     * Does only include bounces AFTER sending the email
+     */
+    @column({ type: 'integer' })
+    hardBouncesCount = 0;
+
+    /**
+     * Does only include bounces AFTER sending the email
+     */
+    @column({ type: 'integer' })
+    softBouncesCount = 0;
+
+    /**
+     * Does only include bounces AFTER sending the email
+     */
+    @column({ type: 'integer' })
+    spamComplaintsCount = 0;
 
     @column({ type: 'string' })
     status = EmailStatus.Draft;
@@ -70,6 +154,18 @@ export class Email extends QueryableModel {
     @column({ type: 'string' })
     recipientsStatus = EmailRecipientsStatus.NotCreated;
 
+    /**
+     * Errors related to creating the recipients.
+     */
+    @column({ type: 'json', nullable: true, decoder: SimpleErrors })
+    recipientsErrors: SimpleErrors | null = null;
+
+    /**
+     * Errors related to sending the email.
+     */
+    @column({ type: 'json', nullable: true, decoder: SimpleErrors })
+    emailErrors: SimpleErrors | null = null;
+
     /**
      * todo: ignore automatically
      */
@@ -109,6 +205,8 @@ export class Email extends QueryableModel {
         count(request: LimitedFilteredRequest, subfilter: StamhoofdFilter | null): Promise<number>;
     }> = new Map();
 
+    static pendingNotificationCountUpdates: Map<string, { timer: NodeJS.Timeout | null; lastUpdate: Date | null }> = new Map();
+
     throwIfNotReadyToSend() {
         if (this.subject == null || this.subject.length == 0) {
             throw new SimpleError({
@@ -142,6 +240,14 @@ export class Email extends QueryableModel {
             });
         }
 
+        if (this.recipientsErrors !== null && this.recipientsStatus !== EmailRecipientsStatus.Created) {
+            throw new SimpleError({
+                code: 'invalid_recipients',
+                message: 'Failed to build recipients (count)',
+                human: $t(`Er ging iets mis bij het aanmaken van de ontvangers. Probeer je selectie aan te passen. Neem contact op als het probleem zich blijft voordoen.`) + ' ' + this.recipientsErrors.getHuman(),
+            });
+        }
+
         this.validateAttachments();
     }
 
@@ -264,13 +370,10 @@ export class Email extends QueryableModel {
         return true;
     }
 
-    async send(): Promise<Email | null> {
-        this.throwIfNotReadyToSend();
-        await this.save();
-
+    async lock<T>(callback: (upToDate: Email, options: QueueHandlerOptions) => Promise<T> | T): Promise<T> {
         const id = this.id;
-        return await QueueHandler.schedule('send-email', async function (this: unknown, { abort }) {
-            let upToDate = await Email.getByID(id);
+        return await QueueHandler.schedule('lock-email-' + id, async (options) => {
+            const upToDate = await Email.getByID(id);
             if (!upToDate) {
                 throw new SimpleError({
                     code: 'not_found',
@@ -278,258 +381,546 @@ export class Email extends QueryableModel {
                     human: $t(`55899a7c-f3d4-43fe-a431-70a3a9e78e34`),
                 });
             }
-            if (upToDate.status === EmailStatus.Sent || upToDate.status === EmailStatus.Failed) {
-                // Already done
-                // In other cases -> queue has stopped and we can retry
-                return upToDate;
-            }
+            const c = await callback(upToDate, options);
+            this.copyFrom(upToDate);
+            return c;
+        });
+    }
 
-            if (upToDate.status === EmailStatus.Sending) {
-                // This is a retry.
-                if (upToDate.emailType) {
-                    // Not eligible for retry
-                    upToDate.status = EmailStatus.Failed;
-                    await upToDate.save();
-                    return upToDate;
-                }
-                if (upToDate.createdAt < new Date(new Date().getTime() - 1000 * 60 * 60 * 24 * 2)) {
-                    // Too long
-                    console.error('Email has been sending for too long. Marking as failed...', upToDate.id);
-                    upToDate.status = EmailStatus.Failed;
-                    await upToDate.save();
-                    return upToDate;
-                }
+    static async bumpNotificationCount(emailId: string, type: 'hard-bounce' | 'soft-bounce' | 'complaint') {
+        // Send an update query
+        const base = Email.update()
+            .where('id', emailId);
+
+        switch (type) {
+            case 'hard-bounce': {
+                base.set('hardBouncesCount', new SQLCalculation(
+                    SQL.column('hardBouncesCount'),
+                    new SQLPlusSign(),
+                    readDynamicSQLExpression(1),
+                ));
+                break;
             }
-            const organization = upToDate.organizationId ? await Organization.getByID(upToDate.organizationId) : null;
-            upToDate.throwIfNotReadyToSend();
+            case 'soft-bounce': {
+                base.set('softBouncesCount', new SQLCalculation(
+                    SQL.column('softBouncesCount'),
+                    new SQLPlusSign(),
+                    readDynamicSQLExpression(1),
+                ));
+                break;
+            }
+            case 'complaint': {
+                base.set('spamComplaintsCount', new SQLCalculation(
+                    SQL.column('spamComplaintsCount'),
+                    new SQLPlusSign(),
+                    readDynamicSQLExpression(1),
+                ));
+                break;
+            }
+        }
 
-            let from = upToDate.getDefaultFromAddress(organization);
-            let replyTo: EmailInterfaceRecipient | null = upToDate.getFromAddress();
+        await base.update();
 
-            if (!from) {
-                throw new SimpleError({
-                    code: 'invalid_field',
-                    message: 'Missing from',
-                    human: $t(`e92cd077-b0f1-4b0a-82a0-8a8baa82e73a`),
-                });
-            }
+        await this.checkNeedsNotificationCountUpdate(emailId, true);
+    }
 
-            // Can we send from this e-mail or reply-to?
-            if (upToDate.fromAddress && await canSendFromEmail(upToDate.fromAddress, organization ?? null)) {
-                from = upToDate.getFromAddress();
-                replyTo = null;
-            }
+    static async checkNeedsNotificationCountUpdate(emailId: string, didUpdate = false) {
+        const existing = this.pendingNotificationCountUpdates.get(emailId);
+        const object = existing ?? {
+            timer: null,
+            lastUpdate: didUpdate ? new Date() : null,
+        };
 
-            abort.throwIfAborted();
-            upToDate.status = EmailStatus.Sending;
-            upToDate.sentAt = upToDate.sentAt ?? new Date();
-            await upToDate.save();
+        if (didUpdate) {
+            object.lastUpdate = new Date();
+        }
 
-            // Create recipients if not yet created
-            await upToDate.buildRecipients();
-            abort.throwIfAborted();
+        if (existing) {
+            this.pendingNotificationCountUpdates.set(emailId, object);
+        }
 
-            // Refresh model
-            upToDate = await Email.getByID(id);
-            if (!upToDate) {
-                throw new SimpleError({
-                    code: 'not_found',
-                    message: 'Email not found',
-                    human: $t(`55899a7c-f3d4-43fe-a431-70a3a9e78e34`),
-                });
-            }
+        if (object.lastUpdate && object.lastUpdate < new Date(Date.now() - 5 * 60 * 1000)) {
+            // After 5 minutes without notifications, run an update.
+            await this.updateNotificationsCounts(emailId);
 
-            if (upToDate.recipientsStatus !== EmailRecipientsStatus.Created) {
-                throw new SimpleError({
-                    code: 'recipients_not_created',
-                    message: 'Failed to create recipients',
-                    human: $t(`f660b2eb-e382-4d21-86e4-673ca7bc2d4a`),
-                });
+            // Stop here
+            return;
+        }
+
+        // Schedule a slow update of all counts
+        if (!object.timer) {
+            object.timer = setTimeout(() => {
+                object.timer = null;
+                this.checkNeedsNotificationCountUpdate(emailId).catch(console.error);
+            }, 1 * 60 * 1000);
+        }
+    }
+
+    static async updateNotificationsCounts(emailId: string) {
+        QueueHandler.cancel('updateNotificationsCounts-' + emailId);
+        return await QueueHandler.schedule('updateNotificationsCounts-' + emailId, async () => {
+            const query = SQL.select(
+                new SQLSelectAs(
+                    new SQLCount(
+                        SQL.column('hardBounceError'),
+                    ),
+                    new SQLAlias('data__hardBounces'),
+                ),
+                // If the current amount_due is negative, we can ignore that negative part if there is a future due item
+                new SQLSelectAs(
+                    new SQLCount(
+                        SQL.column('softBounceError'),
+                    ),
+                    new SQLAlias('data__softBounces'),
+                ),
+
+                new SQLSelectAs(
+                    new SQLCount(
+                        SQL.column('spamComplaintError'),
+                    ),
+                    new SQLAlias('data__complaints'),
+                ),
+            )
+                .from(EmailRecipient.table)
+                .where('emailId', emailId);
+
+            const result = await query.fetch();
+            if (result.length !== 1) {
+                console.error('Unexpected result', result);
+                return;
+            }
+            const row = result[0]['data'];
+            if (!row) {
+                console.error('Unexpected result row', result);
+                return;
             }
 
-            // Start actually sending in batches of recipients that are not yet sent
-            let idPointer = '';
-            const batchSize = 100;
-            const recipientsSet = new Set<string>();
+            const hardBounces = row['hardBounces'];
+            const softBounces = row['softBounces'];
+            const complaints = row['complaints'];
 
-            // Create a buffer of all attachments
-            const attachments: { filename: string; path?: string; href?: string; content?: string | Buffer; contentType?: string; encoding?: string }[] = [];
+            if (typeof hardBounces !== 'number' || typeof softBounces !== 'number' || typeof complaints !== 'number') {
+                console.error('Unexpected result values', row);
+                return;
+            }
 
-            for (const attachment of upToDate.attachments) {
-                if (!attachment.content && !attachment.file) {
-                    console.warn('Attachment without content found, skipping', attachment);
-                    continue;
-                }
+            console.log('Updating email notification counts', emailId, hardBounces, softBounces, complaints);
 
-                let filename = $t('b1291584-d2ad-4ebd-88ed-cbda4f3755b4');
+            // Send an update query
+            await Email.update()
+                .where('id', emailId)
+                .set('hardBouncesCount', hardBounces)
+                .set('softBouncesCount', softBounces)
+                .set('spamComplaintsCount', complaints)
+                .update();
+        });
+    }
 
-                if (attachment.contentType === 'application/pdf') {
-                    // tmp solution for pdf only
-                    filename += '.pdf';
-                }
+    async queueForSending(waitForSending = false) {
+        this.throwIfNotReadyToSend();
+        await this.lock(async (upToDate) => {
+            if (upToDate.status === EmailStatus.Draft) {
+                upToDate.status = EmailStatus.Queued;
+            }
+            await upToDate.save();
+        });
+        if (waitForSending) {
+            return await this.resumeSending();
+        }
+        else {
+            this.resumeSending().catch(console.error);
+        }
+        return this;
+    }
 
-                if (attachment.file?.name) {
-                    filename = attachment.file.name.toLowerCase().replace(/[^a-z0-9.]+/g, '-').replace(/^-+/, '').replace(/-+$/, '');
+    async resumeSending(): Promise<Email | null> {
+        const id = this.id;
+        return await QueueHandler.schedule('send-email', async ({ abort }) => {
+            return await this.lock(async function (upToDate: Email) {
+                if (upToDate.status === EmailStatus.Sent) {
+                    // Already done
+                    // In other cases -> queue has stopped and we can retry
+                    return upToDate;
                 }
 
-                // Correct file name if needed
-                if (attachment.filename) {
-                    filename = attachment.filename.toLowerCase().replace(/[^a-z0-9.]+/g, '-').replace(/^-+/, '').replace(/-+$/, '');
+                if (upToDate.status === EmailStatus.Sending) {
+                    // This is an automatic retry.
+                    if (upToDate.emailType) {
+                        // Not eligible for retry
+                        upToDate.status = EmailStatus.Failed;
+                        await upToDate.save();
+                        return upToDate;
+                    }
+                    if (upToDate.createdAt < new Date(new Date().getTime() - 1000 * 60 * 60 * 24 * 2)) {
+                        // Too long
+                        console.error('Email has been sending for too long. Marking as failed...', upToDate.id);
+                        upToDate.status = EmailStatus.Failed;
+                        await upToDate.save();
+                        return upToDate;
+                    }
                 }
-
-                if (attachment.content) {
-                    attachments.push({
-                        filename: filename,
-                        content: attachment.content,
-                        contentType: attachment.contentType ?? undefined,
-                        encoding: 'base64',
-                    });
+                else if (upToDate.status !== EmailStatus.Queued) {
+                    console.error('Email is not queued or sending, cannot send', upToDate.id, upToDate.status);
+                    return upToDate;
                 }
-                else {
-                    // Note: because we send lots of emails, we better download the file here so we can reuse it in every email instead of downloading it every time
-                    const withSigned = await attachment.file!.withSignedUrl();
-                    if (!withSigned || !withSigned.signedUrl) {
+                const organization = upToDate.organizationId ? await Organization.getByID(upToDate.organizationId) : null;
+                let from = upToDate.getDefaultFromAddress(organization);
+                let replyTo: EmailInterfaceRecipient | null = upToDate.getFromAddress();
+                const attachments: { filename: string; path?: string; href?: string; content?: string | Buffer; contentType?: string; encoding?: string }[] = [];
+                let succeededCount = 0;
+                let softFailedCount = 0;
+                let failedCount = 0;
+
+                try {
+                    upToDate.throwIfNotReadyToSend();
+
+                    if (!from) {
                         throw new SimpleError({
-                            code: 'attachment_not_found',
-                            message: 'Attachment not found',
-                            human: $t(`ce6ddaf0-8347-42c5-b4b7-fbe860c7b7f2`),
+                            code: 'invalid_field',
+                            message: 'Missing from',
+                            human: $t(`e92cd077-b0f1-4b0a-82a0-8a8baa82e73a`),
                         });
                     }
 
-                    const filePath = withSigned.signedUrl;
-                    let fileBuffer: Buffer | null = null;
-                    try {
-                        const response = await fetch(filePath);
-                        fileBuffer = Buffer.from(await response.arrayBuffer());
+                    // Can we send from this e-mail or reply-to?
+                    if (upToDate.fromAddress && await canSendFromEmail(upToDate.fromAddress, organization ?? null)) {
+                        from = upToDate.getFromAddress();
+                        replyTo = null;
                     }
-                    catch (e) {
+
+                    abort.throwIfAborted();
+                    upToDate.status = EmailStatus.Sending;
+                    upToDate.sentAt = upToDate.sentAt ?? new Date();
+                    await upToDate.save();
+
+                    // Create recipients if not yet created
+                    await upToDate.buildRecipients();
+                    abort.throwIfAborted();
+
+                    // Refresh model
+                    const c = (await Email.getByID(id))!;
+                    if (!c) {
                         throw new SimpleError({
-                            code: 'attachment_not_found',
-                            message: 'Attachment not found',
-                            human: $t(`ce6ddaf0-8347-42c5-b4b7-fbe860c7b7f2`),
+                            code: 'not_found',
+                            message: 'Email not found',
+                            human: $t(`55899a7c-f3d4-43fe-a431-70a3a9e78e34`),
                         });
                     }
+                    upToDate.copyFrom(c);
 
-                    attachments.push({
-                        filename: filename,
-                        contentType: attachment.contentType ?? undefined,
-                        content: fileBuffer,
-                    });
-                }
-            }
+                    if (upToDate.recipientsStatus !== EmailRecipientsStatus.Created) {
+                        throw new SimpleError({
+                            code: 'recipients_not_created',
+                            message: 'Failed to create recipients',
+                            human: $t(`f660b2eb-e382-4d21-86e4-673ca7bc2d4a`),
+                        });
+                    }
 
-            while (true) {
-                abort.throwIfAborted();
-                const data = await SQL.select()
-                    .from('email_recipients')
-                    .where('emailId', upToDate.id)
-                    .where('sentAt', null)
-                    .where('id', SQLWhereSign.Greater, idPointer)
-                    .orderBy(SQL.column('id'), 'ASC')
-                    .limit(batchSize)
-                    .fetch();
-
-                const recipients = EmailRecipient.fromRows(data, 'email_recipients');
-
-                if (recipients.length === 0) {
-                    break;
-                }
+                    // Create a buffer of all attachments
+                    for (const attachment of upToDate.attachments) {
+                        if (!attachment.content && !attachment.file) {
+                            console.warn('Attachment without content found, skipping', attachment);
+                            continue;
+                        }
 
-                const sendingPromises: Promise<void>[] = [];
+                        let filename = $t('b1291584-d2ad-4ebd-88ed-cbda4f3755b4');
 
-                for (const recipient of recipients) {
-                    if (recipientsSet.has(recipient.id)) {
-                        console.error('Found duplicate recipient while sending email', recipient.id);
-                        continue;
-                    }
+                        if (attachment.contentType === 'application/pdf') {
+                            // tmp solution for pdf only
+                            filename += '.pdf';
+                        }
 
-                    recipientsSet.add(recipient.email);
-                    idPointer = recipient.id;
+                        if (attachment.file?.name) {
+                            filename = attachment.file.name.toLowerCase().replace(/[^a-z0-9.]+/g, '-').replace(/^-+/, '').replace(/-+$/, '');
+                        }
 
-                    let promiseResolve: (value: void | PromiseLike<void>) => void;
-                    const promise = new Promise<void>((resolve) => {
-                        promiseResolve = resolve;
-                    });
+                        // Correct file name if needed
+                        if (attachment.filename) {
+                            filename = attachment.filename.toLowerCase().replace(/[^a-z0-9.]+/g, '-').replace(/^-+/, '').replace(/-+$/, '');
+                        }
+
+                        if (attachment.content) {
+                            attachments.push({
+                                filename: filename,
+                                content: attachment.content,
+                                contentType: attachment.contentType ?? undefined,
+                                encoding: 'base64',
+                            });
+                        }
+                        else {
+                            // Note: because we send lots of emails, we better download the file here so we can reuse it in every email instead of downloading it every time
+                            const withSigned = await attachment.file!.withSignedUrl();
+                            if (!withSigned || !withSigned.signedUrl) {
+                                throw new SimpleError({
+                                    code: 'attachment_not_found',
+                                    message: 'Attachment not found',
+                                    human: $t(`ce6ddaf0-8347-42c5-b4b7-fbe860c7b7f2`),
+                                });
+                            }
+
+                            const filePath = withSigned.signedUrl;
+                            let fileBuffer: Buffer | null = null;
+                            try {
+                                const response = await fetch(filePath);
+                                fileBuffer = Buffer.from(await response.arrayBuffer());
+                            }
+                            catch (e) {
+                                throw new SimpleError({
+                                    code: 'attachment_not_found',
+                                    message: 'Attachment not found',
+                                    human: $t(`ce6ddaf0-8347-42c5-b4b7-fbe860c7b7f2`),
+                                });
+                            }
 
-                    const virtualRecipient = recipient.getRecipient();
+                            attachments.push({
+                                filename: filename,
+                                contentType: attachment.contentType ?? undefined,
+                                content: fileBuffer,
+                            });
+                        }
+                    }
 
-                    let resolved = false;
-                    const callback = async (error: Error | null) => {
-                        if (resolved) {
+                    // Start actually sending in batches of recipients that are not yet sent
+                    let idPointer = '';
+                    const batchSize = 100;
+                    const recipientsSet = new Set<string>();
+                    let isSavingStatus = false;
+                    let lastStatusSave = new Date();
+
+                    async function saveStatus() {
+                        if (!upToDate) {
+                            return;
+                        }
+                        if (isSavingStatus) {
+                            return;
+                        }
+                        if ((new Date().getTime() - lastStatusSave.getTime()) < 1000 * 5) {
+                            // Save at most every 5 seconds
+                            return;
+                        }
+                        if (succeededCount < upToDate.succeededCount || softFailedCount < upToDate.softFailedCount || failedCount < upToDate.failedCount) {
+                            // Do not update on retries
                             return;
                         }
-                        resolved = true;
+
+                        lastStatusSave = new Date();
+                        isSavingStatus = true;
+                        upToDate.succeededCount = succeededCount;
+                        upToDate.softFailedCount = softFailedCount;
+                        upToDate.failedCount = failedCount;
 
                         try {
-                            if (error === null) {
-                                // Mark saved
-                                recipient.sentAt = new Date();
+                            await upToDate.save();
+                        }
+                        finally {
+                            isSavingStatus = false;
+                        }
+                    }
 
-                                // Update repacements that have been generated
-                                recipient.replacements = virtualRecipient.replacements;
-                                await recipient.save();
+                    while (true) {
+                        abort.throwIfAborted();
+                        const data = await SQL.select()
+                            .from('email_recipients')
+                            .where('emailId', upToDate.id)
+                            .where('id', SQLWhereSign.Greater, idPointer)
+                            .orderBy(SQL.column('id'), 'ASC')
+                            .limit(batchSize)
+                            .fetch();
+
+                        const recipients = EmailRecipient.fromRows(data, 'email_recipients');
+
+                        if (recipients.length === 0) {
+                            break;
+                        }
+
+                        const sendingPromises: Promise<void>[] = [];
+                        let skipped = 0;
+
+                        for (const recipient of recipients) {
+                            idPointer = recipient.id;
+
+                            if (recipient.sentAt) {
+                                // Already sent
+                                if (recipient.email) {
+                                    recipientsSet.add(recipient.email);
+                                }
+                                succeededCount += 1;
+                                await saveStatus();
+                                skipped++;
+                                continue;
                             }
-                            else {
+
+                            if (!recipient.email) {
+                                skipped++;
+                                continue;
+                            }
+
+                            if (recipientsSet.has(recipient.id)) {
+                                console.error('Found duplicate recipient while sending email', recipient.id);
+                                softFailedCount += 1;
+
                                 recipient.failCount += 1;
-                                recipient.failErrorMessage = error.message;
+                                recipient.failErrorMessage = 'Duplicate recipient';
+                                recipient.failError = new SimpleErrors(
+                                    new SimpleError({
+                                        code: 'email_skipped_duplicate_recipient',
+                                        message: 'Duplicate recipient',
+                                        human: $t('Dit e-mailadres staat meerdere keren tussen de ontvangers en werd daarom overgeslagen'),
+                                    }),
+                                );
+
                                 recipient.firstFailedAt = recipient.firstFailedAt ?? new Date();
                                 recipient.lastFailedAt = new Date();
                                 await recipient.save();
+
+                                await saveStatus();
+                                skipped++;
+                                continue;
                             }
+                            recipientsSet.add(recipient.email);
+
+                            let promiseResolve: (value: void | PromiseLike<void>) => void;
+                            const promise = new Promise<void>((resolve) => {
+                                promiseResolve = resolve;
+                            });
+
+                            const virtualRecipient = recipient.getRecipient();
+
+                            let resolved = false;
+                            const callback = async (error: Error | null) => {
+                                if (resolved) {
+                                    return;
+                                }
+                                resolved = true;
+
+                                try {
+                                    if (error === null) {
+                                        // Mark saved
+                                        recipient.sentAt = new Date();
+
+                                        // Update repacements that have been generated
+                                        recipient.replacements = virtualRecipient.replacements;
+
+                                        succeededCount += 1;
+                                        await recipient.save();
+                                        await saveStatus();
+                                    }
+                                    else {
+                                        recipient.failCount += 1;
+                                        recipient.failErrorMessage = error.message;
+                                        recipient.failError = errorToSimpleErrors(error);
+                                        recipient.firstFailedAt = recipient.firstFailedAt ?? new Date();
+                                        recipient.lastFailedAt = new Date();
+
+                                        if (isSoftEmailRecipientError(recipient.failError)) {
+                                            softFailedCount += 1;
+                                        }
+                                        else {
+                                            failedCount += 1;
+                                        }
+                                        await recipient.save();
+                                        await saveStatus();
+                                    }
+                                }
+                                catch (e) {
+                                    console.error(e);
+                                }
+                                promiseResolve();
+                            };
+
+                            // Do send the email
+                            // Create e-mail builder
+                            const builder = await getEmailBuilder(organization ?? null, {
+                                recipients: [
+                                    virtualRecipient,
+                                ],
+                                from,
+                                replyTo,
+                                subject: upToDate.subject!,
+                                html: upToDate.html!,
+                                type: upToDate.emailType ? 'transactional' : 'broadcast',
+                                attachments,
+                                callback(error: Error | null) {
+                                    callback(error).catch(console.error);
+                                },
+                                headers: {
+                                    'X-Email-Id': upToDate.id,
+                                    'X-Email-Recipient-Id': recipient.id,
+                                },
+                            });
+                            abort.throwIfAborted(); // do not schedule if aborted
+                            EmailClass.schedule(builder);
+                            sendingPromises.push(promise);
                         }
-                        catch (e) {
-                            console.error(e);
+
+                        if (sendingPromises.length > 0 || skipped > 0) {
+                            await Promise.all(sendingPromises);
                         }
-                        promiseResolve();
-                    };
-
-                    // Do send the email
-                    // Create e-mail builder
-                    const builder = await getEmailBuilder(organization ?? null, {
-                        recipients: [
-                            virtualRecipient,
-                        ],
-                        from,
-                        replyTo,
-                        subject: upToDate.subject!,
-                        html: upToDate.html!,
-                        type: upToDate.emailType ? 'transactional' : 'broadcast',
-                        attachments,
-                        callback(error: Error | null) {
-                            callback(error).catch(console.error);
-                        },
-                    });
-                    abort.throwIfAborted(); // do not schedule if aborted
-                    EmailClass.schedule(builder);
-                    sendingPromises.push(promise);
+                        else {
+                            break;
+                        }
+                    }
                 }
+                catch (e) {
+                    if (!upToDate) {
+                        throw e;
+                    }
 
-                if (sendingPromises.length > 0) {
-                    await Promise.all(sendingPromises);
+                    upToDate.succeededCount = succeededCount;
+                    upToDate.softFailedCount = softFailedCount;
+                    upToDate.failedCount = failedCount;
+
+                    if (isAbortedError(e) || ((isSimpleError(e) || isSimpleErrors(e)) && e.hasCode('SHUTDOWN'))) {
+                        // Keep sending status: we'll resume after the reboot
+                        await upToDate.save();
+                        throw e;
+                    }
+
+                    upToDate.emailErrors = errorToSimpleErrors(e);
+                    upToDate.status = EmailStatus.Failed;
+                    await upToDate.save();
+                    throw e;
+                }
+
+                if (upToDate.emailRecipientsCount === 0 && upToDate.userId === null) {
+                    // We only delete automated emails (email type) if they have no recipients
+                    console.log('No recipients found for email ', upToDate.id, ' deleting...');
+                    await upToDate.delete();
+                    return null;
+                }
+
+                console.log('Finished sending email', upToDate.id);
+                // Mark email as sent
+
+                if ((succeededCount + failedCount + softFailedCount) === 0) {
+                    upToDate.status = EmailStatus.Failed;
+                    upToDate.emailErrors = new SimpleErrors(
+                        new SimpleError({
+                            code: 'no_recipients',
+                            message: 'No recipients',
+                            human: $t(`Geen ontvangers gevonden`),
+                        }),
+                    );
                 }
                 else {
-                    break;
+                    upToDate.status = EmailStatus.Sent;
+                    upToDate.emailErrors = null;
                 }
-            }
 
-            if (upToDate.recipientCount === 0 && upToDate.userId === null) {
-                // We only delete automated emails (email type) if they have no recipients
-                console.log('No recipients found for email ', upToDate.id, ' deleting...');
-                await upToDate.delete();
-                return null;
-            }
+                upToDate.succeededCount = succeededCount;
+                upToDate.softFailedCount = softFailedCount;
+                upToDate.failedCount = failedCount;
 
-            console.log('Finished sending email', upToDate.id);
-            // Mark email as sent
-            upToDate.status = EmailStatus.Sent;
-            await upToDate.save();
-            return upToDate;
+                await upToDate.save();
+                return upToDate;
+            });
         });
     }
 
     updateCount() {
         const id = this.id;
-        QueueHandler.schedule('email-count-' + this.id, async function () {
+        QueueHandler.abort('email-count-' + this.id);
+        QueueHandler.schedule('email-count-' + this.id, async function ({ abort }) {
             let upToDate = await Email.getByID(id);
 
             if (!upToDate || upToDate.sentAt || !upToDate.id || upToDate.status !== EmailStatus.Draft) {
@@ -558,12 +949,14 @@ export class Email extends QueryableModel {
                         search: subfilter.search,
                     });
 
+                    abort.throwIfAborted();
                     const c = await loader.count(request, subfilter.subfilter);
 
                     count += c;
                 }
+                abort.throwIfAborted();
 
-                // Check if we have a more reliable recipientCount in the meantime
+                // Check if we have a more reliable emailRecipientsCount in the meantime
                 upToDate = await Email.getByID(id);
 
                 if (!upToDate) {
@@ -572,12 +965,27 @@ export class Email extends QueryableModel {
                 if (upToDate.recipientsStatus === EmailRecipientsStatus.Created) {
                     return;
                 }
-                upToDate.recipientCount = count;
+                upToDate.emailRecipientsCount = count;
                 await upToDate.save();
             }
             catch (e) {
+                if (isAbortedError(e)) {
+                    return;
+                }
                 console.error('Failed to update count for email', id);
                 console.error(e);
+
+                // Check if we have a more reliable emailRecipientsCount in the meantime
+                upToDate = await Email.getByID(id);
+
+                if (!upToDate) {
+                    return;
+                }
+                if (upToDate.recipientsStatus === EmailRecipientsStatus.Created) {
+                    return;
+                }
+                upToDate.recipientsErrors = errorToSimpleErrors(e);
+                await upToDate.save();
             }
         }).catch(console.error);
     }
@@ -606,7 +1014,10 @@ export class Email extends QueryableModel {
             upToDate.recipientsStatus = EmailRecipientsStatus.Creating;
             await upToDate.save();
 
+            const membersSet = new Set<string>();
+
             let count = 0;
+            let countWithoutEmail = 0;
 
             try {
                 // Delete all recipients
@@ -639,10 +1050,10 @@ export class Email extends QueryableModel {
                         const response = await loader.fetch(request, subfilter.subfilter);
 
                         for (const item of response.results) {
-                            if (!item.email) {
+                            if (!item.email && !item.memberId && !item.userId) {
                                 continue;
                             }
-                            count += 1;
+
                             const recipient = new EmailRecipient();
                             recipient.emailType = upToDate.emailType;
                             recipient.objectId = item.objectId;
@@ -651,8 +1062,22 @@ export class Email extends QueryableModel {
                             recipient.firstName = item.firstName;
                             recipient.lastName = item.lastName;
                             recipient.replacements = item.replacements;
+                            recipient.memberId = item.memberId ?? null;
+                            recipient.userId = item.userId ?? null;
+                            recipient.organizationId = upToDate.organizationId ?? null;
 
                             await recipient.save();
+
+                            if (recipient.memberId) {
+                                membersSet.add(recipient.memberId);
+                            }
+
+                            if (!recipient.email) {
+                                countWithoutEmail += 1;
+                            }
+                            else {
+                                count += 1;
+                            }
                         }
 
                         request = response.next ?? null;
@@ -660,13 +1085,17 @@ export class Email extends QueryableModel {
                 }
 
                 upToDate.recipientsStatus = EmailRecipientsStatus.Created;
-                upToDate.recipientCount = count;
+                upToDate.emailRecipientsCount = count;
+                upToDate.otherRecipientsCount = countWithoutEmail;
+                upToDate.recipientsErrors = null;
+                upToDate.membersCount = membersSet.size;
                 await upToDate.save();
             }
-            catch (e) {
+            catch (e: unknown) {
                 console.error('Failed to build recipients for email', id);
                 console.error(e);
                 upToDate.recipientsStatus = EmailRecipientsStatus.NotCreated;
+                upToDate.recipientsErrors = errorToSimpleErrors(e);
                 await upToDate.save();
             }
         });