@stamhoofd/models 2.97.2 → 2.98.0

package/src/models/Email.ts

@@ -15,6 +15,8 @@ import { Organization } from './Organization';
 import { User } from './User';
 import { Platform } from './Platform';
 
+type Attachment = { filename: string; path?: string; href?: string; content?: string | Buffer; contentType?: string; encoding?: string };
+
 function errorToSimpleErrors(e: unknown) {
     if (isSimpleErrors(e)) {
         return e;
@@ -601,40 +603,191 @@ export class Email extends QueryableModel {
         return this;
     }
 
-    async resumeSending(): Promise<Email | null> {
+    private async loadAttachments(): Promise<Attachment[]> {
+        const attachments: Attachment[] = [];
+        for (const attachment of this.attachments) {
+            if (!attachment.content && !attachment.file) {
+                console.warn('Attachment without content found, skipping', attachment);
+                continue;
+            }
+
+            let filename = $t('b1291584-d2ad-4ebd-88ed-cbda4f3755b4');
+
+            if (attachment.contentType === 'application/pdf') {
+                // tmp solution for pdf only
+                filename += '.pdf';
+            }
+
+            if (attachment.file?.name) {
+                filename = attachment.file.name.toLowerCase().replace(/[^a-z0-9.]+/g, '-').replace(/^-+/, '').replace(/-+$/, '');
+            }
+
+            // Correct file name if needed
+            if (attachment.filename) {
+                filename = attachment.filename.toLowerCase().replace(/[^a-z0-9.]+/g, '-').replace(/^-+/, '').replace(/-+$/, '');
+            }
+
+            if (attachment.content) {
+                attachments.push({
+                    filename: filename,
+                    content: attachment.content,
+                    contentType: attachment.contentType ?? undefined,
+                    encoding: 'base64',
+                });
+            }
+            else {
+                // Note: because we send lots of emails, we better download the file here so we can reuse it in every email instead of downloading it every time
+                const withSigned = await attachment.file!.withSignedUrl();
+                if (!withSigned || !withSigned.signedUrl) {
+                    throw new SimpleError({
+                        code: 'attachment_not_found',
+                        message: 'Attachment not found',
+                        human: $t(`ce6ddaf0-8347-42c5-b4b7-fbe860c7b7f2`),
+                    });
+                }
+
+                const filePath = withSigned.signedUrl;
+                let fileBuffer: Buffer | null = null;
+                try {
+                    const response = await fetch(filePath);
+                    fileBuffer = Buffer.from(await response.arrayBuffer());
+                }
+                catch (e) {
+                    throw new SimpleError({
+                        code: 'attachment_not_found',
+                        message: 'Attachment not found',
+                        human: $t(`ce6ddaf0-8347-42c5-b4b7-fbe860c7b7f2`),
+                    });
+                }
+
+                attachments.push({
+                    filename: filename,
+                    contentType: attachment.contentType ?? undefined,
+                    content: fileBuffer,
+                });
+            }
+        }
+        return attachments;
+    }
+
+    private async sendSingleRecipient(recipient: EmailRecipient, organization: Organization | null, data: { from: EmailInterfaceRecipient; replyTo: EmailInterfaceRecipient | null; attachments: Attachment[] }) {
+        let promiseResolve: (value: void | PromiseLike<void>) => void;
+        const promise = new Promise<void>((resolve) => {
+            promiseResolve = resolve;
+        });
+
+        const virtualRecipient = recipient.getRecipient();
+
+        let resolved = false;
+        const callback = async (error: Error | null) => {
+            if (resolved) {
+                return;
+            }
+            resolved = true;
+
+            try {
+                if (error === null) {
+                    // Mark saved
+                    recipient.sentAt = new Date();
+                    recipient.failErrorMessage = null;
+
+                    if (recipient.failError) {
+                        recipient.previousFailError = recipient.failError;
+                    }
+                    recipient.failError = null;
+
+                    // Update repacements that have been generated
+                    recipient.replacements = virtualRecipient.replacements;
+                    await recipient.save();
+                }
+                else {
+                    recipient.failCount += 1;
+                    recipient.failErrorMessage = error.message;
+                    if (recipient.failError) {
+                        recipient.previousFailError = recipient.failError;
+                    }
+                    recipient.failError = errorToSimpleErrors(error);
+                    recipient.firstFailedAt = recipient.firstFailedAt ?? new Date();
+                    recipient.lastFailedAt = new Date();
+                    await recipient.save();
+                }
+            }
+            catch (e) {
+                console.error(e);
+            }
+            promiseResolve();
+        };
+
+        // Do send the email
+        // Create e-mail builder
+        const builder = await getEmailBuilder(organization ?? null, {
+            recipients: [
+                virtualRecipient,
+            ],
+            from: data.from,
+            replyTo: data.replyTo,
+            subject: this.subject!,
+            html: this.html!,
+            type: this.emailType ? 'transactional' : 'broadcast',
+            attachments: data.attachments,
+            callback(error: Error | null) {
+                callback(error).catch(console.error);
+            },
+            headers: {
+                'X-Email-Id': this.id,
+                'X-Email-Recipient-Id': recipient.id,
+            },
+        });
+        EmailClass.schedule(builder);
+        return await promise;
+    }
+
+    async resumeSending(singleRecipientId: string | null = null): Promise<Email | null> {
         const id = this.id;
         return await QueueHandler.schedule('send-email', async ({ abort }) => {
             return await this.lock(async function (upToDate: Email) {
                 if (upToDate.status === EmailStatus.Sent) {
                     // Already done
                     // In other cases -> queue has stopped and we can retry
-                    return upToDate;
+                    if (!singleRecipientId) {
+                        console.log('Email already sent, skipping...', upToDate.id);
+                        return upToDate;
+                    }
                 }
+                else {
+                    if (singleRecipientId) {
+                        // Not possible
+                        throw new SimpleError({
+                            code: 'invalid_state',
+                            message: 'Cannot retry single recipient for email that is not yet sent',
+                        });
+                    }
 
-                if (upToDate.status === EmailStatus.Sending) {
+                    if (upToDate.status === EmailStatus.Sending) {
                     // This is an automatic retry.
-                    if (upToDate.emailType) {
+                        if (upToDate.emailType) {
                         // Not eligible for retry
-                        upToDate.status = EmailStatus.Failed;
-                        await upToDate.save();
-                        return upToDate;
-                    }
-                    if (upToDate.createdAt < new Date(new Date().getTime() - 1000 * 60 * 60 * 24 * 2)) {
+                            upToDate.status = EmailStatus.Failed;
+                            await upToDate.save();
+                            return upToDate;
+                        }
+                        if (upToDate.createdAt < new Date(new Date().getTime() - 1000 * 60 * 60 * 24 * 2)) {
                         // Too long
-                        console.error('Email has been sending for too long. Marking as failed...', upToDate.id);
-                        upToDate.status = EmailStatus.Failed;
-                        await upToDate.save();
+                            console.error('Email has been sending for too long. Marking as failed...', upToDate.id);
+                            upToDate.status = EmailStatus.Failed;
+                            await upToDate.save();
+                            return upToDate;
+                        }
+                    }
+                    else if (upToDate.status !== EmailStatus.Queued) {
+                        console.error('Email is not queued or sending, cannot send', upToDate.id, upToDate.status);
                         return upToDate;
                     }
                 }
-                else if (upToDate.status !== EmailStatus.Queued) {
-                    console.error('Email is not queued or sending, cannot send', upToDate.id, upToDate.status);
-                    return upToDate;
-                }
+
                 const organization = upToDate.organizationId ? await Organization.getByID(upToDate.organizationId) : null;
                 let from = upToDate.getDefaultFromAddress(organization);
                 let replyTo: EmailInterfaceRecipient | null = upToDate.getFromAddress();
-                const attachments: { filename: string; path?: string; href?: string; content?: string | Buffer; contentType?: string; encoding?: string }[] = [];
                 let succeededCount = 0;
                 let softFailedCount = 0;
                 let failedCount = 0;
@@ -658,7 +811,10 @@ export class Email extends QueryableModel {
                     }
 
                     abort.throwIfAborted();
-                    upToDate.status = EmailStatus.Sending;
+
+                    if (!singleRecipientId) {
+                        upToDate.status = EmailStatus.Sending;
+                    }
                     upToDate.sentAt = upToDate.sentAt ?? new Date();
                     await upToDate.save();
 
@@ -687,68 +843,7 @@ export class Email extends QueryableModel {
 
                     // Create a buffer of all attachments
                     if (upToDate.sendAsEmail === true) {
-                        for (const attachment of upToDate.attachments) {
-                            if (!attachment.content && !attachment.file) {
-                                console.warn('Attachment without content found, skipping', attachment);
-                                continue;
-                            }
-
-                            let filename = $t('b1291584-d2ad-4ebd-88ed-cbda4f3755b4');
-
-                            if (attachment.contentType === 'application/pdf') {
-                            // tmp solution for pdf only
-                                filename += '.pdf';
-                            }
-
-                            if (attachment.file?.name) {
-                                filename = attachment.file.name.toLowerCase().replace(/[^a-z0-9.]+/g, '-').replace(/^-+/, '').replace(/-+$/, '');
-                            }
-
-                            // Correct file name if needed
-                            if (attachment.filename) {
-                                filename = attachment.filename.toLowerCase().replace(/[^a-z0-9.]+/g, '-').replace(/^-+/, '').replace(/-+$/, '');
-                            }
-
-                            if (attachment.content) {
-                                attachments.push({
-                                    filename: filename,
-                                    content: attachment.content,
-                                    contentType: attachment.contentType ?? undefined,
-                                    encoding: 'base64',
-                                });
-                            }
-                            else {
-                            // Note: because we send lots of emails, we better download the file here so we can reuse it in every email instead of downloading it every time
-                                const withSigned = await attachment.file!.withSignedUrl();
-                                if (!withSigned || !withSigned.signedUrl) {
-                                    throw new SimpleError({
-                                        code: 'attachment_not_found',
-                                        message: 'Attachment not found',
-                                        human: $t(`ce6ddaf0-8347-42c5-b4b7-fbe860c7b7f2`),
-                                    });
-                                }
-
-                                const filePath = withSigned.signedUrl;
-                                let fileBuffer: Buffer | null = null;
-                                try {
-                                    const response = await fetch(filePath);
-                                    fileBuffer = Buffer.from(await response.arrayBuffer());
-                                }
-                                catch (e) {
-                                    throw new SimpleError({
-                                        code: 'attachment_not_found',
-                                        message: 'Attachment not found',
-                                        human: $t(`ce6ddaf0-8347-42c5-b4b7-fbe860c7b7f2`),
-                                    });
-                                }
-
-                                attachments.push({
-                                    filename: filename,
-                                    contentType: attachment.contentType ?? undefined,
-                                    content: fileBuffer,
-                                });
-                            }
-                        }
+                        const attachments = await upToDate.loadAttachments();
 
                         // Start actually sending in batches of recipients that are not yet sent
                         let idPointer = '';
@@ -757,6 +852,11 @@ export class Email extends QueryableModel {
                         let lastStatusSave = new Date();
 
                         async function saveStatus() {
+                            if (singleRecipientId) {
+                                // Don't save during looping
+                                return;
+                            }
+
                             if (!upToDate) {
                                 return;
                             }
@@ -825,78 +925,42 @@ export class Email extends QueryableModel {
                                     continue;
                                 }
 
-                                let promiseResolve: (value: void | PromiseLike<void>) => void;
-                                const promise = new Promise<void>((resolve) => {
-                                    promiseResolve = resolve;
-                                });
-
-                                const virtualRecipient = recipient.getRecipient();
-
-                                let resolved = false;
-                                const callback = async (error: Error | null) => {
-                                    if (resolved) {
-                                        return;
-                                    }
-                                    resolved = true;
-
-                                    try {
-                                        if (error === null) {
-                                        // Mark saved
-                                            recipient.sentAt = new Date();
-
-                                            // Update repacements that have been generated
-                                            recipient.replacements = virtualRecipient.replacements;
-
-                                            succeededCount += 1;
-                                            await recipient.save();
-                                            await saveStatus();
+                                if (singleRecipientId) {
+                                    if (recipient.id !== singleRecipientId) {
+                                        // Failed or soft-failed
+                                        if (recipient.failError && isSoftEmailRecipientError(recipient.failError)) {
+                                            softFailedCount += 1;
                                         }
                                         else {
-                                            recipient.failCount += 1;
-                                            recipient.failErrorMessage = error.message;
-                                            recipient.failError = errorToSimpleErrors(error);
-                                            recipient.firstFailedAt = recipient.firstFailedAt ?? new Date();
-                                            recipient.lastFailedAt = new Date();
-
-                                            if (isSoftEmailRecipientError(recipient.failError)) {
-                                                softFailedCount += 1;
-                                            }
-                                            else {
-                                                failedCount += 1;
-                                            }
-                                            await recipient.save();
-                                            await saveStatus();
+                                            failedCount += 1;
                                         }
+                                        skipped++;
+                                        await saveStatus();
+                                        continue;
                                     }
-                                    catch (e) {
-                                        console.error(e);
-                                    }
-                                    promiseResolve();
-                                };
-
-                                // Do send the email
-                                // Create e-mail builder
-                                const builder = await getEmailBuilder(organization ?? null, {
-                                    recipients: [
-                                        virtualRecipient,
-                                    ],
+                                }
+
+                                const promise = upToDate.sendSingleRecipient(recipient, organization ?? null, {
                                     from,
                                     replyTo,
-                                    subject: upToDate.subject!,
-                                    html: upToDate.html!,
-                                    type: upToDate.emailType ? 'transactional' : 'broadcast',
                                     attachments,
-                                    callback(error: Error | null) {
-                                        callback(error).catch(console.error);
-                                    },
-                                    headers: {
-                                        'X-Email-Id': upToDate.id,
-                                        'X-Email-Recipient-Id': recipient.id,
-                                    },
                                 });
-                                abort.throwIfAborted(); // do not schedule if aborted
-                                EmailClass.schedule(builder);
-                                sendingPromises.push(promise);
+                                sendingPromises.push(promise.then(async () => {
+                                    if (recipient.sentAt) {
+                                        succeededCount += 1;
+                                        await saveStatus();
+                                    }
+                                    else {
+                                        // Failed or soft-failed
+                                        if (recipient.failError && isSoftEmailRecipientError(recipient.failError)) {
+                                            softFailedCount += 1;
+                                        }
+                                        else {
+                                            failedCount += 1;
+                                        }
+                                        await saveStatus();
+                                    }
+                                }));
                             }
 
                             if (sendingPromises.length > 0 || skipped > 0) {
@@ -913,45 +977,49 @@ export class Email extends QueryableModel {
                         throw e;
                     }
 
-                    upToDate.succeededCount = succeededCount;
-                    upToDate.softFailedCount = softFailedCount;
-                    upToDate.failedCount = failedCount;
+                    if (!singleRecipientId) {
+                        upToDate.succeededCount = succeededCount;
+                        upToDate.softFailedCount = softFailedCount;
+                        upToDate.failedCount = failedCount;
 
-                    if (isAbortedError(e) || ((isSimpleError(e) || isSimpleErrors(e)) && e.hasCode('SHUTDOWN'))) {
-                        // Keep sending status: we'll resume after the reboot
+                        if (isAbortedError(e) || ((isSimpleError(e) || isSimpleErrors(e)) && e.hasCode('SHUTDOWN'))) {
+                            // Keep sending status: we'll resume after the reboot
+                            await upToDate.save();
+                            throw e;
+                        }
+
+                        upToDate.emailErrors = errorToSimpleErrors(e);
+                        upToDate.status = EmailStatus.Failed;
                         await upToDate.save();
-                        throw e;
                     }
-
-                    upToDate.emailErrors = errorToSimpleErrors(e);
-                    upToDate.status = EmailStatus.Failed;
-                    await upToDate.save();
                     throw e;
                 }
 
-                if (upToDate.sendAsEmail && upToDate.emailRecipientsCount === 0 && upToDate.userId === null) {
-                    // We only delete automated emails (email type) if they have no recipients
-                    console.log('No recipients found for email ', upToDate.id, ' deleting...');
-                    await upToDate.delete();
-                    return null;
-                }
+                if (!singleRecipientId) {
+                    if (upToDate.sendAsEmail && upToDate.emailRecipientsCount === 0 && upToDate.userId === null) {
+                        // We only delete automated emails (email type) if they have no recipients
+                        console.log('No recipients found for email ', upToDate.id, ' deleting...');
+                        await upToDate.delete();
+                        return null;
+                    }
 
-                console.log('Finished sending email', upToDate.id);
-                // Mark email as sent
-
-                if (upToDate.sendAsEmail && !upToDate.showInMemberPortal && (succeededCount + failedCount + softFailedCount) === 0) {
-                    upToDate.status = EmailStatus.Failed;
-                    upToDate.emailErrors = new SimpleErrors(
-                        new SimpleError({
-                            code: 'no_recipients',
-                            message: 'No recipients',
-                            human: $t(`9fe3de8e-090c-4949-97da-4810ce9e61c7`),
-                        }),
-                    );
-                }
-                else {
-                    upToDate.status = EmailStatus.Sent;
-                    upToDate.emailErrors = null;
+                    console.log('Finished sending email', upToDate.id);
+                    // Mark email as sent
+
+                    if (upToDate.sendAsEmail && !upToDate.showInMemberPortal && (succeededCount + failedCount + softFailedCount) === 0) {
+                        upToDate.status = EmailStatus.Failed;
+                        upToDate.emailErrors = new SimpleErrors(
+                            new SimpleError({
+                                code: 'no_recipients',
+                                message: 'No recipients',
+                                human: $t(`9fe3de8e-090c-4949-97da-4810ce9e61c7`),
+                            }),
+                        );
+                    }
+                    else {
+                        upToDate.status = EmailStatus.Sent;
+                        upToDate.emailErrors = null;
+                    }
                 }
 
                 upToDate.succeededCount = succeededCount;
@@ -1365,37 +1433,53 @@ export class Email extends QueryableModel {
         const cleanedRecipients: EmailRecipient[] = [...recipientsMap.values()];
         const structures = await EmailRecipient.getStructures(cleanedRecipients);
 
-        const virtualRecipients = structures.map((struct) => {
-            const recipient = struct.getRecipient();
-
-            return {
-                struct,
-                recipient,
-            };
-        });
-        for (const { struct, recipient } of virtualRecipients) {
+        for (const struct of structures) {
             if (!(struct.userId === user.id || struct.email === user.email) && !((struct.userId === null && struct.email === null))) {
-                stripSensitiveRecipientReplacements(recipient, {
+                stripSensitiveRecipientReplacements(struct, {
                     organization,
                     willFill: true,
                 });
             }
 
-            recipient.email = user.email;
-            recipient.userId = user.id;
+            struct.firstName = user.firstName;
+            struct.lastName = user.lastName;
+            struct.email = user.email;
+            struct.userId = user.id;
 
             // We always refresh the data when we display it on the web (so everything is up to date)
-            await fillRecipientReplacements(recipient, {
+            await fillRecipientReplacements(struct, {
                 organization,
                 from: this.getFromAddress(),
                 replyTo: null,
                 forPreview: false,
                 forceRefresh: true,
             });
-            stripRecipientReplacementsForWebDisplay(recipient, {
+            stripRecipientReplacementsForWebDisplay(struct, {
                 organization,
             });
-            struct.replacements = recipient.replacements;
+            if (this.html) {
+                struct.replacements = removeUnusedReplacements(this.html, struct.replacements);
+            }
+        }
+
+        // Loop structures and remove if they have exactly the same content
+        // We do this here, because it is possible the user didn't receive any emails, so
+        // the merging at time of sending the emails didn't happen (uniqueness happened on email)
+        const uniqueStructures: EmailRecipientStruct[] = [];
+        for (const struct of structures) {
+            let found = false;
+            for (const unique of uniqueStructures) {
+                const merged = mergeReplacementsIfEqual(unique.replacements, struct.replacements);
+                if (merged !== false) {
+                    unique.replacements = merged;
+                    found = true;
+                    break;
+                }
+            }
+
+            if (!found) {
+                uniqueStructures.push(struct);
+            }
         }
 
         let organizationStruct: BaseOrganization | null = null;
@@ -1406,7 +1490,7 @@ export class Email extends QueryableModel {
         return EmailWithRecipients.create({
             ...this,
             organization: organizationStruct,
-            recipients: structures,
+            recipients: uniqueStructures,
 
             // Remove private-like data
             softBouncesCount: 0,

package/src/models/EmailRecipient.ts

@@ -57,6 +57,9 @@ export class EmailRecipient extends QueryableModel {
     @column({ type: 'json', nullable: true, decoder: SimpleErrors })
     failError: SimpleErrors | null = null;
 
+    @column({ type: 'json', nullable: true, decoder: SimpleErrors })
+    previousFailError: SimpleErrors | null = null;
+
     @column({ type: 'string', nullable: true })
     organizationId: string | null = null;