@stamhoofd/models 2.93.0 → 2.95.0

package/src/models/Email.ts

@@ -1,5 +1,5 @@
 import { column } from '@simonbackx/simple-database';
-import { EmailAttachment, EmailPreview, EmailRecipientFilter, EmailRecipientFilterType, EmailRecipientsStatus, EmailRecipient as EmailRecipientStruct, EmailStatus, Email as EmailStruct, EmailTemplateType, getExampleRecipient, LimitedFilteredRequest, PaginatedResponse, SortItemDirection, StamhoofdFilter } from '@stamhoofd/structures';
+import { User as UserStruct, EmailAttachment, EmailPreview, EmailRecipientFilter, EmailRecipientFilterType, EmailRecipientsStatus, EmailRecipient as EmailRecipientStruct, EmailStatus, Email as EmailStruct, EmailTemplateType, EmailWithRecipients, getExampleRecipient, isSoftEmailRecipientError, LimitedFilteredRequest, PaginatedResponse, Replacement, SortItemDirection, StamhoofdFilter, BaseOrganization } from '@stamhoofd/structures';
 import { v4 as uuidv4 } from 'uuid';
 
 import { AnyDecoder, ArrayDecoder } from '@simonbackx/simple-encoding';
@@ -8,10 +8,11 @@ import { I18n } from '@stamhoofd/backend-i18n';
 import { Email as EmailClass, EmailInterfaceRecipient } from '@stamhoofd/email';
 import { isAbortedError, QueueHandler, QueueHandlerOptions } from '@stamhoofd/queues';
 import { QueryableModel, readDynamicSQLExpression, SQL, SQLAlias, SQLCalculation, SQLCount, SQLPlusSign, SQLSelectAs, SQLWhereSign } from '@stamhoofd/sql';
-import { canSendFromEmail, fillRecipientReplacements, getEmailBuilder } from '../helpers/EmailBuilder';
+import { canSendFromEmail, fillRecipientReplacements, getEmailBuilder, mergeReplacementsIfEqual, removeUnusedReplacements, stripRecipientReplacementsForWebDisplay, stripSensitiveRecipientReplacements } from '../helpers/EmailBuilder';
 import { EmailRecipient } from './EmailRecipient';
 import { EmailTemplate } from './EmailTemplate';
 import { Organization } from './Organization';
+import { User } from './User';
 
 function errorToSimpleErrors(e: unknown) {
     if (isSimpleErrors(e)) {
@@ -31,16 +32,6 @@ function errorToSimpleErrors(e: unknown) {
     }
 }
 
-export function isSoftEmailRecipientError(error: SimpleErrors) {
-    if (error.hasCode('email_skipped_unsubscribed')) {
-        return true;
-    }
-    if (error.hasCode('email_skipped_duplicate_recipient')) {
-        return true;
-    }
-    return false;
-}
-
 export class Email extends QueryableModel {
     static table = 'emails';
 
@@ -178,6 +169,12 @@ export class Email extends QueryableModel {
     })
     sentAt: Date | null = null;
 
+    @column({
+        type: 'datetime',
+        nullable: true,
+    })
+    deletedAt: Date | null = null;
+
     @column({
         type: 'datetime', beforeSave(old?: any) {
             if (old !== undefined) {
@@ -240,7 +237,7 @@ export class Email extends QueryableModel {
             });
         }
 
-        if (this.recipientsErrors !== null && this.recipientsStatus !== EmailRecipientsStatus.Created) {
+        if (this.status === EmailStatus.Draft && this.recipientsErrors !== null && this.recipientsStatus !== EmailRecipientsStatus.Created) {
             throw new SimpleError({
                 code: 'invalid_recipients',
                 message: 'Failed to build recipients (count)',
@@ -248,6 +245,14 @@ export class Email extends QueryableModel {
             });
         }
 
+        if (this.deletedAt) {
+            throw new SimpleError({
+                code: 'invalid_state',
+                message: 'Email is deleted',
+                human: $t(`Deze e-mail is verwijderd en kan niet verzonden worden.`),
+            });
+        }
+
         this.validateAttachments();
     }
 
@@ -522,6 +527,10 @@ export class Email extends QueryableModel {
             if (upToDate.status === EmailStatus.Draft) {
                 upToDate.status = EmailStatus.Queued;
             }
+            if (upToDate.status === EmailStatus.Failed) {
+                // Retry failed email
+                upToDate.status = EmailStatus.Queued;
+            }
             await upToDate.save();
         });
         if (waitForSending) {
@@ -683,7 +692,6 @@ export class Email extends QueryableModel {
                     // Start actually sending in batches of recipients that are not yet sent
                     let idPointer = '';
                     const batchSize = 100;
-                    const recipientsSet = new Set<string>();
                     let isSavingStatus = false;
                     let lastStatusSave = new Date();
 
@@ -740,10 +748,6 @@ export class Email extends QueryableModel {
                             idPointer = recipient.id;
 
                             if (recipient.sentAt) {
-                                // Already sent
-                                if (recipient.email) {
-                                    recipientsSet.add(recipient.email);
-                                }
                                 succeededCount += 1;
                                 await saveStatus();
                                 skipped++;
@@ -755,29 +759,10 @@ export class Email extends QueryableModel {
                                 continue;
                             }
 
-                            if (recipientsSet.has(recipient.id)) {
-                                console.error('Found duplicate recipient while sending email', recipient.id);
-                                softFailedCount += 1;
-
-                                recipient.failCount += 1;
-                                recipient.failErrorMessage = 'Duplicate recipient';
-                                recipient.failError = new SimpleErrors(
-                                    new SimpleError({
-                                        code: 'email_skipped_duplicate_recipient',
-                                        message: 'Duplicate recipient',
-                                        human: $t('Dit e-mailadres staat meerdere keren tussen de ontvangers en werd daarom overgeslagen'),
-                                    }),
-                                );
-
-                                recipient.firstFailedAt = recipient.firstFailedAt ?? new Date();
-                                recipient.lastFailedAt = new Date();
-                                await recipient.save();
-
-                                await saveStatus();
+                            if (recipient.duplicateOfRecipientId) {
                                 skipped++;
                                 continue;
                             }
-                            recipientsSet.add(recipient.email);
 
                             let promiseResolve: (value: void | PromiseLike<void>) => void;
                             const promise = new Promise<void>((resolve) => {
@@ -1009,12 +994,15 @@ export class Email extends QueryableModel {
 
             abort.throwIfAborted();
 
+            console.log('Building recipients for email', id);
+
             // If it is already creating -> something went wrong (e.g. server restart) and we can safely try again
 
             upToDate.recipientsStatus = EmailRecipientsStatus.Creating;
             await upToDate.save();
 
             const membersSet = new Set<string>();
+            const emailsSet = new Set<string>();
 
             let count = 0;
             let countWithoutEmail = 0;
@@ -1054,6 +1042,39 @@ export class Email extends QueryableModel {
                                 continue;
                             }
 
+                            item.replacements = removeUnusedReplacements(upToDate.html ?? '', item.replacements);
+
+                            let duplicateOfRecipientId: string | null = null;
+                            if (item.email && emailsSet.has(item.email)) {
+                                console.log('Found duplicate email recipient', item.email);
+
+                                // Try to merge
+                                const existing = await EmailRecipient.select()
+                                    .where('emailId', upToDate.id)
+                                    .where('email', item.email)
+                                    .fetch();
+
+                                for (const other of existing) {
+                                    const merged = mergeReplacementsIfEqual(other.replacements, item.replacements);
+                                    if (merged !== false) {
+                                        console.log('Found duplicate email recipient', item.email, other.id);
+                                        duplicateOfRecipientId = other.id;
+
+                                        other.replacements = merged;
+                                        other.firstName = other.firstName || item.firstName;
+                                        other.lastName = other.lastName || item.lastName;
+                                        await other.save();
+
+                                        item.replacements = merged;
+
+                                        break;
+                                    }
+                                    else {
+                                        console.log('Could not merge duplicate email recipient', item.email, other.id, 'keeping both', other.replacements, item.replacements);
+                                    }
+                                }
+                            }
+
                             const recipient = new EmailRecipient();
                             recipient.emailType = upToDate.emailType;
                             recipient.objectId = item.objectId;
@@ -1065,6 +1086,7 @@ export class Email extends QueryableModel {
                             recipient.memberId = item.memberId ?? null;
                             recipient.userId = item.userId ?? null;
                             recipient.organizationId = upToDate.organizationId ?? null;
+                            recipient.duplicateOfRecipientId = duplicateOfRecipientId;
 
                             await recipient.save();
 
@@ -1072,7 +1094,11 @@ export class Email extends QueryableModel {
                                 membersSet.add(recipient.memberId);
                             }
 
-                            if (!recipient.email) {
+                            if (recipient.email) {
+                                emailsSet.add(recipient.email);
+                            }
+
+                            if (!recipient.email || duplicateOfRecipientId) {
                                 countWithoutEmail += 1;
                             }
                             else {
@@ -1146,13 +1172,21 @@ export class Email extends QueryableModel {
 
                         for (const item of response.results) {
                             const recipient = new EmailRecipient();
+                            recipient.emailType = upToDate.emailType;
                             recipient.emailId = upToDate.id;
+                            recipient.objectId = item.objectId;
                             recipient.email = item.email;
                             recipient.firstName = item.firstName;
                             recipient.lastName = item.lastName;
                             recipient.replacements = item.replacements;
+                            recipient.memberId = item.memberId ?? null;
+                            recipient.userId = item.userId ?? null;
+                            recipient.organizationId = upToDate.organizationId ?? null;
                             await recipient.save();
-                            return;
+
+                            if (recipient.email || recipient.userId) {
+                                return;
+                            }
                         }
 
                         request = null;
@@ -1175,12 +1209,13 @@ export class Email extends QueryableModel {
     async getPreviewStructure() {
         const emailRecipient = await EmailRecipient.select()
             .where('emailId', this.id)
+            .where('email', '!=', null)
             .first(false);
 
         let recipientRow: EmailRecipientStruct | undefined;
 
         if (emailRecipient) {
-            recipientRow = emailRecipient.getStructure();
+            recipientRow = await emailRecipient.getStructure();
         }
 
         if (!recipientRow) {
@@ -1188,18 +1223,124 @@ export class Email extends QueryableModel {
         }
 
         const virtualRecipient = recipientRow.getRecipient();
+        const organization = this.organizationId ? (await Organization.getByID(this.organizationId))! : null;
 
         await fillRecipientReplacements(virtualRecipient, {
-            organization: this.organizationId ? (await Organization.getByID(this.organizationId))! : null,
+            organization,
             from: this.getFromAddress(),
             replyTo: null,
+            forPreview: true,
+            forceRefresh: !this.sentAt,
         });
-
         recipientRow.replacements = virtualRecipient.replacements;
 
+        let user: UserStruct | null = null;
+        if (this.userId) {
+            const u = await User.getByID(this.userId);
+            if (u) {
+                user = u.getStructure();
+            }
+        }
+
+        let organizationStruct: BaseOrganization | null = null;
+        if (organization) {
+            organizationStruct = organization.getBaseStructure();
+        }
+
         return EmailPreview.create({
             ...this,
+            user,
+            organization: organizationStruct,
             exampleRecipient: recipientRow,
         });
     }
+
+    async getStructureForUser(user: User, memberIds: string[]) {
+        const emailRecipients = await EmailRecipient.select()
+            .where('emailId', this.id)
+            .where('memberId', memberIds)
+            .fetch();
+        const organization = this.organizationId ? (await Organization.getByID(this.organizationId))! : null;
+
+        const recipientsMap: Map<string, EmailRecipient> = new Map();
+        for (const memberId of memberIds) {
+            const preferred = emailRecipients.find(e => e.memberId === memberId && (e.userId === user.id || e.email === user.email));
+            if (preferred) {
+                recipientsMap.set(preferred.duplicateOfRecipientId ?? preferred.id, preferred);
+                continue;
+            }
+
+            const byMember = emailRecipients.find(e => e.memberId === memberId && e.userId === null && e.email === null);
+            if (byMember) {
+                recipientsMap.set(byMember.duplicateOfRecipientId ?? byMember.id, byMember);
+                continue;
+            }
+            const anyData = emailRecipients.find(e => e.memberId === memberId);
+            if (anyData) {
+                recipientsMap.set(anyData.duplicateOfRecipientId ?? anyData.id, anyData);
+                continue;
+            }
+        }
+
+        // Remove duplicates that are marked as the same recipient
+        const cleanedRecipients: EmailRecipient[] = [...recipientsMap.values()];
+        const structures = await EmailRecipient.getStructures(cleanedRecipients);
+
+        const virtualRecipients = structures.map((struct) => {
+            const recipient = struct.getRecipient();
+
+            return {
+                struct,
+                recipient,
+            };
+        });
+        for (const { struct, recipient } of virtualRecipients) {
+            if (!(struct.userId === user.id || struct.email === user.email) && !((struct.userId === null && struct.email === null))) {
+                stripSensitiveRecipientReplacements(recipient, {
+                    organization,
+                    willFill: true,
+                });
+            }
+
+            recipient.email = user.email;
+            recipient.userId = user.id;
+
+            // We always refresh the data when we display it on the web (so everything is up to date)
+            await fillRecipientReplacements(recipient, {
+                organization,
+                from: this.getFromAddress(),
+                replyTo: null,
+                forPreview: false,
+                forceRefresh: true,
+            });
+            stripRecipientReplacementsForWebDisplay(recipient, {
+                organization,
+            });
+            struct.replacements = recipient.replacements;
+        }
+
+        let organizationStruct: BaseOrganization | null = null;
+        if (organization) {
+            organizationStruct = organization.getBaseStructure();
+        }
+
+        return EmailWithRecipients.create({
+            ...this,
+            organization: organizationStruct,
+            recipients: structures,
+
+            // Remove private-like data
+            softBouncesCount: 0,
+            failedCount: 0,
+            emailErrors: null,
+            recipientsErrors: null,
+            succeededCount: 1,
+            emailRecipientsCount: 1,
+            hardBouncesCount: 0,
+            spamComplaintsCount: 0,
+            recipientFilter: EmailRecipientFilter.create({}),
+            membersCount: 1,
+            otherRecipientsCount: 0,
+        });
+    }
 }

package/src/models/EmailRecipient.ts

@@ -1,10 +1,12 @@
 import { column } from '@simonbackx/simple-database';
-import { EmailRecipient as EmailRecipientStruct, Recipient, Replacement } from '@stamhoofd/structures';
+import { EmailRecipient as EmailRecipientStruct, Recipient, Replacement, TinyMember } from '@stamhoofd/structures';
 import { v4 as uuidv4 } from 'uuid';
 
 import { ArrayDecoder } from '@simonbackx/simple-encoding';
 import { QueryableModel } from '@stamhoofd/sql';
 import { SimpleErrors } from '@simonbackx/simple-errors';
+import { Formatter } from '@stamhoofd/utility';
+import { Member } from './Member';
 
 export class EmailRecipient extends QueryableModel {
     static table = 'email_recipients';
@@ -70,6 +72,12 @@ export class EmailRecipient extends QueryableModel {
     @column({ type: 'string', nullable: true })
     userId: string | null = null;
 
+    /**
+     * Won't get send if it is a duplicate of another email recipient.
+     */
+    @column({ type: 'string', nullable: true })
+    duplicateOfRecipientId: string | null = null;
+
     /**
      * Set when the email was send, but we received a hard bounce for this specific email
      * Contains the full output we received in the bounce
@@ -134,11 +142,32 @@ export class EmailRecipient extends QueryableModel {
     })
     updatedAt: Date;
 
-    getStructure() {
+    async getStructure() {
+        return (await EmailRecipient.getStructures([this]))[0];
+    }
+
+    getStructureWithoutRelations() {
         return EmailRecipientStruct.create(this);
     }
 
+    static async getStructures(models: EmailRecipient[]) {
+        const memberIds = Formatter.uniqueArray(models.map(m => m.memberId).filter(m => m) as string[]);
+        const members = await Member.getByIDs(...memberIds);
+        return models.map((m) => {
+            const struct = EmailRecipientStruct.create(m);
+
+            if (m.memberId) {
+                const member = members.find(me => me.id === m.memberId);
+                if (member) {
+                    struct.member = TinyMember.create(member);
+                }
+            }
+
+            return struct;
+        });
+    }
+
     getRecipient() {
-        return this.getStructure().getRecipient();
+        return this.getStructureWithoutRelations().getRecipient();
     }
 }

package/src/models/Event.ts

@@ -87,6 +87,8 @@ export class Event extends QueryableModel {
         group.settings.requireGroupIds = this.meta.groups?.map(g => g.id) ?? [];
         group.settings.startDate = this.startDate;
         group.settings.endDate = this.endDate;
+        group.settings.minAge = this.meta.minAge ?? null;
+        group.settings.maxAge = this.meta.maxAge ?? null;
 
         if (group.type === GroupType.EventRegistration) {
             // Don't change the name of the waiting list

package/src/models/Member.ts

@@ -315,7 +315,7 @@ export class Member extends QueryableModel {
     /**
      * Fetch all members with their corresponding (valid) registrations or waiting lists and payments
      */
-    static async getMemberIdsWithRegistrationForUser(user: User): Promise<string[]> {
+    static async getMemberIdsForUser(user: User): Promise<string[]> {
         const query = SQL
             .select('id')
             .from(Member.table)
@@ -332,14 +332,14 @@ export class Member extends QueryableModel {
             );
 
         const data = await query.fetch();
-        return data.map(r => r.members.id as string);
+        return Formatter.uniqueArray(data.map(r => r.members.id as string));
     }
 
     /**
      * Fetch all members with their corresponding (valid) registrations or waiting lists and payments
      */
     static async getMembersWithRegistrationForUser(user: User): Promise<MemberWithRegistrations[]> {
-        return this.getBlobByIds(...(await this.getMemberIdsWithRegistrationForUser(user)));
+        return this.getBlobByIds(...(await this.getMemberIdsForUser(user)));
     }
 
     static getRegistrationWithTinyMemberStructure(registration: RegistrationWithMember & { group: import('./Group').Group }): RegistrationWithTinyMember {

package/src/models/Platform.ts

@@ -25,6 +25,9 @@ export class Platform extends QueryableModel {
     @column({ type: 'string', nullable: true })
     previousPeriodId: string | null = null;
 
+    @column({ type: 'string', nullable: true })
+    nextPeriodId: string | null = null;
+
     @column({ type: 'string', nullable: true })
     membershipOrganizationId: string | null = null;
 
@@ -56,6 +59,7 @@ export class Platform extends QueryableModel {
     async setPreviousPeriodId() {
         const period = await RegistrationPeriod.getByID(this.periodId);
         this.previousPeriodId = period?.previousPeriodId ?? null;
+        this.nextPeriodId = period?.nextPeriodId ?? null;
     }
 
     static async getSharedPrivateStruct(): Promise<PlatformStruct & { privateConfig: PlatformPrivateConfig }> {

package/src/models/RegistrationPeriod.ts

@@ -13,9 +13,15 @@ export class RegistrationPeriod extends QueryableModel {
     })
     id!: string;
 
+    @column({ type: 'string', nullable: true })
+    customName: string | null = null;
+
     @column({ type: 'string', nullable: true })
     previousPeriodId: string | null = null;
 
+    @column({ type: 'string', nullable: true })
+    nextPeriodId: string | null = null;
+
     @column({ type: 'string', nullable: true })
     organizationId: string | null = null;
 
@@ -74,12 +80,25 @@ export class RegistrationPeriod extends QueryableModel {
         return RegistrationPeriod.fromRow(result[this.table]) ?? null;
     }
 
-    async setPreviousPeriodId() {
-        const allPeriods = await RegistrationPeriod.where({ organizationId: this.organizationId });
+    async updatePreviousNextPeriods() {
+        return await RegistrationPeriod.updatePreviousNextPeriods(this.organizationId, this);
+    }
+
+    static async updatePreviousNextPeriods(organizationId: string | null, existingReference?: RegistrationPeriod) {
+        const allPeriods = await RegistrationPeriod.select().where('organizationId', organizationId).fetch();
 
         // Include self if not yet in database
-        if (!this.existsInDatabase) {
-            allPeriods.push(this);
+        if (existingReference) {
+            if (!existingReference.existsInDatabase) {
+                allPeriods.push(existingReference);
+            }
+            else {
+            // Replace self with this
+                const index = allPeriods.findIndex(p => p.id === existingReference.id);
+                if (index !== -1) {
+                    allPeriods[index] = existingReference;
+                }
+            }
         }
 
         // Sort by start date
@@ -89,12 +108,18 @@ export class RegistrationPeriod extends QueryableModel {
         let previousPeriod: RegistrationPeriod | null = null;
 
         for (const period of allPeriods) {
-            if (period.id === this.id) {
-                break;
+            period.previousPeriodId = previousPeriod?.id ?? null;
+            period.nextPeriodId = null;
+
+            if (previousPeriod) {
+                previousPeriod.nextPeriodId = period.id;
             }
             previousPeriod = period;
         }
 
-        this.previousPeriodId = previousPeriod?.id ?? null;
+        // Save all
+        for (const period of allPeriods) {
+            await period.save();
+        }
     }
 }

package/src/models/User.ts

@@ -228,13 +228,19 @@ export class User extends QueryableModel {
         await Database.update(query, [this.id, other.id]);
 
         // Update payments
-        const query2 = 'UPDATE payments SET userId = ? WHERE userId = ?';
+        const query2 = 'UPDATE payments SET payingUserId = ? WHERE payingUserId = ?';
         await Database.update(query2, [this.id, other.id]);
 
         // Update orders
         const query3 = 'UPDATE webshop_orders SET userId = ? WHERE userId = ?';
         await Database.update(query3, [this.id, other.id]);
 
+        // Update emails
+        await Database.update('UPDATE emails SET userId = ? WHERE userId = ?', [this.id, other.id]);
+
+        // Update email recipients
+        await Database.update('UPDATE email_recipients SET userId = ? WHERE userId = ?', [this.id, other.id]);
+
         await other.delete();
     }