@stamhoofd/models 2.63.0 → 2.65.0

package/src/models/Email.ts

@@ -1,5 +1,5 @@
-import { column, Model } from '@simonbackx/simple-database';
-import { EditorSmartButton, EditorSmartVariable, EmailAttachment, EmailPreview, EmailRecipientFilter, EmailRecipientFilterType, EmailRecipientsStatus, EmailRecipient as EmailRecipientStruct, EmailStatus, Email as EmailStruct, getExampleRecipient, LimitedFilteredRequest, PaginatedResponse, Recipient, SortItemDirection, StamhoofdFilter } from '@stamhoofd/structures';
+import { column } from '@simonbackx/simple-database';
+import { EmailAttachment, EmailPreview, EmailRecipientFilter, EmailRecipientFilterType, EmailRecipientsStatus, EmailRecipient as EmailRecipientStruct, EmailStatus, Email as EmailStruct, EmailTemplateType, getExampleRecipient, LimitedFilteredRequest, PaginatedResponse, Recipient, Replacement, SortItemDirection, StamhoofdFilter } from '@stamhoofd/structures';
 import { v4 as uuidv4 } from 'uuid';
 
 import { AnyDecoder, ArrayDecoder } from '@simonbackx/simple-encoding';
@@ -7,13 +7,14 @@ import { SimpleError } from '@simonbackx/simple-errors';
 import { I18n } from '@stamhoofd/backend-i18n';
 import { Email as EmailClass } from '@stamhoofd/email';
 import { QueueHandler } from '@stamhoofd/queues';
-import { SQL, SQLWhereSign } from '@stamhoofd/sql';
+import { QueryableModel, SQL, SQLWhereSign } from '@stamhoofd/sql';
 import { Formatter } from '@stamhoofd/utility';
-import { getEmailBuilder } from '../helpers/EmailBuilder';
+import { fillRecipientReplacements, getEmailBuilder } from '../helpers/EmailBuilder';
 import { EmailRecipient } from './EmailRecipient';
 import { Organization } from './Organization';
+import { EmailTemplate } from './EmailTemplate';
 
-export class Email extends Model {
+export class Email extends QueryableModel {
     static table = 'emails';
 
     @column({
@@ -32,6 +33,13 @@ export class Email extends Model {
     @column({ type: 'json', decoder: EmailRecipientFilter })
     recipientFilter: EmailRecipientFilter = EmailRecipientFilter.create({});
 
+    /**
+     * Helper to prevent sending too many emails to the same person.
+     * Allows for filtering on objects that didn't receive a specific email yet
+     */
+    @column({ type: 'string', nullable: true })
+    emailType: string | null = null;
+
     @column({ type: 'string', nullable: true })
     subject: string | null;
 
@@ -204,7 +212,29 @@ export class Email extends Model {
         return '"' + cleanedName + '" <' + address + '>';
     }
 
-    async send() {
+    async setFromTemplate(type: EmailTemplateType) {
+        // Most specific template: for specific group
+        let templates = (await EmailTemplate.where({ type, organizationId: this.organizationId, groupId: null, webshopId: null }));
+
+        // Then default
+        if (templates.length === 0 && this.organizationId) {
+            templates = (await EmailTemplate.where({ type, organizationId: null, groupId: null, webshopId: null }));
+        }
+
+        if (templates.length === 0) {
+            // No default
+            return false;
+        }
+        const defaultTemplate = templates[0];
+        this.html = defaultTemplate.html;
+        this.text = defaultTemplate.text;
+        this.subject = defaultTemplate.subject;
+        this.json = defaultTemplate.json;
+
+        return true;
+    }
+
+    async send(): Promise<Email | null> {
         this.throwIfNotReadyToSend();
         await this.save();
 
@@ -218,10 +248,27 @@ export class Email extends Model {
                     human: 'De e-mail die je probeert te versturen bestaat niet meer',
                 });
             }
-            if (upToDate.status === EmailStatus.Sent) {
+            if (upToDate.status === EmailStatus.Sent || upToDate.status === EmailStatus.Failed) {
                 // Already done
                 // In other cases -> queue has stopped and we can retry
-                return;
+                return upToDate;
+            }
+
+            if (upToDate.status === EmailStatus.Sending) {
+                // This is a retry.
+                if (upToDate.emailType) {
+                    // Not eligible for retry
+                    upToDate.status = EmailStatus.Failed;
+                    await upToDate.save();
+                    return upToDate;
+                }
+                if (upToDate.createdAt < new Date(new Date().getTime() - 1000 * 60 * 60 * 24 * 2)) {
+                    // Too long
+                    console.error('Email has been sending for too long. Marking as failed...', upToDate.id);
+                    upToDate.status = EmailStatus.Failed;
+                    await upToDate.save();
+                    return upToDate;
+                }
             }
             const organization = upToDate.organizationId ? await Organization.getByID(upToDate.organizationId) : null;
             upToDate.throwIfNotReadyToSend();
@@ -345,9 +392,7 @@ export class Email extends Model {
                     });
                     sendingPromises.push(promise);
 
-                    const virtualRecipient = Recipient.create({
-                        ...recipient,
-                    });
+                    const virtualRecipient = recipient.getRecipient();
 
                     const callback = async (error: Error | null) => {
                         if (error === null) {
@@ -378,7 +423,7 @@ export class Email extends Model {
                         replyTo,
                         subject: upToDate.subject!,
                         html: upToDate.html!,
-                        type: 'broadcast',
+                        type: upToDate.emailType ? 'transactional' : 'broadcast',
                         attachments,
                         callback(error: Error | null) {
                             callback(error).catch(console.error);
@@ -391,11 +436,18 @@ export class Email extends Model {
                 await Promise.all(sendingPromises);
             }
 
-            console.log('Finished sending email', upToDate.id);
+            if (upToDate.recipientCount === 0 && upToDate.userId === null) {
+                // We only delete automated emails (email type) if they have no recipients
+                console.log('No recipients found for email ', upToDate.id, ' deleting...');
+                await upToDate.delete();
+                return null;
+            }
 
+            console.log('Finished sending email', upToDate.id);
             // Mark email as sent
             upToDate.status = EmailStatus.Sent;
             await upToDate.save();
+            return upToDate;
         });
     }
 
@@ -498,18 +550,19 @@ export class Email extends Model {
                     let request: LimitedFilteredRequest | null = new LimitedFilteredRequest({
                         filter: subfilter.filter,
                         sort: [{ key: 'id', order: SortItemDirection.ASC }],
-                        limit: 1000,
+                        limit: 100,
                         search: subfilter.search,
                     });
 
                     while (request) {
-                        console.log('Loading email page', subfilter.type, request);
                         const response = await loader.fetch(request, subfilter.subfilter);
 
                         count += response.results.length;
 
                         for (const item of response.results) {
                             const recipient = new EmailRecipient();
+                            recipient.emailType = upToDate.emailType;
+                            recipient.objectId = item.objectId;
                             recipient.emailId = upToDate.id;
                             recipient.email = item.email;
                             recipient.firstName = item.firstName;
@@ -523,10 +576,6 @@ export class Email extends Model {
                     }
                 }
 
-                // todo: loop all members that match the filter in batches of 1000
-                // create a new row for every member + calculate the replacement values
-                // todo: do intermediate checks on whether the email was deleted, and stop processing if needed
-
                 upToDate.recipientsStatus = EmailRecipientsStatus.Created;
                 upToDate.recipientCount = count;
                 await upToDate.save();
@@ -578,7 +627,6 @@ export class Email extends Model {
                     });
 
                     while (request) {
-                        console.log('Loading email page', subfilter.type, request);
                         const response = await loader.fetch(request, subfilter.subfilter);
 
                         // Note: it is possible that a result in the database doesn't return a recipient (in memory filtering)
@@ -613,32 +661,32 @@ export class Email extends Model {
     }
 
     async getPreviewStructure() {
-        const recipient = await SQL.select()
-            .from(SQL.table(EmailRecipient.table))
-            .where(SQL.column('emailId'), this.id)
+        const emailRecipient = await EmailRecipient.select()
+            .where('emailId', this.id)
             .first(false);
 
         let recipientRow: EmailRecipientStruct | undefined;
 
-        if (recipient) {
-            const emailRecipient = EmailRecipient.fromRow(recipient[EmailRecipient.table]);
-            if (emailRecipient) {
-                recipientRow = emailRecipient.getStructure();
-            }
+        if (emailRecipient) {
+            recipientRow = emailRecipient.getStructure();
         }
 
         if (!recipientRow) {
             recipientRow = getExampleRecipient();
         }
 
-        const smartVariables = recipientRow ? EditorSmartVariable.forRecipient(recipientRow) : [];
-        const smartButtons = recipientRow ? EditorSmartButton.forRecipient(recipientRow) : [];
+        const virtualRecipient = recipientRow.getRecipient();
+
+        await fillRecipientReplacements(virtualRecipient, {
+            organization: this.organizationId ? (await Organization.getByID(this.organizationId))! : null,
+            fromAddress: this.fromAddress,
+        });
+
+        recipientRow.replacements = virtualRecipient.replacements;
 
         return EmailPreview.create({
             ...this,
             exampleRecipient: recipientRow,
-            smartVariables,
-            smartButtons,
         });
     }
 }

package/src/models/EmailRecipient.ts

@@ -1,11 +1,11 @@
-import { column, Model, SQLResultNamespacedRow } from '@simonbackx/simple-database';
-import { EmailRecipient as EmailRecipientStruct, Replacement } from '@stamhoofd/structures';
+import { column } from '@simonbackx/simple-database';
+import { EmailRecipient as EmailRecipientStruct, Recipient, Replacement } from '@stamhoofd/structures';
 import { v4 as uuidv4 } from 'uuid';
 
 import { ArrayDecoder } from '@simonbackx/simple-encoding';
-import { SQL, SQLSelect } from '@stamhoofd/sql';
+import { QueryableModel } from '@stamhoofd/sql';
 
-export class EmailRecipient extends Model {
+export class EmailRecipient extends QueryableModel {
     static table = 'email_recipients';
 
     @column({
@@ -18,6 +18,20 @@ export class EmailRecipient extends Model {
     @column({ type: 'string' })
     emailId: string;
 
+    /**
+     * Helper to prevent sending too many emails to the same person.
+     * Allows for filtering on objects that didn't receive a specific email yet
+     */
+    @column({ type: 'string', nullable: true })
+    objectId: string | null = null;
+
+    /**
+     * Helper to prevent sending too many emails to the same person.
+     * Allows for filtering on objects that didn't receive a specific email yet
+     */
+    @column({ type: 'string', nullable: true })
+    emailType: string | null = null;
+
     @column({ type: 'string', nullable: true })
     firstName: string | null = null;
 
@@ -80,21 +94,7 @@ export class EmailRecipient extends Model {
         return EmailRecipientStruct.create(this);
     }
 
-    /**
-     * Experimental: needs to move to library
-     */
-    static select() {
-        const transformer = (row: SQLResultNamespacedRow): EmailRecipient => {
-            const d = (this as typeof EmailRecipient & typeof Model).fromRow(row[this.table] as any) as EmailRecipient | undefined;
-
-            if (!d) {
-                throw new Error('EmailTemplate not found');
-            }
-
-            return d;
-        };
-
-        const select = new SQLSelect(transformer, SQL.wildcard());
-        return select.from(SQL.table(this.table));
+    getRecipient() {
+        return this.getStructure().getRecipient();
     }
 }

package/src/models/EmailTemplate.ts

@@ -1,13 +1,13 @@
-import { column, Model, SQLResultNamespacedRow } from '@simonbackx/simple-database';
+import { column } from '@simonbackx/simple-database';
 import { AnyDecoder } from '@simonbackx/simple-encoding';
-import { SQL, SQLSelect } from '@stamhoofd/sql';
+import { QueryableModel } from '@stamhoofd/sql';
 import { EmailTemplate as EmailTemplateStruct, EmailTemplateType } from '@stamhoofd/structures';
 import { v4 as uuidv4 } from 'uuid';
 
 /**
  * Holds the challenges for a given email. User should not exist, since that would allow user enumeration attacks
  */
-export class EmailTemplate extends Model {
+export class EmailTemplate extends QueryableModel {
     static table = 'email_templates';
 
     @column({
@@ -65,22 +65,4 @@ export class EmailTemplate extends Model {
     getStructure() {
         return EmailTemplateStruct.create(this);
     }
-
-    /**
-     * Experimental: needs to move to library
-     */
-    static select() {
-        const transformer = (row: SQLResultNamespacedRow): EmailTemplate => {
-            const d = (this as typeof EmailTemplate & typeof Model).fromRow(row[this.table] as any) as EmailTemplate | undefined;
-
-            if (!d) {
-                throw new Error('EmailTemplate not found');
-            }
-
-            return d;
-        };
-
-        const select = new SQLSelect(transformer, SQL.wildcard());
-        return select.from(SQL.table(this.table));
-    }
 }

package/src/models/EmailVerificationCode.ts

@@ -1,6 +1,7 @@
-import { column, Model } from '@simonbackx/simple-database';
+import { column } from '@simonbackx/simple-database';
 import { SimpleError } from '@simonbackx/simple-errors';
 import { I18n } from '@stamhoofd/backend-i18n';
+import { QueryableModel } from '@stamhoofd/sql';
 import { EmailTemplateType, Recipient, Replacement } from '@stamhoofd/structures';
 import basex from 'base-x';
 import crypto from 'crypto';
@@ -38,7 +39,7 @@ async function randomInt(max: number): Promise<number> {
 /**
  * Holds the verificationCodes for a given email (not a user, since a user can switch email addresses and might avoid verification that way)
  */
-export class EmailVerificationCode extends Model {
+export class EmailVerificationCode extends QueryableModel {
     static table = 'email_verification_codes';
 
     @column({

package/src/models/Event.ts

@@ -1,9 +1,10 @@
-import { column, Model } from '@simonbackx/simple-database';
+import { column } from '@simonbackx/simple-database';
+import { QueryableModel } from '@stamhoofd/sql';
 import { EventMeta, Event as EventStruct, GroupType } from '@stamhoofd/structures';
 import { v4 as uuidv4 } from 'uuid';
 import { Group } from './Group';
 
-export class Event extends Model {
+export class Event extends QueryableModel {
     static table = 'events';
 
     @column({ primary: true, type: 'string', beforeSave(value) {

package/src/models/Group.ts

@@ -1,10 +1,10 @@
-import { column, Database, ManyToOneRelation, Model, OneToManyRelation, SQLResultNamespacedRow } from '@simonbackx/simple-database';
+import { column, Database, ManyToOneRelation, OneToManyRelation } from '@simonbackx/simple-database';
 import { GroupCategory, GroupPrivateSettings, GroupSettings, GroupStatus, Group as GroupStruct, GroupType, StockReservation } from '@stamhoofd/structures';
 import { v4 as uuidv4 } from 'uuid';
 
 import { ArrayDecoder } from '@simonbackx/simple-encoding';
 import { QueueHandler } from '@stamhoofd/queues';
-import { SQL, SQLSelect } from '@stamhoofd/sql';
+import { QueryableModel } from '@stamhoofd/sql';
 import { Formatter } from '@stamhoofd/utility';
 import { Member, MemberWithRegistrations, OrganizationRegistrationPeriod, Payment, Registration, User } from './';
 
@@ -21,7 +21,7 @@ if (Registration === undefined) {
     throw new Error('Import Registration is undefined');
 }
 
-export class Group extends Model {
+export class Group extends QueryableModel {
     static table = 'groups';
 
     @column({
@@ -100,10 +100,9 @@ export class Group extends Model {
     static async getAll(organizationId: string, periodId: string | null, active = true) {
         const w: any = periodId ? { periodId } : {};
         if (active) {
-            // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
             return await Group.where({ organizationId, deletedAt: null, ...w });
         }
-        // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+
         return await Group.where({ organizationId, ...w });
     }
 
@@ -299,24 +298,6 @@ export class Group extends Model {
     static async freeStockReservations(groupId: string, reservations: StockReservation[]) {
         return await this.applyStockReservations(groupId, reservations, true);
     }
-
-    /**
-     * Experimental: needs to move to library
-     */
-    static select() {
-        const transformer = (row: SQLResultNamespacedRow): Group => {
-            const d = (this as typeof Group & typeof Model).fromRow(row[this.table] as any) as Group | undefined;
-
-            if (!d) {
-                throw new Error('EmailTemplate not found');
-            }
-
-            return d;
-        };
-
-        const select = new SQLSelect(transformer, SQL.wildcard());
-        return select.from(SQL.table(this.table));
-    }
 }
 
 Registration.group = new ManyToOneRelation(Group, 'group');

package/src/models/Image.ts

@@ -1,12 +1,13 @@
-import { column, Model } from '@simonbackx/simple-database';
+import { column } from '@simonbackx/simple-database';
 import { ArrayDecoder } from '@simonbackx/simple-encoding';
 import { SimpleError } from '@simonbackx/simple-errors';
+import { QueryableModel } from '@stamhoofd/sql';
 import { File, Resolution, ResolutionRequest } from '@stamhoofd/structures';
 import AWS from 'aws-sdk';
 import sharp from 'sharp';
 import { v4 as uuidv4 } from 'uuid';
 
-export class Image extends Model {
+export class Image extends QueryableModel {
     static table = 'images';
 
     @column({ primary: true, type: 'string', beforeSave(value) {

package/src/models/Member.ts

@@ -1,6 +1,6 @@
-import { column, Database, ManyToManyRelation, ManyToOneRelation, Model, OneToManyRelation } from '@simonbackx/simple-database';
-import { SQL } from '@stamhoofd/sql';
-import { MemberDetails, MemberWithRegistrationsBlob, RegistrationWithMember as RegistrationWithMemberStruct, TinyMember } from '@stamhoofd/structures';
+import { column, Database, ManyToManyRelation, ManyToOneRelation, OneToManyRelation } from '@simonbackx/simple-database';
+import { QueryableModel, SQL } from '@stamhoofd/sql';
+import { MemberDetails, RegistrationWithMember as RegistrationWithMemberStruct, TinyMember } from '@stamhoofd/structures';
 import { Formatter } from '@stamhoofd/utility';
 import { v4 as uuidv4 } from 'uuid';
 
@@ -13,7 +13,7 @@ export type MemberWithRegistrations = Member & {
 // Defined here to prevent cycles
 export type RegistrationWithMember = Registration & { member: Member };
 
-export class Member extends Model {
+export class Member extends QueryableModel {
     static table = 'members';
 
     // #region Columns
@@ -63,7 +63,8 @@ export class Member extends Model {
     details: MemberDetails;
 
     /**
-     * Not yet paid balance
+     * @deprecated
+     * Unreliable since a member can have outstanding balance to multiple organizations now
      */
     @column({ type: 'integer' })
     outstandingBalance = 0;
@@ -103,44 +104,6 @@ export class Member extends Model {
         return (await this.getBlobByIds(id))[0] ?? null;
     }
 
-    /**
-     * Update the outstanding balance of multiple members in one go (or all members)
-     */
-    static async updateOutstandingBalance(memberIds: string[] | 'all') {
-        if (memberIds !== 'all' && memberIds.length == 0) {
-            return;
-        }
-
-        const params: any[] = [];
-        let firstWhere = '';
-        let secondWhere = '';
-
-        if (memberIds !== 'all') {
-            firstWhere = ` AND memberId IN (?)`;
-            params.push(memberIds);
-
-            secondWhere = `WHERE members.id IN (?)`;
-            params.push(memberIds);
-        }
-
-        const query = `UPDATE
-            members
-            LEFT JOIN (
-                SELECT
-                    memberId,
-                    sum(unitPrice * amount) - sum(pricePaid) AS outstandingBalance
-                FROM
-                    balance_items
-                WHERE status != 'Hidden'${firstWhere}
-                GROUP BY
-                    memberId
-            ) i ON i.memberId = members.id 
-        SET members.outstandingBalance = COALESCE(i.outstandingBalance, 0)
-        ${secondWhere}`;
-
-        await Database.update(query, params);
-    }
-
     /**
      * Fetch all registrations with members with their corresponding (valid) registrations
      */
@@ -377,15 +340,6 @@ export class Member extends Model {
         return this.getBlobByIds(...(await this.getMemberIdsWithRegistrationForUser(user)));
     }
 
-    getStructureWithRegistrations(this: MemberWithRegistrations, forOrganization: null | boolean = null) {
-        return MemberWithRegistrationsBlob.create({
-            ...this,
-            registrations: this.registrations.map(r => r.getStructure()),
-            details: this.details,
-            users: this.users.map(u => u.getStructure()),
-        });
-    }
-
     static getRegistrationWithMemberStructure(registration: RegistrationWithMember & { group: import('./Group').Group }): RegistrationWithMemberStruct {
         return RegistrationWithMemberStruct.create({
             ...registration.getStructure(),