@stamhoofd/backend 2.92.0 → 2.94.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@stamhoofd/backend",
-    "version": "2.92.0",
+    "version": "2.94.0",
     "main": "./dist/index.js",
     "exports": {
         ".": {
@@ -45,14 +45,14 @@
         "@simonbackx/simple-encoding": "2.22.0",
         "@simonbackx/simple-endpoints": "1.20.1",
         "@simonbackx/simple-logging": "^1.0.1",
-        "@stamhoofd/backend-i18n": "2.92.0",
-        "@stamhoofd/backend-middleware": "2.92.0",
-        "@stamhoofd/email": "2.92.0",
-        "@stamhoofd/models": "2.92.0",
-        "@stamhoofd/queues": "2.92.0",
-        "@stamhoofd/sql": "2.92.0",
-        "@stamhoofd/structures": "2.92.0",
-        "@stamhoofd/utility": "2.92.0",
+        "@stamhoofd/backend-i18n": "2.94.0",
+        "@stamhoofd/backend-middleware": "2.94.0",
+        "@stamhoofd/email": "2.94.0",
+        "@stamhoofd/models": "2.94.0",
+        "@stamhoofd/queues": "2.94.0",
+        "@stamhoofd/sql": "2.94.0",
+        "@stamhoofd/structures": "2.94.0",
+        "@stamhoofd/utility": "2.94.0",
         "archiver": "^7.0.1",
         "axios": "^1.8.2",
         "cookie": "^0.7.0",
@@ -70,5 +70,5 @@
     "publishConfig": {
         "access": "public"
     },
-    "gitHead": "43d1edfd8061dada1d418a02691fe5fb158aca6a"
+    "gitHead": "7a1a0be03249d69ad0664ee85d446c7b596768f0"
 }

package/src/audit-logs/EmailLogger.ts

@@ -58,8 +58,8 @@ export const EmailLogger = new ModelLogger(Email, {
                 type: AuditLogReplacementType.Email,
             })],
             ['c', AuditLogReplacement.create({
-                value: Formatter.integer(model.recipientCount ?? 0),
-                count: model.recipientCount ?? 0,
+                value: Formatter.integer(model.emailRecipientsCount ?? 0),
+                count: model.emailRecipientsCount ?? 0,
             })],
         ]);
         if (options.data.recipient) {

package/src/crons/amazon-ses.ts

@@ -6,9 +6,10 @@ import {
 } from '@aws-sdk/client-sqs';
 import { registerCron } from '@stamhoofd/crons';
 import { Email, EmailAddress } from '@stamhoofd/email';
-import { AuditLog, Organization } from '@stamhoofd/models';
+import { AuditLog, EmailRecipient, Organization } from '@stamhoofd/models';
 import { AuditLogReplacement, AuditLogReplacementType, AuditLogSource, AuditLogType } from '@stamhoofd/structures';
 import { ForwardHandler } from '../helpers/ForwardHandler';
+import { Email as EmailModel } from '@stamhoofd/models';
 
 registerCron('checkComplaints', checkComplaints);
 registerCron('checkReplies', checkReplies);
@@ -52,13 +53,88 @@ async function saveLog({ email, organization, type, subType, subject, response,
     await log.save();
 }
 
+async function storeEmailStatus({ headers, type, message }: { headers: Record<string, string>; type: 'hard-bounce' | 'soft-bounce' | 'complaint'; message: string }) {
+    const emailId = headers['x-email-id'];
+    const recipientId = headers['x-email-recipient-id'];
+
+    if (emailId && recipientId) {
+        // check
+        const emailRecipient = await EmailRecipient.select()
+            .where('id', recipientId)
+            .where('emailId', emailId)
+            .first(false);
+
+        if (!emailRecipient) {
+            console.log('[AWS FORWARDING] Invalid email or recipient id in headers', headers);
+            return;
+        }
+
+        let isNew = true;
+
+        switch (type) {
+            case 'hard-bounce': {
+                if (emailRecipient.hardBounceError) {
+                    isNew = false;
+                }
+                emailRecipient.hardBounceError = message;
+                break;
+            }
+            case 'soft-bounce': {
+                if (emailRecipient.softBounceError) {
+                    isNew = false;
+                }
+                emailRecipient.softBounceError = message;
+                break;
+            }
+            case 'complaint': {
+                if (emailRecipient.spamComplaintError) {
+                    isNew = false;
+                }
+                emailRecipient.spamComplaintError = message;
+                break;
+            }
+        }
+
+        console.log('[AWS FORWARDING] Marking email recipient ' + recipientId + ' for email ' + emailId + ' as ' + type);
+        if (await emailRecipient.save()) {
+            if (isNew) {
+                await EmailModel.bumpNotificationCount(emailId, type);
+            }
+        }
+    }
+}
+
+function readHeaders(message: any) {
+    try {
+        const mail = message.mail;
+        const headers: Record<string, string> = {};
+        if (mail && typeof mail === 'object' && mail !== null && Array.isArray(mail.headers)) {
+            for (const header of mail.headers) {
+                if (header.name && header.value) {
+                    headers[(header.name as string).toLowerCase()] = header.value;
+                }
+            }
+        }
+        else {
+            console.log('[AWS] Missing mail headers', message);
+        }
+        return headers;
+    }
+    catch (e) {
+        console.log('[AWS] Failed to read headers', e, message);
+        return {};
+    }
+}
+
 async function handleBounce(message: any) {
     if (message.bounce) {
+        console.log('[AWS BOUNCES] Handling bounce message', message);
+        const headers = readHeaders(message);
+
         const b = message.bounce;
         // Block all receivers that generate a permanent bounce
         const type = b.bounceType;
         const subtype = b.bounceSubType;
-
         const source = message.mail.source;
 
         // try to find organization that is responsible for this e-mail address
@@ -80,6 +156,12 @@ async function handleBounce(message: any) {
                 emailAddress.hardBounce = true;
                 await emailAddress.save();
 
+                await storeEmailStatus({
+                    headers,
+                    type: 'hard-bounce',
+                    message: recipient.diagnosticCode || 'Permanent bounce',
+                });
+
                 await saveLog({
                     id: b.feedbackId,
                     email,
@@ -95,6 +177,13 @@ async function handleBounce(message: any) {
                 type === 'Transient'
             ) {
                 const organization: Organization | undefined = source ? await Organization.getByEmail(source) : undefined;
+
+                await storeEmailStatus({
+                    headers,
+                    type: 'soft-bounce',
+                    message: recipient.diagnosticCode || 'Soft bounce',
+                });
+
                 await saveLog({
                     id: b.feedbackId,
                     email,
@@ -116,6 +205,7 @@ async function handleBounce(message: any) {
 
 async function handleComplaint(message: any) {
     if (message.complaint) {
+        const headers = readHeaders(message);
         const b = message.complaint;
         const source = message.mail.source;
         const organization: Organization | undefined = source ? await Organization.getByEmail(source) : undefined;
@@ -129,10 +219,16 @@ async function handleComplaint(message: any) {
             await emailAddress.save();
 
             if (type !== 'not-spam') {
+                await storeEmailStatus({
+                    headers,
+                    type: 'complaint',
+                    message: recipient.diagnosticCode || type || 'Complaint',
+                });
+
                 if (type === 'virus' || type === 'fraud') {
                     await saveLog({
                         id: b.feedbackId,
-                        email: source,
+                        email,
                         organization,
                         type: AuditLogType.EmailAddressFraudComplaint,
                         subType: type || 'unknown',
@@ -144,7 +240,7 @@ async function handleComplaint(message: any) {
                 else {
                     await saveLog({
                         id: b.feedbackId,
-                        email: source,
+                        email,
                         organization,
                         type: AuditLogType.EmailAddressMarkedAsSpam,
                         subType: type || 'unknown',

package/src/crons/balance-emails.ts

@@ -198,7 +198,7 @@ async function sendTemplate({
 
     try {
         const upToDate = await ContextInstance.startForUser(systemUser, organization, async () => {
-            return await model.send();
+            return await model.queueForSending(true);
         });
 
         if (!upToDate) {

package/src/email-recipient-loaders/receivable-balances.ts

@@ -1,5 +1,5 @@
 import { CachedBalance, Email } from '@stamhoofd/models';
-import { BalanceItem as BalanceItemStruct, compileToInMemoryFilter, EmailRecipient, EmailRecipientFilterType, LimitedFilteredRequest, PaginatedResponse, receivableBalanceObjectContactInMemoryFilterCompilers, Replacement, StamhoofdFilter } from '@stamhoofd/structures';
+import { BalanceItem as BalanceItemStruct, compileToInMemoryFilter, EmailRecipient, EmailRecipientFilterType, LimitedFilteredRequest, PaginatedResponse, receivableBalanceObjectContactInMemoryFilterCompilers, ReceivableBalanceType, Replacement, StamhoofdFilter } from '@stamhoofd/structures';
 import { Formatter } from '@stamhoofd/utility';
 import { GetReceivableBalancesEndpoint } from '../endpoints/organization/dashboard/receivable-balances/GetReceivableBalancesEndpoint';
 
@@ -22,6 +22,8 @@ async function fetch(query: LimitedFilteredRequest, subfilter: StamhoofdFilter |
             for (const email of contact.emails) {
                 const recipient = EmailRecipient.create({
                     objectId: balance.id, // Note: not set member, user or organization id here - should be the queryable balance id
+                    userId: balance.objectType === ReceivableBalanceType.user ? balance.object.id : null,
+                    memberId: balance.objectType === ReceivableBalanceType.member ? balance.object.id : null,
                     firstName: contact.firstName,
                     lastName: contact.lastName,
                     email,

package/src/endpoints/global/email/CreateEmailEndpoint.ts

@@ -120,8 +120,32 @@ export class CreateEmailEndpoint extends Endpoint<Params, Query, Body, ResponseB
         await model.buildExampleRecipient();
         model.updateCount();
 
-        if (request.body.status === EmailStatus.Sending || request.body.status === EmailStatus.Sent) {
-            model.send().catch(console.error);
+        if (request.body.status === EmailStatus.Sending || request.body.status === EmailStatus.Sent || request.body.status === EmailStatus.Queued) {
+            if (!await Context.auth.canSendEmail(model)) {
+                throw Context.auth.error({
+                    message: 'Cannot send emails from this sender',
+                    human: $t('1b509614-30b0-484c-af72-57d4bc9ea788'),
+                });
+            }
+            await model.queueForSending();
+        }
+
+        // Delete open drafts with the same content, from the same user
+        const duplicates = await Email.select()
+            .where('userId', user.id)
+            .where('organizationId', model.organizationId)
+            .where('status', EmailStatus.Draft)
+            .where('subject', model.subject)
+            .where('html', model.html)
+            .where('text', model.text)
+            .where('deletedAt', null)
+            .whereNot('id', model.id)
+            .limit(100)
+            .fetch();
+
+        for (const duplicate of duplicates) {
+            duplicate.deletedAt = new Date();
+            await duplicate.save();
         }
 
         return new Response(await model.getPreviewStructure());

package/src/endpoints/global/email/GetAdminEmailsEndpoint.ts

@@ -104,7 +104,8 @@ export class GetAdminEmailsEndpoint extends Endpoint<Params, Query, Body, Respon
             };
         }
 
-        const query = Email.select();
+        const query = Email.select()
+            .where('deletedAt', null);
 
         if (scopeFilter) {
             query.where(await compileToSQLFilter(scopeFilter, filterCompilers));
@@ -138,8 +139,6 @@ export class GetAdminEmailsEndpoint extends Endpoint<Params, Query, Body, Respon
             query.limit(q.limit);
         }
 
-        console.log('Building query for GetAdminEmailsEndpoint', query.getSQL());
-
         return query;
     }
 

package/src/endpoints/global/email/PatchEmailEndpoint.ts

@@ -1,6 +1,6 @@
 import { DecodedRequest, Endpoint, Request, Response } from '@simonbackx/simple-endpoints';
 import { Email, Platform } from '@stamhoofd/models';
-import { EmailPreview, EmailStatus, Email as EmailStruct, PermissionLevel } from '@stamhoofd/structures';
+import { EmailPreview, EmailRecipientsStatus, EmailStatus, Email as EmailStruct, PermissionLevel } from '@stamhoofd/structures';
 
 import { AutoEncoderPatchType, Decoder, patchObject } from '@simonbackx/simple-encoding';
 import { SimpleError } from '@simonbackx/simple-errors';
@@ -50,15 +50,6 @@ export class PatchEmailEndpoint extends Endpoint<Params, Query, Body, ResponseBo
             throw Context.auth.error();
         }
 
-        if (model.status !== EmailStatus.Draft) {
-            throw new SimpleError({
-                code: 'not_draft',
-                human: 'Email is not a draft',
-                message: $t(`298b5a46-2899-4aa1-89df-9b634c20806b`),
-                statusCode: 400,
-            });
-        }
-
         let rebuild = false;
 
         if (request.body.subject !== undefined) {
@@ -78,6 +69,11 @@ export class PatchEmailEndpoint extends Endpoint<Params, Query, Body, ResponseBo
                 model.senderId = sender.id;
                 model.fromAddress = sender.email;
                 model.fromName = sender.name;
+
+                // Check if we still have write access to the email
+                if (!await Context.auth.canAccessEmail(model, PermissionLevel.Write)) {
+                    throw Context.auth.error();
+                }
             }
             else {
                 throw new SimpleError({
@@ -119,6 +115,15 @@ export class PatchEmailEndpoint extends Endpoint<Params, Query, Body, ResponseBo
         }
 
         if (request.body.recipientFilter) {
+            if (model.status !== EmailStatus.Draft) {
+                throw new SimpleError({
+                    code: 'not_draft',
+                    human: 'Email is not a draft',
+                    message: $t(`Je kan de ontvangerslijst alleen aanpassen als de e-mail nog een concept is`),
+                    statusCode: 400,
+                });
+            }
+
             model.recipientFilter = patchObject(model.recipientFilter, request.body.recipientFilter);
             rebuild = true;
         }
@@ -129,17 +134,24 @@ export class PatchEmailEndpoint extends Endpoint<Params, Query, Body, ResponseBo
             model.validateAttachments();
         }
 
+        if (request.body.deletedAt !== undefined) {
+            if (!await Context.auth.canAccessEmail(model, PermissionLevel.Full)) {
+                throw Context.auth.error();
+            }
+            model.deletedAt = request.body.deletedAt;
+        }
+
         await model.save();
 
         if (rebuild) {
             await model.buildExampleRecipient();
-            model.updateCount();
 
             // Force null - because we have stale data
-            model.recipientCount = null;
+            model.emailRecipientsCount = null;
+            model.updateCount();
         }
 
-        if (request.body.status === EmailStatus.Sending || request.body.status === EmailStatus.Sent) {
+        if (request.body.status === EmailStatus.Sending || request.body.status === EmailStatus.Sent || request.body.status === EmailStatus.Queued) {
             if (!await Context.auth.canSendEmail(model)) {
                 throw Context.auth.error({
                     message: 'Cannot send emails from this sender',
@@ -175,7 +187,8 @@ export class PatchEmailEndpoint extends Endpoint<Params, Query, Body, ResponseBo
                 }
             }
 
-            model.send().catch(console.error);
+            // Preview the sending status
+            await model.queueForSending();
         }
 
         return new Response(await model.getPreviewStructure());

package/src/endpoints/global/email-recipients/GetEmailRecipientsCountEndpoint.ts

@@ -0,0 +1,47 @@
+import { Decoder } from '@simonbackx/simple-encoding';
+import { DecodedRequest, Endpoint, Request, Response } from '@simonbackx/simple-endpoints';
+import { CountFilteredRequest, CountResponse } from '@stamhoofd/structures';
+
+import { Context } from '../../../helpers/Context';
+import { GetEmailRecipientsEndpoint } from './GetEmailRecipientsEndpoint';
+
+type Params = Record<string, never>;
+type Query = CountFilteredRequest;
+type Body = undefined;
+type ResponseBody = CountResponse;
+
+export class GetEmailRecipientsCountEndpoint extends Endpoint<Params, Query, Body, ResponseBody> {
+    queryDecoder = CountFilteredRequest as Decoder<CountFilteredRequest>;
+
+    protected doesMatch(request: Request): [true, Params] | [false] {
+        if (request.method !== 'GET') {
+            return [false];
+        }
+
+        const params = Endpoint.parseParameters(request.url, '/email-recipients/count', {});
+
+        if (params) {
+            return [true, params as Params];
+        }
+        return [false];
+    }
+
+    async handle(request: DecodedRequest<Params, Query, Body>) {
+        const organization = await Context.setOptionalOrganizationScope();
+        await Context.authenticate();
+
+        if (!await Context.auth.canReadEmails(organization)) {
+            throw Context.auth.error();
+        }
+        const query = await GetEmailRecipientsEndpoint.buildQuery(request.query);
+
+        const count = await query
+            .count();
+
+        return new Response(
+            CountResponse.create({
+                count,
+            }),
+        );
+    }
+}

package/src/endpoints/global/email-recipients/GetEmailRecipientsEndpoint.test.ts

@@ -0,0 +1,225 @@
+import { STExpect, TestUtils } from '@stamhoofd/test-utils';
+import { GetEmailRecipientsEndpoint } from './GetEmailRecipientsEndpoint';
+import { AccessRight, EmailStatus, LimitedFilteredRequest, OrganizationEmail, PermissionLevel, Permissions, PermissionsResourceType, ResourcePermissions } from '@stamhoofd/structures';
+import { Email, EmailRecipient, Organization, OrganizationFactory, RegistrationPeriod, RegistrationPeriodFactory, Token, User, UserFactory } from '@stamhoofd/models';
+import { Request } from '@simonbackx/simple-endpoints';
+import { testServer } from '../../../../tests/helpers/TestServer';
+
+const baseUrl = `/email-recipients`;
+
+describe('Endpoint.GetEmailRecipients', () => {
+    const endpoint = new GetEmailRecipientsEndpoint();
+    let period: RegistrationPeriod;
+    let organization: Organization;
+    let token: Token;
+    let user: User;
+    let sender: OrganizationEmail;
+    let sender2: OrganizationEmail;
+
+    let token2: Token;
+    let user2: User;
+
+    beforeAll(async () => {
+        TestUtils.setPermanentEnvironment('userMode', 'platform');
+        period = await new RegistrationPeriodFactory({
+            startDate: new Date(2023, 0, 1),
+            endDate: new Date(2023, 11, 31),
+        }).create();
+
+        organization = await new OrganizationFactory({ period })
+            .create();
+
+        sender = OrganizationEmail.create({
+            email: 'groepsleiding@voorbeeld.com',
+            name: 'Groepsleiding',
+        });
+        sender2 = OrganizationEmail.create({
+            email: 'kapoenen@voorbeeld.com',
+            name: 'Kapoenen',
+        });
+
+        organization.privateMeta.emails.push(sender);
+        organization.privateMeta.emails.push(sender2);
+        await organization.save();
+
+        user = await new UserFactory({
+            organization,
+            permissions: Permissions.create({
+                level: PermissionLevel.None,
+                resources: new Map([
+                    [PermissionsResourceType.Senders, new Map([['', ResourcePermissions.create({
+                        resourceName: sender.name!,
+                        level: PermissionLevel.Read,
+                    })]])],
+                ]),
+            }),
+        })
+            .create();
+
+        token = await Token.createToken(user);
+
+        user2 = await new UserFactory({
+            organization,
+            permissions: Permissions.create({
+                level: PermissionLevel.None,
+                resources: new Map([
+                    [PermissionsResourceType.Senders, new Map([[sender2.id, ResourcePermissions.create({
+                        resourceName: sender.name!,
+                        level: PermissionLevel.Read,
+                    })]])],
+                ]),
+            }),
+        })
+            .create();
+
+        token2 = await Token.createToken(user2);
+    });
+
+    test('It can request all email recipients if read permission for all senders', async () => {
+        const email = new Email();
+        email.subject = 'test subject';
+        email.status = EmailStatus.Draft;
+        email.text = 'test email';
+        email.html = `<p style="margin: 0; padding: 0; line-height: 1.4;">test email</p>`;
+        email.json = {};
+        email.organizationId = organization.id;
+        email.senderId = sender.id;
+        await email.save();
+
+        const emailRecipient = new EmailRecipient();
+        emailRecipient.email = 'jan.janssens@geenemail.com';
+        emailRecipient.firstName = 'Jan';
+        emailRecipient.lastName = 'Janssens';
+        emailRecipient.emailId = email.id;
+        emailRecipient.organizationId = organization.id;
+        await emailRecipient.save();
+
+        const request = Request.get({
+            path: baseUrl,
+            host: organization.getApiHost(),
+            query: new LimitedFilteredRequest({
+                filter: {},
+                limit: 10,
+            }),
+            headers: {
+                authorization: 'Bearer ' + token.accessToken,
+            },
+        });
+        const result = await testServer.test(endpoint, request);
+        expect(result.body.results).toHaveLength(1);
+        expect(result.body.results[0]).toMatchObject({
+            id: emailRecipient.id,
+        });
+    });
+
+    test('It can not request all email recipients if not read permission for all senders', async () => {
+        const email = new Email();
+        email.subject = 'test subject';
+        email.status = EmailStatus.Draft;
+        email.text = 'test email';
+        email.html = `<p style="margin: 0; padding: 0; line-height: 1.4;">test email</p>`;
+        email.json = {};
+        email.organizationId = organization.id;
+        email.senderId = sender2.id;
+        await email.save();
+
+        const emailRecipient = new EmailRecipient();
+        emailRecipient.email = 'jan.janssens@geenemail.com';
+        emailRecipient.firstName = 'Jan';
+        emailRecipient.lastName = 'Janssens';
+        emailRecipient.emailId = email.id;
+        emailRecipient.organizationId = organization.id;
+        await emailRecipient.save();
+
+        const request = Request.get({
+            path: baseUrl,
+            host: organization.getApiHost(),
+            query: new LimitedFilteredRequest({
+                filter: {},
+                limit: 10,
+            }),
+            headers: {
+                authorization: 'Bearer ' + token2.accessToken,
+            },
+        });
+        await expect(testServer.test(endpoint, request))
+            .rejects
+            .toThrow(STExpect.errorWithCode('permission_denied'));
+    });
+
+    test('It request all email recipients of a single email if read permission for that sender', async () => {
+        const email = new Email();
+        email.subject = 'test subject';
+        email.status = EmailStatus.Draft;
+        email.text = 'test email';
+        email.html = `<p style="margin: 0; padding: 0; line-height: 1.4;">test email</p>`;
+        email.json = {};
+        email.organizationId = organization.id;
+        email.senderId = sender2.id;
+        await email.save();
+
+        const emailRecipient = new EmailRecipient();
+        emailRecipient.email = 'jan.janssens@geenemail.com';
+        emailRecipient.firstName = 'Jan';
+        emailRecipient.lastName = 'Janssens';
+        emailRecipient.emailId = email.id;
+        emailRecipient.organizationId = organization.id;
+        await emailRecipient.save();
+
+        const request = Request.get({
+            path: baseUrl,
+            host: organization.getApiHost(),
+            query: new LimitedFilteredRequest({
+                filter: {
+                    emailId: email.id,
+                },
+                limit: 10,
+            }),
+            headers: {
+                authorization: 'Bearer ' + token2.accessToken,
+            },
+        });
+        const result = await testServer.test(endpoint, request);
+        expect(result.body.results).toHaveLength(1);
+        expect(result.body.results[0]).toMatchObject({
+            id: emailRecipient.id,
+        });
+    });
+
+    test('It cannot request all email recipients of a single email if read permission for another sender', async () => {
+        const email = new Email();
+        email.subject = 'test subject';
+        email.status = EmailStatus.Draft;
+        email.text = 'test email';
+        email.html = `<p style="margin: 0; padding: 0; line-height: 1.4;">test email</p>`;
+        email.json = {};
+        email.organizationId = organization.id;
+        email.senderId = sender.id;
+        await email.save();
+
+        const emailRecipient = new EmailRecipient();
+        emailRecipient.email = 'jan.janssens@geenemail.com';
+        emailRecipient.firstName = 'Jan';
+        emailRecipient.lastName = 'Janssens';
+        emailRecipient.emailId = email.id;
+        emailRecipient.organizationId = organization.id;
+        await emailRecipient.save();
+
+        const request = Request.get({
+            path: baseUrl,
+            host: organization.getApiHost(),
+            query: new LimitedFilteredRequest({
+                filter: {
+                    emailId: email.id,
+                },
+                limit: 10,
+            }),
+            headers: {
+                authorization: 'Bearer ' + token2.accessToken,
+            },
+        });
+        await expect(testServer.test(endpoint, request))
+            .rejects
+            .toThrow(STExpect.errorWithCode('permission_denied'));
+    });
+});