@stamhoofd/backend 2.91.0 → 2.93.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@stamhoofd/backend",
-    "version": "2.91.0",
+    "version": "2.93.0",
     "main": "./dist/index.js",
     "exports": {
         ".": {
@@ -45,14 +45,14 @@
         "@simonbackx/simple-encoding": "2.22.0",
         "@simonbackx/simple-endpoints": "1.20.1",
         "@simonbackx/simple-logging": "^1.0.1",
-        "@stamhoofd/backend-i18n": "2.91.0",
-        "@stamhoofd/backend-middleware": "2.91.0",
-        "@stamhoofd/email": "2.91.0",
-        "@stamhoofd/models": "2.91.0",
-        "@stamhoofd/queues": "2.91.0",
-        "@stamhoofd/sql": "2.91.0",
-        "@stamhoofd/structures": "2.91.0",
-        "@stamhoofd/utility": "2.91.0",
+        "@stamhoofd/backend-i18n": "2.93.0",
+        "@stamhoofd/backend-middleware": "2.93.0",
+        "@stamhoofd/email": "2.93.0",
+        "@stamhoofd/models": "2.93.0",
+        "@stamhoofd/queues": "2.93.0",
+        "@stamhoofd/sql": "2.93.0",
+        "@stamhoofd/structures": "2.93.0",
+        "@stamhoofd/utility": "2.93.0",
         "archiver": "^7.0.1",
         "axios": "^1.8.2",
         "cookie": "^0.7.0",
@@ -70,5 +70,5 @@
     "publishConfig": {
         "access": "public"
     },
-    "gitHead": "37ff6e89f07320c0736d32c9b911c8c1c35dc421"
+    "gitHead": "05b8228a7fc3871aa767609a95a2c1511132e548"
 }

package/src/audit-logs/EmailLogger.ts

@@ -1,5 +1,5 @@
-import { Email, EmailRecipient, replaceHtml } from '@stamhoofd/models';
-import { AuditLogReplacement, AuditLogReplacementType, AuditLogType, EmailStatus } from '@stamhoofd/structures';
+import { Email, EmailRecipient } from '@stamhoofd/models';
+import { AuditLogReplacement, AuditLogReplacementType, AuditLogType, EmailStatus, replaceEmailHtml } from '@stamhoofd/structures';
 import { Formatter } from '@stamhoofd/utility';
 import { ModelLogger } from './ModelLogger';
 
@@ -54,17 +54,17 @@ export const EmailLogger = new ModelLogger(Email, {
         const map = new Map([
             ['e', AuditLogReplacement.create({
                 id: model.id,
-                value: replaceHtml(model.subject ?? '', options.data.recipient?.replacements ?? []),
+                value: replaceEmailHtml(model.subject ?? '', options.data.recipient?.replacements ?? []),
                 type: AuditLogReplacementType.Email,
             })],
             ['c', AuditLogReplacement.create({
-                value: Formatter.integer(model.recipientCount ?? 0),
-                count: model.recipientCount ?? 0,
+                value: Formatter.integer(model.emailRecipientsCount ?? 0),
+                count: model.emailRecipientsCount ?? 0,
             })],
         ]);
         if (options.data.recipient) {
             map.set('html', AuditLogReplacement.html(
-                replaceHtml(model.html ?? '', options.data.recipient?.replacements ?? []),
+                replaceEmailHtml(model.html ?? '', options.data.recipient?.replacements ?? []),
             ));
         }
         return map;

package/src/crons/amazon-ses.ts

@@ -6,9 +6,10 @@ import {
 } from '@aws-sdk/client-sqs';
 import { registerCron } from '@stamhoofd/crons';
 import { Email, EmailAddress } from '@stamhoofd/email';
-import { AuditLog, Organization } from '@stamhoofd/models';
+import { AuditLog, EmailRecipient, Organization } from '@stamhoofd/models';
 import { AuditLogReplacement, AuditLogReplacementType, AuditLogSource, AuditLogType } from '@stamhoofd/structures';
 import { ForwardHandler } from '../helpers/ForwardHandler';
+import { Email as EmailModel } from '@stamhoofd/models';
 
 registerCron('checkComplaints', checkComplaints);
 registerCron('checkReplies', checkReplies);
@@ -52,13 +53,88 @@ async function saveLog({ email, organization, type, subType, subject, response,
     await log.save();
 }
 
+async function storeEmailStatus({ headers, type, message }: { headers: Record<string, string>; type: 'hard-bounce' | 'soft-bounce' | 'complaint'; message: string }) {
+    const emailId = headers['x-email-id'];
+    const recipientId = headers['x-email-recipient-id'];
+
+    if (emailId && recipientId) {
+        // check
+        const emailRecipient = await EmailRecipient.select()
+            .where('id', recipientId)
+            .where('emailId', emailId)
+            .first(false);
+
+        if (!emailRecipient) {
+            console.log('[AWS FORWARDING] Invalid email or recipient id in headers', headers);
+            return;
+        }
+
+        let isNew = true;
+
+        switch (type) {
+            case 'hard-bounce': {
+                if (emailRecipient.hardBounceError) {
+                    isNew = false;
+                }
+                emailRecipient.hardBounceError = message;
+                break;
+            }
+            case 'soft-bounce': {
+                if (emailRecipient.softBounceError) {
+                    isNew = false;
+                }
+                emailRecipient.softBounceError = message;
+                break;
+            }
+            case 'complaint': {
+                if (emailRecipient.spamComplaintError) {
+                    isNew = false;
+                }
+                emailRecipient.spamComplaintError = message;
+                break;
+            }
+        }
+
+        console.log('[AWS FORWARDING] Marking email recipient ' + recipientId + ' for email ' + emailId + ' as ' + type);
+        if (await emailRecipient.save()) {
+            if (isNew) {
+                await EmailModel.bumpNotificationCount(emailId, type);
+            }
+        }
+    }
+}
+
+function readHeaders(message: any) {
+    try {
+        const mail = message.mail;
+        const headers: Record<string, string> = {};
+        if (mail && typeof mail === 'object' && mail !== null && Array.isArray(mail.headers)) {
+            for (const header of mail.headers) {
+                if (header.name && header.value) {
+                    headers[(header.name as string).toLowerCase()] = header.value;
+                }
+            }
+        }
+        else {
+            console.log('[AWS] Missing mail headers', message);
+        }
+        return headers;
+    }
+    catch (e) {
+        console.log('[AWS] Failed to read headers', e, message);
+        return {};
+    }
+}
+
 async function handleBounce(message: any) {
     if (message.bounce) {
+        console.log('[AWS BOUNCES] Handling bounce message', message);
+        const headers = readHeaders(message);
+
         const b = message.bounce;
         // Block all receivers that generate a permanent bounce
         const type = b.bounceType;
         const subtype = b.bounceSubType;
-
         const source = message.mail.source;
 
         // try to find organization that is responsible for this e-mail address
@@ -80,6 +156,12 @@ async function handleBounce(message: any) {
                 emailAddress.hardBounce = true;
                 await emailAddress.save();
 
+                await storeEmailStatus({
+                    headers,
+                    type: 'hard-bounce',
+                    message: recipient.diagnosticCode || 'Permanent bounce',
+                });
+
                 await saveLog({
                     id: b.feedbackId,
                     email,
@@ -95,6 +177,13 @@ async function handleBounce(message: any) {
                 type === 'Transient'
             ) {
                 const organization: Organization | undefined = source ? await Organization.getByEmail(source) : undefined;
+
+                await storeEmailStatus({
+                    headers,
+                    type: 'soft-bounce',
+                    message: recipient.diagnosticCode || 'Soft bounce',
+                });
+
                 await saveLog({
                     id: b.feedbackId,
                     email,
@@ -116,6 +205,7 @@ async function handleBounce(message: any) {
 
 async function handleComplaint(message: any) {
     if (message.complaint) {
+        const headers = readHeaders(message);
         const b = message.complaint;
         const source = message.mail.source;
         const organization: Organization | undefined = source ? await Organization.getByEmail(source) : undefined;
@@ -129,10 +219,16 @@ async function handleComplaint(message: any) {
             await emailAddress.save();
 
             if (type !== 'not-spam') {
+                await storeEmailStatus({
+                    headers,
+                    type: 'complaint',
+                    message: recipient.diagnosticCode || type || 'Complaint',
+                });
+
                 if (type === 'virus' || type === 'fraud') {
                     await saveLog({
                         id: b.feedbackId,
-                        email: source,
+                        email,
                         organization,
                         type: AuditLogType.EmailAddressFraudComplaint,
                         subType: type || 'unknown',
@@ -144,7 +240,7 @@ async function handleComplaint(message: any) {
                 else {
                     await saveLog({
                         id: b.feedbackId,
-                        email: source,
+                        email,
                         organization,
                         type: AuditLogType.EmailAddressMarkedAsSpam,
                         subType: type || 'unknown',

package/src/crons/balance-emails.ts

@@ -198,7 +198,7 @@ async function sendTemplate({
 
     try {
         const upToDate = await ContextInstance.startForUser(systemUser, organization, async () => {
-            return await model.send();
+            return await model.queueForSending(true);
         });
 
         if (!upToDate) {

package/src/crons/endFunctionsOfUsersWithoutRegistration.ts

@@ -33,6 +33,12 @@ export async function endFunctionsOfUsersWithoutRegistration() {
         return;
     }
 
+    // If period is ending within 15 days, also skip cleanup
+    if (period.endDate && period.endDate < new Date(Date.now() + 1000 * 60 * 60 * 24 * 15)) {
+        console.warn('Current registration period is ending within 15 days or has ended, skipping cleanup.');
+        return;
+    }
+
     await FlagMomentCleanup.endFunctionsOfUsersWithoutRegistration();
     lastCleanupYear = currentYear;
     lastCleanupMonth = currentMonth;

package/src/email-recipient-loaders/receivable-balances.ts

@@ -1,5 +1,5 @@
 import { CachedBalance, Email } from '@stamhoofd/models';
-import { BalanceItem as BalanceItemStruct, compileToInMemoryFilter, EmailRecipient, EmailRecipientFilterType, LimitedFilteredRequest, PaginatedResponse, receivableBalanceObjectContactInMemoryFilterCompilers, Replacement, StamhoofdFilter } from '@stamhoofd/structures';
+import { BalanceItem as BalanceItemStruct, compileToInMemoryFilter, EmailRecipient, EmailRecipientFilterType, LimitedFilteredRequest, PaginatedResponse, receivableBalanceObjectContactInMemoryFilterCompilers, ReceivableBalanceType, Replacement, StamhoofdFilter } from '@stamhoofd/structures';
 import { Formatter } from '@stamhoofd/utility';
 import { GetReceivableBalancesEndpoint } from '../endpoints/organization/dashboard/receivable-balances/GetReceivableBalancesEndpoint';
 
@@ -22,6 +22,8 @@ async function fetch(query: LimitedFilteredRequest, subfilter: StamhoofdFilter |
             for (const email of contact.emails) {
                 const recipient = EmailRecipient.create({
                     objectId: balance.id, // Note: not set member, user or organization id here - should be the queryable balance id
+                    userId: balance.objectType === ReceivableBalanceType.user ? balance.object.id : null,
+                    memberId: balance.objectType === ReceivableBalanceType.member ? balance.object.id : null,
                     firstName: contact.firstName,
                     lastName: contact.lastName,
                     email,

package/src/endpoints/global/email/CreateEmailEndpoint.ts

@@ -1,9 +1,10 @@
 import { Decoder } from '@simonbackx/simple-encoding';
 import { DecodedRequest, Endpoint, Request, Response } from '@simonbackx/simple-endpoints';
-import { Email, RateLimiter } from '@stamhoofd/models';
+import { Email, Platform, RateLimiter } from '@stamhoofd/models';
 import { EmailPreview, EmailStatus, Email as EmailStruct, EmailTemplate as EmailTemplateStruct } from '@stamhoofd/structures';
 
 import { Context } from '../../../helpers/Context';
+import { SimpleError } from '@simonbackx/simple-errors';
 
 type Params = Record<string, never>;
 type Query = undefined;
@@ -64,8 +65,11 @@ export class CreateEmailEndpoint extends Endpoint<Params, Query, Body, ResponseB
         const organization = await Context.setOptionalOrganizationScope();
         const { user } = await Context.authenticate();
 
-        if (!Context.auth.canSendEmails()) {
-            throw Context.auth.error();
+        if (!await Context.auth.canSendEmails(organization)) {
+            throw Context.auth.error({
+                message: 'Cannot send emails',
+                human: $t('f7b7ac75-f7df-49cc-8961-b2478d9683e3'),
+            });
         }
 
         const model = new Email();
@@ -79,8 +83,28 @@ export class CreateEmailEndpoint extends Endpoint<Params, Query, Body, ResponseB
         model.json = request.body.json;
         model.status = request.body.status;
         model.attachments = request.body.attachments;
-        model.fromAddress = request.body.fromAddress;
-        model.fromName = request.body.fromName;
+
+        const list = organization ? organization.privateMeta.emails : (await Platform.getShared()).privateConfig.emails;
+        const sender = list.find(e => e.id === request.body.senderId);
+        if (sender) {
+            if (!await Context.auth.canSendEmailsFrom(organization, sender.id)) {
+                throw Context.auth.error({
+                    message: 'Cannot send emails from this sender',
+                    human: $t('1b509614-30b0-484c-af72-57d4bc9ea788'),
+                });
+            }
+            model.senderId = sender.id;
+            model.fromAddress = sender.email;
+            model.fromName = sender.name;
+        }
+        else {
+            throw new SimpleError({
+                code: 'invalid_sender',
+                human: 'Sender not found',
+                message: $t(`94adb4e0-2ef1-4ee8-9f02-5a76efa51c1d`),
+                statusCode: 400,
+            });
+        }
 
         model.validateAttachments();
 
@@ -96,8 +120,14 @@ export class CreateEmailEndpoint extends Endpoint<Params, Query, Body, ResponseB
         await model.buildExampleRecipient();
         model.updateCount();
 
-        if (request.body.status === EmailStatus.Sending || request.body.status === EmailStatus.Sent) {
-            model.send().catch(console.error);
+        if (request.body.status === EmailStatus.Sending || request.body.status === EmailStatus.Sent || request.body.status === EmailStatus.Queued) {
+            if (!await Context.auth.canSendEmail(model)) {
+                throw Context.auth.error({
+                    message: 'Cannot send emails from this sender',
+                    human: $t('1b509614-30b0-484c-af72-57d4bc9ea788'),
+                });
+            }
+            await model.queueForSending();
         }
 
         return new Response(await model.getPreviewStructure());

package/src/endpoints/global/email/GetAdminEmailsEndpoint.ts

@@ -0,0 +1,205 @@
+import { DecodedRequest, Endpoint, Request, Response } from '@simonbackx/simple-endpoints';
+import { assertSort, CountFilteredRequest, EmailPreview, EmailStatus, getSortFilter, LimitedFilteredRequest, PaginatedResponse, StamhoofdFilter } from '@stamhoofd/structures';
+
+import { Decoder } from '@simonbackx/simple-encoding';
+import { SimpleError } from '@simonbackx/simple-errors';
+import { Email, Platform } from '@stamhoofd/models';
+import { applySQLSorter, compileToSQLFilter, SQLFilterDefinitions, SQLSortDefinitions } from '@stamhoofd/sql';
+import { Context } from '../../../helpers/Context';
+import { emailFilterCompilers } from '../../../sql-filters/emails';
+import { emailSorters } from '../../../sql-sorters/emails';
+
+type Params = Record<string, never>;
+type Query = LimitedFilteredRequest;
+type Body = undefined;
+type ResponseBody = PaginatedResponse<EmailPreview[], LimitedFilteredRequest>;
+
+const filterCompilers: SQLFilterDefinitions = emailFilterCompilers;
+const sorters: SQLSortDefinitions<Email> = emailSorters;
+
+export class GetAdminEmailsEndpoint extends Endpoint<Params, Query, Body, ResponseBody> {
+    queryDecoder = LimitedFilteredRequest as Decoder<LimitedFilteredRequest>;
+
+    protected doesMatch(request: Request): [true, Params] | [false] {
+        if (request.method !== 'GET') {
+            return [false];
+        }
+
+        const params = Endpoint.parseParameters(request.url, '/email', {});
+
+        if (params) {
+            return [true, params as Params];
+        }
+        return [false];
+    }
+
+    static async buildQuery(q: CountFilteredRequest | LimitedFilteredRequest) {
+        const organization = Context.organization;
+        const user = Context.user;
+        if (!user) {
+            throw new Error('Not authenticated');
+        }
+
+        let scopeFilter: StamhoofdFilter | undefined = undefined;
+
+        const canReadAllEmails = await Context.auth.canReadAllEmails(organization ?? null);
+        scopeFilter = {
+            organizationId: organization?.id ?? null,
+        };
+
+        if (!canReadAllEmails) {
+            const senders = organization ? organization.privateMeta.emails : (await Platform.getShared()).privateConfig.emails;
+            const ids: string[] = [];
+            for (const sender of senders) {
+                if (await Context.auth.canReadAllEmails(organization ?? null, sender.id)) {
+                    ids.push(sender.id);
+                }
+            }
+            if (ids.length === 0) {
+                throw Context.auth.error();
+            }
+
+            scopeFilter = {
+                $and: [
+                    {
+                        organizationId: organization?.id ?? null,
+                    },
+                    {
+                        $or: [
+                            {
+                                senderId: {
+                                    $in: ids,
+                                },
+                                status: {
+                                    $neq: EmailStatus.Draft,
+                                },
+                            },
+                            {
+                                userId: user.id,
+                            },
+                        ],
+                    },
+                ],
+            };
+        }
+        else {
+            scopeFilter = {
+                $and: [
+                    {
+                        organizationId: organization?.id ?? null,
+                    },
+                    {
+                        $or: [
+                            {
+                                status: {
+                                    $neq: EmailStatus.Draft,
+                                },
+                            },
+                            {
+                                userId: user.id,
+                            },
+                        ],
+                    },
+                ],
+            };
+        }
+
+        const query = Email.select();
+
+        if (scopeFilter) {
+            query.where(await compileToSQLFilter(scopeFilter, filterCompilers));
+        }
+
+        if (q.filter) {
+            query.where(await compileToSQLFilter(q.filter, filterCompilers));
+        }
+
+        if (q.search) {
+            let searchFilter: StamhoofdFilter | null = null;
+
+            searchFilter = {
+                subject: {
+                    $contains: q.search,
+                },
+            };
+
+            if (searchFilter) {
+                query.where(await compileToSQLFilter(searchFilter, filterCompilers));
+            }
+        }
+
+        if (q instanceof LimitedFilteredRequest) {
+            if (q.pageFilter) {
+                query.where(await compileToSQLFilter(q.pageFilter, filterCompilers));
+            }
+
+            q.sort = assertSort(q.sort, [{ key: 'id' }]);
+            applySQLSorter(query, q.sort, sorters);
+            query.limit(q.limit);
+        }
+
+        return query;
+    }
+
+    static async buildData(requestQuery: LimitedFilteredRequest) {
+        const query = await GetAdminEmailsEndpoint.buildQuery(requestQuery);
+        const emails = await query.fetch();
+
+        let next: LimitedFilteredRequest | undefined;
+
+        if (emails.length >= requestQuery.limit) {
+            const lastObject = emails[emails.length - 1];
+            const nextFilter = getSortFilter(lastObject, sorters, requestQuery.sort);
+
+            next = new LimitedFilteredRequest({
+                filter: requestQuery.filter,
+                pageFilter: nextFilter,
+                sort: requestQuery.sort,
+                limit: requestQuery.limit,
+                search: requestQuery.search,
+            });
+
+            if (JSON.stringify(nextFilter) === JSON.stringify(requestQuery.pageFilter)) {
+                console.error('Found infinite loading loop for', requestQuery);
+                next = undefined;
+            }
+        }
+
+        return new PaginatedResponse<EmailPreview[], LimitedFilteredRequest>({
+            results: await Promise.all(emails.map(email => email.getPreviewStructure())),
+            next,
+        });
+    }
+
+    async handle(request: DecodedRequest<Params, Query, Body>) {
+        const organization = await Context.setOptionalOrganizationScope();
+        await Context.authenticate();
+
+        if (!await Context.auth.canReadEmails(organization)) {
+            // This is a first fast check, we'll limit it later in the scope query
+            throw Context.auth.error();
+        }
+
+        const maxLimit = Context.auth.hasSomePlatformAccess() ? 1000 : 100;
+
+        if (request.query.limit > maxLimit) {
+            throw new SimpleError({
+                code: 'invalid_field',
+                field: 'limit',
+                message: 'Limit can not be more than ' + maxLimit,
+            });
+        }
+
+        if (request.query.limit < 1) {
+            throw new SimpleError({
+                code: 'invalid_field',
+                field: 'limit',
+                message: 'Limit can not be less than 1',
+            });
+        }
+
+        return new Response(
+            await GetAdminEmailsEndpoint.buildData(request.query),
+        );
+    }
+}

package/src/endpoints/global/email/GetEmailEndpoint.ts

@@ -32,7 +32,7 @@ export class GetEmailEndpoint extends Endpoint<Params, Query, Body, ResponseBody
         const organization = await Context.setOptionalOrganizationScope();
         const { user } = await Context.authenticate();
 
-        if (!Context.auth.canSendEmails()) {
+        if (!await Context.auth.canReadEmails(organization)) {
             throw Context.auth.error();
         }
 
@@ -46,6 +46,10 @@ export class GetEmailEndpoint extends Endpoint<Params, Query, Body, ResponseBody
             });
         }
 
+        if (!await Context.auth.canAccessEmail(model)) {
+            throw Context.auth.error();
+        }
+
         return new Response(await model.getPreviewStructure());
     }
 }