@stamhoofd/backend 2.79.7 → 2.80.0

package/.nvmrc

@@ -0,0 +1 @@
+20.12

package/index.ts

@@ -1,5 +1,5 @@
 import backendEnv from '@stamhoofd/backend-env';
-backendEnv.load();
+backendEnv.load({ service: 'api' });
 
 import { Column, Database, Migration } from '@simonbackx/simple-database';
 import { CORSPreflightEndpoint, Router, RouterServer } from '@simonbackx/simple-endpoints';

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@stamhoofd/backend",
-    "version": "2.79.7",
+    "version": "2.80.0",
     "main": "./dist/index.js",
     "exports": {
         ".": {
@@ -35,17 +35,17 @@
         "@bwip-js/node": "^4.5.1",
         "@mollie/api-client": "3.7.0",
         "@simonbackx/simple-database": "1.29.1",
-        "@simonbackx/simple-encoding": "2.21.0",
+        "@simonbackx/simple-encoding": "2.22.0",
         "@simonbackx/simple-endpoints": "1.19.1",
         "@simonbackx/simple-logging": "^1.0.1",
-        "@stamhoofd/backend-i18n": "2.79.7",
-        "@stamhoofd/backend-middleware": "2.79.7",
-        "@stamhoofd/email": "2.79.7",
-        "@stamhoofd/models": "2.79.7",
-        "@stamhoofd/queues": "2.79.7",
-        "@stamhoofd/sql": "2.79.7",
-        "@stamhoofd/structures": "2.79.7",
-        "@stamhoofd/utility": "2.79.7",
+        "@stamhoofd/backend-i18n": "2.80.0",
+        "@stamhoofd/backend-middleware": "2.80.0",
+        "@stamhoofd/email": "2.80.0",
+        "@stamhoofd/models": "2.80.0",
+        "@stamhoofd/queues": "2.80.0",
+        "@stamhoofd/sql": "2.80.0",
+        "@stamhoofd/structures": "2.80.0",
+        "@stamhoofd/utility": "2.80.0",
         "archiver": "^7.0.1",
         "aws-sdk": "^2.885.0",
         "axios": "1.6.8",
@@ -65,5 +65,5 @@
     "publishConfig": {
         "access": "public"
     },
-    "gitHead": "d8675cbc91c2fcdcc6bbd3b3925ccd8063b15522"
+    "gitHead": "252faf1dbd8ebdd8469810e9c51e510d93f21fc2"
 }

package/src/audit-logs/ModelLogger.ts

@@ -1,9 +1,9 @@
 import { Model, ModelEvent } from '@simonbackx/simple-database';
 import { AuditLog } from '@stamhoofd/models';
+import { ObjectDiffer } from '@stamhoofd/object-differ';
 import { AuditLogPatchItem, AuditLogPatchItemType, AuditLogReplacement, AuditLogSource, AuditLogType } from '@stamhoofd/structures';
 import { ContextInstance } from '../helpers/Context';
 import { AuditLogService } from '../services/AuditLogService';
-import { diffUnknown } from '../services/diff';
 
 export type ModelEventLogOptions<D> = {
     type: AuditLogType;
@@ -180,7 +180,7 @@ export class ModelLogger<ModelType extends typeof Model, M extends InstanceType<
                             }));
                         }
                         else {
-                            log.patchList.push(...diffUnknown(
+                            log.patchList.push(...ObjectDiffer.diff(
                                 key in oldModel ? oldModel[key] : undefined,
                                 key in event.model ? event.model[key] : undefined,
                                 AuditLogReplacement.key(renamedKey),

package/src/crons/amazon-ses.ts

@@ -27,13 +27,17 @@ async function saveLog({ email, organization, type, subType, subject, response,
             type: AuditLogReplacementType.EmailAddress,
         })],
         ['subType', AuditLogReplacement.key(subType || 'unknown')],
-        ['response', AuditLogReplacement.longText(response)],
         ['subject', AuditLogReplacement.string(subject)],
         ['sender', AuditLogReplacement.create({
             value: sender,
             type: AuditLogReplacementType.EmailAddress,
         })],
     ]);
+
+    if (response) {
+        log.replacements.set('response', AuditLogReplacement.longText(response));
+    }
+
     // Check if we already logged this bounce
     const existing = await AuditLog.select().where('externalId', log.externalId).first(false);
     if (existing) {
@@ -44,140 +48,180 @@ async function saveLog({ email, organization, type, subType, subject, response,
     await log.save();
 }
 
-async function checkBounces() {
-    if (STAMHOOFD.environment !== 'production' || !STAMHOOFD.AWS_ACCESS_KEY_ID) {
-        return;
-    }
-
-    console.log('[AWS BOUNCES] Checking bounces from AWS SQS');
-    const sqs = new AWS.SQS();
-    const messages = await sqs.receiveMessage({ QueueUrl: 'https://sqs.eu-west-1.amazonaws.com/118244293157/stamhoofd-bounces-queue', MaxNumberOfMessages: 10 }).promise();
-    if (messages.Messages) {
-        for (const message of messages.Messages) {
-            console.log('[AWS BOUNCES] Received bounce message');
-            console.log('[AWS BOUNCES]', message);
-
-            if (message.ReceiptHandle) {
-                if (STAMHOOFD.environment === 'production') {
-                    await sqs.deleteMessage({
-                        QueueUrl: 'https://sqs.eu-west-1.amazonaws.com/118244293157/stamhoofd-bounces-queue',
-                        ReceiptHandle: message.ReceiptHandle,
-                    }).promise();
-                    console.log('[AWS BOUNCES] Deleted from queue');
+async function handleBounce(message: any) {
+    if (message.bounce) {
+        const b = message.bounce;
+        // Block all receivers that generate a permanent bounce
+        const type = b.bounceType;
+        const subtype = b.bounceSubType;
+
+        const source = message.mail.source;
+
+        // try to find organization that is responsible for this e-mail address
+
+        for (const recipient of b.bouncedRecipients) {
+            const email = recipient.emailAddress;
+
+            if (
+                type === 'Permanent'
+                || (
+                    recipient.diagnosticCode && (
+                        (recipient.diagnosticCode as string).toLowerCase().includes('invalid domain')
+                        || (recipient.diagnosticCode as string).toLowerCase().includes('unable to lookup dns')
+                    )
+                )
+            ) {
+                const organization: Organization | undefined = source ? await Organization.getByEmail(source) : undefined;
+                if (organization) {
+                    const emailAddress = await EmailAddress.getOrCreate(email, organization.id);
+                    emailAddress.hardBounce = true;
+                    await emailAddress.save();
+                }
+                else {
+                    console.error('[AWS BOUNCES] Unknown organization for email address ' + source);
                 }
-            }
-
-            try {
-                if (message.Body) {
-                    // decode the JSON value
-                    const bounce = JSON.parse(message.Body);
 
-                    if (bounce.Message) {
-                        const message = JSON.parse(bounce.Message);
+                await saveLog({
+                    id: b.feedbackId,
+                    email,
+                    organization,
+                    type: AuditLogType.EmailAddressHardBounced,
+                    subType: subtype || 'unknown',
+                    sender: source,
+                    response: recipient.diagnosticCode || '',
+                    subject: message.mail.commonHeaders?.subject || '',
+                });
+            }
+            else if (
+                type === 'Transient'
+            ) {
+                const organization: Organization | undefined = source ? await Organization.getByEmail(source) : undefined;
+                await saveLog({
+                    id: b.feedbackId,
+                    email,
+                    organization,
+                    type: AuditLogType.EmailAddressSoftBounced,
+                    subType: subtype || 'unknown',
+                    sender: source,
+                    response: recipient.diagnosticCode || '',
+                    subject: message.mail.commonHeaders?.subject || '',
+                });
+            }
+        }
+        console.log('[AWS BOUNCES] For domain ' + source);
+    }
+    else {
+        console.log("[AWS BOUNCES] 'bounce' field missing in bounce message");
+    }
+}
 
-                        if (message.bounce) {
-                            const b = message.bounce;
-                            // Block all receivers that generate a permanent bounce
-                            const type = b.bounceType;
-                            const subtype = b.bounceSubType;
-
-                            const source = message.mail.source;
-
-                            // try to find organization that is responsible for this e-mail address
-
-                            for (const recipient of b.bouncedRecipients) {
-                                const email = recipient.emailAddress;
-
-                                if (
-                                    type === 'Permanent'
-                                    || (
-                                        recipient.diagnosticCode && (
-                                            (recipient.diagnosticCode as string).toLowerCase().includes('invalid domain')
-                                            || (recipient.diagnosticCode as string).toLowerCase().includes('unable to lookup dns')
-                                        )
-                                    )
-                                ) {
-                                    const organization: Organization | undefined = source ? await Organization.getByEmail(source) : undefined;
-                                    if (organization) {
-                                        const emailAddress = await EmailAddress.getOrCreate(email, organization.id);
-                                        emailAddress.hardBounce = true;
-                                        await emailAddress.save();
-                                    }
-                                    else {
-                                        console.error('[AWS BOUNCES] Unknown organization for email address ' + source);
-                                    }
-
-                                    await saveLog({
-                                        id: b.feedbackId,
-                                        email,
-                                        organization,
-                                        type: AuditLogType.EmailAddressHardBounced,
-                                        subType: subtype || 'unknown',
-                                        sender: source,
-                                        response: b.diagnosticCode || '',
-                                        subject: message.mail.commonHeaders?.subject || '',
-                                    });
-                                }
-                                else if (
-                                    type === 'Transient'
-                                ) {
-                                    const organization: Organization | undefined = source ? await Organization.getByEmail(source) : undefined;
-                                    await saveLog({
-                                        id: b.feedbackId,
-                                        email,
-                                        organization,
-                                        type: AuditLogType.EmailAddressSoftBounced,
-                                        subType: subtype || 'unknown',
-                                        sender: source,
-                                        response: b.diagnosticCode || '',
-                                        subject: message.mail.commonHeaders?.subject || '',
-                                    });
-                                }
-                            }
-                            console.log('[AWS BOUNCES] For domain ' + source);
-                        }
-                        else {
-                            console.log("[AWS BOUNCES] 'bounce' field missing in bounce message");
-                        }
+async function handleComplaint(message: any) {
+    if (message.complaint) {
+        const b = message.complaint;
+        const source = message.mail.source;
+        const organization: Organization | undefined = source ? await Organization.getByEmail(source) : undefined;
+
+        const type: 'abuse' | 'auth-failure' | 'fraud' | 'not-spam' | 'other' | 'virus' = b.complaintFeedbackType;
+
+        if (organization) {
+            for (const recipient of b.complainedRecipients) {
+                const email = recipient.emailAddress;
+                const emailAddress = await EmailAddress.getOrCreate(email, organization.id);
+                emailAddress.markedAsSpam = type !== 'not-spam';
+                await emailAddress.save();
+
+                if (type !== 'not-spam') {
+                    if (type === 'virus' || type === 'fraud') {
+                        await saveLog({
+                            id: b.feedbackId,
+                            email: source,
+                            organization,
+                            type: AuditLogType.EmailAddressFraudComplaint,
+                            subType: type || 'unknown',
+                            sender: source,
+                            response: recipient.diagnosticCode || '',
+                            subject: message.mail.commonHeaders?.subject || '',
+                        });
                     }
                     else {
-                        console.log("[AWS BOUNCES] 'Message' field missing in bounce message");
+                        await saveLog({
+                            id: b.feedbackId,
+                            email: source,
+                            organization,
+                            type: AuditLogType.EmailAddressMarkedAsSpam,
+                            subType: type || 'unknown',
+                            sender: source,
+                            response: recipient.diagnosticCode || '',
+                            subject: message.mail.commonHeaders?.subject || '',
+                        });
                     }
                 }
-                else {
-                    console.log('[AWS BOUNCES] Message Body missing in bounce');
-                }
             }
-            catch (e) {
-                console.log('[AWS BOUNCES] Bounce message processing failed:');
-                console.error('[AWS BOUNCES] Bounce message processing failed:');
-                console.error('[AWS BOUNCES]', e);
+        }
+        else {
+            console.error('[AWS COMPLAINTS] Unknown organization for email address ' + source);
+        }
+
+        if (type === 'virus' || type === 'fraud') {
+            console.error('[AWS COMPLAINTS] Received virus / fraud complaint!');
+            console.error('[AWS COMPLAINTS]', message.complaint);
+            if (STAMHOOFD.environment !== 'development') {
+                Email.sendWebmaster({
+                    subject: 'Received a ' + type + ' email notification',
+                    text: 'We received a ' + type + ' notification for an e-mail from the organization: ' + organization?.name + '. Please check and adjust if needed.\n',
+                });
             }
         }
     }
+    else {
+        console.log('[AWS COMPLAINTS] Missing complaint field');
+    }
 }
 
-async function checkReplies() {
-    if (STAMHOOFD.environment !== 'production' || !STAMHOOFD.AWS_ACCESS_KEY_ID) {
-        return;
+async function handleForward(message: any) {
+    if (message.mail && message.content && message.receipt) {
+        const content = message.content;
+        const receipt = message.receipt as {
+            recipients: string[];
+            spamVerdict: { status: 'PASS' | string };
+            virusVerdict: { status: 'PASS' | string };
+            spfVerdict: { status: 'PASS' | string };
+            dkimVerdict: { status: 'PASS' | string };
+            dmarcVerdict: { status: 'PASS' | string };
+        };
+
+        const options = await ForwardHandler.handle(content, receipt);
+        if (options) {
+            if (STAMHOOFD.environment !== 'development') {
+                Email.send(options);
+            }
+        }
+    }
+    else {
+        console.log('[AWS FORWARDING] Missing mail, content or receipt field');
     }
+}
 
-    console.log('Checking replies from AWS SQS');
+async function readFromQueue(queueUrl: string) {
+    console.log('[AWS Queue] Checking ' + queueUrl);
     const sqs = new AWS.SQS();
-    const messages = await sqs.receiveMessage({ QueueUrl: 'https://sqs.eu-west-1.amazonaws.com/118244293157/stamhoofd-email-forwarding', MaxNumberOfMessages: 10 }).promise();
+    const messages = await sqs.receiveMessage({ QueueUrl: queueUrl, MaxNumberOfMessages: 10 }).promise();
+    let didProcess = 0;
     if (messages.Messages) {
         for (const message of messages.Messages) {
-            console.log('Received message from forwarding queue');
+            console.log('[AWS Queue] Received message');
+            console.log('[AWS Queue]', message);
 
             if (message.ReceiptHandle) {
-                if (STAMHOOFD.environment === 'production') {
+                if (STAMHOOFD.environment !== 'development') {
                     await sqs.deleteMessage({
-                        QueueUrl: 'https://sqs.eu-west-1.amazonaws.com/118244293157/stamhoofd-email-forwarding',
+                        QueueUrl: queueUrl,
                         ReceiptHandle: message.ReceiptHandle,
                     }).promise();
-                    console.log('Deleted from queue');
+                    console.log('[AWS Queue] Deleted from queue');
                 }
             }
+            didProcess += 1;
 
             try {
                 if (message.Body) {
@@ -187,138 +231,82 @@ async function checkReplies() {
                     if (bounce.Message) {
                         const message = JSON.parse(bounce.Message);
 
+                        // Docs: https://docs.aws.amazon.com/ses/latest/dg/event-publishing-retrieving-sns-contents.html
+                        if (message.bounce) {
+                            await handleBounce(message);
+                        }
+                        else if (message.complaint) {
+                            await handleComplaint(message);
+                        }
                         // Read message content
-                        if (message.mail && message.content && message.receipt) {
-                            const content = message.content;
-                            const receipt = message.receipt as {
-                                recipients: string[];
-                                spamVerdict: { status: 'PASS' | string };
-                                virusVerdict: { status: 'PASS' | string };
-                                spfVerdict: { status: 'PASS' | string };
-                                dkimVerdict: { status: 'PASS' | string };
-                                dmarcVerdict: { status: 'PASS' | string };
-                            };
-
-                            const options = await ForwardHandler.handle(content, receipt);
-                            if (options) {
-                                if (STAMHOOFD.environment === 'production') {
-                                    Email.send(options);
-                                }
-                            }
+                        // https://docs.aws.amazon.com/ses/latest/dg/receiving-email-notifications-contents.html
+                        else if (message.mail && message.content && message.receipt) {
+                            await handleForward(message);
+                        }
+                        else {
+                            console.log('[AWS Queue] Unsupported message');
                         }
                     }
+                    else {
+                        console.log("[AWS Queue] 'Message' field missing");
+                    }
+                }
+                else {
+                    console.log('[AWS Queue] Message Body missing in bounce');
                 }
             }
             catch (e) {
-                console.error(e);
+                console.log('[AWS Queue] Message processing failed:');
+                console.log('[AWS Queue]', e);
+
+                console.error('[AWS Queue] Message processing failed:');
+                console.error('[AWS Queue]', e);
             }
         }
     }
+
+    if (didProcess) {
+        console.log(`[AWS Queue] Processed ${didProcess} message(s) from queue`);
+    }
+    else {
+        console.log(`[AWS Queue] No message to process from queue`);
+    }
+    return didProcess;
 }
 
-async function checkComplaints() {
-    if (STAMHOOFD.environment !== 'production' || !STAMHOOFD.AWS_ACCESS_KEY_ID) {
+async function readAllFromQueue(queueUrl: string) {
+    let readCount = 0;
+    while (readCount < 20) {
+        const didProcess = await readFromQueue(queueUrl);
+        if (!didProcess) {
+            break;
+        }
+        readCount += didProcess;
+    }
+    console.log(`[AWS Queue] Finished processing all messages from queue (${readCount} messages processed)`);
+    return true;
+}
+
+async function checkBounces() {
+    if (!STAMHOOFD.AWS_ACCESS_KEY_ID || !STAMHOOFD.AWS_BOUNCE_QUEUE_URL) {
         return;
     }
 
-    console.log('[AWS COMPLAINTS] Checking complaints from AWS SQS');
-    const sqs = new AWS.SQS();
-    const messages = await sqs.receiveMessage({ QueueUrl: 'https://sqs.eu-west-1.amazonaws.com/118244293157/stamhoofd-complaints-queue', MaxNumberOfMessages: 10 }).promise();
-    if (messages.Messages) {
-        for (const message of messages.Messages) {
-            console.log('[AWS COMPLAINTS] Received complaint message');
-            console.log('[AWS COMPLAINTS]', message);
+    await readAllFromQueue(STAMHOOFD.AWS_BOUNCE_QUEUE_URL);
+}
 
-            if (message.ReceiptHandle) {
-                if (STAMHOOFD.environment === 'production') {
-                    await sqs.deleteMessage({
-                        QueueUrl: 'https://sqs.eu-west-1.amazonaws.com/118244293157/stamhoofd-complaints-queue',
-                        ReceiptHandle: message.ReceiptHandle,
-                    }).promise();
-                    console.log('[AWS COMPLAINTS] Deleted from queue');
-                }
-            }
+async function checkReplies() {
+    if (!STAMHOOFD.AWS_ACCESS_KEY_ID || !STAMHOOFD.AWS_FORWARDING_QUEUE_URL) {
+        return;
+    }
 
-            try {
-                if (message.Body) {
-                    // decode the JSON value
-                    const complaint = JSON.parse(message.Body);
-                    console.log('[AWS COMPLAINTS]', complaint);
-
-                    if (complaint.Message) {
-                        const message = JSON.parse(complaint.Message);
-
-                        if (message.complaint) {
-                            const b = message.complaint;
-                            const source = message.mail.source;
-                            const organization: Organization | undefined = source ? await Organization.getByEmail(source) : undefined;
-
-                            const type: 'abuse' | 'auth-failure' | 'fraud' | 'not-spam' | 'other' | 'virus' = b.complaintFeedbackType;
-
-                            if (organization) {
-                                for (const recipient of b.complainedRecipients) {
-                                    const email = recipient.emailAddress;
-                                    const emailAddress = await EmailAddress.getOrCreate(email, organization.id);
-                                    emailAddress.markedAsSpam = type !== 'not-spam';
-                                    await emailAddress.save();
-
-                                    if (type !== 'not-spam') {
-                                        if (type === 'virus' || type === 'fraud') {
-                                            await saveLog({
-                                                id: b.feedbackId,
-                                                email: source,
-                                                organization,
-                                                type: AuditLogType.EmailAddressFraudComplaint,
-                                                subType: type || 'unknown',
-                                                sender: source,
-                                                response: b.diagnosticCode || '',
-                                                subject: message.mail.commonHeaders?.subject || '',
-                                            });
-                                        }
-                                        else {
-                                            await saveLog({
-                                                id: b.feedbackId,
-                                                email: source,
-                                                organization,
-                                                type: AuditLogType.EmailAddressMarkedAsSpam,
-                                                subType: type || 'unknown',
-                                                sender: source,
-                                                response: b.diagnosticCode || '',
-                                                subject: message.mail.commonHeaders?.subject || '',
-                                            });
-                                        }
-                                    }
-                                }
-                            }
-                            else {
-                                console.error('[AWS COMPLAINTS] Unknown organization for email address ' + source);
-                            }
-
-                            if (type === 'virus' || type === 'fraud') {
-                                console.error('[AWS COMPLAINTS] Received virus / fraud complaint!');
-                                console.error('[AWS COMPLAINTS]', complaint);
-                                if (STAMHOOFD.environment === 'production') {
-                                    Email.sendWebmaster({
-                                        subject: 'Received a ' + type + ' email notification',
-                                        text: 'We received a ' + type + ' notification for an e-mail from the organization: ' + organization?.name + '. Please check and adjust if needed.\n',
-                                    });
-                                }
-                            }
-                        }
-                        else {
-                            console.log('[AWS COMPLAINTS] Missing complaint field');
-                        }
-                    }
-                    else {
-                        console.log('[AWS COMPLAINTS] Missing message field in complaint');
-                    }
-                }
-            }
-            catch (e) {
-                console.log('[AWS COMPLAINTS] Complain message processing failed:');
-                console.error('[AWS COMPLAINTS] Complain message processing failed:');
-                console.error('[AWS COMPLAINTS]', e);
-            }
-        }
+    await readAllFromQueue(STAMHOOFD.AWS_FORWARDING_QUEUE_URL);
+}
+
+async function checkComplaints() {
+    if (!STAMHOOFD.AWS_ACCESS_KEY_ID || !STAMHOOFD.AWS_COMPLAINTS_QUEUE_URL) {
+        return;
     }
+
+    await readAllFromQueue(STAMHOOFD.AWS_COMPLAINTS_QUEUE_URL);
 }

package/src/crons/clearExcelCache.test.ts

@@ -4,7 +4,7 @@ import { clearExcelCacheHelper } from './clearExcelCache';
 
 const testPath = '/Users/user/project/backend/app/api/.cache';
 jest.mock('fs/promises');
-const fsMock = jest.mocked(fs, true);
+const fsMock = jest.mocked(fs, { shallow: true });
 
 describe('clearExcelCacheHelper', () => {
     it('should only run between 3 and 6 AM', async () => {