@stamhoofd/backend 2.83.5 → 2.84.1

package/index.ts

@@ -16,10 +16,13 @@ import { GlobalHelper } from './src/helpers/GlobalHelper';
 import { SetupStepUpdater } from './src/helpers/SetupStepUpdater';
 import { ContextMiddleware } from './src/middleware/ContextMiddleware';
 import { AuditLogService } from './src/services/AuditLogService';
+import { BalanceItemService } from './src/services/BalanceItemService';
 import { DocumentService } from './src/services/DocumentService';
 import { FileSignService } from './src/services/FileSignService';
 import { PlatformMembershipService } from './src/services/PlatformMembershipService';
 import { UniqueUserService } from './src/services/UniqueUserService';
+import { QueueHandler } from '@stamhoofd/queues';
+import { SimpleError } from '@simonbackx/simple-errors';
 
 process.on('unhandledRejection', (error: Error) => {
     console.error('unhandledRejection');
@@ -115,8 +118,10 @@ const start = async () => {
 
     // Register Email Recipient loaders
     await import('./src/email-recipient-loaders/members');
+    await import('./src/email-recipient-loaders/registrations');
     await import('./src/email-recipient-loaders/orders');
     await import('./src/email-recipient-loaders/receivable-balances');
+    await import('./src/excel-loaders/registrations');
 
     console.log('Opening port...');
     routerServer.listen(STAMHOOFD.PORT ?? 9090);
@@ -163,12 +168,21 @@ const start = async () => {
             console.error(err);
         }
 
+        await BalanceItemService.flushAll();
         await waitForCrons();
+        QueueHandler.abortAll(
+            new SimpleError({
+                code: 'SHUTDOWN',
+                message: 'Shutting down',
+                statusCode: 503,
+            }),
+        );
+        await QueueHandler.awaitAll();
 
         try {
             while (Email.currentQueue.length > 0) {
-                console.log('Emails still in queue. Waiting 2 seconds...');
-                await sleep(2000);
+                console.log(`${Email.currentQueue.length} emails still in queue. Waiting 500ms...`);
+                await sleep(500);
             }
         }
         catch (err) {
@@ -207,13 +221,14 @@ const start = async () => {
 
     // Register crons
     await import('./src/crons');
-    startCrons();
-    seeds().catch(console.error);
 
     AuditLogService.listen();
     PlatformMembershipService.listen();
     DocumentService.listen();
     SetupStepUpdater.listen();
+
+    startCrons();
+    seeds().catch(console.error);
 };
 
 start().catch((error) => {

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@stamhoofd/backend",
-    "version": "2.83.5",
+    "version": "2.84.1",
     "main": "./dist/index.js",
     "exports": {
         ".": {
@@ -32,22 +32,26 @@
         "sinon": "^18.0.0"
     },
     "dependencies": {
+        "@aws-sdk/client-s3": "3.823.0",
+        "@aws-sdk/client-ses": "3.823.0",
+        "@aws-sdk/client-sesv2": "3.823.0",
+        "@aws-sdk/client-sqs": "3.823.0",
+        "@aws-sdk/s3-request-presigner": "3.823.0",
         "@bwip-js/node": "^4.5.1",
         "@mollie/api-client": "3.7.0",
-        "@simonbackx/simple-database": "1.29.1",
+        "@simonbackx/simple-database": "1.32.0",
         "@simonbackx/simple-encoding": "2.22.0",
-        "@simonbackx/simple-endpoints": "1.19.1",
+        "@simonbackx/simple-endpoints": "1.20.1",
         "@simonbackx/simple-logging": "^1.0.1",
-        "@stamhoofd/backend-i18n": "2.83.5",
-        "@stamhoofd/backend-middleware": "2.83.5",
-        "@stamhoofd/email": "2.83.5",
-        "@stamhoofd/models": "2.83.5",
-        "@stamhoofd/queues": "2.83.5",
-        "@stamhoofd/sql": "2.83.5",
-        "@stamhoofd/structures": "2.83.5",
-        "@stamhoofd/utility": "2.83.5",
+        "@stamhoofd/backend-i18n": "2.84.1",
+        "@stamhoofd/backend-middleware": "2.84.1",
+        "@stamhoofd/email": "2.84.1",
+        "@stamhoofd/models": "2.84.1",
+        "@stamhoofd/queues": "2.84.1",
+        "@stamhoofd/sql": "2.84.1",
+        "@stamhoofd/structures": "2.84.1",
+        "@stamhoofd/utility": "2.84.1",
         "archiver": "^7.0.1",
-        "aws-sdk": "^2.885.0",
         "axios": "1.6.8",
         "cookie": "^0.5.0",
         "formidable": "3.5.1",
@@ -56,7 +60,7 @@
         "luxon": "3.4.4",
         "mailparser": "3.7.0",
         "mockdate": "^3.0.2",
-        "mysql": "^2.18.1",
+        "mysql2": "^3.14.1",
         "node-rsa": "1.1.1",
         "openid-client": "^5.4.0",
         "postmark": "^4.0.5",
@@ -65,5 +69,5 @@
     "publishConfig": {
         "access": "public"
     },
-    "gitHead": "6ba039bba88e801583188774a254c61e5ab28922"
+    "gitHead": "59df29b75109b599d5b90af17b0f69a3089b9d7e"
 }

package/src/crons/amazon-ses.ts

@@ -1,9 +1,13 @@
 /* eslint-disable @typescript-eslint/no-unsafe-argument */
+import {
+    DeleteMessageCommand,
+    ReceiveMessageCommand,
+    SQSClient,
+} from '@aws-sdk/client-sqs';
 import { registerCron } from '@stamhoofd/crons';
 import { Email, EmailAddress } from '@stamhoofd/email';
 import { AuditLog, Organization } from '@stamhoofd/models';
 import { AuditLogReplacement, AuditLogReplacementType, AuditLogSource, AuditLogType } from '@stamhoofd/structures';
-import AWS from 'aws-sdk';
 import { ForwardHandler } from '../helpers/ForwardHandler';
 
 registerCron('checkComplaints', checkComplaints);
@@ -192,10 +196,25 @@ async function handleForward(message: any) {
     }
 }
 
+let sharedClient: SQSClient | null = null;
+function getClient() {
+    if (!sharedClient) {
+        sharedClient = new SQSClient({});
+    }
+    return sharedClient;
+}
+
 async function readFromQueue(queueUrl: string) {
     console.log('[AWS Queue] Checking ' + queueUrl);
-    const sqs = new AWS.SQS();
-    const messages = await sqs.receiveMessage({ QueueUrl: queueUrl, MaxNumberOfMessages: 10 }).promise();
+
+    const client = getClient();
+
+    const cmd = new ReceiveMessageCommand({
+        MaxNumberOfMessages: 10,
+        QueueUrl: queueUrl,
+    });
+
+    const messages = await client.send(cmd);
     let didProcess = 0;
     if (messages.Messages) {
         for (const message of messages.Messages) {
@@ -204,10 +223,12 @@ async function readFromQueue(queueUrl: string) {
 
             if (message.ReceiptHandle) {
                 if (STAMHOOFD.environment !== 'development') {
-                    await sqs.deleteMessage({
+                    const deleteCmd = new DeleteMessageCommand({
                         QueueUrl: queueUrl,
                         ReceiptHandle: message.ReceiptHandle,
-                    }).promise();
+                    });
+
+                    await client.send(deleteCmd);
                     console.log('[AWS Queue] Deleted from queue');
                 }
             }

package/src/crons/balance-emails.ts

@@ -61,8 +61,13 @@ async function balanceEmails() {
             systemUser,
             templateType: EmailTemplateType.UserBalanceIncreaseNotification,
             filter: {
-                reminderAmountIncreased: true,
-                reminderEmailCount: 0,
+                $or: [
+                    // The amount has increased since the last reminder
+                    { reminderAmountIncreased: true },
+
+                    // Or we didn't send a reminder at all yet (since the last time it was zero)
+                    { reminderEmailCount: 0 },
+                ],
             },
         });
 
@@ -74,8 +79,13 @@ async function balanceEmails() {
                 systemUser,
                 templateType: EmailTemplateType.OrganizationBalanceIncreaseNotification,
                 filter: {
-                    reminderAmountIncreased: true,
-                    reminderEmailCount: 0,
+                    $or: [
+                        // The amount has increased since the last reminder
+                        { reminderAmountIncreased: true },
+
+                        // Or we didn't send a reminder at all yet (since the last time it was zero)
+                        { reminderEmailCount: 0 },
+                    ],
                 },
                 subfilter: organization.privateMeta.balanceNotificationSettings.organizationContactsFilter,
             });
@@ -147,7 +157,10 @@ async function sendTemplate({
     subfilter?: StamhoofdFilter;
 }) {
     // Do not send to persons that already received a similar email before this date
-    const weekAgo = new Date(new Date().getTime() - 1000 * 60 * 60 * 24 * organization.privateMeta.balanceNotificationSettings.minimumDaysBetween); // 5 instead of 7 so the email received is on another working day
+    const weekAgo = new Date(
+        new Date().getTime() - 1000 * 60 * 60 * 24 * Math.max(1, organization.privateMeta.balanceNotificationSettings.minimumDaysBetween)
+            + 12 * 1000 * 60 * 60, // Add a half day offset so we don't get trapped in small differences in time of sending
+    ); // 5 instead of 7 so the email received is on another working day
 
     const model = new Email();
     model.userId = null; // This is a system e-mail
@@ -172,18 +185,6 @@ async function sendTemplate({
                             ],
                         },
                         filter,
-                        /* {
-                            // Do not send if already received any email very recently
-                            $not: {
-                                emails: {
-                                    $elemMatch: {
-                                        sentAt: {
-                                            $gt: weekAgo,
-                                        },
-                                    },
-                                },
-                            },
-                        }, */
                     ],
 
                 },

package/src/email-recipient-loaders/registrations.ts

@@ -0,0 +1,87 @@
+import { Email, Member } from '@stamhoofd/models';
+import { SQL } from '@stamhoofd/sql';
+import { EmailRecipient, EmailRecipientFilterType, LimitedFilteredRequest, PaginatedResponse, RegistrationsBlob, mergeFilters } from '@stamhoofd/structures';
+import { GetRegistrationsEndpoint } from '../endpoints/global/registration/GetRegistrationsEndpoint';
+import { memberJoin } from '../sql-filters/registrations';
+
+async function getRecipients(result: PaginatedResponse<RegistrationsBlob, LimitedFilteredRequest>, type: 'member' | 'parents' | 'unverified') {
+    const recipients: EmailRecipient[] = [];
+
+    for (const registration of result.results.registrations) {
+        const memberRecipients = registration.member.getEmailRecipients([type]);
+        recipients.push(...memberRecipients);
+    }
+    return recipients;
+}
+
+Email.recipientLoaders.set(EmailRecipientFilterType.RegistrationMembers, {
+    fetch: async (query: LimitedFilteredRequest) => {
+        const result = await GetRegistrationsEndpoint.buildData(query);
+
+        return new PaginatedResponse({
+            results: await getRecipients(result, 'member'),
+            next: result.next,
+        });
+    },
+
+    count: async (query: LimitedFilteredRequest) => {
+        query.filter = mergeFilters([query.filter, {
+            member: {
+                $elemMatch: {
+                    $and: [
+                        {
+                            email: {
+                                $neq: null,
+                            },
+                        },
+                        {
+                            email: {
+                                $neq: '',
+                            },
+                        },
+                    ],
+                },
+            },
+        }]);
+        const q = await GetRegistrationsEndpoint.buildQuery(query);
+        return await q.count();
+    },
+});
+
+Email.recipientLoaders.set(EmailRecipientFilterType.RegistrationParents, {
+    fetch: async (query: LimitedFilteredRequest) => {
+        const result = await GetRegistrationsEndpoint.buildData(query);
+
+        return new PaginatedResponse({
+            results: await getRecipients(result, 'parents'),
+            next: result.next,
+        });
+    },
+
+    count: async (query: LimitedFilteredRequest) => {
+        const q = (await GetRegistrationsEndpoint.buildQuery(query)).join(memberJoin);
+
+        return await q.sum(
+            SQL.jsonLength(SQL.column(Member.table, 'details'), '$.value.parents[*].email'),
+        );
+    },
+});
+
+Email.recipientLoaders.set(EmailRecipientFilterType.RegistrationUnverified, {
+    fetch: async (query: LimitedFilteredRequest) => {
+        const result = await GetRegistrationsEndpoint.buildData(query);
+
+        return new PaginatedResponse({
+            results: await getRecipients(result, 'unverified'),
+            next: result.next,
+        });
+    },
+
+    count: async (query: LimitedFilteredRequest) => {
+        const q = (await GetRegistrationsEndpoint.buildQuery(query)).join(memberJoin);
+
+        return await q.sum(
+            SQL.jsonLength(SQL.column('details'), '$.value.unverifiedEmails'),
+        );
+    },
+});

package/src/endpoints/global/addresses/SearchRegionsEndpoint.ts

@@ -1,7 +1,6 @@
 import { AutoEncoder, Decoder, field, StringDecoder } from '@simonbackx/simple-encoding';
 import { DecodedRequest, Endpoint, Request, Response } from '@simonbackx/simple-endpoints';
-import { City } from '@stamhoofd/models';
-import { Province } from '@stamhoofd/models';
+import { City, Province } from '@stamhoofd/models';
 import { City as CityStruct, Country, Province as ProvinceStruct, SearchRegions } from '@stamhoofd/structures';
 import { StringCompare } from '@stamhoofd/utility';
 
@@ -71,6 +70,10 @@ export class SearchRegionsEndpoint extends Endpoint<Params, Query, Body, Respons
         const cities = await City.where({ name: match }, {
             limit: 5,
             sort: [
+                {
+                    column: { name: rawQuery },
+                    direction: 'DESC',
+                },
                 {
                     column: { name: match },
                     direction: 'DESC',

package/src/endpoints/global/email/PatchEmailEndpoint.test.ts

@@ -1,9 +1,9 @@
 import { Request } from '@simonbackx/simple-endpoints';
-import { Email, Organization, OrganizationFactory, RegistrationPeriod, RegistrationPeriodFactory, Token, User, UserFactory } from "@stamhoofd/models";
-import { EmailStatus, Email as EmailStruct, PermissionLevel, Permissions, Version } from "@stamhoofd/structures";
-import { TestUtils } from "@stamhoofd/test-utils";
-import { testServer } from "../../../../tests/helpers/TestServer";
-import { PatchEmailEndpoint } from "./PatchEmailEndpoint";
+import { Email, Organization, OrganizationFactory, RegistrationPeriod, RegistrationPeriodFactory, Token, User, UserFactory } from '@stamhoofd/models';
+import { EmailStatus, Email as EmailStruct, PermissionLevel, Permissions, Version } from '@stamhoofd/structures';
+import { TestUtils } from '@stamhoofd/test-utils';
+import { testServer } from '../../../../tests/helpers/TestServer';
+import { PatchEmailEndpoint } from './PatchEmailEndpoint';
 
 const baseUrl = `/v${Version}/email`;
 
@@ -32,7 +32,7 @@ describe('Endpoint.PatchEmailEndpoint', () => {
         }).create();
 
         organization = await new OrganizationFactory({ period })
-        .create();
+            .create();
 
         user = await new UserFactory({
             organization,
@@ -40,7 +40,7 @@ describe('Endpoint.PatchEmailEndpoint', () => {
                 level: PermissionLevel.Read,
             }),
         })
-        .create();
+            .create();
 
         token = await Token.createToken(user);
     });
@@ -50,7 +50,7 @@ describe('Endpoint.PatchEmailEndpoint', () => {
         email.subject = 'test subject';
         email.status = EmailStatus.Draft;
         email.text = 'test email {{unsubscribeUrl}}';
-        email.html =  `<!DOCTYPE html>
+        email.html = `<!DOCTYPE html>
         <html>
 
         <head>
@@ -66,37 +66,37 @@ describe('Endpoint.PatchEmailEndpoint', () => {
 
         </html>`;
         email.json = {
-            "content": [
-              {
-                "content": [
-                  {
-                    "text": "test email",
-                    "type": "text"
-                  }
-                ],
-                "type": "paragraph"
-              }
+            content: [
+                {
+                    content: [
+                        {
+                            text: 'test email',
+                            type: 'text',
+                        },
+                    ],
+                    type: 'paragraph',
+                },
             ],
-            "type": "doc"
+            type: 'doc',
         };
         email.userId = user.id;
         email.organizationId = organization.id;
 
         await email.save();
 
-        const body = EmailStruct.create({...email, fromAddress:'test@test.be', status: EmailStatus.Sending})
+        const body = EmailStruct.create({ ...email, fromAddress: 'test@test.be', status: EmailStatus.Sending });
 
         await expect(async () => await patchEmail(body, token, organization))
-        .rejects
-        .toThrow('Missing unsubscribe button');
-    })
+            .rejects
+            .toThrow('Missing unsubscribe button');
+    });
 
     test('Should throw error if no unsubscribe button in email text', async () => {
         const email = new Email();
         email.subject = 'test subject';
         email.status = EmailStatus.Draft;
         email.text = 'test email';
-        email.html =  `<!DOCTYPE html>
+        email.html = `<!DOCTYPE html>
         <html>
 
         <head>
@@ -112,28 +112,28 @@ describe('Endpoint.PatchEmailEndpoint', () => {
 
         </html>`;
         email.json = {
-            "content": [
-              {
-                "content": [
-                  {
-                    "text": "test email",
-                    "type": "text"
-                  }
-                ],
-                "type": "paragraph"
-              }
+            content: [
+                {
+                    content: [
+                        {
+                            text: 'test email',
+                            type: 'text',
+                        },
+                    ],
+                    type: 'paragraph',
+                },
             ],
-            "type": "doc"
+            type: 'doc',
         };
         email.userId = user.id;
         email.organizationId = organization.id;
 
         await email.save();
 
-        const body = EmailStruct.create({...email, fromAddress:'test@test.be', status: EmailStatus.Sending})
+        const body = EmailStruct.create({ ...email, fromAddress: 'test@test.be', status: EmailStatus.Sending });
 
         await expect(async () => await patchEmail(body, token, organization))
-        .rejects
-        .toThrow('Missing unsubscribe button');
-    })
-})
+            .rejects
+            .toThrow('Missing unsubscribe button');
+    });
+});