@stamhoofd/backend 2.85.4 → 2.87.0

package/.nvmrc

@@ -1 +1 @@
-20.12
+22

package/index.ts

@@ -1,237 +1,15 @@
 import backendEnv from '@stamhoofd/backend-env';
-backendEnv.load({ service: 'api' });
 
-import { Column, Database, Migration } from '@simonbackx/simple-database';
-import { CORSPreflightEndpoint, Router, RouterServer } from '@simonbackx/simple-endpoints';
-import { CORSMiddleware, LogMiddleware, VersionMiddleware } from '@stamhoofd/backend-middleware';
-import { Email } from '@stamhoofd/email';
-import { loadLogger } from '@stamhoofd/logging';
-import { Version } from '@stamhoofd/structures';
-import { sleep } from '@stamhoofd/utility';
-
-import { startCrons, stopCrons, waitForCrons } from '@stamhoofd/crons';
-import { Platform } from '@stamhoofd/models';
-import { resumeEmails } from './src/helpers/EmailResumer';
-import { GlobalHelper } from './src/helpers/GlobalHelper';
-import { SetupStepUpdater } from './src/helpers/SetupStepUpdater';
-import { ContextMiddleware } from './src/middleware/ContextMiddleware';
-import { AuditLogService } from './src/services/AuditLogService';
-import { BalanceItemService } from './src/services/BalanceItemService';
-import { DocumentService } from './src/services/DocumentService';
-import { FileSignService } from './src/services/FileSignService';
-import { PlatformMembershipService } from './src/services/PlatformMembershipService';
-import { UniqueUserService } from './src/services/UniqueUserService';
-import { QueueHandler } from '@stamhoofd/queues';
-import { SimpleError } from '@simonbackx/simple-errors';
-
-process.on('unhandledRejection', (error: Error) => {
-    console.error('unhandledRejection');
-    console.error(error.message, error.stack);
+backendEnv.load({ service: 'api' }).catch((error) => {
+    console.error('Failed to load environment:', error);
     process.exit(1);
-});
-
-// Set version of saved structures
-Column.setJSONVersion(Version);
-
-// Set timezone
-process.env.TZ = 'UTC';
-
-// Quick check
-if (new Date().getTimezoneOffset() !== 0) {
-    throw new Error('Process should always run in UTC timezone');
-}
-
-const seeds = async () => {
-    try {
-        // Internal
-        await AuditLogService.disable(async () => {
-            await Migration.runAll(__dirname + '/src/seeds');
-        });
-    }
-    catch (e) {
-        console.error('Failed to run seeds:');
-        console.error(e);
+}).then(async () => {
+    if (STAMHOOFD.environment === 'development') {
+        const { run } = await import('./src/migrate');
+        await run();
     }
-};
-const bootTime = process.hrtime();
-
-const start = async () => {
-    console.log('Running server at v' + Version);
-    loadLogger();
-
-    await GlobalHelper.load();
-    await UniqueUserService.check();
-
-    // Init platform shared struct: otherwise permissions won't work with missing responsibilities
-    console.log('Loading platform...');
-    await Platform.loadCaches();
-
-    const router = new Router();
-
-    console.log('Loading endpoints...');
-
-    // Note: we should load endpoints one by once to have a reliable order of url matching
-    await router.loadAllEndpoints(__dirname + '/src/endpoints/global/*');
-    await router.loadAllEndpoints(__dirname + '/src/endpoints/admin/*');
-    await router.loadAllEndpoints(__dirname + '/src/endpoints/auth');
-    await router.loadAllEndpoints(__dirname + '/src/endpoints/organization/dashboard/*');
-    await router.loadAllEndpoints(__dirname + '/src/endpoints/organization/registration');
-    await router.loadAllEndpoints(__dirname + '/src/endpoints/organization/webshops');
-    await router.loadAllEndpoints(__dirname + '/src/endpoints/organization/shared');
-    await router.loadAllEndpoints(__dirname + '/src/endpoints/organization/shared/*');
-
-    router.endpoints.push(new CORSPreflightEndpoint());
-
-    console.log('Creating router...');
-    const routerServer = new RouterServer(router);
-    routerServer.verbose = false;
-
-    // Log requests and errors
-    routerServer.addRequestMiddleware(LogMiddleware);
-    routerServer.addResponseMiddleware(LogMiddleware);
-
-    routerServer.addResponseMiddleware(FileSignService);
-    routerServer.addRequestMiddleware(FileSignService);
-
-    // Contexts
-    routerServer.addRequestMiddleware(ContextMiddleware);
-
-    // Add version headers and minimum version
-    const versionMiddleware = new VersionMiddleware({
-        latestVersions: {
-            android: STAMHOOFD.LATEST_ANDROID_VERSION,
-            ios: STAMHOOFD.LATEST_IOS_VERSION,
-            web: Version,
-        },
-        minimumVersion: 331,
-    });
-    routerServer.addRequestMiddleware(versionMiddleware);
-    routerServer.addResponseMiddleware(versionMiddleware);
-
-    // Add CORS headers
-    routerServer.addResponseMiddleware(CORSMiddleware);
-
-    console.log('Loading loaders...');
-
-    // Register Excel loaders
-    await import('./src/excel-loaders');
-
-    // Register Email Recipient loaders
-    await import('./src/email-recipient-loaders/members');
-    await import('./src/email-recipient-loaders/registrations');
-    await import('./src/email-recipient-loaders/orders');
-    await import('./src/email-recipient-loaders/receivable-balances');
-    await import('./src/excel-loaders/registrations');
-
-    console.log('Opening port...');
-    routerServer.listen(STAMHOOFD.PORT ?? 9090);
-
-    const hrend = process.hrtime(bootTime);
-    console.log('🟢 HTTP server started in ' + Math.ceil(hrend[0] * 1000 + hrend[1] / 1000000) + 'ms');
-
-    resumeEmails().catch(console.error);
-
-    if (routerServer.server) {
-        // Default timeout is a bit too short
-        routerServer.server.timeout = 61000;
-    }
-
-    let shuttingDown = false;
-    const shutdown = async () => {
-        if (shuttingDown) {
-            return;
-        }
-        shuttingDown = true;
-        console.log('Shutting down...');
-        // Disable keep alive
-        routerServer.defaultHeaders = Object.assign(routerServer.defaultHeaders, { Connection: 'close' });
-        if (routerServer.server) {
-            routerServer.server.headersTimeout = 5000;
-            routerServer.server.keepAliveTimeout = 1;
-        }
-
-        stopCrons();
-
-        if (STAMHOOFD.environment === 'development') {
-            setTimeout(() => {
-                console.error('Forcing exit after 5 seconds');
-                process.exit(1);
-            }, 5000);
-        }
-
-        try {
-            await routerServer.close();
-            console.log('HTTP server stopped');
-        }
-        catch (err) {
-            console.error('Failed to stop HTTP server:');
-            console.error(err);
-        }
-
-        await BalanceItemService.flushAll();
-        await waitForCrons();
-        QueueHandler.abortAll(
-            new SimpleError({
-                code: 'SHUTDOWN',
-                message: 'Shutting down',
-                statusCode: 503,
-            }),
-        );
-        await QueueHandler.awaitAll();
-
-        try {
-            while (Email.currentQueue.length > 0) {
-                console.log(`${Email.currentQueue.length} emails still in queue. Waiting 500ms...`);
-                await sleep(500);
-            }
-        }
-        catch (err) {
-            console.error('Failed to wait for emails to finish:');
-            console.error(err);
-        }
-
-        try {
-            await Database.end();
-            console.log('MySQL connections closed');
-        }
-        catch (err) {
-            console.error('Failed to close MySQL connection:');
-            console.error(err);
-        }
-
-        // Should not be needed, but added for security as sometimes a promise hangs somewhere
-        process.exit(0);
-    };
-
-    process.on('SIGTERM', () => {
-        console.info('SIGTERM signal received.');
-        shutdown().catch((e) => {
-            console.error(e);
-            process.exit(1);
-        });
-    });
-
-    process.on('SIGINT', () => {
-        console.info('SIGINT signal received.');
-        shutdown().catch((e) => {
-            console.error(e);
-            process.exit(1);
-        });
-    });
-
-    // Register crons
-    await import('./src/crons');
-
-    AuditLogService.listen();
-    PlatformMembershipService.listen();
-    DocumentService.listen();
-    SetupStepUpdater.listen();
-
-    startCrons();
-    seeds().catch(console.error);
-};
-
-start().catch((error) => {
-    console.error('unhandledRejection', error);
+    await import('./src/boot');
+}).catch((error) => {
+    console.error('Failed to start the API:', error);
     process.exit(1);
 });

package/migrations.ts

@@ -1,35 +1,13 @@
 import backendEnv from '@stamhoofd/backend-env';
-backendEnv.load();
 
-import { Column, Migration } from '@simonbackx/simple-database';
-import { Version } from '@stamhoofd/structures';
-import path from 'path';
-
-Column.setJSONVersion(Version);
-process.env.TZ = 'UTC';
-
-const emailPath = require.resolve('@stamhoofd/email');
-const modelsPath = require.resolve('@stamhoofd/models');
-
-// Validate UTC timezone
-if (new Date().getTimezoneOffset() !== 0) {
-    throw new Error('Process should always run in UTC timezone');
-}
-
-const start = async () => {
-    // External migrations
-    await Migration.runAll(path.dirname(modelsPath) + '/migrations');
-    await Migration.runAll(path.dirname(emailPath) + '/migrations');
-
-    // Internal
-    await Migration.runAll(__dirname + '/src/migrations');
-};
-
-start()
-    .catch((error) => {
-        console.error('unhandledRejection', error);
-        process.exit(1);
-    })
-    .finally(() => {
-        process.exit();
-    });
+backendEnv.load({ service: 'api' }).catch((error) => {
+    console.error('Failed to load environment:', error);
+    process.exit(1);
+}).then(async () => {
+    const { run } = await import('./src/migrate');
+    await run();
+    process.exit(0);
+}).catch((error) => {
+    console.error('Failed to run migrations:', error);
+    process.exit(1);
+});

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@stamhoofd/backend",
-    "version": "2.85.4",
+    "version": "2.87.0",
     "main": "./dist/index.js",
     "exports": {
         ".": {
@@ -9,24 +9,25 @@
     },
     "license": "UNLICENCED",
     "scripts": {
-        "dev": "echo 'Waiting for shared backend packages...' && wait-on ../../shared/middleware/dist/index.js && echo 'Start building backend API' && concurrently -r 'yarn build --watch --preserveWatchOutput' \"wait-on ./dist/index.js && nodemon --quiet --inspect=5858 --watch dist --watch '../../../shared/*/dist/' --watch '../../shared/*/dist/' --ext .ts,.json,.sql,.js --watch .env.json --delay 1000ms --exec 'node --enable-source-maps ./dist/index.js' --signal SIGTERM\"",
-        "dev:backend": "yarn dev",
+        "dev": "wait-on ../../shared/middleware/dist/index.js && concurrently -r 'yarn -s build --watch --preserveWatchOutput' \"yarn -s dev:watch\"",
+        "dev:watch": "wait-on ./dist/index.js && nodemon --quiet --inspect=5858 --watch dist --watch '../../../shared/*/dist/' --watch '../../shared/*/dist/' --ext .ts,.json,.sql,.js --delay 2000ms --exec 'node --enable-source-maps ./dist/index.js' --signal SIGTERM",
+        "dev:backend": "yarn -s dev",
         "build": "rm -rf ./dist/src/migrations && rm -rf ./dist/src/seeds && tsc -b",
-        "build:full": "yarn clear && yarn build",
+        "build:full": "yarn -s clear && yarn -s build",
         "clear": "rm -rf ./dist",
-        "start": "yarn build && node --enable-source-maps ./dist/index.js",
+        "start": "yarn -s build && node --enable-source-maps ./dist/index.js",
         "test": "jest --runInBand",
         "test:build": "yarn -s build:full && yarn -s test",
-        "test:reset": "yarn build:full && jest --runInBand",
-        "migrations": "yarn build:full && node ./dist/migrations.js",
+        "test:reset": "yarn -s build:full && jest --runInBand",
+        "migrations": "yarn -s build:full && node ./dist/migrations.js",
         "lint": "eslint"
     },
     "devDependencies": {
-        "@types/cookie": "^0.5.1",
+        "@types/cookie": "^0.6.0",
         "@types/luxon": "3.4.2",
         "@types/mailparser": "3.4.4",
         "@types/mysql": "^2.15.20",
-        "@types/node": "^20.12",
+        "@types/node": "^22",
         "nock": "^13.5.1",
         "qs": "^6.11.2",
         "sinon": "^18.0.0"
@@ -39,22 +40,22 @@
         "@aws-sdk/s3-request-presigner": "3.823.0",
         "@bwip-js/node": "^4.5.1",
         "@mollie/api-client": "3.7.0",
-        "@simonbackx/simple-database": "1.32.0",
+        "@simonbackx/simple-database": "1.33.0",
         "@simonbackx/simple-encoding": "2.22.0",
         "@simonbackx/simple-endpoints": "1.20.1",
         "@simonbackx/simple-logging": "^1.0.1",
-        "@stamhoofd/backend-i18n": "2.85.4",
-        "@stamhoofd/backend-middleware": "2.85.4",
-        "@stamhoofd/email": "2.85.4",
-        "@stamhoofd/models": "2.85.4",
-        "@stamhoofd/queues": "2.85.4",
-        "@stamhoofd/sql": "2.85.4",
-        "@stamhoofd/structures": "2.85.4",
-        "@stamhoofd/utility": "2.85.4",
+        "@stamhoofd/backend-i18n": "2.87.0",
+        "@stamhoofd/backend-middleware": "2.87.0",
+        "@stamhoofd/email": "2.87.0",
+        "@stamhoofd/models": "2.87.0",
+        "@stamhoofd/queues": "2.87.0",
+        "@stamhoofd/sql": "2.87.0",
+        "@stamhoofd/structures": "2.87.0",
+        "@stamhoofd/utility": "2.87.0",
         "archiver": "^7.0.1",
-        "axios": "1.6.8",
-        "cookie": "^0.5.0",
-        "formidable": "3.5.1",
+        "axios": "^1.8.2",
+        "cookie": "^0.7.0",
+        "formidable": "3.5.4",
         "handlebars": "^4.7.7",
         "jsonwebtoken": "9.0.0",
         "luxon": "3.4.4",
@@ -63,11 +64,10 @@
         "mysql2": "^3.14.1",
         "node-rsa": "1.1.1",
         "openid-client": "^5.4.0",
-        "postmark": "^4.0.5",
         "stripe": "^16.6.0"
     },
     "publishConfig": {
         "access": "public"
     },
-    "gitHead": "ecceacc395d167a6761f1425e77cfc06a31fe8a9"
+    "gitHead": "372713e1e4b626d6b4ce4f3d148f14948528312e"
 }

package/src/boot.ts

@@ -0,0 +1,240 @@
+import { Column, Database, Migration } from '@simonbackx/simple-database';
+import { CORSPreflightEndpoint, Router, RouterServer } from '@simonbackx/simple-endpoints';
+import { CORSMiddleware, LogMiddleware, VersionMiddleware } from '@stamhoofd/backend-middleware';
+import { Email } from '@stamhoofd/email';
+import { loadLogger } from '@stamhoofd/logging';
+import { Version } from '@stamhoofd/structures';
+import { sleep } from '@stamhoofd/utility';
+
+import { startCrons, stopCrons, waitForCrons } from '@stamhoofd/crons';
+import { Platform } from '@stamhoofd/models';
+import { resumeEmails } from './helpers/EmailResumer';
+import { GlobalHelper } from './helpers/GlobalHelper';
+import { SetupStepUpdater } from './helpers/SetupStepUpdater';
+import { ContextMiddleware } from './middleware/ContextMiddleware';
+import { AuditLogService } from './services/AuditLogService';
+import { BalanceItemService } from './services/BalanceItemService';
+import { DocumentService } from './services/DocumentService';
+import { FileSignService } from './services/FileSignService';
+import { PlatformMembershipService } from './services/PlatformMembershipService';
+import { UniqueUserService } from './services/UniqueUserService';
+import { QueueHandler } from '@stamhoofd/queues';
+import { SimpleError } from '@simonbackx/simple-errors';
+
+process.on('unhandledRejection', (error: Error) => {
+    console.error('unhandledRejection');
+    console.error(error.message, error.stack);
+    process.exit(1);
+});
+
+// Set version of saved structures
+Column.setJSONVersion(Version);
+
+// Set timezone
+process.env.TZ = 'UTC';
+
+// Quick check
+if (new Date().getTimezoneOffset() !== 0) {
+    throw new Error('Process should always run in UTC timezone');
+}
+
+const seeds = async () => {
+    try {
+        // Internal
+        await AuditLogService.disable(async () => {
+            await Migration.runAll(__dirname + '/seeds');
+        });
+    }
+    catch (e) {
+        console.error('Failed to run seeds:');
+        console.error(e);
+    }
+};
+const bootTime = process.hrtime();
+
+function productionLog(message: string) {
+    if (STAMHOOFD.environment !== 'development') {
+        console.log(message);
+    }
+}
+
+const start = async () => {
+    productionLog('Running server at v' + Version);
+    loadLogger();
+
+    await GlobalHelper.load();
+    await UniqueUserService.check();
+
+    // Init platform shared struct: otherwise permissions won't work with missing responsibilities
+    productionLog('Loading platform...');
+    await Platform.loadCaches();
+
+    const router = new Router();
+
+    productionLog('Loading endpoints...');
+
+    // Note: we should load endpoints one by once to have a reliable order of url matching
+    await router.loadAllEndpoints(__dirname + '/endpoints/global/*');
+    await router.loadAllEndpoints(__dirname + '/endpoints/admin/*');
+    await router.loadAllEndpoints(__dirname + '/endpoints/auth');
+    await router.loadAllEndpoints(__dirname + '/endpoints/organization/dashboard/*');
+    await router.loadAllEndpoints(__dirname + '/endpoints/organization/registration');
+    await router.loadAllEndpoints(__dirname + '/endpoints/organization/webshops');
+    await router.loadAllEndpoints(__dirname + '/endpoints/organization/shared');
+    await router.loadAllEndpoints(__dirname + '/endpoints/organization/shared/*');
+
+    router.endpoints.push(new CORSPreflightEndpoint());
+
+    productionLog('Creating router...');
+    const routerServer = new RouterServer(router);
+    routerServer.verbose = false;
+
+    // Log requests and errors
+    routerServer.addRequestMiddleware(LogMiddleware);
+    routerServer.addResponseMiddleware(LogMiddleware);
+
+    routerServer.addResponseMiddleware(FileSignService);
+    routerServer.addRequestMiddleware(FileSignService);
+
+    // Contexts
+    routerServer.addRequestMiddleware(ContextMiddleware);
+
+    // Add version headers and minimum version
+    const versionMiddleware = new VersionMiddleware({
+        latestVersions: {
+            android: STAMHOOFD.LATEST_ANDROID_VERSION,
+            ios: STAMHOOFD.LATEST_IOS_VERSION,
+            web: Version,
+        },
+        minimumVersion: 331,
+    });
+    routerServer.addRequestMiddleware(versionMiddleware);
+    routerServer.addResponseMiddleware(versionMiddleware);
+
+    // Add CORS headers
+    routerServer.addResponseMiddleware(CORSMiddleware);
+
+    productionLog('Loading loaders...');
+
+    // Register Excel loaders
+    await import('./excel-loaders');
+
+    // Register Email Recipient loaders
+    await import('./email-recipient-loaders/members');
+    await import('./email-recipient-loaders/registrations');
+    await import('./email-recipient-loaders/orders');
+    await import('./email-recipient-loaders/receivable-balances');
+    await import('./excel-loaders/registrations');
+
+    productionLog('Opening port...');
+    routerServer.listen(STAMHOOFD.PORT ?? 9090);
+
+    const hrend = process.hrtime(bootTime);
+    productionLog('🟢 HTTP server started in ' + Math.ceil(hrend[0] * 1000 + hrend[1] / 1000000) + 'ms');
+
+    resumeEmails().catch(console.error);
+
+    if (routerServer.server) {
+        // Default timeout is a bit too short
+        routerServer.server.timeout = 61000;
+    }
+
+    let shuttingDown = false;
+    const shutdown = async () => {
+        if (shuttingDown) {
+            return;
+        }
+        shuttingDown = true;
+        productionLog('Shutting down...');
+        // Disable keep alive
+        routerServer.defaultHeaders = Object.assign(routerServer.defaultHeaders, { Connection: 'close' });
+        if (routerServer.server) {
+            routerServer.server.headersTimeout = 5000;
+            routerServer.server.keepAliveTimeout = 1;
+        }
+
+        stopCrons();
+
+        if (STAMHOOFD.environment === 'development') {
+            setTimeout(() => {
+                console.error('Forcing exit after 5 seconds');
+                process.exit(1);
+            }, 5000);
+        }
+
+        try {
+            await routerServer.close();
+            productionLog('HTTP server stopped');
+        }
+        catch (err) {
+            console.error('Failed to stop HTTP server:');
+            console.error(err);
+        }
+
+        await BalanceItemService.flushAll();
+        await waitForCrons();
+        QueueHandler.abortAll(
+            new SimpleError({
+                code: 'SHUTDOWN',
+                message: 'Shutting down',
+                statusCode: 503,
+            }),
+        );
+        await QueueHandler.awaitAll();
+
+        try {
+            while (Email.currentQueue.length > 0) {
+                console.log(`${Email.currentQueue.length} emails still in queue. Waiting 500ms...`);
+                await sleep(500);
+            }
+        }
+        catch (err) {
+            console.error('Failed to wait for emails to finish:');
+            console.error(err);
+        }
+
+        try {
+            await Database.end();
+            productionLog('MySQL connections closed');
+        }
+        catch (err) {
+            console.error('Failed to close MySQL connection:');
+            console.error(err);
+        }
+
+        // Should not be needed, but added for security as sometimes a promise hangs somewhere
+        process.exit(0);
+    };
+
+    process.on('SIGTERM', () => {
+        productionLog('SIGTERM signal received.');
+        shutdown().catch((e) => {
+            console.error(e);
+            process.exit(1);
+        });
+    });
+
+    process.on('SIGINT', () => {
+        productionLog('SIGINT signal received.');
+        shutdown().catch((e) => {
+            console.error(e);
+            process.exit(1);
+        });
+    });
+
+    // Register crons
+    await import('./crons');
+
+    AuditLogService.listen();
+    PlatformMembershipService.listen();
+    DocumentService.listen();
+    SetupStepUpdater.listen();
+
+    startCrons();
+    seeds().catch(console.error);
+};
+
+start().catch((error) => {
+    console.error('unhandledRejection', error);
+    process.exit(1);
+});

package/src/crons/clearExcelCache.test.ts

@@ -131,7 +131,10 @@ describe('clearExcelCacheHelper', () => {
             dir.isDirectory = () => true;
         });
 
-        fsMock.readdir.mockReturnValue(Promise.resolve([...directories, file1]));
+        fsMock.readdir.mockReturnValue(
+            // eslint-disable-next-line @typescript-eslint/no-unsafe-argument
+            Promise.resolve([...directories, file1]) as any,
+        );
         // #endregion
 
         // act

package/src/crons/delete-old-email-drafts.ts

@@ -0,0 +1,37 @@
+import { registerCron } from '@stamhoofd/crons';
+import { Email } from '@stamhoofd/models';
+import { EmailStatus } from '@stamhoofd/structures';
+
+let lastRunDate: number | null = null;
+
+registerCron('deleteOldEmailDrafts', deleteOldEmailDrafts);
+
+/**
+ * Run every night at 5 AM.
+ */
+export async function deleteOldEmailDrafts() {
+    const now = new Date();
+
+    if (now.getDate() === lastRunDate) {
+        return;
+    }
+
+    const hour = now.getHours();
+
+    // between 5 and 6 AM
+    if (hour !== 5 && STAMHOOFD.environment !== 'development') {
+        return;
+    }
+
+    // Clear old email drafts older than 7 days
+    const sevenDaysAgo = new Date(now.getTime() - 7 * 24 * 60 * 60 * 1000);
+    const result = await Email.delete()
+        .where('status', EmailStatus.Draft)
+        .where('createdAt', '<', sevenDaysAgo)
+        .where('updatedAt', '<', sevenDaysAgo)
+        .where('sentAt', null);
+
+    console.log(`Deleted ${result.affectedRows} old email drafts.`);
+
+    lastRunDate = now.getDate();
+}