@stamhoofd/backend 2.137.1 → 2.137.3

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@stamhoofd/backend",
-    "version": "2.137.1",
+    "version": "2.137.3",
     "type": "module",
     "main": "./dist/index.js",
     "exports": {
@@ -51,29 +51,29 @@
         "@aws-sdk/client-sesv2": "3.1090.0",
         "@aws-sdk/client-sqs": "3.1090.0",
         "@aws-sdk/s3-request-presigner": "3.1090.0",
-        "@bwip-js/node": "4.11.1",
+        "@bwip-js/node": "4.11.2",
         "@mollie/api-client": "4.6.0",
         "@simonbackx/simple-database": "1.37.1",
         "@simonbackx/simple-encoding": "2.26.11",
-        "@simonbackx/simple-endpoints": "1.21.1",
+        "@simonbackx/simple-endpoints": "1.22.0",
         "@simonbackx/simple-errors": "1.5.0",
         "@simonbackx/simple-logging": "1.0.1",
-        "@stamhoofd/backend-env": "2.137.1",
-        "@stamhoofd/backend-i18n": "2.137.1",
-        "@stamhoofd/backend-middleware": "2.137.1",
-        "@stamhoofd/crons": "2.137.1",
-        "@stamhoofd/email": "2.137.1",
-        "@stamhoofd/excel-writer": "2.137.1",
-        "@stamhoofd/logging": "2.137.1",
-        "@stamhoofd/models": "2.137.1",
-        "@stamhoofd/object-differ": "2.137.1",
-        "@stamhoofd/queues": "2.137.1",
-        "@stamhoofd/sql": "2.137.1",
-        "@stamhoofd/structures": "2.137.1",
-        "@stamhoofd/types": "2.137.1",
-        "@stamhoofd/utility": "2.137.1",
+        "@stamhoofd/backend-env": "2.137.3",
+        "@stamhoofd/backend-i18n": "2.137.3",
+        "@stamhoofd/backend-middleware": "2.137.3",
+        "@stamhoofd/crons": "2.137.3",
+        "@stamhoofd/email": "2.137.3",
+        "@stamhoofd/excel-writer": "2.137.3",
+        "@stamhoofd/logging": "2.137.3",
+        "@stamhoofd/models": "2.137.3",
+        "@stamhoofd/object-differ": "2.137.3",
+        "@stamhoofd/queues": "2.137.3",
+        "@stamhoofd/sql": "2.137.3",
+        "@stamhoofd/structures": "2.137.3",
+        "@stamhoofd/types": "2.137.3",
+        "@stamhoofd/utility": "2.137.3",
         "archiver": "7.0.1",
-        "axios": "1.18.0",
+        "axios": "1.18.1",
         "base-x": "3.0.11",
         "chalk": "5.6.2",
         "cookie": "0.7.2",
@@ -82,31 +82,31 @@
         "jose": "5.10.0",
         "jsonwebtoken": "9.0.3",
         "jsvat-next": "3.0.4",
-        "libphonenumber-js": "1.13.6",
+        "libphonenumber-js": "1.13.9",
         "luxon": "3.7.2",
-        "mailparser": "3.9.9",
+        "mailparser": "3.9.14",
         "mockdate": "3.0.5",
         "mysql2": "3.23.1",
         "node-rsa": "2.0.0",
         "openid-client": "5.7.1",
         "stripe": "16.12.0",
-        "uuid": "14.0.0"
+        "uuid": "14.0.1"
     },
     "devDependencies": {
-        "@stamhoofd/test-utils": "2.137.1",
+        "@stamhoofd/test-utils": "2.137.3",
         "@types/cookie": "0.6.0",
-        "@types/luxon": "3.7.1",
+        "@types/luxon": "3.7.2",
         "@types/mailparser": "3.4.6",
         "@types/node": "22.20.1",
         "@types/uuid": "8.3.4",
-        "nock": "14.0.15",
-        "qs": "6.15.2",
+        "nock": "14.0.16",
+        "qs": "6.15.3",
         "sinon": "18.0.1",
-        "vitest": "4.1.8"
+        "vitest": "4.1.10"
     },
     "license": "UNLICENCED",
     "publishConfig": {
         "access": "public"
     },
-    "gitHead": "951f73f30cf648612da5e4ff22d9db19f5a8f31e"
+    "gitHead": "5f2515f1eab66fa40ddb0c19620d2181ccf3d945"
 }

package/src/boot.ts

@@ -4,6 +4,7 @@ import { CORSMiddleware, LogMiddleware, VersionMiddleware } from '@stamhoofd/bac
 import { checkReadOnly } from '@stamhoofd/crons';
 import { Email } from '@stamhoofd/email';
 import { loadLogger } from '@stamhoofd/logging';
+import { CpuService } from '@stamhoofd/logging/CpuService';
 import { Version } from '@stamhoofd/structures';
 import { sleep } from '@stamhoofd/utility';
 
@@ -20,14 +21,13 @@ import { ContextMiddleware } from './middleware/ContextMiddleware.js';
 import { AuditLogService } from './services/AuditLogService.js';
 import { BalanceItemService } from './services/BalanceItemService.js';
 import { BootChecksService } from './services/BootChecksService.js';
-import { CpuService } from './services/CpuService.js';
 import { DocumentService } from './services/DocumentService.js';
 import { FileSignService } from './services/FileSignService.js';
 import { PlatformMembershipService } from './services/PlatformMembershipService.js';
 import { UitpasService } from './services/uitpas/UitpasService.js';
 import { UniqueMemberNumberService } from './services/UniqueMemberNumberService.js';
 import { UniqueUserService } from './services/UniqueUserService.js';
-import { QueryableModel } from '@stamhoofd/sql';
+import { QueryableModel, SQLLogger } from '@stamhoofd/sql';
 
 process.on('unhandledRejection', (error: Error) => {
     console.error('unhandledRejection');
@@ -176,6 +176,19 @@ export const boot = async (options: { killProcess: boolean }) => {
     resumeEmails().catch(console.error);
 
     if (STAMHOOFD.environment !== 'development' && STAMHOOFD.environment !== 'test') {
+        // Tune SQL slow-query logging based on CPU load: under heavy load timings are unreliable,
+        // and when idle we can afford to log all slow queries.
+        CpuService.addSampleListener((average5s) => {
+            if (average5s > 80) {
+                // Danger zone: don't log slow queries any longer because the timings won't be trustworthy.
+                SQLLogger.slowQueryThresholdMs = null;
+            } else if (average5s < 20) {
+                // No load, safe to log all slow queries
+                SQLLogger.slowQueryThresholdMs = 300;
+            } else {
+                SQLLogger.slowQueryThresholdMs = 500;
+            }
+        });
         CpuService.startMonitoring();
     } else if (STAMHOOFD.environment === 'development') {
         const { loadDebugFunctions } = await import('./debug.js');

package/src/endpoints/organization/dashboard/invoices/GetInvoicesEndpoint.ts

@@ -187,16 +187,6 @@ export class GetInvoicesEndpoint extends Endpoint<Params, Query, Body, ResponseB
             }
         }
 
-        for (const invoice of invoices) {
-            if (invoice.number && !invoice.pdf && STAMHOOFD.environment === 'development') {
-                await InvoicePdfService.generatePdf(invoice);
-            }
-
-            if (invoice.number && STAMHOOFD.environment === 'development') {
-                await InvoiceXMlService.generateXml(invoice);
-            }
-        }
-
         return new PaginatedResponse<InvoiceStruct[], LimitedFilteredRequest>({
             results: await AuthenticatedStructures.invoices(invoices, true),
             next,

package/src/endpoints/organization/dashboard/invoices/PatchInvoicesEndpoint.ts

@@ -57,6 +57,24 @@ export class PatchInvoicesEndpoint extends Endpoint<Params, Query, Body, Respons
             invoices.push(model);
         }
 
+        for (const patch of request.body.getPatches()) {
+            const model = await Invoice.getByID(patch.id);
+            if (!model || model.organizationId !== organization.id) {
+                throw Context.auth.notFoundOrNoAccess($t('%ZcE'));
+            }
+
+            if (!model.number) {
+                // ignore
+                continue;
+            }
+
+            // Create PDF
+            if (patch.pdf === null) {
+                await InvoiceService.retryInvoiceGenerationAndSending(model);
+            }
+            invoices.push(model);
+        }
+
         for (const id of request.body.getDeletes()) {
             const model = await Invoice.getByID(id);
             if (!model || model.organizationId !== organization.id) {

package/src/endpoints/system/HealthEndpoint.ts

@@ -2,7 +2,7 @@ import { ArrayDecoder, AutoEncoder, field, StringDecoder } from '@simonbackx/sim
 import type { DecodedRequest, Request } from '@simonbackx/simple-endpoints';
 import { Endpoint, Response } from '@simonbackx/simple-endpoints';
 import { SimpleError } from '@simonbackx/simple-errors';
-import { CpuService } from '../../services/CpuService.js';
+import { CpuService } from '@stamhoofd/logging/CpuService';
 import { StartupHealthService } from '../../services/StartupHealthService.js';
 
 type Params = Record<string, never>;

package/src/services/InvoicePdfService.ts

@@ -258,26 +258,62 @@ export class InvoicePdfService {
         const timeout = setTimeout(() => controller.abort(), 30_000);
 
         try {
-            // Issue with system trusted CA in development
-            const result = await fetch((STAMHOOFD.environment === 'development' ? 'http://' : 'https://') + STAMHOOFD.domains.rendererApi + '/v' + Version + '/html-to-pdf', {
-                method: 'POST',
-                body: form,
-                signal: controller.signal,
-            });
-            if (result.status === 200) {
-                // todo
-                const buffer = Buffer.from(await result.arrayBuffer());
-                const file = await this.uploadPdf(invoice, buffer);
+            if (STAMHOOFD.environment === 'test') {
+                const file = new File({
+                    id: uuidv4(),
+                    server: 'https://' + STAMHOOFD.SPACES_BUCKET + '.' + STAMHOOFD.SPACES_ENDPOINT,
+                    path: 'test.pdf',
+                    size: 0,
+                    name: (invoice.number ?? invoice.id),
+                    isPrivate: false,
+                    contentType: 'application/pdf',
+                });
                 invoice.pdf = file;
                 await invoice.save();
             } else {
+            // Issue with system trusted CA in development
+                const result = await fetch((STAMHOOFD.environment === 'development' ? 'http://' : 'https://') + STAMHOOFD.domains.rendererApi + '/v' + Version + '/html-to-pdf', {
+                    method: 'POST',
+                    body: form,
+                    signal: controller.signal,
+                });
+                if (result.status === 200) {
                 // todo
+                    const buffer = Buffer.from(await result.arrayBuffer());
+                    const file = await this.uploadPdf(invoice, buffer);
+                    invoice.pdf = file;
+                    await invoice.save();
+                } else {
+                    let txt = '';
+                    try {
+                        txt = await result.text();
+                    } catch (e) {
+                        txt = '<unreadable stream>';
+                        console.error('Could not read response', e);
+                    }
+                    console.error('Failed to generate invoice PDF. Non 200 statuscode: ', result.status, txt);
+                    throw new SimpleError({
+                        code: 'pdf_creation_timeout',
+                        message: 'Failed to generate invoice PDF',
+                        human: $t('Er ging iets mis bij het aanmaken van de factuur'),
+                    });
+                }
             }
         } catch (err) {
+            console.error('Failed to generate invoice PDF', err);
+
             if (err instanceof DOMException && err.name === 'AbortError') {
-                throw new Error('Request timed out after 30s');
+                throw new SimpleError({
+                    code: 'pdf_creation_timeout',
+                    message: 'Failed to generate invoice PDF: timeout',
+                    human: $t('Er ging iets mis bij het aanmaken van de factuur'),
+                });
             }
-            console.error(err);
+            throw new SimpleError({
+                code: 'pdf_creation_failed',
+                message: 'Failed to generate invoice PDF',
+                human: $t('Er ging iets mis bij het aanmaken van de factuur'),
+            });
         } finally {
             clearTimeout(timeout);
         }

package/src/services/InvoiceService.ts

@@ -302,6 +302,28 @@ export class InvoiceService {
         return model;
     }
 
+    static async retryInvoiceGenerationAndSending(model: Invoice) {
+        const hadPdf = !!model.pdf;
+        if (!model.pdf) {
+            // Create PDF
+            await InvoicePdfService.generatePdf(model);
+        }
+
+        if (!model.xml) {
+            await InvoiceXMlService.generateXml(model);
+        }
+
+        const organization = await Organization.getByID(model.organizationId);
+        if (!organization) {
+            console.error('Unexpected missing organization for ' + model.id);
+            return;
+        }
+
+        if (!await this.forwardInvoice(model, organization) && !model.didSendPeppol && !hadPdf) {
+            await this.sendCustomerEmail(model, organization);
+        }
+    }
+
     /**
      * Permanently delete an invoice.
      *

package/src/services/MollieService.ts

@@ -250,11 +250,11 @@ export class MollieService {
 
             details: PaymentMandateDetails.create({
                 name: ('consumerName' in details ? details.consumerName : details.cardHolder) ?? undefined,
-                cardNumber: 'cardNumber' in details ? details.cardNumber : null,
-                iban: 'consumerAccount' in details ? Formatter.iban(details.consumerAccount) : null,
-                bic: ('consumerBic' in details ? details.consumerBic : undefined),
-                expiryDate: ('cardExpiryDate' in details ? DateTime.fromISO(details.cardExpiryDate, { zone: Formatter.timezone }).toJSDate() : null), // todo: parse date correctly in Brussels timezone!
-                brand: ('cardLabel' in details ? details.cardLabel : null),
+                cardNumber: 'cardNumber' in details && details.cardNumber ? details.cardNumber : null,
+                iban: 'consumerAccount' in details && details.consumerAccount ? Formatter.iban(details.consumerAccount) : null,
+                bic: ('consumerBic' in details && details.consumerBic ? details.consumerBic : undefined),
+                expiryDate: ('cardExpiryDate' in details && details.cardExpiryDate ? DateTime.fromISO(details.cardExpiryDate, { zone: Formatter.timezone }).toJSDate() : null), // todo: parse date correctly in Brussels timezone!
+                brand: ('cardLabel' in details && details.cardLabel ? details.cardLabel : null),
             }),
         });
     }

package/src/services/SSOService.ts

@@ -287,7 +287,6 @@ export class SSOService {
             httpOnly: true,
             secure: STAMHOOFD.environment !== 'development',
             expires: data.expires,
-            sameSite: 'lax',
         });
     }
 

package/src/services/CpuService.ts

@@ -1,120 +0,0 @@
-import { SQLLogger } from '@stamhoofd/sql';
-
-class StaticCpuService {
-    samples: number[];
-    private maxSamples: number;
-    private interval?: NodeJS.Timeout;
-    // Current index = the last saved
-    private currentIndex = 0;
-
-    constructor() {
-        this.maxSamples = 5 * 60; // 5 minutes of data
-        this.samples = new Array(this.maxSamples).fill(0);
-    }
-
-    /**
-     * Get a live sample of CPU usage
-     * @param samplingInterval
-     * @returns
-     */
-    async takeSample(samplingInterval = 500): Promise<number> {
-        const startUsage = process.cpuUsage();
-        const startTime = process.hrtime.bigint();
-
-        return new Promise((resolve) => {
-            setTimeout(() => {
-                const elapsedUsage = process.cpuUsage(startUsage);
-                const elapsedTime = process.hrtime.bigint() - startTime;
-
-                // Convert to microseconds
-                const elapsedTimeMs = Number(elapsedTime) / 1000;
-
-                // CPU time in microseconds
-                const totalCPUTime = elapsedUsage.user + elapsedUsage.system;
-
-                // Calculate percentage
-                const cpuPercent = (totalCPUTime / elapsedTimeMs) * 100;
-
-                resolve(cpuPercent);
-            }, samplingInterval);
-        });
-    }
-
-    getCpuUsage(): number | undefined {
-        if (this.currentIndex === 0) {
-            return this.samples[this.samples.length - 1];
-        }
-        return this.samples[this.currentIndex - 1];
-    }
-
-    getAverage(size = this.maxSamples): number {
-        if (size > this.maxSamples) {
-            size = this.maxSamples;
-        }
-        if (size <= 0) {
-            return NaN;
-        }
-        if (size === this.maxSamples) {
-            return this.samples.reduce((a, b) => a + b, 0) / size;
-        } else {
-            // To read before current index
-            const toReadBeforeIndex = Math.min(size, this.currentIndex);
-            const toReadFromEnd = size - toReadBeforeIndex;
-
-            // Sum performantly
-            let sum = 0;
-            for (let i = 0; i < toReadBeforeIndex; i++) {
-                sum += this.samples[this.currentIndex - 1 - i];
-            }
-            for (let i = 0; i < toReadFromEnd; i++) {
-                sum += this.samples[this.samples.length - 1 - i];
-            }
-
-            return sum / size;
-        }
-    }
-
-    private async saveSample() {
-        const sample = await this.takeSample(1000);
-        this.samples[this.currentIndex] = sample;
-        this.currentIndex = (this.currentIndex + 1) % this.maxSamples;
-        const five = this.getAverage(5);
-
-        if (this.currentIndex % 5 === 0 || five > 80) {
-            const min = this.getAverage(60);
-            console.log(`[CPU] 5s: ${five.toFixed(2)}%\n[CPU] 1 min: ${min.toFixed(2)}%\n[CPU] 5 min: ${this.getAverage(60 * 5).toFixed(2)}%`);
-        }
-
-        if (five > 80) {
-            // Danger zone, in this case we don't want to log all slow queries any longer because the information won't be trustworthy.
-            SQLLogger.slowQueryThresholdMs = null;
-        } else {
-            if (five < 20) {
-                // No load, safe to log all slow queries
-                SQLLogger.slowQueryThresholdMs = 300;
-            } else {
-                SQLLogger.slowQueryThresholdMs = 500;
-            }
-        }
-    }
-
-    startMonitoring() {
-        if (this.interval) {
-            return;
-        }
-        this.interval = setInterval(() => {
-            this.saveSample().catch((error) => {
-                console.error('Failed to take CPU sample:', error);
-            });
-        }, 1000); // Sample every second
-    }
-
-    stopMonitoring() {
-        if (this.interval) {
-            clearInterval(this.interval);
-            this.interval = undefined;
-        }
-    }
-}
-
-export const CpuService = new StaticCpuService();