@stamhoofd/backend 2.83.5 → 2.84.1

package/src/helpers/LimitedFilteredRequestHelper.ts

@@ -1,5 +1,6 @@
 import { SimpleError } from '@simonbackx/simple-errors';
-import { LimitedFilteredRequest } from '@stamhoofd/structures';
+import { SQLSortDefinitions } from '@stamhoofd/sql';
+import { getSortFilter, LimitedFilteredRequest } from '@stamhoofd/structures';
 
 export class LimitedFilteredRequestHelper {
     static throwIfInvalidLimit({ request, maxLimit }: { request: LimitedFilteredRequest; maxLimit: number }) {
@@ -21,4 +22,28 @@ export class LimitedFilteredRequestHelper {
             });
         }
     }
+
+    static fixInfiniteLoadingLoop<T>({ request, results, sorters }: { request: LimitedFilteredRequest; results: T[]; sorters: SQLSortDefinitions<T> }): LimitedFilteredRequest | undefined {
+        let next: LimitedFilteredRequest | undefined;
+
+        if (results.length >= request.limit) {
+            const lastObject = results[results.length - 1];
+            const nextFilter = getSortFilter(lastObject, sorters, request.sort);
+
+            next = new LimitedFilteredRequest({
+                filter: request.filter,
+                pageFilter: nextFilter,
+                sort: request.sort,
+                limit: request.limit,
+                search: request.search,
+            });
+
+            if (JSON.stringify(nextFilter) === JSON.stringify(request.pageFilter)) {
+                console.error('Found infinite loading loop for', request);
+                next = undefined;
+            }
+        }
+
+        return next;
+    }
 }

package/src/helpers/MemberCharger.ts

@@ -1,6 +1,5 @@
 import { BalanceItem } from '@stamhoofd/models';
 import { BalanceItemType, MemberWithRegistrationsBlob } from '@stamhoofd/structures';
-import { BalanceItemService } from '../services/BalanceItemService';
 
 export class MemberCharger {
     static async chargeMany({ chargingOrganizationId, membersToCharge, price, amount, description, dueAt, createdAt }: { chargingOrganizationId: string; membersToCharge: MemberWithRegistrationsBlob[]; price: number; amount?: number; description: string; dueAt: Date | null; createdAt: Date | null }) {
@@ -15,10 +14,6 @@ export class MemberCharger {
         }));
 
         await Promise.all(balanceItems.map(balanceItem => balanceItem.save()));
-        await BalanceItem.updateOutstanding(balanceItems);
-
-        // Reallocate
-        await BalanceItemService.reallocate(balanceItems, chargingOrganizationId);
     }
 
     private static createBalanceItem({ price, amount, description, chargingOrganizationId, memberBeingCharged, dueAt, createdAt }: { price: number; amount?: number; description: string; chargingOrganizationId: string; memberBeingCharged: MemberWithRegistrationsBlob; dueAt: Date | null; createdAt: Date | null }): BalanceItem {

package/src/helpers/MembershipCharger.ts

@@ -1,9 +1,8 @@
 import { SimpleError } from '@simonbackx/simple-errors';
 import { BalanceItem, Member, MemberPlatformMembership, Platform } from '@stamhoofd/models';
 import { SQL, SQLOrderBy, SQLWhereSign } from '@stamhoofd/sql';
-import { BalanceItemRelation, BalanceItemRelationType, BalanceItemType } from '@stamhoofd/structures';
+import { BalanceItemRelation, BalanceItemRelationType, BalanceItemType, TranslatedString } from '@stamhoofd/structures';
 import { Formatter } from '@stamhoofd/utility';
-import { BalanceItemService } from '../services/BalanceItemService';
 
 export const MembershipCharger = {
     async charge() {
@@ -104,14 +103,14 @@ export const MembershipCharger = {
                         BalanceItemRelationType.Member,
                         BalanceItemRelation.create({
                             id: member.id,
-                            name: member.details.name,
+                            name: new TranslatedString(member.details.name),
                         }),
                     ],
                     [
                         BalanceItemRelationType.MembershipType,
                         BalanceItemRelation.create({
                             id: type.id,
-                            name: type.name,
+                            name: new TranslatedString(type.name),
                         }),
                     ],
                 ]);
@@ -131,11 +130,6 @@ export const MembershipCharger = {
                 createdPrice += membership.price;
             }
 
-            await BalanceItem.updateOutstanding(createdBalanceItems);
-
-            // Reallocate
-            await BalanceItemService.reallocate(createdBalanceItems, chargeVia);
-
             if (memberships.length < chunkSize) {
                 break;
             }

package/src/helpers/OrganizationCharger.ts

@@ -1,6 +1,5 @@
 import { BalanceItem } from '@stamhoofd/models';
 import { BalanceItemType, Organization as OrganizationStruct } from '@stamhoofd/structures';
-import { BalanceItemService } from '../services/BalanceItemService';
 
 export class OrganizationCharger {
     static async chargeMany({ chargingOrganizationId, organizationsToCharge, price, amount, description, dueAt, createdAt }: { chargingOrganizationId: string; organizationsToCharge: OrganizationStruct[]; price: number; amount?: number; description: string; dueAt: Date | null; createdAt: Date | null }) {
@@ -15,10 +14,6 @@ export class OrganizationCharger {
         }));
 
         await Promise.all(balanceItems.map(balanceItem => balanceItem.save()));
-        await BalanceItem.updateOutstanding(balanceItems);
-
-        // Reallocate
-        await BalanceItemService.reallocate(balanceItems, chargingOrganizationId);
     }
 
     private static createBalanceItem({ price, amount, description, chargingOrganizationId, organizationBeingCharged, dueAt, createdAt }: { price: number; amount?: number; description: string; chargingOrganizationId: string; organizationBeingCharged: OrganizationStruct; dueAt: Date | null; createdAt: Date | null }): BalanceItem {

package/src/helpers/ThrottledQueue.test.ts

@@ -0,0 +1,194 @@
+import { ThrottledQueue } from './ThrottledQueue';
+
+describe('ThrottledQueue', () => {
+    // Mock timers for controlling setTimeout
+    beforeEach(() => {
+        jest.useFakeTimers();
+    });
+
+    afterEach(() => {
+        jest.useRealTimers();
+    });
+
+    afterAll(() => {
+        jest.clearAllMocks();
+    });
+
+    it('should create an instance with the provided handler', () => {
+        const handler = jest.fn();
+        const queue = new ThrottledQueue(handler);
+
+        expect(queue.handler).toBe(handler);
+        expect(queue.items.size).toBe(0);
+        expect(queue.timeout).toBeNull();
+    });
+
+    it('should add items to the queue', () => {
+        const handler = jest.fn();
+        const queue = new ThrottledQueue(handler);
+
+        queue.addItem(1);
+        expect(queue.items.size).toBe(1);
+        expect(queue.items.has(1)).toBe(true);
+    });
+
+    it('should add multiple items to the queue', () => {
+        const handler = jest.fn();
+        const queue = new ThrottledQueue(handler);
+
+        queue.addItems([1, 2, 3]);
+        expect(queue.items.size).toBe(3);
+        expect(queue.items.has(1)).toBe(true);
+        expect(queue.items.has(2)).toBe(true);
+        expect(queue.items.has(3)).toBe(true);
+    });
+
+    it('should flush automatically when reaching maxBatchSize', async () => {
+        const handler = jest.fn().mockResolvedValue(undefined);
+        const queue = new ThrottledQueue(handler);
+        queue.maxBatchSize = 3;
+
+        // Spy on flushAll to check if it's called
+        const flushAllSpy = jest.spyOn(queue, 'flushAll');
+
+        queue.addItems([1, 2]);
+        expect(flushAllSpy).not.toHaveBeenCalled();
+
+        queue.addItem(3);
+        expect(flushAllSpy).toHaveBeenCalled();
+
+        // Wait for all promises to resolve
+        await queue.wait();
+
+        expect(handler).toHaveBeenCalledWith([1, 2, 3]);
+        expect(queue.items.size).toBe(0);
+    });
+
+    it('should process items in batches when exceeding maxBatchSize', async () => {
+        const handler = jest.fn().mockResolvedValue(undefined);
+        const queue = new ThrottledQueue(handler);
+        queue.maxBatchSize = 3;
+
+        const items = [1, 2, 3, 4, 5, 6, 7];
+        queue.addItems(items);
+        await queue.wait();
+
+        // Expecting multiple calls with batches of appropriate size
+        expect(handler).toHaveBeenCalledTimes(3);
+
+        // Check if all items were processed
+        const processedItems = new Set([
+            ...handler.mock.calls[0][0],
+            ...handler.mock.calls[1][0],
+            ...handler.mock.calls[2][0],
+        ]);
+
+        expect(processedItems.size).toBe(items.length);
+        items.forEach(item => expect(processedItems.has(item)).toBe(true));
+    });
+
+    it('should call emptyHandler when queue becomes empty', async () => {
+        const handler = jest.fn().mockResolvedValue(undefined);
+        const queue = new ThrottledQueue(handler);
+        const emptyHandler = jest.fn();
+        queue.emptyHandler = emptyHandler;
+
+        queue.maxBatchSize = 3;
+        queue.addItems([1, 2, 3]);
+        await queue.wait();
+
+        expect(emptyHandler).toHaveBeenCalled();
+    });
+
+    it('should not call emptyHandler when queue is filled during processing', async () => {
+        const handler = jest.fn().mockResolvedValue(undefined);
+        const queue = new ThrottledQueue(handler);
+        const emptyHandler = jest.fn();
+        queue.emptyHandler = emptyHandler;
+
+        queue.maxBatchSize = 3;
+        queue.addItems([1, 2, 3]);
+        // flush happened automatically above
+        queue.addItems([1]);
+        await queue.wait();
+
+        expect(emptyHandler).not.toHaveBeenCalled();
+
+        await queue.flushAndWait();
+        expect(emptyHandler).toHaveBeenCalled();
+    });
+
+    it('should not fail if handler throws an error', async () => {
+        const error = new Error('Test error');
+        const handler = jest.fn().mockRejectedValue(error);
+        const queue = new ThrottledQueue(handler);
+
+        // Spy on console.error
+        const consoleErrorSpy = jest.spyOn(console, 'error').mockImplementation(() => {});
+
+        queue.addItem(1);
+        await queue.flushAndWait();
+
+        expect(handler).toHaveBeenCalledWith([1]);
+        expect(consoleErrorSpy).toHaveBeenCalled();
+        expect(consoleErrorSpy.mock.calls[0][0]).toContain('Error processing batch in ThrottledQueue:');
+
+        consoleErrorSpy.mockRestore();
+    });
+
+    it('should handle flushAndWait when queue is empty', async () => {
+        const handler = jest.fn();
+        const queue = new ThrottledQueue(handler);
+
+        await queue.flushAndWait();
+        expect(handler).not.toHaveBeenCalled();
+    });
+
+    it('should allow adding items while processing', async () => {
+        // Setup a handler that adds more items to the queue when called
+        const queue = new ThrottledQueue(async (items: number[]) => {
+            if (items.includes(1)) {
+                queue.addItem(4);
+            }
+        });
+
+        queue.addItems([1, 2, 3]);
+        await queue.flushAndWait();
+
+        expect(queue.items.size).toBe(1);
+        expect(queue.items.has(4)).toBe(true);
+    });
+
+    it('should handle startTimeout and stopTimeout correctly', async () => {
+        const handler = jest.fn();
+        const queue = new ThrottledQueue(handler);
+        queue.maxDelay = 1000; // Set a max delay for the timeout
+
+        queue.addItem(1);
+        expect(queue.timeout).not.toBeNull();
+
+        await queue.flushAndWait();
+        expect(queue.timeout).toBeNull();
+    });
+
+    it('should automatically flush after maxDelay', async () => {
+        const handler = jest.fn().mockResolvedValue(undefined);
+        const queue = new ThrottledQueue(handler);
+        queue.maxDelay = 1000; // Set a max delay for the timeout
+
+        queue.addItem(1);
+        expect(queue.timeout).not.toBeNull();
+
+        jest.advanceTimersByTime(500);
+        await queue.wait();
+        expect(handler).not.toHaveBeenCalled();
+
+        // Fast-forward time
+        jest.advanceTimersByTime(500);
+
+        await queue.wait();
+        expect(handler).toHaveBeenCalledWith([1]);
+        expect(queue.items.size).toBe(0);
+        expect(queue.timeout).toBeNull();
+    });
+});

package/src/helpers/ThrottledQueue.ts

@@ -0,0 +1,145 @@
+/**
+ * Define an expensive action that needs to run on a list of items.
+ * Instead of running the action per item, it will call it on a batch of items or if a certain time has passed.
+ *
+ * Note that the handler can be called in parallel in some situations.
+ */
+export class ThrottledQueue<T> {
+    items: Set<T> = new Set<T>();
+    timeout: NodeJS.Timeout | null = null;
+
+    maxBatchSize = 100;
+
+    /**
+     * The queue is cleared at least after this duration.
+     * Set to null (default) to disable the timeout.
+     * In milliseconds.
+     */
+    maxDelay: number | null = 10_000;
+
+    handler: (items: T[]) => Promise<void> | void;
+    emptyHandler?: () => void;
+
+    constructor(handler: (items: T[]) => Promise<void> | void, options: { maxBatchSize?: number; maxDelay?: number | null; emptyHandler?: () => void } = {}) {
+        this.handler = handler;
+        this.maxBatchSize = options.maxBatchSize ?? 100;
+        this.maxDelay = options.maxDelay ?? 10_000;
+        this.emptyHandler = options.emptyHandler;
+    }
+
+    addItems(items: T[]): void {
+        for (const item of items) {
+            this.items.add(item);
+        }
+
+        if (this.items.size >= this.maxBatchSize) {
+            this.flushAll();
+            return;
+        }
+        this.startTimeout();
+    }
+
+    addItem(item: T): void {
+        this.items.add(item);
+
+        if (this.items.size >= this.maxBatchSize) {
+            this.flushAll();
+            return;
+        }
+        this.startTimeout();
+    }
+
+    stopTimeout() {
+        if (this.timeout) {
+            clearTimeout(this.timeout);
+            this.timeout = null;
+        }
+    }
+
+    startTimeout() {
+        if (this.timeout || this.maxDelay === null) {
+            // Keep existing
+            return;
+        }
+
+        this.timeout = setTimeout(() => {
+            this.timeout = null;
+            this.flushAll();
+        }, this.maxDelay); // Adjust the timeout duration as needed
+    }
+
+    pendingFlushes: Set<Promise<void>> = new Set<Promise<void>>();
+
+    private async process(items: T[]) {
+        // Calculate an efficient batch size (not simply using maxBatchSize)
+        const batchCount = Math.ceil(items.length / this.maxBatchSize);
+        if (batchCount === 0) {
+            return;
+        }
+        const batchSize = Math.ceil(items.length / batchCount);
+
+        for (let i = 0; i < items.length; i += batchSize) {
+            // Avoid cloning the array if the batch size is larger than the remaining items
+            const batch = batchSize < items.length ? items.slice(i, i + batchSize) : items;
+            try {
+                await this.handler(batch);
+            }
+            catch (error) {
+                console.error('Error processing batch in ThrottledQueue:', error);
+            }
+        }
+    }
+
+    processErrorHandler(error: Error) {
+        console.error('Error in ThrottledQueue flush:', error);
+    }
+
+    /**
+     * Flushes all items in the queue.
+     * Items that are added while processing will not be processed automatically, but rather when the tiemout is reached or the maximum is reached.
+     */
+    flushAll() {
+        if (this.items.size === 0) {
+            // Nothing to flush or clear
+            return;
+        }
+
+        // Move items, so we can continue adding items while we are processing
+        const items = [...this.items];
+        this.items.clear();
+
+        const flush = this.process(items)
+            .catch(this.processErrorHandler)
+            .finally(() => {
+                this.pendingFlushes.delete(flush);
+
+                if (this.pendingFlushes.size === 0 && this.items.size === 0) {
+                // Call empty handler to signal that the queue is empty
+                    this.stopTimeout();
+
+                    if (this.emptyHandler) {
+                        this.emptyHandler();
+                    }
+                }
+            });
+
+        this.pendingFlushes.add(flush);
+    }
+
+    async wait() {
+        if (this.pendingFlushes.size === 0) {
+            return;
+        }
+
+        // Wait for all pending flushes to complete
+        await Promise.all(this.pendingFlushes);
+    }
+
+    /**
+     * Note: this won't wait on new items that are added while flushing.
+     */
+    async flushAndWait() {
+        this.flushAll();
+        await this.wait();
+    }
+}

package/src/helpers/XlsxTransformerColumnHelper.ts

@@ -1,4 +1,4 @@
-import { XlsxTransformerColumn } from '@stamhoofd/excel-writer';
+import { isXlsxTransformerConcreteColumn, XlsxTransformerColumn, XlsxTransformerConcreteColumn } from '@stamhoofd/excel-writer';
 import { Address, CountryHelper, Parent, ParentTypeHelper, PlatformMember, RecordAnswer, RecordCategory, RecordSettings, RecordType } from '@stamhoofd/structures';
 
 export class XlsxTransformerColumnHelper {
@@ -239,4 +239,47 @@ export class XlsxTransformerColumnHelper {
             },
         };
     }
+
+    /**
+     * Makes it possible to reuse an XlsxTransformerColumn, for example member columns exist, PlatformRegistration has
+     * a property member, so we can reuse the member columns for PlatformRegistration.
+     * @param param0
+     * @returns
+     */
+    private static transformConcreteColumnForProperty<T, O>({ column, key, getPropertyValue }: { column: XlsxTransformerConcreteColumn<T>; key: string; getPropertyValue: (object: O) => T }): XlsxTransformerConcreteColumn<O> {
+        return {
+            ...column,
+            id: `${key}.${column.id}`,
+            getValue: (object: O) => column.getValue(getPropertyValue(object)),
+        };
+    }
+
+    /**
+     * Makes it possible to reuse an XlsxTransformerColumn, for example member columns exist, PlatformRegistration has
+     * a property member, so we can reuse the member columns for PlatformRegistration.
+     * @param param0
+     * @returns
+     */
+    static transformColumnForProperty<T, O>({ column, key, getPropertyValue }: { column: XlsxTransformerColumn<T>; key: string; getPropertyValue: (object: O) => T }): XlsxTransformerColumn<O> {
+        if (isXlsxTransformerConcreteColumn(column)) {
+            return this.transformConcreteColumnForProperty({ column, key, getPropertyValue });
+        }
+
+        return {
+            match: (id: string) => {
+                if (!id.startsWith(key + '.')) {
+                    return;
+                }
+
+                const subId = id.substring(key.length + 1);
+                const subColumns = column.match(subId);
+
+                if (!subColumns) {
+                    return;
+                }
+
+                return subColumns.map(column => XlsxTransformerColumnHelper.transformConcreteColumnForProperty({ column, key, getPropertyValue }));
+            },
+        };
+    }
 }

package/src/middleware/ContextMiddleware.ts

@@ -10,7 +10,7 @@ export const ContextMiddleware: RequestMiddleware = {
         return ContextInstance.start(request, run);
     },
 
-    handleRequest: function (request: Request) {
+    handleRequest() {
         // Noop
     },
 };

package/src/seeds/1728928974-update-cached-outstanding-balance-from-items.ts

@@ -1,6 +1,7 @@
 import { Migration } from '@simonbackx/simple-database';
 import { logger } from '@simonbackx/simple-logging';
 import { BalanceItem } from '@stamhoofd/models';
+import { BalanceItemService } from '../services/BalanceItemService';
 
 /**
  * This migration is required to keep '1733994455-balance-item-status-open' working
@@ -16,7 +17,7 @@ export default new Migration(async () => {
 
     await logger.setContext({ tags: ['silent-seed', 'seed'] }, async () => {
         for await (const items of BalanceItem.select().limit(1000).allBatched()) {
-            await BalanceItem.updateOutstanding(items);
+            await BalanceItemService.updatePaidAndPending(items);
 
             c += items.length;
             process.stdout.write('.');

package/src/seeds/1735577912-update-cached-outstanding-balance-from-items.ts

@@ -1,6 +1,7 @@
 import { Migration } from '@simonbackx/simple-database';
 import { logger } from '@simonbackx/simple-logging';
 import { BalanceItem } from '@stamhoofd/models';
+import { BalanceItemService } from '../services/BalanceItemService';
 
 export default new Migration(async () => {
     if (STAMHOOFD.environment == 'test') {
@@ -13,7 +14,7 @@ export default new Migration(async () => {
 
     await logger.setContext({ tags: ['silent-seed', 'seed'] }, async () => {
         for await (const items of BalanceItem.select().limit(1000).allBatched()) {
-            await BalanceItem.updateOutstanding(items);
+            await BalanceItemService.updatePaidAndPending(items);
 
             c += items.length;
             process.stdout.write('.');

package/src/services/BalanceItemPaymentService.ts

@@ -1,44 +1,12 @@
 import { ManyToOneRelation } from '@simonbackx/simple-database';
 import { BalanceItemPayment, Organization } from '@stamhoofd/models';
-import { BalanceItemStatus } from '@stamhoofd/structures';
 import { BalanceItemService } from './BalanceItemService';
 
 type Loaded<T> = (T) extends ManyToOneRelation<infer Key, infer Model> ? Record<Key, Model> : never;
 
 export const BalanceItemPaymentService = {
     async markPaid(balanceItemPayment: BalanceItemPayment & Loaded<typeof BalanceItemPayment.balanceItem> & Loaded<typeof BalanceItemPayment.payment>, organization: Organization) {
-        const wasPaid = balanceItemPayment.balanceItem.isPaid;
-
-        // Update cached amountPaid of the balance item (balanceItemPayment will get overwritten later, but we need it to calculate the status)
-        balanceItemPayment.balanceItem.pricePaid += balanceItemPayment.price;
-
-        if (balanceItemPayment.balanceItem.status === BalanceItemStatus.Hidden && balanceItemPayment.balanceItem.pricePaid !== 0) {
-            balanceItemPayment.balanceItem.status = BalanceItemStatus.Due;
-        }
-
-        await balanceItemPayment.balanceItem.save();
-        const isPaid = balanceItemPayment.balanceItem.isPaid;
-
-        // Do logic of balance item
-        if (isPaid && !wasPaid && balanceItemPayment.price >= 0 && balanceItemPayment.balanceItem.status === BalanceItemStatus.Due) {
-            // Only call markPaid once (if it wasn't (partially) paid before)
-            await BalanceItemService.markPaid(balanceItemPayment.balanceItem, balanceItemPayment.payment, organization);
-        }
-        else {
-            await BalanceItemService.markUpdated(balanceItemPayment.balanceItem, balanceItemPayment.payment, organization);
-        }
-    },
-
-    /**
-     * Safe method to correct balance items that missed a markPaid call, but avoid double marking an order as valid.
-     */
-    async markPaidRepeated(balanceItemPayment: BalanceItemPayment & Loaded<typeof BalanceItemPayment.balanceItem> & Loaded<typeof BalanceItemPayment.payment>, organization: Organization) {
-        const isPaid = balanceItemPayment.balanceItem.isPaid;
-
-        // Do logic of balance item
-        if (isPaid && balanceItemPayment.price >= 0 && balanceItemPayment.balanceItem.status === BalanceItemStatus.Due) {
-            await BalanceItemService.markPaidRepeated(balanceItemPayment.balanceItem, balanceItemPayment.payment, organization);
-        }
+        await BalanceItemService.markPaid(balanceItemPayment.balanceItem, balanceItemPayment.payment, organization);
     },
 
     /**