@stamhoofd/backend 2.34.1 → 2.36.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@stamhoofd/backend",
-    "version": "2.34.1",
+    "version": "2.36.0",
     "main": "./dist/index.js",
     "exports": {
         ".": {
@@ -25,7 +25,7 @@
         "@types/luxon": "3.4.2",
         "@types/mailparser": "3.4.4",
         "@types/mysql": "^2.15.20",
-        "@types/node": "^18.11.17",
+        "@types/node": "^20.12",
         "nock": "^13.5.1",
         "qs": "^6.11.2",
         "sinon": "^18.0.0"
@@ -36,14 +36,14 @@
         "@simonbackx/simple-encoding": "2.15.1",
         "@simonbackx/simple-endpoints": "1.14.0",
         "@simonbackx/simple-logging": "^1.0.1",
-        "@stamhoofd/backend-i18n": "2.34.1",
-        "@stamhoofd/backend-middleware": "2.34.1",
-        "@stamhoofd/email": "2.34.1",
-        "@stamhoofd/models": "2.34.1",
-        "@stamhoofd/queues": "2.34.1",
-        "@stamhoofd/sql": "2.34.1",
-        "@stamhoofd/structures": "2.34.1",
-        "@stamhoofd/utility": "2.34.1",
+        "@stamhoofd/backend-i18n": "2.36.0",
+        "@stamhoofd/backend-middleware": "2.36.0",
+        "@stamhoofd/email": "2.36.0",
+        "@stamhoofd/models": "2.36.0",
+        "@stamhoofd/queues": "2.36.0",
+        "@stamhoofd/sql": "2.36.0",
+        "@stamhoofd/structures": "2.36.0",
+        "@stamhoofd/utility": "2.36.0",
         "archiver": "^7.0.1",
         "aws-sdk": "^2.885.0",
         "axios": "1.6.8",
@@ -57,8 +57,8 @@
         "mysql": "^2.18.1",
         "node-rsa": "1.1.1",
         "openid-client": "^5.4.0",
-        "postmark": "4.0.2",
+        "postmark": "^4.0.5",
         "stripe": "^16.6.0"
     },
-    "gitHead": "4faa18ab00da5608caedb6f7e83143f41a046d4f"
+    "gitHead": "8d1f0861fee07ca2047d13b0402011e6a9d272c3"
 }

package/src/crons/clear-excel-cache.test.ts

@@ -0,0 +1,152 @@
+import { Dirent } from "fs";
+import fs from "fs/promises";
+import { clearExcelCacheHelper } from "./clear-excel-cache";
+
+const testPath = '/Users/user/project/backend/app/api/.cache';
+jest.mock("fs/promises");
+const fsMock =  jest.mocked(fs, true)
+
+
+describe("clearExcelCacheHelper", () => {
+
+    it('should only run between 3 and 6 AM', async () => {
+        //#region arrange
+        const shouldFail = [
+            new Date(2024,1,1,0,0),
+            new Date(2024,1,1,2,50),
+            new Date(2024,1,1,2,59, 59, 999),
+            new Date(2024,1,1,6,0)
+        ]
+
+        const shouldPass = [
+            new Date(2024,1,1,3,0),
+            new Date(2024,1,1,3,55),
+            new Date(2024,1,1,5,59, 59, 999)
+        ]
+
+        fsMock.readdir.mockReturnValue(Promise.resolve([]));
+        //#endregion
+
+        //#region act and assert
+        for(const date of shouldFail) {
+            const didClear = await clearExcelCacheHelper({
+                lastExcelClear: null,
+                now: date,
+                cachePath: testPath,
+                environment: 'production'
+            })
+
+            expect(didClear).toBeFalse();
+        }
+
+        for(const date of shouldPass) {
+            const didClear = await clearExcelCacheHelper({
+                lastExcelClear: null,
+                now: date,
+                cachePath: testPath,
+                environment: 'production'
+            })
+
+            expect(didClear).toBeTrue();
+        }
+        //#endregion
+    })
+
+    it('should only run once each day', async () => {
+        //#region arrange
+        const firstTry  = new Date(2024,1,1,3,0);
+        const secondTry = new Date(2024,1,1,3,5);
+        const thirdTry  = new Date(2024,1,2,3,0);
+        const fourthTry  = new Date(2024,1,2,3,5);
+
+        fsMock.readdir.mockReturnValue(Promise.resolve([]));
+        //#endregion
+
+        //#region act and assert
+
+        // second try, should fail because 5 min earlier the cache was cleared
+        const didClearSecondTry = await clearExcelCacheHelper({
+            lastExcelClear: firstTry.getTime(),
+            now: secondTry,
+            cachePath: testPath,
+            environment: 'production'
+        });
+
+        expect(didClearSecondTry).toBeFalse();
+
+        // third try, should pass because the last clear was more than a day ago
+        const didClearThirdTry = await clearExcelCacheHelper({
+            lastExcelClear: secondTry.getTime(),
+            now: thirdTry,
+            cachePath: testPath,
+            environment: 'production'
+        });
+
+        expect(didClearThirdTry).toBeTrue();
+
+        // fourth try, should fail because 5 min earlier the cache was cleared
+        const didClearFourthTry = await clearExcelCacheHelper({
+            lastExcelClear: thirdTry.getTime(),
+            now: fourthTry,
+            cachePath: testPath,
+            environment: 'production'
+        });
+
+        expect(didClearFourthTry).toBeFalse();
+        //#endregion
+    })
+
+    it('should delete old cache only', async () => {
+        //#region arrange
+        const now = new Date(2024,0,5,3,5);
+
+        const dir1 = new Dirent();
+        dir1.name = '2024-01-01';
+
+        const dir2 = new Dirent();
+        dir2.name = '2024-01-02';
+
+        const dir3 = new Dirent();
+        dir3.name = '2024-01-03';
+
+        const dir4 = new Dirent();
+        dir4.name = '2024-01-04';
+
+        const dir5 = new Dirent();
+        dir5.name = '2024-01-05';
+
+        const dir6 = new Dirent();
+        dir6.name = 'not-a-date';
+
+        const dir7 = new Dirent();
+        dir7.name = 'noDate';
+
+        const file1 = new Dirent();
+        file1.name = 'some-file';
+        file1.parentPath = testPath;
+        file1.isDirectory = () => false;
+
+        const directories = [dir1, dir2, dir3, dir4, dir5, dir6, dir7];
+
+        directories.forEach(dir => {
+            dir.parentPath = testPath;
+            dir.isDirectory = () => true;
+        })
+
+        fsMock.readdir.mockReturnValue(Promise.resolve([...directories, file1]));
+        //#endregion
+
+        // act
+        await clearExcelCacheHelper({
+            lastExcelClear: null,
+            now,
+            cachePath: testPath,
+            environment: 'production'
+        });
+
+        // assert
+        expect(fsMock.rm).toHaveBeenCalledTimes(2);
+        expect(fsMock.rm).toHaveBeenCalledWith("/Users/user/project/backend/app/api/.cache/2024-01-01", { recursive: true, force: true });
+        expect(fsMock.rm).toHaveBeenCalledWith("/Users/user/project/backend/app/api/.cache/2024-01-02",{ recursive: true, force: true });
+    })
+})

package/src/crons/clear-excel-cache.ts

@@ -0,0 +1,92 @@
+import fs from "fs/promises";
+
+const msIn22Hours = 79200000;
+let lastExcelClear: number | null = null;
+
+export async function clearExcelCache() {
+    const now = new Date();
+
+    const didClear = await clearExcelCacheHelper({
+        lastExcelClear,
+        now,
+        cachePath: STAMHOOFD.CACHE_PATH,
+        environment: STAMHOOFD.environment
+    });
+
+    if(didClear) {
+        lastExcelClear = now.getTime();
+    }
+}
+
+export async function clearExcelCacheHelper
+({lastExcelClear, now, cachePath, environment}: {lastExcelClear: number | null, now: Date, cachePath: string, environment: "production" | "development" | "staging" | "test"}): Promise<boolean> {
+    if (environment === "development") {
+        return false;
+    }
+
+    const hour = now.getHours();
+
+    // between 3 and 6 AM
+    if(hour < 3 || hour > 5) {
+        return false;
+    }
+
+    // only run once each day
+    if(lastExcelClear !== null && lastExcelClear + msIn22Hours > now.getTime()) {
+        return false;
+    }
+
+    const currentYear = now.getFullYear();
+    const currentMonth = now.getMonth();
+    const currentDay = now.getDate();
+
+    const maxDaysInCache = 2;
+
+    const dateLimit = new Date(currentYear, currentMonth, currentDay - maxDaysInCache, 0,0,0,0);
+
+    const files = await fs.readdir(cachePath, {withFileTypes: true});
+
+    for (const file of files) {
+        if (file.isDirectory()) {
+            try {
+                const date = getDateFromDirectoryName(file.name);
+                const shouldDelete = date < dateLimit;
+
+                if(shouldDelete) {
+                    const path = file.parentPath + '/' + file.name;
+                    await fs.rm(path, { recursive: true, force: true })
+                    console.log("Removed", path)
+                }
+            } catch(error) {
+                console.error(error);
+            }
+        }
+    }
+
+    return true;
+}
+
+function getDateFromDirectoryName(file: string): Date {
+    const parts = file.split('-');
+
+    if (parts.length != 3) {
+        throw new Error(`Invalid directory ${file} in cache.`);
+    }
+
+    const year = parseInt(parts[0]);
+    if(isNaN(year)) {
+        throw new Error(`Year is NAN for directory ${file} in cache.`);
+    }
+
+    const month = parseInt(parts[1]);
+    if(isNaN(month)) {
+        throw new Error(`Month is NAN for directory ${file} in cache.`);
+    }
+
+    const day = parseInt(parts[2]);
+    if(isNaN(day)) {
+        throw new Error(`Day is NAN for directory ${file} in cache.`);
+    }
+
+    return new Date(year, month - 1, day);
+}

package/src/crons.ts

@@ -7,6 +7,7 @@ import { Formatter, sleep } from '@stamhoofd/utility';
 import AWS from 'aws-sdk';
 import { DateTime } from 'luxon';
 
+import { clearExcelCache } from './crons/clear-excel-cache';
 import { ExchangePaymentEndpoint } from './endpoints/organization/shared/ExchangePaymentEndpoint';
 import { checkSettlements } from './helpers/CheckSettlements';
 import { ForwardHandler } from './helpers/ForwardHandler';
@@ -575,6 +576,10 @@ async function checkReservedUntil() {
         }
     })
 
+    for(const registration of registrations) {
+        registration.scheduleStockUpdate()
+    }
+
     // Update occupancy
     for (const group of groups) {
         await group.updateOccupancy()
@@ -712,6 +717,12 @@ registeredCronJobs.push({
     running: false
 });
 
+registeredCronJobs.push({
+    name: 'clearExcelCache',
+    method: clearExcelCache,
+    running: false
+})
+
 async function run(name: string, handler: () => Promise<void>) {
     try {
         await logger.setContext({

package/src/endpoints/global/events/PatchEventsEndpoint.ts

@@ -256,6 +256,24 @@ export class PatchEventsEndpoint extends Endpoint<Params, Query, Body, ResponseB
             events.push(event)
         }
 
+        for (const id of request.body.getDeletes()) {
+            const event = await Event.getByID(id);
+            if (!event) {
+                throw new SimpleError({ code: "not_found", message: "Event not found", statusCode: 404 });
+            }
+
+            if (!(await Context.auth.canAccessEvent(event, PermissionLevel.Full))) {
+                throw Context.auth.error()
+            }
+
+            if(event.groupId) {
+                await PatchOrganizationRegistrationPeriodsEndpoint.deleteGroup(event.groupId)
+                event.groupId = null;
+            }
+            
+            await event.delete();
+        }
+
         return new Response(
             await AuthenticatedStructures.events(events)
         );

package/src/endpoints/global/files/ExportToExcelEndpoint.ts

@@ -99,7 +99,8 @@ export class ExportToExcelEndpoint extends Endpoint<Params, Query, Body, Respons
                                 token: 'downloadUrl',
                                 value: url
                             }))
-                        ]
+                        ],
+                        type: 'transactional'
                     })
 
                 }
@@ -113,7 +114,8 @@ export class ExportToExcelEndpoint extends Endpoint<Params, Query, Body, Respons
                         },
                         recipients: [
                             user.createRecipient()
-                        ]
+                        ],
+                        type: 'transactional'
                     })
                 }
                 throw error

package/src/endpoints/global/members/PatchOrganizationMembersEndpoint.ts

@@ -71,10 +71,8 @@ export class PatchOrganizationMembersEndpoint extends Endpoint<Params, Query, Bo
             }
             return null
         }
-        const updateGroups = new Map<string, Group>()
-
-        const balanceItemMemberIds: string[] = []
-        const balanceItemRegistrationIdsPerOrganization: Map<string, string[]> = new Map()
+        const updateGroups = new Map<string, Group>();
+        const updateRegistrations = new Map<string, Registration>();
         const updateMembershipMemberIds = new Set<string>()
 
         // Loop all members one by one
@@ -144,7 +142,6 @@ export class PatchOrganizationMembersEndpoint extends Endpoint<Params, Query, Bo
 
             await member.save()
             members.push(member)
-            balanceItemMemberIds.push(member.id)
             updateMembershipMemberIds.add(member.id)
 
             // Auto link users based on data
@@ -505,10 +502,10 @@ export class PatchOrganizationMembersEndpoint extends Endpoint<Params, Query, Bo
             await member.delete()
             shouldUpdateSetupSteps = true
 
-            // Update occupancy of this member because we removed registrations
-            const groupIds = member.registrations.flatMap(r => r.groupId)
-            for (const id of groupIds) {
-                const group = await getGroup(id)
+            for(const registration of member.registrations) {
+                const groupId = registration.groupId;
+                const group = await getGroup(groupId);
+                updateRegistrations.set(registration.id, registration);
                 if (group) {
                     // We need to update this group occupancy because we moved one member away from it
                     updateGroups.set(group.id, group)
@@ -516,26 +513,16 @@ export class PatchOrganizationMembersEndpoint extends Endpoint<Params, Query, Bo
             }
         }
 
-        await Member.updateOutstandingBalance(Formatter.uniqueArray(balanceItemMemberIds))
-        for (const [organizationId, balanceItemRegistrationIds] of balanceItemRegistrationIdsPerOrganization) {
-            await Registration.updateOutstandingBalance(Formatter.uniqueArray(balanceItemRegistrationIds), organizationId)
+        for(const registration of updateRegistrations.values()) {
+            registration.scheduleStockUpdate();
         }
-        
+
         // Loop all groups and update occupancy if needed
         for (const group of updateGroups.values()) {
             await group.updateOccupancy()
             await group.save()
         }
         
-        // We need to refetch the outstanding amounts of members that have changed
-        const updatedMembers = balanceItemMemberIds.length > 0 ? await Member.getBlobByIds(...balanceItemMemberIds) : []
-        for (const member of updatedMembers) {
-            const index = members.findIndex(m => m.id === member.id)
-            if (index !== -1) {
-                members[index] = member
-            }
-        }
-
         for (const member of members) {
             if (updateMembershipMemberIds.has(member.id)) {
                 await member.updateMemberships()

package/src/endpoints/global/registration/GetUserBillingStatusEndpoint.ts

@@ -0,0 +1,80 @@
+import { DecodedRequest, Endpoint, Request, Response } from "@simonbackx/simple-endpoints";
+import { CachedOutstandingBalance, Member, Organization } from "@stamhoofd/models";
+import { OrganizationBillingStatus, OrganizationBillingStatusItem } from "@stamhoofd/structures";
+
+import { Formatter } from "@stamhoofd/utility";
+import { AuthenticatedStructures } from "../../../helpers/AuthenticatedStructures";
+import { Context } from "../../../helpers/Context";
+
+type Params = Record<string, never>;
+type Query = undefined
+type Body = undefined
+type ResponseBody = OrganizationBillingStatus
+
+export class GetUserBilingStatusEndpoint extends Endpoint<Params, Query, Body, ResponseBody> {
+    protected doesMatch(request: Request): [true, Params] | [false] {
+        if (request.method != "GET") {
+            return [false];
+        }
+
+        const params = Endpoint.parseParameters(request.url, "/user/billing/status", {});
+
+        if (params) {
+            return [true, params as Params];
+        }
+        return [false];
+    }
+
+    async handle(_: DecodedRequest<Params, Query, Body>) {
+        const organization = await Context.setUserOrganizationScope();
+        const {user} = await Context.authenticate()
+
+        const memberIds = await Member.getMemberIdsWithRegistrationForUser(user)
+
+        return new Response(await GetUserBilingStatusEndpoint.getBillingStatusForObjects([user.id, ...memberIds], organization));
+    }
+
+    static async getBillingStatusForObjects(objectIds: string[], organization?: Organization|null) {
+        // Load cached balances
+        const cachedOutstandingBalances = await CachedOutstandingBalance.getForObjects(objectIds, organization?.id)
+
+        const organizationIds = Formatter.uniqueArray(cachedOutstandingBalances.map(b => b.organizationId))
+
+        let addOrganization = false
+        const i = organization ? organizationIds.indexOf(organization.id) : -1
+        if (i !== -1) {
+            organizationIds.splice(i, 1)
+            addOrganization = true
+        }
+
+        const organizations = await Organization.getByIDs(...organizationIds)
+
+        if (addOrganization && organization) {
+            organizations.push(organization)
+        }
+
+        const authenticatedOrganizations = await AuthenticatedStructures.organizations(organizations)
+
+        const billingStatus = OrganizationBillingStatus.create({})
+
+        for (const organization of authenticatedOrganizations) {
+            const items = cachedOutstandingBalances.filter(b => b.organizationId === organization.id)
+
+            let amount = 0;
+            let amountPending = 0;
+
+            for (const item of items) {
+                amount += item.amount
+                amountPending += item.amountPending
+            }
+
+            billingStatus.organizations.push(OrganizationBillingStatusItem.create({
+                organization,
+                amount,
+                amountPending
+            }))
+        }
+
+        return billingStatus
+    }
+}

package/src/endpoints/global/registration/GetUserDetailedBillingStatusEndpoint.ts

@@ -0,0 +1,83 @@
+import { DecodedRequest, Endpoint, Request, Response } from "@simonbackx/simple-endpoints";
+import { BalanceItem, Member, Organization, Payment } from "@stamhoofd/models";
+import { OrganizationDetailedBillingStatus, OrganizationDetailedBillingStatusItem } from "@stamhoofd/structures";
+
+import { Formatter } from "@stamhoofd/utility";
+import { AuthenticatedStructures } from "../../../helpers/AuthenticatedStructures";
+import { Context } from "../../../helpers/Context";
+
+type Params = Record<string, never>;
+type Query = undefined
+type Body = undefined
+type ResponseBody = OrganizationDetailedBillingStatus
+
+export class GetUserDetailedBilingStatusEndpoint extends Endpoint<Params, Query, Body, ResponseBody> {
+    protected doesMatch(request: Request): [true, Params] | [false] {
+        if (request.method != "GET") {
+            return [false];
+        }
+
+        const params = Endpoint.parseParameters(request.url, "/user/billing/status/detailed", {});
+
+        if (params) {
+            return [true, params as Params];
+        }
+        return [false];
+    }
+
+    async handle(_: DecodedRequest<Params, Query, Body>) {
+        const organization = await Context.setUserOrganizationScope();
+        const {user} = await Context.authenticate()
+
+        const memberIds = await Member.getMemberIdsWithRegistrationForUser(user)
+
+        const balanceItemModels = await BalanceItem.balanceItemsForUsersAndMembers(organization?.id ?? null, [user.id], memberIds);
+        
+        // todo: this is a duplicate query
+        const {payments, balanceItemPayments} = await BalanceItem.loadPayments(balanceItemModels)
+
+        return new Response(await GetUserDetailedBilingStatusEndpoint.getDetailedBillingStatus(balanceItemModels, payments));
+    }
+
+    static async getDetailedBillingStatus(balanceItemModels: BalanceItem[], paymentModels: Payment[]) {
+        const organizationIds = Formatter.uniqueArray([
+            ...balanceItemModels.map(b => b.organizationId),
+            ...paymentModels.map(p => p.organizationId).filter(p => p !== null)
+        ])
+        
+        // Group by organization you'll have to pay to
+        if (organizationIds.length === 0) {
+            return OrganizationDetailedBillingStatus.create({})
+        }
+
+        // Optimization: prevent fetching the organization we already have
+        const organization = Context.organization
+
+        let addOrganization = false
+        const i = organization ? organizationIds.indexOf(organization.id) : -1
+        if (i !== -1) {
+            organizationIds.splice(i, 1)
+            addOrganization = true
+        }
+
+        const organizationModels = await Organization.getByIDs(...organizationIds)
+
+        if (addOrganization && organization) {
+            organizationModels.push(organization)
+        }
+
+        const balanceItems = await BalanceItem.getStructureWithPayments(balanceItemModels)
+        const organizations = await AuthenticatedStructures.organizations(organizationModels)
+        const payments = await AuthenticatedStructures.paymentsGeneral(paymentModels, false)
+
+        return OrganizationDetailedBillingStatus.create({
+            organizations: organizations.map(o => {
+                return OrganizationDetailedBillingStatusItem.create({
+                    organization: o,
+                    balanceItems: balanceItems.filter(b => b.organizationId == o.id),
+                    payments: payments.filter(p => p.organizationId === o.id)
+                })
+            })
+        })
+    }
+}

package/src/endpoints/global/registration/RegisterMembersEndpoint.ts

@@ -3,7 +3,6 @@ import { ManyToOneRelation } from '@simonbackx/simple-database';
 import { Decoder } from '@simonbackx/simple-encoding';
 import { DecodedRequest, Endpoint, Request, Response } from "@simonbackx/simple-endpoints";
 import { SimpleError } from '@simonbackx/simple-errors';
-import { I18n } from '@stamhoofd/backend-i18n';
 import { Email } from '@stamhoofd/email';
 import { BalanceItem, BalanceItemPayment, Group, Member, MemberWithRegistrations, MolliePayment, MollieToken, Organization, PayconiqPayment, Payment, Platform, RateLimiter, Registration, User } from '@stamhoofd/models';
 import { BalanceItemRelation, BalanceItemRelationType, BalanceItemStatus, BalanceItemType, BalanceItemWithPayments, IDRegisterCheckout, PaymentCustomer, PaymentMethod, PaymentMethodHelper, PaymentProvider, PaymentStatus, Payment as PaymentStruct, PermissionLevel, PlatformFamily, PlatformMember, RegisterItem, RegisterResponse, Version } from "@stamhoofd/structures";
@@ -100,8 +99,23 @@ export class RegisterMembersEndpoint extends Endpoint<Params, Query, Body, Respo
         const deleteRegistrationIds = request.body.cart.deleteRegistrationIds
         const deleteRegistrationModels = (deleteRegistrationIds.length ? (await Registration.getByIDs(...deleteRegistrationIds)) : []).filter(r => r.organizationId === organization.id)
 
+        // Validate balance items (can only happen serverside)
+        const balanceItemIds = request.body.cart.balanceItems.map(i => i.item.id)
+        let memberBalanceItemsStructs: BalanceItemWithPayments[] = []
+        let balanceItemsModels: BalanceItem[] = []
+        if (balanceItemIds.length > 0) {
+            balanceItemsModels = await BalanceItem.where({ id: { sign:'IN', value: balanceItemIds }, organizationId: organization.id })
+            if (balanceItemsModels.length != balanceItemIds.length) {
+                throw new SimpleError({
+                    code: "invalid_data",
+                    message: "Oeps, één of meerdere openstaande bedragen in jouw winkelmandje zijn aangepast. Herlaad de pagina en probeer opnieuw."
+                })
+            }
+            memberBalanceItemsStructs = await BalanceItem.getStructureWithPayments(balanceItemsModels)
+        }
+
         const memberIds = Formatter.uniqueArray(
-            [...request.body.memberIds, ...deleteRegistrationModels.map(i => i.memberId)]
+            [...request.body.memberIds, ...deleteRegistrationModels.map(i => i.memberId), ...balanceItemsModels.map(i => i.memberId).filter(m => m !== null)]
         )
         const members = await Member.getBlobByIds(...memberIds)
         const groupIds = request.body.groupIds
@@ -160,6 +174,7 @@ export class RegisterMembersEndpoint extends Endpoint<Params, Query, Body, Respo
         
         const registrations: RegistrationWithMemberAndGroup[] = []
         const payRegistrations: {registration: RegistrationWithMemberAndGroup, item: RegisterItem}[] = []
+        const deactivatedRegistrationGroupIds: string[] = [];
 
         if (checkout.cart.isEmpty) {
             throw new SimpleError({
@@ -168,27 +183,6 @@ export class RegisterMembersEndpoint extends Endpoint<Params, Query, Body, Respo
             })
         }
 
-        // Update occupancies
-        // TODO: might not be needed in the future (for performance)
-        for (const group of groups) {
-            await group.updateOccupancy()
-        }
-
-        // Validate balance items (can only happen serverside)
-        const balanceItemIds = request.body.cart.balanceItems.map(i => i.item.id)
-        let memberBalanceItemsStructs: BalanceItemWithPayments[] = []
-        let balanceItemsModels: BalanceItem[] = []
-        if (balanceItemIds.length > 0) {
-            balanceItemsModels = await BalanceItem.where({ id: { sign:'IN', value: balanceItemIds }, organizationId: organization.id })
-            if (balanceItemsModels.length != balanceItemIds.length) {
-                throw new SimpleError({
-                    code: "invalid_data",
-                    message: "Oeps, één of meerdere openstaande bedragen in jouw winkelmandje zijn aangepast. Herlaad de pagina en probeer opnieuw."
-                })
-            }
-            memberBalanceItemsStructs = await BalanceItem.getStructureWithPayments(balanceItemsModels)
-        }
-
         // Validate the cart
         checkout.validate({memberBalanceItems: memberBalanceItemsStructs})
 
@@ -309,6 +303,7 @@ export class RegisterMembersEndpoint extends Endpoint<Params, Query, Body, Respo
         console.log('Registering members using whoWillPayNow', whoWillPayNow, checkout.paymentMethod, totalPrice)
 
         const createdBalanceItems: BalanceItem[] = []
+        const unrelatedCreatedBalanceItems: BalanceItem[] = []
         const shouldMarkValid = whoWillPayNow === 'nobody' || checkout.paymentMethod === PaymentMethod.Transfer || checkout.paymentMethod === PaymentMethod.PointOfSale || checkout.paymentMethod === PaymentMethod.Unknown
         
         // Create negative balance items
@@ -352,6 +347,7 @@ export class RegisterMembersEndpoint extends Endpoint<Params, Query, Body, Respo
             
             // Clear the registration
             await existingRegistration.deactivate()
+            deactivatedRegistrationGroupIds.push(existingRegistration.groupId);
 
             const group = groups.find(g => g.id === existingRegistration.groupId)
             if (!group) {
@@ -408,6 +404,7 @@ export class RegisterMembersEndpoint extends Endpoint<Params, Query, Body, Respo
                 await balanceItem2.save();
 
                 // do not add to createdBalanceItems array because we don't want to add this to the payment if we create a payment
+                unrelatedCreatedBalanceItems.push(balanceItem2)
             } else {
                 balanceItem.memberId = registration.memberId;
                 balanceItem.userId = user.id
@@ -590,27 +587,31 @@ export class RegisterMembersEndpoint extends Endpoint<Params, Query, Body, Respo
             }
 
             // Make sure every price is accurate before creating a payment
-            await BalanceItem.updateOutstanding(createdBalanceItems, organization.id)
-            const response = await this.createPayment({
-                balanceItems: mappedBalanceItems,
-                organization,
-                user,
-                checkout: request.body,
-                members
-            })
+            await BalanceItem.updateOutstanding([...createdBalanceItems, ...unrelatedCreatedBalanceItems])
+            try {
+                const response = await this.createPayment({
+                    balanceItems: mappedBalanceItems,
+                    organization,
+                    user,
+                    checkout: request.body,
+                    members
+                })
 
-            if (response) {
-                paymentUrl = response.paymentUrl
-                payment = response.payment
+                if (response) {
+                    paymentUrl = response.paymentUrl
+                    payment = response.payment
+                }
+            } finally {
+                // Update cached balance items pending amount (only created balance items, because those are involved in the payment)
+                await BalanceItem.updateOutstanding(createdBalanceItems)
             }
         } else {
-            await BalanceItem.updateOutstanding(createdBalanceItems, organization.id)
+            await BalanceItem.updateOutstanding([...createdBalanceItems, ...unrelatedCreatedBalanceItems])
         }
 
-
         // Update occupancy
         for (const group of groups) {
-            if (registrations.find(p => p.groupId === group.id) || checkout.cart.deleteRegistrations.find(p => p.groupId === group.id)) {
+            if (registrations.some(r => r.groupId === group.id) || deactivatedRegistrationGroupIds.some(id => id === group.id)) {
                 await group.updateOccupancy()
                 await group.save()
             }
@@ -636,12 +637,20 @@ export class RegisterMembersEndpoint extends Endpoint<Params, Query, Body, Respo
                 throw new Error('Unexpected balance item from other organization')
             }
 
-            if (price < 0 || (price > 0 && price > balanceItem.price - balanceItem.pricePaid)) {
+            if (price > 0 && price > Math.max(balanceItem.priceOpen, balanceItem.price - balanceItem.pricePaid)) {
                 throw new SimpleError({
                     code: "invalid_data",
-                    message: "Oeps, het bedrag dat je probeert te betalen is ongeldig. Herlaad de pagina en probeer opnieuw."
+                    message: "Oeps, het bedrag dat je probeert te betalen is ongeldig (het bedrag is hoger dan het bedrag dat je moet betalen). Herlaad de pagina en probeer opnieuw."
                 })
             }
+
+            if (price < 0 && price < Math.min(balanceItem.priceOpen, balanceItem.price - balanceItem.pricePaid)) {
+                throw new SimpleError({
+                    code: "invalid_data",
+                    message: "Oeps, het bedrag dat je probeert te betalen is ongeldig (het terug te krijgen bedrag is hoger dan het bedrag dat je kan terug krijgen). Herlaad de pagina en probeer opnieuw."
+                })
+            }
+
             totalPrice += price
 
             if (price > 0 && balanceItem.memberId) {

package/src/endpoints/organization/dashboard/billing/{GetBillingStatusEndpoint.ts → GetOrganizationBillingStatusEndpoint.ts}

@@ -2,19 +2,20 @@ import { DecodedRequest, Endpoint, Request, Response } from "@simonbackx/simple-
 import { OrganizationBillingStatus } from "@stamhoofd/structures";
 
 import { Context } from "../../../../helpers/Context";
+import { GetUserBilingStatusEndpoint } from "../../../global/registration/GetUserBillingStatusEndpoint";
 
 type Params = Record<string, never>;
 type Query = undefined;
 type ResponseBody = OrganizationBillingStatus;
 type Body = undefined;
 
-export class GetBillingStatusEndpoint extends Endpoint<Params, Query, Body, ResponseBody> {
+export class GetDetailedBillingStatusEndpoint extends Endpoint<Params, Query, Body, ResponseBody> {
     protected doesMatch(request: Request): [true, Params] | [false] {
         if (request.method != "GET") {
             return [false];
         }
 
-        const params = Endpoint.parseParameters(request.url, "/billing/status", {});
+        const params = Endpoint.parseParameters(request.url, "/organization/billing/status", {});
 
         if (params) {
             return [true, params as Params];
@@ -30,9 +31,7 @@ export class GetBillingStatusEndpoint extends Endpoint<Params, Query, Body, Resp
         if (!await Context.auth.canManageFinances(organization.id)) {
             throw Context.auth.error()
         }
-
-        return new Response(
-            OrganizationBillingStatus.create({})
-        )
+        
+        return new Response(await GetUserBilingStatusEndpoint.getBillingStatusForObjects([organization.id], null));
     }
 }

package/src/endpoints/organization/dashboard/billing/GetOrganizationDetailedBillingStatusEndpoint.ts

@@ -0,0 +1,58 @@
+import { DecodedRequest, Endpoint, Request, Response } from "@simonbackx/simple-endpoints";
+import { OrganizationDetailedBillingStatus, PaymentStatus } from "@stamhoofd/structures";
+
+import { BalanceItem, Payment } from "@stamhoofd/models";
+import { SQL } from "@stamhoofd/sql";
+import { Context } from "../../../../helpers/Context";
+import { GetUserDetailedBilingStatusEndpoint } from "../../../global/registration/GetUserDetailedBillingStatusEndpoint";
+
+type Params = Record<string, never>;
+type Query = undefined;
+type ResponseBody = OrganizationDetailedBillingStatus;
+type Body = undefined;
+
+export class GetOrganizationDetailedBillingStatusEndpoint extends Endpoint<Params, Query, Body, ResponseBody> {
+    protected doesMatch(request: Request): [true, Params] | [false] {
+        if (request.method != "GET") {
+            return [false];
+        }
+
+        if (request.getVersion() <= 334) {
+            // Deprecated
+            const params = Endpoint.parseParameters(request.url, "/billing/status/detailed", {});
+
+            if (params) {
+                return [true, params as Params];
+            }
+            return [false];
+        }
+
+        const params = Endpoint.parseParameters(request.url, "/organization/billing/status/detailed", {});
+
+        if (params) {
+            return [true, params as Params];
+        }
+        return [false];
+    }
+
+    async handle(_: DecodedRequest<Params, Query, Body>) {
+        const organization = await Context.setOrganizationScope();
+        await Context.authenticate()
+
+        // If the user has permission, we'll also search if he has access to the organization's key
+        if (!await Context.auth.canManageFinances(organization.id)) {
+            throw Context.auth.error()
+        }
+        
+        const balanceItemModels = await BalanceItem.balanceItemsForOrganization(organization.id);
+
+        const paymentModels = await Payment.select()
+            .where('payingOrganizationId', organization.id)
+            .andWhere(
+                SQL.whereNot('status', PaymentStatus.Failed)
+            )
+            .fetch()
+
+        return new Response(await GetUserDetailedBilingStatusEndpoint.getDetailedBillingStatus(balanceItemModels, paymentModels));
+    }
+}

package/src/endpoints/organization/dashboard/payments/PatchBalanceItemsEndpoint.ts

@@ -1,10 +1,9 @@
 import { AutoEncoderPatchType, Decoder, PatchableArrayAutoEncoder, PatchableArrayDecoder, StringDecoder } from '@simonbackx/simple-encoding';
 import { DecodedRequest, Endpoint, Request, Response } from "@simonbackx/simple-endpoints";
 import { SimpleError } from "@simonbackx/simple-errors";
-import { BalanceItem, Member, Order, Registration, User } from '@stamhoofd/models';
+import { BalanceItem, Member, Order, User } from '@stamhoofd/models';
 import { QueueHandler } from '@stamhoofd/queues';
 import { BalanceItemStatus, BalanceItemType, BalanceItemWithPayments, PermissionLevel } from "@stamhoofd/structures";
-import { Formatter } from '@stamhoofd/utility';
 
 import { Context } from '../../../../helpers/Context';
 
@@ -43,6 +42,7 @@ export class PatchBalanceItemsEndpoint extends Endpoint<Params, Query, Body, Res
         }
 
         const returnedModels: BalanceItem[] = []
+        const updateOutstandingBalance: BalanceItem[] = []
 
         // Keep track of updates
         const memberIds: string[] = []
@@ -80,6 +80,8 @@ export class PatchBalanceItemsEndpoint extends Endpoint<Params, Query, Body, Res
 
                 await model.save();
                 returnedModels.push(model);
+
+                updateOutstandingBalance.push(model)
             }
 
             for (const patch of request.body.getPatches()) {
@@ -145,11 +147,14 @@ export class PatchBalanceItemsEndpoint extends Endpoint<Params, Query, Body, Res
 
                 await model.save();
                 returnedModels.push(model);
+                
+                if (patch.unitPrice || patch.amount || patch.status) {
+                    updateOutstandingBalance.push(model)
+                }
             }
         });
 
-        await Member.updateOutstandingBalance(Formatter.uniqueArray(memberIds))
-        await Registration.updateOutstandingBalance(Formatter.uniqueArray(registrationIds), organization.id)
+        await BalanceItem.updateOutstanding(updateOutstandingBalance)
 
          return new Response(
             await BalanceItem.getStructureWithPayments(returnedModels)

package/src/endpoints/organization/dashboard/payments/PatchPaymentsEndpoint.ts

@@ -153,6 +153,8 @@ export class PatchPaymentsEndpoint extends Endpoint<Params, Query, Body, Respons
                 for (const balanceItem of balanceItems) {
                     await balanceItem.markUpdated(payment, organization)
                 }
+
+                await BalanceItem.updateOutstanding(balanceItems)
             }
 
             changedPayments.push(payment)

package/src/endpoints/organization/dashboard/registration-periods/PatchOrganizationRegistrationPeriodsEndpoint.ts

@@ -455,7 +455,6 @@ export class PatchOrganizationRegistrationPeriodsEndpoint extends Endpoint<Param
             }
         }
 
-        await model.updateOccupancy()
         await model.save();
         return model;
     }

package/src/endpoints/organization/shared/ExchangePaymentEndpoint.ts

@@ -72,8 +72,8 @@ export class ExchangePaymentEndpoint extends Endpoint<Params, Query, Body, Respo
     static async handlePaymentStatusUpdate(payment: Payment, organization: Organization, status: PaymentStatus) {
         if (payment.status === status) {
             return;
-        }
-        // const wasPaid = payment.paidAt !== null
+        }        
+
         if (status === PaymentStatus.Succeeded) {
             payment.status = PaymentStatus.Succeeded
             payment.paidAt = new Date()
@@ -90,39 +90,20 @@ export class ExchangePaymentEndpoint extends Endpoint<Params, Query, Body, Respo
                     await balanceItemPayment.markPaid(organization);
                 }
 
-                await BalanceItem.updateOutstanding(balanceItemPayments.map(p => p.balanceItem), organization.id)
+                await BalanceItem.updateOutstanding(balanceItemPayments.map(p => p.balanceItem))
             })
-
-            //if (!wasPaid && payment.provider === PaymentProvider.Buckaroo && payment.method) {
-            //    // Charge transaction fees
-            //    let fee = 0
-            //
-            //    if (payment.method === PaymentMethod.iDEAL) {
-            //        fee = calculateFee(payment.price, 21, 20); // € 0,21 + 0,2%
-            //    } else if (payment.method === PaymentMethod.Bancontact || payment.method === PaymentMethod.Payconiq) {
-            //        fee = calculateFee(payment.price, 24, 20); // € 0,24 + 0,2%
-            //    } else {
-            //        fee = calculateFee(payment.price, 25, 150); // € 0,25 + 1,5%
-            //    }
-            //
-            //    const name = "Transactiekosten voor "+PaymentMethodHelper.getName(payment.method)
-            //    const item = STInvoiceItem.create({
-            //        name,
-            //        description: "Via Buckaroo",
-            //        amount: 1,
-            //        unitPrice: fee,
-            //        canUseCredits: false
-            //    })
-            //    console.log("Scheduling transaction fee charge for ", payment.id, item)
-            //    await QueueHandler.schedule("billing/invoices-"+organization.id, async () => {
-            //        await STPendingInvoice.addItems(organization, [item])
-            //    });
-            //}
             return;
         }
 
+        const oldStatus = payment.status
+
+        // Save before updating balance items
+        payment.status = status
+        payment.paidAt = null
+        await payment.save();
+
         // If OLD status was succeeded, we need to revert the actions
-        if (payment.status === PaymentStatus.Succeeded) {
+        if (oldStatus === PaymentStatus.Succeeded) {
             // No longer succeeded
             await QueueHandler.schedule("balance-item-update/"+organization.id, async () => {
                 const balanceItemPayments = await BalanceItemPayment.balanceItem.load(
@@ -133,10 +114,11 @@ export class ExchangePaymentEndpoint extends Endpoint<Params, Query, Body, Respo
                     await balanceItemPayment.undoPaid(organization);
                 }
 
-                await BalanceItem.updateOutstanding(balanceItemPayments.map(p => p.balanceItem), organization.id)
+                await BalanceItem.updateOutstanding(balanceItemPayments.map(p => p.balanceItem))
             })
         }
         
+        // Moved to failed
         if (status == PaymentStatus.Failed) {
             await QueueHandler.schedule("balance-item-update/"+organization.id, async () => {
                 const balanceItemPayments = await BalanceItemPayment.balanceItem.load(
@@ -147,12 +129,12 @@ export class ExchangePaymentEndpoint extends Endpoint<Params, Query, Body, Respo
                     await balanceItemPayment.markFailed(organization);
                 }
 
-                await BalanceItem.updateOutstanding(balanceItemPayments.map(p => p.balanceItem), organization.id)
+                await BalanceItem.updateOutstanding(balanceItemPayments.map(p => p.balanceItem))
             })
         }
 
         // If OLD status was FAILED, we need to revert the actions
-        if (payment.status === PaymentStatus.Failed) { // OLD FAILED!! -> NOW PENDING
+        if (oldStatus === PaymentStatus.Failed) { // OLD FAILED!! -> NOW PENDING
             await QueueHandler.schedule("balance-item-update/"+organization.id, async () => {
                 const balanceItemPayments = await BalanceItemPayment.balanceItem.load(
                     (await BalanceItemPayment.where({paymentId: payment.id})).map(r => r.setRelation(BalanceItemPayment.payment, payment))
@@ -161,12 +143,10 @@ export class ExchangePaymentEndpoint extends Endpoint<Params, Query, Body, Respo
                 for (const balanceItemPayment of balanceItemPayments) {
                     await balanceItemPayment.undoFailed(organization);
                 }
+
+                await BalanceItem.updateOutstanding(balanceItemPayments.map(p => p.balanceItem))
             })
         }
-
-        payment.status = status
-        payment.paidAt = null
-        await payment.save();
     }
 
     /**

package/src/helpers/AdminPermissionChecker.ts

@@ -992,16 +992,21 @@ export class AdminPermissionChecker {
             })
         }
 
-        if (data.details.securityCode !== undefined) {
-            // Unset silently
-            data.details.securityCode = undefined
-        }
-
         const hasRecordAnswers = !!data.details.recordAnswers;
         const hasNotes = data.details.notes !== undefined;
         const isSetFinancialSupportTrue = data.details.shouldApplyReducedPrice;
         const isUserManager = this.isUserManager(member);
 
+        if (data.details.securityCode !== undefined) {
+            const hasFullAccess = await this.canAccessMember(member, PermissionLevel.Full);
+
+            // can only be set to null, and only if can access member with full access
+            if(!hasFullAccess || data.details.securityCode !== null) {
+                // Unset silently
+                data.details.securityCode = undefined
+            }
+        }
+
         if (hasRecordAnswers) {
             if (!(data.details.recordAnswers instanceof PatchMap)) {
                 throw new SimpleError({

package/src/seeds/1726055544-balance-item-paid.ts

@@ -0,0 +1,11 @@
+import { Migration } from '@simonbackx/simple-database';
+import { BalanceItem } from '@stamhoofd/models';
+
+export default new Migration(async () => {
+    if (STAMHOOFD.environment == "test") {
+        console.log("skipped in tests")
+        return;
+    }
+
+    await BalanceItem.updatePricePaid('all')
+})

package/src/seeds/1726055545-balance-item-pending.ts

@@ -0,0 +1,11 @@
+import { Migration } from '@simonbackx/simple-database';
+import { BalanceItem } from '@stamhoofd/models';
+
+export default new Migration(async () => {
+    if (STAMHOOFD.environment == "test") {
+        console.log("skipped in tests")
+        return;
+    }
+
+    await BalanceItem.updatePricePending('all')
+})

package/src/endpoints/global/registration/GetUserBalanceEndpoint.ts

@@ -1,39 +0,0 @@
-import { DecodedRequest, Endpoint, Request, Response } from "@simonbackx/simple-endpoints";
-import { BalanceItem, Member } from "@stamhoofd/models";
-import { BalanceItemWithPayments } from "@stamhoofd/structures";
-
-import { Context } from "../../../helpers/Context";
-
-type Params = Record<string, never>;
-type Query = undefined
-type Body = undefined
-type ResponseBody = BalanceItemWithPayments[]
-
-export class GetUserBalanceEndpoint extends Endpoint<Params, Query, Body, ResponseBody> {
-    protected doesMatch(request: Request): [true, Params] | [false] {
-        if (request.method != "GET") {
-            return [false];
-        }
-
-        const params = Endpoint.parseParameters(request.url, "/balance", {});
-
-        if (params) {
-            return [true, params as Params];
-        }
-        return [false];
-    }
-
-    async handle(_: DecodedRequest<Params, Query, Body>) {
-        const organization = await Context.setUserOrganizationScope();
-        const {user} = await Context.authenticate()
-
-        const members = await Member.getMembersWithRegistrationForUser(user)
-
-        // Get all balance items for this member or users
-        const balanceItems = await BalanceItem.balanceItemsForUsersAndMembers(organization?.id ?? null, [user.id], members.map(m => m.id))
-
-        return new Response(
-            await BalanceItem.getStructureWithPayments(balanceItems)
-        );
-    }
-}

package/src/endpoints/organization/dashboard/billing/GetDetailedBillingStatusEndpoint.ts

@@ -1,77 +0,0 @@
-import { DecodedRequest, Endpoint, Request, Response } from "@simonbackx/simple-endpoints";
-import { OrganizationDetailedBillingStatus, OrganizationDetailedBillingStatusItem, PaymentMethod, PaymentStatus } from "@stamhoofd/structures";
-
-import { BalanceItem, Organization, Payment } from "@stamhoofd/models";
-import { SQL } from "@stamhoofd/sql";
-import { Formatter } from "@stamhoofd/utility";
-import { AuthenticatedStructures } from "../../../../helpers/AuthenticatedStructures";
-import { Context } from "../../../../helpers/Context";
-
-type Params = Record<string, never>;
-type Query = undefined;
-type ResponseBody = OrganizationDetailedBillingStatus;
-type Body = undefined;
-
-export class GetDetailedBillingStatusEndpoint extends Endpoint<Params, Query, Body, ResponseBody> {
-    protected doesMatch(request: Request): [true, Params] | [false] {
-        if (request.method != "GET") {
-            return [false];
-        }
-
-        const params = Endpoint.parseParameters(request.url, "/billing/status/detailed", {});
-
-        if (params) {
-            return [true, params as Params];
-        }
-        return [false];
-    }
-
-    async handle(_: DecodedRequest<Params, Query, Body>) {
-        const organization = await Context.setOrganizationScope();
-        await Context.authenticate()
-
-        // If the user has permission, we'll also search if he has access to the organization's key
-        if (!await Context.auth.canManageFinances(organization.id)) {
-            throw Context.auth.error()
-        }
-        
-        const balanceItemModels = await BalanceItem.balanceItemsForOrganization(organization.id);
-
-        // Hide pending online payments
-        const paymentModels = await Payment.select()
-            .where('payingOrganizationId', organization.id)
-            .andWhere(
-                SQL.whereNot('status', PaymentStatus.Failed)
-            )
-            .fetch()
-
-        const organizationIds = Formatter.uniqueArray([
-            ...balanceItemModels.map(b => b.organizationId),
-            ...paymentModels.map(p => p.organizationId).filter(p => p !== null)
-        ])
-        
-        // Group by organization you'll have to pay to
-        if (organizationIds.length === 0) {
-            return new Response(
-                OrganizationDetailedBillingStatus.create({})
-            )
-        }
-
-        const balanceItems = await BalanceItem.getStructureWithPayments(balanceItemModels)
-        const organizationModels = await Organization.getByIDs(...organizationIds)
-        const organizations = await AuthenticatedStructures.organizations(organizationModels)
-        const payments = await AuthenticatedStructures.paymentsGeneral(paymentModels, false)
-
-        return new Response(
-            OrganizationDetailedBillingStatus.create({
-                organizations: organizations.map(o => {
-                    return OrganizationDetailedBillingStatusItem.create({
-                        organization: o,
-                        balanceItems: balanceItems.filter(b => b.organizationId == o.id),
-                        payments: payments.filter(p => p.organizationId === o.id)
-                    })
-                })
-            })
-        )
-    }
-}