@stamhoofd/backend 2.115.1 → 2.116.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@stamhoofd/backend",
-    "version": "2.115.1",
+    "version": "2.116.0",
     "main": "./dist/index.js",
     "exports": {
         ".": {
@@ -55,14 +55,14 @@
         "@simonbackx/simple-encoding": "2.23.1",
         "@simonbackx/simple-endpoints": "1.20.1",
         "@simonbackx/simple-logging": "^1.0.1",
-        "@stamhoofd/backend-i18n": "2.115.1",
-        "@stamhoofd/backend-middleware": "2.115.1",
-        "@stamhoofd/email": "2.115.1",
-        "@stamhoofd/models": "2.115.1",
-        "@stamhoofd/queues": "2.115.1",
-        "@stamhoofd/sql": "2.115.1",
-        "@stamhoofd/structures": "2.115.1",
-        "@stamhoofd/utility": "2.115.1",
+        "@stamhoofd/backend-i18n": "2.116.0",
+        "@stamhoofd/backend-middleware": "2.116.0",
+        "@stamhoofd/email": "2.116.0",
+        "@stamhoofd/models": "2.116.0",
+        "@stamhoofd/queues": "2.116.0",
+        "@stamhoofd/sql": "2.116.0",
+        "@stamhoofd/structures": "2.116.0",
+        "@stamhoofd/utility": "2.116.0",
         "archiver": "^7.0.1",
         "axios": "^1.13.2",
         "cookie": "^0.7.0",
@@ -80,5 +80,5 @@
     "publishConfig": {
         "access": "public"
     },
-    "gitHead": "4b692d62820d095e00524e3c0214e8f73a69506e"
+    "gitHead": "842040d2dcbde452fb25178f3586c8d5293b974f"
 }

package/src/audit-logs/ModelLogger.ts

@@ -2,8 +2,8 @@ import { Model, ModelEvent } from '@simonbackx/simple-database';
 import { AuditLog } from '@stamhoofd/models';
 import { ObjectDiffer } from '@stamhoofd/object-differ';
 import { AuditLogPatchItem, AuditLogPatchItemType, AuditLogReplacement, AuditLogSource, AuditLogType } from '@stamhoofd/structures';
-import { ContextInstance } from '../helpers/Context';
-import { AuditLogService } from '../services/AuditLogService';
+import { ContextInstance } from '../helpers/Context.js';
+import { AuditLogService } from '../services/AuditLogService.js';
 
 export type ModelEventLogOptions<D> = {
     type: AuditLogType;
@@ -197,7 +197,6 @@ export class ModelLogger<ModelType extends typeof Model, M extends InstanceType<
                     }
                 }
                 else {
-                    console.log('No changes');
                     return false;
                 }
             }

package/src/boot.ts

@@ -16,13 +16,12 @@ import { SetupStepUpdater } from './helpers/SetupStepUpdater.js';
 import { ContextMiddleware } from './middleware/ContextMiddleware.js';
 import { AuditLogService } from './services/AuditLogService.js';
 import { BalanceItemService } from './services/BalanceItemService.js';
+import { CpuService } from './services/CpuService.js';
 import { DocumentService } from './services/DocumentService.js';
 import { FileSignService } from './services/FileSignService.js';
 import { PlatformMembershipService } from './services/PlatformMembershipService.js';
 import { UitpasService } from './services/uitpas/UitpasService.js';
 import { UniqueUserService } from './services/UniqueUserService.js';
-import { CpuService } from './services/CpuService.js';
-import { SQLLogger } from '@stamhoofd/sql';
 
 process.on('unhandledRejection', (error: Error) => {
     console.error('unhandledRejection');
@@ -149,8 +148,8 @@ export const boot = async (options: { killProcess: boolean }) => {
         CpuService.startMonitoring();
     }
     else if (STAMHOOFD.environment === 'development') {
-        SQLLogger.slowQueryThresholdMs = 200;
-        SQLLogger.explainAllAndLogInefficient = true;
+        const { loadDebugFunctions } = await import('./debug.js');
+        loadDebugFunctions({ routerServer });
     }
 
     if (routerServer.server) {

package/src/crons/delete-archived-data.ts

@@ -0,0 +1,47 @@
+/**
+ * We cannot keep deletedAt data forever. We only keep it that way to debug things or be able to easily restore data.
+ * After 1 month of deletion, we should delete it permanently.
+ */
+
+import { registerCron } from '@stamhoofd/crons';
+import { Group } from '@stamhoofd/models';
+
+let lastRunDate: number | null = null;
+const keepDeletedDataForDays = 30 * 24 * 60 * 60 * 1000; // 30 days in milliseconds
+
+registerCron('delete-archived-data', deleteArchivedData);
+
+function shouldRun() {
+    const now = new Date();
+
+    if (now.getDate() === lastRunDate) {
+        return false;
+    }
+
+    const hour = now.getHours();
+
+    // between 4 and 5 AM - except in development
+    if (hour !== 4 && STAMHOOFD.environment !== 'development') {
+        return false;
+    }
+
+    return true;
+}
+
+async function deleteArchivedData() {
+    if (!shouldRun()) {
+        return;
+    }
+
+    const now = new Date();
+    lastRunDate = now.getDate();
+
+    console.log('Deleting archived data...');
+
+    const result = await Group.delete()
+        .where('deletedAt', '<', new Date(now.getTime() - keepDeletedDataForDays))
+        .delete();
+    console.log('Deleted groups:', result.affectedRows);
+
+    // For now, we don't delete orders yet (can 'fuck' up pagination because deleted orders are still used in the frontend to remove cached orders)
+}

package/src/crons/index.ts

@@ -4,3 +4,4 @@ import './endFunctionsOfUsersWithoutRegistration.js';
 import './update-cached-balances.js';
 import './balance-emails.js';
 import './delete-old-email-drafts.js';
+import './delete-archived-data.js';

package/src/debug.ts

@@ -0,0 +1,230 @@
+import { Column } from '@simonbackx/simple-database';
+import { EncodedResponse, ResponseMiddleware, Router, RouterServer } from '@simonbackx/simple-endpoints';
+import { logger } from '@simonbackx/simple-logging';
+import { requestPrefix } from '@stamhoofd/backend-middleware';
+
+import { AutoEncoder } from '@simonbackx/simple-encoding';
+import { Request, Response } from '@simonbackx/simple-endpoints';
+import { SQLLogger } from '@stamhoofd/sql';
+import { ContextInstance } from './helpers/Context.js';
+
+export function loadDebugFunctions({ routerServer }: { routerServer: RouterServer }) {
+    SQLLogger.slowQueryThresholdMs = 75;
+    SQLLogger.explainAllAndLogInefficient = true;
+
+    if (process.env.LOG_REQUEST_QUERIES) {
+        loadRequestQueries({ routerServer });
+    }
+
+    if (process.env.LOG_TIMERS) {
+        loadTimers({ routerServer });
+    }
+}
+
+export function loadRequestQueries({ routerServer }: { routerServer: RouterServer }) {
+    SQLLogger.customLoggers.push((query, params, elapsedTimeMs) => {
+        const context = ContextInstance.optional;
+        if (context) {
+            context.queries.push({ query, time: elapsedTimeMs });
+        }
+    });
+    const LogQueriesMiddleware: ResponseMiddleware = {
+        handleResponse(request: Request, response: Response, error?: Error) {
+            const prefix = !error ? [] : requestPrefix(request, 'error');
+            const context = (request as any)._context as ContextInstance | undefined;
+
+            if (context && request.method !== 'OPTIONS') {
+                logger.log(
+                    ...prefix,
+                    ' - Queries ' + context.queries.map(q => `\n- ${q.query} (${q.time?.toFixed(2) ?? '-'}ms)`).join(''),
+                );
+            }
+        },
+    };
+    routerServer.addResponseMiddleware(LogQueriesMiddleware);
+}
+
+export function loadTimers({ routerServer }: { routerServer: RouterServer }) {
+    const jsonStringify = JSON.stringify;
+    JSON.stringify = function (...args) {
+        const startTime = process.hrtime.bigint();
+        const result = jsonStringify.apply(this, args);
+        const elapsedTime = process.hrtime.bigint() - startTime;
+        const elapsedTimeMs = Number(elapsedTime) / 1000 / 1000;
+
+        if (elapsedTimeMs > 1) {
+            console.log('JSON.stringify took ' + elapsedTimeMs.toFixed(2) + 'ms');
+        }
+        // Count total request
+        const context = ContextInstance.optional;
+        if (context) {
+            context.timers.set('JSON.stringify', (context.timers.get('JSON.stringify') ?? 0) + elapsedTimeMs);
+        }
+        return result;
+    };
+
+    const jsonParse = JSON.parse;
+    JSON.parse = function (...args) {
+        const startTime = process.hrtime.bigint();
+        const result = jsonParse.apply(this, args);
+        const elapsedTime = process.hrtime.bigint() - startTime;
+        const elapsedTimeMs = Number(elapsedTime) / 1000 / 1000;
+
+        if (elapsedTimeMs > 1) {
+            console.log('JSON.parse took ' + elapsedTimeMs.toFixed(2) + 'ms');
+        }
+
+        // Count total request
+        const context = ContextInstance.optional;
+        if (context) {
+            context.timers.set('JSON.parse', (context.timers.get('JSON.parse') ?? 0) + elapsedTimeMs);
+        }
+        return result;
+    };
+
+    const routerDecode = Router.prototype.decode;
+    Router.prototype.decode = function (...args) {
+        const startTime = process.hrtime.bigint();
+        const result = routerDecode.apply(this, args);
+        const elapsedTime = process.hrtime.bigint() - startTime;
+        const elapsedTimeMs = Number(elapsedTime) / 1000 / 1000;
+
+        if (elapsedTimeMs > 1) {
+            console.log('Router.decode took ' + elapsedTimeMs.toFixed(2) + 'ms');
+        }
+
+        // Count total request
+        const context = ContextInstance.optional;
+        if (context) {
+            context.timers.set('Router.decode', (context.timers.get('Router.decode') ?? 0) + elapsedTimeMs);
+        }
+        return result;
+    };
+
+    const encodedResponseEncode = EncodedResponse.encode;
+    EncodedResponse.encode = function (...args) {
+        const startTime = process.hrtime.bigint();
+        const result = encodedResponseEncode.apply(this, args);
+        const elapsedTime = process.hrtime.bigint() - startTime;
+        const elapsedTimeMs = Number(elapsedTime) / 1000 / 1000;
+
+        if (elapsedTimeMs > 1) {
+            console.log('EncodedResponse.encode took ' + elapsedTimeMs.toFixed(2) + 'ms');
+        }
+
+        // Count total request
+        const context = ContextInstance.optional;
+        if (context) {
+            context.timers.set('EncodedResponse.encode', (context.timers.get('EncodedResponse.encode') ?? 0) + elapsedTimeMs);
+        }
+
+        return result;
+    };
+
+    // Change
+    const columnTo = Column.prototype.to;
+    Column.prototype.to = function (this: Column, value: any) {
+        const startTime = process.hrtime.bigint();
+        const result = columnTo.apply(this, [value]);
+        const elapsedTime = process.hrtime.bigint() - startTime;
+        const elapsedTimeMs = Number(elapsedTime) / 1000 / 1000;
+
+        if (elapsedTimeMs > 1) {
+            console.log(`Column.to for ${this.constructor.name} took ${elapsedTimeMs.toFixed(2)}ms`);
+        }
+
+        // Count total request
+        const context = ContextInstance.optional;
+        if (context) {
+            context.timers.set(`Column.to`, (context.timers.get(`Column.to`) ?? 0) + elapsedTimeMs);
+        }
+        return result;
+    };
+
+    const columnFrom = Column.prototype.from;
+    Column.prototype.from = function (this: Column, value: any) {
+        const startTime = process.hrtime.bigint();
+        const result = columnFrom.apply(this, [value]);
+        const elapsedTime = process.hrtime.bigint() - startTime;
+        const elapsedTimeMs = Number(elapsedTime) / 1000 / 1000;
+
+        if (elapsedTimeMs > 1) {
+            console.log(`Column.from for ${this.constructor.name} took ${elapsedTimeMs.toFixed(2)}ms`);
+        }
+
+        // Count total request
+        const context = ContextInstance.optional;
+        if (context) {
+            context.timers.set(`Column.from`, (context.timers.get(`Column.from`) ?? 0) + elapsedTimeMs);
+        }
+        return result;
+    };
+
+    const decode = AutoEncoder.decode;
+    AutoEncoder.decode = function (...args) {
+        const c = args[0].context as any;
+        if (c._didTrack) {
+            return decode.apply(this, args);
+        }
+        c._didTrack = true;
+
+        const startTime = process.hrtime.bigint();
+        const result = decode.apply(this, args);
+        const elapsedTime = process.hrtime.bigint() - startTime;
+        const elapsedTimeMs = Number(elapsedTime) / 1000 / 1000;
+
+        if (elapsedTimeMs > 1) {
+            console.log('AutoEncoder.decode took ' + elapsedTimeMs.toFixed(2) + 'ms');
+        }
+
+        const context = ContextInstance.optional;
+        if (context) {
+            context.timers.set('AutoEncoder.decode', (context.timers.get('AutoEncoder.decode') ?? 0) + elapsedTimeMs);
+        }
+
+        return result;
+    };
+
+    const encode = AutoEncoder.prototype.encode;
+    AutoEncoder.prototype.encode = function (this: AutoEncoder, ...args) {
+        const c = args[0] as any;
+        if (c._didTrack) {
+            return encode.apply(this, args);
+        }
+        c._didTrack = true;
+
+        const startTime = process.hrtime.bigint();
+        const result = encode.apply(this, args);
+        const elapsedTime = process.hrtime.bigint() - startTime;
+        const elapsedTimeMs = Number(elapsedTime) / 1000 / 1000;
+
+        if (elapsedTimeMs > 1) {
+            console.log('AutoEncoder.encode took ' + elapsedTimeMs.toFixed(2) + 'ms');
+        }
+
+        const context = ContextInstance.optional;
+        if (context) {
+            context.timers.set('AutoEncoder.encode', (context.timers.get('AutoEncoder.encode') ?? 0) + elapsedTimeMs);
+        }
+
+        return result;
+    };
+
+    // Add middleware to log timers
+    const LogTimersMiddleware: ResponseMiddleware = {
+        handleResponse(request: Request, response: Response, error?: Error) {
+            const prefix = !error ? [] : requestPrefix(request, 'error');
+            const context = (request as any)._context as ContextInstance | undefined;
+
+            if (context && request.method !== 'OPTIONS') {
+                logger.log(
+                    ...prefix,
+                    ' - Timers: ' + Array.from(context.timers.entries()).sort((a, b) => {
+                        return b[1] - a[1];
+                    }).map(([key, value]) => `\n- ${key}: ${value.toFixed(2)}ms`).join(''),
+                );
+            }
+        },
+    };
+    routerServer.addResponseMiddleware(LogTimersMiddleware);
+}

package/src/email-recipient-loaders/documents.ts

@@ -9,7 +9,7 @@ async function fetch(query: LimitedFilteredRequest) {
     const recipients: EmailRecipient[] = [];
     const memberIds = new Set(result.results.map(doc => doc.memberId).filter(id => id !== null)); // silently skip null memberIds
 
-    const members = await Member.getBlobByIds(...memberIds);
+    const members = await Member.getByIdsWithUsers(...memberIds);
     for (const member of members) {
         const emails = member.details.getNotificationEmails();
         for (const user of member.users) {

package/src/email-recipient-loaders/payments.ts

@@ -254,7 +254,7 @@ async function getMemberRecipients(ids: { memberId: string; payment: PaymentGene
     }
 
     const allMemberIds = Formatter.uniqueArray(ids.map(i => i.memberId));
-    const members = await Member.getBlobByIds(...allMemberIds);
+    const members = await Member.getByIdsWithUsers(...allMemberIds);
 
     const results: EmailRecipient[] = [];
 

package/src/endpoints/auth/PatchUserEndpoint.ts

@@ -4,9 +4,9 @@ import { SimpleError } from '@simonbackx/simple-errors';
 import { EmailVerificationCode, Member, PasswordToken, Platform, Token, User } from '@stamhoofd/models';
 import { LoginMethod, NewUser, PermissionLevel, SignupResponse, UserPermissions, UserWithMembers } from '@stamhoofd/structures';
 
-import { Context } from '../../helpers/Context';
-import { MemberUserSyncer } from '../../helpers/MemberUserSyncer';
-import { AuthenticatedStructures } from '../../helpers/AuthenticatedStructures';
+import { Context } from '../../helpers/Context.js';
+import { MemberUserSyncer } from '../../helpers/MemberUserSyncer.js';
+import { AuthenticatedStructures } from '../../helpers/AuthenticatedStructures.js';
 
 type Params = { id: string };
 type Query = undefined;
@@ -49,7 +49,7 @@ export class PatchUserEndpoint extends Endpoint<Params, Query, Body, ResponseBod
 
         if (await Context.auth.canEditUserName(editUser)) {
             if (editUser.memberId) {
-                const member = await Member.getWithRegistrations(editUser.memberId);
+                const member = await Member.getByID(editUser.memberId);
                 if (member) {
                     member.details.firstName = request.body.firstName ?? member.details.firstName;
                     member.details.lastName = request.body.lastName ?? member.details.lastName;

package/src/endpoints/global/members/GetMemberFamilyEndpoint.ts

@@ -1,9 +1,9 @@
 import { DecodedRequest, Endpoint, Request, Response } from '@simonbackx/simple-endpoints';
-import { Member, MemberWithRegistrations } from '@stamhoofd/models';
+import { Member, MemberWithUsersRegistrationsAndGroups } from '@stamhoofd/models';
 
 import { MembersBlob } from '@stamhoofd/structures';
-import { AuthenticatedStructures } from '../../../helpers/AuthenticatedStructures';
-import { Context } from '../../../helpers/Context';
+import { AuthenticatedStructures } from '../../../helpers/AuthenticatedStructures.js';
+import { Context } from '../../../helpers/Context.js';
 type Params = { id: string };
 type Query = undefined;
 type Body = undefined;
@@ -47,7 +47,7 @@ export class GetMemberFamilyEndpoint extends Endpoint<Params, Query, Body, Respo
 
         let foundMember = false;
 
-        const validatedMembers: MemberWithRegistrations[] = [];
+        const validatedMembers: MemberWithUsersRegistrationsAndGroups[] = [];
 
         for (const member of members) {
             if (member.id === request.params.id) {

package/src/endpoints/global/members/GetMembersEndpoint.ts

@@ -126,10 +126,7 @@ export class GetMembersEndpoint extends Endpoint<Params, Query, Body, ResponseBo
             }
         }
 
-        const query = SQL
-            .select(
-                SQL.column('members', 'id'),
-            )
+        const query = Member.select()
             .setMaxExecutionTime(15 * 1000)
             .from(
                 SQL.table('members'),
@@ -257,7 +254,7 @@ export class GetMembersEndpoint extends Endpoint<Params, Query, Body, ResponseBo
 
     static async buildData(requestQuery: LimitedFilteredRequest, permissionLevel = PermissionLevel.Read) {
         const query = await GetMembersEndpoint.buildQuery(requestQuery, permissionLevel);
-        let data: SQLResultNamespacedRow[];
+        let data: Member[];
 
         try {
             data = await query.fetch();
@@ -273,16 +270,7 @@ export class GetMembersEndpoint extends Endpoint<Params, Query, Body, ResponseBo
             throw error;
         }
 
-        const memberIds = data.map((r) => {
-            if (typeof r.members.id === 'string') {
-                return r.members.id;
-            }
-            throw new Error('Expected string');
-        });
-
-        const _members = await Member.getBlobByIds(...memberIds);
-        // Make sure members is in same order as memberIds
-        const members = memberIds.map(id => _members.find(m => m.id === id)!);
+        const members = await Member.loadRegistrationsAndUsers(data, true);
 
         for (const member of members) {
             if (!await Context.auth.canAccessMember(member, permissionLevel)) {
@@ -293,7 +281,7 @@ export class GetMembersEndpoint extends Endpoint<Params, Query, Body, ResponseBo
 
         let next: LimitedFilteredRequest | undefined;
 
-        if (memberIds.length >= requestQuery.limit) {
+        if (members.length >= requestQuery.limit) {
             const lastObject = members[members.length - 1];
             const nextFilter = getSortFilter(lastObject, sorters, requestQuery.sort);
 

package/src/endpoints/global/members/PatchOrganizationMembersEndpoint.ts

@@ -2,7 +2,7 @@ import { OneToManyRelation } from '@simonbackx/simple-database';
 import { AutoEncoderPatchType, ConvertArrayToPatchableArray, Decoder, isEmptyPatch, isPatchableArray, PatchableArray, PatchableArrayAutoEncoder, PatchableArrayDecoder, StringDecoder } from '@simonbackx/simple-encoding';
 import { DecodedRequest, Endpoint, Request, Response } from '@simonbackx/simple-endpoints';
 import { SimpleError } from '@simonbackx/simple-errors';
-import { AuditLog, BalanceItem, Document, Group, Member, MemberFactory, MemberPlatformMembership, MemberResponsibilityRecord, MemberWithRegistrations, mergeTwoMembers, Organization, Platform, RateLimiter, Registration, RegistrationPeriod, User } from '@stamhoofd/models';
+import { AuditLog, BalanceItem, Document, Group, Member, MemberFactory, MemberPlatformMembership, MemberResponsibilityRecord, MemberWithRegistrations, MemberWithUsersAndRegistrations, MemberWithUsersRegistrationsAndGroups, mergeTwoMembers, Organization, Platform, RateLimiter, Registration, RegistrationPeriod, User } from '@stamhoofd/models';
 import { AuditLogReplacement, AuditLogReplacementType, AuditLogSource, AuditLogType, EmergencyContact, GroupType, MemberDetails, MemberResponsibility, MembersBlob, MemberWithRegistrationsBlob, Parent, PermissionLevel, SetupStepType } from '@stamhoofd/structures';
 import { Formatter } from '@stamhoofd/utility';
 
@@ -72,7 +72,7 @@ export class PatchOrganizationMembersEndpoint extends Endpoint<Params, Query, Bo
             }
         }
 
-        const members: MemberWithRegistrations[] = [];
+        const members: MemberWithUsersRegistrationsAndGroups[] = [];
 
         const platform = await Platform.getShared();
 
@@ -717,7 +717,7 @@ export class PatchOrganizationMembersEndpoint extends Endpoint<Params, Query, Bo
 
         // Loop all members one by one
         for (const id of ids) {
-            const member = await Member.getWithRegistrations(id);
+            const member = await Member.getByIdWithUsersAndRegistrations(id);
             if (!member || !await Context.auth.canDeleteMember(member)) {
                 throw Context.auth.error($t(`39f5696c-3755-429f-b0da-a0ca920ed11e`));
             }
@@ -754,9 +754,10 @@ export class PatchOrganizationMembersEndpoint extends Endpoint<Params, Query, Bo
         }
     }
 
-    static async mergeDuplicateRelations(member: MemberWithRegistrations, patch: AutoEncoderPatchType<MemberDetails> | MemberDetails) {
-        const _familyMembers = await Member.getFamilyWithRegistrations(member.id);
-        const familyMembers: typeof _familyMembers = [];
+    static async mergeDuplicateRelations(member: Member, patch: AutoEncoderPatchType<MemberDetails> | MemberDetails) {
+        const __familyMembers = await Member.getFamily(member.id);
+        const _familyMembers = await Member.loadRegistrationsAndUsers(__familyMembers);
+        const familyMembers: Member[] = [];
         // Only modify members if we have write access to them (this avoids issues with overriding data)
         for (const member of _familyMembers) {
             if (await Context.auth.canAccessMember(member, PermissionLevel.Write)) {
@@ -901,7 +902,7 @@ export class PatchOrganizationMembersEndpoint extends Endpoint<Params, Query, Bo
         }
     }
 
-    static async checkSecurityCode(member: MemberWithRegistrations, securityCode: string | null | undefined, type: 'put' | 'patch') {
+    static async checkSecurityCode(member: MemberWithUsersRegistrationsAndGroups, securityCode: string | null | undefined, type: 'put' | 'patch') {
         if ((type === 'put' && await member.isSafeToMergeDuplicateWithoutSecurityCode()) || await Context.auth.canAccessMember(member, PermissionLevel.Write)) {
             console.log('checkSecurityCode: without security code: allowed for ' + member.id);
         }

package/src/endpoints/global/registration/GetUserDetailedPayableBalanceEndpoint.ts

@@ -3,8 +3,8 @@ import { BalanceItem, Member, Organization, Payment } from '@stamhoofd/models';
 import { DetailedPayableBalanceCollection, DetailedPayableBalance } from '@stamhoofd/structures';
 
 import { Formatter } from '@stamhoofd/utility';
-import { AuthenticatedStructures } from '../../../helpers/AuthenticatedStructures';
-import { Context } from '../../../helpers/Context';
+import { AuthenticatedStructures } from '../../../helpers/AuthenticatedStructures.js';
+import { Context } from '../../../helpers/Context.js';
 
 type Params = Record<string, never>;
 type Query = undefined;

package/src/endpoints/global/registration/GetUserDocumentsEndpoint.ts

@@ -3,7 +3,7 @@ import { Document, DocumentTemplate, Member } from '@stamhoofd/models';
 import { Document as DocumentStruct, DocumentStatus } from '@stamhoofd/structures';
 import { Sorter } from '@stamhoofd/utility';
 
-import { Context } from '../../../helpers/Context';
+import { Context } from '../../../helpers/Context.js';
 type Params = Record<string, never>;
 type Query = undefined;
 type Body = undefined;
@@ -31,7 +31,7 @@ export class GetUserDocumentsEndpoint extends Endpoint<Params, Query, Body, Resp
         const organization = await Context.setUserOrganizationScope();
         const { user } = await Context.authenticate();
 
-        const members = await Member.getMembersWithRegistrationForUser(user);
+        const members = await Member.getMembersForUser(user);
         let templates = organization ? await DocumentTemplate.where({ status: 'Published', organizationId: organization.id }) : null;
         const memberIds = members.map(m => m.id);
         const templateIds = templates ? templates.map(t => t.id) : null;

package/src/endpoints/global/registration/RegisterMembersEndpoint.test.ts

@@ -1,7 +1,7 @@
 import { PatchMap } from '@simonbackx/simple-encoding';
 import { Request } from '@simonbackx/simple-endpoints';
 import { EmailMocker } from '@stamhoofd/email';
-import { BalanceItemFactory, Group, GroupFactory, Member, MemberFactory, MemberWithRegistrations, Organization, OrganizationFactory, OrganizationRegistrationPeriodFactory, Registration, RegistrationFactory, RegistrationPeriod, RegistrationPeriodFactory, Token, UserFactory } from '@stamhoofd/models';
+import { BalanceItemFactory, Group, GroupFactory, Member, MemberFactory, MemberWithUsersRegistrationsAndGroups, Organization, OrganizationFactory, OrganizationRegistrationPeriodFactory, Registration, RegistrationFactory, RegistrationPeriod, RegistrationPeriodFactory, Token, UserFactory } from '@stamhoofd/models';
 import { AccessRight, BalanceItemCartItem, BalanceItemStatus, BalanceItemType, BooleanStatus, Company, GroupOption, GroupOptionMenu, IDRegisterCart, IDRegisterCheckout, IDRegisterItem, OrganizationPackages, PaymentCustomer, PaymentMethod, PermissionLevel, Permissions, PermissionsResourceType, ReduceablePrice, RegisterItemOption, ResourcePermissions, STPackageStatus, STPackageType, UitpasNumberDetails, UitpasSocialTariff, UitpasSocialTariffStatus, UserPermissions, Version } from '@stamhoofd/structures';
 import { STExpect, TestUtils } from '@stamhoofd/test-utils';
 import { v4 as uuidv4 } from 'uuid';
@@ -80,7 +80,7 @@ describe('Endpoint.RegisterMembers', () => {
         const member = await new MemberFactory({ organization, user: linkMembersToUser ? user : undefined })
             .create();
 
-        const otherMembers: MemberWithRegistrations[] = [];
+        const otherMembers: MemberWithUsersRegistrationsAndGroups[] = [];
 
         for (let i = 0; i < otherMemberAmount; i++) {
             otherMembers.push(await new MemberFactory({ organization, user: linkMembersToUser ? user : undefined })

package/src/endpoints/global/registration/RegisterMembersEndpoint.ts

@@ -4,9 +4,9 @@ import { Decoder } from '@simonbackx/simple-encoding';
 import { DecodedRequest, Endpoint, Request, Response } from '@simonbackx/simple-endpoints';
 import { SimpleError } from '@simonbackx/simple-errors';
 import { Email } from '@stamhoofd/email';
-import { BalanceItem, BalanceItemPayment, CachedBalance, Group, Member, MemberWithRegistrations, MolliePayment, MollieToken, Organization, PayconiqPayment, Payment, Platform, RateLimiter, Registration, User } from '@stamhoofd/models';
+import { BalanceItem, BalanceItemPayment, CachedBalance, Group, Member, MemberWithUsersRegistrationsAndGroups, MolliePayment, MollieToken, Organization, PayconiqPayment, Payment, Platform, RateLimiter, Registration, User } from '@stamhoofd/models';
 import { BalanceItemRelation, BalanceItemRelationType, BalanceItemStatus, BalanceItem as BalanceItemStruct, BalanceItemType, IDRegisterCheckout, PaymentCustomer, PaymentMethod, PaymentMethodHelper, PaymentProvider, PaymentStatus, Payment as PaymentStruct, PaymentType, PermissionLevel, PlatformFamily, PlatformMember, ReceivableBalanceType, RegisterItem, RegisterResponse, TranslatedString, Version } from '@stamhoofd/structures';
-import { Formatter } from '@stamhoofd/utility';
+import { Formatter, sleep } from '@stamhoofd/utility';
 
 import { AuthenticatedStructures } from '../../../helpers/AuthenticatedStructures.js';
 import { BuckarooHelper } from '../../../helpers/BuckarooHelper.js';
@@ -145,7 +145,7 @@ export class RegisterMembersEndpoint extends Endpoint<Params, Query, Body, Respo
             memberBalanceItemsStructs = balanceItemsModels.map(i => i.getStructure());
         }
 
-        let members: MemberWithRegistrations[] = [];
+        let members: MemberWithUsersRegistrationsAndGroups[] = [];
         if (request.body.asOrganizationId) {
             const memberIds = Formatter.uniqueArray(
                 [...request.body.memberIds, ...deleteRegistrationModels.map(i => i.memberId), ...balanceItemsModels.map(i => i.memberId).filter(m => m !== null)],
@@ -900,21 +900,26 @@ export class RegisterMembersEndpoint extends Endpoint<Params, Query, Body, Respo
             }
         }
 
-        const updatedMembers = await Member.getBlobByIds(...members.map(m => m.id));
+        // Flush caches so data is up to date in response
+        await BalanceItemService.flushCaches(organization.id);
+
+        // Force reload registrations and group data
+        Member.unloadRegistrations(members);
+        await Member.loadRegistrations(members, true);
 
         return new Response(RegisterResponse.create({
             payment: payment ? PaymentStruct.create(payment) : null,
-            members: await AuthenticatedStructures.membersBlob(updatedMembers),
+            members: await AuthenticatedStructures.membersBlob(members),
             registrations: registrations.map(r => Member.getRegistrationWithTinyMemberStructure(r)),
             paymentUrl,
             paymentQRCode,
         }));
     }
 
-    async createPayment({ balanceItems, organization, user, checkout, members }: { balanceItems: Map<BalanceItem, number>; organization: Organization; user: User; checkout: IDRegisterCheckout; members: MemberWithRegistrations[] }) {
+    async createPayment({ balanceItems, organization, user, checkout, members }: { balanceItems: Map<BalanceItem, number>; organization: Organization; user: User; checkout: IDRegisterCheckout; members: MemberWithUsersRegistrationsAndGroups[] }) {
         // Calculate total price to pay
         let totalPrice = 0;
-        const payMembers: MemberWithRegistrations[] = [];
+        const payMembers: MemberWithUsersRegistrationsAndGroups[] = [];
         let hasNegative = false;
 
         for (const [balanceItem, price] of balanceItems) {

package/src/endpoints/organization/dashboard/documents/PatchDocumentEndpoint.ts

@@ -4,7 +4,7 @@ import { SimpleError } from '@simonbackx/simple-errors';
 import { Document, DocumentTemplate, Member, Registration } from '@stamhoofd/models';
 import { DocumentStatus, Document as DocumentStruct, PermissionLevel } from '@stamhoofd/structures';
 
-import { Context } from '../../../../helpers/Context';
+import { Context } from '../../../../helpers/Context.js';
 
 type Params = Record<string, never>;
 type Query = undefined;
@@ -104,7 +104,7 @@ export class PatchDocumentEndpoint extends Endpoint<Params, Query, Body, Respons
                 put.memberId = registration.memberId;
             }
             if (put.memberId) {
-                const member = await Member.getWithRegistrations(put.memberId);
+                const member = await Member.getByIdWithUsersAndRegistrations(put.memberId);
                 if (!member || !await Context.auth.canAccessMember(member, PermissionLevel.Read)) {
                     throw new SimpleError({
                         code: 'not_found',