@stamhoofd/backend 2.90.3 → 2.92.0

package/src/helpers/GroupedThrottledQueue.ts

@@ -15,13 +15,15 @@ export class GroupedThrottledQueue<T> {
      * In milliseconds.
      */
     maxDelay: number | null = 10_000;
+    itemDelay: number | null = null;
 
     constructor(
         handler: (group: string, items: T[]) => Promise<void> | void,
-        options: { maxDelay?: number | null } = {},
+        options: { maxDelay?: number | null; itemDelay?: number | null } = {},
     ) {
         this.handler = handler;
-        this.maxDelay = options.maxDelay ?? 10_000;
+        this.maxDelay = options.maxDelay !== null ? (options.maxDelay ?? 10_000) : null;
+        this.itemDelay = options.itemDelay ?? null;
     }
 
     addItems(group: string, items: T[]): void {
@@ -32,7 +34,7 @@ export class GroupedThrottledQueue<T> {
             return;
         }
         const newQueue = new ThrottledQueue<T>(items => this.handler(group, items), {
-            maxDelay: null,
+            maxDelay: this.itemDelay,
             emptyHandler: () => {
                 this.queues.delete(group);
             },

package/src/helpers/PeriodHelper.ts

@@ -1,11 +1,10 @@
 import { SimpleError } from '@simonbackx/simple-errors';
-import { Group, Member, MemberResponsibilityRecord, Organization, OrganizationRegistrationPeriod, Platform, RegistrationPeriod } from '@stamhoofd/models';
+import { Group, Organization, OrganizationRegistrationPeriod, RegistrationPeriod } from '@stamhoofd/models';
 import { QueueHandler } from '@stamhoofd/queues';
-import { AuditLogSource, Group as GroupStruct, PermissionLevel } from '@stamhoofd/structures';
+import { AuditLogSource, Group as GroupStruct } from '@stamhoofd/structures';
 import { PatchOrganizationRegistrationPeriodsEndpoint } from '../endpoints/organization/dashboard/registration-periods/PatchOrganizationRegistrationPeriodsEndpoint';
-import { AuthenticatedStructures } from './AuthenticatedStructures';
-import { MemberUserSyncer } from './MemberUserSyncer';
 import { AuditLogService } from '../services/AuditLogService';
+import { AuthenticatedStructures } from './AuthenticatedStructures';
 import { SetupStepUpdater } from './SetupStepUpdater';
 
 export class PeriodHelper {
@@ -18,88 +17,6 @@ export class PeriodHelper {
         });
     }
 
-    static async stopAllResponsibilities() {
-        console.log('Stopping all responsibilities');
-        const platform = await Platform.getSharedPrivateStruct();
-        const keepPlatformResponsibilityIds = platform.config.responsibilities.filter(r => !r.organizationBased).map(r => r.id);
-        const keepResponsibilityIds = platform.config.responsibilities.filter(r => !r.organizationBased || r.permissions?.level === PermissionLevel.Full).map(r => r.id);
-        const batchSize = 100;
-
-        let lastId = '';
-        let c = 0;
-
-        while (true) {
-            const records = await MemberResponsibilityRecord.where(
-                {
-                    id: { sign: '>', value: lastId },
-                    endDate: null,
-                },
-                {
-                    limit: batchSize,
-                    sort: ['id'],
-                },
-            );
-
-            for (const record of records) {
-                lastId = record.id;
-
-                const invalid = keepPlatformResponsibilityIds.includes(record.responsibilityId) && record.organizationId;
-
-                if (!keepResponsibilityIds.includes(record.responsibilityId) || invalid) {
-                    record.endDate = new Date();
-                    await record.save();
-                    c++;
-                }
-            }
-
-            if (records.length < batchSize) {
-                break;
-            }
-        }
-
-        console.log('Done: stopped all responsibilities: ' + c);
-    }
-
-    static async syncAllMemberUsers() {
-        console.log('Syncing all members');
-
-        let c = 0;
-        let lastId: string = '';
-
-        while (true) {
-            const rawMembers = await Member.where({
-                id: {
-                    value: lastId,
-                    sign: '>',
-                },
-            }, { limit: 500, sort: ['id'] });
-
-            if (rawMembers.length === 0) {
-                break;
-            }
-
-            const membersWithRegistrations = await Member.getBlobByIds(...rawMembers.map(m => m.id));
-
-            const promises: Promise<any>[] = [];
-
-            for (const memberWithRegistrations of membersWithRegistrations) {
-                promises.push((async () => {
-                    await MemberUserSyncer.onChangeMember(memberWithRegistrations);
-                    c++;
-
-                    if (c % 10000 === 0) {
-                        console.log('Synced ' + c + ' members');
-                    }
-                })());
-            }
-
-            await Promise.all(promises);
-            lastId = rawMembers[rawMembers.length - 1].id;
-        }
-
-        console.log('Done: synced all members: ' + c);
-    }
-
     static async createOrganizationPeriodForPeriod(organization: Organization, period: RegistrationPeriod) {
         const oPeriods = await OrganizationRegistrationPeriod.where({ periodId: period.id, organizationId: organization.id }, { limit: 1 });
 
@@ -128,38 +45,15 @@ export class PeriodHelper {
             });
         }
 
-        const batchSize = 10;
         await QueueHandler.schedule(tag, async () => {
-            let lastId = '';
-
-            while (true) {
-                const organizations = await Organization.where(
-                    {
-                        id: { sign: '>', value: lastId },
-                    },
-                    {
-                        limit: batchSize,
-                        sort: ['id'],
-                    },
-                );
-
-                for (const organization of organizations) {
-                    try {
-                        await this.moveOrganizationToPeriod(organization, period);
-                    }
-                    catch (error) {
-                        console.error('Error moving organization to period', organization.id, error);
-                    }
-                    lastId = organization.id;
+            for await (const organization of Organization.select().all()) {
+                try {
+                    await this.moveOrganizationToPeriod(organization, period);
                 }
-
-                if (organizations.length < batchSize) {
-                    break;
+                catch (error) {
+                    console.error('Error moving organization to period', organization.id, error);
                 }
             }
-
-            await this.stopAllResponsibilities();
-            await this.syncAllMemberUsers();
         });
 
         // When done: update setup steps
@@ -175,31 +69,10 @@ export class PeriodHelper {
 
         console.log(tag);
 
-        const batchSize = 100;
         await QueueHandler.schedule(tag, async () => {
             await AuditLogService.setContext({ source: AuditLogSource.System }, async () => {
-                let lastId = '';
-
-                while (true) {
-                    const groups = await Group.where(
-                        {
-                            id: { sign: '>', value: lastId },
-                            periodId: period.id,
-                        },
-                        {
-                            limit: batchSize,
-                            sort: ['id'],
-                        },
-                    );
-
-                    for (const group of groups) {
-                        await PatchOrganizationRegistrationPeriodsEndpoint.patchGroup(GroupStruct.patch({ id: group.id }), period);
-                        lastId = group.id;
-                    }
-
-                    if (groups.length < batchSize) {
-                        break;
-                    }
+                for await (const group of Group.select().where('periodId', period.id).all()) {
+                    await PatchOrganizationRegistrationPeriodsEndpoint.patchGroup(GroupStruct.patch({ id: group.id }), period);
                 }
             });
         });

package/src/helpers/SetupStepUpdater.ts

@@ -7,6 +7,7 @@ import {
     Organization,
     OrganizationRegistrationPeriod,
     Platform,
+    Registration,
 } from '@stamhoofd/models';
 import { QueueHandler } from '@stamhoofd/queues';
 import { SQL, SQLWhereSign } from '@stamhoofd/sql';
@@ -15,11 +16,14 @@ import {
     GroupType,
     MemberResponsibility,
     Platform as PlatformStruct,
+    RecordCategory,
     SetupStepType,
     SetupSteps,
 } from '@stamhoofd/structures';
 import { Formatter } from '@stamhoofd/utility';
 import { AuditLogService } from '../services/AuditLogService';
+import { GroupedThrottledQueue } from './GroupedThrottledQueue';
+import { AuthenticatedStructures } from './AuthenticatedStructures';
 
 type SetupStepOperation = (setupSteps: SetupSteps, organization: Organization, platform: PlatformStruct) => void | Promise<void>;
 
@@ -81,10 +85,25 @@ export class SetupStepUpdater {
             if (before !== after) {
                 console.log('Updating setups steps for organization', event.model.organizationId, 'because of change in members in group', event.model.id, 'from', before, 'to', after, '(limited)');
                 // We need to do a recalculation
-                SetupStepUpdater.updateForOrganizationId(event.model.organizationId)
+                SetupStepUpdater.updateForOrganizationId(event.model.organizationId, { types: [SetupStepType.Registrations] })
                     .catch(console.error);
             }
         });
+
+        const updateResponsibilitiesQueue = new GroupedThrottledQueue(async (organizationId: string) => {
+            console.log('(delayed) Updating setups steps for organization', organizationId, 'because of change in registrations');
+            SetupStepUpdater.updateForOrganizationId(organizationId, { types: [SetupStepType.Responsibilities] }).catch(console.error);
+        }, { maxDelay: 3_000 });
+
+        Model.modelEventBus.addListener({}, async (event) => {
+            if (!(event.model instanceof Registration)) {
+                return;
+            }
+
+            if ((event.type === 'updated' && ('registeredAt' in event.changedFields || 'deactivatedAt' in event.changedFields)) || (event.type === 'created' && event.model.registeredAt)) {
+                updateResponsibilitiesQueue.addItem(event.model.organizationId, 1);
+            }
+        });
     }
 
     private static readonly STEP_TYPE_OPERATIONS: Record<
@@ -170,7 +189,7 @@ export class SetupStepUpdater {
         });
     }
 
-    static async updateForOrganizationId(id: string) {
+    static async updateForOrganizationId(id: string, options?: { types?: SetupStepType[] }) {
         const organization = await Organization.getByID(id);
         if (!organization) {
             throw new SimpleError({
@@ -180,7 +199,7 @@ export class SetupStepUpdater {
             });
         }
 
-        await this.updateForOrganization(organization);
+        await this.updateForOrganization(organization, options);
     }
 
     static async updateForOrganization(
@@ -188,9 +207,11 @@ export class SetupStepUpdater {
         {
             platform,
             organizationRegistrationPeriod,
+            types,
         }: {
             platform?: PlatformStruct;
             organizationRegistrationPeriod?: OrganizationRegistrationPeriod;
+            types?: SetupStepType[];
         } = {},
     ) {
         if (!platform) {
@@ -222,6 +243,7 @@ export class SetupStepUpdater {
             organizationRegistrationPeriod,
             platform,
             organization,
+            types,
         );
     }
 
@@ -229,12 +251,13 @@ export class SetupStepUpdater {
         organizationRegistrationPeriod: OrganizationRegistrationPeriod,
         platform: PlatformStruct,
         organization: Organization,
+        types?: SetupStepType[],
     ) {
         console.log('Updating setup steps for organization', organization.id);
         const setupSteps = organizationRegistrationPeriod.setupSteps;
 
         await AuditLogService.setContext({ source: AuditLogSource.System }, async () => {
-            for (const stepType of Object.values(SetupStepType)) {
+            for (const stepType of types ?? Object.values(SetupStepType)) {
                 const operation = this.STEP_TYPE_OPERATIONS[stepType];
                 await operation(setupSteps, organization, platform);
             }
@@ -309,15 +332,28 @@ export class SetupStepUpdater {
     private static updateStepCompanies(
         setupSteps: SetupSteps,
         organization: Organization,
-        _platform: PlatformStruct,
+        platform: PlatformStruct,
     ) {
         const totalSteps = 1;
         let finishedSteps = 0;
 
         if (organization.meta.companies.length) {
             finishedSteps = 1;
+
+            try {
+                RecordCategory.validate(
+                    platform.config.organizationLevelRecordsConfiguration.recordCategories,
+                    organization.getBaseStructureWithPrivateMeta(), // private data needed for answers
+                );
+            }
+            catch (error) {
+                console.error('Error validating record categories for organization', organization.id, error);
+                finishedSteps = 0;
+            }
         }
 
+        console.log('Updating companies step for organization', organization.id, 'with total steps', totalSteps, 'and finished steps', finishedSteps);
+
         setupSteps.update(SetupStepType.Companies, {
             totalSteps,
             finishedSteps,
@@ -346,11 +382,14 @@ export class SetupStepUpdater {
         // Remove invalid responsibilities: members that are not registered in the current period
         const memberIds = Formatter.uniqueArray(allRecords.map(r => r.memberId));
         const members = await Member.getBlobByIds(...memberIds);
-        const validMembers = members.filter(m => m.registrations.some(r => r.organizationId === organization.id && r.periodId === organization.periodId && r.group.type === GroupType.Membership && r.deactivatedAt === null && r.registeredAt !== null));
+        const validMembers = members.filter(m => m.registrations.some(r => r.organizationId === organization.id && r.periodId === organization.periodId && r.group.defaultAgeGroupId && r.group.type === GroupType.Membership && r.deactivatedAt === null && r.registeredAt !== null));
+        const invalidMembers = members.filter(m => !m.registrations.some(r => r.organizationId === organization.id && r.periodId === organization.periodId && r.group.defaultAgeGroupId && r.group.type === GroupType.Membership && r.deactivatedAt === null && r.registeredAt !== null));
 
         const validMembersIds = validMembers.map(m => m.id);
+        const invalidMembersIds = invalidMembers.map(m => m.id);
 
         const records = allRecords.filter(r => validMembersIds.includes(r.memberId));
+        const invalidRecords = allRecords.filter(r => invalidMembersIds.includes(r.memberId));
 
         let totalSteps = 0;
         let finishedSteps = 0;
@@ -381,6 +420,13 @@ export class SetupStepUpdater {
         for (const { responsibility, group } of flatResponsibilities) {
             const { minimumMembers: min, maximumMembers: max } = responsibility;
 
+            const invalidMembersWithThisResponsibility = !!invalidRecords.find(r => r.responsibilityId === responsibility.id && (group ? r.groupId === group.id : true));
+            if (invalidMembersWithThisResponsibility) {
+                totalSteps += Math.max(min ?? 1, 1);
+                // Step not okay: we have an invalid member with this responsibility
+                continue;
+            }
+
             if (min === null) {
                 continue;
             }
@@ -406,13 +452,14 @@ export class SetupStepUpdater {
             }
 
             if (max !== null && totalRecordsWithThisResponsibility > max) {
-                // Not added
+                // Not added (not okay)
                 continue;
             }
 
             finishedSteps += Math.min(min, totalRecordsWithThisResponsibility);
         }
 
+        console.log('Updating responsibilities step for organization', organization.id, 'with total steps', totalSteps, 'and finished steps', finishedSteps);
         setupSteps.update(SetupStepType.Responsibilities, {
             totalSteps,
             finishedSteps,

package/src/helpers/UitpasTokenRepository.ts

@@ -83,7 +83,7 @@ export class UitpasTokenRepository {
                 throw new SimpleError({
                     code: 'uitpas_api_not_configured_for_platform',
                     message: 'UiTPAS api is not configured for the platform',
-                    human: $t('UiTPAS is niet volledig geconfigureerd, contacteer de platformbeheerder.'),
+                    human: $t('71a8218b-c58e-4e95-9626-551b80eb8367'),
                 });
             }
             model = new UitpasClientCredential();
@@ -123,7 +123,7 @@ export class UitpasTokenRepository {
                 throw new SimpleError({
                     code: 'invalid_uitpas_client_credentials',
                     message: `Invalid UiTPAS client credentials`,
-                    human: $t(`De opgegeven UiTPAS client credentials zijn ongeldig. Controleer ze en probeer opnieuw.`),
+                    human: $t(`1086bb24-5df4-4faf-9dc0-ab5a955b0d8f`),
                 });
             }
             console.error(`Unsuccessful response when fetching UiTPAS token for organization with id ${this.uitpasClientCredential.organizationId}:`, response.statusText);
@@ -193,7 +193,7 @@ export class UitpasTokenRepository {
                     throw new SimpleError({
                         code: 'uitpas_api_not_configured_for_this_organization',
                         message: `UiTPAS api not configured for organization with id ${organizationId}`,
-                        human: $t('UiTPAS is nog niet volledig geconfigureerd voor deze organisatie.'),
+                        human: $t('6b333cb4-21fd-42be-b0c9-6d899f2fd348'),
                     });
                 }
                 repo = UitpasTokenRepository.setRepoInMemory(organizationId, new UitpasTokenRepository(model)); // store in memory

package/src/seeds/1750090030-records-configuration.ts

@@ -1,10 +1,14 @@
 import { Migration } from '@simonbackx/simple-database';
-import { Organization } from '@stamhoofd/models';
+import { Organization, Webshop } from '@stamhoofd/models';
 
 export async function startRecordsConfigurationMigration() {
     for await (const organization of Organization.select().all()) {
         await organization.save();
     }
+
+    for await (const webshop of Webshop.select().all()) {
+        await webshop.save();
+    }
 }
 
 export default new Migration(async () => {