@stamhoofd/backend 2.120.6 → 2.122.0

package/src/helpers/RecordAnswerHelper.ts

@@ -0,0 +1,116 @@
+import type { AutoEncoderPatchType, Encodeable, Identifiable, Patchable, PatchableArray } from '@simonbackx/simple-encoding';
+import { isPatchableArray } from '@simonbackx/simple-encoding';
+import { SimpleError } from '@simonbackx/simple-errors';
+import type { RecordCategory, RecordSettings, RecordType } from '@stamhoofd/structures';
+import { RecordAnswerDecoder } from '@stamhoofd/structures';
+
+export class RecordAnswerHelper {
+    static throwIfPatchOrPutIsInvalid(original: RecordCategory[], patchOrPut: RecordCategory[] | PatchableArray<string, RecordCategory, AutoEncoderPatchType<RecordCategory>>) {
+        if (original.length === 0) {
+            return;
+        }
+
+        const updatedRecordSettings = this.getAllRecords(patchOrPut)
+            // only check if the types changed
+            .filter(p => p.type !== undefined || p.fileType !== undefined);
+
+        if (updatedRecordSettings.length === 0) {
+            return;
+        }
+
+        const updatRecordSettingsMap = new Map<string, RecordSettings | AutoEncoderPatchType<RecordSettings>>(updatedRecordSettings.map(p => [p.id, p]));
+
+        for (const category of original) {
+            for (const originalRecord of category.getAllRecords()) {
+                const updatedSettings = updatRecordSettingsMap.get(originalRecord.id);
+                if (updatedSettings) {
+                    this.throwIfInvalidRecordSettingsUpdate(originalRecord, updatedSettings);
+
+                    // remove from map
+                    updatRecordSettingsMap.delete(originalRecord.id);
+
+                    // stop looping if all updated records are checked
+                    if (updatRecordSettingsMap.size === 0) {
+                        break;
+                    }
+                }
+            }
+        }
+    }
+
+    static throwIfPatchOrPutsAreInvalid<
+        Key extends string & keyof Put,
+        Id extends string | number,
+        Put extends (Identifiable<Id> & Encodeable & Patchable<Patch> & Record<Key, RecordCategory[]>),
+        Patch extends (Identifiable<Id> & Encodeable) | Put,
+    >(originals: Put[], patchOrPuts: Put[] | Patch[] | PatchableArray<Id, Put, Patch>, key: string) {
+        if (isPatchableArray(patchOrPuts)) {
+            const puts = patchOrPuts.getPuts().map(p => p.put);
+            this.throwIfPatchOrPutsAreInvalid<Key, Id, Put, Patch>(originals, puts, key);
+            const patches = patchOrPuts.getPatches();
+            this.throwIfPatchOrPutsAreInvalid<Key, Id, Put, Patch>(originals, patches, key);
+        } else {
+            for (const patchOrPut of patchOrPuts) {
+                const original = originals.find(o => o.id === patchOrPut.id);
+                if (original) {
+                    const originalRecordCategories: RecordCategory[] = original[key];
+                    const newRecordCategories: RecordCategory[] | PatchableArray<string, RecordCategory, AutoEncoderPatchType<RecordCategory>> = patchOrPut[key];
+                    if (originalRecordCategories && newRecordCategories) {
+                        this.throwIfPatchOrPutIsInvalid(originalRecordCategories, newRecordCategories);
+                    }
+                }
+            }
+        }
+    }
+
+    private static throwIfInvalidRecordSettingsUpdate(original: RecordSettings, updated: RecordSettings | AutoEncoderPatchType<RecordSettings>) {
+        if (updated.type !== undefined
+            && original.type !== updated.type
+            // changing between types with the same class should be allowed
+            && !this.haveTypesSameClass(original.type, updated.type)) {
+            throw new SimpleError({
+                code: 'invalid_record_type',
+                message: `Cannot change record type from ${original.type} to ${updated.type}`,
+                human: $t(`%1Qn`, { name: original.name }),
+            });
+        }
+
+        if (updated.fileType !== undefined && ((original.fileType ?? null) !== (updated.fileType ?? null))) {
+            throw new SimpleError({
+                code: 'invalid_record_type',
+                message: `Cannot change record file type from ${original.fileType} to ${updated.fileType}`,
+                human: $t(`%1SP`, { name: original.name }),
+            });
+        }
+    }
+
+    static haveTypesSameClass(a: RecordType, b: RecordType) {
+        return RecordAnswerDecoder.getClassForType(a) === RecordAnswerDecoder.getClassForType(b);
+    }
+
+    private static getAllRecords(patchOrPut: RecordCategory[] | PatchableArray<string, RecordCategory, AutoEncoderPatchType<RecordCategory>>): (RecordSettings | AutoEncoderPatchType<RecordSettings>)[] {
+        if (isPatchableArray(patchOrPut)) {
+            const patches = patchOrPut.getPatches().flatMap(p => this.getAllRecordsFromRecordCategoryPatch(p));
+            const puts = patchOrPut.getPuts().flatMap(p => p.put.getAllRecords());
+            return [...patches, ...puts];
+        }
+        return patchOrPut.flatMap(c => c.getAllRecords());
+    }
+
+    private static getAllRecordsFromRecordCategoryPatch(category: AutoEncoderPatchType<RecordCategory>) {
+        const results: (RecordSettings | AutoEncoderPatchType<RecordSettings>)[] = [
+            ...category.records.getPatches(),
+            ...category.records.getPuts().map(p => p.put),
+        ];
+
+        if (category.childCategories) {
+            const childCategoryPatches = category.childCategories.getPatches().flatMap(p => this.getAllRecordsFromRecordCategoryPatch(p));
+            results.push(...childCategoryPatches);
+
+            const childCategoryPuts = category.childCategories.getPuts().flatMap(p => p.put.getAllRecords());
+            results.push(...childCategoryPuts);
+        }
+
+        return results;
+    }
+}

package/src/helpers/SeedTools.ts

@@ -0,0 +1,106 @@
+import { Database } from '@simonbackx/simple-database';
+import type { SQLSelect } from '@stamhoofd/sql';
+import type { ProgressLogger } from '@stamhoofd/utility';
+import { LoggingTools } from '@stamhoofd/utility';
+
+/**
+ * If an error is thrown, first await all pending items before throwing, otherwise we risk quiting mid running actions
+ */
+export function allSettledButThrowFirst<T>(promises: Promise<T>[]): Promise<T[]> {
+    return Promise.allSettled(promises).then((results) => {
+        return results.map((r) => {
+            if (r.status === 'fulfilled') {
+                return r.value;
+            }
+            throw r.reason;
+        });
+    });
+}
+
+export class SeedTools {
+    static createBatchProcessor<T>(options: BatchProcessorArgs<T>) {
+        return new BatchProcessor(options);
+    }
+
+    static async loop<T extends { id: string }>(options: BatchProcessorArgs<T> & { query: SQLSelect<T> }) {
+        const batchProcessor = SeedTools.createBatchProcessor(options);
+
+        const progressLogger = await LoggingTools.createProgressLoggerFromQuery(options.query.clone());
+        batchProcessor.setProgressLogger(progressLogger);
+
+        for await (const item of options.query.limit(options.batchSize).all()) {
+            await batchProcessor.execute(item as T);
+        }
+        await batchProcessor.finish();
+
+        return {
+            total: progressLogger.total,
+        };
+    }
+
+    static async loopBatched<T extends { id: string }>(options: { batchSize: number; batchAction: (items: T[]) => Promise<void>; query: SQLSelect<T> }) {
+        const progressLogger = await LoggingTools.createProgressLoggerFromQuery(options.query.clone());
+
+        for await (const batch of options.query.limit(options.batchSize).allBatched()) {
+            await options.batchAction(batch as T[]);
+            progressLogger.update(batch.length);
+        }
+
+        return {
+            total: progressLogger.total,
+        };
+    }
+}
+
+type BatchProcessorArgs<T> = {
+    batchSize: number;
+    useTransactionPerBatch?: boolean;
+    action: (item: T) => Promise<void>;
+};
+
+class BatchProcessor<T> {
+    private currentBatch: Promise<void>[] = [];
+    private readonly batchSize: number;
+    private readonly originalAction: (item: T) => Promise<void>;
+    private action: (item: T) => Promise<void>;
+    private readonly useTransactionPerBatch: boolean;
+
+    constructor({ batchSize, action, useTransactionPerBatch }: BatchProcessorArgs<T>) {
+        this.batchSize = batchSize;
+        this.originalAction = action;
+        this.action = action;
+        this.useTransactionPerBatch = useTransactionPerBatch ?? false;
+    }
+
+    setProgressLogger(progressLogger: ProgressLogger) {
+        this.action = async (item: T) => {
+            await this.originalAction(item);
+            progressLogger.update();
+        };
+    }
+
+    async execute(item: T): Promise<void> {
+        if (this.currentBatch.length >= this.batchSize) {
+            await this.finishCurrentBatch();
+        }
+
+        this.currentBatch.push(this.action(item));
+    }
+
+    private async finishCurrentBatch() {
+        const batch = this.currentBatch;
+        this.currentBatch = [];
+
+        if (this.useTransactionPerBatch) {
+            await Database.beginTransaction(async () => {
+                await allSettledButThrowFirst(batch);
+            });
+        } else {
+            await allSettledButThrowFirst(batch);
+        }
+    }
+
+    async finish(): Promise<void> {
+        await this.finishCurrentBatch();
+    }
+}

package/src/helpers/ServiceFeeHelper.ts

@@ -1,21 +1,64 @@
-import type { Organization, Payment } from '@stamhoofd/models';
-import { calculateVATPercentage, PaymentProvider, STPackageType } from '@stamhoofd/structures';
+import { SimpleError } from '@simonbackx/simple-errors';
+import type { Payment, StripeAccount } from '@stamhoofd/models';
+import { Organization, Platform } from '@stamhoofd/models';
+import { PaymentMethod, PaymentProvider, STPackageType } from '@stamhoofd/structures';
+import { VATService } from '../services/VATService.js';
+import { STMath } from '@stamhoofd/utility';
+
+const CACHE_TIMEOUT = 1000 * 60 * 5; // 5 minutes
 
 export class ServiceFeeHelper {
-    static setServiceFee(payment: Payment, organization: Organization, type: 'webshop' | 'members' | 'tickets' | 'system', itemPrices: number[]): void {
-        if (type === 'system') {
-            return;
+    static #cachedOrganization: Organization | null = null;
+    static #cachedOrganizationDate: Date | null = null;
+
+    /**
+     * Organization that is invoicing
+     */
+    static async getInvoicingOrganization() {
+        if (this.#cachedOrganization && this.#cachedOrganizationDate && this.#cachedOrganizationDate > new Date(Date.now() - CACHE_TIMEOUT)) {
+            return this.#cachedOrganization;
+        }
+        const platform = await Platform.getShared();
+        const membershipId = platform.membershipOrganizationId;
+        if (!membershipId) {
+            return null;
         }
 
-        let vat = 0;
+        const org = await Organization.getByID(membershipId, true);
+        this.#cachedOrganization = org;
+        this.#cachedOrganizationDate = new Date();
 
-        // Calculate item count
-        if (payment.provider === PaymentProvider.Stripe && payment.stripeAccountId) {
-            // Don't chagne
-            vat = calculateVATPercentage(organization.address, organization.meta.VATNumber);
+        return org;
+    }
+
+    /**
+     * VAT rate used to charge service fees or transfer fees to organization by the invoicing organization (membershipOrganizationId)
+     */
+    static async getVATPercentage(organization: Organization) {
+        const invoicingOrganization = await this.getInvoicingOrganization();
+        if (!invoicingOrganization) {
+            throw new SimpleError({
+                code: 'unavailable',
+                message: 'Payments are temporarily unavailable due to a lack of VAT settings',
+                human: $t('%1ay'),
+            });
         }
-        else {
-            // don't charge VAT (we'll add the VAT to the invoice)
+
+        const customer = organization.defaultCompanies[0];
+
+        if (VATService.isVATExcempt({
+            sellingOrganization: invoicingOrganization,
+            company: customer,
+        })) {
+            return 0;
+        }
+
+        return 21;
+    }
+
+    static async setServiceFee(payment: Payment, organization: Organization, type: 'webshop' | 'members' | 'tickets' | 'system', itemPrices: number[]): Promise<void> {
+        if (type === 'system') {
+            return;
         }
 
         function calculateFee(price: number, minimumFee: number | null, maximumFee: number | null, fixed: number, percentageTimes100: number) {
@@ -55,7 +98,18 @@ export class ServiceFeeHelper {
         }
 
         // Add VAT
-        serviceFee = serviceFee * (100 + vat) / 100;
+        if (serviceFee !== 0) {
+            let vat = 0;
+
+            // Calculate item count
+            if (payment.provider === PaymentProvider.Stripe && payment.stripeAccountId) {
+                // Don't chagne
+                vat = await this.getVATPercentage(organization);
+            } else {
+                // don't charge VAT (we'll add the VAT to the invoice)
+            }
+            serviceFee = serviceFee * (100 + vat) / 100;
+        }
 
         // Round service fee to 2 decimal places
         serviceFee = Math.round(serviceFee / 100) * 100;
@@ -63,9 +117,65 @@ export class ServiceFeeHelper {
         console.log('Service fee for payment', payment.id, type, 'is', serviceFee);
         if (payment.provider === PaymentProvider.Stripe && payment.stripeAccountId) {
             payment.serviceFeePayout = serviceFee;
-        }
-        else {
+        } else {
             payment.serviceFeeManual = serviceFee;
         }
     }
+
+    /**
+     * Depends on the situation.
+     */
+    static async setTransferFee({
+        payment,
+        organization,
+        stripeAccount,
+    }: {
+        payment: Payment;
+        organization: Organization;
+        stripeAccount?: StripeAccount | null | undefined;
+    }) {
+        if (payment.provider !== PaymentProvider.Stripe && payment.provider !== PaymentProvider.Buckaroo) {
+            return;
+        }
+
+        if (payment.provider === PaymentProvider.Stripe && !stripeAccount) {
+            throw new Error('Missing StripeAccount');
+        }
+
+        const totalPrice = STMath.round(payment.price / 100); // Convert from 4 decimal places to 2 decimal places for this calculation
+
+        if (totalPrice <= 0) {
+            return;
+        }
+
+        if (stripeAccount && stripeAccount.meta.type === 'standard') {
+            // Submerchant is charged by Stripe for the fees directly
+            return;
+        }
+
+        let fee = 0;
+        const vat = await this.getVATPercentage(organization);
+
+        function calculateFee(fixed: number, percentageTimes100: number) {
+            return STMath.round(STMath.round(fixed + Math.max(1, totalPrice * percentageTimes100 / 100 / 100)) * (100 + vat) / 100); // € 0,21 + 0,2%
+        }
+
+        if (payment.method === PaymentMethod.iDEAL) {
+            fee = calculateFee(21, 20); // € 0,21 + 0,2%
+        } else if (payment.method === PaymentMethod.Bancontact) {
+            fee = calculateFee(24, 20); // € 0,24 + 0,2%
+        } else {
+            fee = calculateFee(25, 150); // € 0,25 + 1,5%
+        }
+
+        payment.transferFee = fee * 100; // Convert back to 4 decimal places for storage
+
+        if (payment.provider === PaymentProvider.Buckaroo) {
+            // We'll need to charge
+            payment.transferFeeManual = payment.transferFee;
+            console.log('Transfer fee for payment', payment.id, 'is', payment.transferFee, '(manually charged)');
+        } else {
+            console.log('Transfer fee for payment', payment.id, 'is', payment.transferFee);
+        }
+    }
 }

package/src/helpers/SetupStepUpdater.ts

@@ -14,17 +14,17 @@ import { SQL, SQLWhereSign } from '@stamhoofd/sql';
 import type {
     MemberResponsibility,
     Platform as PlatformStruct,
-    SetupSteps} from '@stamhoofd/structures';
+    SetupSteps,
+} from '@stamhoofd/structures';
 import {
     AuditLogSource,
     GroupType,
     RecordCategory,
-    SetupStepType
+    SetupStepType,
 } from '@stamhoofd/structures';
 import { Formatter } from '@stamhoofd/utility';
 import { AuditLogService } from '../services/AuditLogService.js';
 import { GroupedThrottledQueue } from './GroupedThrottledQueue.js';
-import { AuthenticatedStructures } from './AuthenticatedStructures.js';
 
 type SetupStepOperation = (setupSteps: SetupSteps, organization: Organization, platform: PlatformStruct) => void | Promise<void>;
 
@@ -71,6 +71,10 @@ export class SetupStepUpdater {
     static isListening = false;
 
     static listen() {
+        if (STAMHOOFD.userMode === 'organization') {
+            return;
+        }
+
         if (this.isListening) {
             return;
         }
@@ -111,14 +115,14 @@ export class SetupStepUpdater {
         SetupStepType,
         SetupStepOperation
     > = {
-            [SetupStepType.Responsibilities]: this.updateStepResponsibilities,
-            [SetupStepType.Companies]: this.updateStepCompanies,
-            [SetupStepType.Groups]: this.updateStepGroups,
-            [SetupStepType.Premises]: this.updateStepPremises,
-            [SetupStepType.Emails]: this.updateStepEmails,
-            [SetupStepType.Payment]: this.updateStepPayment,
-            [SetupStepType.Registrations]: this.updateStepRegistrations,
-        };
+        [SetupStepType.Responsibilities]: this.updateStepResponsibilities,
+        [SetupStepType.Companies]: this.updateStepCompanies,
+        [SetupStepType.Groups]: this.updateStepGroups,
+        [SetupStepType.Premises]: this.updateStepPremises,
+        [SetupStepType.Emails]: this.updateStepEmails,
+        [SetupStepType.Payment]: this.updateStepPayment,
+        [SetupStepType.Registrations]: this.updateStepRegistrations,
+    };
 
     static async updateSetupStepsForAllOrganizationsInCurrentPeriod({
         batchSize,
@@ -191,6 +195,10 @@ export class SetupStepUpdater {
     }
 
     static async updateForOrganizationId(id: string, options?: { types?: SetupStepType[] }) {
+        if (STAMHOOFD.userMode === 'organization') {
+            return;
+        }
+
         const organization = await Organization.getByID(id);
         if (!organization) {
             throw new SimpleError({
@@ -215,6 +223,10 @@ export class SetupStepUpdater {
             types?: SetupStepType[];
         } = {},
     ) {
+        if (STAMHOOFD.userMode === 'organization') {
+            return;
+        }
+
         if (!platform) {
             platform = await Platform.getSharedPrivateStruct();
             if (!platform) {
@@ -312,8 +324,7 @@ export class SetupStepUpdater {
                 totalSteps,
                 finishedSteps,
             });
-        }
-        else {
+        } else {
             // if no premise types, remove step
             setupSteps.remove(SetupStepType.Premises);
         }
@@ -346,8 +357,7 @@ export class SetupStepUpdater {
                     platform.config.organizationLevelRecordsConfiguration.recordCategories,
                     organization.getBaseStructureWithPrivateMeta(), // private data needed for answers
                 );
-            }
-            catch (error) {
+            } catch (error) {
                 console.error('Error validating record categories for organization', organization.id, error);
                 finishedSteps = 0;
             }
@@ -375,10 +385,10 @@ export class SetupStepUpdater {
         const allRecords = responsibilityIds.length === 0
             ? []
             : await MemberResponsibilityRecord.select()
-                .where('responsibilityId', responsibilityIds)
-                .where('organizationId', organization.id)
-                .where(SQL.where('endDate', SQLWhereSign.Greater, now).or('endDate', null))
-                .fetch();
+                    .where('responsibilityId', responsibilityIds)
+                    .where('organizationId', organization.id)
+                    .where(SQL.where('endDate', SQLWhereSign.Greater, now).or('endDate', null))
+                    .fetch();
 
         // Remove invalid responsibilities: members that are not registered in the current period
         const memberIds = Formatter.uniqueArray(allRecords.map(r => r.memberId));
@@ -444,8 +454,7 @@ export class SetupStepUpdater {
                         totalRecordsWithThisResponsibility++;
                     }
                 }
-            }
-            else {
+            } else {
                 for (const record of records) {
                     if (record.responsibilityId === responsibilityId && record.groupId === group.id) {
                         totalRecordsWithThisResponsibility++;
@@ -488,8 +497,7 @@ export class SetupStepUpdater {
 
         if (finishedSteps >= totalSteps) {
             setupSteps.markReviewed(SetupStepType.Emails, { userId: 'backend', userName: 'backend' });
-        }
-        else {
+        } else {
             setupSteps.resetReviewed(SetupStepType.Emails);
         }
     }
@@ -510,11 +518,11 @@ export class SetupStepUpdater {
 
         const groupsWithDefaultAgeGroups = defaultAgeGroupIds.length > 0
             ? await Group.select()
-                .where('organizationId', organization.id)
-                .where('periodId', platform.period.id)
-                .where('defaultAgeGroupId', defaultAgeGroupIds)
-                .where('deletedAt', null)
-                .fetch()
+                    .where('organizationId', organization.id)
+                    .where('periodId', platform.period.id)
+                    .where('defaultAgeGroupId', defaultAgeGroupIds)
+                    .where('deletedAt', null)
+                    .fetch()
             : [];
 
         let totalSteps = 0;
@@ -530,8 +538,7 @@ export class SetupStepUpdater {
             if (!processedDefaultAgeGroupIds.has(defaultAgeGroup.id)) {
                 totalSteps += defaultAgeGroup.minimumRequiredMembers;
                 processedDefaultAgeGroupIds.set(defaultAgeGroup.id, Math.min(defaultAgeGroup.minimumRequiredMembers, group.settings.registeredMembers ?? 0));
-            }
-            else {
+            } else {
                 processedDefaultAgeGroupIds.set(defaultAgeGroup.id,
                     Math.min(
                         defaultAgeGroup.minimumRequiredMembers,
@@ -550,8 +557,7 @@ export class SetupStepUpdater {
 
         if (finishedSteps >= totalSteps) {
             setupSteps.markReviewed(SetupStepType.Registrations, { userId: 'backend', userName: 'backend' });
-        }
-        else {
+        } else {
             setupSteps.resetReviewed(SetupStepType.Registrations);
         }
     }

package/src/helpers/StamhoofdFilterAccessHelper.ts

@@ -0,0 +1,52 @@
+import { SimpleError } from '@simonbackx/simple-errors';
+import type { StamhoofdCompareValue, StamhoofdFilter } from '@stamhoofd/structures';
+import { FilterWrapperMarker, getMagicRelationValue, isMagicFilter, isMagicRelationFilter, unwrapFilter } from '@stamhoofd/structures';
+
+export class StamhoofdFilterAccessHelper {
+    static getGroupIdsFromFilter(filter: StamhoofdFilter): StamhoofdFilter<StamhoofdCompareValue>[] {
+        let rawGroupIds: StamhoofdFilter<StamhoofdCompareValue>[] | null = null;
+
+        if (typeof filter === 'string' || isMagicFilter(filter)) {
+            rawGroupIds = [filter];
+        } else {
+            const { markerValue } = unwrapFilter(filter, {
+                $in: FilterWrapperMarker,
+            });
+
+            if (!Array.isArray(markerValue)) {
+                throw new SimpleError({
+                    code: 'invalid_field',
+                    field: 'filter',
+                    message: 'You must filter on a group of the organization you are trying to access',
+                    human: $t(`%1HG`),
+                });
+            }
+
+            if (markerValue.length === 0) {
+                throw new SimpleError({
+                    code: 'invalid_field',
+                    field: 'filter',
+                    message: 'Filtering on an empty list of groups is not supported',
+                });
+            }
+
+            rawGroupIds = markerValue;
+        }
+
+        return rawGroupIds.map((raw) => {
+            if (typeof raw === 'string') {
+                return raw;
+            }
+
+            if (isMagicRelationFilter(raw)) {
+                return getMagicRelationValue(raw);
+            }
+
+            throw new SimpleError({
+                code: 'invalid_field',
+                field: 'filter',
+                message: 'Invalid group ID in filter',
+            });
+        });
+    }
+}