@stamhoofd/backend 2.80.1 → 2.82.0

package/src/excel-loaders/members.ts

@@ -6,7 +6,7 @@ import { ExportToExcelEndpoint } from '../endpoints/global/files/ExportToExcelEn
 import { GetMembersEndpoint } from '../endpoints/global/members/GetMembersEndpoint';
 import { AuthenticatedStructures } from '../helpers/AuthenticatedStructures';
 import { Context } from '../helpers/Context';
-import { XlsxTransformerColumnHelper } from '../helpers/xlsxAddressTransformerColumnFactory';
+import { XlsxTransformerColumnHelper } from '../helpers/XlsxTransformerColumnHelper';
 
 // Assign to a typed variable to assure we have correct type checking in place
 const sheet: XlsxTransformerSheet<PlatformMember, PlatformMember> = {
@@ -16,7 +16,7 @@ const sheet: XlsxTransformerSheet<PlatformMember, PlatformMember> = {
         {
             id: 'id',
             name: 'ID',
-            width: 20,
+            width: 40,
             getValue: ({ patchedMember: object }: PlatformMember) => ({
                 value: object.id,
             }),
@@ -89,7 +89,7 @@ const sheet: XlsxTransformerSheet<PlatformMember, PlatformMember> = {
         {
             id: 'email',
             name: 'E-mailadres',
-            width: 20,
+            width: 40,
             getValue: ({ patchedMember: object }: PlatformMember) => ({
                 value: object.details.email,
             }),
@@ -247,42 +247,19 @@ const sheet: XlsxTransformerSheet<PlatformMember, PlatformMember> = {
         },
 
         // Dynamic records
-        {
-            match(id) {
-                if (id.startsWith('recordAnswers.')) {
-                    const platform = PlatformStruct.shared;
-                    const organization = Context.organization;
-
-                    const recordSettings = [
-                        ...(organization?.meta.recordsConfiguration.recordCategories.flatMap(category => category.getAllRecords()) ?? []),
-                        ...platform.config.recordsConfiguration.recordCategories.flatMap(category => category.getAllRecords()),
-                    ];
-
-                    const recordSettingId = id.split('.')[1];
-                    console.log('recordSettingId', recordSettingId);
-                    const recordSetting = recordSettings.find(r => r.id === recordSettingId);
-
-                    if (!recordSetting) {
-                        // Will throw a proper error itself
-                        console.log('recordSetting not found', recordSettings);
-                        return;
-                    }
-
-                    const columns = recordSetting.excelColumns;
-
-                    return columns.map((columnName, index) => {
-                        return {
-                            id: `recordAnswers.${recordSettingId}.${index}`,
-                            name: columnName,
-                            width: 20,
-                            getValue: ({ patchedMember: object }: PlatformMember) => ({
-                                value: object.details.recordAnswers.get(recordSettingId)?.excelValues[index]?.value ?? '',
-                            }),
-                        };
-                    });
-                }
+        XlsxTransformerColumnHelper.createRecordAnswersColumns({
+            matchId: 'recordAnswers',
+            getRecordAnswers: ({ patchedMember: object }: PlatformMember) => object.details.recordAnswers,
+            getRecordCategories: () => {
+                const platform = PlatformStruct.shared;
+                const organization = Context.organization;
+
+                return [
+                    ...(organization?.meta.recordsConfiguration.recordCategories ?? []),
+                    ...platform.config.recordsConfiguration.recordCategories,
+                ];
             },
-        },
+        }),
 
         // Registration records
         {

package/src/excel-loaders/organizations.ts

@@ -2,7 +2,7 @@ import { XlsxTransformerSheet } from '@stamhoofd/excel-writer';
 import { Platform as PlatformStruct, ExcelExportType, LimitedFilteredRequest, Organization as OrganizationStruct, MemberResponsibilityRecord as MemberResponsibilityRecordStruct, PaginatedResponse, MemberWithRegistrationsBlob, Premise } from '@stamhoofd/structures';
 import { GetOrganizationsEndpoint } from '../endpoints/admin/organizations/GetOrganizationsEndpoint';
 import { ExportToExcelEndpoint } from '../endpoints/global/files/ExportToExcelEndpoint';
-import { XlsxTransformerColumnHelper } from '../helpers/xlsxAddressTransformerColumnFactory';
+import { XlsxTransformerColumnHelper } from '../helpers/XlsxTransformerColumnHelper';
 import { Group, Member, MemberResponsibilityRecord } from '@stamhoofd/models';
 import { Formatter, Sorter } from '@stamhoofd/utility';
 import { ArrayDecoder, field } from '@simonbackx/simple-encoding';
@@ -33,7 +33,7 @@ const sheet: XlsxTransformerSheet<Object, Object> = {
         {
             id: 'id',
             name: 'ID',
-            width: 35,
+            width: 40,
             getValue: (object: Object) => ({
                 value: object.id,
             }),

package/src/excel-loaders/payments.ts

@@ -5,7 +5,7 @@ import { BalanceItemPaymentDetailed, BalanceItemRelationType, ExcelExportType, g
 import { Formatter } from '@stamhoofd/utility';
 import { ExportToExcelEndpoint } from '../endpoints/global/files/ExportToExcelEndpoint';
 import { GetPaymentsEndpoint } from '../endpoints/organization/dashboard/payments/GetPaymentsEndpoint';
-import { XlsxTransformerColumnHelper } from '../helpers/xlsxAddressTransformerColumnFactory';
+import { XlsxTransformerColumnHelper } from '../helpers/XlsxTransformerColumnHelper';
 
 type PaymentWithItem = {
     payment: PaymentGeneral;
@@ -475,7 +475,7 @@ function getPayingOrganizationColumns(): XlsxTransformerColumn<PaymentGeneral>[]
         {
             id: 'payingOrganization.id',
             name: 'ID betalende groep',
-            width: 30,
+            width: 40,
             getValue: (object: PaymentGeneralWithStripeAccount) => {
                 return {
                     value: object.payingOrganization?.id || '',

package/src/helpers/AuthenticatedStructures.ts

@@ -423,6 +423,14 @@ export class AuthenticatedStructures {
             }
         }
 
+        if (includeContextOrganization && STAMHOOFD.singleOrganization && !Context.auth.organization) {
+            const found = organizations.get(STAMHOOFD.singleOrganization);
+            if (!found) {
+                const organization = await Context.auth.getOrganization(STAMHOOFD.singleOrganization);
+                organizations.set(organization.id, organization);
+            }
+        }
+
         const memberBlobs: MemberWithRegistrationsBlob[] = [];
         for (const member of members) {
             for (const registration of member.registrations) {

package/src/helpers/CheckSettlements.ts

@@ -23,7 +23,7 @@ type MolliePaymentJSON = {
 let lastSettlementCheck: Date | null = null;
 
 export async function checkAllStripePayouts(checkAll = false) {
-    if (STAMHOOFD.environment !== 'production') {
+    if (STAMHOOFD.environment !== 'production' || !STAMHOOFD.STRIPE_SECRET_KEY) {
         console.log('Skip settlement check');
         return;
     }

package/src/helpers/Context.ts

@@ -4,29 +4,42 @@ import { I18n } from '@stamhoofd/backend-i18n';
 import { Organization, Platform, RateLimiter, Token, User } from '@stamhoofd/models';
 import { AsyncLocalStorage } from 'async_hooks';
 
+import { AutoEncoder, Decoder, field, StringDecoder } from '@simonbackx/simple-encoding';
+import { ApiUserRateLimits } from '@stamhoofd/structures';
 import { AdminPermissionChecker } from './AdminPermissionChecker';
-import { AutoEncoder, field, Decoder, StringDecoder } from '@simonbackx/simple-encoding';
 
 export const apiUserRateLimiter = new RateLimiter({
     limits: [
         {
-            // Block heavy bursts (5req/s for 5s)
-            limit: 25,
+            limit: {
+                '': 25, // (5req/s for 5s)
+                [ApiUserRateLimits.Medium]: 10 * 5, // (10req/s for 5s)
+                [ApiUserRateLimits.High]: 25 * 5, // (100req/s for 5s)
+            },
             duration: 5 * 1000,
         },
         {
-            // max 1req/s during 150s
-            limit: 150,
-            duration: 150 * 1000,
+            limit: {
+                '': 120, // max 1req/s during 150s
+                [ApiUserRateLimits.Medium]: 240, // (2req/s for 150s)
+                [ApiUserRateLimits.High]: 480, // (4req/s for 150s)
+            },
+            duration: 120 * 1000,
         },
         {
-            // 1000 requests per hour
-            limit: 1000,
+            limit: {
+                '': 1000, // ± 0.27 request/s sustained for an hour = 3.6s between each request
+                [ApiUserRateLimits.Medium]: 2000, // ± 0.56 request/s sustained for an hour
+                [ApiUserRateLimits.High]: 4000, // ± 1.11 request/s sustained for an hour
+            },
             duration: 60 * 1000 * 60,
         },
         {
-            // 2000 requests per day
-            limit: 2000,
+            limit: {
+                '': 2_000, // max 2000 requests per day
+                [ApiUserRateLimits.Medium]: 14_400, // max 4000 requests per day
+                [ApiUserRateLimits.High]: 18_000, // max 10 requests per minute, sustained for a full day
+            },
             duration: 24 * 60 * 1000 * 60,
         },
     ],
@@ -169,7 +182,10 @@ export class ContextInstance {
             return await this.authenticate({ allowWithoutAccount });
         }
         catch (e) {
-            return {};
+            if (e.code === 'not_authenticated') {
+                return {};
+            }
+            throw e;
         }
     }
 
@@ -235,7 +251,7 @@ export class ContextInstance {
 
         // Rate limits for api users
         if (token.user.isApiUser) {
-            apiUserRateLimiter.track(this.organization?.id ?? token.user.id);
+            apiUserRateLimiter.track(this.organization?.id ?? token.user.id, 1, token.user.meta?.rateLimits ?? undefined);
         }
 
         const user = token.user;

package/src/helpers/StripeHelper.ts

@@ -6,7 +6,18 @@ import { Formatter } from '@stamhoofd/utility';
 import Stripe from 'stripe';
 
 export class StripeHelper {
+    static get notConfiguredError() {
+        return new SimpleError({
+            code: 'not_configured',
+            message: 'Stripe is not yet configured for this platform',
+            human: $t('Stripe is nog niet geconfigureerd voor dit platform'),
+        });
+    }
+
     static getInstance(accountId: string | null = null) {
+        if (!STAMHOOFD.STRIPE_SECRET_KEY) {
+            throw this.notConfiguredError;
+        }
         return new Stripe(STAMHOOFD.STRIPE_SECRET_KEY, { apiVersion: '2024-06-20', typescript: true, maxNetworkRetries: 0, timeout: 10000, stripeAccount: accountId ?? undefined });
     }
 
@@ -81,7 +92,7 @@ export class StripeHelper {
     }
 
     static async getStatus(payment: Payment, cancel = false, testMode = false): Promise<PaymentStatus> {
-        if (testMode && !STAMHOOFD.STRIPE_SECRET_KEY.startsWith('sk_test_')) {
+        if (testMode && !STAMHOOFD.STRIPE_SECRET_KEY?.startsWith('sk_test_')) {
             // Do not query anything
             return payment.status;
         }

package/src/helpers/{xlsxAddressTransformerColumnFactory.ts → XlsxTransformerColumnHelper.ts}

@@ -1,5 +1,5 @@
 import { XlsxTransformerColumn } from '@stamhoofd/excel-writer';
-import { Address, CountryHelper, Parent, ParentTypeHelper, PlatformMember } from '@stamhoofd/structures';
+import { Address, CountryHelper, Parent, ParentTypeHelper, PlatformMember, RecordAnswer, RecordCategory, RecordSettings, RecordType } from '@stamhoofd/structures';
 
 export class XlsxTransformerColumnHelper {
     static formatBoolean(value: boolean | undefined | null): string {
@@ -115,7 +115,8 @@ export class XlsxTransformerColumnHelper {
                         {
                             id: getId('street'),
                             name: `Straat`,
-                            width: 30,
+                            defaultCategory: 'Adres', // Ignore this name
+                            width: 40,
                             getValue: (object: T) => {
                                 const address = getAddress(object);
                                 return {
@@ -126,6 +127,7 @@ export class XlsxTransformerColumnHelper {
                         {
                             id: getId('number'),
                             name: 'Nummer',
+                            defaultCategory: 'Adres', // Ignore this name
                             width: 20,
                             getValue: (object: T) => {
                                 const address = getAddress(object);
@@ -137,6 +139,7 @@ export class XlsxTransformerColumnHelper {
                         {
                             id: getId('postalCode'),
                             name: 'Postcode',
+                            defaultCategory: 'Adres', // Ignore this name
                             width: 20,
                             getValue: (object: T) => {
                                 const address = getAddress(object);
@@ -148,6 +151,7 @@ export class XlsxTransformerColumnHelper {
                         {
                             id: getId('city'),
                             name: 'Stad',
+                            defaultCategory: 'Adres', // Ignore this name
                             width: 20,
                             getValue: (object: T) => {
                                 const address = getAddress(object);
@@ -159,6 +163,7 @@ export class XlsxTransformerColumnHelper {
                         {
                             id: getId('country'),
                             name: 'Land',
+                            defaultCategory: 'Adres', // Ignore this name
                             width: 20,
                             getValue: (object: T) => {
                                 const address = getAddress(object);
@@ -173,4 +178,65 @@ export class XlsxTransformerColumnHelper {
             },
         };
     }
+
+    static createRecordAnswersColumns<T>({ matchId, getRecordCategories, getRecordAnswers }: { matchId: string; getRecordCategories: () => RecordCategory[]; getRecordAnswers: (object: T) => Map<string, RecordAnswer> }): XlsxTransformerColumn<T> {
+        return {
+            match(id) {
+                if (id.startsWith(matchId + '.')) {
+                    const recordCategories = getRecordCategories();
+                    const flattenedCategories = RecordCategory.flattenCategoriesWith(recordCategories, r => r.excelColumns.length > 0);
+
+                    let recordCategory: RecordCategory | undefined;
+                    let recordSetting: RecordSettings | undefined;
+                    const recordSettingId = id.split('.')[1];
+
+                    for (const category of flattenedCategories) {
+                        const recordSettings = category.getAllRecords();
+                        const rr = recordSettings.find(r => r.id === recordSettingId);
+
+                        if (rr) {
+                            recordSetting = rr;
+                            recordCategory = category;
+                            break;
+                        }
+                    }
+
+                    if (!recordSetting || !recordCategory) {
+                        // Will throw a proper error itself
+                        console.log('recordSetting not found');
+                        return;
+                    }
+
+                    const columns = recordSetting.excelColumns;
+
+                    return columns.map(({ name, width, defaultCategory }, index) => {
+                        return {
+                            id: `${matchId}.${recordSettingId}.${index}`,
+                            name,
+                            width: width ?? 20,
+                            defaultCategory,
+                            category: recordCategory.name,
+                            getValue: (object: T) => {
+                                const answers = getRecordAnswers(object);
+                                const b = (answers.get(recordSettingId)?.excelValues[index] ?? {
+                                    value: '',
+                                });
+
+                                return {
+                                    ...b,
+                                    style: {
+                                        ...b.style,
+                                        alignment: {
+                                            ...b.style?.alignment,
+                                            wrapText: recordSetting.type === RecordType.Textarea,
+                                        },
+                                    },
+                                };
+                            },
+                        };
+                    });
+                }
+            },
+        };
+    }
 }

package/tests/e2e/api-rate-limits.test.ts

@@ -0,0 +1,188 @@
+/* eslint-disable jest/no-conditional-expect */
+import { Request } from '@simonbackx/simple-endpoints';
+import { Organization, OrganizationFactory, Token, UserFactory } from '@stamhoofd/models';
+
+import { PatchMap } from '@simonbackx/simple-encoding';
+import { ApiUser, ApiUserRateLimits, PermissionLevel, Permissions, PermissionsResourceType, ResourcePermissions, UserMeta, UserPermissions } from '@stamhoofd/structures';
+import { SHExpect, TestUtils } from '@stamhoofd/test-utils';
+import { CreateApiUserEndpoint } from '../../src/endpoints/organization/dashboard/users/CreateApiUserEndpoint';
+import { testServer } from '../helpers/TestServer';
+import { GetUserEndpoint } from '../../src/endpoints/auth/GetUserEndpoint';
+
+describe('E2E.APIRateLimits', () => {
+    // Test endpoint
+    const createEndpoint = new CreateApiUserEndpoint();
+    const getUserEndpoint = new GetUserEndpoint();
+    let organization: Organization;
+
+    beforeEach(async () => {
+        TestUtils.setEnvironment('userMode', 'platform');
+        organization = await new OrganizationFactory({}).create();
+    });
+
+    /**
+     * Note: we don't use a factory because this is an E2E test and
+     * we also want to check if the created tokens with the API are actually marked as API-keys and not normal users.
+     */
+    async function createAPIToken(rateLimits: ApiUserRateLimits | null) {
+        const user = await new UserFactory({
+            globalPermissions: Permissions.create({
+                level: PermissionLevel.Full,
+            }),
+        }).create();
+        const token = await Token.createToken(user);
+
+        const createRequest = Request.buildJson('POST', '/api-keys', organization.getApiHost(), ApiUser.create({
+            permissions: UserPermissions.create({
+                organizationPermissions: new Map([
+                    [organization.id, Permissions.create({ level: PermissionLevel.Read })],
+                ]),
+            }),
+            meta: UserMeta.create({
+                rateLimits,
+            }),
+        }));
+        createRequest.headers.authorization = 'Bearer ' + token.accessToken;
+        const createResponse = await testServer.test(createEndpoint, createRequest);
+
+        expect(createResponse.body.meta?.rateLimits ?? undefined).toEqual(rateLimits ?? undefined);
+
+        return createResponse.body.token;
+    }
+
+    test('By default throws after 25 requests in less than 5s', async () => {
+        const token = await createAPIToken(null);
+
+        // Start firing
+        for (let i = 0; i < 30; i++) {
+            const request = Request.buildJson('GET', '/user', organization.getApiHost());
+            request.headers.authorization = 'Bearer ' + token;
+            const promise = testServer.test(getUserEndpoint, request);
+
+            if (i < 25) {
+                try {
+                    await expect(promise).toResolve();
+                }
+                catch (e) {
+                    let error: any = null;
+                    try {
+                        await promise;
+                    }
+                    catch (e) {
+                        error = e;
+                    }
+                    throw new Error('The endpoint rejected at call ' + i + ' with error message ' + error?.message);
+                }
+            }
+            else {
+                await expect(promise).rejects.toThrow(
+                    SHExpect.simpleError({
+                        code: 'rate_limit',
+                    }),
+                );
+            }
+        }
+    });
+
+    test('Normal rate limit throws after 25 requests in less than 5s', async () => {
+        const token = await createAPIToken(ApiUserRateLimits.Normal);
+
+        // Start firing
+        for (let i = 0; i < 30; i++) {
+            const request = Request.buildJson('GET', '/user', organization.getApiHost());
+            request.headers.authorization = 'Bearer ' + token;
+            const promise = testServer.test(getUserEndpoint, request);
+
+            if (i < 25) {
+                try {
+                    await expect(promise).toResolve();
+                }
+                catch (e) {
+                    let error: any = null;
+                    try {
+                        await promise;
+                    }
+                    catch (e) {
+                        error = e;
+                    }
+                    throw new Error('The endpoint rejected at call ' + i + ' with error message ' + error?.message);
+                }
+            }
+            else {
+                await expect(promise).rejects.toThrow(
+                    SHExpect.simpleError({
+                        code: 'rate_limit',
+                    }),
+                );
+            }
+        }
+    });
+
+    test('Medium rate limits throw after 50 requests in less than 5s', async () => {
+        const token = await createAPIToken(ApiUserRateLimits.Medium);
+
+        // Start firing
+        for (let i = 0; i < 60; i++) {
+            const request = Request.buildJson('GET', '/user', organization.getApiHost());
+            request.headers.authorization = 'Bearer ' + token;
+            const promise = testServer.test(getUserEndpoint, request);
+
+            if (i < 50) {
+                try {
+                    await expect(promise).toResolve();
+                }
+                catch (e) {
+                    let error: any = null;
+                    try {
+                        await promise;
+                    }
+                    catch (e) {
+                        error = e;
+                    }
+                    throw new Error('The endpoint rejected at call ' + i + ' with error message ' + error?.message);
+                }
+            }
+            else {
+                await expect(promise).rejects.toThrow(
+                    SHExpect.simpleError({
+                        code: 'rate_limit',
+                    }),
+                );
+            }
+        }
+    });
+
+    test('High rate limits throw after 125 requests in less than 5s', async () => {
+        const token = await createAPIToken(ApiUserRateLimits.High);
+
+        // Start firing
+        for (let i = 0; i < 140; i++) {
+            const request = Request.buildJson('GET', '/user', organization.getApiHost());
+            request.headers.authorization = 'Bearer ' + token;
+            const promise = testServer.test(getUserEndpoint, request);
+
+            if (i < 125) {
+                try {
+                    await expect(promise).toResolve();
+                }
+                catch (e) {
+                    let error: any = null;
+                    try {
+                        await promise;
+                    }
+                    catch (e) {
+                        error = e;
+                    }
+                    throw new Error('The endpoint rejected at call ' + i + ' with error message ' + error?.message);
+                }
+            }
+            else {
+                await expect(promise).rejects.toThrow(
+                    SHExpect.simpleError({
+                        code: 'rate_limit',
+                    }),
+                );
+            }
+        }
+    });
+});

package/tests/helpers/StripeMocker.ts

@@ -165,9 +165,15 @@ export class StripeMocker {
         const endpoint = new StripeWebookEndpoint();
 
         const r = Request.buildJson('POST', `/stripe/webhooks`, undefined, payload);
+        const secret = payload.account ? STAMHOOFD.STRIPE_CONNECT_ENDPOINT_SECRET : STAMHOOFD.STRIPE_ENDPOINT_SECRET;
+
+        if (!secret) {
+            throw new Error('No stripe secret set in env');
+        }
+
         r.headers['stripe-signature'] = stripe.webhooks.generateTestHeaderString({
             payload: await r.body,
-            secret: payload.account ? STAMHOOFD.STRIPE_CONNECT_ENDPOINT_SECRET : STAMHOOFD.STRIPE_ENDPOINT_SECRET,
+            secret: secret,
         });
         await testServer.test(endpoint, r);
     }