@stamhoofd/backend 2.143.0 → 2.144.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@stamhoofd/backend",
-    "version": "2.143.0",
+    "version": "2.144.0",
     "type": "module",
     "main": "./dist/index.js",
     "exports": {
@@ -65,20 +65,20 @@
         "@simonbackx/simple-errors": "1.5.0",
         "@simonbackx/simple-logging": "1.0.1",
         "@simplewebauthn/server": "13.3.2",
-        "@stamhoofd/backend-env": "2.143.0",
-        "@stamhoofd/backend-i18n": "2.143.0",
-        "@stamhoofd/backend-middleware": "2.143.0",
-        "@stamhoofd/crons": "2.143.0",
-        "@stamhoofd/email": "2.143.0",
-        "@stamhoofd/excel-writer": "2.143.0",
-        "@stamhoofd/logging": "2.143.0",
-        "@stamhoofd/models": "2.143.0",
-        "@stamhoofd/object-differ": "2.143.0",
-        "@stamhoofd/queues": "2.143.0",
-        "@stamhoofd/sql": "2.143.0",
-        "@stamhoofd/structures": "2.143.0",
-        "@stamhoofd/types": "2.143.0",
-        "@stamhoofd/utility": "2.143.0",
+        "@stamhoofd/backend-env": "2.144.0",
+        "@stamhoofd/backend-i18n": "2.144.0",
+        "@stamhoofd/backend-middleware": "2.144.0",
+        "@stamhoofd/crons": "2.144.0",
+        "@stamhoofd/email": "2.144.0",
+        "@stamhoofd/excel-writer": "2.144.0",
+        "@stamhoofd/logging": "2.144.0",
+        "@stamhoofd/models": "2.144.0",
+        "@stamhoofd/object-differ": "2.144.0",
+        "@stamhoofd/queues": "2.144.0",
+        "@stamhoofd/sql": "2.144.0",
+        "@stamhoofd/structures": "2.144.0",
+        "@stamhoofd/types": "2.144.0",
+        "@stamhoofd/utility": "2.144.0",
         "archiver": "7.0.1",
         "argon2": "0.44.0",
         "axios": "1.18.1",
@@ -102,7 +102,7 @@
         "uuid": "14.0.1"
     },
     "devDependencies": {
-        "@stamhoofd/test-utils": "2.143.0",
+        "@stamhoofd/test-utils": "2.144.0",
         "@types/cookie": "0.6.0",
         "@types/luxon": "3.7.2",
         "@types/mailparser": "3.4.6",
@@ -117,5 +117,5 @@
     "publishConfig": {
         "access": "public"
     },
-    "gitHead": "8959c4f720d332be9769e033d40bb25f9e6aa322"
+    "gitHead": "31bc31b2a56d9516e8bc6be5779bc58af0022a88"
 }

package/src/boot.ts

@@ -43,6 +43,7 @@ Column.setJSONVersion(Version);
 
 // Set timezone to UTC
 process.env.TZ = 'UTC';
+Error.stackTraceLimit = 50;
 
 if (STAMHOOFD.environment === 'development') {
     Error.stackTraceLimit = 100;

package/src/crons/fake-settlements.test.ts

@@ -90,8 +90,8 @@ describe('Cron.fake-settlements', () => {
             settledAt: Formatter.luxon(inWeek(2)).startOf('week').plus({ weeks: 1, days: 2 }).toJSDate(),
 
             // Refunds are taken out of the payout, just like the provider does, and so are the
-            // fake Mollie costs: 3 payments x 0.30 + 21% VAT
-            amount: 65_0000 - 90_00 - 18_90,
+            // fake Mollie fees: 3 payments x 0.363
+            amount: 65_0000 - 3 * 36_30,
         });
 
         // All payments of the week point to the same payout
@@ -135,8 +135,7 @@ describe('Cron.fake-settlements', () => {
         ];
 
         // Same week, but four accounts: four payouts, each holding only its own money. The Mollie
-        // payouts are reduced by the fake cost of one payment (0.30 + 21% VAT); the Stripe payments
-        // carry no fees here
+        // payouts are reduced by the fake fee of one payment; the Stripe payments carry no fees here
         expect(settlements.map(s => s!.amount)).toEqual([50_0000 - 36_30, 30_0000 - 36_30, 20_0000, 15_0000]);
         expect(new Set(settlements.map(s => s!.id)).size).toBe(4);
         expect(new Set(settlements.map(s => s!.reference)).size).toBe(4);
@@ -192,7 +191,7 @@ describe('Cron.fake-settlements', () => {
             return await Settlement.select().where('externalId', reference!.id).first(true);
         };
 
-        test('A Mollie payout stores payment lines and cost rows, and reconciles to zero', async () => {
+        test('A Mollie payout stores payment lines and fee rows, and reconciles to zero', async () => {
             const first = await createPayment({ price: 50_0000 });
             const second = await createPayment({ price: -10_0000, type: PaymentType.Refund });
 
@@ -201,7 +200,7 @@ describe('Cron.fake-settlements', () => {
             const settlement = await getSettlementRow(first);
             expect(settlement.provider).toBe(PaymentProvider.Mollie);
             expect(settlement.organizationId).toBe(organization.id);
-            expect(settlement.amount).toBe(40_0000 - 60_00 - 12_60);
+            expect(settlement.amount).toBe(40_0000 - 2 * 36_30);
             expect(settlement.unexplainedAmount).toBe(0);
             expect(settlement.syncedAt).not.toBeNull();
 
@@ -211,12 +210,15 @@ describe('Cron.fake-settlements', () => {
                 [second.id, -10_0000],
             ].sort());
 
-            const settledMonth = settlement.settledAt.getFullYear() + '-' + (settlement.settledAt.getMonth() + 1).toString().padStart(2, '0');
             const charges = await SettlementCharge.select().where('settlementId', settlement.id).fetch();
-            expect(charges.map(c => ({ type: c.type, amount: c.amount, providerInvoiceId: c.providerInvoiceId })).sort((a, b) => a.amount - b.amount)).toEqual([
-                { type: SettlementChargeType.ProviderTransactionFee, amount: -60_00, providerInvoiceId: 'fake-invoice-mollie-' + settledMonth },
-                { type: SettlementChargeType.Tax, amount: -12_60, providerInvoiceId: 'fake-invoice-mollie-' + settledMonth },
-            ]);
+            expect(charges.map(c => [c.paymentId, c.type, c.amount]).sort()).toEqual([
+                [first.id, SettlementChargeType.ProviderTransactionFee, -36_30],
+                [second.id, SettlementChargeType.ProviderTransactionFee, -36_30],
+            ].sort());
+
+            // Each settled payment holds its own fee
+            expect((await Payment.getByID(first.id))!.transferFee).toBe(36_30);
+            expect((await Payment.getByID(second.id))!.transferFee).toBe(36_30);
         });
 
         test('A Stripe application fee is mirrored on the payout and a monthly platform payout', async () => {

package/src/crons/fake-settlements.ts

@@ -34,10 +34,9 @@ const PAYOUT_DELAY_DAYS = 2;
 const BATCH_SIZE = 1000;
 
 /**
- * Fake Mollie transaction fee per settled payment, excluding VAT.
+ * Fake Mollie transaction fee per settled payment, including VAT.
  */
-const MOLLIE_FEE_PER_PAYMENT = 30_00;
-const MOLLIE_FEE_VAT_PERCENTAGE = 21;
+const MOLLIE_FEE_PER_PAYMENT = 36_30;
 
 export async function createFakeSettlements() {
     if (STAMHOOFD.environment !== 'development') {
@@ -183,8 +182,7 @@ async function settleWeek(payments: Payment[], weekStart: DateTime, settledAt: D
                 return total + serviceFee + transferFee;
             }, 0);
         } else if (provider === PaymentProvider.Mollie) {
-            const net = MOLLIE_FEE_PER_PAYMENT * group.length;
-            fees = net + Math.round(net * MOLLIE_FEE_VAT_PERCENTAGE / 100);
+            fees = MOLLIE_FEE_PER_PAYMENT * group.length;
         }
 
         const grossAmount = group.reduce((total, payment) => total + payment.price, 0);
@@ -223,7 +221,7 @@ async function settleWeek(payments: Payment[], weekStart: DateTime, settledAt: D
         if (provider === PaymentProvider.Stripe) {
             await createStripeFeeRows(settlement, group, settledAt, platformOrganizationId);
         } else if (provider === PaymentProvider.Mollie) {
-            await createMollieCostRows(settlement, group, settledAt);
+            await createMollieFeeRows(settlement, group);
         }
 
         await finishFakeSync(settlement);
@@ -322,43 +320,29 @@ async function createStripeFeeRows(settlement: Settlement, group: Payment[], set
 
     // A dev database can contain fee payments (e.g. copied history): keep their derived lines
     // in sync like a real payout walk does
-    await SettlementService.updatePaymentSettlementsForAccountDeductionBalanceItems([...invoicedFeeBalanceItemIds]);
+    await SettlementService.updatePaymentSettlementsForApplicationFeeBalanceItems([...invoicedFeeBalanceItemIds]);
 
     await finishFakeSync(platformSettlement);
 }
 
 /**
- * Mollie deducts its own costs from the settlement: one fake transaction fee line plus its VAT, so
- * the settlement reconciles to 0 like a real one.
+ * Mollie deducts its fee per transaction: one fake fee charge per settled payment, linked to it
+ * like the real balance transaction walk stores, so the settlement reconciles to 0.
  */
-async function createMollieCostRows(settlement: Settlement, group: Payment[], settledAt: Date) {
-    const externalId = settlement.externalId + ':cost:0';
-    const existing = await SettlementCharge.select().where('externalId', externalId).first(false);
-
-    const addedNet = MOLLIE_FEE_PER_PAYMENT * group.length;
-    const net = (existing ? -existing.amount : 0) + addedNet;
-    const vat = Math.round(net * MOLLIE_FEE_VAT_PERCENTAGE / 100);
-    const providerInvoiceId = 'fake-invoice-mollie-' + SettlementService.getPeriodKey(settledAt);
-
-    await SettlementService.upsertCharge({
-        type: SettlementChargeType.ProviderTransactionFee,
-        externalId,
-        amount: -net,
-        settlementId: settlement.id,
-        organizationId: settlement.organizationId,
-        providerInvoiceId,
-        description: 'DEV Mollie transactiekosten',
-        occurredAt: settledAt,
-    });
+async function createMollieFeeRows(settlement: Settlement, group: Payment[]) {
+    for (const payment of group) {
+        await SettlementService.upsertCharge({
+            type: SettlementChargeType.ProviderTransactionFee,
+            externalId: 'fake-baltr-' + payment.id,
+            amount: -MOLLIE_FEE_PER_PAYMENT,
+            settlementId: settlement.id,
+            paymentId: payment.id,
+            organizationId: settlement.organizationId,
+            description: 'DEV Mollie transactiekosten',
+            occurredAt: payment.paidAt!,
+        });
 
-    await SettlementService.upsertCharge({
-        type: SettlementChargeType.Tax,
-        externalId: externalId + ':tax',
-        amount: -vat,
-        settlementId: settlement.id,
-        organizationId: settlement.organizationId,
-        providerInvoiceId,
-        description: 'DEV BTW op Mollie transactiekosten',
-        occurredAt: settledAt,
-    });
+        payment.transferFee = MOLLIE_FEE_PER_PAYMENT;
+        await payment.save();
+    }
 }

package/src/endpoints/global/files/UploadFile.ts

@@ -1,4 +1,4 @@
-import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3'; // ES Modules import
+import { PutObjectCommand } from '@aws-sdk/client-s3'; // ES Modules import
 import type { DecodedRequest, Request } from '@simonbackx/simple-endpoints';
 import { Endpoint, Response } from '@simonbackx/simple-endpoints';
 import { SimpleError } from '@simonbackx/simple-errors';
@@ -11,7 +11,8 @@ import { v4 as uuidv4 } from 'uuid';
 import type { Decoder } from '@simonbackx/simple-encoding';
 import { AutoEncoder, BooleanDecoder, field } from '@simonbackx/simple-encoding';
 import { Context } from '../../../helpers/Context.js';
-import { limiter } from './UploadImage.js';
+import { limiter, organizationLimiter } from './UploadImage.js';
+import type { Organization } from '@stamhoofd/models';
 import { Image } from '@stamhoofd/models';
 
 type Params = Record<string, never>;
@@ -58,11 +59,20 @@ export class UploadFile extends Endpoint<Params, Query, Body, ResponseBody> {
     }
 
     async handle(request: DecodedRequest<Params, Query, Body>) {
-        await Context.setOptionalOrganizationScope();
-        const { user } = await Context.authenticate();
+        const organization = await Context.setOptionalOrganizationScope();
+        const { user } = await Context.optionalAuthenticate();
 
-        if (!Context.auth.canUpload({ private: request.query.isPrivate })) {
-            throw Context.auth.error();
+        // Webshops allow for file upload, but don't have a user attached
+        if (user) {
+            if (!Context.auth?.canUpload({ private: request.query.isPrivate })) {
+                throw Context.auth.error();
+            }
+        } else if (!organization) {
+            throw new SimpleError({
+                code: 'permission_denied',
+                message: 'Endpoints needs organization if no user is present',
+                human: $t('%ZlM'),
+            });
         }
 
         if (!STAMHOOFD.SPACES_BUCKET || !STAMHOOFD.SPACES_ENDPOINT || !STAMHOOFD.SPACES_KEY || !STAMHOOFD.SPACES_SECRET) {
@@ -77,7 +87,14 @@ export class UploadFile extends Endpoint<Params, Query, Body, ResponseBody> {
             throw new Error('Not supported without real request');
         }
 
-        limiter.track(user.id, 1);
+        if (user) {
+            limiter.track(user.id, 1);
+        } else {
+            limiter.track(request.request.getIP(), 1);
+            if (organization) {
+                organizationLimiter.track(organization.id, 1);
+            }
+        }
 
         const form = formidable({ maxFileSize: 20 * 1024 * 1024, maxFields: 1, keepExtensions: true });
         const file = await new Promise<FormidableFile>((resolve, reject) => {
@@ -89,34 +106,43 @@ export class UploadFile extends Endpoint<Params, Query, Body, ResponseBody> {
                 }));
                 return;
             }
-            form.parse(request.request.request, (err, fields, files) => {
-                if (err) {
-                    reject(err);
-                    return;
+            try {
+                if (request.request.request?.destroyed) {
+                    throw new Error('Request destroyed before parsing');
                 }
-
-                if (!files.file || !Array.isArray(files.file) || files.file.length !== 1) {
-                    reject(new SimpleError({
-                        code: 'missing_field',
-                        message: 'Missing file',
-                        field: 'file',
-                    }));
-                    return;
-                }
-
-                resolve(files.file[0]);
-            });
+                form.parse(request.request.request, (err, fields, files) => {
+                    console.log('form.parse', err, fields, files);
+                    if (err) {
+                        reject(err);
+                        return;
+                    }
+
+                    if (!files.file || !Array.isArray(files.file) || files.file.length !== 1) {
+                        reject(new SimpleError({
+                            code: 'missing_field',
+                            message: 'Missing file',
+                            field: 'file',
+                        }));
+                        return;
+                    }
+
+                    resolve(files.file[0]);
+                });
+            } catch (e) {
+                console.error(e);
+                reject(e);
+            }
         });
 
         try {
-            return await this.upload(request, file, user);
+            return await this.upload(request, file, organization, user);
         } finally {
             // Formidable wrote the upload to a temporary file
             await fs.rm(file.filepath, { force: true }).catch(() => { /* we can't do anything about this */ });
         }
     }
 
-    private async upload(request: DecodedRequest<Params, Query, Body>, file: FormidableFile, user: { id: string }) {
+    private async upload(request: DecodedRequest<Params, Query, Body>, file: FormidableFile, organization: Organization | null, user?: { id: string }) {
         if (!STAMHOOFD.SPACES_BUCKET || !STAMHOOFD.SPACES_ENDPOINT || !STAMHOOFD.SPACES_KEY || !STAMHOOFD.SPACES_SECRET) {
             throw new SimpleError({
                 code: 'not_available',
@@ -156,6 +182,8 @@ export class UploadFile extends Endpoint<Params, Query, Body, ResponseBody> {
         if (request.query.isPrivate && user) {
             // Private files
             prefix += 'users/' + user.id + '/';
+        } else if (request.query.isPrivate && !user && organization) {
+            prefix += 'anonymous/' + organization.id + '/';
         } else {
             // Public files
             prefix += 'p/';

package/src/endpoints/global/files/UploadImage.ts

@@ -3,6 +3,7 @@ import { AutoEncoder, BooleanDecoder, field, ObjectData } from '@simonbackx/simp
 import type { DecodedRequest, Request } from '@simonbackx/simple-endpoints';
 import { Endpoint, Response } from '@simonbackx/simple-endpoints';
 import { SimpleError } from '@simonbackx/simple-errors';
+import type { Organization } from '@stamhoofd/models';
 import { Image, RateLimiter } from '@stamhoofd/models';
 import { Image as ImageStruct, ResolutionRequest, supportedImageTypes } from '@stamhoofd/structures';
 import formidable from 'formidable';
@@ -44,6 +45,15 @@ export const limiter = new RateLimiter({
         },
     ],
 });
+export const organizationLimiter = new RateLimiter({
+    limits: [
+        {
+            // Max 200 per day
+            limit: 200,
+            duration: 60 * 1000 * 60 * 24,
+        },
+    ],
+});
 
 export class UploadImage extends Endpoint<Params, Query, Body, ResponseBody> {
     queryDecoder = Query as Decoder<Query>;
@@ -63,11 +73,20 @@ export class UploadImage extends Endpoint<Params, Query, Body, ResponseBody> {
     }
 
     async handle(request: DecodedRequest<Params, Query, Body>) {
-        await Context.setOptionalOrganizationScope();
-        const { user } = await Context.authenticate();
+        const organization = await Context.setOptionalOrganizationScope();
+        const { user } = await Context.optionalAuthenticate();
 
-        if (!Context.auth.canUpload({ private: request.query.isPrivate })) {
-            throw Context.auth.error();
+        // Webshops allow for file upload, but don't have a user attached
+        if (user) {
+            if (!Context.auth?.canUpload({ private: request.query.isPrivate })) {
+                throw Context.auth.error();
+            }
+        } else if (!organization) {
+            throw new SimpleError({
+                code: 'permission_denied',
+                message: 'Endpoints needs organization if no user is present',
+                human: $t('%ZlM'),
+            });
         }
 
         if (!STAMHOOFD.SPACES_BUCKET || !STAMHOOFD.SPACES_ENDPOINT || !STAMHOOFD.SPACES_KEY || !STAMHOOFD.SPACES_SECRET) {
@@ -82,7 +101,14 @@ export class UploadImage extends Endpoint<Params, Query, Body, ResponseBody> {
             throw new Error('Not supported without real request');
         }
 
-        limiter.track(user.id, 1);
+        if (user) {
+            limiter.track(user.id, 1);
+        } else {
+            limiter.track(request.request.getIP(), 1);
+            if (organization) {
+                organizationLimiter.track(organization.id, 1);
+            }
+        }
 
         const form = formidable({
             maxFileSize: 5 * 1024 * 1024,
@@ -147,7 +173,7 @@ export class UploadImage extends Endpoint<Params, Query, Body, ResponseBody> {
             }
 
             const fileContent = await fs.readFile(file.filepath);
-            const image = await Image.create(fileContent, imageType.contentType, resolutions, request.query.isPrivate, user);
+            const image = await Image.create(fileContent, imageType.contentType, resolutions, request.query.isPrivate, user, organization as Organization | null | undefined);
             return new Response(ImageStruct.create(image));
         } finally {
             // Formidable wrote the upload to a temporary file

package/src/helpers/ApplicationFeeInvoicer.ts

@@ -340,7 +340,7 @@ export class ApplicationFeeInvoicer {
         // Paid now, not in the billed month: the invoices cron only picks up recent payments, so a
         // month that is billed late would otherwise never end up on an invoice
         await PaymentService.handlePaymentStatusUpdate(payment, sellingOrganization, PaymentStatus.Succeeded, new Date());
-        await SettlementService.updatePaymentSettlementsForAccountDeductionPayment(payment);
+        await SettlementService.updatePaymentSettlementsForApplicationFeePayment(payment);
     }
 
     /**

package/src/helpers/Context.ts

@@ -47,11 +47,6 @@ export const apiUserRateLimiter = new RateLimiter({
     ],
 });
 
-export class AuthorizationPostBody extends AutoEncoder {
-    @field({ decoder: StringDecoder })
-    header_authorization: string;
-}
-
 export class ContextInstance {
     request: Request;
     queries: { query: string; time?: number }[] = [];
@@ -241,16 +236,7 @@ export class ContextInstance {
     async authenticate(options: { allowMFASetupToken: true; allowWithoutAccount?: boolean; allowUnscoped?: boolean }): Promise<{ user: User; token: Token } | { user: User; setupToken: MFAToken }>;
     async authenticate(options?: { allowMFASetupToken?: false | undefined; allowWithoutAccount?: boolean; allowUnscoped?: boolean }): Promise<{ user: User; token: Token }>;
     async authenticate({ allowWithoutAccount = false, allowUnscoped = false, allowMFASetupToken = false }: { allowMFASetupToken?: boolean; allowWithoutAccount?: boolean; allowUnscoped?: boolean } = {}): Promise<{ user: User; token: Token } | { user: User; setupToken: MFAToken }> {
-        let header = this.request.headers.authorization;
-
-        if (!header && this.request.method === 'POST') {
-            try {
-                const decoded = await DecodedRequest.fromRequest(this.request, undefined, undefined, AuthorizationPostBody as Decoder<AuthorizationPostBody>);
-                header = decoded.body.header_authorization;
-            } catch (e) {
-                // Ignore: failed to read from body
-            }
-        }
+        const header = this.request.headers.authorization;
 
         if (!header) {
             throw new SimpleError({