@stamhoofd/backend 2.140.0 → 2.141.0

package/src/services/ApplicationFeeService.test.ts

@@ -0,0 +1,256 @@
+import type { Organization } from '@stamhoofd/models';
+import { BalanceItem, BalanceItemPayment, Invoice, OrganizationFactory, Payment, StripeAccount } from '@stamhoofd/models';
+import { ApplicationFee } from '@stamhoofd/models/models/ApplicationFee.js';
+import { SettlementCharge } from '@stamhoofd/models/models/SettlementCharge.js';
+import { BalanceItemStatus, BalanceItemType, PaymentMethod, PaymentProvider, PaymentStatus } from '@stamhoofd/structures';
+import { ApplicationFeeType } from '@stamhoofd/structures/settlements/ApplicationFeeType.js';
+import { SettlementChargeType } from '@stamhoofd/structures/settlements/SettlementChargeType.js';
+import { v4 as uuidv4 } from 'uuid';
+
+import { initMembershipOrganization } from '../../tests/init/initMembershipOrganization.js';
+import { ApplicationFeeService, LEGACY_FEE_PAYMENT_REFERENCE_PREFIX } from './ApplicationFeeService.js';
+import { SettlementService } from './SettlementService.js';
+
+describe('ApplicationFeeService', () => {
+    let membershipOrganization: Organization;
+
+    // A month the legacy invoicer would have billed
+    const occurredAt = new Date(2025, 2, 14);
+    const legacyReference = LEGACY_FEE_PAYMENT_REFERENCE_PREFIX + '2025-03-01';
+
+    beforeAll(async () => {
+        membershipOrganization = await initMembershipOrganization();
+    });
+
+    beforeEach(() => {
+        ApplicationFeeService.resetWarnings();
+    });
+
+    const init = async () => {
+        const organization = await new OrganizationFactory({}).create();
+        const stripeAccount = new StripeAccount();
+        stripeAccount.organizationId = organization.id;
+        stripeAccount.accountId = 'acct_' + uuidv4();
+        await stripeAccount.save();
+        return { organization, stripeAccount };
+    };
+
+    /**
+     * The fee payment and balance items the legacy invoicer created for a month.
+     */
+    const createLegacyFeePayment = async (organization: Organization, stripeAccount: StripeAccount | null, { types = [BalanceItemType.ServiceFee, BalanceItemType.TransferFee], price = 2_50_00 } = {}) => {
+        const payment = new Payment();
+        payment.organizationId = membershipOrganization.id;
+        payment.payingOrganizationId = organization.id;
+        payment.stripeAccountId = stripeAccount?.id ?? null;
+        payment.method = PaymentMethod.AccountDeductions;
+        payment.provider = PaymentProvider.Stripe;
+        payment.status = PaymentStatus.Succeeded;
+        payment.reference = legacyReference;
+        payment.price = price;
+        payment.paidAt = occurredAt;
+        await payment.save();
+
+        const balanceItems: BalanceItem[] = [];
+        for (const type of types) {
+            const item = new BalanceItem();
+            item.type = type;
+            item.organizationId = membershipOrganization.id;
+            item.payingOrganizationId = organization.id;
+            item.unitPrice = price;
+            item.quantity = 1;
+            item.status = BalanceItemStatus.Hidden;
+            await item.save();
+            balanceItems.push(item);
+
+            const balanceItemPayment = new BalanceItemPayment();
+            balanceItemPayment.balanceItemId = item.id;
+            balanceItemPayment.paymentId = payment.id;
+            balanceItemPayment.organizationId = membershipOrganization.id;
+            balanceItemPayment.price = price;
+            await balanceItemPayment.save();
+        }
+
+        return { payment, balanceItems };
+    };
+
+    const upsertFee = async (organization: Organization, stripeAccount: StripeAccount, { type = ApplicationFeeType.Service, amount = 30_00 } = {}) => {
+        const externalId = 'fee_' + uuidv4();
+        const charge = await SettlementService.upsertCharge({
+            type: type === ApplicationFeeType.Service ? SettlementChargeType.ApplicationFeeService : SettlementChargeType.ApplicationFeeTransfer,
+            externalId: externalId + ':' + type,
+            amount: -amount,
+            applicationFeeId: externalId,
+            organizationId: organization.id,
+            stripeAccountId: stripeAccount.id,
+            occurredAt,
+        });
+
+        const fee = await ApplicationFeeService.upsertFee({
+            externalId,
+            type,
+            amount,
+            organizationId: membershipOrganization.id,
+            payingOrganizationId: organization.id,
+            payingStripeAccountId: stripeAccount.id,
+            settlementChargeId: charge.id,
+            occurredAt,
+        });
+
+        return { fee, charge };
+    };
+
+    describe('legacy linking', () => {
+        test('a fee of a month the legacy invoicer billed is linked right away', async () => {
+            const { organization, stripeAccount } = await init();
+            const { balanceItems } = await createLegacyFeePayment(organization, stripeAccount);
+
+            const { fee } = await upsertFee(organization, stripeAccount, { type: ApplicationFeeType.Service });
+            const { fee: transferFee } = await upsertFee(organization, stripeAccount, { type: ApplicationFeeType.Transfer });
+
+            expect(fee.balanceItemId).toBe(balanceItems.find(i => i.type === BalanceItemType.ServiceFee)!.id);
+            expect(transferFee.balanceItemId).toBe(balanceItems.find(i => i.type === BalanceItemType.TransferFee)!.id);
+        });
+
+        test('legacy payments without a Stripe account are found too', async () => {
+            const { organization, stripeAccount } = await init();
+            const { balanceItems } = await createLegacyFeePayment(organization, null);
+
+            const { fee } = await upsertFee(organization, stripeAccount);
+
+            expect(fee.balanceItemId).toBe(balanceItems.find(i => i.type === BalanceItemType.ServiceFee)!.id);
+        });
+
+        test('a month that was never billed stays uninvoiced', async () => {
+            const { organization, stripeAccount } = await init();
+
+            const { fee } = await upsertFee(organization, stripeAccount);
+
+            expect(fee.balanceItemId).toBeNull();
+        });
+
+        test('a legacy payment without the matching balance item leaves the fee uninvoiced', async () => {
+            const { organization, stripeAccount } = await init();
+            await createLegacyFeePayment(organization, stripeAccount, { types: [BalanceItemType.ServiceFee] });
+
+            const { fee } = await upsertFee(organization, stripeAccount, { type: ApplicationFeeType.Transfer });
+
+            expect(fee.balanceItemId).toBeNull();
+        });
+
+        test('re-storing the same fee links it as soon as the legacy payment exists', async () => {
+            const { organization, stripeAccount } = await init();
+            const externalId = 'fee_' + uuidv4();
+            const charge = await SettlementService.upsertCharge({
+                type: SettlementChargeType.ApplicationFeeService,
+                externalId: externalId + ':' + ApplicationFeeType.Service,
+                amount: -30_00,
+                applicationFeeId: externalId,
+                organizationId: organization.id,
+                occurredAt,
+            });
+
+            const data = {
+                externalId,
+                type: ApplicationFeeType.Service,
+                amount: 30_00,
+                organizationId: membershipOrganization.id,
+                payingOrganizationId: organization.id,
+                payingStripeAccountId: stripeAccount.id,
+                settlementChargeId: charge.id,
+                occurredAt,
+            };
+
+            const unlinked = await ApplicationFeeService.upsertFee(data);
+            expect(unlinked.balanceItemId).toBeNull();
+
+            const { balanceItems } = await createLegacyFeePayment(organization, stripeAccount);
+            const linked = await ApplicationFeeService.upsertFee(data);
+
+            expect(linked.id).toBe(unlinked.id);
+            expect(linked.balanceItemId).toBe(balanceItems.find(i => i.type === BalanceItemType.ServiceFee)!.id);
+        });
+    });
+
+    describe('providerInvoiceId', () => {
+        const createInvoice = async (payment: Payment, number: string | null) => {
+            const invoice = new Invoice();
+            invoice.organizationId = membershipOrganization.id;
+            invoice.payingOrganizationId = payment.payingOrganizationId;
+            invoice.number = number;
+            invoice.totalWithVAT = payment.price;
+            await invoice.save();
+
+            payment.invoiceId = invoice.id;
+            await payment.save();
+            return invoice;
+        };
+
+        test('the invoice number is stamped on the deduction charge when the fee is linked', async () => {
+            const { organization, stripeAccount } = await init();
+            const { payment } = await createLegacyFeePayment(organization, stripeAccount);
+            await createInvoice(payment, '2025001');
+
+            const { charge } = await upsertFee(organization, stripeAccount);
+
+            expect((await SettlementCharge.getByID(charge.id))!.providerInvoiceId).toBe('2025001');
+        });
+
+        test('a draft invoice has no number to stamp yet', async () => {
+            const { organization, stripeAccount } = await init();
+            const { payment } = await createLegacyFeePayment(organization, stripeAccount);
+            await createInvoice(payment, null);
+
+            const { charge } = await upsertFee(organization, stripeAccount);
+
+            expect((await SettlementCharge.getByID(charge.id))!.providerInvoiceId).toBeNull();
+        });
+
+        test('stampInvoicedPayments stamps and clears the charges of an invoice', async () => {
+            const { organization, stripeAccount } = await init();
+            const { payment } = await createLegacyFeePayment(organization, stripeAccount);
+            const { charge, fee } = await upsertFee(organization, stripeAccount);
+            expect(fee.balanceItemId).not.toBeNull();
+
+            const invoice = await createInvoice(payment, '2025042');
+            await ApplicationFeeService.stampInvoicedPayments([payment], invoice);
+            expect((await SettlementCharge.getByID(charge.id))!.providerInvoiceId).toBe('2025042');
+
+            await ApplicationFeeService.stampInvoicedPayments([payment], null);
+            expect((await SettlementCharge.getByID(charge.id))!.providerInvoiceId).toBeNull();
+        });
+
+        test('an invoice without application fees is a no-op', async () => {
+            const { organization } = await init();
+            const payment = new Payment();
+            payment.organizationId = membershipOrganization.id;
+            payment.payingOrganizationId = organization.id;
+            payment.method = PaymentMethod.Transfer;
+            payment.status = PaymentStatus.Succeeded;
+            payment.price = 10_00;
+            await payment.save();
+
+            await expect(ApplicationFeeService.stampInvoicedPayments([payment], null)).resolves.toBeUndefined();
+        });
+    });
+
+    test('upserting the same fee twice keeps one row', async () => {
+        const { organization, stripeAccount } = await init();
+        const { fee } = await upsertFee(organization, stripeAccount);
+
+        const again = await ApplicationFeeService.upsertFee({
+            externalId: fee.externalId,
+            type: fee.type,
+            amount: 35_00,
+            organizationId: membershipOrganization.id,
+            payingOrganizationId: organization.id,
+            payingStripeAccountId: stripeAccount.id,
+            settlementChargeId: fee.settlementChargeId,
+            occurredAt,
+        });
+
+        expect(again.id).toBe(fee.id);
+        expect(again.amount).toBe(35_00);
+        expect(await ApplicationFee.select().where('externalId', fee.externalId).count()).toBe(1);
+    });
+});

package/src/services/ApplicationFeeService.ts

@@ -0,0 +1,283 @@
+import { BalanceItem, BalanceItemPayment, Invoice, Payment } from '@stamhoofd/models';
+import { ApplicationFee } from '@stamhoofd/models/models/ApplicationFee.js';
+import { SQL } from '@stamhoofd/sql';
+import { BalanceItemType, PaymentMethod, PaymentProvider, PaymentStatus } from '@stamhoofd/structures';
+import { ApplicationFeeType } from '@stamhoofd/structures/settlements/ApplicationFeeType.js';
+import { Formatter } from '@stamhoofd/utility';
+
+import { WebmasterReport } from '../helpers/WebmasterReport.js';
+import { SettlementService } from './SettlementService.js';
+
+/**
+ * Reference prefix of the fee payments the legacy invoicer created, one per (account, month).
+ * Only used to detect them: new fee payments use FEE_PAYMENT_REFERENCE_PREFIX.
+ */
+export const LEGACY_FEE_PAYMENT_REFERENCE_PREFIX = 'stripe-fees-';
+
+export const FEE_PAYMENT_REFERENCE_PREFIX = 'application-fees-';
+
+/**
+ * Fees read per batch when a whole invoice is stamped at once.
+ */
+const FEE_BATCH_SIZE = 500;
+
+export type ApplicationFeeData = {
+    externalId: string;
+    type: ApplicationFeeType;
+    amount: number;
+
+    /**
+     * The organization that charged (and receives) the fee.
+     */
+    organizationId: string;
+
+    payingOrganizationId: string;
+    payingStripeAccountId: string;
+    payingPaymentId?: string | null;
+    settlementChargeId: string;
+    settlementId?: string | null;
+    occurredAt: Date;
+};
+
+/**
+ * All writes to application_fees go through this service (the settlement sweep's unlink excepted).
+ * Every write is an upsert on the unique (externalId, type) key, so re-running a sync can never
+ * duplicate rows.
+ *
+ * `balanceItemId === null` must always mean "not yet invoiced": upsertFee immediately links fees of
+ * months the legacy invoicer already billed (detected by the legacy payment's reference), so the
+ * invoicer only ever sees fees that still need billing.
+ */
+export class ApplicationFeeService {
+    /**
+     * Emails about a legacy month that can't be linked are sent once per (account, month) per
+     * process, not once per fee.
+     */
+    private static reportedLegacyProblems = new Set<string>();
+
+    /**
+     * The legacy fee payment of a (paying account, month), remembered per process: a month has one
+     * answer for all of its fees, and a sync walks thousands of them. Only found payments are
+     * remembered, so a fee stored before its payment existed still links on the next pass.
+     */
+    private static legacyFeePayments = new Map<string, Payment[]>();
+
+    /**
+     * For tests only.
+     */
+    static resetWarnings() {
+        this.reportedLegacyProblems.clear();
+        this.legacyFeePayments.clear();
+    }
+
+    /**
+     * Upsert on (externalId, type). Fields that are undefined keep their stored value, so the fee
+     * walk (which doesn't know our platform payout yet) can't unlink a fee the payout sync attached
+     * earlier. balanceItemId is only written through markInvoiced or the inline legacy detection.
+     */
+    static async upsertFee(data: ApplicationFeeData): Promise<ApplicationFee> {
+        const fee = await ApplicationFee.select()
+            .where('externalId', data.externalId)
+            .where('type', data.type)
+            .first(false) ?? new ApplicationFee();
+
+        fee.externalId = data.externalId;
+        fee.type = data.type;
+        fee.amount = data.amount;
+        fee.organizationId = data.organizationId;
+        fee.payingOrganizationId = data.payingOrganizationId;
+        fee.payingStripeAccountId = data.payingStripeAccountId;
+        fee.settlementChargeId = data.settlementChargeId;
+        fee.occurredAt = data.occurredAt;
+
+        if (data.settlementId !== undefined) {
+            fee.settlementId = data.settlementId;
+        }
+        if (data.payingPaymentId !== undefined) {
+            fee.payingPaymentId = data.payingPaymentId;
+        }
+        await fee.save();
+
+        if (fee.balanceItemId === null) {
+            await this.linkLegacyInvoicedFee(fee);
+        }
+
+        return fee;
+    }
+
+    /**
+     * Records which balance item bills the fee to the payer. Pass the fee payment when the caller
+     * already has it, or an explicit null when the balance item is known to have no payment yet
+     * (the invoicer stamps before creating the payment): both skip resolving it again.
+     */
+    static async markInvoiced(fee: ApplicationFee, balanceItemId: string, { payment }: { payment?: Payment | null } = {}) {
+        fee.balanceItemId = balanceItemId;
+        await fee.save();
+        if (payment === null) {
+            return;
+        }
+        await this.stampProviderInvoiceId(fee, { payment });
+    }
+
+    /**
+     * The fee payments the legacy invoicer created for a (payer, month), matched on their
+     * reference. The OR NULL clause covers legacy payments created without a stripeAccountId.
+     */
+    static async findLegacyFeePayments({ organizationId, payingOrganizationId, payingStripeAccountId, periodStart }: {
+        organizationId: string;
+        payingOrganizationId: string;
+        payingStripeAccountId: string;
+        periodStart: Date;
+    }): Promise<Payment[]> {
+        const reference = LEGACY_FEE_PAYMENT_REFERENCE_PREFIX + Formatter.dateIso(periodStart);
+        const key = organizationId + ':' + payingStripeAccountId + ':' + reference;
+
+        const cached = this.legacyFeePayments.get(key);
+        if (cached && cached.length > 0) {
+            return cached;
+        }
+
+        const payments = await Payment.select()
+            .where('organizationId', organizationId)
+            .where('payingOrganizationId', payingOrganizationId)
+            .where(
+                SQL.where('stripeAccountId', payingStripeAccountId)
+                    .or('stripeAccountId', null),
+            )
+            .where('reference', reference)
+            .where('method', PaymentMethod.AccountDeductions)
+            .where('provider', PaymentProvider.Stripe)
+            .where('status', PaymentStatus.Succeeded)
+            .fetch();
+
+        this.legacyFeePayments.set(key, payments);
+        return payments;
+    }
+
+    /**
+     * Links a fee the legacy invoicer already billed to that month's balance item, the moment the
+     * fee row is created: service fees to the month's ServiceFee item, transfer fees to the
+     * TransferFee item. A month without a legacy payment is simply not invoiced yet. A legacy month
+     * that can't be linked stays unlinked and is emailed: the invoicer's legacy guard prevents it
+     * from being billed twice.
+     */
+    private static async linkLegacyInvoicedFee(fee: ApplicationFee) {
+        if (!fee.payingStripeAccountId) {
+            return;
+        }
+
+        const periodStart = SettlementService.getPeriodStart(fee.occurredAt);
+        const payments = await this.findLegacyFeePayments({
+            organizationId: fee.organizationId,
+            payingOrganizationId: fee.payingOrganizationId,
+            payingStripeAccountId: fee.payingStripeAccountId,
+            periodStart,
+        });
+
+        if (payments.length === 0) {
+            return;
+        }
+
+        const problem = (reason: string) => {
+            const key = fee.payingStripeAccountId + ':' + SettlementService.getPeriodKey(periodStart);
+            if (this.reportedLegacyProblems.has(key)) {
+                return;
+            }
+            this.reportedLegacyProblems.add(key);
+            console.error('Legacy fee linking failed for ' + key + ': ' + reason);
+            WebmasterReport.report(
+                'Applicatiekosten van account ' + fee.payingStripeAccountId + ' voor ' + SettlementService.getPeriodKey(periodStart) + ' koppelen aan oude facturatie mislukt',
+                'Ze worden niet opnieuw aangerekend. ' + reason,
+            );
+        };
+
+        if (payments.length > 1) {
+            problem('Found ' + payments.length + ' legacy fee payments for the same month');
+            return;
+        }
+
+        const payment = payments[0];
+        const balanceItem = await this.findFeeBalanceItem(payment, fee.type);
+        if (!balanceItem) {
+            problem('The legacy fee payment ' + payment.id + ' has no ' + fee.type + ' balance item');
+            return;
+        }
+
+        await this.markInvoiced(fee, balanceItem.id, { payment });
+    }
+
+    private static async findFeeBalanceItem(payment: Payment, type: ApplicationFeeType): Promise<BalanceItem | null> {
+        const balanceItemPayments = await BalanceItemPayment.select()
+            .where('paymentId', payment.id)
+            .fetch();
+        if (balanceItemPayments.length === 0) {
+            return null;
+        }
+        const balanceItems = await BalanceItem.select()
+            .where('id', balanceItemPayments.map(b => b.balanceItemId))
+            .fetch();
+        const balanceItemType = type === ApplicationFeeType.Service ? BalanceItemType.ServiceFee : BalanceItemType.TransferFee;
+        return balanceItems.find(item => item.type === balanceItemType) ?? null;
+    }
+
+    /**
+     * Stamps the Stamhoofd invoice number that bills this fee on the payer's deduction charge, so
+     * the payer can link the cost in their settlement export to our invoice. No invoice (yet) means
+     * no stamp: stampInvoicedPayments runs when the invoice is created later.
+     */
+    private static async stampProviderInvoiceId(fee: ApplicationFee, { payment }: { payment?: Payment } = {}) {
+        if (!payment) {
+            if (!fee.balanceItemId) {
+                return;
+            }
+            const balanceItemPayment = await BalanceItemPayment.select()
+                .where('balanceItemId', fee.balanceItemId)
+                .first(false);
+            if (!balanceItemPayment) {
+                return;
+            }
+            payment = await Payment.getByID(balanceItemPayment.paymentId) ?? undefined;
+        }
+
+        if (!payment?.invoiceId) {
+            return;
+        }
+
+        const invoice = await Invoice.getByID(payment.invoiceId);
+        if (!invoice?.number) {
+            return;
+        }
+
+        await SettlementService.setChargeProviderInvoiceId(fee.settlementChargeId, invoice.number);
+    }
+
+    /**
+     * Called when an invoice for these payments is created (with the invoice) or deleted (with
+     * null): stamps or clears the invoice number on the deduction charges of every application fee
+     * the payments billed. A no-op for invoices without fee payments.
+     */
+    static async stampInvoicedPayments(payments: Payment[], invoice: Invoice | null) {
+        if (payments.length === 0) {
+            return;
+        }
+
+        const balanceItemPayments = await BalanceItemPayment.select()
+            .where('paymentId', payments.map(p => p.id))
+            .fetch();
+        if (balanceItemPayments.length === 0) {
+            return;
+        }
+
+        // An invoice can bill a whole month of fees, so the charges are collected and stamped in
+        // batches instead of one query pair per fee
+        const settlementChargeIds: string[] = [];
+        for await (const fees of ApplicationFee.select()
+            .where('balanceItemId', balanceItemPayments.map(b => b.balanceItemId))
+            .limit(FEE_BATCH_SIZE)
+            .allBatched()) {
+            settlementChargeIds.push(...fees.map(fee => fee.settlementChargeId));
+        }
+
+        await SettlementService.setChargeProviderInvoiceIds(settlementChargeIds, invoice?.number ?? null);
+    }
+}

package/src/services/InvoiceService.ts

@@ -7,6 +7,8 @@ import type { Invoice as InvoiceStruct } from '@stamhoofd/structures';
 import { EmailTemplateType, PaymentStatus, Recipient, Replacement } from '@stamhoofd/structures';
 import { Formatter } from '@stamhoofd/utility';
 import { ViesHelper } from '../helpers/ViesHelper.js';
+import { WebmasterReport } from '../helpers/WebmasterReport.js';
+import { ApplicationFeeService } from './ApplicationFeeService.js';
 import { BalanceItemService } from './BalanceItemService.js';
 import { InvoicePdfService } from './InvoicePdfService.js';
 import { InvoiceXMlService } from './InvoiceXMLService.js';
@@ -301,6 +303,16 @@ export class InvoiceService {
             throw e;
         }
 
+        try {
+            // The deduction charges of application fees billed by these payments now link to a
+            // numbered invoice. Only after the try/catch: a stamping error must never delete an
+            // invoice that was already sent
+            await ApplicationFeeService.stampInvoicedPayments(payments, model);
+        } catch (e) {
+            console.error('Failed to stamp application fee charges for invoice ' + model.id, e);
+            WebmasterReport.report('Factuurnummer op applicatiekosten zetten mislukt voor factuur ' + (model.number ?? model.id) + ' (de factuur zelf is wel verstuurd)', e);
+        }
+
         return model;
     }
 
@@ -337,14 +349,19 @@ export class InvoiceService {
      * After deletion the invoiced cache of the affected balance items is recalculated.
      */
     static async delete(invoice: Invoice) {
-        // Collect the affected balance items before deleting, because the invoiced balance items are cascade deleted.
+        // Collect the affected balance items and payments before deleting, because the invoiced
+        // balance items are cascade deleted and the payments' invoiceId is reset
         const { invoicedBalanceItems } = await Invoice.loadBalanceItems([invoice]);
         const balanceItemIds = Formatter.uniqueArray(invoicedBalanceItems.map(i => i.balanceItemId));
+        const payments = await Payment.select().where('invoiceId', invoice.id).fetch();
 
         await invoice.delete();
 
         // Recalculate the invoiced amount cache of the balance items that were invoiced by this invoice.
         await BalanceItemService.updateInvoiced(balanceItemIds);
+
+        // The invoice number stamped on application fee deduction charges no longer exists
+        await ApplicationFeeService.stampInvoicedPayments(payments, null);
     }
 
     private static shouldForwardInvoice(invoice: Invoice, organization: Organization) {