@stamhoofd/backend 2.91.0 → 2.93.0

package/src/seeds/1755532883-update-email-sender-ids.ts

@@ -0,0 +1,47 @@
+import { Migration } from '@simonbackx/simple-database';
+import { Email, Organization, Platform } from '@stamhoofd/models';
+
+export default new Migration(async () => {
+    if (STAMHOOFD.environment === 'test') {
+        console.log('skipped in tests');
+        return;
+    }
+
+    console.log('Start setting senderId of emails.');
+
+    const batchSize = 100;
+    let count = 0;
+    const platform = await Platform.getShared();
+
+    for await (const email of Email.select()
+        .where('senderId', null).limit(batchSize).all()) {
+        if (!email.fromAddress) {
+            continue;
+        }
+        const organization = email.organizationId ? await Organization.getByID(email.organizationId) : null;
+        if (!organization) {
+            const sender = platform.privateConfig.emails.find(s => s.email === email.fromAddress);
+            if (sender) {
+                email.senderId = sender.id;
+                await email.save();
+                count += 1;
+            }
+            else {
+                console.warn(`No sender found for email ${email.fromAddress} in platform config`);
+            }
+        }
+        else {
+            const sender = organization.privateMeta.emails.find(s => s.email === email.fromAddress);
+            if (sender) {
+                email.senderId = sender.id;
+                await email.save();
+                count += 1;
+            }
+            else {
+                console.warn(`No sender found for email ${email.fromAddress} in organization ${organization.id}`);
+            }
+        }
+    }
+
+    console.log('Finished saving ' + count + ' emails.');
+});

package/src/seeds/1755790070-fill-email-recipient-errors.ts

@@ -0,0 +1,96 @@
+import { Migration } from '@simonbackx/simple-database';
+import { SimpleError, SimpleErrors } from '@simonbackx/simple-errors';
+import { EmailRecipient } from '@stamhoofd/models';
+
+function stringToError(message: string) {
+    if (message === 'Recipient has hard bounced') {
+        return new SimpleErrors(
+            new SimpleError({
+                code: 'email_skipped_hard_bounce',
+                message: 'The recipient has hard bounced. This means that the email address is invalid or no longer exists.',
+                human: $t(`af49a569-ce88-48d9-ac37-81e594e16c03`),
+            }),
+        );
+    }
+
+    if (message === 'Recipient has marked as spam') {
+        return new SimpleErrors(
+            new SimpleError({
+                code: 'email_skipped_spam',
+                message: 'Recipient has marked as spam',
+                human: $t(`e6523f56-397e-4127-8bf7-8396f6f25a62`),
+            }),
+        );
+    }
+
+    if (message === 'Recipient has unsubscribed') {
+        return new SimpleErrors(
+            new SimpleError({
+                code: 'email_skipped_unsubscribed',
+                message: 'Recipient has unsubscribed',
+                human: $t('De ontvanger heeft zich afgemeld voor e-mails'),
+            }),
+        );
+    }
+
+    if (message === 'Recipient has unsubscribed from marketing') {
+        return new SimpleErrors(
+            new SimpleError({
+                code: 'email_skipped_unsubscribed',
+                message: 'Recipient has unsubscribed from marketing',
+                human: $t('De ontvanger heeft zich afgemeld voor e-mails'),
+            }),
+        );
+    }
+
+    if (message === 'All recipients are filtered due to hard bounce or spam') {
+        return new SimpleErrors(
+            new SimpleError({
+                code: 'all_filtered',
+                message: 'All recipients are filtered due to hard bounce or spam',
+                human: $t('Deze ontvanger komt voor op de gedeelde bounce of spamlijst. De ontvanger was eerder permanent onbereikbaar of heeft eerder een e-mail als spam gemarkeerd.'),
+            }),
+        );
+    }
+
+    if (message === 'Invalid email address') {
+        return new SimpleErrors(
+            new SimpleError({
+                code: 'invalid_email_address',
+                message: 'Invalid email address',
+                human: $t(`cbbff442-758c-4f76-b8c2-26bb176fefcc`),
+            }),
+        );
+    }
+
+    return new SimpleErrors(
+        new SimpleError({
+            code: 'unknown_error',
+            message: message,
+        }),
+    );
+}
+
+export default new Migration(async () => {
+    if (STAMHOOFD.environment === 'test') {
+        console.log('skipped in tests');
+        return;
+    }
+
+    console.log('Start setting failError object on email recipients.');
+
+    const batchSize = 100;
+    let count = 0;
+
+    for await (const r of EmailRecipient.select()
+        .whereNot('failErrorMessage', null).limit(batchSize).all()) {
+        if (!r.failErrorMessage) {
+            continue;
+        }
+        r.failError = stringToError(r.failErrorMessage);
+        await r.save();
+        count++;
+    }
+
+    console.log('Finished saving ' + count + ' recipients with error object.');
+});

package/src/seeds/1755876819-remove-duplicate-members.ts

@@ -0,0 +1,145 @@
+import { Migration, SQLResultNamespacedRow } from '@simonbackx/simple-database';
+import { Member, mergeTwoMembers } from '@stamhoofd/models';
+import { SQL, SQLSelect } from '@stamhoofd/sql';
+import { PatchOrganizationMembersEndpoint } from '../endpoints/global/members/PatchOrganizationMembersEndpoint';
+
+type MergeType = {
+    a: {
+        id: string;
+        firstName: string;
+        lastName: string;
+        createdAt: Date;
+    };
+    b: {
+        id: string;
+        createdAt: Date;
+    };
+};
+
+export default new Migration(async () => {
+    if (STAMHOOFD.environment === 'test') {
+        console.log('skipped in tests');
+        return;
+    }
+
+    const q = new SQLSelect(
+        (row: SQLResultNamespacedRow): MergeType => {
+            return {
+                a: {
+                    id: row['a'].id as string,
+                    firstName: row['a'].firstName as string,
+                    lastName: row['a'].lastName as string,
+                    createdAt: row['a'].createdAt as Date,
+                },
+                b: {
+                    id: row['b'].id as string,
+                    createdAt: row['b'].createdAt as Date,
+                },
+            };
+        },
+        SQL.wildcard('a'),
+        SQL.wildcard('b'),
+    )
+        .from(Member.table, 'a')
+        .join(
+            SQL.join(Member.table, 'b')
+                .where(
+                    SQL.column('b', 'id'),
+                    '!=',
+                    SQL.column('a', 'id'),
+                )
+                .andWhere(
+                    SQL.column('b', 'firstName'),
+                    SQL.column('a', 'firstName'),
+                )
+                .andWhere(
+                    SQL.column('b', 'lastName'),
+                    SQL.column('a', 'lastName'),
+                )
+                .andWhere(
+                    SQL.column('b', 'birthDay'),
+                    SQL.column('a', 'birthDay'),
+                ),
+        )
+        .where(
+            SQL.where(
+                SQL.column('a', 'createdAt'),
+                '<',
+                SQL.column('b', 'createdAt'),
+            ).or(
+                SQL.where(
+                    SQL.column('b', 'createdAt'),
+                    SQL.column('a', 'createdAt'),
+                ).and(
+                    SQL.column('a', 'id'),
+                    '<',
+                    SQL.column('b', 'id'),
+                ),
+            ),
+        )
+        .orderBy(
+            SQL.column('a', 'createdAt'),
+            'ASC',
+        );
+
+    if (STAMHOOFD.userMode === 'organization') {
+        q.where(
+            SQL.column('b', 'organizationId'),
+            SQL.column('organizationId'),
+        );
+    }
+
+    const duplicates = await q.fetch();
+
+    const deletedSet = new Set<string>();
+    const mergedIntoSet = new Set<string>();
+
+    for (const duplicate of duplicates) {
+        if (mergedIntoSet.has(duplicate.b.id)) {
+            console.log('Found chained duplicate in wrong order', duplicate.a.id, 'and', duplicate.b.id);
+            continue; // Already merged this one
+        }
+
+        if (deletedSet.has(duplicate.b.id)) {
+            continue; // Already deleted this one
+        }
+
+        if (deletedSet.has(duplicate.a.id)) {
+            console.log('Skipping duplicate', duplicate.a.id, 'because it was already deleted');
+            continue; // Already deleted this one
+        }
+
+        console.log(
+            'Found duplicate member',
+            duplicate.a.id,
+            duplicate.a.createdAt,
+            'and',
+            duplicate.b.id,
+            duplicate.b.createdAt,
+            'with name',
+            duplicate.a.firstName, duplicate.a.lastName);
+        deletedSet.add(duplicate.b.id);
+        mergedIntoSet.add(duplicate.a.id);
+
+        // Run the merge
+        const [memberA] = await Member.getBlobByIds(duplicate.a.id);
+        const [memberB] = await Member.getBlobByIds(duplicate.b.id);
+
+        if (memberA.details.name !== memberB.details.name) {
+            console.warn('Member names do not match', memberA.details.name, 'and', memberB.details.name);
+            continue; // Names do not match, cannot merge
+        }
+
+        if (memberA.details.birthDayFormatted === null || memberA.details.birthDayFormatted !== memberB.details.birthDayFormatted) {
+            console.warn('Member birthday do not match', memberA.details.birthDayFormatted, 'and', memberB.details.birthDayFormatted);
+            continue; // Names do not match, cannot merge
+        }
+
+        if (!PatchOrganizationMembersEndpoint.shouldCheckIfMemberIsDuplicate(memberA)) {
+            console.log('Skipping merge because not eligible for duplicate check');
+            continue;
+        }
+
+        await mergeTwoMembers(memberA, memberB);
+    }
+});

package/src/seeds/1756115432-remove-old-drafts.ts

@@ -0,0 +1,16 @@
+import { Migration } from '@simonbackx/simple-database';
+import { Email } from '@stamhoofd/models';
+import { EmailStatus } from '@stamhoofd/structures';
+
+export default new Migration(async () => {
+    if (STAMHOOFD.environment === 'test') {
+        console.log('skipped in tests');
+        return;
+    }
+
+    const { affectedRows } = await Email.delete()
+        .where('status', EmailStatus.Draft)
+        .where('createdAt', '<', new Date(Date.now() - 1000 * 60 * 60 * 24 * 30)) // older than 30 days
+        .delete();
+    console.log('Deleted ' + affectedRows + ' old draft emails.');
+});

package/src/seeds/1756115433-fill-email-recipient-organization-id.ts

@@ -0,0 +1,30 @@
+import { Migration } from '@simonbackx/simple-database';
+import { Email, EmailRecipient } from '@stamhoofd/models';
+import { SQL } from '@stamhoofd/sql';
+
+export default new Migration(async () => {
+    if (STAMHOOFD.environment === 'test') {
+        console.log('skipped in tests');
+        return;
+    }
+
+    console.log('Start setting organizationId object on email recipients.');
+
+    const batchSize = 100;
+    let count = 0;
+
+    for await (const email of Email.select().where('organizationId', '!=', null).limit(batchSize).all()) {
+        if (!email.organizationId) {
+            continue;
+        }
+
+        await SQL.update(EmailRecipient.table)
+            .set('organizationId', email.organizationId)
+            .where('emailId', email.id)
+            .update();
+
+        count++;
+    }
+
+    console.log('Finished saving email recipients of ' + count + ' emails with organization id.');
+});

package/src/services/uitpas/UitpasService.ts

@@ -1,6 +1,6 @@
 import { Model } from '@simonbackx/simple-database';
 import { Order, WebshopUitpasNumber } from '@stamhoofd/models';
-import { OrderStatus, Product, ProductPrice, UitpasClientCredentialsStatus, UitpasOrganizersResponse } from '@stamhoofd/structures';
+import { Cart, OrderStatus, Product, ProductPrice, UitpasClientCredentialsStatus, UitpasOrganizersResponse } from '@stamhoofd/structures';
 import { v4 as uuidv4 } from 'uuid';
 import { UitpasTokenRepository } from '../../helpers/UitpasTokenRepository';
 import { searchUitpasOrganizers } from './searchUitpasOrganizers';
@@ -13,6 +13,7 @@ import { RegisterTicketSaleRequest, RegisterTicketSaleResponse, registerTicketSa
 import { cancelTicketSales } from './cancelTicketSales';
 import { SimpleError } from '@simonbackx/simple-errors';
 import { QueueHandler } from '@stamhoofd/queues';
+import { Formatter } from '@stamhoofd/utility';
 
 type UitpasTicketSale = {
     basePrice: number;
@@ -39,7 +40,7 @@ type InsertUitpasNumber = {
     uitpasEventUrl: string | null; // null for non-official flow
 };
 
-function shouldReserveUitpasNumbers(status: OrderStatus): boolean {
+export function shouldReserveUitpasNumbers(status: OrderStatus): boolean {
     return status !== OrderStatus.Canceled && status !== OrderStatus.Deleted;
 }
 
@@ -356,4 +357,72 @@ export class UitpasService {
         // Clear the uitpas client credentials for the organization
         await UitpasTokenRepository.clearClientCredentialsFor(organizationId);
     }
+
+    static async areThereRegisteredTicketSales(webshopId: string): Promise<boolean> {
+        return await WebshopUitpasNumber.areThereRegisteredTicketSales(webshopId);
+    }
+
+    static async validateCart(organizationId: string, webshopId: string, cart: Cart, exisitingOrderId?: string): Promise<Cart> {
+        let access_token_org: string | null = null;
+        let access_token_platform: string | null = null;
+        for (const item of cart.items) {
+            if (item.uitpasNumbers.length === 0) {
+                continue;
+            }
+
+            // verify the UiTPAS numbers are not already used for this product
+            const hasBeenUsed = await WebshopUitpasNumber.areUitpasNumbersUsed(webshopId, item.product.id, item.uitpasNumbers.map(p => p.uitpasNumber), item.product.uitpasEvent?.url, exisitingOrderId);
+            if (hasBeenUsed) {
+                throw new SimpleError({
+                    code: 'uitpas_number_already_used',
+                    message: 'One or more uitpas numbers are already used',
+                    human: $t('f3daff19-a227-4e45-b19a-c770bd7a6687'),
+                    field: 'uitpasNumbers',
+                });
+            }
+
+            if (item.product.uitpasEvent) {
+                // official flow
+                const basePrice = item.product.prices.filter(price => price.id === item.productPrice.uitpasBaseProductPriceId)[0]?.price;
+                if (!basePrice) {
+                    throw new SimpleError({
+                        code: 'missing_uitpas_base_product_price',
+                        message: `Missing UiTPAS base product price`,
+                        human: $t(`3d08a166-11a7-4429-8ff7-84458bbe3e9a`),
+                    });
+                }
+
+                access_token_org = access_token_org ?? await UitpasTokenRepository.getAccessTokenFor(organizationId);
+                const verified = await getSocialTariffForUitpasNumbers(access_token_org, item.uitpasNumbers.map(p => p.uitpasNumber), basePrice, item.product.uitpasEvent.url);
+                if (verified.length < item.uitpasNumbers.length) {
+                    throw new SimpleError({
+                        code: 'uitpas_social_tariff_price_mismatch',
+                        message: 'UiTPAS wrong number of prices returned',
+                        human: $t('83c472b8-4bc5-4282-bbc9-1c6a2d382171'),
+                        field: 'uitpasNumbers',
+                    });
+                }
+                for (let i = 0; i < verified.length; i++) {
+                    if (item.uitpasNumbers[i].uitpasTariffId !== verified[i].uitpasTariffId) {
+                        // silently update
+                        item.uitpasNumbers[i].uitpasTariffId = verified[i].uitpasTariffId;
+                    }
+                    if (item.uitpasNumbers[i].price !== verified[i].price) {
+                        throw new SimpleError({
+                            code: 'uitpas_social_tariff_price_mismatch',
+                            message: 'UiTPAS social tariff have a different price',
+                            human: $t('9a0ad099-99e3-4341-beac-f14feb3fb9d1', { correctPrice: Formatter.price(verified[i].price), orderPrice: Formatter.price(item.uitpasNumbers[i].price) }),
+                            field: 'uitpasNumbers.' + i.toString(),
+                        });
+                    }
+                }
+            }
+            else {
+                // non-official flow
+                access_token_platform = access_token_platform ?? await UitpasTokenRepository.getAccessTokenFor();
+                await checkUitpasNumbers(access_token_platform, item.uitpasNumbers.map(p => p.uitpasNumber));
+            }
+        }
+        return cart;
+    }
 };

package/src/services/uitpas/checkUitpasNumbers.ts

@@ -64,6 +64,7 @@ async function checkUitpasNumber(access_token: string, uitpasNumber: string) {
             ),
         });
     }
+
     const baseUrl = 'https://api-test.uitpas.be'; // TO DO: Use the URL from environment variables
 
     const url = `${baseUrl}/passes/${uitpasNumber}`;

package/src/sql-filters/email-recipients.ts

@@ -0,0 +1,59 @@
+import { baseSQLFilterCompilers, createColumnFilter, SQL, SQLConcat, SQLFilterDefinitions, SQLScalar, SQLValueType } from '@stamhoofd/sql';
+
+export const emailRecipientsFilterCompilers: SQLFilterDefinitions = {
+    ...baseSQLFilterCompilers,
+    id: createColumnFilter({
+        expression: SQL.column('id'),
+        type: SQLValueType.String,
+        nullable: false,
+    }),
+    email: createColumnFilter({
+        expression: SQL.column('email'),
+        type: SQLValueType.String,
+        nullable: true,
+    }),
+    name: createColumnFilter({
+        expression: new SQLConcat(
+            SQL.column('firstName'),
+            new SQLScalar(' '),
+            SQL.column('lastName'),
+        ),
+        type: SQLValueType.String,
+        nullable: true,
+    }),
+    organizationId: createColumnFilter({
+        expression: SQL.column('organizationId'),
+        type: SQLValueType.String,
+        nullable: true,
+    }),
+    emailId: createColumnFilter({
+        expression: SQL.column('emailId'),
+        type: SQLValueType.String,
+        nullable: false,
+    }),
+    sentAt: createColumnFilter({
+        expression: SQL.column('sentAt'),
+        type: SQLValueType.Datetime,
+        nullable: true,
+    }),
+    failError: createColumnFilter({
+        expression: SQL.column('failError'),
+        type: SQLValueType.JSONObject,
+        nullable: true,
+    }),
+    spamComplaintError: createColumnFilter({
+        expression: SQL.column('spamComplaintError'),
+        type: SQLValueType.String,
+        nullable: true,
+    }),
+    softBounceError: createColumnFilter({
+        expression: SQL.column('softBounceError'),
+        type: SQLValueType.String,
+        nullable: true,
+    }),
+    hardBounceError: createColumnFilter({
+        expression: SQL.column('hardBounceError'),
+        type: SQLValueType.String,
+        nullable: true,
+    }),
+};

package/src/sql-filters/emails.ts

@@ -0,0 +1,95 @@
+import { baseSQLFilterCompilers, createColumnFilter, SQL, SQLFilterDefinitions, SQLValueType } from '@stamhoofd/sql';
+
+export const emailFilterCompilers: SQLFilterDefinitions = {
+    ...baseSQLFilterCompilers,
+    id: createColumnFilter({
+        expression: SQL.column('id'),
+        type: SQLValueType.String,
+        nullable: false,
+    }),
+    organizationId: createColumnFilter({
+        expression: SQL.column('organizationId'),
+        type: SQLValueType.String,
+        nullable: true,
+    }),
+    userId: createColumnFilter({
+        expression: SQL.column('userId'),
+        type: SQLValueType.String,
+        nullable: true,
+    }),
+    emailType: createColumnFilter({
+        expression: SQL.column('emailType'),
+        type: SQLValueType.String,
+        nullable: true,
+    }),
+    subject: createColumnFilter({
+        expression: SQL.column('subject'),
+        type: SQLValueType.String,
+        nullable: true,
+    }),
+    fromAddress: createColumnFilter({
+        expression: SQL.column('fromAddress'),
+        type: SQLValueType.String,
+        nullable: true,
+    }),
+    text: createColumnFilter({
+        expression: SQL.column('text'),
+        type: SQLValueType.String,
+        nullable: true,
+    }),
+    status: createColumnFilter({
+        expression: SQL.column('status'),
+        type: SQLValueType.String,
+        nullable: false,
+    }),
+    recipientStatus: createColumnFilter({
+        expression: SQL.column('recipientStatus'),
+        type: SQLValueType.String,
+        nullable: false,
+    }),
+    emailRecipientsCount: createColumnFilter({
+        expression: SQL.column('emailRecipientsCount'),
+        type: SQLValueType.Number,
+        nullable: true,
+    }),
+    failedCount: createColumnFilter({
+        expression: SQL.column('failedCount'),
+        type: SQLValueType.Number,
+        nullable: false,
+    }),
+    softFailedCount: createColumnFilter({
+        expression: SQL.column('softFailedCount'),
+        type: SQLValueType.Number,
+        nullable: false,
+    }),
+    hardBouncesCount: createColumnFilter({
+        expression: SQL.column('hardBouncesCount'),
+        type: SQLValueType.Number,
+        nullable: false,
+    }),
+    softBouncesCount: createColumnFilter({
+        expression: SQL.column('softBouncesCount'),
+        type: SQLValueType.Number,
+        nullable: false,
+    }),
+    spamComplaintsCount: createColumnFilter({
+        expression: SQL.column('spamComplaintsCount'),
+        type: SQLValueType.Number,
+        nullable: false,
+    }),
+    createdAt: createColumnFilter({
+        expression: SQL.column('createdAt'),
+        type: SQLValueType.Datetime,
+        nullable: true,
+    }),
+    sentAt: createColumnFilter({
+        expression: SQL.column('sentAt'),
+        type: SQLValueType.Datetime,
+        nullable: true,
+    }),
+    senderId: createColumnFilter({
+        expression: SQL.column('senderId'),
+        type: SQLValueType.String,
+        nullable: true,
+    }),
+};

package/src/sql-filters/members.ts

@@ -1,5 +1,5 @@
 import { SimpleError } from '@simonbackx/simple-errors';
-import { Member } from '@stamhoofd/models';
+import { Email, Member } from '@stamhoofd/models';
 import { baseSQLFilterCompilers, createColumnFilter, createExistsFilter, SQL, SQLAge, SQLCast, SQLConcat, SQLFilterDefinitions, SQLValueType, SQLScalar, createWildcardColumnFilter, SQLJsonExtract } from '@stamhoofd/sql';
 import { AccessRight } from '@stamhoofd/structures';
 import { Context } from '../helpers/Context';
@@ -413,6 +413,47 @@ export const memberFilterCompilers: SQLFilterDefinitions = {
             ),
         organizationFilterCompilers,
     ),
+    'emails': createExistsFilter(
+        SQL.select()
+            .from(
+                SQL.table('email_recipients'),
+            ).where(
+                SQL.column('memberId'),
+                SQL.column('members', 'id'),
+            ),
+        {
+            ...baseSQLFilterCompilers,
+            id: createColumnFilter({
+                expression: SQL.column('emailId'),
+                type: SQLValueType.String,
+                nullable: false,
+                checkPermission: async (filter) => {
+                    if (typeof filter !== 'string') {
+                        throw new SimpleError({
+                            code: 'invalid_filter',
+                            message: 'This filter structure is not supported here.',
+                        });
+                    }
+                    const id = filter;
+                    const email = await Email.getByID(id);
+                    if (!email) {
+                        throw new SimpleError({
+                            code: 'not_found',
+                            message: 'This email does not exist.',
+                            human: $t('Deze e-mail bestaat niet (meer)'),
+                            statusCode: 404,
+                        });
+                    }
+                    if (!await Context.auth.canAccessEmail(email)) {
+                        throw Context.auth.error({
+                            message: 'No permissions to access this email.',
+                            human: $t('Je hebt niet voldoende toegangsrechten om te filteren op deze e-mail'),
+                        });
+                    }
+                },
+            }),
+        },
+    ),
     'details': {
         ...baseSQLFilterCompilers,
         recordAnswers: createWildcardColumnFilter(

package/src/sql-filters/registration-periods.ts

@@ -7,6 +7,11 @@ export const registrationPeriodFilterCompilers: SQLFilterDefinitions = {
         type: SQLValueType.String,
         nullable: false,
     }),
+    organizationId: createColumnFilter({
+        expression: SQL.column('organizationId'),
+        type: SQLValueType.String,
+        nullable: false,
+    }),
     startDate: createColumnFilter({
         expression: SQL.column('startDate'),
         type: SQLValueType.Datetime,