@stamhoofd/backend 2.120.6 → 2.122.0

package/src/migrations/1780933598-document-groups.ts

@@ -0,0 +1,88 @@
+import { Migration } from '@simonbackx/simple-database';
+import { DocumentTemplate, Group, RegistrationPeriod, V1GroupMigrationData } from '@stamhoofd/models';
+import { GroupType, NamedObject } from '@stamhoofd/structures';
+import { Formatter } from '@stamhoofd/utility';
+import { SeedTools } from '../helpers/SeedTools.js';
+
+export default new Migration(async () => {
+    if (STAMHOOFD.environment === 'test') {
+        console.log('skipped in tests');
+        return;
+    }
+
+    if (STAMHOOFD.platformName.toLowerCase() !== 'stamhoofd') {
+        console.log('skipped for platform (only runs for Stamhoofd): ' + STAMHOOFD.platformName);
+        return;
+    }
+
+    let totalGroups = 0;
+    let totalNotFound = 0;
+
+    await SeedTools.loop({
+        batchSize: 100,
+        query: DocumentTemplate.select(),
+        useTransactionPerBatch: true,
+        action: async (template: DocumentTemplate) => {
+            const groups = template.privateSettings.groups;
+
+            for (const group of groups) {
+                totalGroups += 1;
+                const newGroup = await getNewGroup(group.group.id, group.cycle, template.id);
+
+                // it is possible that the group was deleted
+                if (newGroup) {
+                    const period = await RegistrationPeriod.getByID(newGroup.periodId);
+
+                    group.group = NamedObject.create({
+                        id: newGroup.id,
+                        name: newGroup.settings.name.toString(),
+                        description: newGroup.type === GroupType.Membership && period ? getPeriodName(period) : (Formatter.dateRange(newGroup.settings.startDate, newGroup.settings.endDate)),
+                    });
+
+                    await template.save({ forceSave: true });
+                } else {
+                    totalNotFound += 1;
+                    console.log(`Group not found, id: ${group.group.id}, cycle: ${group.cycle}`);
+                }
+            }
+        },
+    });
+
+    console.log('Finished updating groups on document templates:');
+    // check if reasonable amount of groups had been found
+    console.log(`totalGroups: ${totalGroups}, totalNotFound: ${totalNotFound} (${totalNotFound / totalGroups * 100}%)`);
+});
+
+// see get name on period structure (same logic)
+function getPeriodName(period: RegistrationPeriod) {
+    if (period.customName) {
+        return period.customName;
+    }
+
+    if (Formatter.year(period.endDate) === Formatter.year(period.startDate)) {
+        return $t(`%7Z`) + ' ' + Formatter.year(period.startDate);
+    }
+
+    return $t(`%7Z`) + ' ' + Formatter.year(period.startDate) + ' - ' + Formatter.year(period.endDate);
+}
+
+async function getNewGroup(oldGroupId: string, cycle: number, templateId: string) {
+    try {
+        const migrationData = await V1GroupMigrationData.select()
+            .where('oldGroupId', oldGroupId)
+            .where('oldCycle', cycle)
+            // the group could have been deleted
+            .first(false);
+
+        if (!migrationData) {
+            return null;
+        }
+
+        const newGroupId: string = migrationData.newGroupId;
+
+        return await Group.getByID(newGroupId);
+    } catch (e) {
+        console.error(`oldGroupId: ${oldGroupId}, cycle: ${cycle}, templateId: ${templateId}`);
+        throw e;
+    }
+}

package/src/migrations/1781692342-balance-item-relations.ts

@@ -0,0 +1,13 @@
+import { Migration } from '@simonbackx/simple-database';
+
+export default new Migration(async () => {
+    if (STAMHOOFD.environment === 'test') {
+        // The backfill is covered directly by BalanceItemRelationsBackfiller.test.ts
+        console.log('skipped in tests');
+        return;
+    }
+
+    // Imported dynamically so the migration loader does not need to resolve the helper at link time.
+    const { backfillBalanceItemRelations } = await import('../helpers/BalanceItemRelationsBackfiller.js');
+    await backfillBalanceItemRelations();
+});

package/src/migrations/1781857866-platform-admins.ts

@@ -0,0 +1,103 @@
+import { column, Migration } from '@simonbackx/simple-database';
+import { User } from '@stamhoofd/models';
+import { QueryableModel } from '@stamhoofd/sql';
+import { PermissionLevel, Permissions, UserPermissions } from '@stamhoofd/structures';
+
+export class Admin extends QueryableModel {
+    static table = 'admins';
+
+    // Columns
+    @column({
+        primary: true, type: 'string',
+    })
+    id!: string;
+
+    @column({ type: 'string' })
+    email: string;
+
+    @column({ type: 'string' })
+    password?: string;
+
+    @column({
+        type: 'datetime', beforeSave(old?: Date) {
+            if (old !== undefined) {
+                return old;
+            }
+            const date = new Date();
+            date.setMilliseconds(0);
+            return date;
+        },
+    })
+    createdAt: Date;
+
+    @column({
+        type: 'datetime', beforeSave() {
+            const date = new Date();
+            date.setMilliseconds(0);
+            return date;
+        },
+        skipUpdate: true,
+    })
+    updatedAt: Date;
+}
+
+export default new Migration(async () => {
+    if (STAMHOOFD.environment === 'test') {
+        console.log('skipped in tests');
+        return;
+    }
+
+    if (STAMHOOFD.platformName !== 'stamhoofd') {
+        return;
+    }
+
+    if (STAMHOOFD.userMode !== 'organization') {
+        return;
+    }
+
+    try {
+        await Admin.select().first(false);
+    } catch (e) {
+        if (e.code === 'ER_NO_SUCH_TABLE') {
+            console.error('Migration skipped: admins table does not exist');
+            return;
+        }
+        throw e;
+    }
+
+    for await (const admin of Admin.select().all()) {
+        if (!admin.password) {
+            continue;
+        }
+        const email = admin.email;
+
+        if (!email || !email.includes('@')) {
+            continue;
+        }
+
+        console.log('Converting Platform Admin ', email);
+
+        // Delete all existing users with this email address
+        const deleted = await User.delete().where('email', email);
+        console.log('Deleted ' + deleted.affectedRows + ' users to migrate to new platform user');
+
+        const user = new User();
+        user.email = email;
+        user.password = admin.password;
+
+        user.organizationId = null;
+        user.permissions = UserPermissions.create({
+            globalPermissions: Permissions.create({
+                level: PermissionLevel.Full,
+            }),
+        });
+
+        user.createdAt = admin.createdAt;
+        user.updatedAt = new Date();
+        user.verified = true;
+
+        await user.save();
+
+        console.log('Created Platform Admin ', email);
+    }
+});

package/src/migrations/1781857867-rm-old-platform-admins.ts

@@ -0,0 +1,25 @@
+import { Migration } from '@simonbackx/simple-database';
+import { User } from '@stamhoofd/models';
+
+export default new Migration(async () => {
+    if (STAMHOOFD.environment === 'test') {
+        console.log('skipped in tests');
+        return;
+    }
+
+    if (STAMHOOFD.platformName !== 'stamhoofd') {
+        return;
+    }
+
+    if (STAMHOOFD.userMode !== 'organization') {
+        return;
+    }
+
+    if (STAMHOOFD.environment !== 'production') {
+        return;
+    }
+
+    // Replaced with new global admins
+    await User.delete().where('email', 'hallo@stamhoofd.be');
+    await User.delete().where('email', 'hallo@stamhoofd.nl');
+});

package/src/seeds/0000000001-throw.ts

@@ -0,0 +1,5 @@
+import { Migration } from '@simonbackx/simple-database';
+
+export default new Migration(async () => {
+    // throw new Error('Stop here')
+});

package/src/seeds/1760702452-update-balance-price-paid.ts

@@ -0,0 +1,19 @@
+import { Migration } from '@simonbackx/simple-database';
+import { BalanceItem } from '@stamhoofd/models';
+
+export default new Migration(async () => {
+    if (STAMHOOFD.environment === 'test') {
+        console.log('skipped in tests');
+        return;
+    }
+
+    if (STAMHOOFD.platformName !== 'stamhoofd') {
+        return;
+    }
+
+    if (STAMHOOFD.userMode !== 'organization') {
+        return;
+    }
+
+    await BalanceItem.updatePricePaid('all');
+});

package/src/seeds/1760702453-update-balance-invoiced.ts

@@ -0,0 +1,19 @@
+import { Migration } from '@simonbackx/simple-database';
+import { BalanceItem } from '@stamhoofd/models';
+
+export default new Migration(async () => {
+    if (STAMHOOFD.environment === 'test') {
+        console.log('skipped in tests');
+        return;
+    }
+
+    if (STAMHOOFD.platformName !== 'stamhoofd') {
+        return;
+    }
+
+    if (STAMHOOFD.userMode !== 'organization') {
+        return;
+    }
+
+    await BalanceItem.updateInvoiced('all');
+});

package/src/seeds/1760702454-update-cached-outstanding-balance-from-items.ts

@@ -1,29 +1,66 @@
 import { Migration } from '@simonbackx/simple-database';
-import { logger } from '@simonbackx/simple-logging';
-import { BalanceItem } from '@stamhoofd/models';
-import { BalanceItemService } from '../services/BalanceItemService.js';
+import { CachedBalance, Member, Organization, Platform, Registration, User } from '@stamhoofd/models';
+import { SeedTools } from '../helpers/SeedTools.js';
 
 export default new Migration(async () => {
-    if (STAMHOOFD.environment == 'test') {
+    if (STAMHOOFD.environment === 'test') {
         console.log('skipped in tests');
         return;
     }
 
-    process.stdout.write('\n');
-    let c = 0;
+    if (STAMHOOFD.platformName !== 'stamhoofd') {
+        return;
+    }
+
+    if (STAMHOOFD.userMode !== 'organization') {
+        return;
+    }
+
+    // Loop all members
+    await SeedTools.loopBatched({
+        query: Organization.select(),
+        // useTransactionPerBatch: true,
+        batchSize: 100,
+        batchAction: async (organizations) => {
+            for (const organization of organizations) {
+                for await (const batch of Member.select('id').where('organizationId', organization.id).limit(1000).allBatched()) {
+                    await CachedBalance.updateForMembers(organization.id, batch.map(m => m.id));
+                }
 
-    await logger.setContext({ tags: ['silent-seed', 'seed'] }, async () => {
-        for await (const items of BalanceItem.select().limit(1000).allBatched()) {
-            await BalanceItemService.updatePaidAndPending(items);
+                for await (const batch of User.select('id').where('organizationId', organization.id).limit(1000).allBatched()) {
+                    await CachedBalance.updateForUsers(organization.id, batch.map(m => m.id));
+                }
 
-            c += items.length;
-            process.stdout.write('.');
-        }
+                for await (const batch of Registration.select('id').where('organizationId', organization.id).limit(1000).allBatched()) {
+                    await CachedBalance.updateForRegistrations(organization.id, batch.map(m => m.id));
+                }
+            }
+        },
     });
 
-    await BalanceItemService.flushAll();
+    const membershipOrganizationId = (await Platform.getShared()).membershipOrganizationId;
+    if (membershipOrganizationId) {
+        const membershipOrganization = await Organization.getByID(membershipOrganizationId, true);
+        await SeedTools.loopBatched({
+            query: Organization.select('id').where('id', '!=', membershipOrganization.id),
+            // useTransactionPerBatch: true,
+            batchSize: 1000,
+            batchAction: async (organizations) => {
+                await CachedBalance.updateForOrganizations(membershipOrganization.id, organizations.map(m => m.id));
+            },
+        });
+    }
+
+    // await logger.setContext({ tags: ['silent-seed', 'seed'] }, async () => {
+    //    for await (const items of BalanceItem.select().limit(10_000).allBatched()) {
+    //        await BalanceItemService.updatePaidAndPending(items);
+    //        await BalanceItemService.flushAll();
+    //
+    //        progressLogger.update(items.length);
+    //    }
+    // });
 
-    console.log('Updated outstanding balance for ' + c + ' items');
+    // console.log('Updated outstanding balance for ' + progressLogger.total + ' items');
 
     // Do something here
     return Promise.resolve();

package/src/seeds/1773754928-force-save-members.ts

@@ -1,15 +1,48 @@
 import { Migration } from '@simonbackx/simple-database';
-import { Member, Registration } from '@stamhoofd/models';
+import { Member } from '@stamhoofd/models';
+import { SeedTools } from '../helpers/SeedTools.js';
+import { QueryableModel } from '@stamhoofd/sql';
 
 export default new Migration(async () => {
-    process.stdout.write('\n');
-    for await (const member of Member.select().all()) {
-        member.forceSaveProperty('details');
-        await member.save();
+    if (STAMHOOFD.environment === 'test') {
+        // The customer determination is covered directly by PaymentCustomerResolver.test.ts
+        console.log('skipped in tests');
+        return;
     }
 
-    for await (const registration of Registration.select().all()) {
-        registration.forceSaveProperty('groupPrice');
-        await registration.save();
+    if (STAMHOOFD.platformName !== 'stamhoofd') {
+        // In v1 only the Stamhoofd platform stored payments without a customer.
+        return;
     }
+
+    process.stdout.write('\n');
+
+    await migrateMembers();
 });
+
+async function migrateMembers() {
+    console.log('Start updating members.');
+
+    let realUpdate = 0;
+    const result = await SeedTools.loop({
+        query: Member.select().where('updatedAt', '<', new Date(Date.now() - 1_000 * 60 * 60 * 5)),
+        batchSize: 200,
+        useTransactionPerBatch: true,
+        action: async (member: Member) => {
+            member.forceSaveProperty('details');
+
+            if (await member.save({
+                skipMarkSaved: true,
+                skipSendEvents: true,
+            })) {
+                realUpdate += 1;
+            }
+
+            if (QueryableModel.shutdownMigrations) {
+                throw new Error('Stopping migration gracefully');
+            }
+        },
+    });
+
+    console.log('Updated ' + realUpdate + ' members out of ' + result.total + ' looped members');
+}

package/src/seeds/1780665427-schedule-stock-updates.ts

@@ -0,0 +1,45 @@
+import { Migration } from '@simonbackx/simple-database';
+import { Registration } from '@stamhoofd/models';
+import { SeedTools } from '../helpers/SeedTools.js';
+import { RegistrationService } from '../services/RegistrationService.js';
+import { QueryableModel } from '@stamhoofd/sql';
+import { QueueHandler } from '@stamhoofd/queues';
+
+export default new Migration(async () => {
+    if (STAMHOOFD.environment == 'test') {
+        console.log('skipped in tests');
+        return;
+    }
+
+    if (STAMHOOFD.userMode === 'platform') {
+        console.log('skipped seed schedule-stock-updates because usermode platform');
+        return;
+    }
+
+    const result = await SeedTools.loopBatched({
+        query: Registration.select('id'),
+        batchSize: 2_000,
+        // dot not use transactions here: this breaks the stock behaviour
+        batchAction: async (registrations: Registration[]) => {
+            for (const registration of registrations) {
+                await RegistrationService.scheduleStockUpdateAsync(registration.id);
+
+                if (QueryableModel.shutdownMigrations) {
+                    break;
+                }
+            }
+
+            // Immediate tasks scheduled on group updates
+            await QueueHandler.awaitAll();
+
+            // Recursive tasks
+            await QueueHandler.awaitAll();
+
+            if (QueryableModel.shutdownMigrations) {
+                throw new Error('Stopping migration gracefully');
+            }
+        },
+    });
+
+    console.log('Looped ' + result.total + ' registrations');
+});

package/src/seeds/1780665428-group-update-occupancy.ts

@@ -0,0 +1,31 @@
+import { Migration } from '@simonbackx/simple-database';
+import { Group } from '@stamhoofd/models';
+import { SeedTools } from '../helpers/SeedTools.js';
+
+export default new Migration(async () => {
+    if (STAMHOOFD.environment == 'test') {
+        console.log('skipped in tests');
+        return;
+    }
+
+    if (STAMHOOFD.userMode !== 'platform') {
+        console.log('skipped seed group-update-occupancy because usermode not platform');
+        return;
+    }
+
+    await SeedTools.loop<Group>({
+        query: Group.select(),
+        useTransactionPerBatch: true,
+        batchSize: 200,
+        action: async (group) => {
+            await group.updateOccupancy();
+            await group.save({
+                skipMarkSaved: true,
+                skipSendEvents: true,
+            });
+        },
+    });
+
+    // Do something here
+    return Promise.resolve();
+});

package/src/seeds/1780915001-fill-payment-customer.ts

@@ -0,0 +1,70 @@
+import { Migration } from '@simonbackx/simple-database';
+import { Member, Order, Organization, Payment, User } from '@stamhoofd/models';
+import { Formatter } from '@stamhoofd/utility';
+import { SeedTools } from '../helpers/SeedTools.js';
+import { determinePaymentCustomer } from '../helpers/PaymentCustomerResolver.js';
+
+export default new Migration(async () => {
+    if (STAMHOOFD.environment === 'test') {
+        // The customer determination is covered directly by PaymentCustomerResolver.test.ts
+        console.log('skipped in tests');
+        return;
+    }
+
+    if (STAMHOOFD.platformName !== 'stamhoofd') {
+        // In v1 only the Stamhoofd platform stored payments without a customer.
+        return;
+    }
+
+    console.log('Start filling in payment customers.');
+
+    let filled = 0;
+
+    const result = await SeedTools.loopBatched({
+        query: Payment.select().where('customer', null),
+        batchSize: 1000,
+        batchAction: async (payments: Payment[]) => {
+            // Load all the related data needed to determine the customers in bulk.
+            const { balanceItemPayments, balanceItems } = await Payment.loadBalanceItems(payments);
+            const { payingOrganizations } = await Payment.loadPayingOrganizations(payments);
+
+            const memberIds = Formatter.uniqueArray(balanceItems.map(b => b.memberId).filter((id): id is string => id !== null));
+            const userIds = Formatter.uniqueArray([
+                ...payments.map(p => p.payingUserId),
+                ...balanceItems.map(b => b.userId),
+            ].filter((id): id is string => id !== null));
+            const orderIds = Formatter.uniqueArray(balanceItems.map(b => b.orderId).filter((id): id is string => id !== null));
+            const balanceItemPayingOrganizationIds = Formatter.uniqueArray(balanceItems.map(b => b.payingOrganizationId).filter((id): id is string => id !== null));
+
+            const members = memberIds.length ? await Member.getByIDs(...memberIds) : [];
+            const users = userIds.length ? await User.getByIDs(...userIds) : [];
+            const orders = orderIds.length ? await Order.getByIDs(...orderIds) : [];
+            const extraOrganizations = balanceItemPayingOrganizationIds.length ? await Organization.getByIDs(...balanceItemPayingOrganizationIds) : [];
+            const allPayingOrganizations = [...payingOrganizations, ...extraOrganizations.filter(o => !payingOrganizations.some(p => p.id === o.id))];
+
+            for (const payment of payments) {
+                const paymentBalanceItemIds = balanceItemPayments.filter(bip => bip.paymentId === payment.id).map(bip => bip.balanceItemId);
+                const paymentBalanceItems = balanceItems.filter(b => paymentBalanceItemIds.includes(b.id));
+
+                const customer = determinePaymentCustomer(payment, {
+                    balanceItems: paymentBalanceItems,
+                    members,
+                    users,
+                    orders,
+                    payingOrganizations: allPayingOrganizations,
+                });
+
+                if (customer) {
+                    payment.customer = customer;
+                    await payment.save({
+                        skipMarkSaved: true,
+                        skipSendEvents: true,
+                    });
+                    filled++;
+                }
+            }
+        },
+    });
+
+    console.log(`Finished filling in payment customers: set ${filled} of ${result.total} payments.`);
+});

package/src/seeds/1780915002-update-orders.ts

@@ -0,0 +1,39 @@
+import { Migration } from '@simonbackx/simple-database';
+import { Order } from '@stamhoofd/models';
+import { QueryableModel } from '@stamhoofd/sql';
+import { SeedTools } from '../helpers/SeedTools.js';
+
+export default new Migration(async () => {
+    if (STAMHOOFD.environment === 'test') {
+        console.log('skipped in tests');
+        return;
+    }
+
+    if (STAMHOOFD.platformName !== 'stamhoofd') {
+        // Already ran, or not required for new servers
+        return;
+    }
+
+    console.log('Start saving orders.');
+    let realUpdate = 0;
+
+    const result = await SeedTools.loop({
+        query: Order.select().where('updatedAt', '<', new Date(Date.now() - 1_000 * 60 * 60 * 5)),
+        batchSize: 200,
+        useTransactionPerBatch: true,
+        action: async (order: Order) => {
+            if (await order.save({
+                skipMarkSaved: true,
+                skipSendEvents: true,
+            })) {
+                realUpdate += 1;
+            }
+
+            if (QueryableModel.shutdownMigrations) {
+                throw new Error('Stopping migration gracefully');
+            }
+        },
+    });
+
+    console.log('Updated ' + realUpdate + ' orders out of ' + result.total + ' looped orders');
+});

package/src/seeds/1782047708-organization-company-address.ts

@@ -0,0 +1,44 @@
+import { Migration } from '@simonbackx/simple-database';
+import { Organization } from '@stamhoofd/models';
+import { SeedTools } from '../helpers/SeedTools.js';
+
+export default new Migration(async () => {
+    if (STAMHOOFD.environment === 'test') {
+        // The customer determination is covered directly by PaymentCustomerResolver.test.ts
+        console.log('skipped in tests');
+        return;
+    }
+
+    if (STAMHOOFD.platformName !== 'stamhoofd') {
+        // In v1 only the Stamhoofd platform stored payments without a customer.
+        return;
+    }
+
+    process.stdout.write('\n');
+
+    await migrateOrganizations();
+});
+
+async function migrateOrganizations() {
+    console.log('Start updating organizations.');
+
+    let realUpdate = 0;
+    const result = await SeedTools.loop({
+        query: Organization.select(),
+        batchSize: 200,
+        useTransactionPerBatch: true,
+        action: async (organization: Organization) => {
+            if (organization.meta.companies.length && !organization.meta.companies[0].address) {
+                for (const c of organization.meta.companies) {
+                    if (!c.address) {
+                        c.address = organization.address;
+                    }
+                }
+                await organization.save();
+                realUpdate += 1;
+            }
+        },
+    });
+
+    console.log('Updated ' + realUpdate + ' organizations out of ' + result.total + ' looped organizations');
+}

package/src/seeds-temporary/1769088653-uitpas-status.ts

@@ -76,8 +76,7 @@ async function migrateMember(member: Member) {
 
         await member.save();
         idOfLastUpdatedMember = member.id;
-    }
-    catch (error) {
+    } catch (error) {
         if (isSimpleError(error)) {
             // if rate limited
             if (error.statusCode === 429) {