@stamhoofd/backend 2.121.0 → 2.122.0

package/src/audit-logs/EventNotificationLogger.ts

@@ -0,0 +1,40 @@
+import { EventNotification, Platform } from '@stamhoofd/models';
+import { AuditLogReplacement, AuditLogReplacementType, AuditLogType } from '@stamhoofd/structures';
+import { getDefaultGenerator, ModelLogger } from './ModelLogger.js';
+
+const defaultGenerator = getDefaultGenerator<EventNotification>({
+    created: AuditLogType.EventNotificationAdded,
+    updated: AuditLogType.EventNotificationEdited,
+    deleted: AuditLogType.EventNotificationDeleted,
+});
+
+export const EventNotificationLogger = new ModelLogger(EventNotification, {
+    skipKeys: ['createdBy', 'submittedBy', 'submittedAt', 'acceptedRecordAnswers'],
+
+    async optionsGenerator(event) {
+        const result = await defaultGenerator(event);
+        if (!result) {
+            return;
+        }
+
+        const platform = await Platform.getSharedPrivateStruct();
+        const type = platform.config.eventNotificationTypes.find(type => type.id === event.model.typeId);
+
+        return {
+            ...result,
+            data: {
+                title: type?.title || event.model.id,
+            },
+        };
+    },
+
+    createReplacements(model, { data }) {
+        return new Map([
+            ['n', AuditLogReplacement.create({
+                id: model.id,
+                value: data.title,
+                type: AuditLogReplacementType.EventNotification,
+            })],
+        ]);
+    },
+});

package/src/audit-logs/ModelLogger.ts

@@ -125,8 +125,7 @@ export class ModelLogger<ModelType extends typeof Model, M extends InstanceType<
                 if (!settings?.source) {
                     if (settings) {
                         settings.source = AuditLogSource.System;
-                    }
-                    else {
+                    } else {
                         settings = { source: AuditLogSource.System };
                     }
                 }
@@ -148,11 +147,9 @@ export class ModelLogger<ModelType extends typeof Model, M extends InstanceType<
 
             if (settings?.source) {
                 log.source = settings.source;
-            }
-            else if (context) {
+            } else if (context) {
                 log.source = userId ? AuditLogSource.User : AuditLogSource.Anonymous;
-            }
-            else {
+            } else {
                 log.source = AuditLogSource.System;
             }
 
@@ -179,25 +176,34 @@ export class ModelLogger<ModelType extends typeof Model, M extends InstanceType<
                                 key: AuditLogReplacement.key(renamedKey),
                                 type: AuditLogPatchItemType.Changed,
                             }));
-                        }
-                        else {
+                        } else {
+                            const column = event.model.static.columns?.get(key);
                             log.patchList.push(...ObjectDiffer.diff(
                                 key in oldModel ? oldModel[key] : undefined,
                                 key in event.model ? event.model[key] : undefined,
                                 AuditLogReplacement.key(renamedKey),
+                                column?.decoder,
                             ));
                         }
                     }
 
                     // Remove skipped keys
-                    log.patchList = log.patchList.filter(p => !this.skipKeys?.includes(p.key.toKey()));
+                    if (this.skipKeys) {
+                        log.patchList = log.patchList.filter((p) => {
+                            for (const s of this.skipKeys) {
+                                if (p.key.toKey().startsWith(s)) {
+                                    return false;
+                                }
+                            }
+                            return true;
+                        });
+                    }
 
                     if (log.patchList.length === 0 && !log.description) {
                         // No changes or all skipped
                         return false;
                     }
-                }
-                else {
+                } else {
                     return false;
                 }
             }
@@ -235,8 +241,7 @@ export class ModelLogger<ModelType extends typeof Model, M extends InstanceType<
 
             try {
                 log.validate();
-            }
-            catch (e) {
+            } catch (e) {
                 console.error('AuditLog dit not pass validation', log);
             }
 
@@ -250,8 +255,7 @@ export class ModelLogger<ModelType extends typeof Model, M extends InstanceType<
                 events: d.slice(0, 1),
             });
             return saved;
-        }
-        catch (e) {
+        } catch (e) {
             console.error('Failed to save log', e, log);
         }
         return false;

package/src/audit-logs/OrganizationLogger.ts

@@ -3,7 +3,7 @@ import { AuditLogReplacement, AuditLogReplacementType, AuditLogType } from '@sta
 import { getDefaultGenerator, ModelLogger } from './ModelLogger.js';
 
 export const OrganizationLogger = new ModelLogger(Organization, {
-    skipKeys: ['searchIndex', 'serverMeta'],
+    skipKeys: ['searchIndex', 'serverMeta', 'meta.packages'],
     optionsGenerator: getDefaultGenerator<Organization>({
         created: AuditLogType.OrganizationAdded,
         updated: AuditLogType.OrganizationEdited,

package/src/audit-logs/RegistrationInvitationLogger.ts

@@ -3,33 +3,33 @@ import { AuditLogReplacement, AuditLogReplacementType, AuditLogType } from '@sta
 import { getDefaultGenerator, ModelLogger } from './ModelLogger.js';
 
 const defaultGenerator = getDefaultGenerator<RegistrationInvitation>({
-        created: AuditLogType.RegistrationInvitationAdded,
-        deleted: AuditLogType.RegistrationInvitationDeleted,
+    created: AuditLogType.RegistrationInvitationAdded,
+    deleted: AuditLogType.RegistrationInvitationDeleted,
 });
 
 export const RegistrationInvitationLogger = new ModelLogger(RegistrationInvitation, {
-        async optionsGenerator(event) {
-            const result = await defaultGenerator(event);
-            if (!result) {
-                return;
-            }
-    
-            const member = await Member.getByID(event.model.memberId);
-            const group = await Group.getByID(event.model.groupId);
-    
-            if (!member || !group) {
-                return;
-            }
-    
-            return {
-                ...result,
-                data: {
-                    member,
-                    group,
-                },
-            };
-        },
-        createReplacements(_, { data }) {
+    async optionsGenerator(event) {
+        const result = await defaultGenerator(event);
+        if (!result) {
+            return;
+        }
+
+        const member = await Member.getByID(event.model.memberId);
+        const group = await Group.getByID(event.model.groupId);
+
+        if (!member || !group) {
+            return;
+        }
+
+        return {
+            ...result,
+            data: {
+                member,
+                group,
+            },
+        };
+    },
+    createReplacements(_, { data }) {
         return new Map([
             ['m', AuditLogReplacement.create({
                 id: data.member.id,

package/src/audit-logs/init.ts

@@ -1,8 +1,10 @@
+import { AuditLogService, modelLogDefinitions } from '../services/AuditLogService.js';
 import { DocumentTemplateLogger } from './DocumentTemplateLogger.js';
 import { EmailAddressLogger } from './EmailAddressLogger.js';
 import { EmailLogger } from './EmailLogger.js';
 import { EmailTemplateLogger } from './EmailTemplateLogger.js';
 import { EventLogger } from './EventLogger.js';
+import { EventNotificationLogger } from './EventNotificationLogger.js';
 import { GroupLogger } from './GroupLogger.js';
 import { MemberLogger } from './MemberLogger.js';
 import { MemberPlatformMembershipLogger } from './MemberPlatformMembershipLogger.js';
@@ -12,19 +14,19 @@ import { OrganizationLogger } from './OrganizationLogger.js';
 import { OrganizationRegistrationPeriodLogger } from './OrganizationRegistrationPeriodLogger.js';
 import { PaymentLogger } from './PaymentLogger.js';
 import { PlatformLogger } from './PlatformLogger.js';
+import { RegistrationInvitationLogger } from './RegistrationInvitationLogger.js';
 import { RegistrationLogger } from './RegistrationLogger.js';
 import { RegistrationPeriodLogger } from './RegistrationPeriodLogger.js';
 import { StripeAccountLogger } from './StripeAccountLogger.js';
 import { UserLogger } from './UserLogger.js';
 import { WebshopLogger } from './WebshopLogger.js';
-import { modelLogDefinitions } from '../services/AuditLogService.js';
-import { RegistrationInvitationLogger } from './RegistrationInvitationLogger.js';
 
 modelLogDefinitions.set(RegistrationLogger.model, RegistrationLogger);
 modelLogDefinitions.set(GroupLogger.model, GroupLogger);
 modelLogDefinitions.set(OrganizationLogger.model, OrganizationLogger);
 modelLogDefinitions.set(PlatformLogger.model, PlatformLogger);
 modelLogDefinitions.set(EventLogger.model, EventLogger);
+modelLogDefinitions.set(EventNotificationLogger.model, EventNotificationLogger);
 modelLogDefinitions.set(RegistrationPeriodLogger.model, RegistrationPeriodLogger);
 modelLogDefinitions.set(OrganizationRegistrationPeriodLogger.model, OrganizationRegistrationPeriodLogger);
 modelLogDefinitions.set(StripeAccountLogger.model, StripeAccountLogger);
@@ -40,3 +42,7 @@ modelLogDefinitions.set(EmailTemplateLogger.model, EmailTemplateLogger);
 modelLogDefinitions.set(EmailAddressLogger.model, EmailAddressLogger);
 modelLogDefinitions.set(UserLogger.model, UserLogger);
 modelLogDefinitions.set(RegistrationInvitationLogger.model, RegistrationInvitationLogger);
+
+if (STAMHOOFD.environment !== 'production') {
+    AuditLogService.validateAuditLogEnumTranslations();
+}

package/src/boot.ts

@@ -1,14 +1,16 @@
 import { Column, Database, Migration } from '@simonbackx/simple-database';
 import { CORSPreflightEndpoint, Router, RouterServer } from '@simonbackx/simple-endpoints';
 import { CORSMiddleware, LogMiddleware, VersionMiddleware } from '@stamhoofd/backend-middleware';
+import { checkReadOnly } from '@stamhoofd/crons';
 import { Email } from '@stamhoofd/email';
 import { loadLogger } from '@stamhoofd/logging';
 import { Version } from '@stamhoofd/structures';
 import { sleep } from '@stamhoofd/utility';
-import { checkReadOnly } from '@stamhoofd/crons';
 
+import { AutoEncoder } from '@simonbackx/simple-encoding';
 import { SimpleError } from '@simonbackx/simple-errors';
-import { startCrons, stopCrons, waitForCrons } from '@stamhoofd/crons';
+import { StyledText } from '@simonbackx/simple-logging';
+import { markAllSeedsFinished, startCrons, stopCrons, waitForCrons } from '@stamhoofd/crons';
 import { Platform } from '@stamhoofd/models';
 import { QueueHandler } from '@stamhoofd/queues';
 import { resumeEmails } from './helpers/EmailResumer.js';
@@ -23,9 +25,9 @@ import { DocumentService } from './services/DocumentService.js';
 import { FileSignService } from './services/FileSignService.js';
 import { PlatformMembershipService } from './services/PlatformMembershipService.js';
 import { UitpasService } from './services/uitpas/UitpasService.js';
+import { UniqueMemberNumberService } from './services/UniqueMemberNumberService.js';
 import { UniqueUserService } from './services/UniqueUserService.js';
-import { AutoEncoder } from '@simonbackx/simple-encoding';
-import { StyledText } from '@simonbackx/simple-logging';
+import { QueryableModel } from '@stamhoofd/sql';
 
 process.on('unhandledRejection', (error: Error) => {
     console.error('unhandledRejection');
@@ -36,7 +38,7 @@ process.on('unhandledRejection', (error: Error) => {
 // Set version of saved structures
 Column.setJSONVersion(Version);
 
-// Set timezone
+// Set timezone to UTC
 process.env.TZ = 'UTC';
 
 if (STAMHOOFD.environment === 'development') {
@@ -48,7 +50,7 @@ if (new Date().getTimezoneOffset() !== 0) {
     throw new Error('Process should always run in UTC timezone');
 }
 
-const seeds = async (options: {shutdown: () => Promise<void>}) => {
+const seeds = async (options: { shutdown: () => Promise<void> }) => {
     if (await checkReadOnly()) {
         console.error(new StyledText(`[SEEDS] `).addClass('migration', 'tag'), new StyledText('MySQL is in read-only mode: Seeds are disabled.').addClass('error'));
         return;
@@ -63,10 +65,11 @@ const seeds = async (options: {shutdown: () => Promise<void>}) => {
                 if (STAMHOOFD.environment === 'test' || STAMHOOFD.environment === 'development') {
                     await options.shutdown();
                 }
+            } else {
+                markAllSeedsFinished();
             }
         });
-    }
-    catch (e) {
+    } catch (e) {
         console.error('Failed to run seeds:');
         console.error(e);
     }
@@ -81,19 +84,20 @@ function productionLog(message: string) {
 
 export const boot = async (options: { killProcess: boolean }) => {
     // Make sure we use the current environment for connecting to the database
-    await Database.reload({})
+    await Database.reload({});
 
     productionLog('Running server at v' + Version);
     productionLog('Running server at port ' + STAMHOOFD.PORT);
     productionLog('Running server on DB ' + process.env.DB_DATABASE); // note, should always use process env here
     console.log('Node version is: ' + process.version);
-    
+
     AutoEncoder.skipDefaultValuesVersion = 10000; // todo
 
     loadLogger();
 
     await GlobalHelper.load();
     await UniqueUserService.check();
+    await UniqueMemberNumberService.check();
     await BootChecksService.checkDatabaseCollation();
 
     // Init platform shared struct: otherwise permissions won't work with missing responsibilities
@@ -172,8 +176,7 @@ export const boot = async (options: { killProcess: boolean }) => {
 
     if (STAMHOOFD.environment !== 'development' && STAMHOOFD.environment !== 'test') {
         CpuService.startMonitoring();
-    }
-    else if (STAMHOOFD.environment === 'development') {
+    } else if (STAMHOOFD.environment === 'development') {
         const { loadDebugFunctions } = await import('./debug.js');
         loadDebugFunctions({ routerServer });
     }
@@ -189,6 +192,7 @@ export const boot = async (options: { killProcess: boolean }) => {
             return;
         }
         shuttingDown = true;
+        QueryableModel.shutdownMigrations = true;
         productionLog('Shutting down...');
         // Disable keep alive
         routerServer.defaultHeaders = Object.assign(routerServer.defaultHeaders, { Connection: 'close' });
@@ -209,8 +213,7 @@ export const boot = async (options: { killProcess: boolean }) => {
         try {
             await routerServer.close();
             productionLog('HTTP server stopped');
-        }
-        catch (err) {
+        } catch (err) {
             console.error('Failed to stop HTTP server:');
             console.error(err);
         }
@@ -231,8 +234,7 @@ export const boot = async (options: { killProcess: boolean }) => {
                 console.log(`${Email.currentQueue.length} emails still in queue. Waiting 500ms...`);
                 await sleep(500);
             }
-        }
-        catch (err) {
+        } catch (err) {
             console.error('Failed to wait for emails to finish:');
             console.error(err);
         }
@@ -240,8 +242,7 @@ export const boot = async (options: { killProcess: boolean }) => {
         try {
             await Database.end();
             productionLog('MySQL connections closed');
-        }
-        catch (err) {
+        } catch (err) {
             console.error('Failed to close MySQL connection:');
             console.error(err);
         }

package/src/crons/amazon-ses.ts

@@ -114,13 +114,11 @@ function readHeaders(message: any) {
                     headers[(header.name as string).toLowerCase()] = header.value;
                 }
             }
-        }
-        else {
+        } else {
             console.log('[AWS] Missing mail headers', message);
         }
         return headers;
-    }
-    catch (e) {
+    } catch (e) {
         console.log('[AWS] Failed to read headers', e, message);
         return {};
     }
@@ -172,8 +170,7 @@ async function handleBounce(message: any) {
                     response: recipient.diagnosticCode || '',
                     subject: message.mail.commonHeaders?.subject || '',
                 });
-            }
-            else if (
+            } else if (
                 type === 'Transient'
             ) {
                 const organization: Organization | undefined = source ? await Organization.getByEmail(source) : undefined;
@@ -197,8 +194,7 @@ async function handleBounce(message: any) {
             }
         }
         console.log('[AWS BOUNCES] For domain ' + source);
-    }
-    else {
+    } else {
         console.log("[AWS BOUNCES] 'bounce' field missing in bounce message");
     }
 }
@@ -236,8 +232,7 @@ async function handleComplaint(message: any) {
                         response: recipient.diagnosticCode || '',
                         subject: message.mail.commonHeaders?.subject || '',
                     });
-                }
-                else {
+                } else {
                     await saveLog({
                         id: b.feedbackId,
                         email,
@@ -262,8 +257,7 @@ async function handleComplaint(message: any) {
                 });
             }
         }
-    }
-    else {
+    } else {
         console.log('[AWS COMPLAINTS] Missing complaint field');
     }
 }
@@ -286,8 +280,7 @@ async function handleForward(message: any) {
                 Email.send(options);
             }
         }
-    }
-    else {
+    } else {
         console.log('[AWS FORWARDING] Missing mail, content or receipt field');
     }
 }
@@ -341,28 +334,23 @@ async function readFromQueue(queueUrl: string) {
                         // Docs: https://docs.aws.amazon.com/ses/latest/dg/event-publishing-retrieving-sns-contents.html
                         if (message.bounce) {
                             await handleBounce(message);
-                        }
-                        else if (message.complaint) {
+                        } else if (message.complaint) {
                             await handleComplaint(message);
                         }
                         // Read message content
                         // https://docs.aws.amazon.com/ses/latest/dg/receiving-email-notifications-contents.html
                         else if (message.mail && message.content && message.receipt) {
                             await handleForward(message);
-                        }
-                        else {
+                        } else {
                             console.log('[AWS Queue] Unsupported message');
                         }
-                    }
-                    else {
+                    } else {
                         console.log("[AWS Queue] 'Message' field missing");
                     }
-                }
-                else {
+                } else {
                     console.log('[AWS Queue] Message Body missing in bounce');
                 }
-            }
-            catch (e) {
+            } catch (e) {
                 console.log('[AWS Queue] Message processing failed:');
                 console.log('[AWS Queue]', e);
 
@@ -374,8 +362,7 @@ async function readFromQueue(queueUrl: string) {
 
     if (didProcess) {
         console.log(`[AWS Queue] Processed ${didProcess} message(s) from queue`);
-    }
-    else {
+    } else {
         console.log(`[AWS Queue] No message to process from queue`);
     }
     return didProcess;

package/src/crons/balance-emails.ts

@@ -205,8 +205,7 @@ async function sendTemplate({
 
         if (!upToDate) {
             console.log('No recipients found for organization', organization.name, organization.id);
-        }
-        else {
+        } else {
             const now = new Date();
             now.setMilliseconds(0);
 
@@ -221,7 +220,7 @@ async function sendTemplate({
                     .set(
                         'reminderEmailCount',
                         SQL.calculation(SQL.column('reminderEmailCount'))
-                            .add(readDynamicSQLExpression(1))
+                            .add(readDynamicSQLExpression(1)),
                     )
                     .where('id', balanceItemIds)
                     .where('organizationId', organization.id)
@@ -230,8 +229,7 @@ async function sendTemplate({
                     .update();
             }
         }
-    }
-    catch (e) {
+    } catch (e) {
         console.error('Error sending email for organization', e, organization.name, organization.id);
     }
 }

package/src/crons/clearExcelCache.test.ts

@@ -132,7 +132,7 @@ describe('clearExcelCacheHelper', () => {
         });
 
         fsMock.readdir.mockReturnValue(
-             
+
             Promise.resolve([...directories, file1]) as any,
         );
         // #endregion

package/src/crons/clearExcelCache.ts

@@ -59,8 +59,7 @@ export async function clearExcelCacheHelper({ lastExcelClear, now, cachePath, en
                     await fs.rm(path, { recursive: true, force: true });
                     console.log('Removed', path);
                 }
-            }
-            catch (error) {
+            } catch (error) {
                 console.error(error);
             }
         }

package/src/crons/drip-emails.ts

@@ -0,0 +1,42 @@
+import { registerCron } from '@stamhoofd/crons';
+import { Organization } from '@stamhoofd/models';
+import { isOutside } from './helpers/isOutside.js';
+import { useSavedIterator } from './helpers/useSavedIterator.js';
+
+const { iterate, isHoursAgo } = useSavedIterator(() => {
+    return Organization.select();
+}, { limit: 10, maxQueries: 5 });
+
+async function checkDrips() {
+    if (STAMHOOFD.environment === 'development') {
+        // For now reenabled on development for testing
+        // return;
+    }
+
+    if (STAMHOOFD.userMode === 'platform') {
+        return;
+    }
+
+    // Only send emails between 8:00 - 18:00 CET
+    if (isOutside('08:00', '18:00') && STAMHOOFD.environment === 'production') {
+        console.log('Skip Drip check: outside hours');
+        return;
+    }
+
+    if (!isHoursAgo(6)) {
+        return;
+    }
+
+    // Get the next x organization to send e-mails for
+    for await (const organization of iterate()) {
+        try {
+            await organization.checkDrips();
+        } catch (e) {
+            console.error(e);
+        }
+    }
+}
+
+if (STAMHOOFD.userMode !== 'platform') {
+    registerCron('checkDrips', checkDrips);
+}

package/src/crons/endFunctionsOfUsersWithoutRegistration.ts

@@ -12,6 +12,10 @@ let lastCleanupMonth: number = now.getMonth();
 registerCron('endFunctionsOfUsersWithoutRegistration', endFunctionsOfUsersWithoutRegistration);
 
 export async function endFunctionsOfUsersWithoutRegistration() {
+    if (STAMHOOFD.userMode === 'organization') {
+        return;
+    }
+
     const now = new Date();
     const currentYear = now.getFullYear();
     const currentMonth = now.getMonth();

package/src/crons/helpers/isOutside.ts

@@ -0,0 +1,11 @@
+import { Formatter } from '@stamhoofd/utility';
+
+type TimeString = `${0 | 1 | 2}${0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9}:${0 | 1 | 2 | 3 | 4 | 5 | 6}${0 | 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9}`;
+export function isOutside(start: TimeString, end: TimeString) {
+    // Only send emails between 8:00 - 18:00 CET
+    const CETTime = Formatter.timeIso(new Date());
+    if ((CETTime < start || CETTime > end)) {
+        return true;
+    }
+    return false;
+}

package/src/crons/helpers/useSavedIterator.ts

@@ -0,0 +1,32 @@
+import type { SQLResultNamespacedRow } from '@simonbackx/simple-database';
+import type { IterableSQLSelect, SQLSelect } from '@stamhoofd/sql';
+
+export function useSavedIterator<T extends object = SQLResultNamespacedRow>(start: () => SQLSelect<T>, options: { limit: number; maxQueries?: number }) {
+    let savedIterator: IterableSQLSelect<T> | null = null;
+    let lastFullRun: Date = new Date(0);
+
+    function updateStatus() {
+        if (savedIterator && savedIterator.isDone) {
+            savedIterator = null;
+            lastFullRun = new Date();
+
+            console.log('Done full loop');
+        }
+    }
+
+    const iterate = function () {
+        if (savedIterator === null) {
+            savedIterator = start().limit(options.limit).all({
+                nextEachHook: updateStatus,
+            });
+        }
+
+        return savedIterator.maxQueries(options.maxQueries ?? 1);
+    };
+
+    const isHoursAgo = function (hours: number) {
+        return lastFullRun.getTime() < new Date().getTime() - 1000 * 60 * 60 * hours;
+    };
+
+    return { iterate, lastFullRun, isHoursAgo };
+}

package/src/crons/index.ts

@@ -7,3 +7,8 @@ import './delete-old-email-drafts.js';
 import './delete-archived-data.js';
 import './mollie-chargebacks.js';
 import './invoices.js';
+import './service-fees.js';
+import './members-fees.js';
+import './stripe-invoices.js';
+import './transfer-fees.js';
+import './drip-emails.js';