@stamhoofd/models 2.12.0 → 2.16.0

package/src/helpers/MemberMerger.ts

@@ -0,0 +1,497 @@
+import { Model } from "@simonbackx/simple-database";
+import { SQL } from "@stamhoofd/sql";
+import {
+    Address,
+    BooleanStatus,
+    Gender,
+    MemberDetails,
+    Parent,
+    ParentType,
+    RecordAnswer,
+} from "@stamhoofd/structures";
+import { Formatter } from "@stamhoofd/utility";
+import {
+    BalanceItem,
+    Document,
+    Member,
+    MemberPlatformMembership,
+    MemberResponsibilityRecord,
+    MergedMember,
+    Registration,
+    User,
+} from "../models";
+
+export async function mergeMultipleMembers(members: Member[]) {
+    const { base, others } = selectBaseMember(members);
+
+    for (const other of others) {
+        await mergeTwoMembers(base, other);
+    }
+}
+
+export async function findEqualMembers({
+    firstName,
+    lastName,
+    birthDay,
+}: {
+    firstName: string;
+    lastName: string;
+    birthDay: string;
+}): Promise<Member[]> {
+    return await Member.where({
+        firstName,
+        lastName,
+        birthDay,
+    });
+}
+
+async function mergeTwoMembers(base: Member, other: Member): Promise<void> {
+    mergeMemberDetails(base, other);
+
+    await mergeRegistrations(base, other);
+    await mergeUsers(base, other);
+    await mergeResponsibilities(base, other);
+    await mergeBalanceItems(base, other);
+    await mergeDocuments(base, other);
+    await mergeMemberPlatformMemberships(base, other);
+
+    await base.save();
+    // store other member in merged_member table
+    const mergedMember = MergedMember.fromMember(other, base.id);
+    await mergedMember.save();
+    await other.delete();
+}
+
+async function mergeRegistrations(base: Member, other: Member) {
+    await mergeModels(base, other, Registration);
+}
+
+async function mergeUsers(base: Member, other: Member) {
+    await mergeModels(base, other, User);
+}
+
+async function mergeResponsibilities(base: Member, other: Member) {
+    async function getResponsibilities(memberId: string) {
+        const rows = await SQL.select()
+            .from(SQL.table(MemberResponsibilityRecord.table))
+            .where(SQL.column("memberId"), memberId)
+            .fetch();
+
+        return MemberResponsibilityRecord.fromRows(
+            rows,
+            MemberResponsibilityRecord.table
+        );
+    }
+
+    const otherResponsibilities = await getResponsibilities(other.id);
+    const baseResponsibilities = await getResponsibilities(base.id);
+
+    // Delete duplicate responsibilities where endDate is null -> keep responsibility with oldest start date
+    for (const otherResponsibility of otherResponsibilities) {
+        // check if equal responsibilities exist
+        const otherResponsibilitiesWithoutCurrent =
+            otherResponsibilities.filter(
+                (o) => o.id !== otherResponsibility.id
+            );
+        const equalResponsibilities = baseResponsibilities
+            .concat(otherResponsibilitiesWithoutCurrent)
+            .filter((baseResponsibility) => {
+                return (
+                    baseResponsibility.responsibilityId ===
+                        otherResponsibility.responsibilityId &&
+                    baseResponsibility.organizationId ===
+                        otherResponsibility.organizationId &&
+                    baseResponsibility.groupId ===
+                        otherResponsibility.groupId &&
+                    baseResponsibility.endDate === null &&
+                    otherResponsibility.endDate === null
+                );
+            });
+
+        if (equalResponsibilities.length > 0) {
+            const allEqualResponsibilities = [
+                ...equalResponsibilities,
+                otherResponsibility,
+            ]
+                // sort on startDate
+                .sort((a, b) => a.startDate.getTime() - b.startDate.getTime());
+
+            const responsibilityWithOldestStartDate =
+                allEqualResponsibilities[0];
+
+            const responsibilitiesToDelete = allEqualResponsibilities.slice(
+                1,
+                undefined
+            );
+
+            for (const responsibilityToDelete of responsibilitiesToDelete) {
+                const baseIndex = baseResponsibilities.indexOf(
+                    responsibilityToDelete
+                );
+                if (baseIndex !== -1) baseResponsibilities.splice(baseIndex, 1);
+                else {
+                    const otherIndex = otherResponsibilities.indexOf(
+                        responsibilityToDelete
+                    );
+                    if (otherIndex !== -1)
+                        otherResponsibilities.splice(otherIndex, 1);
+                }
+
+                if (responsibilityToDelete.existsInDatabase) {
+                    await responsibilityToDelete.delete();
+                }
+            }
+
+            if (responsibilityWithOldestStartDate.memberId !== base.id) {
+                responsibilityWithOldestStartDate.memberId = base.id;
+                await responsibilityWithOldestStartDate.save();
+            }
+        } else {
+            otherResponsibility.memberId = base.id;
+            await otherResponsibility.save();
+        }
+    }
+}
+
+async function mergeBalanceItems(base: Member, other: Member) {
+    await mergeModels(base, other, BalanceItem);
+}
+
+async function mergeDocuments(base: Member, other: Member) {
+    await mergeModels(base, other, Document);
+}
+
+async function mergeMemberPlatformMemberships(base: Member, other: Member) {
+    await mergeModels(base, other, MemberPlatformMembership);
+}
+
+class ModelWithMemberId extends Model {
+    memberId: string | null;
+}
+
+async function mergeModels<M extends typeof ModelWithMemberId>(
+    base: Member,
+    other: Member,
+    model: M
+) {
+    const baseId = base.id;
+    const otherModels = await model.where({
+        memberId: other.id,
+    });
+
+    for (const otherModel of otherModels) {
+        otherModel.memberId = baseId;
+        await otherModel.save();
+    }
+}
+
+export function mergeMemberDetails(base: Member, other: Member): void {
+    const baseDetails = base.details;
+    const otherDetails = other.details;
+
+    // string details
+    mergeStringIfBaseNotSet(baseDetails, otherDetails, "firstName");
+    mergeStringIfBaseNotSet(baseDetails, otherDetails, "lastName");
+
+    mergeStringIfBaseNotSet(baseDetails, otherDetails, "memberNumber");
+    mergeStringIfBaseNotSet(baseDetails, otherDetails, "uitpasNumber");
+
+    // email
+    mergeEmail(baseDetails, otherDetails, baseDetails.unverifiedEmails);
+
+    // phone
+    mergePhone(baseDetails, otherDetails, baseDetails);
+
+    // gender
+    if (baseDetails.gender === Gender.Other) {
+        baseDetails.gender = otherDetails.gender;
+    }
+
+    // notes
+    mergeNotes(baseDetails, otherDetails);
+
+    // date
+    mergeIfBaseNotSet(baseDetails, otherDetails, "birthDay");
+
+    // boolean status
+    mergeBooleanStatusIfBaseNotSet(
+        baseDetails,
+        otherDetails,
+        "requiresFinancialSupport"
+    );
+
+    mergeBooleanStatusIfBaseNotSet(
+        baseDetails,
+        otherDetails,
+        "dataPermissions"
+    );
+
+    // address
+    mergeAddress(baseDetails, otherDetails, baseDetails);
+
+    // parents
+    mergeParents(baseDetails, otherDetails);
+
+    // emergency contacts
+    baseDetails.emergencyContacts = baseDetails.emergencyContacts.concat(
+        // add contacts that are not yet in the list
+        otherDetails.emergencyContacts.filter(
+            (otherContact) =>
+                !baseDetails.emergencyContacts.some((baseContact) =>
+                    baseContact.isEqual(otherContact)
+                )
+        )
+    );
+
+    // review times
+    // todo: is this correct?
+    baseDetails.reviewTimes.merge(otherDetails.reviewTimes);
+
+    // answers
+    mergeAnswers(baseDetails, otherDetails);
+
+    // unverified data
+    baseDetails.unverifiedEmails = Formatter.uniqueArray(
+        baseDetails.unverifiedEmails.concat(
+            otherDetails.unverifiedEmails.filter(
+                (email) => !isNullOrEmpty(email)
+            )
+        )
+    );
+    baseDetails.unverifiedPhones = Formatter.uniqueArray(
+        baseDetails.unverifiedPhones.concat(
+            otherDetails.unverifiedPhones.filter(
+                (phone) => !isNullOrEmpty(phone)
+            )
+        )
+    );
+
+    // unverified addresses
+    for (const address of otherDetails.unverifiedAddresses) {
+        if (!baseDetails.unverifiedAddresses.some((a) => a.id === address.id)) {
+            baseDetails.unverifiedAddresses.push(address);
+        }
+    }
+}
+
+export function selectBaseMember(members: Member[]): {
+    base: Member;
+    others: Member[];
+} {
+    if (members.length < 2) {
+        throw Error("Members array length is less than 2.");
+    }
+    const sorted = members.sort(
+        (m1, m2) => m2.createdAt.getTime() - m1.createdAt.getTime()
+    );
+
+    return { base: sorted[0], others: sorted.slice(1, undefined) };
+}
+
+function mergeAnswers(base: MemberDetails, other: MemberDetails) {
+    const newAnswers: Map<string, RecordAnswer> = new Map(base.recordAnswers);
+    for (const otherAnswer of other.recordAnswers.values()) {
+        const otherId = otherAnswer.settings.id;
+        const baseAnswer = newAnswers.get(otherId);
+
+        if (!baseAnswer) {
+            newAnswers.set(otherId, otherAnswer);
+        } else if (otherAnswer.date >= baseAnswer.date) {
+            newAnswers.set(otherId, otherAnswer);
+        } else {
+            // keep existing, this one is more up-to-date, don't add the other answer
+        }
+    }
+    base.recordAnswers = newAnswers;
+}
+
+function mergeNotes(base: MemberDetails, other: MemberDetails) {
+    if (base.notes && other.notes) {
+        base.notes = `${base.notes}\n${other.notes}`;
+    } else if (base.notes) {
+        return;
+    } else {
+        base.notes = other.notes;
+    }
+}
+
+function mergeParents(base: MemberDetails, other: MemberDetails) {
+    const baseParents = base.parents;
+    const otherParents = other.parents;
+    const parentsToAdd: Parent[] = [];
+
+    for (const otherParent of otherParents) {
+        // equal if same first and last name
+        const equalBaseParent = baseParents.find(
+            (baseParent) =>
+                hasEqualStringValue(baseParent, otherParent, "firstName") &&
+                hasEqualStringValue(baseParent, otherParent, "lastName")
+        );
+
+        if (!equalBaseParent) {
+            parentsToAdd.push(otherParent);
+            continue;
+        }
+
+        mergeParent(equalBaseParent, otherParent, base);
+    }
+
+    base.parents = baseParents.concat(parentsToAdd);
+}
+
+function mergeParent(base: Parent, other: Parent, baseDetails: MemberDetails) {
+    if (base.type === ParentType.Other) {
+        base.type = other.type;
+    }
+    mergeStringIfBaseNotSet(base, other, "firstName");
+    mergeStringIfBaseNotSet(base, other, "lastName");
+    // add other emails to alternative emails
+    mergeEmail(base, other, base.alternativeEmails);
+    mergePhone(base, other, baseDetails);
+    mergeAddress(base, other, baseDetails);
+}
+
+function mergeEmail(
+    base: { email: string | null | undefined },
+    other: { email: string | null | undefined },
+    alternativeEmails: string[]
+) {
+    const isEmailMerged = mergeStringIfBaseNotSet(base, other, "email");
+    const otherEmail = other.email;
+    if (!isEmailMerged && !isNullOrEmpty(otherEmail)) {
+        if (!alternativeEmails.some((email) => email === otherEmail)) {
+            alternativeEmails.push(otherEmail!);
+        }
+    }
+}
+
+function mergePhone(
+    base: { phone: string | null | undefined },
+    other: { phone: string | null | undefined },
+    baseDetails: MemberDetails
+) {
+    const isPhoneMerged = mergeStringIfBaseNotSet(base, other, "phone");
+    const otherPhone = other.phone;
+    if (!isPhoneMerged && !isNullOrEmpty(otherPhone)) {
+        if (
+            !baseDetails.unverifiedPhones.some((phone) => phone === otherPhone)
+        ) {
+            baseDetails.unverifiedPhones.push(otherPhone!);
+        }
+    }
+}
+
+function mergeAddress(
+    base: { address: Address | null | undefined },
+    other: { address: Address | null | undefined },
+    baseDetails: MemberDetails
+) {
+    const baseAddress = base.address;
+    const otherAddress = other.address;
+
+    if (!baseAddress) {
+        base.address = otherAddress;
+    } else if (otherAddress && baseAddress.id !== otherAddress.id) {
+        // add other address to unverified addresses
+        if (
+            !baseDetails.unverifiedAddresses.some(
+                (address) => address.id === otherAddress.id
+            )
+        ) {
+            baseDetails.unverifiedAddresses.push(otherAddress);
+        }
+    }
+}
+
+function mergeStringIfBaseNotSet<T, K extends keyof T>(
+    base: T,
+    other: T,
+    key: K & (T[K] extends string | null | undefined ? K : never)
+): boolean {
+    const baseValue = base[key] as string | null | undefined;
+    if (!isNullOrEmpty(baseValue)) {
+        return false;
+    }
+    const otherValue = other[key] as string | null | undefined;
+    if (isNullOrEmpty(otherValue)) {
+        return false;
+    }
+
+    (base[key] as string | null | undefined) = otherValue;
+
+    return true;
+}
+
+function mergeIfBaseNotSet<T, K extends keyof T>(
+    base: T,
+    other: T,
+    key: K &
+        (T[K] extends number | Date | boolean | null | undefined ? K : never)
+): boolean {
+    const baseValue = base[key] as number | Date | boolean | null | undefined;
+    if (!(baseValue === null || baseValue === undefined)) return false;
+    const otherValue = other[key] as number | Date | boolean | null | undefined;
+    if (otherValue === null || otherValue === undefined) return false;
+    (base[key] as number | Date | boolean | null | undefined) = otherValue;
+    return true;
+}
+
+function mergeBooleanStatusIfBaseNotSet<T, K extends keyof T>(
+    base: T,
+    other: T,
+    key: K & (T[K] extends BooleanStatus | null | undefined ? K : never)
+): boolean {
+    const otherValue = other[key] as BooleanStatus | null | undefined;
+    if (otherValue === null || otherValue === undefined) return false;
+    const baseValue = base[key] as BooleanStatus | null | undefined;
+    if (!(baseValue === null || baseValue === undefined)) {
+        if (baseValue.date < otherValue.date) {
+            (base[key] as BooleanStatus | null | undefined) = otherValue;
+            return true;
+        }
+        return false;
+    }
+
+    (base[key] as BooleanStatus | null | undefined) = otherValue;
+    return true;
+}
+
+/**
+ * Returns true if the values of the key for a and b
+ * are not null or undefined
+ * and both are equal.
+ * @param a
+ * @param b
+ * @param key
+ * @returns
+ */
+function hasEqualStringValue<T, K extends keyof T>(
+    a: T,
+    b: T,
+    key: K & (T[K] extends string | null | undefined ? K : never)
+) {
+    return hasValueAndIsEqual(
+        a[key] as string | null | undefined,
+        b[key] as string | null | undefined
+    );
+}
+
+function hasValueAndIsEqual(
+    a: string | null | undefined,
+    b: string | null | undefined
+): boolean {
+    if (isNullOrEmpty(a) || isNullOrEmpty(b)) return false;
+    return isStringEqual(a as string, b as string);
+}
+
+function isStringEqual(a: string, b: string): boolean {
+    return toLowerTrim(a) === toLowerTrim(b);
+}
+
+function toLowerTrim(name: string) {
+    return name.toLowerCase().trim();
+}
+
+function isNullOrEmpty(value: string | null | undefined) {
+    return value === null || value === undefined || value.trim() === "";
+}

package/src/helpers/RateLimiter.ts

@@ -42,6 +42,7 @@ export class RateLimitWindow {
             throw new SimpleError({
                 code: 'rate_limit',
                 message: `Rate limit exceeded (${w} ${amount > 1 ? '('+amount+' added)' : ''} requests in ${Math.round(this.age/1000)}s). Retry after ${retryAfter}s. Check your code and try to reduce the number of (parallel) requests you make. Add waiting periods if needed.`,
+                human: `Oeps! Te veel aanvragen. Om spam te vermijden is jouw aanvraag tijdelijk geblokkeerd. Probeer het over ${retryAfter} seconden opnieuw.`,
                 statusCode: 429
             })
         }

package/src/index.ts

@@ -15,7 +15,8 @@ export * from "./helpers/EmailBuilder"
 export * from "./helpers/GroupBuilder"
 export * from "./helpers/RateLimiter"
 export * from "./helpers/WebshopCounter"
+export * from "./helpers/MemberMerger"
 
 // Models
 export * from "./models"
-export * from "./structures/OrganizationServerMetaData"
+export * from "./structures/OrganizationServerMetaData"

package/src/migrations/1723736282-merged-members.sql

@@ -0,0 +1,19 @@
+CREATE TABLE `merged_members` (
+  `id` varchar(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL,
+  `firstName` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL,
+  `lastName` varchar(200) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL,
+  `birthDay` varchar(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL,
+  `organizationId` varchar(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL,
+  `details` json NOT NULL,
+  `createdAt` datetime NOT NULL,
+  `updatedAt` datetime NOT NULL,
+  `outstandingBalance` int NOT NULL DEFAULT '0',
+  `memberNumber` varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL,
+  `mergedToId` varchar(36) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL,
+  `mergedAt` datetime NOT NULL,
+  PRIMARY KEY (`id`),
+  KEY `organizationId` (`organizationId`),
+  CONSTRAINT `merged_members_ibfk_1` FOREIGN KEY (`organizationId`) REFERENCES `organizations` (`id`) ON DELETE CASCADE ON UPDATE CASCADE,
+  KEY `mergedToId` (`mergedToId`),
+  CONSTRAINT `merged_members_ibfk_2` FOREIGN KEY (`mergedToId`) REFERENCES `members` (`id`) ON DELETE CASCADE ON UPDATE CASCADE
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;

package/src/models/Email.ts

@@ -241,17 +241,15 @@ export class Email extends Model {
 
             // eslint-disable-next-line no-constant-condition
             while (true) {
-                const q = SQL.select()
-                    .from(SQL.table('email_recipients'))
-                    .where(SQL.column('emailId'), upToDate.id)
-                    .where(SQL.column('sentAt'), null)
-                    .where(SQL.column('id'), SQLWhereSign.Greater, idPointer);
-
-                q.orderBy(SQL.column('id'), 'ASC')
-                q.limit(batchSize)
+                const data = await SQL.select()
+                    .from('email_recipients')
+                    .where('emailId', upToDate.id)
+                    .where('sentAt', null)
+                    .where('id', SQLWhereSign.Greater, idPointer)
+                    .orderBy(SQL.column('id'), 'ASC')
+                    .limit(batchSize)
+                    .fetch();
                 
-                const data = await q.fetch();
-
                 const recipients = EmailRecipient.fromRows(data, 'email_recipients');
 
                 if (recipients.length == 0) {
@@ -301,7 +299,7 @@ export class Email extends Model {
                         from, 
                         replyTo,
                         subject: upToDate.subject!, 
-                        html: upToDate.html,
+                        html: upToDate.html!,
                         type: "broadcast",
                         callback(error: Error|null ) {
                             callback(error).catch(console.error)

package/src/models/EmailTemplate.ts

@@ -1,5 +1,6 @@
-import { column, Model } from "@simonbackx/simple-database";
+import { column, Model, SQLResultNamespacedRow } from "@simonbackx/simple-database";
 import { AnyDecoder } from "@simonbackx/simple-encoding";
+import { SQL, SQLSelect } from "@stamhoofd/sql";
 import { EmailTemplate as EmailTemplateStruct, EmailTemplateType } from "@stamhoofd/structures";
 import { v4 as uuidv4 } from "uuid";
 
@@ -65,4 +66,22 @@ export class EmailTemplate extends Model {
     getStructure() {
         return EmailTemplateStruct.create(this)
     }
+
+    /**
+     * Experimental: needs to move to library
+     */
+    static select() {
+        const transformer = (row: SQLResultNamespacedRow): EmailTemplate => {
+            const d = (this as typeof EmailTemplate & typeof Model).fromRow(row[this.table] as any) as EmailTemplate|undefined
+
+            if (!d) {
+                throw new Error("EmailTemplate not found")
+            }
+
+            return d;
+        }
+        
+        const select = new SQLSelect(transformer, SQL.wildcard())
+        return select.from(SQL.table(this.table))
+    }
 }

package/src/models/Member.ts

@@ -18,7 +18,7 @@ export type RegistrationWithMember = Registration & { member: Member }
 export class Member extends Model {
     static table = "members"
 
-    // Columns
+    //#region Columns
     @column({
         primary: true, type: "string", beforeSave(value) {
             return value ?? uuidv4();
@@ -91,6 +91,7 @@ export class Member extends Model {
         skipUpdate: true
     })
     updatedAt: Date
+    //#endregion
 
     static registrations = new OneToManyRelation(Member, Registration, "registrations", "memberId")
 
@@ -354,14 +355,12 @@ export class Member extends Model {
      */
     static async getMembersWithRegistrationForUser(user: User): Promise<MemberWithRegistrations[]> {
         const query = SQL
-            .select(
-                SQL.column('id')
-            )
-            .from(SQL.table(Member.table))
+            .select('id')
+            .from(Member.table)
             .join(
-                SQL.leftJoin(
-                    SQL.table('_members_users')
-                ).where(
+                SQL
+                .leftJoin('_members_users')
+                .where(
                     SQL.column('_members_users', 'membersId'),
                     SQL.column(Member.table, 'id'),
                 )