appwrite-utils-cli 0.0.286 → 0.9.0

package/dist/migrations/dataLoader.js

@@ -45,6 +45,7 @@ export class DataLoader {
     // Maps to hold email and phone to user ID mappings for unique-ness in User Accounts
     emailToUserIdMap = new Map();
     phoneToUserIdMap = new Map();
+    userIdSet = new Set();
     userExistsMap = new Map();
     shouldWriteFile = false;
     // Constructor to initialize the DataLoader with necessary configurations
@@ -65,6 +66,7 @@ export class DataLoader {
      * It iterates through the target object's keys and updates the source object if:
      * - The source object has the key.
      * - The target object's value for that key is not null, undefined, or an empty string.
+     * - If the target object has an array value, it concatenates the values and removes duplicates.
      *
      * @param source - The source object to be updated.
      * @param target - The target object with values to update the source object.
@@ -73,29 +75,74 @@ export class DataLoader {
     mergeObjects(source, update) {
         // Create a new object to hold the merged result
         const result = { ...source };
-        Object.keys(update).forEach((key) => {
-            const sourceValue = source[key];
-            const updateValue = update[key];
-            // If the update value is an array, concatenate and remove duplicates
-            if (Array.isArray(updateValue)) {
-                const sourceArray = Array.isArray(sourceValue) ? sourceValue : [];
-                result[key] = [...new Set([...sourceArray, ...updateValue])];
-            }
-            // If the update value is an object, recursively merge
-            else if (updateValue !== null &&
-                typeof updateValue === "object" &&
-                !(updateValue instanceof Date)) {
-                result[key] = this.mergeObjects(sourceValue, updateValue);
-            }
-            // If the update value is not nullish, overwrite the source value
-            else if (updateValue !== null && updateValue !== undefined) {
-                result[key] = updateValue;
-            }
-            // If the update value is nullish, keep the original value unless it doesn't exist
-            else if (sourceValue === undefined) {
-                result[key] = updateValue;
+        // Loop through the keys of the object we care about
+        for (const [key, value] of Object.entries(source)) {
+            // Check if the key exists in the target object
+            if (!Object.hasOwn(update, key)) {
+                // If the key doesn't exist, we can just skip it like bad cheese
+                continue;
             }
-        });
+            if (update[key] === value) {
+                continue;
+            }
+            // If the value ain't here, we can just do whatever man
+            if (value === undefined || value === null || value === "") {
+                // If the update key is defined
+                if (update[key] !== undefined &&
+                    update[key] !== null &&
+                    update[key] !== "") {
+                    // might as well use it eh?
+                    result[key] = update[key];
+                }
+                // ELSE if the value is an array, because it would then not be === to those things above
+            }
+            else if (Array.isArray(value)) {
+                // Get the update value
+                const updateValue = update[key];
+                // If the update value is an array, concatenate and remove duplicates
+                // and poopy data
+                if (Array.isArray(updateValue)) {
+                    result[key] = [...new Set([...value, ...updateValue])].filter((item) => item !== null && item !== undefined && item !== "");
+                }
+                else {
+                    // If the update value is not an array, just use it
+                    result[key] = [...value, updateValue].filter((item) => item !== null && item !== undefined && item !== "");
+                }
+            }
+            else if (typeof value === "object" && !Array.isArray(value)) {
+                // If the value is an object, we need to merge it
+                if (typeof update[key] === "object" && !Array.isArray(update[key])) {
+                    result[key] = this.mergeObjects(value, update[key]);
+                }
+            }
+            else {
+                // Finally, the source value is defined, and not an array, so we don't care about the update value
+                continue;
+            }
+        }
+        // Because the objects should technically always be validated FIRST, we can assume the update keys are also defined on the source object
+        for (const [key, value] of Object.entries(update)) {
+            if (value === undefined || value === null || value === "") {
+                continue;
+            }
+            else if (!Object.hasOwn(source, key)) {
+                result[key] = value;
+            }
+            else if (typeof source[key] === "object" &&
+                typeof value === "object" &&
+                !Array.isArray(source[key]) &&
+                !Array.isArray(value)) {
+                result[key] = this.mergeObjects(source[key], value);
+            }
+            else if (Array.isArray(source[key]) && Array.isArray(value)) {
+                result[key] = [...new Set([...source[key], ...value])].filter((item) => item !== null && item !== undefined && item !== "");
+            }
+            else if (source[key] === undefined ||
+                source[key] === null ||
+                source[key] === "") {
+                result[key] = value;
+            }
+        }
         return result;
     }
     // Method to load data from a file specified in the import definition
@@ -125,8 +172,21 @@ export class DataLoader {
     // Method to generate a unique ID that doesn't conflict with existing IDs
     getTrueUniqueId(collectionName) {
         let newId = ID.unique();
-        while (this.checkMapValuesForId(newId, collectionName)) {
+        let condition = this.checkMapValuesForId(newId, collectionName) ||
+            this.userExistsMap.has(newId) ||
+            this.userIdSet.has(newId) ||
+            this.importMap
+                .get(this.getCollectionKey("users"))
+                ?.data.some((user) => user.finalData.docId === newId || user.finalData.userId === newId);
+        while (condition) {
             newId = ID.unique();
+            condition =
+                this.checkMapValuesForId(newId, collectionName) ||
+                    this.userExistsMap.has(newId) ||
+                    this.userIdSet.has(newId) ||
+                    this.importMap
+                        .get(this.getCollectionKey("users"))
+                        ?.data.some((user) => user.finalData.docId === newId || user.finalData.userId === newId);
         }
         return newId;
     }
@@ -203,12 +263,35 @@ export class DataLoader {
         // Iterate over the users and setup our maps ahead of time for email and phone
         for (const user of allUsers) {
             if (user.email) {
-                this.emailToUserIdMap.set(user.email, user.$id);
+                this.emailToUserIdMap.set(user.email.toLowerCase(), user.$id);
             }
             if (user.phone) {
                 this.phoneToUserIdMap.set(user.phone, user.$id);
             }
             this.userExistsMap.set(user.$id, true);
+            this.userIdSet.add(user.$id);
+            let importData = this.importMap.get(this.getCollectionKey("users"));
+            if (!importData) {
+                importData = {
+                    data: [],
+                };
+            }
+            importData.data.push({
+                finalData: {
+                    ...user,
+                    email: user.email?.toLowerCase(),
+                    userId: user.$id,
+                    docId: user.$id,
+                },
+                context: {
+                    ...user,
+                    email: user.email?.toLowerCase(),
+                    userId: user.$id,
+                    docId: user.$id,
+                },
+                rawData: user,
+            });
+            this.importMap.set(this.getCollectionKey("users"), importData);
         }
         return allUsers;
     }
@@ -219,7 +302,8 @@ export class DataLoader {
         console.log("---------------------------------");
         await this.setupMaps(dbId);
         const allUsers = await this.getAllUsers();
-        console.log(`Fetched ${allUsers.length} users`);
+        console.log(`Fetched ${allUsers.length} users, waiting a few seconds to let the program catch up...`);
+        await new Promise((resolve) => setTimeout(resolve, 5000));
         // Iterate over the configured databases to find the matching one
         for (const db of this.config.databases) {
             if (db.$id !== dbId) {
@@ -234,17 +318,19 @@ export class DataLoader {
                 // Determine if this is the users collection
                 let isUsersCollection = this.getCollectionKey(this.config.usersCollectionName) ===
                     this.getCollectionKey(collection.name);
+                const collectionDefs = collection.importDefs;
+                if (!collectionDefs || !collectionDefs.length) {
+                    continue;
+                }
                 // Process create and update definitions for the collection
                 const createDefs = collection.importDefs.filter((def) => def.type === "create" || !def.type);
                 const updateDefs = collection.importDefs.filter((def) => def.type === "update");
                 for (const createDef of createDefs) {
-                    if (!isUsersCollection) {
-                        console.log(`${collection.name} is not users collection`);
+                    if (!isUsersCollection || !createDef.createUsers) {
                         await this.prepareCreateData(db, collection, createDef);
                     }
                     else {
                         // Special handling for users collection if needed
-                        console.log(`${collection.name} is users collection`);
                         await this.prepareUserCollectionCreateData(db, collection, createDef);
                     }
                 }
@@ -254,12 +340,12 @@ export class DataLoader {
                         continue;
                     }
                     // Prepare the update data for the collection
-                    await this.prepareUpdateData(db, collection, updateDef);
+                    this.prepareUpdateData(db, collection, updateDef);
                 }
             }
             console.log("Running update references");
-            await this.dealWithMergedUsers();
-            await this.updateOldReferencesForNew();
+            // this.dealWithMergedUsers();
+            this.updateOldReferencesForNew();
             console.log("Done running update references");
         }
         // for (const collection of this.config.collections) {
@@ -272,49 +358,70 @@ export class DataLoader {
             this.writeMapsToJsonFile();
         }
     }
-    async dealWithMergedUsers() {
-        const usersCollectionKey = this.getCollectionKey(this.config.usersCollectionName);
-        const usersCollectionPrimaryKeyFields = new Set();
-        if (!this.config.collections) {
-            return;
+    /**
+     * Deals with merged users by iterating through all collections in the configuration.
+     * We have merged users if there are duplicate emails or phones in the import data.
+     * This function will iterate through all collections that are the same name as the
+     * users collection and pull out their primaryKeyField's. It will then loop through
+     * each collection and find any documents that have a
+     *
+     * @return {void} This function does not return anything.
+     */
+    // dealWithMergedUsers() {
+    //   const usersCollectionKey = this.getCollectionKey(
+    //     this.config.usersCollectionName
+    //   );
+    //   const usersCollectionData = this.importMap.get(usersCollectionKey);
+    //   if (!this.config.collections) {
+    //     console.log("No collections found in configuration.");
+    //     return;
+    //   }
+    //   let needsUpdate = false;
+    //   let numUpdates = 0;
+    //   for (const collectionConfig of this.config.collections) {
+    //     const collectionKey = this.getCollectionKey(collectionConfig.name);
+    //     const collectionData = this.importMap.get(collectionKey);
+    //     const collectionImportDefs = collectionConfig.importDefs;
+    //     const collectionIdMappings = collectionImportDefs
+    //       .map((importDef) => importDef.idMappings)
+    //       .flat()
+    //       .filter((idMapping) => idMapping !== undefined && idMapping !== null);
+    //     if (!collectionData || !collectionData.data) continue;
+    //     for (const dataItem of collectionData.data) {
+    //       for (const idMapping of collectionIdMappings) {
+    //         // We know it's the users collection here
+    //         if (this.getCollectionKey(idMapping.targetCollection) === usersCollectionKey) {
+    //           const targetFieldKey = idMapping.targetFieldToMatch || idMapping.targetField;
+    //           if (targetFieldKey === )
+    //           const targetValue = dataItem.finalData[targetFieldKey];
+    //           const targetCollectionData = this.importMap.get(this.getCollectionKey(idMapping.targetCollection));
+    //           if (!targetCollectionData || !targetCollectionData.data) continue;
+    //           const foundData = targetCollectionData.data.filter(({ context }) => {
+    //             const targetValue = context[targetFieldKey];
+    //             const isMatch = `${targetValue}` === `${valueToMatch}`;
+    //             return isMatch && targetValue !== undefined && targetValue !== null;
+    //           });
+    //         }
+    //       }
+    //     }
+    //   }
+    // }
+    /**
+     * Gets the value to match for a given key in the final data or context.
+     * @param finalData - The final data object.
+     * @param context - The context object.
+     * @param key - The key to get the value for.
+     * @returns The value to match for from finalData or Context
+     */
+    getValueFromData(finalData, context, key) {
+        if (context[key] !== undefined &&
+            context[key] !== null &&
+            context[key] !== "") {
+            return context[key];
         }
-        // Collect primary key fields from the users collection definitions
-        this.config.collections.forEach((collection) => {
-            if (this.getCollectionKey(collection.name) === usersCollectionKey) {
-                collection.importDefs.forEach((importDef) => {
-                    if (importDef.primaryKeyField) {
-                        usersCollectionPrimaryKeyFields.add(importDef.primaryKeyField);
-                    }
-                });
-            }
-        });
-        // Iterate over all collections to update references based on merged users
-        this.config.collections.forEach((collection) => {
-            const collectionData = this.importMap.get(this.getCollectionKey(collection.name));
-            if (!collectionData || !collectionData.data)
-                return;
-            collection.importDefs.forEach((importDef) => {
-                importDef.idMappings?.forEach((idMapping) => {
-                    if (this.getCollectionKey(idMapping.targetCollection) ===
-                        usersCollectionKey) {
-                        if (usersCollectionPrimaryKeyFields.has(idMapping.targetField)) {
-                            // Process each item in the collection
-                            collectionData.data.forEach((item) => {
-                                const oldId = item.context[idMapping.sourceField];
-                                const newId = this.mergedUserMap.get(oldId);
-                                if (newId) {
-                                    // Update context to use new user ID
-                                    item.context[idMapping.fieldToSet || idMapping.targetField] =
-                                        newId;
-                                }
-                            });
-                        }
-                    }
-                });
-            });
-        });
+        return finalData[key];
     }
-    async updateOldReferencesForNew() {
+    updateOldReferencesForNew() {
         if (!this.config.collections) {
             return;
         }
@@ -333,158 +440,159 @@ export class DataLoader {
                             for (const idMapping of importDef.idMappings) {
                                 const targetCollectionKey = this.getCollectionKey(idMapping.targetCollection);
                                 const fieldToSetKey = idMapping.fieldToSet || idMapping.sourceField;
-                                const valueToMatch = collectionData.data[i].context[idMapping.sourceField];
-                                if (!valueToMatch || _.isEmpty(valueToMatch))
+                                const targetFieldKey = idMapping.targetFieldToMatch || idMapping.targetField;
+                                const sourceValue = this.getValueFromData(collectionData.data[i].finalData, collectionData.data[i].context, idMapping.sourceField);
+                                // Skip if value to match is missing or empty
+                                if (!sourceValue ||
+                                    _.isEmpty(sourceValue) ||
+                                    sourceValue === null)
                                     continue;
+                                const isFieldToSetArray = collectionConfig.attributes.find((attribute) => attribute.key === fieldToSetKey)?.array;
                                 const targetCollectionData = this.importMap.get(targetCollectionKey);
                                 if (!targetCollectionData || !targetCollectionData.data)
                                     continue;
-                                const foundData = targetCollectionData.data.filter((data) => {
-                                    const targetValue = data.context[idMapping.targetField];
-                                    const isMatch = `${targetValue}` === `${valueToMatch}`;
-                                    // Debugging output to understand what's being compared
-                                    logger.warn(`Comparing target: ${targetValue} with match: ${valueToMatch} - Result: ${isMatch}`);
-                                    return isMatch;
-                                });
-                                if (!foundData.length) {
-                                    console.log(`No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey}`);
-                                    logger.error(`No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(idMapping, null, 2)}`);
+                                // Handle cases where sourceValue is an array
+                                const sourceValues = Array.isArray(sourceValue)
+                                    ? sourceValue.map((sourceValue) => `${sourceValue}`)
+                                    : [`${sourceValue}`];
+                                let newData = [];
+                                for (const valueToMatch of sourceValues) {
+                                    // Find matching data in the target collection
+                                    const foundData = targetCollectionData.data.filter(({ context, finalData }) => {
+                                        const targetValue = this.getValueFromData(finalData, context, targetFieldKey);
+                                        const isMatch = `${targetValue}` === `${valueToMatch}`;
+                                        // Ensure the targetValue is defined and not null
+                                        return (isMatch &&
+                                            targetValue !== undefined &&
+                                            targetValue !== null);
+                                    });
+                                    if (foundData.length) {
+                                        newData.push(...foundData.map((data) => {
+                                            const newValue = this.getValueFromData(data.finalData, data.context, idMapping.targetField);
+                                            return newValue;
+                                        }));
+                                    }
+                                    else {
+                                        logger.info(`No data found for collection: ${targetCollectionKey} with value: ${valueToMatch} for field: ${fieldToSetKey} -- idMapping: ${JSON.stringify(idMapping, null, 2)}`);
+                                    }
                                     continue;
                                 }
-                                needsUpdate = true;
-                                // Properly handle arrays and non-arrays
-                                if (Array.isArray(collectionData.data[i].finalData[fieldToSetKey])) {
-                                    collectionData.data[i].finalData[fieldToSetKey] =
-                                        foundData.map((data) => data.finalData);
-                                }
-                                else {
-                                    collectionData.data[i].finalData[fieldToSetKey] =
-                                        foundData[0].finalData;
-                                }
-                            }
-                        }
-                    }
-                }
-            }
-            if (needsUpdate) {
-                this.importMap.set(collectionKey, collectionData);
-            }
-        }
-    }
-    async updateReferencesInRelatedCollections() {
-        if (!this.config.collections) {
-            return;
-        }
-        // Iterate over each collection configuration
-        for (const collectionConfig of this.config.collections) {
-            const collectionKey = this.getCollectionKey(collectionConfig.name);
-            const collectionData = this.importMap.get(collectionKey);
-            if (!collectionData || !collectionData.data)
-                continue;
-            console.log(`Updating references for collection: ${collectionConfig.name}`);
-            // Iterate over each data item in the current collection
-            for (const item of collectionData.data) {
-                let needsUpdate = false;
-                // Check if the current collection has import definitions with idMappings
-                if (collectionConfig.importDefs) {
-                    for (const importDef of collectionConfig.importDefs) {
-                        if (importDef.idMappings) {
-                            // Iterate over each idMapping defined for the current import definition
-                            for (const idMapping of importDef.idMappings) {
-                                const oldIds = Array.isArray(item.context[idMapping.sourceField])
-                                    ? item.context[idMapping.sourceField]
-                                    : [item.context[idMapping.sourceField]];
-                                const resolvedNewIds = [];
-                                oldIds.forEach((oldId) => {
-                                    // Attempt to find a new ID for the old ID
-                                    let newIdForOldId = this.findNewIdForOldId(oldId, idMapping, importDef);
-                                    if (newIdForOldId &&
-                                        !resolvedNewIds.includes(newIdForOldId)) {
-                                        resolvedNewIds.push(newIdForOldId);
+                                const getCurrentDataFiltered = (currentData) => {
+                                    if (Array.isArray(currentData.finalData[fieldToSetKey])) {
+                                        return currentData.finalData[fieldToSetKey].filter((data) => !sourceValues.includes(`${data}`));
+                                    }
+                                    return currentData.finalData[fieldToSetKey];
+                                };
+                                // Get the current data to be updated
+                                const currentDataFiltered = getCurrentDataFiltered(collectionData.data[i]);
+                                if (newData.length) {
+                                    needsUpdate = true;
+                                    // Handle cases where current data is an array
+                                    if (isFieldToSetArray) {
+                                        if (!currentDataFiltered) {
+                                            // Set new data if current data is undefined
+                                            collectionData.data[i].finalData[fieldToSetKey] =
+                                                Array.isArray(newData) ? newData : [newData];
+                                        }
+                                        else {
+                                            if (Array.isArray(currentDataFiltered)) {
+                                                // Convert current data to array and merge if new data is non-empty array
+                                                collectionData.data[i].finalData[fieldToSetKey] = [
+                                                    ...new Set([...currentDataFiltered, ...newData].filter((value) => value !== null &&
+                                                        value !== undefined &&
+                                                        value !== "")),
+                                                ];
+                                            }
+                                            else {
+                                                // Merge arrays if new data is non-empty array and filter for uniqueness
+                                                collectionData.data[i].finalData[fieldToSetKey] = [
+                                                    ...new Set([
+                                                        ...(Array.isArray(currentDataFiltered)
+                                                            ? currentDataFiltered
+                                                            : [currentDataFiltered]),
+                                                        ...newData,
+                                                    ].filter((value) => value !== null &&
+                                                        value !== undefined &&
+                                                        value !== "" &&
+                                                        !sourceValues.includes(`${value}`))),
+                                                ];
+                                            }
+                                        }
                                     }
                                     else {
-                                        logger.error(`No new ID found for old ID ${oldId} in collection ${collectionConfig.name}`);
+                                        if (!currentDataFiltered) {
+                                            // Set new data if current data is undefined
+                                            collectionData.data[i].finalData[fieldToSetKey] =
+                                                Array.isArray(newData) ? newData[0] : newData;
+                                        }
+                                        else if (Array.isArray(newData) && newData.length > 0) {
+                                            // Convert current data to array and merge if new data is non-empty array, then filter for uniqueness
+                                            // and take the first value, because it's an array and the attribute is not an array
+                                            collectionData.data[i].finalData[fieldToSetKey] = [
+                                                ...new Set([currentDataFiltered, ...newData].filter((value) => value !== null &&
+                                                    value !== undefined &&
+                                                    value !== "" &&
+                                                    !sourceValues.includes(`${value}`))),
+                                            ].slice(0, 1)[0];
+                                        }
+                                        else if (!Array.isArray(newData) &&
+                                            newData !== undefined) {
+                                            // Simply update the field if new data is not an array and defined
+                                            collectionData.data[i].finalData[fieldToSetKey] = newData;
+                                        }
                                     }
-                                });
-                                if (resolvedNewIds.length) {
-                                    const targetField = idMapping.fieldToSet || idMapping.targetField;
-                                    const isArray = collectionConfig.attributes.some((attribute) => attribute.key === targetField && attribute.array);
-                                    // Set the target field based on whether it's an array or single value
-                                    item.finalData[targetField] = isArray
-                                        ? resolvedNewIds
-                                        : resolvedNewIds[0];
-                                    needsUpdate = true;
                                 }
                             }
                         }
                     }
                 }
-                // Update the importMap if changes were made to the item
-                if (needsUpdate) {
-                    this.importMap.set(collectionKey, collectionData);
-                    logger.info(`Updated item: ${JSON.stringify(item.finalData, undefined, 2)}`);
-                }
-            }
-        }
-    }
-    findNewIdForOldId(oldId, idMapping, importDef) {
-        // First, check if this ID mapping is related to the users collection.
-        const targetCollectionKey = this.getCollectionKey(idMapping.targetCollection);
-        const isUsersCollection = targetCollectionKey ===
-            this.getCollectionKey(this.config.usersCollectionName);
-        // If handling users, check the mergedUserMap for any existing new ID.
-        if (isUsersCollection) {
-            for (const [newUserId, oldIds] of this.mergedUserMap.entries()) {
-                if (oldIds.includes(oldId)) {
-                    return newUserId;
-                }
             }
-        }
-        // If not a user or no merged ID found, check the regular ID mapping from old to new.
-        const targetCollectionData = this.importMap.get(targetCollectionKey);
-        if (targetCollectionData) {
-            const foundEntry = targetCollectionData.data.find((entry) => entry.context[importDef.primaryKeyField] === oldId);
-            if (foundEntry) {
-                return foundEntry.context.docId; // Assuming `docId` stores the new ID after import
+            // Update the import map if any changes were made
+            if (needsUpdate) {
+                this.importMap.set(collectionKey, collectionData);
             }
         }
-        logger.error(`No corresponding new ID found for ${oldId} in ${targetCollectionKey}`);
-        return null; // Return null if no new ID is found
     }
     writeMapsToJsonFile() {
-        const outputDir = path.resolve(process.cwd());
-        const outputFile = path.join(outputDir, "dataLoaderOutput.json");
-        const dataToWrite = {
-            // Convert Maps to arrays of entries for serialization
-            oldIdToNewIdPerCollectionMap: Array.from(this.oldIdToNewIdPerCollectionMap.entries()).map(([key, value]) => {
-                return {
-                    collection: key,
-                    data: Array.from(value.entries()),
-                };
-            }),
-            mergedUserMap: Array.from(this.mergedUserMap.entries()),
-            dataFromCollections: Array.from(this.importMap.entries()).map(([key, value]) => {
-                return {
-                    collection: key,
-                    data: value.data.map((item) => item.finalData),
-                };
-            }),
-            // emailToUserIdMap: Array.from(this.emailToUserIdMap.entries()),
-            // phoneToUserIdMap: Array.from(this.phoneToUserIdMap.entries()),
-        };
-        // Use JSON.stringify with a replacer function to handle Maps
-        const replacer = (key, value) => {
-            if (value instanceof Map) {
-                return Array.from(value.entries());
-            }
-            return value;
+        const outputDir = path.resolve(process.cwd(), "zlogs");
+        // Ensure the logs directory exists
+        if (!fs.existsSync(outputDir)) {
+            fs.mkdirSync(outputDir);
+        }
+        // Helper function to write data to a file
+        const writeToFile = (fileName, data) => {
+            const outputFile = path.join(outputDir, fileName);
+            fs.writeFile(outputFile, JSON.stringify(data, null, 2), "utf8", (err) => {
+                if (err) {
+                    console.error(`Error writing data to ${fileName}:`, err);
+                    return;
+                }
+                console.log(`Data successfully written to ${fileName}`);
+            });
         };
-        fs.writeFile(outputFile, JSON.stringify(dataToWrite, replacer, 2), "utf8", (err) => {
-            if (err) {
-                console.error("Error writing data to JSON file:", err);
-                return;
-            }
-            console.log(`Data successfully written to ${outputFile}`);
+        // Convert Maps to arrays of entries for serialization
+        const oldIdToNewIdPerCollectionMap = Array.from(this.oldIdToNewIdPerCollectionMap.entries()).map(([key, value]) => {
+            return {
+                collection: key,
+                data: Array.from(value.entries()),
+            };
+        });
+        const mergedUserMap = Array.from(this.mergedUserMap.entries());
+        // Write each part to a separate file
+        writeToFile("oldIdToNewIdPerCollectionMap.json", oldIdToNewIdPerCollectionMap);
+        writeToFile("mergedUserMap.json", mergedUserMap);
+        // Write each collection's data to a separate file
+        this.importMap.forEach((value, key) => {
+            const data = {
+                collection: key,
+                data: value.data.map((item) => {
+                    return {
+                        finalData: item.finalData,
+                        context: item.context,
+                    };
+                }),
+            };
+            writeToFile(`${key}.json`, data);
         });
     }
     /**
@@ -495,46 +603,109 @@ export class DataLoader {
      * @param attributeMappings - The attribute mappings for the item.
      * @returns The transformed item with user-specific keys removed.
      */
-    async prepareUserData(item, attributeMappings, primaryKeyField, newId) {
-        // Transform the item data based on the attribute mappings
+    prepareUserData(item, attributeMappings, primaryKeyField, newId) {
+        if (this.userIdSet.has(newId) ||
+            this.userExistsMap.has(newId) ||
+            Array.from(this.emailToUserIdMap.values()).includes(newId) ||
+            Array.from(this.phoneToUserIdMap.values()).includes(newId)) {
+            newId = this.getTrueUniqueId(this.getCollectionKey("users"));
+        }
         let transformedItem = this.transformData(item, attributeMappings);
-        const userData = AuthUserCreateSchema.safeParse(transformedItem);
-        if (!userData.success) {
-            logger.error(`Invalid user data: ${JSON.stringify(userData.error.errors, undefined, 2)}`);
-            return transformedItem;
+        let userData = AuthUserCreateSchema.safeParse(transformedItem);
+        if (userData.data?.email) {
+            userData.data.email = userData.data.email.toLowerCase();
         }
-        const email = userData.data.email;
+        if (!userData.success || !(userData.data.email || userData.data.phone)) {
+            logger.error(`Invalid user data: ${JSON.stringify(userData.error?.errors, undefined, 2)} or missing email/phone`);
+            const userKeys = ["email", "phone", "name", "labels", "prefs"];
+            userKeys.forEach((key) => {
+                if (transformedItem.hasOwnProperty(key)) {
+                    delete transformedItem[key];
+                }
+            });
+            return {
+                transformedItem,
+                existingId: undefined,
+                userData: {
+                    rawData: item,
+                    finalData: transformedItem,
+                },
+            };
+        }
+        const email = userData.data.email?.toLowerCase();
         const phone = userData.data.phone;
         let existingId;
-        // Check for duplicate email and add to emailToUserIdMap if not found
-        if (email && email.length > 0) {
-            if (this.emailToUserIdMap.has(email)) {
-                existingId = this.emailToUserIdMap.get(email);
+        // Check for duplicate email and phone
+        if (email && this.emailToUserIdMap.has(email)) {
+            existingId = this.emailToUserIdMap.get(email);
+            if (phone && !this.phoneToUserIdMap.has(phone)) {
+                this.phoneToUserIdMap.set(phone, newId);
             }
-            else {
+        }
+        else if (phone && this.phoneToUserIdMap.has(phone)) {
+            existingId = this.phoneToUserIdMap.get(phone);
+            if (email && !this.emailToUserIdMap.has(email)) {
                 this.emailToUserIdMap.set(email, newId);
             }
         }
-        // Check for duplicate phone and add to phoneToUserIdMap if not found
-        if (phone && phone.length > 0) {
-            if (this.phoneToUserIdMap.has(phone)) {
-                existingId = this.phoneToUserIdMap.get(phone);
-            }
-            else {
+        else {
+            if (email)
+                this.emailToUserIdMap.set(email, newId);
+            if (phone)
                 this.phoneToUserIdMap.set(phone, newId);
-            }
-        }
-        if (!existingId) {
-            existingId = newId;
         }
-        // If existingId is found, add to mergedUserMap
         if (existingId) {
             userData.data.userId = existingId;
             const mergedUsers = this.mergedUserMap.get(existingId) || [];
             mergedUsers.push(`${item[primaryKeyField]}`);
             this.mergedUserMap.set(existingId, mergedUsers);
+            const userFound = this.importMap
+                .get(this.getCollectionKey("users"))
+                ?.data.find((userDataExisting) => {
+                let userIdToMatch;
+                if (userDataExisting?.finalData?.userId) {
+                    userIdToMatch = userDataExisting?.finalData?.userId;
+                }
+                else if (userDataExisting?.finalData?.docId) {
+                    userIdToMatch = userDataExisting?.finalData?.docId;
+                }
+                else if (userDataExisting?.context?.userId) {
+                    userIdToMatch = userDataExisting.context.userId;
+                }
+                else if (userDataExisting?.context?.docId) {
+                    userIdToMatch = userDataExisting.context.docId;
+                }
+                return userIdToMatch === existingId;
+            });
+            if (userFound) {
+                userFound.finalData.userId = existingId;
+                userFound.finalData.docId = existingId;
+                this.userIdSet.add(existingId);
+                transformedItem = {
+                    ...transformedItem,
+                    userId: existingId,
+                    docId: existingId,
+                };
+            }
+            const userKeys = ["email", "phone", "name", "labels", "prefs"];
+            userKeys.forEach((key) => {
+                if (transformedItem.hasOwnProperty(key)) {
+                    delete transformedItem[key];
+                }
+            });
+            return {
+                transformedItem,
+                existingId,
+                userData: {
+                    rawData: userFound?.rawData,
+                    finalData: userFound?.finalData,
+                },
+            };
+        }
+        else {
+            existingId = newId;
+            userData.data.userId = existingId;
         }
-        // Remove user-specific keys from the transformed item
         const userKeys = ["email", "phone", "name", "labels", "prefs"];
         userKeys.forEach((key) => {
             if (transformedItem.hasOwnProperty(key)) {
@@ -546,11 +717,15 @@ export class DataLoader {
             rawData: item,
             finalData: userData.data,
         };
-        // Directly update the importMap with the new user data, without pushing to usersMap.data first
         this.importMap.set(this.getCollectionKey("users"), {
             data: [...(usersMap?.data || []), userDataToAdd],
         });
-        return [transformedItem, existingId, userDataToAdd];
+        this.userIdSet.add(existingId);
+        return {
+            transformedItem,
+            existingId,
+            userData: userDataToAdd,
+        };
     }
     /**
      * Prepares the data for creating user collection documents.
@@ -572,7 +747,6 @@ export class DataLoader {
             this.oldIdToNewIdPerCollectionMap
                 .set(this.getCollectionKey(collection.name), oldIdToNewIdMap)
                 .get(this.getCollectionKey(collection.name));
-        console.log(`${collection.name} -- collectionOldIdToNewIdMap: ${collectionOldIdToNewIdMap}`);
         if (!operationId) {
             throw new Error(`No import operation found for collection ${collection.name}`);
         }
@@ -595,18 +769,31 @@ export class DataLoader {
         // Iterate through each item in the raw data
         for (const item of rawData) {
             // Prepare user data, check for duplicates, and remove user-specific keys
-            let [transformedItem, existingId, userData] = await this.prepareUserData(item, importDef.attributeMappings, importDef.primaryKeyField, this.getTrueUniqueId(this.getCollectionKey("users")));
+            let { transformedItem, existingId, userData } = this.prepareUserData(item, importDef.attributeMappings, importDef.primaryKeyField, this.getTrueUniqueId(this.getCollectionKey("users")));
             logger.info(`In create user -- transformedItem: ${JSON.stringify(transformedItem, null, 2)}`);
             // Generate a new unique ID for the item or use existing ID
-            if (!existingId) {
+            if (!existingId && !userData.finalData?.userId) {
                 // No existing user ID, generate a new unique ID
-                existingId = this.getTrueUniqueId(this.getCollectionKey("users"));
-                transformedItem.userId = existingId; // Assign the new ID to the transformed data's userId field
+                existingId = this.getTrueUniqueId(this.getCollectionKey(collection.name));
+                transformedItem = {
+                    ...transformedItem,
+                    userId: existingId,
+                    docId: existingId,
+                };
+            }
+            else if (!existingId && userData.finalData?.userId) {
+                // Existing user ID, use it as the new ID
+                existingId = userData.finalData.userId;
+                transformedItem = {
+                    ...transformedItem,
+                    userId: existingId,
+                    docId: existingId,
+                };
             }
             // Create a context object for the item, including the new ID
             let context = this.createContext(db, collection, item, existingId);
             // Merge the transformed data into the context
-            context = { ...context, ...transformedItem };
+            context = { ...context, ...transformedItem, ...userData.finalData };
             // If a primary key field is defined, handle the ID mapping and check for duplicates
             if (importDef.primaryKeyField) {
                 const oldId = item[importDef.primaryKeyField];
@@ -617,7 +804,9 @@ export class DataLoader {
                     // Found a duplicate oldId, now decide how to merge or handle these duplicates
                     for (const data of currentData.data) {
                         if (data.finalData.docId === oldId ||
-                            data.finalData.userId === oldId) {
+                            data.finalData.userId === oldId ||
+                            data.context.docId === oldId ||
+                            data.context.userId === oldId) {
                             transformedItem = this.mergeObjects(data.finalData, transformedItem);
                         }
                     }
@@ -627,15 +816,22 @@ export class DataLoader {
                     collectionOldIdToNewIdMap?.set(`${oldId}`, `${existingId}`);
                 }
             }
-            // Merge the final user data into the context
-            context = { ...context, ...userData.finalData };
             // Handle merging for currentUserData
             for (let i = 0; i < currentUserData.data.length; i++) {
-                if ((currentUserData.data[i].finalData.docId === existingId ||
-                    currentUserData.data[i].finalData.userId === existingId) &&
-                    !_.isEqual(currentUserData.data[i], userData)) {
-                    this.mergeObjects(currentUserData.data[i].finalData, userData.finalData);
-                    console.log("Merging user data", currentUserData.data[i].finalData);
+                const currentUserDataItem = currentUserData.data[i];
+                const samePhones = currentUserDataItem.finalData.phone &&
+                    transformedItem.phone &&
+                    currentUserDataItem.finalData.phone === transformedItem.phone;
+                const sameEmails = currentUserDataItem.finalData.email &&
+                    transformedItem.email &&
+                    currentUserDataItem.finalData.email === transformedItem.email;
+                if ((currentUserDataItem.finalData.docId === existingId ||
+                    currentUserDataItem.finalData.userId === existingId) &&
+                    (samePhones || sameEmails) &&
+                    currentUserDataItem.finalData &&
+                    userData.finalData) {
+                    const userDataMerged = this.mergeObjects(currentUserData.data[i].finalData, userData.finalData);
+                    currentUserData.data[i].finalData = userDataMerged;
                     this.importMap.set(this.getCollectionKey("users"), currentUserData);
                 }
             }
@@ -646,27 +842,41 @@ export class DataLoader {
                 ...importDef,
                 attributeMappings: mappingsWithActions,
             };
+            const updatedData = this.importMap.get(this.getCollectionKey(collection.name));
             let foundData = false;
-            for (let i = 0; i < currentData.data.length; i++) {
-                if (currentData.data[i].finalData.docId === existingId ||
-                    currentData.data[i].finalData.userId === existingId) {
-                    currentData.data[i].finalData = this.mergeObjects(currentData.data[i].finalData, transformedItem);
-                    currentData.data[i].context = context;
-                    currentData.data[i].importDef = newImportDef;
-                    this.importMap.set(this.getCollectionKey(collection.name), currentData);
+            for (let i = 0; i < updatedData.data.length; i++) {
+                if (updatedData.data[i].finalData.docId === existingId ||
+                    updatedData.data[i].finalData.userId === existingId ||
+                    updatedData.data[i].context.docId === existingId ||
+                    updatedData.data[i].context.userId === existingId) {
+                    updatedData.data[i].finalData = this.mergeObjects(updatedData.data[i].finalData, transformedItem);
+                    updatedData.data[i].context = this.mergeObjects(updatedData.data[i].context, context);
+                    const mergedImportDef = {
+                        ...updatedData.data[i].importDef,
+                        idMappings: [
+                            ...(updatedData.data[i].importDef?.idMappings || []),
+                            ...(newImportDef.idMappings || []),
+                        ],
+                        attributeMappings: [
+                            ...(updatedData.data[i].importDef?.attributeMappings || []),
+                            ...(newImportDef.attributeMappings || []),
+                        ],
+                    };
+                    updatedData.data[i].importDef = mergedImportDef;
+                    this.importMap.set(this.getCollectionKey(collection.name), updatedData);
                     this.oldIdToNewIdPerCollectionMap.set(this.getCollectionKey(collection.name), collectionOldIdToNewIdMap);
                     foundData = true;
                 }
             }
             if (!foundData) {
                 // Add new data to the associated collection
-                currentData.data.push({
+                updatedData.data.push({
                     rawData: item,
                     context: context,
                     importDef: newImportDef,
                     finalData: transformedItem,
                 });
-                this.importMap.set(this.getCollectionKey(collection.name), currentData);
+                this.importMap.set(this.getCollectionKey(collection.name), updatedData);
                 this.oldIdToNewIdPerCollectionMap.set(this.getCollectionKey(collection.name), collectionOldIdToNewIdMap);
             }
         }
@@ -697,17 +907,20 @@ export class DataLoader {
             this.oldIdToNewIdPerCollectionMap
                 .set(this.getCollectionKey(collection.name), oldIdToNewIdMapNew)
                 .get(this.getCollectionKey(collection.name));
-        console.log(`${collection.name} -- collectionOldIdToNewIdMap: ${collectionOldIdToNewIdMap}`);
+        const isRegions = collection.name.toLowerCase() === "regions";
         // Iterate through each item in the raw data
         for (const item of rawData) {
             // Generate a new unique ID for the item
             const itemIdNew = this.getTrueUniqueId(this.getCollectionKey(collection.name));
+            if (isRegions) {
+                logger.info(`Creating region: ${JSON.stringify(item, null, 2)}`);
+            }
             // Retrieve the current collection data from the import map
             const currentData = this.importMap.get(this.getCollectionKey(collection.name));
             // Create a context object for the item, including the new ID
             let context = this.createContext(db, collection, item, itemIdNew);
             // Transform the item data based on the attribute mappings
-            const transformedData = this.transformData(item, importDef.attributeMappings);
+            let transformedData = this.transformData(item, importDef.attributeMappings);
             // If a primary key field is defined, handle the ID mapping and check for duplicates
             if (importDef.primaryKeyField) {
                 const oldId = item[importDef.primaryKeyField];
@@ -720,7 +933,7 @@ export class DataLoader {
             // Merge the transformed data into the context
             context = { ...context, ...transformedData };
             // Validate the item before proceeding
-            const isValid = await this.importDataActions.validateItem(transformedData, importDef.attributeMappings, context);
+            const isValid = this.importDataActions.validateItem(transformedData, importDef.attributeMappings, context);
             if (!isValid) {
                 continue;
             }
@@ -759,69 +972,88 @@ export class DataLoader {
      * @param importDef - The import definition containing the attribute mappings and other relevant info.
      */
     async prepareUpdateData(db, collection, importDef) {
-        // Retrieve the current collection data and old-to-new ID map from the import map
         const currentData = this.importMap.get(this.getCollectionKey(collection.name));
         const oldIdToNewIdMap = this.oldIdToNewIdPerCollectionMap.get(this.getCollectionKey(collection.name));
-        // Log an error and return if no current data is found for the collection
         if (!(currentData?.data && currentData?.data.length > 0) &&
             !oldIdToNewIdMap) {
             logger.error(`No data found for collection ${collection.name} for updateDef but it says it's supposed to have one...`);
             return;
         }
-        // Load the raw data based on the import definition
         const rawData = this.loadData(importDef);
         const operationId = this.collectionImportOperations.get(this.getCollectionKey(collection.name));
         if (!operationId) {
             throw new Error(`No import operation found for collection ${collection.name}`);
         }
         for (const item of rawData) {
-            // Transform the item data based on the attribute mappings
             let transformedData = this.transformData(item, importDef.attributeMappings);
             let newId;
             let oldId;
-            // Determine the new ID for the item based on the primary key field or update mapping
-            if (importDef.primaryKeyField) {
-                oldId = item[importDef.primaryKeyField];
-            }
-            else if (importDef.updateMapping) {
-                oldId = item[importDef.updateMapping.originalIdField];
-            }
-            if (oldId) {
-                newId = oldIdToNewIdMap?.get(`${oldId}`);
-                if (!newId &&
-                    this.getCollectionKey(this.config.usersCollectionName) ===
-                        this.getCollectionKey(collection.name)) {
-                    for (const [key, value] of this.mergedUserMap.entries()) {
-                        if (value.includes(`${oldId}`)) {
-                            newId = key;
-                            break;
+            let itemDataToUpdate;
+            // Try to find itemDataToUpdate using updateMapping
+            if (importDef.updateMapping) {
+                oldId =
+                    item[importDef.updateMapping.originalIdField] ||
+                        transformedData[importDef.updateMapping.originalIdField];
+                if (oldId) {
+                    itemDataToUpdate = currentData?.data.find(({ context, finalData }) => {
+                        const targetField = importDef.updateMapping.targetField ??
+                            importDef.updateMapping.originalIdField;
+                        return (`${context[targetField]}` === `${oldId}` ||
+                            `${finalData[targetField]}` === `${oldId}`);
+                    });
+                    if (itemDataToUpdate) {
+                        newId = itemDataToUpdate.context.docId;
+                    }
+                }
+            }
+            // If updateMapping is not defined or did not find the item, use primaryKeyField
+            if (!itemDataToUpdate && importDef.primaryKeyField) {
+                oldId =
+                    item[importDef.primaryKeyField] ||
+                        transformedData[importDef.primaryKeyField];
+                if (oldId && oldId.length > 0) {
+                    newId = oldIdToNewIdMap?.get(`${oldId}`);
+                    if (!newId &&
+                        this.getCollectionKey(this.config.usersCollectionName) ===
+                            this.getCollectionKey(collection.name)) {
+                        for (const [key, value] of this.mergedUserMap.entries()) {
+                            if (value.includes(`${oldId}`)) {
+                                newId = key;
+                                break;
+                            }
                         }
                     }
                 }
+                if (oldId && !itemDataToUpdate) {
+                    itemDataToUpdate = currentData?.data.find((data) => `${data.context[importDef.primaryKeyField]}` === `${oldId}`);
+                }
             }
-            else {
+            if (!oldId) {
                 logger.error(`No old ID found (to update another document with) in prepareUpdateData for ${collection.name}, ${JSON.stringify(item, null, 2)}`);
                 continue;
             }
-            // Log an error and continue to the next item if no new ID is found
-            if (!newId) {
-                logger.error(`No new id found for collection ${collection.name} for updateDef ${JSON.stringify(item, null, 2)} but it says it's supposed to have one...`);
+            if (!newId && !itemDataToUpdate) {
+                logger.error(`No new id && no data found for collection ${collection.name} for updateDef ${JSON.stringify(item, null, 2)} but it says it's supposed to have one...`);
                 continue;
             }
-            const itemDataToUpdate = this.importMap
-                .get(this.getCollectionKey(collection.name))
-                ?.data.find((data) => data.rawData[importDef.primaryKeyField] === oldId);
-            if (!itemDataToUpdate) {
-                logger.error(`No data found for collection ${collection.name} for updateDef ${JSON.stringify(item, null, 2)} but it says it's supposed to have one...`);
+            else if (itemDataToUpdate) {
+                newId =
+                    itemDataToUpdate.finalData.docId || itemDataToUpdate.context.docId;
+                if (!newId) {
+                    logger.error(`No new id found for collection ${collection.name} for updateDef ${JSON.stringify(item, null, 2)} but has itemDataToUpdate ${JSON.stringify(itemDataToUpdate, null, 2)} but it says it's supposed to have one...`);
+                    continue;
+                }
+            }
+            if (!itemDataToUpdate || !newId) {
+                logger.error(`No data or ID (docId) found for collection ${collection.name} for updateDef ${JSON.stringify(item, null, 2)} but it says it's supposed to have one...`);
                 continue;
             }
             transformedData = this.mergeObjects(itemDataToUpdate.finalData, transformedData);
             // Create a context object for the item, including the new ID and transformed data
-            let context = this.createContext(db, collection, item, newId);
+            let context = itemDataToUpdate.context;
             context = this.mergeObjects(context, transformedData);
             // Validate the item before proceeding
-            const isValid = await this.importDataActions.validateItem(item, importDef.attributeMappings, context);
-            // Log info and continue to the next item if it's invalid
+            const isValid = this.importDataActions.validateItem(item, importDef.attributeMappings, context);
             if (!isValid) {
                 logger.info(`Skipping item: ${JSON.stringify(item, null, 2)} because it's invalid`);
                 continue;
@@ -833,15 +1065,26 @@ export class DataLoader {
                 ...importDef,
                 attributeMappings: mappingsWithActions,
             };
-            // Add the item with its context and final data to the current collection data
             if (itemDataToUpdate) {
-                // Update the existing item's finalData and context in place
                 itemDataToUpdate.finalData = this.mergeObjects(itemDataToUpdate.finalData, transformedData);
                 itemDataToUpdate.context = context;
-                itemDataToUpdate.importDef = newImportDef;
+                // Merge existing importDef with new importDef, focusing only on postImportActions
+                itemDataToUpdate.importDef = {
+                    ...itemDataToUpdate.importDef,
+                    attributeMappings: itemDataToUpdate.importDef?.attributeMappings.map((attrMapping, index) => ({
+                        ...attrMapping,
+                        postImportActions: [
+                            ...(attrMapping.postImportActions || []),
+                            ...(newImportDef.attributeMappings[index]
+                                ?.postImportActions || []),
+                        ],
+                    })) || [],
+                };
+                if (collection.name.toLowerCase() === "councils") {
+                    console.log(`Mappings in update councils: ${JSON.stringify(itemDataToUpdate.importDef.attributeMappings, null, 2)}`);
+                }
             }
             else {
-                // If no existing item matches, then add the new item
                 currentData.data.push({
                     rawData: item,
                     context: context,